From 6887b104d6844419a224ce0d7b437ae38e3f1813 Mon Sep 17 00:00:00 2001 From: alexion Date: Sat, 19 Sep 2026 14:53:22 -0400 Subject: [PATCH] build(rust): vendor googletest crate --- Cargo.lock | 98 +- Cargo.toml | 1 + src/xatu/Cargo.toml | 3 + third_party/crates/BUILD.bazel | 24 + .../aho-corasick-1.1.5/.cargo-checksum.json | 1 + .../aho-corasick-1.1.5/.cargo_vcs_info.json | 6 + .../aho-corasick-1.1.5/.github/FUNDING.yml | 1 + .../aho-corasick-1.1.5/.github/dependabot.yml | 14 + .../.github/workflows/ci.yml | 154 + .../aho-corasick-1.1.5/.vim/coc-settings.json | 12 + .../crates/aho-corasick-1.1.5/BUILD.bazel | 117 + third_party/crates/aho-corasick-1.1.5/COPYING | 3 + .../crates/aho-corasick-1.1.5/Cargo.lock | 30 + .../crates/aho-corasick-1.1.5/Cargo.toml | 80 + .../crates/aho-corasick-1.1.5/Cargo.toml.orig | 74 + .../crates/aho-corasick-1.1.5/DESIGN.md | 481 + .../crates/aho-corasick-1.1.5/LICENSE-MIT | 21 + .../crates/aho-corasick-1.1.5/README.md | 174 + .../crates/aho-corasick-1.1.5/UNLICENSE | 24 + .../crates/aho-corasick-1.1.5/rustfmt.toml | 2 + .../aho-corasick-1.1.5/src/ahocorasick.rs | 2789 ++++ .../aho-corasick-1.1.5/src/automaton.rs | 1608 ++ .../crates/aho-corasick-1.1.5/src/dfa.rs | 835 ++ .../crates/aho-corasick-1.1.5/src/lib.rs | 326 + .../crates/aho-corasick-1.1.5/src/macros.rs | 18 + .../aho-corasick-1.1.5/src/nfa/contiguous.rs | 1141 ++ .../crates/aho-corasick-1.1.5/src/nfa/mod.rs | 40 + .../src/nfa/noncontiguous.rs | 1762 +++ .../aho-corasick-1.1.5/src/packed/api.rs | 691 + .../aho-corasick-1.1.5/src/packed/ext.rs | 39 + .../aho-corasick-1.1.5/src/packed/mod.rs | 120 + .../aho-corasick-1.1.5/src/packed/pattern.rs | 480 + .../src/packed/rabinkarp.rs | 168 + .../src/packed/teddy/README.md | 386 + .../src/packed/teddy/builder.rs | 792 + .../src/packed/teddy/generic.rs | 1382 ++ .../src/packed/teddy/mod.rs | 9 + .../aho-corasick-1.1.5/src/packed/tests.rs | 583 + .../aho-corasick-1.1.5/src/packed/vector.rs | 1757 +++ .../crates/aho-corasick-1.1.5/src/tests.rs | 1664 +++ .../aho-corasick-1.1.5/src/transducer.rs | 270 + .../aho-corasick-1.1.5/src/util/alphabet.rs | 409 + .../aho-corasick-1.1.5/src/util/buffer.rs | 124 + .../src/util/byte_frequencies.rs | 258 + .../aho-corasick-1.1.5/src/util/debug.rs | 26 + .../aho-corasick-1.1.5/src/util/error.rs | 259 + .../crates/aho-corasick-1.1.5/src/util/int.rs | 278 + .../crates/aho-corasick-1.1.5/src/util/mod.rs | 12 + .../aho-corasick-1.1.5/src/util/prefilter.rs | 924 ++ .../aho-corasick-1.1.5/src/util/primitives.rs | 759 + .../aho-corasick-1.1.5/src/util/remapper.rs | 214 + .../aho-corasick-1.1.5/src/util/search.rs | 1158 ++ .../aho-corasick-1.1.5/src/util/special.rs | 42 + .../crates/autocfg-1.5.1/.cargo-checksum.json | 1 + .../crates/autocfg-1.5.1/.cargo_vcs_info.json | 6 + third_party/crates/autocfg-1.5.1/BUILD.bazel | 110 + third_party/crates/autocfg-1.5.1/Cargo.lock | 6 + third_party/crates/autocfg-1.5.1/Cargo.toml | 75 + .../crates/autocfg-1.5.1/Cargo.toml.orig | 26 + .../crates/autocfg-1.5.1/LICENSE-APACHE | 201 + third_party/crates/autocfg-1.5.1/LICENSE-MIT | 25 + third_party/crates/autocfg-1.5.1/README.md | 121 + .../crates/autocfg-1.5.1/examples/integers.rs | 9 + .../crates/autocfg-1.5.1/examples/nightly.rs | 18 + .../crates/autocfg-1.5.1/examples/paths.rs | 22 + .../crates/autocfg-1.5.1/examples/traits.rs | 26 + .../crates/autocfg-1.5.1/examples/versions.rs | 9 + third_party/crates/autocfg-1.5.1/src/error.rs | 81 + third_party/crates/autocfg-1.5.1/src/lib.rs | 636 + third_party/crates/autocfg-1.5.1/src/rustc.rs | 89 + third_party/crates/autocfg-1.5.1/src/tests.rs | 54 + .../crates/autocfg-1.5.1/src/version.rs | 65 + .../crates/autocfg-1.5.1/tests/no_std.rs | 28 + .../crates/autocfg-1.5.1/tests/rustflags.rs | 34 + .../crates/autocfg-1.5.1/tests/support/mod.rs | 21 + .../crates/autocfg-1.5.1/tests/tests.rs | 189 + .../crates/autocfg-1.5.1/tests/wrappers.rs | 83 + third_party/crates/crates.bzl | 8 + .../googletest-0.14.3/.cargo-checksum.json | 1 + .../googletest-0.14.3/.cargo_vcs_info.json | 6 + .../crates/googletest-0.14.3/BUILD.bazel | 118 + .../crates/googletest-0.14.3/Cargo.lock | 527 + .../crates/googletest-0.14.3/Cargo.toml | 160 + .../crates/googletest-0.14.3/Cargo.toml.orig | 43 + third_party/crates/googletest-0.14.3/LICENSE | 202 + .../crates/googletest-0.14.3/README.md | 367 + .../crates/googletest-0.14.3/crate_docs.md | 544 + .../googletest-0.14.3/src/assertions.rs | 1918 +++ .../googletest-0.14.3/src/description.rs | 406 + .../crates/googletest-0.14.3/src/fixtures.rs | 317 + .../crates/googletest-0.14.3/src/fmt.rs | 67 + .../src/internal/description_renderer.rs | 626 + .../googletest-0.14.3/src/internal/glob.rs | 312 + .../googletest-0.14.3/src/internal/mod.rs | 23 + .../src/internal/scoped_trace.rs | 126 + .../src/internal/test_data.rs | 29 + .../src/internal/test_filter.rs | 261 + .../src/internal/test_outcome.rs | 325 + .../src/internal/test_sharding.rs | 107 + .../crates/googletest-0.14.3/src/lib.rs | 381 + .../crates/googletest-0.14.3/src/matcher.rs | 333 + .../src/matcher_support/auto_eq.rs | 107 + .../src/matcher_support/count_elements.rs | 79 + .../src/matcher_support/edit_distance.rs | 649 + .../src/matcher_support/match_matrix.rs | 518 + .../src/matcher_support/mod.rs | 31 + .../src/matcher_support/summarize_diff.rs | 591 + .../src/matcher_support/zipped_iterator.rs | 85 + .../src/matchers/all_matcher.rs | 148 + .../src/matchers/any_matcher.rs | 155 + .../src/matchers/anything_matcher.rs | 80 + .../src/matchers/bool_matcher.rs | 100 + .../src/matchers/char_count_matcher.rs | 164 + .../src/matchers/conjunction_matcher.rs | 218 + .../src/matchers/container_eq_matcher.rs | 517 + .../src/matchers/contains_matcher.rs | 279 + .../src/matchers/contains_regex_matcher.rs | 137 + .../src/matchers/derefs_to_matcher.rs | 100 + .../src/matchers/disjunction_matcher.rs | 152 + .../src/matchers/display_matcher.rs | 124 + .../src/matchers/each_matcher.rs | 231 + .../src/matchers/elements_are_matcher.rs | 185 + .../src/matchers/empty_matcher.rs | 90 + .../src/matchers/eq_matcher.rs | 422 + .../src/matchers/err_matcher.rs | 282 + .../src/matchers/field_matcher.rs | 360 + .../src/matchers/ge_matcher.rs | 219 + .../src/matchers/gt_matcher.rs | 258 + .../src/matchers/has_entry_matcher.rs | 178 + .../src/matchers/is_encoded_string_matcher.rs | 176 + .../src/matchers/is_finite_matcher.rs | 84 + .../src/matchers/is_infinite_matcher.rs | 84 + .../src/matchers/is_matcher.rs | 65 + .../src/matchers/is_nan_matcher.rs | 64 + .../src/matchers/le_matcher.rs | 218 + .../src/matchers/len_matcher.rs | 209 + .../src/matchers/lt_matcher.rs | 227 + .../src/matchers/matches_pattern.rs | 523 + .../src/matchers/matches_regex_matcher.rs | 203 + .../googletest-0.14.3/src/matchers/mod.rs | 130 + .../src/matchers/ne_matcher.rs | 108 + .../src/matchers/near_matcher.rs | 363 + .../src/matchers/none_matcher.rs | 129 + .../src/matchers/not_matcher.rs | 105 + .../src/matchers/ok_matcher.rs | 291 + .../src/matchers/points_to_matcher.rs | 88 + .../src/matchers/pointwise_matcher.rs | 232 + .../src/matchers/predicate_matcher.rs | 215 + .../src/matchers/property_matcher.rs | 384 + .../src/matchers/result_of_matcher.rs | 441 + .../src/matchers/some_matcher.rs | 250 + .../src/matchers/str_matcher.rs | 1322 ++ .../src/matchers/subset_of_matcher.rs | 253 + .../src/matchers/superset_of_matcher.rs | 253 + .../src/matchers/tuple_matcher.rs | 255 + .../unordered_elements_are_matcher.rs | 609 + .../tests/all_matcher_test.rs | 155 + .../tests/any_matcher_test.rs | 145 + .../tests/assertions_test.rs | 428 + .../tests/colorized_diff_test.rs | 50 + .../tests/composition_test.rs | 187 + .../tests/elements_are_matcher_test.rs | 147 + .../tests/field_matcher_test.rs | 315 + .../googletest-0.14.3/tests/fmt_test.rs | 58 + .../tests/matches_pattern_enum_test.rs | 469 + .../matches_pattern_struct_and_enum_test.rs | 234 + .../tests/matches_pattern_struct_test.rs | 1250 ++ .../matches_pattern_tuple_struct_test.rs | 237 + .../googletest-0.14.3/tests/no_color_test.rs | 50 + .../googletest-0.14.3/tests/no_std_test.rs | 37 + .../tests/pointwise_matcher_test.rs | 191 + .../tests/property_matcher_test.rs | 396 + .../tests/proptest_integration_test.rs | 30 + .../tests/reference_type_test.rs | 51 + .../tests/scoped_trace_panic_test.golden | 3 + .../tests/scoped_trace_panic_test.rs | 27 + .../tests/scoped_trace_panic_test.sh | 17 + .../tests/scoped_trace_test.rs | 46 + .../tests/tuple_matcher_test.rs | 265 + .../unordered_elements_are_matcher_test.rs | 466 + .../.cargo-checksum.json | 1 + .../.cargo_vcs_info.json | 6 + .../googletest_macro-0.14.3/BUILD.bazel | 115 + .../crates/googletest_macro-0.14.3/Cargo.lock | 47 + .../crates/googletest_macro-0.14.3/Cargo.toml | 54 + .../googletest_macro-0.14.3/Cargo.toml.orig | 35 + .../crates/googletest_macro-0.14.3/LICENSE | 202 + .../crates/googletest_macro-0.14.3/README.md | 5 + .../crates/googletest_macro-0.14.3/src/lib.rs | 415 + .../src/matches_pattern.rs | 415 + .../src/verify_pred.rs | 183 + .../num-traits-0.2.19/.cargo-checksum.json | 1 + .../num-traits-0.2.19/.cargo_vcs_info.json | 6 + .../crates/num-traits-0.2.19/BUILD.bazel | 190 + .../crates/num-traits-0.2.19/Cargo.toml | 54 + .../crates/num-traits-0.2.19/Cargo.toml.orig | 34 + .../crates/num-traits-0.2.19/LICENSE-APACHE | 201 + .../crates/num-traits-0.2.19/LICENSE-MIT | 25 + .../crates/num-traits-0.2.19/README.md | 58 + .../crates/num-traits-0.2.19/RELEASES.md | 307 + third_party/crates/num-traits-0.2.19/build.rs | 7 + .../crates/num-traits-0.2.19/src/bounds.rs | 148 + .../crates/num-traits-0.2.19/src/cast.rs | 770 + .../crates/num-traits-0.2.19/src/float.rs | 2513 ++++ .../num-traits-0.2.19/src/identities.rs | 238 + .../crates/num-traits-0.2.19/src/int.rs | 562 + .../crates/num-traits-0.2.19/src/lib.rs | 635 + .../crates/num-traits-0.2.19/src/macros.rs | 44 + .../crates/num-traits-0.2.19/src/ops/bytes.rs | 317 + .../num-traits-0.2.19/src/ops/checked.rs | 261 + .../num-traits-0.2.19/src/ops/euclid.rs | 278 + .../crates/num-traits-0.2.19/src/ops/inv.rs | 47 + .../crates/num-traits-0.2.19/src/ops/mod.rs | 8 + .../num-traits-0.2.19/src/ops/mul_add.rs | 149 + .../num-traits-0.2.19/src/ops/overflowing.rs | 96 + .../num-traits-0.2.19/src/ops/saturating.rs | 130 + .../num-traits-0.2.19/src/ops/wrapping.rs | 327 + .../crates/num-traits-0.2.19/src/pow.rs | 242 + .../crates/num-traits-0.2.19/src/real.rs | 834 ++ .../crates/num-traits-0.2.19/src/sign.rs | 216 + .../crates/num-traits-0.2.19/tests/cast.rs | 387 + .../crates/regex-1.13.1/.cargo-checksum.json | 1 + .../crates/regex-1.13.1/.cargo_vcs_info.json | 6 + third_party/crates/regex-1.13.1/BUILD.bazel | 135 + third_party/crates/regex-1.13.1/CHANGELOG.md | 1813 +++ third_party/crates/regex-1.13.1/Cargo.lock | 383 + third_party/crates/regex-1.13.1/Cargo.toml | 216 + .../crates/regex-1.13.1/Cargo.toml.orig | 287 + .../crates/regex-1.13.1/LICENSE-APACHE | 201 + third_party/crates/regex-1.13.1/LICENSE-MIT | 25 + third_party/crates/regex-1.13.1/README.md | 339 + third_party/crates/regex-1.13.1/UNICODE.md | 258 + .../crates/regex-1.13.1/bench/README.md | 2 + .../crates/regex-1.13.1/src/builders.rs | 2539 ++++ third_party/crates/regex-1.13.1/src/bytes.rs | 95 + third_party/crates/regex-1.13.1/src/error.rs | 101 + .../crates/regex-1.13.1/src/find_byte.rs | 17 + third_party/crates/regex-1.13.1/src/lib.rs | 1386 ++ .../crates/regex-1.13.1/src/pattern.rs | 67 + .../crates/regex-1.13.1/src/regex/bytes.rs | 2775 ++++ .../crates/regex-1.13.1/src/regex/mod.rs | 2 + .../crates/regex-1.13.1/src/regex/string.rs | 2674 ++++ .../crates/regex-1.13.1/src/regexset/bytes.rs | 728 + .../crates/regex-1.13.1/src/regexset/mod.rs | 2 + .../regex-1.13.1/src/regexset/string.rs | 724 + .../regex-1.13.1/testdata/anchored.toml | 127 + .../crates/regex-1.13.1/testdata/bytes.toml | 235 + .../crates/regex-1.13.1/testdata/crazy.toml | 315 + .../crates/regex-1.13.1/testdata/crlf.toml | 117 + .../regex-1.13.1/testdata/earliest.toml | 52 + .../crates/regex-1.13.1/testdata/empty.toml | 113 + .../regex-1.13.1/testdata/expensive.toml | 23 + .../crates/regex-1.13.1/testdata/flags.toml | 68 + .../crates/regex-1.13.1/testdata/iter.toml | 143 + .../regex-1.13.1/testdata/leftmost-all.toml | 25 + .../testdata/line-terminator.toml | 109 + .../crates/regex-1.13.1/testdata/misc.toml | 99 + .../regex-1.13.1/testdata/multiline.toml | 845 ++ .../regex-1.13.1/testdata/no-unicode.toml | 222 + .../regex-1.13.1/testdata/overlapping.toml | 280 + .../regex-1.13.1/testdata/regex-lite.toml | 98 + .../regex-1.13.1/testdata/regression.toml | 985 ++ .../crates/regex-1.13.1/testdata/set.toml | 641 + .../regex-1.13.1/testdata/substring.toml | 36 + .../crates/regex-1.13.1/testdata/unicode.toml | 517 + .../crates/regex-1.13.1/testdata/utf8.toml | 399 + .../testdata/word-boundary-special.toml | 687 + .../regex-1.13.1/testdata/word-boundary.toml | 781 + third_party/crates/regex-1.13.1/tests/lib.rs | 58 + third_party/crates/regex-1.13.1/tests/misc.rs | 161 + .../crates/regex-1.13.1/tests/regression.rs | 94 + .../regex-1.13.1/tests/regression_fuzz.rs | 61 + .../crates/regex-1.13.1/tests/replace.rs | 183 + .../crates/regex-1.13.1/tests/searcher.rs | 93 + .../crates/regex-1.13.1/tests/suite_bytes.rs | 108 + .../regex-1.13.1/tests/suite_bytes_set.rs | 71 + .../crates/regex-1.13.1/tests/suite_string.rs | 113 + .../regex-1.13.1/tests/suite_string_set.rs | 78 + .../.cargo-checksum.json | 1 + .../.cargo_vcs_info.json | 6 + .../crates/regex-automata-0.4.18/BUILD.bazel | 139 + .../crates/regex-automata-0.4.18/Cargo.lock | 371 + .../crates/regex-automata-0.4.18/Cargo.toml | 216 + .../regex-automata-0.4.18/Cargo.toml.orig | 135 + .../regex-automata-0.4.18/LICENSE-APACHE | 201 + .../crates/regex-automata-0.4.18/LICENSE-MIT | 25 + .../crates/regex-automata-0.4.18/README.md | 117 + .../regex-automata-0.4.18/src/dfa/accel.rs | 517 + .../src/dfa/automaton.rs | 2260 +++ .../regex-automata-0.4.18/src/dfa/dense.rs | 5282 +++++++ .../src/dfa/determinize.rs | 599 + .../regex-automata-0.4.18/src/dfa/minimize.rs | 463 + .../regex-automata-0.4.18/src/dfa/mod.rs | 360 + .../regex-automata-0.4.18/src/dfa/onepass.rs | 3208 ++++ .../regex-automata-0.4.18/src/dfa/regex.rs | 870 ++ .../regex-automata-0.4.18/src/dfa/remapper.rs | 242 + .../regex-automata-0.4.18/src/dfa/search.rs | 652 + .../regex-automata-0.4.18/src/dfa/sparse.rs | 2673 ++++ .../regex-automata-0.4.18/src/dfa/special.rs | 494 + .../regex-automata-0.4.18/src/dfa/start.rs | 74 + .../regex-automata-0.4.18/src/hybrid/dfa.rs | 4434 ++++++ .../regex-automata-0.4.18/src/hybrid/error.rs | 241 + .../regex-automata-0.4.18/src/hybrid/id.rs | 354 + .../regex-automata-0.4.18/src/hybrid/mod.rs | 144 + .../regex-automata-0.4.18/src/hybrid/regex.rs | 895 ++ .../src/hybrid/search.rs | 808 ++ .../crates/regex-automata-0.4.18/src/lib.rs | 651 + .../regex-automata-0.4.18/src/macros.rs | 20 + .../regex-automata-0.4.18/src/meta/error.rs | 241 + .../regex-automata-0.4.18/src/meta/limited.rs | 251 + .../regex-automata-0.4.18/src/meta/literal.rs | 81 + .../regex-automata-0.4.18/src/meta/mod.rs | 64 + .../regex-automata-0.4.18/src/meta/prefix.rs | 236 + .../regex-automata-0.4.18/src/meta/regex.rs | 3781 +++++ .../src/meta/reverse_inner.rs | 303 + .../src/meta/reverse_suffix.rs | 270 + .../regex-automata-0.4.18/src/meta/stopat.rs | 212 + .../src/meta/strategy.rs | 2388 +++ .../src/meta/wrappers.rs | 1336 ++ .../regex-automata-0.4.18/src/nfa/mod.rs | 55 + .../src/nfa/thompson/backtrack.rs | 1908 +++ .../src/nfa/thompson/builder.rs | 1337 ++ .../src/nfa/thompson/compiler.rs | 2368 +++ .../src/nfa/thompson/error.rs | 182 + .../src/nfa/thompson/literal_trie.rs | 528 + .../src/nfa/thompson/map.rs | 296 + .../src/nfa/thompson/mod.rs | 81 + .../src/nfa/thompson/nfa.rs | 2098 +++ .../src/nfa/thompson/pikevm.rs | 2359 +++ .../src/nfa/thompson/range_trie.rs | 1051 ++ .../src/util/alphabet.rs | 1139 ++ .../src/util/captures.rs | 2551 ++++ .../src/util/determinize/mod.rs | 682 + .../src/util/determinize/state.rs | 907 ++ .../regex-automata-0.4.18/src/util/empty.rs | 265 + .../regex-automata-0.4.18/src/util/escape.rs | 84 + .../regex-automata-0.4.18/src/util/int.rs | 246 + .../src/util/interpolate.rs | 576 + .../regex-automata-0.4.18/src/util/iter.rs | 1022 ++ .../regex-automata-0.4.18/src/util/lazy.rs | 461 + .../regex-automata-0.4.18/src/util/look.rs | 2546 ++++ .../regex-automata-0.4.18/src/util/memchr.rs | 93 + .../regex-automata-0.4.18/src/util/mod.rs | 57 + .../regex-automata-0.4.18/src/util/pool.rs | 1269 ++ .../src/util/prefilter/aho_corasick.rs | 153 + .../src/util/prefilter/byteset.rs | 62 + .../src/util/prefilter/memchr.rs | 198 + .../src/util/prefilter/memmem.rs | 92 + .../src/util/prefilter/mod.rs | 726 + .../src/util/prefilter/teddy.rs | 164 + .../src/util/primitives.rs | 776 + .../regex-automata-0.4.18/src/util/search.rs | 1997 +++ .../src/util/sparse_set.rs | 239 + .../regex-automata-0.4.18/src/util/start.rs | 479 + .../regex-automata-0.4.18/src/util/syntax.rs | 482 + .../src/util/unicode_data/mod.rs | 17 + .../src/util/unicode_data/perl_word.rs | 806 + .../regex-automata-0.4.18/src/util/utf8.rs | 191 + .../regex-automata-0.4.18/src/util/wire.rs | 947 ++ .../regex-automata-0.4.18/tests/dfa/api.rs | 162 + .../regex-automata-0.4.18/tests/dfa/mod.rs | 8 + .../tests/dfa/onepass/mod.rs | 3 + .../tests/dfa/onepass/regression.rs | 61 + .../tests/dfa/onepass/suite.rs | 197 + .../tests/dfa/regression.rs | 75 + .../regex-automata-0.4.18/tests/dfa/suite.rs | 443 + .../regex-automata-0.4.18/tests/fuzz/dense.rs | 52 + .../tests/fuzz/sparse.rs | 132 + .../tests/gen/dense/mod.rs | 22 + .../tests/gen/dense/multi_pattern_v2.rs | 43 + .../regex-automata-0.4.18/tests/gen/mod.rs | 2 + .../tests/gen/sparse/mod.rs | 22 + .../tests/gen/sparse/multi_pattern_v2.rs | 37 + .../regex-automata-0.4.18/tests/hybrid/api.rs | 201 + .../regex-automata-0.4.18/tests/hybrid/mod.rs | 3 + .../tests/hybrid/suite.rs | 347 + .../crates/regex-automata-0.4.18/tests/lib.rs | 115 + .../regex-automata-0.4.18/tests/meta/mod.rs | 2 + .../regex-automata-0.4.18/tests/meta/suite.rs | 200 + .../regex-automata-0.4.18/tests/nfa/mod.rs | 1 + .../tests/nfa/thompson/backtrack/mod.rs | 3 + .../nfa/thompson/backtrack/regression.rs | 20 + .../tests/nfa/thompson/backtrack/suite.rs | 213 + .../tests/nfa/thompson/mod.rs | 4 + .../tests/nfa/thompson/pikevm/mod.rs | 3 + .../tests/nfa/thompson/pikevm/regression.rs | 20 + .../tests/nfa/thompson/pikevm/suite.rs | 162 + .../regex-syntax-0.8.11/.cargo-checksum.json | 1 + .../regex-syntax-0.8.11/.cargo_vcs_info.json | 6 + .../crates/regex-syntax-0.8.11/BUILD.bazel | 122 + .../crates/regex-syntax-0.8.11/Cargo.lock | 65 + .../crates/regex-syntax-0.8.11/Cargo.toml | 97 + .../regex-syntax-0.8.11/Cargo.toml.orig | 63 + .../crates/regex-syntax-0.8.11/LICENSE-APACHE | 201 + .../crates/regex-syntax-0.8.11/LICENSE-MIT | 25 + .../crates/regex-syntax-0.8.11/README.md | 96 + .../regex-syntax-0.8.11/benches/bench.rs | 63 + .../crates/regex-syntax-0.8.11/src/ast/mod.rs | 1807 +++ .../regex-syntax-0.8.11/src/ast/parse.rs | 6377 ++++++++ .../regex-syntax-0.8.11/src/ast/print.rs | 577 + .../regex-syntax-0.8.11/src/ast/visitor.rs | 522 + .../crates/regex-syntax-0.8.11/src/debug.rs | 107 + .../crates/regex-syntax-0.8.11/src/either.rs | 8 + .../crates/regex-syntax-0.8.11/src/error.rs | 311 + .../regex-syntax-0.8.11/src/hir/interval.rs | 591 + .../regex-syntax-0.8.11/src/hir/literal.rs | 3214 ++++ .../crates/regex-syntax-0.8.11/src/hir/mod.rs | 3873 +++++ .../regex-syntax-0.8.11/src/hir/print.rs | 608 + .../regex-syntax-0.8.11/src/hir/translate.rs | 3744 +++++ .../regex-syntax-0.8.11/src/hir/visitor.rs | 215 + .../crates/regex-syntax-0.8.11/src/lib.rs | 433 + .../crates/regex-syntax-0.8.11/src/parser.rs | 254 + .../crates/regex-syntax-0.8.11/src/rank.rs | 258 + .../crates/regex-syntax-0.8.11/src/unicode.rs | 1041 ++ .../src/unicode_tables/LICENSE-UNICODE | 57 + .../src/unicode_tables/age.rs | 1846 +++ .../src/unicode_tables/case_folding_simple.rs | 2948 ++++ .../src/unicode_tables/general_category.rs | 6717 +++++++++ .../unicode_tables/grapheme_cluster_break.rs | 1420 ++ .../src/unicode_tables/mod.rs | 57 + .../src/unicode_tables/perl_decimal.rs | 84 + .../src/unicode_tables/perl_space.rs | 23 + .../src/unicode_tables/perl_word.rs | 806 + .../src/unicode_tables/property_bool.rs | 12095 ++++++++++++++++ .../src/unicode_tables/property_names.rs | 281 + .../src/unicode_tables/property_values.rs | 956 ++ .../src/unicode_tables/script.rs | 1300 ++ .../src/unicode_tables/script_extension.rs | 1718 +++ .../src/unicode_tables/sentence_break.rs | 2530 ++++ .../src/unicode_tables/word_break.rs | 1152 ++ .../crates/regex-syntax-0.8.11/src/utf8.rs | 592 + .../rustversion-1.0.23/.cargo-checksum.json | 1 + .../rustversion-1.0.23/.cargo_vcs_info.json | 6 + .../rustversion-1.0.23/.github/FUNDING.yml | 1 + .../.github/workflows/ci.yml | 117 + .../crates/rustversion-1.0.23/BUILD.bazel | 179 + .../crates/rustversion-1.0.23/Cargo.lock | 248 + .../crates/rustversion-1.0.23/Cargo.toml | 69 + .../crates/rustversion-1.0.23/Cargo.toml.orig | 29 + .../crates/rustversion-1.0.23/LICENSE-APACHE | 176 + .../crates/rustversion-1.0.23/LICENSE-MIT | 23 + .../crates/rustversion-1.0.23/README.md | 167 + .../crates/rustversion-1.0.23/src/attr.rs | 35 + .../crates/rustversion-1.0.23/src/bound.rs | 63 + .../crates/rustversion-1.0.23/src/constfn.rs | 58 + .../crates/rustversion-1.0.23/src/date.rs | 50 + .../crates/rustversion-1.0.23/src/error.rs | 56 + .../crates/rustversion-1.0.23/src/expand.rs | 100 + .../crates/rustversion-1.0.23/src/expr.rs | 163 + .../crates/rustversion-1.0.23/src/iter.rs | 42 + .../crates/rustversion-1.0.23/src/lib.rs | 281 + .../crates/rustversion-1.0.23/src/release.rs | 34 + .../crates/rustversion-1.0.23/src/time.rs | 51 + .../crates/rustversion-1.0.23/src/token.rs | 78 + .../crates/rustversion-1.0.23/src/version.rs | 18 + .../rustversion-1.0.23/tests/compiletest.rs | 7 + .../rustversion-1.0.23/tests/test_const.rs | 42 + .../rustversion-1.0.23/tests/test_eval.rs | 20 + .../rustversion-1.0.23/tests/test_parse.rs | 103 + .../rustversion-1.0.23/tests/ui/bad-bound.rs | 7 + .../tests/ui/bad-bound.stderr | 11 + .../rustversion-1.0.23/tests/ui/bad-date.rs | 7 + .../tests/ui/bad-date.stderr | 11 + .../rustversion-1.0.23/tests/ui/bad-not.rs | 7 + .../tests/ui/bad-not.stderr | 11 + .../tests/ui/bad-version.rs | 7 + .../tests/ui/bad-version.stderr | 11 + .../tests/ui/const-not-fn.rs | 4 + .../tests/ui/const-not-fn.stderr | 5 + .../crates/syn-2.0.119/.cargo-checksum.json | 1 + .../crates/syn-2.0.119/.cargo_vcs_info.json | 6 + third_party/crates/syn-2.0.119/BUILD.bazel | 125 + third_party/crates/syn-2.0.119/Cargo.lock | 1912 +++ third_party/crates/syn-2.0.119/Cargo.toml | 272 + .../crates/syn-2.0.119/Cargo.toml.orig | 95 + third_party/crates/syn-2.0.119/LICENSE-APACHE | 176 + third_party/crates/syn-2.0.119/LICENSE-MIT | 23 + third_party/crates/syn-2.0.119/README.md | 282 + .../crates/syn-2.0.119/benches/file.rs | 59 + .../crates/syn-2.0.119/benches/rust.rs | 189 + third_party/crates/syn-2.0.119/src/attr.rs | 841 ++ third_party/crates/syn-2.0.119/src/bigint.rs | 68 + third_party/crates/syn-2.0.119/src/buffer.rs | 437 + .../crates/syn-2.0.119/src/classify.rs | 311 + .../crates/syn-2.0.119/src/custom_keyword.rs | 260 + .../syn-2.0.119/src/custom_punctuation.rs | 305 + third_party/crates/syn-2.0.119/src/data.rs | 425 + third_party/crates/syn-2.0.119/src/derive.rs | 260 + .../crates/syn-2.0.119/src/discouraged.rs | 225 + third_party/crates/syn-2.0.119/src/drops.rs | 58 + third_party/crates/syn-2.0.119/src/error.rs | 473 + third_party/crates/syn-2.0.119/src/export.rs | 73 + third_party/crates/syn-2.0.119/src/expr.rs | 4179 ++++++ third_party/crates/syn-2.0.119/src/ext.rs | 180 + third_party/crates/syn-2.0.119/src/file.rs | 128 + third_party/crates/syn-2.0.119/src/fixup.rs | 773 + .../crates/syn-2.0.119/src/gen/clone.rs | 2267 +++ .../crates/syn-2.0.119/src/gen/debug.rs | 3238 +++++ third_party/crates/syn-2.0.119/src/gen/eq.rs | 2308 +++ .../crates/syn-2.0.119/src/gen/fold.rs | 3906 +++++ .../crates/syn-2.0.119/src/gen/hash.rs | 2878 ++++ .../crates/syn-2.0.119/src/gen/token.css | 737 + .../crates/syn-2.0.119/src/gen/visit.rs | 3941 +++++ .../crates/syn-2.0.119/src/gen/visit_mut.rs | 3761 +++++ .../crates/syn-2.0.119/src/generics.rs | 1485 ++ third_party/crates/syn-2.0.119/src/group.rs | 294 + third_party/crates/syn-2.0.119/src/ident.rs | 109 + third_party/crates/syn-2.0.119/src/item.rs | 3539 +++++ third_party/crates/syn-2.0.119/src/lib.rs | 1016 ++ .../crates/syn-2.0.119/src/lifetime.rs | 155 + third_party/crates/syn-2.0.119/src/lit.rs | 1928 +++ .../crates/syn-2.0.119/src/lookahead.rs | 350 + third_party/crates/syn-2.0.119/src/mac.rs | 225 + third_party/crates/syn-2.0.119/src/macros.rs | 182 + third_party/crates/syn-2.0.119/src/meta.rs | 427 + third_party/crates/syn-2.0.119/src/op.rs | 219 + third_party/crates/syn-2.0.119/src/parse.rs | 1420 ++ .../syn-2.0.119/src/parse_macro_input.rs | 128 + .../crates/syn-2.0.119/src/parse_quote.rs | 244 + third_party/crates/syn-2.0.119/src/pat.rs | 960 ++ third_party/crates/syn-2.0.119/src/path.rs | 965 ++ .../crates/syn-2.0.119/src/precedence.rs | 210 + third_party/crates/syn-2.0.119/src/print.rs | 16 + .../crates/syn-2.0.119/src/punctuated.rs | 1170 ++ .../crates/syn-2.0.119/src/restriction.rs | 180 + .../crates/syn-2.0.119/src/scan_expr.rs | 270 + third_party/crates/syn-2.0.119/src/sealed.rs | 4 + third_party/crates/syn-2.0.119/src/span.rs | 63 + third_party/crates/syn-2.0.119/src/spanned.rs | 118 + third_party/crates/syn-2.0.119/src/stmt.rs | 504 + third_party/crates/syn-2.0.119/src/thread.rs | 60 + third_party/crates/syn-2.0.119/src/token.rs | 1104 ++ third_party/crates/syn-2.0.119/src/tt.rs | 97 + third_party/crates/syn-2.0.119/src/ty.rs | 1286 ++ .../crates/syn-2.0.119/src/verbatim.rs | 32 + .../crates/syn-2.0.119/src/whitespace.rs | 65 + .../crates/syn-2.0.119/tests/common/eq.rs | 926 ++ .../crates/syn-2.0.119/tests/common/mod.rs | 6 + .../crates/syn-2.0.119/tests/common/parse.rs | 51 + .../crates/syn-2.0.119/tests/common/visit.rs | 119 + .../crates/syn-2.0.119/tests/debug/gen.rs | 5238 +++++++ .../crates/syn-2.0.119/tests/debug/mod.rs | 147 + .../crates/syn-2.0.119/tests/macros/mod.rs | 7 + .../crates/syn-2.0.119/tests/regression.rs | 5 + .../syn-2.0.119/tests/regression/issue1108.rs | 5 + .../syn-2.0.119/tests/regression/issue1235.rs | 32 + .../crates/syn-2.0.119/tests/repo/mod.rs | 1023 ++ .../crates/syn-2.0.119/tests/repo/progress.rs | 37 + .../crates/syn-2.0.119/tests/snapshot/mod.rs | 68 + .../syn-2.0.119/tests/test_asyncness.rs | 49 + .../syn-2.0.119/tests/test_attribute.rs | 231 + .../syn-2.0.119/tests/test_derive_input.rs | 785 + .../crates/syn-2.0.119/tests/test_expr.rs | 1702 +++ .../crates/syn-2.0.119/tests/test_generics.rs | 356 + .../crates/syn-2.0.119/tests/test_grouping.rs | 59 + .../crates/syn-2.0.119/tests/test_ident.rs | 87 + .../crates/syn-2.0.119/tests/test_item.rs | 371 + .../crates/syn-2.0.119/tests/test_lit.rs | 335 + .../crates/syn-2.0.119/tests/test_meta.rs | 180 + .../syn-2.0.119/tests/test_parse_buffer.rs | 104 + .../syn-2.0.119/tests/test_parse_quote.rs | 172 + .../syn-2.0.119/tests/test_parse_stream.rs | 187 + .../crates/syn-2.0.119/tests/test_pat.rs | 158 + .../crates/syn-2.0.119/tests/test_path.rs | 116 + .../syn-2.0.119/tests/test_precedence.rs | 556 + .../syn-2.0.119/tests/test_punctuated.rs | 92 + .../crates/syn-2.0.119/tests/test_receiver.rs | 327 + .../syn-2.0.119/tests/test_round_trip.rs | 225 + .../crates/syn-2.0.119/tests/test_shebang.rs | 73 + .../crates/syn-2.0.119/tests/test_size.rs | 54 + .../crates/syn-2.0.119/tests/test_stmt.rs | 337 + .../syn-2.0.119/tests/test_token_trees.rs | 38 + .../crates/syn-2.0.119/tests/test_ty.rs | 471 + .../syn-2.0.119/tests/test_unparenthesize.rs | 70 + .../syn-2.0.119/tests/test_visibility.rs | 191 + .../crates/syn-2.0.119/tests/zzz_stable.rs | 33 + 576 files changed, 284290 insertions(+), 2 deletions(-) create mode 100644 third_party/crates/aho-corasick-1.1.5/.cargo-checksum.json create mode 100644 third_party/crates/aho-corasick-1.1.5/.cargo_vcs_info.json create mode 100644 third_party/crates/aho-corasick-1.1.5/.github/FUNDING.yml create mode 100644 third_party/crates/aho-corasick-1.1.5/.github/dependabot.yml create mode 100644 third_party/crates/aho-corasick-1.1.5/.github/workflows/ci.yml create mode 100644 third_party/crates/aho-corasick-1.1.5/.vim/coc-settings.json create mode 100644 third_party/crates/aho-corasick-1.1.5/BUILD.bazel create mode 100644 third_party/crates/aho-corasick-1.1.5/COPYING create mode 100644 third_party/crates/aho-corasick-1.1.5/Cargo.lock create mode 100644 third_party/crates/aho-corasick-1.1.5/Cargo.toml create mode 100644 third_party/crates/aho-corasick-1.1.5/Cargo.toml.orig create mode 100644 third_party/crates/aho-corasick-1.1.5/DESIGN.md create mode 100644 third_party/crates/aho-corasick-1.1.5/LICENSE-MIT create mode 100644 third_party/crates/aho-corasick-1.1.5/README.md create mode 100644 third_party/crates/aho-corasick-1.1.5/UNLICENSE create mode 100644 third_party/crates/aho-corasick-1.1.5/rustfmt.toml create mode 100644 third_party/crates/aho-corasick-1.1.5/src/ahocorasick.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/automaton.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/dfa.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/lib.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/macros.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/nfa/contiguous.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/nfa/mod.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/nfa/noncontiguous.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/api.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/ext.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/mod.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/pattern.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/rabinkarp.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/teddy/README.md create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/teddy/builder.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/teddy/generic.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/teddy/mod.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/tests.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/packed/vector.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/tests.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/transducer.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/alphabet.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/buffer.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/byte_frequencies.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/debug.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/error.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/int.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/mod.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/prefilter.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/primitives.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/remapper.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/search.rs create mode 100644 third_party/crates/aho-corasick-1.1.5/src/util/special.rs create mode 100644 third_party/crates/autocfg-1.5.1/.cargo-checksum.json create mode 100644 third_party/crates/autocfg-1.5.1/.cargo_vcs_info.json create mode 100644 third_party/crates/autocfg-1.5.1/BUILD.bazel create mode 100644 third_party/crates/autocfg-1.5.1/Cargo.lock create mode 100644 third_party/crates/autocfg-1.5.1/Cargo.toml create mode 100644 third_party/crates/autocfg-1.5.1/Cargo.toml.orig create mode 100644 third_party/crates/autocfg-1.5.1/LICENSE-APACHE create mode 100644 third_party/crates/autocfg-1.5.1/LICENSE-MIT create mode 100644 third_party/crates/autocfg-1.5.1/README.md create mode 100644 third_party/crates/autocfg-1.5.1/examples/integers.rs create mode 100644 third_party/crates/autocfg-1.5.1/examples/nightly.rs create mode 100644 third_party/crates/autocfg-1.5.1/examples/paths.rs create mode 100644 third_party/crates/autocfg-1.5.1/examples/traits.rs create mode 100644 third_party/crates/autocfg-1.5.1/examples/versions.rs create mode 100644 third_party/crates/autocfg-1.5.1/src/error.rs create mode 100644 third_party/crates/autocfg-1.5.1/src/lib.rs create mode 100644 third_party/crates/autocfg-1.5.1/src/rustc.rs create mode 100644 third_party/crates/autocfg-1.5.1/src/tests.rs create mode 100644 third_party/crates/autocfg-1.5.1/src/version.rs create mode 100644 third_party/crates/autocfg-1.5.1/tests/no_std.rs create mode 100644 third_party/crates/autocfg-1.5.1/tests/rustflags.rs create mode 100644 third_party/crates/autocfg-1.5.1/tests/support/mod.rs create mode 100644 third_party/crates/autocfg-1.5.1/tests/tests.rs create mode 100644 third_party/crates/autocfg-1.5.1/tests/wrappers.rs create mode 100644 third_party/crates/googletest-0.14.3/.cargo-checksum.json create mode 100644 third_party/crates/googletest-0.14.3/.cargo_vcs_info.json create mode 100644 third_party/crates/googletest-0.14.3/BUILD.bazel create mode 100644 third_party/crates/googletest-0.14.3/Cargo.lock create mode 100644 third_party/crates/googletest-0.14.3/Cargo.toml create mode 100644 third_party/crates/googletest-0.14.3/Cargo.toml.orig create mode 100644 third_party/crates/googletest-0.14.3/LICENSE create mode 100644 third_party/crates/googletest-0.14.3/README.md create mode 100644 third_party/crates/googletest-0.14.3/crate_docs.md create mode 100644 third_party/crates/googletest-0.14.3/src/assertions.rs create mode 100644 third_party/crates/googletest-0.14.3/src/description.rs create mode 100644 third_party/crates/googletest-0.14.3/src/fixtures.rs create mode 100644 third_party/crates/googletest-0.14.3/src/fmt.rs create mode 100644 third_party/crates/googletest-0.14.3/src/internal/description_renderer.rs create mode 100644 third_party/crates/googletest-0.14.3/src/internal/glob.rs create mode 100644 third_party/crates/googletest-0.14.3/src/internal/mod.rs create mode 100644 third_party/crates/googletest-0.14.3/src/internal/scoped_trace.rs create mode 100644 third_party/crates/googletest-0.14.3/src/internal/test_data.rs create mode 100644 third_party/crates/googletest-0.14.3/src/internal/test_filter.rs create mode 100644 third_party/crates/googletest-0.14.3/src/internal/test_outcome.rs create mode 100644 third_party/crates/googletest-0.14.3/src/internal/test_sharding.rs create mode 100644 third_party/crates/googletest-0.14.3/src/lib.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matcher_support/auto_eq.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matcher_support/count_elements.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matcher_support/edit_distance.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matcher_support/match_matrix.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matcher_support/mod.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matcher_support/summarize_diff.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matcher_support/zipped_iterator.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/all_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/any_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/anything_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/bool_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/char_count_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/conjunction_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/container_eq_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/contains_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/contains_regex_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/derefs_to_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/disjunction_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/display_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/each_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/elements_are_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/empty_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/eq_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/err_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/field_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/ge_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/gt_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/has_entry_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/is_encoded_string_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/is_finite_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/is_infinite_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/is_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/is_nan_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/le_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/len_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/lt_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/matches_pattern.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/matches_regex_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/mod.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/ne_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/near_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/none_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/not_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/ok_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/points_to_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/pointwise_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/predicate_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/property_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/result_of_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/some_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/str_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/subset_of_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/superset_of_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/tuple_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/src/matchers/unordered_elements_are_matcher.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/all_matcher_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/any_matcher_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/assertions_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/colorized_diff_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/composition_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/elements_are_matcher_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/field_matcher_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/fmt_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/matches_pattern_enum_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/matches_pattern_struct_and_enum_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/matches_pattern_struct_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/matches_pattern_tuple_struct_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/no_color_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/no_std_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/pointwise_matcher_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/property_matcher_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/proptest_integration_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/reference_type_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.golden create mode 100644 third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.sh create mode 100644 third_party/crates/googletest-0.14.3/tests/scoped_trace_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/tuple_matcher_test.rs create mode 100644 third_party/crates/googletest-0.14.3/tests/unordered_elements_are_matcher_test.rs create mode 100644 third_party/crates/googletest_macro-0.14.3/.cargo-checksum.json create mode 100644 third_party/crates/googletest_macro-0.14.3/.cargo_vcs_info.json create mode 100644 third_party/crates/googletest_macro-0.14.3/BUILD.bazel create mode 100644 third_party/crates/googletest_macro-0.14.3/Cargo.lock create mode 100644 third_party/crates/googletest_macro-0.14.3/Cargo.toml create mode 100644 third_party/crates/googletest_macro-0.14.3/Cargo.toml.orig create mode 100644 third_party/crates/googletest_macro-0.14.3/LICENSE create mode 100644 third_party/crates/googletest_macro-0.14.3/README.md create mode 100644 third_party/crates/googletest_macro-0.14.3/src/lib.rs create mode 100644 third_party/crates/googletest_macro-0.14.3/src/matches_pattern.rs create mode 100644 third_party/crates/googletest_macro-0.14.3/src/verify_pred.rs create mode 100644 third_party/crates/num-traits-0.2.19/.cargo-checksum.json create mode 100644 third_party/crates/num-traits-0.2.19/.cargo_vcs_info.json create mode 100644 third_party/crates/num-traits-0.2.19/BUILD.bazel create mode 100644 third_party/crates/num-traits-0.2.19/Cargo.toml create mode 100644 third_party/crates/num-traits-0.2.19/Cargo.toml.orig create mode 100644 third_party/crates/num-traits-0.2.19/LICENSE-APACHE create mode 100644 third_party/crates/num-traits-0.2.19/LICENSE-MIT create mode 100644 third_party/crates/num-traits-0.2.19/README.md create mode 100644 third_party/crates/num-traits-0.2.19/RELEASES.md create mode 100644 third_party/crates/num-traits-0.2.19/build.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/bounds.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/cast.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/float.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/identities.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/int.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/lib.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/macros.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/bytes.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/checked.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/euclid.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/inv.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/mod.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/mul_add.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/overflowing.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/saturating.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/ops/wrapping.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/pow.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/real.rs create mode 100644 third_party/crates/num-traits-0.2.19/src/sign.rs create mode 100644 third_party/crates/num-traits-0.2.19/tests/cast.rs create mode 100644 third_party/crates/regex-1.13.1/.cargo-checksum.json create mode 100644 third_party/crates/regex-1.13.1/.cargo_vcs_info.json create mode 100644 third_party/crates/regex-1.13.1/BUILD.bazel create mode 100644 third_party/crates/regex-1.13.1/CHANGELOG.md create mode 100644 third_party/crates/regex-1.13.1/Cargo.lock create mode 100644 third_party/crates/regex-1.13.1/Cargo.toml create mode 100644 third_party/crates/regex-1.13.1/Cargo.toml.orig create mode 100644 third_party/crates/regex-1.13.1/LICENSE-APACHE create mode 100644 third_party/crates/regex-1.13.1/LICENSE-MIT create mode 100644 third_party/crates/regex-1.13.1/README.md create mode 100644 third_party/crates/regex-1.13.1/UNICODE.md create mode 100644 third_party/crates/regex-1.13.1/bench/README.md create mode 100644 third_party/crates/regex-1.13.1/src/builders.rs create mode 100644 third_party/crates/regex-1.13.1/src/bytes.rs create mode 100644 third_party/crates/regex-1.13.1/src/error.rs create mode 100644 third_party/crates/regex-1.13.1/src/find_byte.rs create mode 100644 third_party/crates/regex-1.13.1/src/lib.rs create mode 100644 third_party/crates/regex-1.13.1/src/pattern.rs create mode 100644 third_party/crates/regex-1.13.1/src/regex/bytes.rs create mode 100644 third_party/crates/regex-1.13.1/src/regex/mod.rs create mode 100644 third_party/crates/regex-1.13.1/src/regex/string.rs create mode 100644 third_party/crates/regex-1.13.1/src/regexset/bytes.rs create mode 100644 third_party/crates/regex-1.13.1/src/regexset/mod.rs create mode 100644 third_party/crates/regex-1.13.1/src/regexset/string.rs create mode 100644 third_party/crates/regex-1.13.1/testdata/anchored.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/bytes.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/crazy.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/crlf.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/earliest.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/empty.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/expensive.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/flags.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/iter.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/leftmost-all.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/line-terminator.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/misc.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/multiline.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/no-unicode.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/overlapping.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/regex-lite.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/regression.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/set.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/substring.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/unicode.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/utf8.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/word-boundary-special.toml create mode 100644 third_party/crates/regex-1.13.1/testdata/word-boundary.toml create mode 100644 third_party/crates/regex-1.13.1/tests/lib.rs create mode 100644 third_party/crates/regex-1.13.1/tests/misc.rs create mode 100644 third_party/crates/regex-1.13.1/tests/regression.rs create mode 100644 third_party/crates/regex-1.13.1/tests/regression_fuzz.rs create mode 100644 third_party/crates/regex-1.13.1/tests/replace.rs create mode 100644 third_party/crates/regex-1.13.1/tests/searcher.rs create mode 100644 third_party/crates/regex-1.13.1/tests/suite_bytes.rs create mode 100644 third_party/crates/regex-1.13.1/tests/suite_bytes_set.rs create mode 100644 third_party/crates/regex-1.13.1/tests/suite_string.rs create mode 100644 third_party/crates/regex-1.13.1/tests/suite_string_set.rs create mode 100644 third_party/crates/regex-automata-0.4.18/.cargo-checksum.json create mode 100644 third_party/crates/regex-automata-0.4.18/.cargo_vcs_info.json create mode 100644 third_party/crates/regex-automata-0.4.18/BUILD.bazel create mode 100644 third_party/crates/regex-automata-0.4.18/Cargo.lock create mode 100644 third_party/crates/regex-automata-0.4.18/Cargo.toml create mode 100644 third_party/crates/regex-automata-0.4.18/Cargo.toml.orig create mode 100644 third_party/crates/regex-automata-0.4.18/LICENSE-APACHE create mode 100644 third_party/crates/regex-automata-0.4.18/LICENSE-MIT create mode 100644 third_party/crates/regex-automata-0.4.18/README.md create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/accel.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/automaton.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/dense.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/determinize.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/minimize.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/onepass.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/regex.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/remapper.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/search.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/sparse.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/special.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/dfa/start.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/hybrid/dfa.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/hybrid/error.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/hybrid/id.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/hybrid/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/hybrid/regex.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/hybrid/search.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/lib.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/macros.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/error.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/limited.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/literal.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/prefix.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/regex.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/reverse_inner.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/reverse_suffix.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/stopat.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/strategy.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/meta/wrappers.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/backtrack.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/builder.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/compiler.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/error.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/literal_trie.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/map.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/nfa.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/pikevm.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/nfa/thompson/range_trie.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/alphabet.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/captures.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/determinize/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/determinize/state.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/empty.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/escape.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/int.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/interpolate.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/iter.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/lazy.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/look.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/memchr.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/pool.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/prefilter/aho_corasick.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/prefilter/byteset.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/prefilter/memchr.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/prefilter/memmem.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/prefilter/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/prefilter/teddy.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/primitives.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/search.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/sparse_set.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/start.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/syntax.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/unicode_data/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/unicode_data/perl_word.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/utf8.rs create mode 100644 third_party/crates/regex-automata-0.4.18/src/util/wire.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/dfa/api.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/dfa/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/regression.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/suite.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/dfa/regression.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/dfa/suite.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/fuzz/dense.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/fuzz/sparse.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/gen/dense/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/gen/dense/multi_pattern_v2.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/gen/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/gen/sparse/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/gen/sparse/multi_pattern_v2.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/hybrid/api.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/hybrid/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/hybrid/suite.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/lib.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/meta/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/meta/suite.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/nfa/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/regression.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/suite.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/mod.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/regression.rs create mode 100644 third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/suite.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/.cargo-checksum.json create mode 100644 third_party/crates/regex-syntax-0.8.11/.cargo_vcs_info.json create mode 100644 third_party/crates/regex-syntax-0.8.11/BUILD.bazel create mode 100644 third_party/crates/regex-syntax-0.8.11/Cargo.lock create mode 100644 third_party/crates/regex-syntax-0.8.11/Cargo.toml create mode 100644 third_party/crates/regex-syntax-0.8.11/Cargo.toml.orig create mode 100644 third_party/crates/regex-syntax-0.8.11/LICENSE-APACHE create mode 100644 third_party/crates/regex-syntax-0.8.11/LICENSE-MIT create mode 100644 third_party/crates/regex-syntax-0.8.11/README.md create mode 100644 third_party/crates/regex-syntax-0.8.11/benches/bench.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/ast/mod.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/ast/parse.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/ast/print.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/ast/visitor.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/debug.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/either.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/error.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/hir/interval.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/hir/literal.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/hir/mod.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/hir/print.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/hir/translate.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/hir/visitor.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/lib.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/parser.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/rank.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/LICENSE-UNICODE create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/age.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/case_folding_simple.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/general_category.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/grapheme_cluster_break.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/mod.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_decimal.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_space.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_word.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_bool.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_names.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_values.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/script.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/script_extension.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/sentence_break.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/unicode_tables/word_break.rs create mode 100644 third_party/crates/regex-syntax-0.8.11/src/utf8.rs create mode 100644 third_party/crates/rustversion-1.0.23/.cargo-checksum.json create mode 100644 third_party/crates/rustversion-1.0.23/.cargo_vcs_info.json create mode 100644 third_party/crates/rustversion-1.0.23/.github/FUNDING.yml create mode 100644 third_party/crates/rustversion-1.0.23/.github/workflows/ci.yml create mode 100644 third_party/crates/rustversion-1.0.23/BUILD.bazel create mode 100644 third_party/crates/rustversion-1.0.23/Cargo.lock create mode 100644 third_party/crates/rustversion-1.0.23/Cargo.toml create mode 100644 third_party/crates/rustversion-1.0.23/Cargo.toml.orig create mode 100644 third_party/crates/rustversion-1.0.23/LICENSE-APACHE create mode 100644 third_party/crates/rustversion-1.0.23/LICENSE-MIT create mode 100644 third_party/crates/rustversion-1.0.23/README.md create mode 100644 third_party/crates/rustversion-1.0.23/src/attr.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/bound.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/constfn.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/date.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/error.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/expand.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/expr.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/iter.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/lib.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/release.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/time.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/token.rs create mode 100644 third_party/crates/rustversion-1.0.23/src/version.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/compiletest.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/test_const.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/test_eval.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/test_parse.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/bad-bound.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/bad-bound.stderr create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/bad-date.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/bad-date.stderr create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/bad-not.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/bad-not.stderr create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/bad-version.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/bad-version.stderr create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/const-not-fn.rs create mode 100644 third_party/crates/rustversion-1.0.23/tests/ui/const-not-fn.stderr create mode 100644 third_party/crates/syn-2.0.119/.cargo-checksum.json create mode 100644 third_party/crates/syn-2.0.119/.cargo_vcs_info.json create mode 100644 third_party/crates/syn-2.0.119/BUILD.bazel create mode 100644 third_party/crates/syn-2.0.119/Cargo.lock create mode 100644 third_party/crates/syn-2.0.119/Cargo.toml create mode 100644 third_party/crates/syn-2.0.119/Cargo.toml.orig create mode 100644 third_party/crates/syn-2.0.119/LICENSE-APACHE create mode 100644 third_party/crates/syn-2.0.119/LICENSE-MIT create mode 100644 third_party/crates/syn-2.0.119/README.md create mode 100644 third_party/crates/syn-2.0.119/benches/file.rs create mode 100644 third_party/crates/syn-2.0.119/benches/rust.rs create mode 100644 third_party/crates/syn-2.0.119/src/attr.rs create mode 100644 third_party/crates/syn-2.0.119/src/bigint.rs create mode 100644 third_party/crates/syn-2.0.119/src/buffer.rs create mode 100644 third_party/crates/syn-2.0.119/src/classify.rs create mode 100644 third_party/crates/syn-2.0.119/src/custom_keyword.rs create mode 100644 third_party/crates/syn-2.0.119/src/custom_punctuation.rs create mode 100644 third_party/crates/syn-2.0.119/src/data.rs create mode 100644 third_party/crates/syn-2.0.119/src/derive.rs create mode 100644 third_party/crates/syn-2.0.119/src/discouraged.rs create mode 100644 third_party/crates/syn-2.0.119/src/drops.rs create mode 100644 third_party/crates/syn-2.0.119/src/error.rs create mode 100644 third_party/crates/syn-2.0.119/src/export.rs create mode 100644 third_party/crates/syn-2.0.119/src/expr.rs create mode 100644 third_party/crates/syn-2.0.119/src/ext.rs create mode 100644 third_party/crates/syn-2.0.119/src/file.rs create mode 100644 third_party/crates/syn-2.0.119/src/fixup.rs create mode 100644 third_party/crates/syn-2.0.119/src/gen/clone.rs create mode 100644 third_party/crates/syn-2.0.119/src/gen/debug.rs create mode 100644 third_party/crates/syn-2.0.119/src/gen/eq.rs create mode 100644 third_party/crates/syn-2.0.119/src/gen/fold.rs create mode 100644 third_party/crates/syn-2.0.119/src/gen/hash.rs create mode 100644 third_party/crates/syn-2.0.119/src/gen/token.css create mode 100644 third_party/crates/syn-2.0.119/src/gen/visit.rs create mode 100644 third_party/crates/syn-2.0.119/src/gen/visit_mut.rs create mode 100644 third_party/crates/syn-2.0.119/src/generics.rs create mode 100644 third_party/crates/syn-2.0.119/src/group.rs create mode 100644 third_party/crates/syn-2.0.119/src/ident.rs create mode 100644 third_party/crates/syn-2.0.119/src/item.rs create mode 100644 third_party/crates/syn-2.0.119/src/lib.rs create mode 100644 third_party/crates/syn-2.0.119/src/lifetime.rs create mode 100644 third_party/crates/syn-2.0.119/src/lit.rs create mode 100644 third_party/crates/syn-2.0.119/src/lookahead.rs create mode 100644 third_party/crates/syn-2.0.119/src/mac.rs create mode 100644 third_party/crates/syn-2.0.119/src/macros.rs create mode 100644 third_party/crates/syn-2.0.119/src/meta.rs create mode 100644 third_party/crates/syn-2.0.119/src/op.rs create mode 100644 third_party/crates/syn-2.0.119/src/parse.rs create mode 100644 third_party/crates/syn-2.0.119/src/parse_macro_input.rs create mode 100644 third_party/crates/syn-2.0.119/src/parse_quote.rs create mode 100644 third_party/crates/syn-2.0.119/src/pat.rs create mode 100644 third_party/crates/syn-2.0.119/src/path.rs create mode 100644 third_party/crates/syn-2.0.119/src/precedence.rs create mode 100644 third_party/crates/syn-2.0.119/src/print.rs create mode 100644 third_party/crates/syn-2.0.119/src/punctuated.rs create mode 100644 third_party/crates/syn-2.0.119/src/restriction.rs create mode 100644 third_party/crates/syn-2.0.119/src/scan_expr.rs create mode 100644 third_party/crates/syn-2.0.119/src/sealed.rs create mode 100644 third_party/crates/syn-2.0.119/src/span.rs create mode 100644 third_party/crates/syn-2.0.119/src/spanned.rs create mode 100644 third_party/crates/syn-2.0.119/src/stmt.rs create mode 100644 third_party/crates/syn-2.0.119/src/thread.rs create mode 100644 third_party/crates/syn-2.0.119/src/token.rs create mode 100644 third_party/crates/syn-2.0.119/src/tt.rs create mode 100644 third_party/crates/syn-2.0.119/src/ty.rs create mode 100644 third_party/crates/syn-2.0.119/src/verbatim.rs create mode 100644 third_party/crates/syn-2.0.119/src/whitespace.rs create mode 100644 third_party/crates/syn-2.0.119/tests/common/eq.rs create mode 100644 third_party/crates/syn-2.0.119/tests/common/mod.rs create mode 100644 third_party/crates/syn-2.0.119/tests/common/parse.rs create mode 100644 third_party/crates/syn-2.0.119/tests/common/visit.rs create mode 100644 third_party/crates/syn-2.0.119/tests/debug/gen.rs create mode 100644 third_party/crates/syn-2.0.119/tests/debug/mod.rs create mode 100644 third_party/crates/syn-2.0.119/tests/macros/mod.rs create mode 100644 third_party/crates/syn-2.0.119/tests/regression.rs create mode 100644 third_party/crates/syn-2.0.119/tests/regression/issue1108.rs create mode 100644 third_party/crates/syn-2.0.119/tests/regression/issue1235.rs create mode 100644 third_party/crates/syn-2.0.119/tests/repo/mod.rs create mode 100644 third_party/crates/syn-2.0.119/tests/repo/progress.rs create mode 100644 third_party/crates/syn-2.0.119/tests/snapshot/mod.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_asyncness.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_attribute.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_derive_input.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_expr.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_generics.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_grouping.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_ident.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_item.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_lit.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_meta.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_parse_buffer.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_parse_quote.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_parse_stream.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_pat.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_path.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_precedence.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_punctuated.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_receiver.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_round_trip.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_shebang.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_size.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_stmt.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_token_trees.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_ty.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_unparenthesize.rs create mode 100644 third_party/crates/syn-2.0.119/tests/test_visibility.rs create mode 100644 third_party/crates/syn-2.0.119/tests/zzz_stable.rs diff --git a/Cargo.lock b/Cargo.lock index 83d9f3a..bedb295 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "1.0.0" @@ -62,6 +71,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "clap" version = "4.6.7" @@ -93,7 +108,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -114,6 +129,29 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" +[[package]] +name = "googletest" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b5e2f2b556b7b90297a5a35c8267dd43a537923d2b329beefdba2b4ec19d94" +dependencies = [ + "googletest_macro", + "num-traits", + "regex", + "rustversion", +] + +[[package]] +name = "googletest_macro" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae6abc96141edd26bf5aeec0f119c129c44de3ced09e5073711a02cb74725d0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -152,6 +190,15 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" name = "monorepo" version = "0.1.0" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell_polyfill" version = "1.70.2" @@ -176,6 +223,35 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "rnix" version = "0.14.0" @@ -203,6 +279,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "serde" version = "1.0.229" @@ -230,7 +312,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -256,6 +338,17 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "3.0.6" @@ -305,6 +398,7 @@ name = "xatu" version = "0.1.0" dependencies = [ "clap", + "googletest", "monorepo", "rnix", "rowan", diff --git a/Cargo.toml b/Cargo.toml index bcfcb49..70160ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ resolver = "3" [workspace.dependencies] clap = "4" +googletest = "0.14" monorepo = { path = "src/monorepo" } rnix = "=0.14.0" rowan = "0.16" diff --git a/src/xatu/Cargo.toml b/src/xatu/Cargo.toml index b9aff35..d12e186 100644 --- a/src/xatu/Cargo.toml +++ b/src/xatu/Cargo.toml @@ -16,3 +16,6 @@ monorepo = { workspace = true } rnix = { workspace = true } rowan = { workspace = true } status = { workspace = true } + +[dev-dependencies] +googletest = { workspace = true } diff --git a/third_party/crates/BUILD.bazel b/third_party/crates/BUILD.bazel index efa30e7..46aabd3 100644 --- a/third_party/crates/BUILD.bazel +++ b/third_party/crates/BUILD.bazel @@ -43,6 +43,18 @@ alias( tags = ["manual"], ) +alias( + name = "googletest-0.14.3", + actual = "//third_party/crates/googletest-0.14.3:googletest", + tags = ["manual"], +) + +alias( + name = "googletest", + actual = "//third_party/crates/googletest-0.14.3:googletest", + tags = ["manual"], +) + alias( name = "rnix-0.14.0", actual = "//third_party/crates/rnix-0.14.0:rnix", @@ -55,6 +67,18 @@ alias( tags = ["manual"], ) +alias( + name = "rowan-0.16.1", + actual = "//third_party/crates/rowan-0.16.1:rowan", + tags = ["manual"], +) + +alias( + name = "rowan", + actual = "//third_party/crates/rowan-0.16.1:rowan", + tags = ["manual"], +) + alias( name = "serde-1.0.229", actual = "//third_party/crates/serde-1.0.229:serde", diff --git a/third_party/crates/aho-corasick-1.1.5/.cargo-checksum.json b/third_party/crates/aho-corasick-1.1.5/.cargo-checksum.json new file mode 100644 index 0000000..43783de --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"fa452b0920e10ae97397f4cb336b629f7cbe5de0bed8fd8046ce8c137c06e2a4",".github/FUNDING.yml":"0c65f392d32a8639ba7986bbb42ca124505b462122382f314c89d84c95dd27f1",".github/dependabot.yml":"ea498dc2f28b89e80cf42a1349ac48ba9a9cc4d53f6cb21333481a4611f95abb",".github/workflows/ci.yml":"2d9409bedd406551a2e7e62158f3826c54c38a15d35e2a1a660e47499ea1999e",".vim/coc-settings.json":"8237c8f41db352b0d83f1bb10a60bc2f60f56f3234afbf696b4075c8d4d62d9b","COPYING":"01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f","Cargo.lock":"9b5026fc4b42d1470b586a636f7c0e843ca9a056f4137727bf7ba36baaa9c9b0","Cargo.toml":"7ff243aed78547f286729c3c60d97577fbc685546120263548d7c4f2a3884bc1","Cargo.toml.orig":"6c8341535cb4f736c1ac6b18292df2c572ccb18740bb9325e42b22c67f9e27da","DESIGN.md":"59c960e1b73b1d7fb41e4df6c0c1b1fcf44dd2ebc8a349597a7d0595f8cb5130","LICENSE-MIT":"0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f","README.md":"afc4d559a98cf190029af0bf320fc0022725e349cd2a303aac860254e28f3c53","UNLICENSE":"7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c","rustfmt.toml":"1ca600239a27401c4a43f363cf3f38183a212affc1f31bff3ae93234bbaec228","src/ahocorasick.rs":"c699c07df70be45c666e128509ad571a7649d2073e4ae16ac1efd6793c9c6890","src/automaton.rs":"22258a3e118672413119f8f543a9b912cce954e63524575c0ebfdf9011f9c2dd","src/dfa.rs":"197075923eb9d760a552f4e8652310fd4f657736613a9b1444ae05ef5d525da3","src/lib.rs":"66dea84d227f269b2f14ecc8109a97e96245b56c22eef0e8ce03b2343b8d6e66","src/macros.rs":"c6c52ae05b24433cffaca7b78b3645d797862c5d5feffddf9f54909095ed6e05","src/nfa/contiguous.rs":"f435c131ce84927e5600109722d006533ea21442dddaf18e03286d8caed82389","src/nfa/mod.rs":"ee7b3109774d14bbad5239c16bb980dd6b8185ec136d94fbaf2f0dc27d5ffa15","src/nfa/noncontiguous.rs":"de94f02b04efd8744fb096759a8897c22012b0e0ca3ace161fd87c71befefe04","src/packed/api.rs":"d555a8c7a6da33b75f1f90e21ee970629f92f7bddd4b408b78656cd93fe1a3d6","src/packed/ext.rs":"66be06fde8558429da23a290584d4b9fae665bf64c2578db4fe5f5f3ee864869","src/packed/mod.rs":"0020cd6f07ba5c8955923a9516d7f758864260eda53a6b6f629131c45ddeec62","src/packed/pattern.rs":"0e4bca57d4b941495d31fc8246ad32904eed0cd89e3cda732ad35f4deeba3bef","src/packed/rabinkarp.rs":"403146eb1d838a84601d171393542340513cd1ee7ff750f2372161dd47746586","src/packed/teddy/README.md":"3a43194b64e221543d885176aba3beb1224a927385a20eca842daf6b0ea2f342","src/packed/teddy/builder.rs":"08ec116a4a842a2bb1221d296a2515ef3672c54906bed588fb733364c07855d3","src/packed/teddy/generic.rs":"ea252ab05b32cea7dd9d71e332071d243db7dd0362e049252a27e5881ba2bf39","src/packed/teddy/mod.rs":"17d741f7e2fb9dbac5ba7d1bd4542cf1e35e9f146ace728e23fe6bbed20028b2","src/packed/tests.rs":"8e2f56eb3890ed3876ecb47d3121996e416563127b6430110d7b516df3f83b4b","src/packed/vector.rs":"70c325cfa6f7c5c4c9a6af7b133b75a29e65990a7fe0b9a4c4ce3c3d5a0fe587","src/tests.rs":"c68192ab97b6161d0d6ee96fefd80cc7d14e4486ddcd8d1f82b5c92432c24ed5","src/transducer.rs":"02daa33a5d6dac41dcfd67f51df7c0d4a91c5131c781fb54c4de3520c585a6e1","src/util/alphabet.rs":"6dc22658a38deddc0279892035b18870d4585069e35ba7c7e649a24509acfbcc","src/util/buffer.rs":"f9e37f662c46c6ecd734458dedbe76c3bb0e84a93b6b0117c0d4ad3042413891","src/util/byte_frequencies.rs":"2fb85b381c038c1e44ce94294531cdcd339dca48b1e61f41455666e802cbbc9e","src/util/debug.rs":"ab301ad59aa912529cb97233a54a05914dd3cb2ec43e6fec7334170b97ac5998","src/util/error.rs":"ecccd60e7406305023efcc6adcc826eeeb083ab8f7fbfe3d97469438cd4c4e5c","src/util/int.rs":"e264e6abebf5622b59f6500210773db36048371c4e509c930263334095959a52","src/util/mod.rs":"7ab28d11323ecdbd982087f32eb8bceeee84f1a2583f3aae27039c36d58cf12c","src/util/prefilter.rs":"183e32aa9951d9957f89062e4a6ae7235df7060722a3c91995a3d36db5a98111","src/util/primitives.rs":"f89f3fa1d8db4e37de9ca767c6d05e346404837cade6d063bba68972fafa610b","src/util/remapper.rs":"9f12d911583a325c11806eeceb46d0dfec863cfcfa241aed84d31af73da746e5","src/util/search.rs":"ac16607e69a3ff24d27c0680dc9afcd5bb7b7cb272179d5475300a2035fdf7e3","src/util/special.rs":"7d2f9cb9dd9771f59816e829b2d96b1239996f32939ba98764e121696c52b146"},"package":"c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"} \ No newline at end of file diff --git a/third_party/crates/aho-corasick-1.1.5/.cargo_vcs_info.json b/third_party/crates/aho-corasick-1.1.5/.cargo_vcs_info.json new file mode 100644 index 0000000..d9dcecc --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "5178060ce73d91938f8582d0360e3be031380440" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/third_party/crates/aho-corasick-1.1.5/.github/FUNDING.yml b/third_party/crates/aho-corasick-1.1.5/.github/FUNDING.yml new file mode 100644 index 0000000..2869fec --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [BurntSushi] diff --git a/third_party/crates/aho-corasick-1.1.5/.github/dependabot.yml b/third_party/crates/aho-corasick-1.1.5/.github/dependabot.yml new file mode 100644 index 0000000..0048722 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 + +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: semiannually + open-pull-requests-limit: 1 + cooldown: + default-days: 30 + groups: + actions: + patterns: + - "*" diff --git a/third_party/crates/aho-corasick-1.1.5/.github/workflows/ci.yml b/third_party/crates/aho-corasick-1.1.5/.github/workflows/ci.yml new file mode 100644 index 0000000..05394df --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/.github/workflows/ci.yml @@ -0,0 +1,154 @@ +name: ci +on: + pull_request: + push: + branches: + - master + schedule: + - cron: '00 01 * * *' + +# The section is needed to drop write-all permissions that are granted on +# `schedule` event. By specifying any permission explicitly all others are set +# to none. By using the principle of least privilege the damage a compromised +# workflow can do (because of an injection or compromised third party tool or +# action) is restricted. Currently the worklow doesn't need any additional +# permission except for pulling the code. Adding labels to issues, commenting +# on pull-requests, etc. may need additional permissions: +# +# Syntax for this section: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +# +# Reference for how to assign permissions on a job-by-job basis: +# https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs +# +# Reference for available permissions that we can enable if needed: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token +permissions: + # to fetch code (actions/checkout) + contents: read + +jobs: + test: + name: test + env: + # For some builds, we use cross to test on 32-bit and big-endian + # systems. + CARGO: cargo + # When CARGO is set to CROSS, TARGET is set to `--target matrix.target`. + # Note that we only use cross on Linux, so setting a target on a + # different OS will just use normal cargo. + TARGET: + # Bump this as appropriate. We pin to a version to make sure CI + # continues to work as cross releases in the past have broken things + # in subtle ways. + CROSS_VERSION: v0.2.5 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - build: pinned + os: ubuntu-latest + rust: 1.60.0 + - build: stable + os: ubuntu-latest + rust: stable + - build: stable-x86 + os: ubuntu-latest + rust: stable + target: i686-unknown-linux-gnu + - build: stable-aarch64 + os: ubuntu-latest + rust: stable + target: aarch64-unknown-linux-gnu + - build: stable-powerpc64 + os: ubuntu-latest + rust: stable + target: powerpc64-unknown-linux-gnu + - build: stable-s390x + os: ubuntu-latest + rust: stable + target: s390x-unknown-linux-gnu + - build: beta + os: ubuntu-latest + rust: beta + - build: nightly + os: ubuntu-latest + rust: nightly + - build: macos + os: macos-latest + rust: stable + - build: win-msvc + os: windows-latest + rust: stable + - build: win-gnu + os: windows-latest + rust: stable-x86_64-gnu + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 + with: + toolchain: ${{ matrix.rust }} + - name: Install and configure Cross + if: matrix.os == 'ubuntu-latest' && matrix.target != '' + run: | + # In the past, new releases of 'cross' have broken CI. So for now, we + # pin it. We also use their pre-compiled binary releases because cross + # has over 100 dependencies and takes a bit to compile. + dir="$RUNNER_TEMP/cross-download" + mkdir "$dir" + echo "$dir" >> $GITHUB_PATH + cd "$dir" + curl -LO "https://github.com/cross-rs/cross/releases/download/$CROSS_VERSION/cross-x86_64-unknown-linux-musl.tar.gz" + tar xf cross-x86_64-unknown-linux-musl.tar.gz + + # We used to install 'cross' from master, but it kept failing. So now + # we build from a known-good version until 'cross' becomes more stable + # or we find an alternative. Notably, between v0.2.1 and current + # master (2022-06-14), the number of Cross's dependencies has doubled. + echo "CARGO=cross" >> $GITHUB_ENV + echo "TARGET=--target ${{ matrix.target }}" >> $GITHUB_ENV + - name: Show command used for Cargo + run: | + echo "cargo command is: ${{ env.CARGO }}" + echo "target flag is: ${{ env.TARGET }}" + - name: Show CPU info for debugging + if: matrix.os == 'ubuntu-latest' + run: lscpu + # See: https://github.com/rust-lang/regex/blob/a2887636930156023172e4b376a6febad4e49120/.github/workflows/ci.yml#L145-L163 + - name: Pin memchr to 2.6.2 + if: matrix.build == 'pinned' + run: | + cargo update -p memchr --precise 2.6.2 + cargo update -p log --precise 0.4.20 + - run: ${{ env.CARGO }} build --verbose $TARGET + - run: ${{ env.CARGO }} doc --verbose $TARGET + - run: ${{ env.CARGO }} test --verbose $TARGET + - run: ${{ env.CARGO }} test --lib --verbose --no-default-features --features std,perf-literal $TARGET + - run: ${{ env.CARGO }} test --lib --verbose --no-default-features $TARGET + - run: ${{ env.CARGO }} test --lib --verbose --no-default-features --features std $TARGET + - run: ${{ env.CARGO }} test --lib --verbose --no-default-features --features perf-literal $TARGET + - run: ${{ env.CARGO }} test --lib --verbose --no-default-features --features std,perf-literal,logging $TARGET + - if: matrix.build == 'nightly' + run: ${{ env.CARGO }} build --manifest-path aho-corasick-debug/Cargo.toml $TARGET + + rustfmt: + name: rustfmt + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1 + with: + toolchain: stable + components: rustfmt + - name: Check formatting + run: | + cargo fmt --all -- --check diff --git a/third_party/crates/aho-corasick-1.1.5/.vim/coc-settings.json b/third_party/crates/aho-corasick-1.1.5/.vim/coc-settings.json new file mode 100644 index 0000000..887eb6f --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/.vim/coc-settings.json @@ -0,0 +1,12 @@ +{ + "rust-analyzer.linkedProjects": [ + "aho-corasick-debug/Cargo.toml", + "benchmarks/engines/rust-aho-corasick/Cargo.toml", + "benchmarks/engines/rust-daachorse/Cargo.toml", + "benchmarks/engines/rust-jetscii/Cargo.toml", + "benchmarks/engines/naive/Cargo.toml", + "benchmarks/shared/Cargo.toml", + "fuzz/Cargo.toml", + "Cargo.toml" + ] +} diff --git a/third_party/crates/aho-corasick-1.1.5/BUILD.bazel b/third_party/crates/aho-corasick-1.1.5/BUILD.bazel new file mode 100644 index 0000000..0a9dc8c --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/BUILD.bazel @@ -0,0 +1,117 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_library( + name = "aho_corasick", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "perf-literal", + "std", + ], + crate_root = "src/lib.rs", + edition = "2021", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=aho-corasick", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "1.1.5", + deps = [ + "//third_party/crates/memchr-2.8.3:memchr", + ], +) diff --git a/third_party/crates/aho-corasick-1.1.5/COPYING b/third_party/crates/aho-corasick-1.1.5/COPYING new file mode 100644 index 0000000..bb9c20a --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/COPYING @@ -0,0 +1,3 @@ +This project is dual-licensed under the Unlicense and MIT licenses. + +You may use this code under the terms of either license. diff --git a/third_party/crates/aho-corasick-1.1.5/Cargo.lock b/third_party/crates/aho-corasick-1.1.5/Cargo.lock new file mode 100644 index 0000000..dfbdc85 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/Cargo.lock @@ -0,0 +1,30 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +dependencies = [ + "doc-comment", + "log", + "memchr", +] + +[[package]] +name = "doc-comment" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" diff --git a/third_party/crates/aho-corasick-1.1.5/Cargo.toml b/third_party/crates/aho-corasick-1.1.5/Cargo.toml new file mode 100644 index 0000000..1444ce3 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/Cargo.toml @@ -0,0 +1,80 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.60.0" +name = "aho-corasick" +version = "1.1.5" +authors = ["Andrew Gallant "] +build = false +exclude = [ + "/aho-corasick-debug", + "/benchmarks", + "/tmp", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Fast multiple substring searching." +homepage = "https://github.com/BurntSushi/aho-corasick" +readme = "README.md" +keywords = [ + "string", + "search", + "text", + "pattern", + "multi", +] +categories = ["text-processing"] +license = "Unlicense OR MIT" +repository = "https://github.com/BurntSushi/aho-corasick" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", + "--generate-link-to-definition", +] + +[features] +default = [ + "std", + "perf-literal", +] +logging = ["dep:log"] +perf-literal = ["dep:memchr"] +std = ["memchr?/std"] + +[lib] +name = "aho_corasick" +path = "src/lib.rs" + +[dependencies.log] +version = "0.4.17" +optional = true + +[dependencies.memchr] +version = "2.4.0" +optional = true +default-features = false + +[dev-dependencies.doc-comment] +version = "0.3.3" + +[profile.bench] +debug = 2 + +[profile.release] +debug = 2 diff --git a/third_party/crates/aho-corasick-1.1.5/Cargo.toml.orig b/third_party/crates/aho-corasick-1.1.5/Cargo.toml.orig new file mode 100644 index 0000000..3219bb4 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/Cargo.toml.orig @@ -0,0 +1,74 @@ +[package] +name = "aho-corasick" +version = "1.1.5" #:version +authors = ["Andrew Gallant "] +description = "Fast multiple substring searching." +homepage = "https://github.com/BurntSushi/aho-corasick" +repository = "https://github.com/BurntSushi/aho-corasick" +readme = "README.md" +keywords = ["string", "search", "text", "pattern", "multi"] +license = "Unlicense OR MIT" +categories = ["text-processing"] +autotests = false +exclude = ["/aho-corasick-debug", "/benchmarks", "/tmp"] +edition = "2021" +rust-version = "1.60.0" + +[lib] +name = "aho_corasick" + +[features] +default = ["std", "perf-literal"] +std = ["memchr?/std"] + +# Enables prefilter optimizations that depend on external crates. +perf-literal = ["dep:memchr"] + +# Enable logging via the 'log' crate. This is useful for seeing messages about +# internal decisions and metrics. For example, how the choice of the internal +# Aho-Corasick implementation is used or the heap usage of an automaton. +logging = ["dep:log"] + +# Provides a trait impl for fst::Automaton for nfa::noncontiguous::NFA, +# nfa::contiguous::NFA and dfa::DFA. This is useful for searching an +# FST with an Aho-Corasick automaton. Note that this does not apply +# to the top-level 'AhoCorasick' type, as it does not implement the +# aho_corasick::automaton::Automaton trait, and thus enabling this feature does +# not cause it to implement fst::Automaton either. +# +# NOTE: Currently this feature is not available as `fst` is not at 1.0 yet, +# and this would make `fst` a public dependency. If you absolutely need this, +# you can copy the (very small) src/transducer.rs file to your tree. It +# specifically does not use any private APIs and should work after replacing +# 'crate::' with 'aho_corasick::'. +# +# NOTE: I think my current plan is to flip this around an add an optional +# dependency on 'aho-corasick' to the 'fst' crate and move the trait impls +# there. But I haven't gotten around to it yet. +# transducer = ["fst"] + +[dependencies] +log = { version = "0.4.17", optional = true } +memchr = { version = "2.4.0", default-features = false, optional = true } + +[dev-dependencies] +doc-comment = "0.3.3" +# fst = "0.4.5" + +[package.metadata.docs.rs] +# We want to document all features. +all-features = true +# This opts into a nightly unstable option to show the features that need to be +# enabled for public API items. To do that, we set 'docsrs', and when that's +# enabled, we enable the 'doc_cfg' feature. +# +# To test this locally, run: +# +# RUSTDOCFLAGS="--cfg docsrs" cargo +nightly doc --all-features +rustdoc-args = ["--cfg", "docsrs", "--generate-link-to-definition"] + +[profile.release] +debug = true + +[profile.bench] +debug = true diff --git a/third_party/crates/aho-corasick-1.1.5/DESIGN.md b/third_party/crates/aho-corasick-1.1.5/DESIGN.md new file mode 100644 index 0000000..f911f0c --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/DESIGN.md @@ -0,0 +1,481 @@ +This document describes the internal design of this crate, which is an object +lesson in what happens when you take a fairly simple old algorithm like +Aho-Corasick and make it fast and production ready. + +The target audience of this document is Rust programmers that have some +familiarity with string searching, however, one does not need to know the +Aho-Corasick algorithm in order to read this (it is explained below). One +should, however, know what a trie is. (If you don't, go read its Wikipedia +article.) + +The center-piece of this crate is an implementation of Aho-Corasick. On its +own, Aho-Corasick isn't that complicated. The complex pieces come from the +different variants of Aho-Corasick implemented in this crate. Specifically, +they are: + +* Aho-Corasick as a noncontiguous NFA. States have their transitions + represented sparsely, and each state puts its transitions in its own separate + allocation. Hence the same "noncontiguous." +* Aho-Corasick as a contiguous NFA. This NFA uses a single allocation to + represent the transitions of all states. That is, transitions are laid out + contiguously in memory. Moreover, states near the starting state are + represented densely, such that finding the next state ID takes a constant + number of instructions. +* Aho-Corasick as a DFA. In this case, all states are represented densely in + a transition table that uses one allocation. +* Supporting "standard" match semantics, along with its overlapping variant, + in addition to leftmost-first and leftmost-longest semantics. The "standard" + semantics are typically what you see in a textbook description of + Aho-Corasick. However, Aho-Corasick is also useful as an optimization in + regex engines, which often use leftmost-first or leftmost-longest semantics. + Thus, it is useful to implement those semantics here. The "standard" and + "leftmost" search algorithms are subtly different, and also require slightly + different construction algorithms. +* Support for ASCII case insensitive matching. +* Support for accelerating searches when the patterns all start with a small + number of fixed bytes. Or alternatively, when the patterns all contain a + small number of rare bytes. (Searching for these bytes uses SIMD vectorized + code courtesy of `memchr`.) +* Transparent support for alternative SIMD vectorized search routines for + smaller number of literals, such as the Teddy algorithm. We called these + "packed" search routines because they use SIMD. They can often be an order of + magnitude faster than just Aho-Corasick, but don't scale as well. +* Support for searching streams. This can reuse most of the underlying code, + but does require careful buffering support. +* Support for anchored searches, which permit efficient "is prefix" checks for + a large number of patterns. + +When you combine all of this together along with trying to make everything as +fast as possible, what you end up with is enitrely too much code with too much +`unsafe`. Alas, I was not smart enough to figure out how to reduce it. Instead, +we will explain it. + + +# Basics + +The fundamental problem this crate is trying to solve is to determine the +occurrences of possibly many patterns in a haystack. The naive way to solve +this is to look for a match for each pattern at each position in the haystack: + + for i in 0..haystack.len(): + for p in patterns.iter(): + if haystack[i..].starts_with(p.bytes()): + return Match(p.id(), i, i + p.bytes().len()) + +Those four lines are effectively all this crate does. The problem with those +four lines is that they are very slow, especially when you're searching for a +large number of patterns. + +While there are many different algorithms available to solve this, a popular +one is Aho-Corasick. It's a common solution because it's not too hard to +implement, scales quite well even when searching for thousands of patterns and +is generally pretty fast. Aho-Corasick does well here because, regardless of +the number of patterns you're searching for, it always visits each byte in the +haystack exactly once. This means, generally speaking, adding more patterns to +an Aho-Corasick automaton does not make it slower. (Strictly speaking, however, +this is not true, since a larger automaton will make less effective use of the +CPU's cache.) + +Aho-Corasick can be succinctly described as a trie with state transitions +between some of the nodes that efficiently instruct the search algorithm to +try matching alternative keys in the trie. The trick is that these state +transitions are arranged such that each byte of input needs to be inspected +only once. These state transitions are typically called "failure transitions," +because they instruct the searcher (the thing traversing the automaton while +reading from the haystack) what to do when a byte in the haystack does not +correspond to a valid transition in the current state of the trie. + +More formally, a failure transition points to a state in the automaton that may +lead to a match whose prefix is a proper suffix of the path traversed through +the trie so far. (If no such proper suffix exists, then the failure transition +points back to the start state of the trie, effectively restarting the search.) +This is perhaps simpler to explain pictorally. For example, let's say we built +an Aho-Corasick automaton with the following patterns: 'abcd' and 'cef'. The +trie looks like this: + + a - S1 - b - S2 - c - S3 - d - S4* + / + S0 - c - S5 - e - S6 - f - S7* + +where states marked with a `*` are match states (meaning, the search algorithm +should stop and report a match to the caller). + +So given this trie, it should be somewhat straight-forward to see how it can +be used to determine whether any particular haystack *starts* with either +`abcd` or `cef`. It's easy to express this in code: + + fn has_prefix(trie: &Trie, haystack: &[u8]) -> bool { + let mut state_id = trie.start(); + // If the empty pattern is in trie, then state_id is a match state. + if trie.is_match(state_id) { + return true; + } + for (i, &b) in haystack.iter().enumerate() { + state_id = match trie.next_state(state_id, b) { + Some(id) => id, + // If there was no transition for this state and byte, then we know + // the haystack does not start with one of the patterns in our trie. + None => return false, + }; + if trie.is_match(state_id) { + return true; + } + } + false + } + +And that's pretty much it. All we do is move through the trie starting with the +bytes at the beginning of the haystack. If we find ourselves in a position +where we can't move, or if we've looked through the entire haystack without +seeing a match state, then we know the haystack does not start with any of the +patterns in the trie. + +The meat of the Aho-Corasick algorithm is in how we add failure transitions to +our trie to keep searching efficient. Specifically, it permits us to not only +check whether a haystack *starts* with any one of a number of patterns, but +rather, whether the haystack contains any of a number of patterns *anywhere* in +the haystack. + +As mentioned before, failure transitions connect a proper suffix of the path +traversed through the trie before, with a path that leads to a match that has a +prefix corresponding to that proper suffix. So in our case, for patterns `abcd` +and `cef`, with a haystack `abcef`, we want to transition to state `S5` (from +the diagram above) from `S3` upon seeing that the byte following `c` is not +`d`. Namely, the proper suffix in this example is `c`, which is a prefix of +`cef`. So the modified diagram looks like this: + + + a - S1 - b - S2 - c - S3 - d - S4* + / / + / ---------------- + / / + S0 - c - S5 - e - S6 - f - S7* + +One thing that isn't shown in this diagram is that *all* states have a failure +transition, but only `S3` has a *non-trivial* failure transition. That is, all +other states have a failure transition back to the start state. So if our +haystack was `abzabcd`, then the searcher would transition back to `S0` after +seeing `z`, which effectively restarts the search. (Because there is no pattern +in our trie that has a prefix of `bz` or `z`.) + +The code for traversing this *automaton* or *finite state machine* (it is no +longer just a trie) is not that much different from the `has_prefix` code +above: + + fn contains(fsm: &FiniteStateMachine, haystack: &[u8]) -> bool { + let mut state_id = fsm.start(); + // If the empty pattern is in fsm, then state_id is a match state. + if fsm.is_match(state_id) { + return true; + } + for (i, &b) in haystack.iter().enumerate() { + // While the diagram above doesn't show this, we may wind up needing + // to follow multiple failure transitions before we land on a state + // in which we can advance. Therefore, when searching for the next + // state, we need to loop until we don't see a failure transition. + // + // This loop terminates because the start state has no empty + // transitions. Every transition from the start state either points to + // another state, or loops back to the start state. + loop { + match fsm.next_state(state_id, b) { + Some(id) => { + state_id = id; + break; + } + // Unlike our code above, if there was no transition for this + // state, then we don't quit. Instead, we look for this state's + // failure transition and follow that instead. + None => { + state_id = fsm.next_fail_state(state_id); + } + }; + } + if fsm.is_match(state_id) { + return true; + } + } + false + } + +Other than the complication around traversing failure transitions, this code +is still roughly "traverse the automaton with bytes from the haystack, and quit +when a match is seen." + +And that concludes our section on the basics. While we didn't go deep into how +the automaton is built (see `src/nfa/noncontiguous.rs`, which has detailed +comments about that), the basic structure of Aho-Corasick should be reasonably +clear. + + +# NFAs and DFAs + +There are generally two types of finite automata: non-deterministic finite +automata (NFA) and deterministic finite automata (DFA). The difference between +them is, principally, that an NFA can be in multiple states at once. This is +typically accomplished by things called _epsilon_ transitions, where one could +move to a new state without consuming any bytes from the input. (The other +mechanism by which NFAs can be in more than one state is where the same byte in +a particular state transitions to multiple distinct states.) In contrast, a DFA +can only ever be in one state at a time. A DFA has no epsilon transitions, and +for any given state, a byte transitions to at most one other state. + +By this formulation, the Aho-Corasick automaton described in the previous +section is an NFA. This is because failure transitions are, effectively, +epsilon transitions. That is, whenever the automaton is in state `S`, it is +actually in the set of states that are reachable by recursively following +failure transitions from `S` until you reach the start state. (This means +that, for example, the start state is always active since the start state is +reachable via failure transitions from any state in the automaton.) + +NFAs have a lot of nice properties. They tend to be easier to construct, and +also tend to use less memory. However, their primary downside is that they are +typically slower to execute a search with. For example, the code above showing +how to search with an Aho-Corasick automaton needs to potentially iterate +through many failure transitions for every byte of input. While this is a +fairly small amount of overhead, this can add up, especially if the automaton +has a lot of overlapping patterns with a lot of failure transitions. + +A DFA's search code, by contrast, looks like this: + + fn contains(dfa: &DFA, haystack: &[u8]) -> bool { + let mut state_id = dfa.start(); + // If the empty pattern is in dfa, then state_id is a match state. + if dfa.is_match(state_id) { + return true; + } + for (i, &b) in haystack.iter().enumerate() { + // An Aho-Corasick DFA *never* has a missing state that requires + // failure transitions to be followed. One byte of input advances the + // automaton by one state. Always. + state_id = dfa.next_state(state_id, b); + if dfa.is_match(state_id) { + return true; + } + } + false + } + +The search logic here is much simpler than for the NFA, and this tends to +translate into significant performance benefits as well, since there's a lot +less work being done for each byte in the haystack. How is this accomplished? +It's done by pre-following all failure transitions for all states for all bytes +in the alphabet, and then building a single state transition table. Building +this DFA can be much more costly than building the NFA, and use much more +memory, but the better performance can be worth it. + +Users of this crate can actually choose between using one of two possible NFAs +(noncontiguous or contiguous) or a DFA. By default, a contiguous NFA is used, +in most circumstances, but if the number of patterns is small enough a DFA will +be used. A contiguous NFA is chosen because it uses orders of magnitude less +memory than a DFA, takes only a little longer to build than a noncontiguous +NFA and usually gets pretty close to the search speed of a DFA. (Callers can +override this automatic selection via the `AhoCorasickBuilder::start_kind` +configuration.) + + +# More DFA tricks + +As described in the previous section, one of the downsides of using a DFA +is that it uses more memory and can take longer to build. One small way of +mitigating these concerns is to map the alphabet used by the automaton into +a smaller space. Typically, the alphabet of a DFA has 256 elements in it: +one element for each possible value that fits into a byte. However, in many +cases, one does not need the full alphabet. For example, if all patterns in an +Aho-Corasick automaton are ASCII letters, then this only uses up 52 distinct +bytes. As far as the automaton is concerned, the rest of the 204 bytes are +indistinguishable from one another: they will never disrciminate between a +match or a non-match. Therefore, in cases like that, the alphabet can be shrunk +to just 53 elements. One for each ASCII letter, and then another to serve as a +placeholder for every other unused byte. + +In practice, this library doesn't quite compute the optimal set of equivalence +classes, but it's close enough in most cases. The key idea is that this then +allows the transition table for the DFA to be potentially much smaller. The +downside of doing this, however, is that since the transition table is defined +in terms of this smaller alphabet space, every byte in the haystack must be +re-mapped to this smaller space. This requires an additional 256-byte table. +In practice, this can lead to a small search time hit, but it can be difficult +to measure. Moreover, it can sometimes lead to faster search times for bigger +automata, since it could be difference between more parts of the automaton +staying in the CPU cache or not. + +One other trick for DFAs employed by this crate is the notion of premultiplying +state identifiers. Specifically, the normal way to compute the next transition +in a DFA is via the following (assuming that the transition table is laid out +sequentially in memory, in row-major order, where the rows are states): + + next_state_id = dfa.transitions[current_state_id * 256 + current_byte] + +However, since the value `256` is a fixed constant, we can actually premultiply +the state identifiers in the table when we build the table initially. Then, the +next transition computation simply becomes: + + next_state_id = dfa.transitions[current_state_id + current_byte] + +This doesn't seem like much, but when this is being executed for every byte of +input that you're searching, saving that extra multiplication instruction can +add up. + +The same optimization works even when equivalence classes are enabled, as +described above. The only difference is that the premultiplication is by the +total number of equivalence classes instead of 256. + +There isn't much downside to premultiplying state identifiers, other than it +imposes a smaller limit on the total number of states in the DFA. Namely, with +premultiplied state identifiers, you run out of room in your state identifier +representation more rapidly than if the identifiers are just state indices. + +Both equivalence classes and premultiplication are always enabled. There is a +`AhoCorasickBuilder::byte_classes` configuration, but disabling this just makes +it so there are always 256 equivalence classes, i.e., every class corresponds +to precisely one byte. When it's disabled, the equivalence class map itself is +still used. The purpose of disabling it is when one is debugging the underlying +automaton. It can be easier to comprehend when it uses actual byte values for +its transitions instead of equivalence classes. + + +# Match semantics + +One of the more interesting things about this implementation of Aho-Corasick +that (as far as this author knows) separates it from other implementations, is +that it natively supports leftmost-first and leftmost-longest match semantics. +Briefly, match semantics refer to the decision procedure by which searching +will disambiguate matches when there are multiple to choose from: + +* **standard** match semantics emits matches as soon as they are detected by + the automaton. This is typically equivalent to the textbook non-overlapping + formulation of Aho-Corasick. +* **leftmost-first** match semantics means that 1) the next match is the match + starting at the leftmost position and 2) among multiple matches starting at + the same leftmost position, the match corresponding to the pattern provided + first by the caller is reported. +* **leftmost-longest** is like leftmost-first, except when there are multiple + matches starting at the same leftmost position, the pattern corresponding to + the longest match is returned. + +(The crate API documentation discusses these differences, with examples, in +more depth on the `MatchKind` type.) + +The reason why supporting these match semantics is important is because it +gives the user more control over the match procedure. For example, +leftmost-first permits users to implement match priority by simply putting the +higher priority patterns first. Leftmost-longest, on the other hand, permits +finding the longest possible match, which might be useful when trying to find +words matching a dictionary. Additionally, regex engines often want to use +Aho-Corasick as an optimization when searching for an alternation of literals. +In order to preserve correct match semantics, regex engines typically can't use +the standard textbook definition directly, since regex engines will implement +either leftmost-first (Perl-like) or leftmost-longest (POSIX) match semantics. + +Supporting leftmost semantics requires a couple key changes: + +* Constructing the Aho-Corasick automaton changes a bit in both how the trie is + constructed and how failure transitions are found. Namely, only a subset + of the failure transitions are added. Specifically, only the failure + transitions that either do not occur after a match or do occur after a match + but preserve that match are kept. (More details on this can be found in + `src/nfa/noncontiguous.rs`.) +* The search algorithm changes slightly. Since we are looking for the leftmost + match, we cannot quit as soon as a match is detected. Instead, after a match + is detected, we must keep searching until either the end of the input or + until a dead state is seen. (Dead states are not used for standard match + semantics. Dead states mean that searching should stop after a match has been + found.) + +Most other implementations of Aho-Corasick do support leftmost match semantics, +but they do it with more overhead at search time, or even worse, with a queue +of matches and sophisticated hijinks to disambiguate the matches. While our +construction algorithm becomes a bit more complicated, the correct match +semantics fall out from the structure of the automaton itself. + + +# Overlapping matches + +One of the nice properties of an Aho-Corasick automaton is that it can report +all possible matches, even when they overlap with one another. In this mode, +the match semantics don't matter, since all possible matches are reported. +Overlapping searches work just like regular searches, except the state +identifier at which the previous search left off is carried over to the next +search, so that it can pick up where it left off. If there are additional +matches at that state, then they are reported before resuming the search. + +Enabling leftmost-first or leftmost-longest match semantics causes the +automaton to use a subset of all failure transitions, which means that +overlapping searches cannot be used. Therefore, if leftmost match semantics are +used, attempting to do an overlapping search will return an error (or panic +when using the infallible APIs). Thus, to get overlapping searches, the caller +must use the default standard match semantics. This behavior was chosen because +there are only two alternatives, which were deemed worse: + +* Compile two automatons internally, one for standard semantics and one for + the semantics requested by the caller (if not standard). +* Create a new type, distinct from the `AhoCorasick` type, which has different + capabilities based on the configuration options. + +The first is untenable because of the amount of memory used by the automaton. +The second increases the complexity of the API too much by adding too many +types that do similar things. It is conceptually much simpler to keep all +searching isolated to a single type. + + +# Stream searching + +Since Aho-Corasick is an automaton, it is possible to do partial searches on +partial parts of the haystack, and then resume that search on subsequent pieces +of the haystack. This is useful when the haystack you're trying to search is +not stored contiguously in memory, or if one does not want to read the entire +haystack into memory at once. + +Currently, only standard semantics are supported for stream searching. This is +some of the more complicated code in this crate, and is something I would very +much like to improve. In particular, it currently has the restriction that it +must buffer at least enough of the haystack in memory in order to fit the +longest possible match. The difficulty in getting stream searching right is +that the implementation choices (such as the buffer size) often impact what the +API looks like and what it's allowed to do. + + +# Prefilters + +In some cases, Aho-Corasick is not the fastest way to find matches containing +multiple patterns. Sometimes, the search can be accelerated using highly +optimized SIMD routines. For example, consider searching the following +patterns: + + Sherlock + Moriarty + Watson + +It is plausible that it would be much faster to quickly look for occurrences of +the leading bytes, `S`, `M` or `W`, before trying to start searching via the +automaton. Indeed, this is exactly what this crate will do. + +When there are more than three distinct starting bytes, then this crate will +look for three distinct bytes occurring at any position in the patterns, while +preferring bytes that are heuristically determined to be rare over others. For +example: + + Abuzz + Sanchez + Vasquez + Topaz + Waltz + +Here, we have more than 3 distinct starting bytes, but all of the patterns +contain `z`, which is typically a rare byte. In this case, the prefilter will +scan for `z`, back up a bit, and then execute the Aho-Corasick automaton. + +If all of that fails, then a packed multiple substring algorithm will be +attempted. Currently, the only algorithm available for this is Teddy, but more +may be added in the future. Teddy is unlike the above prefilters in that it +confirms its own matches, so when Teddy is active, it might not be necessary +for Aho-Corasick to run at all. However, the current Teddy implementation +only works in `x86_64` when SSSE3 or AVX2 are available or in `aarch64` +(using NEON), and moreover, only works _well_ when there are a small number +of patterns (say, less than 100). Teddy also requires the haystack to be of a +certain length (more than 16-34 bytes). When the haystack is shorter than that, +Rabin-Karp is used instead. (See `src/packed/rabinkarp.rs`.) + +There is a more thorough description of Teddy at +[`src/packed/teddy/README.md`](src/packed/teddy/README.md). diff --git a/third_party/crates/aho-corasick-1.1.5/LICENSE-MIT b/third_party/crates/aho-corasick-1.1.5/LICENSE-MIT new file mode 100644 index 0000000..3b0a5dc --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Andrew Gallant + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/third_party/crates/aho-corasick-1.1.5/README.md b/third_party/crates/aho-corasick-1.1.5/README.md new file mode 100644 index 0000000..c0f525f --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/README.md @@ -0,0 +1,174 @@ +aho-corasick +============ +A library for finding occurrences of many patterns at once with SIMD +acceleration in some cases. This library provides multiple pattern +search principally through an implementation of the +[Aho-Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm), +which builds a finite state machine for executing searches in linear time. +Features include case insensitive matching, overlapping matches, fast searching +via SIMD and optional full DFA construction and search & replace in streams. + +[![Build status](https://github.com/BurntSushi/aho-corasick/workflows/ci/badge.svg)](https://github.com/BurntSushi/aho-corasick/actions) +[![crates.io](https://img.shields.io/crates/v/aho-corasick.svg)](https://crates.io/crates/aho-corasick) + +Dual-licensed under MIT or the [UNLICENSE](https://unlicense.org/). + + +### Documentation + +https://docs.rs/aho-corasick + + +### Usage + +Run `cargo add aho-corasick` to automatically add this crate as a dependency +in your `Cargo.toml` file. + + +### Example: basic searching + +This example shows how to search for occurrences of multiple patterns +simultaneously. Each match includes the pattern that matched along with the +byte offsets of the match. + +```rust +use aho_corasick::{AhoCorasick, PatternID}; + +let patterns = &["apple", "maple", "Snapple"]; +let haystack = "Nobody likes maple in their apple flavored Snapple."; + +let ac = AhoCorasick::new(patterns).unwrap(); +let mut matches = vec![]; +for mat in ac.find_iter(haystack) { + matches.push((mat.pattern(), mat.start(), mat.end())); +} +assert_eq!(matches, vec![ + (PatternID::must(1), 13, 18), + (PatternID::must(0), 28, 33), + (PatternID::must(2), 43, 50), +]); +``` + + +### Example: ASCII case insensitivity + +This is like the previous example, but matches `Snapple` case insensitively +using `AhoCorasickBuilder`: + +```rust +use aho_corasick::{AhoCorasick, PatternID}; + +let patterns = &["apple", "maple", "snapple"]; +let haystack = "Nobody likes maple in their apple flavored Snapple."; + +let ac = AhoCorasick::builder() + .ascii_case_insensitive(true) + .build(patterns) + .unwrap(); +let mut matches = vec![]; +for mat in ac.find_iter(haystack) { + matches.push((mat.pattern(), mat.start(), mat.end())); +} +assert_eq!(matches, vec![ + (PatternID::must(1), 13, 18), + (PatternID::must(0), 28, 33), + (PatternID::must(2), 43, 50), +]); +``` + + +### Example: replacing matches in a stream + +This example shows how to execute a search and replace on a stream without +loading the entire stream into memory first. + +```rust,ignore +use aho_corasick::AhoCorasick; + +let patterns = &["fox", "brown", "quick"]; +let replace_with = &["sloth", "grey", "slow"]; + +// In a real example, these might be `std::fs::File`s instead. All you need to +// do is supply a pair of `std::io::Read` and `std::io::Write` implementations. +let rdr = "The quick brown fox."; +let mut wtr = vec![]; + +let ac = AhoCorasick::new(patterns).unwrap(); +ac.stream_replace_all(rdr.as_bytes(), &mut wtr, replace_with) + .expect("stream_replace_all failed"); +assert_eq!(b"The slow grey sloth.".to_vec(), wtr); +``` + + +### Example: finding the leftmost first match + +In the textbook description of Aho-Corasick, its formulation is typically +structured such that it reports all possible matches, even when they overlap +with another. In many cases, overlapping matches may not be desired, such as +the case of finding all successive non-overlapping matches like you might with +a standard regular expression. + +Unfortunately the "obvious" way to modify the Aho-Corasick algorithm to do +this doesn't always work in the expected way, since it will report matches as +soon as they are seen. For example, consider matching the regex `Samwise|Sam` +against the text `Samwise`. Most regex engines (that are Perl-like, or +non-POSIX) will report `Samwise` as a match, but the standard Aho-Corasick +algorithm modified for reporting non-overlapping matches will report `Sam`. + +A novel contribution of this library is the ability to change the match +semantics of Aho-Corasick (without additional search time overhead) such that +`Samwise` is reported instead. For example, here's the standard approach: + +```rust +use aho_corasick::AhoCorasick; + +let patterns = &["Samwise", "Sam"]; +let haystack = "Samwise"; + +let ac = AhoCorasick::new(patterns).unwrap(); +let mat = ac.find(haystack).expect("should have a match"); +assert_eq!("Sam", &haystack[mat.start()..mat.end()]); +``` + +And now here's the leftmost-first version, which matches how a Perl-like +regex will work: + +```rust +use aho_corasick::{AhoCorasick, MatchKind}; + +let patterns = &["Samwise", "Sam"]; +let haystack = "Samwise"; + +let ac = AhoCorasick::builder() + .match_kind(MatchKind::LeftmostFirst) + .build(patterns) + .unwrap(); +let mat = ac.find(haystack).expect("should have a match"); +assert_eq!("Samwise", &haystack[mat.start()..mat.end()]); +``` + +In addition to leftmost-first semantics, this library also supports +leftmost-longest semantics, which match the POSIX behavior of a regular +expression alternation. See `MatchKind` in the docs for more details. + + +### Minimum Rust version policy + +This crate's minimum supported `rustc` version is `1.60.0`. + +The current policy is that the minimum Rust version required to use this crate +can be increased in minor version updates. For example, if `crate 1.0` requires +Rust 1.20.0, then `crate 1.0.z` for all values of `z` will also require Rust +1.20.0 or newer. However, `crate 1.y` for `y > 0` may require a newer minimum +version of Rust. + +In general, this crate will be conservative with respect to the minimum +supported version of Rust. + + +### FFI bindings + +* [G-Research/ahocorasick_rs](https://github.com/G-Research/ahocorasick_rs/) +is a Python wrapper for this library. +* [tmikus/ahocorasick_rs](https://github.com/tmikus/ahocorasick_rs) is a Go + wrapper for this library. diff --git a/third_party/crates/aho-corasick-1.1.5/UNLICENSE b/third_party/crates/aho-corasick-1.1.5/UNLICENSE new file mode 100644 index 0000000..68a49da --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/third_party/crates/aho-corasick-1.1.5/rustfmt.toml b/third_party/crates/aho-corasick-1.1.5/rustfmt.toml new file mode 100644 index 0000000..aa37a21 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 79 +use_small_heuristics = "max" diff --git a/third_party/crates/aho-corasick-1.1.5/src/ahocorasick.rs b/third_party/crates/aho-corasick-1.1.5/src/ahocorasick.rs new file mode 100644 index 0000000..2947627 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/ahocorasick.rs @@ -0,0 +1,2789 @@ +use core::{ + fmt::Debug, + panic::{RefUnwindSafe, UnwindSafe}, +}; + +use alloc::{string::String, sync::Arc, vec::Vec}; + +use crate::{ + automaton::{self, Automaton, OverlappingState}, + dfa, + nfa::{contiguous, noncontiguous}, + util::{ + error::{BuildError, MatchError}, + prefilter::Prefilter, + primitives::{PatternID, StateID}, + search::{Anchored, Input, Match, MatchKind, StartKind}, + }, +}; + +/// An automaton for searching multiple strings in linear time. +/// +/// The `AhoCorasick` type supports a few basic ways of constructing an +/// automaton, with the default being [`AhoCorasick::new`]. However, there +/// are a fair number of configurable options that can be set by using +/// [`AhoCorasickBuilder`] instead. Such options include, but are not limited +/// to, how matches are determined, simple case insensitivity, whether to use a +/// DFA or not and various knobs for controlling the space-vs-time trade offs +/// taken when building the automaton. +/// +/// # Resource usage +/// +/// Aho-Corasick automatons are always constructed in `O(p)` time, where +/// `p` is the combined length of all patterns being searched. With that +/// said, building an automaton can be fairly costly because of high constant +/// factors, particularly when enabling the [DFA](AhoCorasickKind::DFA) option +/// with [`AhoCorasickBuilder::kind`]. For this reason, it's generally a good +/// idea to build an automaton once and reuse it as much as possible. +/// +/// Aho-Corasick automatons can also use a fair bit of memory. To get +/// a concrete idea of how much memory is being used, try using the +/// [`AhoCorasick::memory_usage`] method. +/// +/// To give a quick idea of the differences between Aho-Corasick +/// implementations and their resource usage, here's a sample of construction +/// times and heap memory used after building an automaton from 100,000 +/// randomly selected titles from Wikipedia: +/// +/// * 99MB for a [`noncontiguous::NFA`] in 240ms. +/// * 21MB for a [`contiguous::NFA`] in 275ms. +/// * 1.6GB for a [`dfa::DFA`] in 1.88s. +/// +/// (Note that the memory usage above reflects the size of each automaton and +/// not peak memory usage. For example, building a contiguous NFA requires +/// first building a noncontiguous NFA. Once the contiguous NFA is built, the +/// noncontiguous NFA is freed.) +/// +/// This experiment very strongly argues that a contiguous NFA is often the +/// best balance in terms of resource usage. It takes a little longer to build, +/// but its memory usage is quite small. Its search speed (not listed) is +/// also often faster than a noncontiguous NFA, but a little slower than a +/// DFA. Indeed, when no specific [`AhoCorasickKind`] is used (which is the +/// default), a contiguous NFA is used in most cases. +/// +/// The only "catch" to using a contiguous NFA is that, because of its variety +/// of compression tricks, it may not be able to support automatons as large as +/// what the noncontiguous NFA supports. In which case, building a contiguous +/// NFA will fail and (by default) `AhoCorasick` will automatically fall +/// back to a noncontiguous NFA. (This typically only happens when building +/// automatons from millions of patterns.) Otherwise, the small additional time +/// for building a contiguous NFA is almost certainly worth it. +/// +/// # Cloning +/// +/// The `AhoCorasick` type uses thread safe reference counting internally. It +/// is guaranteed that it is cheap to clone. +/// +/// # Search configuration +/// +/// Most of the search routines accept anything that can be cheaply converted +/// to an [`Input`]. This includes `&[u8]`, `&str` and `Input` itself. +/// +/// # Construction failure +/// +/// It is generally possible for building an Aho-Corasick automaton to fail. +/// Construction can fail in generally one way: when the inputs provided are +/// too big. Whether that's a pattern that is too long, too many patterns +/// or some combination of both. A first approximation for the scale at which +/// construction can fail is somewhere around "millions of patterns." +/// +/// For that reason, if you're building an Aho-Corasick automaton from +/// untrusted input (or input that doesn't have any reasonable bounds on its +/// size), then it is strongly recommended to handle the possibility of an +/// error. +/// +/// If you're constructing an Aho-Corasick automaton from static or trusted +/// data, then it is likely acceptable to panic (by calling `unwrap()` or +/// `expect()`) if construction fails. +/// +/// # Fallibility +/// +/// The `AhoCorasick` type provides a number of methods for searching, as one +/// might expect. Depending on how the Aho-Corasick automaton was built and +/// depending on the search configuration, it is possible for a search to +/// return an error. Since an error is _never_ dependent on the actual contents +/// of the haystack, this type provides both infallible and fallible methods +/// for searching. The infallible methods panic if an error occurs, and can be +/// used for convenience and when you know the search will never return an +/// error. +/// +/// For example, the [`AhoCorasick::find_iter`] method is the infallible +/// version of the [`AhoCorasick::try_find_iter`] method. +/// +/// Examples of errors that can occur: +/// +/// * Running a search that requires [`MatchKind::Standard`] semantics (such +/// as a stream or overlapping search) with an automaton that was built with +/// [`MatchKind::LeftmostFirst`] or [`MatchKind::LeftmostLongest`] semantics. +/// * Running an anchored search with an automaton that only supports +/// unanchored searches. (By default, `AhoCorasick` only supports unanchored +/// searches. But this can be toggled with [`AhoCorasickBuilder::start_kind`].) +/// * Running an unanchored search with an automaton that only supports +/// anchored searches. +/// +/// The common thread between the different types of errors is that they are +/// all rooted in the automaton construction and search configurations. If +/// those configurations are a static property of your program, then it is +/// reasonable to call infallible routines since you know an error will never +/// occur. And if one _does_ occur, then it's a bug in your program. +/// +/// To re-iterate, if the patterns, build or search configuration come from +/// user or untrusted data, then you should handle errors at build or search +/// time. If only the haystack comes from user or untrusted data, then there +/// should be no need to handle errors anywhere and it is generally encouraged +/// to `unwrap()` (or `expect()`) both build and search time calls. +/// +/// # Examples +/// +/// This example shows how to search for occurrences of multiple patterns +/// simultaneously in a case insensitive fashion. Each match includes the +/// pattern that matched along with the byte offsets of the match. +/// +/// ``` +/// use aho_corasick::{AhoCorasick, PatternID}; +/// +/// let patterns = &["apple", "maple", "snapple"]; +/// let haystack = "Nobody likes maple in their apple flavored Snapple."; +/// +/// let ac = AhoCorasick::builder() +/// .ascii_case_insensitive(true) +/// .build(patterns) +/// .unwrap(); +/// let mut matches = vec![]; +/// for mat in ac.find_iter(haystack) { +/// matches.push((mat.pattern(), mat.start(), mat.end())); +/// } +/// assert_eq!(matches, vec![ +/// (PatternID::must(1), 13, 18), +/// (PatternID::must(0), 28, 33), +/// (PatternID::must(2), 43, 50), +/// ]); +/// ``` +/// +/// This example shows how to replace matches with some other string: +/// +/// ``` +/// use aho_corasick::AhoCorasick; +/// +/// let patterns = &["fox", "brown", "quick"]; +/// let haystack = "The quick brown fox."; +/// let replace_with = &["sloth", "grey", "slow"]; +/// +/// let ac = AhoCorasick::new(patterns).unwrap(); +/// let result = ac.replace_all(haystack, replace_with); +/// assert_eq!(result, "The slow grey sloth."); +/// ``` +#[derive(Clone)] +pub struct AhoCorasick { + /// The underlying Aho-Corasick automaton. It's one of + /// nfa::noncontiguous::NFA, nfa::contiguous::NFA or dfa::DFA. + aut: Arc, + /// The specific Aho-Corasick kind chosen. This makes it possible to + /// inspect any `AhoCorasick` and know what kind of search strategy it + /// uses. + kind: AhoCorasickKind, + /// The start kind of this automaton as configured by the caller. + /// + /// We don't really *need* to put this here, since the underlying automaton + /// will correctly return errors if the caller requests an unsupported + /// search type. But we do keep this here for API behavior consistency. + /// Namely, the NFAs in this crate support both unanchored and anchored + /// searches unconditionally. There's no way to disable one or the other. + /// They always both work. But the DFA in this crate specifically only + /// supports both unanchored and anchored searches if it's configured to + /// do so. Why? Because for the DFA, supporting both essentially requires + /// two copies of the transition table: one generated by following failure + /// transitions from the original NFA and one generated by not following + /// those failure transitions. + /// + /// So why record the start kind here? Well, consider what happens + /// when no specific 'AhoCorasickKind' is selected by the caller and + /// 'StartKind::Unanchored' is used (both are the default). It *might* + /// result in using a DFA or it might pick an NFA. If it picks an NFA, the + /// caller would then be able to run anchored searches, even though the + /// caller only asked for support for unanchored searches. Maybe that's + /// fine, but what if the DFA was chosen instead? Oops, the caller would + /// get an error. + /// + /// Basically, it seems bad to return an error or not based on some + /// internal implementation choice. So we smooth things out and ensure + /// anchored searches *always* report an error when only unanchored support + /// was asked for (and vice versa), even if the underlying automaton + /// supports it. + start_kind: StartKind, +} + +/// Convenience constructors for an Aho-Corasick searcher. To configure the +/// searcher, use an [`AhoCorasickBuilder`] instead. +impl AhoCorasick { + /// Create a new Aho-Corasick automaton using the default configuration. + /// + /// The default configuration optimizes for less space usage, but at the + /// expense of longer search times. To change the configuration, use + /// [`AhoCorasickBuilder`]. + /// + /// This uses the default [`MatchKind::Standard`] match semantics, which + /// reports a match as soon as it is found. This corresponds to the + /// standard match semantics supported by textbook descriptions of the + /// Aho-Corasick algorithm. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, PatternID}; + /// + /// let ac = AhoCorasick::new(&["foo", "bar", "baz"]).unwrap(); + /// assert_eq!( + /// Some(PatternID::must(1)), + /// ac.find("xxx bar xxx").map(|m| m.pattern()), + /// ); + /// ``` + pub fn new(patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + AhoCorasickBuilder::new().build(patterns) + } + + /// A convenience method for returning a new Aho-Corasick builder. + /// + /// This usually permits one to just import the `AhoCorasick` type. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Match, MatchKind}; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(&["samwise", "sam"]) + /// .unwrap(); + /// assert_eq!(Some(Match::must(0, 0..7)), ac.find("samwise")); + /// ``` + pub fn builder() -> AhoCorasickBuilder { + AhoCorasickBuilder::new() + } +} + +/// Infallible search routines. These APIs panic when the underlying search +/// would otherwise fail. Infallible routines are useful because the errors are +/// a result of both search-time configuration and what configuration is used +/// to build the Aho-Corasick searcher. Both of these things are not usually +/// the result of user input, and thus, an error is typically indicative of a +/// programmer error. In cases where callers want errors instead of panics, use +/// the corresponding `try` method in the section below. +impl AhoCorasick { + /// Returns true if and only if this automaton matches the haystack at any + /// position. + /// + /// `input` may be any type that is cheaply convertible to an `Input`. This + /// includes, but is not limited to, `&str` and `&[u8]`. + /// + /// Aside from convenience, when `AhoCorasick` was built with + /// leftmost-first or leftmost-longest semantics, this might result in a + /// search that visits less of the haystack than [`AhoCorasick::find`] + /// would otherwise. (For standard semantics, matches are always + /// immediately returned once they are seen, so there is no way for this to + /// do less work in that case.) + /// + /// Note that there is no corresponding fallible routine for this method. + /// If you need a fallible version of this, then [`AhoCorasick::try_find`] + /// can be used with [`Input::earliest`] enabled. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::AhoCorasick; + /// + /// let ac = AhoCorasick::new(&[ + /// "foo", "bar", "quux", "baz", + /// ]).unwrap(); + /// assert!(ac.is_match("xxx bar xxx")); + /// assert!(!ac.is_match("xxx qux xxx")); + /// ``` + pub fn is_match<'h, I: Into>>(&self, input: I) -> bool { + self.aut + .try_find(&input.into().earliest(true)) + .expect("AhoCorasick::try_find is not expected to fail") + .is_some() + } + + /// Returns the location of the first match according to the match + /// semantics that this automaton was constructed with. + /// + /// `input` may be any type that is cheaply convertible to an `Input`. This + /// includes, but is not limited to, `&str` and `&[u8]`. + /// + /// This is the infallible version of [`AhoCorasick::try_find`]. + /// + /// # Panics + /// + /// This panics when [`AhoCorasick::try_find`] would return an error. + /// + /// # Examples + /// + /// Basic usage, with standard semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::Standard) // default, not necessary + /// .build(patterns) + /// .unwrap(); + /// let mat = ac.find(haystack).expect("should have a match"); + /// assert_eq!("b", &haystack[mat.start()..mat.end()]); + /// ``` + /// + /// Now with leftmost-first semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let mat = ac.find(haystack).expect("should have a match"); + /// assert_eq!("abc", &haystack[mat.start()..mat.end()]); + /// ``` + /// + /// And finally, leftmost-longest semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostLongest) + /// .build(patterns) + /// .unwrap(); + /// let mat = ac.find(haystack).expect("should have a match"); + /// ``` + /// + /// # Example: configuring a search + /// + /// Because this method accepts anything that can be turned into an + /// [`Input`], it's possible to provide an `Input` directly in order to + /// configure the search. In this example, we show how to use the + /// `earliest` option to force the search to return as soon as it knows + /// a match has occurred. + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Input, MatchKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostLongest) + /// .build(patterns) + /// .unwrap(); + /// let mat = ac.find(Input::new(haystack).earliest(true)) + /// .expect("should have a match"); + /// // The correct leftmost-longest match here is 'abcd', but since we + /// // told the search to quit as soon as it knows a match has occurred, + /// // we get a different match back. + /// assert_eq!("b", &haystack[mat.start()..mat.end()]); + /// ``` + pub fn find<'h, I: Into>>(&self, input: I) -> Option { + self.try_find(input) + .expect("AhoCorasick::try_find is not expected to fail") + } + + /// Returns the location of the first overlapping match in the given + /// input with respect to the current state of the underlying searcher. + /// + /// `input` may be any type that is cheaply convertible to an `Input`. This + /// includes, but is not limited to, `&str` and `&[u8]`. + /// + /// Overlapping searches do not report matches in their return value. + /// Instead, matches can be accessed via [`OverlappingState::get_match`] + /// after a search call. + /// + /// This is the infallible version of + /// [`AhoCorasick::try_find_overlapping`]. + /// + /// # Panics + /// + /// This panics when [`AhoCorasick::try_find_overlapping`] would + /// return an error. For example, when the Aho-Corasick searcher + /// doesn't support overlapping searches. (Only searchers built with + /// [`MatchKind::Standard`] semantics support overlapping searches.) + /// + /// # Example + /// + /// This shows how we can repeatedly call an overlapping search without + /// ever needing to explicitly re-slice the haystack. Overlapping search + /// works this way because searches depend on state saved during the + /// previous search. + /// + /// ``` + /// use aho_corasick::{ + /// automaton::OverlappingState, + /// AhoCorasick, Input, Match, + /// }; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let mut state = OverlappingState::start(); + /// + /// ac.find_overlapping(haystack, &mut state); + /// assert_eq!(Some(Match::must(2, 0..3)), state.get_match()); + /// + /// ac.find_overlapping(haystack, &mut state); + /// assert_eq!(Some(Match::must(0, 0..6)), state.get_match()); + /// + /// ac.find_overlapping(haystack, &mut state); + /// assert_eq!(Some(Match::must(2, 11..14)), state.get_match()); + /// + /// ac.find_overlapping(haystack, &mut state); + /// assert_eq!(Some(Match::must(2, 22..25)), state.get_match()); + /// + /// ac.find_overlapping(haystack, &mut state); + /// assert_eq!(Some(Match::must(0, 22..28)), state.get_match()); + /// + /// ac.find_overlapping(haystack, &mut state); + /// assert_eq!(Some(Match::must(1, 22..31)), state.get_match()); + /// + /// // No more match matches to be found. + /// ac.find_overlapping(haystack, &mut state); + /// assert_eq!(None, state.get_match()); + /// ``` + pub fn find_overlapping<'h, I: Into>>( + &self, + input: I, + state: &mut OverlappingState, + ) { + self.try_find_overlapping(input, state).expect( + "AhoCorasick::try_find_overlapping is not expected to fail", + ) + } + + /// Returns an iterator of non-overlapping matches, using the match + /// semantics that this automaton was constructed with. + /// + /// `input` may be any type that is cheaply convertible to an `Input`. This + /// includes, but is not limited to, `&str` and `&[u8]`. + /// + /// This is the infallible version of [`AhoCorasick::try_find_iter`]. + /// + /// # Panics + /// + /// This panics when [`AhoCorasick::try_find_iter`] would return an error. + /// + /// # Examples + /// + /// Basic usage, with standard semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind, PatternID}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::Standard) // default, not necessary + /// .build(patterns) + /// .unwrap(); + /// let matches: Vec = ac + /// .find_iter(haystack) + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![ + /// PatternID::must(2), + /// PatternID::must(2), + /// PatternID::must(2), + /// ], matches); + /// ``` + /// + /// Now with leftmost-first semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind, PatternID}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let matches: Vec = ac + /// .find_iter(haystack) + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![ + /// PatternID::must(0), + /// PatternID::must(2), + /// PatternID::must(0), + /// ], matches); + /// ``` + /// + /// And finally, leftmost-longest semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind, PatternID}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostLongest) + /// .build(patterns) + /// .unwrap(); + /// let matches: Vec = ac + /// .find_iter(haystack) + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![ + /// PatternID::must(0), + /// PatternID::must(2), + /// PatternID::must(1), + /// ], matches); + /// ``` + pub fn find_iter<'a, 'h, I: Into>>( + &'a self, + input: I, + ) -> FindIter<'a, 'h> { + self.try_find_iter(input) + .expect("AhoCorasick::try_find_iter is not expected to fail") + } + + /// Returns an iterator of overlapping matches. Stated differently, this + /// returns an iterator of all possible matches at every position. + /// + /// `input` may be any type that is cheaply convertible to an `Input`. This + /// includes, but is not limited to, `&str` and `&[u8]`. + /// + /// This is the infallible version of + /// [`AhoCorasick::try_find_overlapping_iter`]. + /// + /// # Panics + /// + /// This panics when `AhoCorasick::try_find_overlapping_iter` would return + /// an error. For example, when the Aho-Corasick searcher is built with + /// either leftmost-first or leftmost-longest match semantics. Stated + /// differently, overlapping searches require one to build the searcher + /// with [`MatchKind::Standard`] (it is the default). + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::{AhoCorasick, PatternID}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let matches: Vec = ac + /// .find_overlapping_iter(haystack) + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![ + /// PatternID::must(2), + /// PatternID::must(0), + /// PatternID::must(2), + /// PatternID::must(2), + /// PatternID::must(0), + /// PatternID::must(1), + /// ], matches); + /// ``` + pub fn find_overlapping_iter<'a, 'h, I: Into>>( + &'a self, + input: I, + ) -> FindOverlappingIter<'a, 'h> { + self.try_find_overlapping_iter(input).expect( + "AhoCorasick::try_find_overlapping_iter is not expected to fail", + ) + } + + /// Replace all matches with a corresponding value in the `replace_with` + /// slice given. Matches correspond to the same matches as reported by + /// [`AhoCorasick::find_iter`]. + /// + /// Replacements are determined by the index of the matching pattern. + /// For example, if the pattern with index `2` is found, then it is + /// replaced by `replace_with[2]`. + /// + /// This is the infallible version of [`AhoCorasick::try_replace_all`]. + /// + /// # Panics + /// + /// This panics when [`AhoCorasick::try_replace_all`] would return an + /// error. + /// + /// This also panics when `replace_with.len()` does not equal + /// [`AhoCorasick::patterns_len`]. + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let result = ac.replace_all(haystack, &["x", "y", "z"]); + /// assert_eq!("x the z to the xage", result); + /// ``` + pub fn replace_all(&self, haystack: &str, replace_with: &[B]) -> String + where + B: AsRef, + { + self.try_replace_all(haystack, replace_with) + .expect("AhoCorasick::try_replace_all is not expected to fail") + } + + /// Replace all matches using raw bytes with a corresponding value in the + /// `replace_with` slice given. Matches correspond to the same matches as + /// reported by [`AhoCorasick::find_iter`]. + /// + /// Replacements are determined by the index of the matching pattern. + /// For example, if the pattern with index `2` is found, then it is + /// replaced by `replace_with[2]`. + /// + /// This is the infallible version of + /// [`AhoCorasick::try_replace_all_bytes`]. + /// + /// # Panics + /// + /// This panics when [`AhoCorasick::try_replace_all_bytes`] would return an + /// error. + /// + /// This also panics when `replace_with.len()` does not equal + /// [`AhoCorasick::patterns_len`]. + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = b"append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let result = ac.replace_all_bytes(haystack, &["x", "y", "z"]); + /// assert_eq!(b"x the z to the xage".to_vec(), result); + /// ``` + pub fn replace_all_bytes( + &self, + haystack: &[u8], + replace_with: &[B], + ) -> Vec + where + B: AsRef<[u8]>, + { + self.try_replace_all_bytes(haystack, replace_with) + .expect("AhoCorasick::try_replace_all_bytes should not fail") + } + + /// Replace all matches using a closure called on each match. + /// Matches correspond to the same matches as reported by + /// [`AhoCorasick::find_iter`]. + /// + /// The closure accepts three parameters: the match found, the text of + /// the match and a string buffer with which to write the replaced text + /// (if any). If the closure returns `true`, then it continues to the next + /// match. If the closure returns `false`, then searching is stopped. + /// + /// Note that any matches with boundaries that don't fall on a valid UTF-8 + /// boundary are silently skipped. + /// + /// This is the infallible version of + /// [`AhoCorasick::try_replace_all_with`]. + /// + /// # Panics + /// + /// This panics when [`AhoCorasick::try_replace_all_with`] would return an + /// error. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let mut result = String::new(); + /// ac.replace_all_with(haystack, &mut result, |mat, _, dst| { + /// dst.push_str(&mat.pattern().as_usize().to_string()); + /// true + /// }); + /// assert_eq!("0 the 2 to the 0age", result); + /// ``` + /// + /// Stopping the replacement by returning `false` (continued from the + /// example above): + /// + /// ``` + /// # use aho_corasick::{AhoCorasick, MatchKind, PatternID}; + /// # let patterns = &["append", "appendage", "app"]; + /// # let haystack = "append the app to the appendage"; + /// # let ac = AhoCorasick::builder() + /// # .match_kind(MatchKind::LeftmostFirst) + /// # .build(patterns) + /// # .unwrap(); + /// let mut result = String::new(); + /// ac.replace_all_with(haystack, &mut result, |mat, _, dst| { + /// dst.push_str(&mat.pattern().as_usize().to_string()); + /// mat.pattern() != PatternID::must(2) + /// }); + /// assert_eq!("0 the 2 to the appendage", result); + /// ``` + pub fn replace_all_with( + &self, + haystack: &str, + dst: &mut String, + replace_with: F, + ) where + F: FnMut(&Match, &str, &mut String) -> bool, + { + self.try_replace_all_with(haystack, dst, replace_with) + .expect("AhoCorasick::try_replace_all_with should not fail") + } + + /// Replace all matches using raw bytes with a closure called on each + /// match. Matches correspond to the same matches as reported by + /// [`AhoCorasick::find_iter`]. + /// + /// The closure accepts three parameters: the match found, the text of + /// the match and a byte buffer with which to write the replaced text + /// (if any). If the closure returns `true`, then it continues to the next + /// match. If the closure returns `false`, then searching is stopped. + /// + /// This is the infallible version of + /// [`AhoCorasick::try_replace_all_with_bytes`]. + /// + /// # Panics + /// + /// This panics when [`AhoCorasick::try_replace_all_with_bytes`] would + /// return an error. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = b"append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let mut result = vec![]; + /// ac.replace_all_with_bytes(haystack, &mut result, |mat, _, dst| { + /// dst.extend(mat.pattern().as_usize().to_string().bytes()); + /// true + /// }); + /// assert_eq!(b"0 the 2 to the 0age".to_vec(), result); + /// ``` + /// + /// Stopping the replacement by returning `false` (continued from the + /// example above): + /// + /// ``` + /// # use aho_corasick::{AhoCorasick, MatchKind, PatternID}; + /// # let patterns = &["append", "appendage", "app"]; + /// # let haystack = b"append the app to the appendage"; + /// # let ac = AhoCorasick::builder() + /// # .match_kind(MatchKind::LeftmostFirst) + /// # .build(patterns) + /// # .unwrap(); + /// let mut result = vec![]; + /// ac.replace_all_with_bytes(haystack, &mut result, |mat, _, dst| { + /// dst.extend(mat.pattern().as_usize().to_string().bytes()); + /// mat.pattern() != PatternID::must(2) + /// }); + /// assert_eq!(b"0 the 2 to the appendage".to_vec(), result); + /// ``` + pub fn replace_all_with_bytes( + &self, + haystack: &[u8], + dst: &mut Vec, + replace_with: F, + ) where + F: FnMut(&Match, &[u8], &mut Vec) -> bool, + { + self.try_replace_all_with_bytes(haystack, dst, replace_with) + .expect("AhoCorasick::try_replace_all_with_bytes should not fail") + } + + /// Returns an iterator of non-overlapping matches in the given + /// stream. Matches correspond to the same matches as reported by + /// [`AhoCorasick::find_iter`]. + /// + /// The matches yielded by this iterator use absolute position offsets in + /// the stream given, where the first byte has index `0`. Matches are + /// yieled until the stream is exhausted. + /// + /// Each item yielded by the iterator is an `Result`, where an error is yielded if there was a problem + /// reading from the reader given. + /// + /// When searching a stream, an internal buffer is used. Therefore, callers + /// should avoiding providing a buffered reader, if possible. + /// + /// This is the infallible version of + /// [`AhoCorasick::try_stream_find_iter`]. Note that both methods return + /// iterators that produce `Result` values. The difference is that this + /// routine panics if _construction_ of the iterator failed. The `Result` + /// values yield by the iterator come from whether the given reader returns + /// an error or not during the search. + /// + /// # Memory usage + /// + /// In general, searching streams will use a constant amount of memory for + /// its internal buffer. The one requirement is that the internal buffer + /// must be at least the size of the longest possible match. In most use + /// cases, the default buffer size will be much larger than any individual + /// match. + /// + /// # Panics + /// + /// This panics when [`AhoCorasick::try_stream_find_iter`] would return + /// an error. For example, when the Aho-Corasick searcher doesn't support + /// stream searches. (Only searchers built with [`MatchKind::Standard`] + /// semantics support stream searches.) + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::{AhoCorasick, PatternID}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let mut matches = vec![]; + /// for result in ac.stream_find_iter(haystack.as_bytes()) { + /// let mat = result?; + /// matches.push(mat.pattern()); + /// } + /// assert_eq!(vec![ + /// PatternID::must(2), + /// PatternID::must(2), + /// PatternID::must(2), + /// ], matches); + /// + /// # Ok::<(), Box>(()) + /// ``` + #[cfg(feature = "std")] + pub fn stream_find_iter<'a, R: std::io::Read>( + &'a self, + rdr: R, + ) -> StreamFindIter<'a, R> { + self.try_stream_find_iter(rdr) + .expect("AhoCorasick::try_stream_find_iter should not fail") + } +} + +/// Fallible search routines. These APIs return an error in cases where the +/// infallible routines would panic. +impl AhoCorasick { + /// Returns the location of the first match according to the match + /// semantics that this automaton was constructed with, and according + /// to the given `Input` configuration. + /// + /// This is the fallible version of [`AhoCorasick::find`]. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the given `Input` configuration. + /// + /// For example, if the Aho-Corasick searcher only supports anchored + /// searches or only supports unanchored searches, then providing an + /// `Input` that requests an anchored (or unanchored) search when it isn't + /// supported would result in an error. + /// + /// # Example: leftmost-first searching + /// + /// Basic usage with leftmost-first semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind, Input}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "foo abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let mat = ac.try_find(haystack)?.expect("should have a match"); + /// assert_eq!("abc", &haystack[mat.span()]); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// # Example: anchored leftmost-first searching + /// + /// This shows how to anchor the search, so that even if the haystack + /// contains a match somewhere, a match won't be reported unless one can + /// be found that starts at the beginning of the search: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Anchored, Input, MatchKind, StartKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "foo abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .start_kind(StartKind::Anchored) + /// .build(patterns) + /// .unwrap(); + /// let input = Input::new(haystack).anchored(Anchored::Yes); + /// assert_eq!(None, ac.try_find(input)?); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// If the beginning of the search is changed to where a match begins, then + /// it will be found: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Anchored, Input, MatchKind, StartKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "foo abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .start_kind(StartKind::Anchored) + /// .build(patterns) + /// .unwrap(); + /// let input = Input::new(haystack).range(4..).anchored(Anchored::Yes); + /// let mat = ac.try_find(input)?.expect("should have a match"); + /// assert_eq!("abc", &haystack[mat.span()]); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// # Example: earliest leftmost-first searching + /// + /// This shows how to run an "earliest" search even when the Aho-Corasick + /// searcher was compiled with leftmost-first match semantics. In this + /// case, the search is stopped as soon as it is known that a match has + /// occurred, even if it doesn't correspond to the leftmost-first match. + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Input, MatchKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "foo abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let input = Input::new(haystack).earliest(true); + /// let mat = ac.try_find(input)?.expect("should have a match"); + /// assert_eq!("b", &haystack[mat.span()]); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn try_find<'h, I: Into>>( + &self, + input: I, + ) -> Result, MatchError> { + let input = input.into(); + enforce_anchored_consistency(self.start_kind, input.get_anchored())?; + self.aut.try_find(&input) + } + + /// Returns the location of the first overlapping match in the given + /// input with respect to the current state of the underlying searcher. + /// + /// Overlapping searches do not report matches in their return value. + /// Instead, matches can be accessed via [`OverlappingState::get_match`] + /// after a search call. + /// + /// This is the fallible version of [`AhoCorasick::find_overlapping`]. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the given `Input` configuration or if overlapping search is not + /// supported. + /// + /// One example is that only Aho-Corasicker searchers built with + /// [`MatchKind::Standard`] semantics support overlapping searches. Using + /// any other match semantics will result in this returning an error. + /// + /// # Example: basic usage + /// + /// This shows how we can repeatedly call an overlapping search without + /// ever needing to explicitly re-slice the haystack. Overlapping search + /// works this way because searches depend on state saved during the + /// previous search. + /// + /// ``` + /// use aho_corasick::{ + /// automaton::OverlappingState, + /// AhoCorasick, Input, Match, + /// }; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let mut state = OverlappingState::start(); + /// + /// ac.try_find_overlapping(haystack, &mut state)?; + /// assert_eq!(Some(Match::must(2, 0..3)), state.get_match()); + /// + /// ac.try_find_overlapping(haystack, &mut state)?; + /// assert_eq!(Some(Match::must(0, 0..6)), state.get_match()); + /// + /// ac.try_find_overlapping(haystack, &mut state)?; + /// assert_eq!(Some(Match::must(2, 11..14)), state.get_match()); + /// + /// ac.try_find_overlapping(haystack, &mut state)?; + /// assert_eq!(Some(Match::must(2, 22..25)), state.get_match()); + /// + /// ac.try_find_overlapping(haystack, &mut state)?; + /// assert_eq!(Some(Match::must(0, 22..28)), state.get_match()); + /// + /// ac.try_find_overlapping(haystack, &mut state)?; + /// assert_eq!(Some(Match::must(1, 22..31)), state.get_match()); + /// + /// // No more match matches to be found. + /// ac.try_find_overlapping(haystack, &mut state)?; + /// assert_eq!(None, state.get_match()); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// # Example: implementing your own overlapping iteration + /// + /// The previous example can be easily adapted to implement your own + /// iteration by repeatedly calling `try_find_overlapping` until either + /// an error occurs or no more matches are reported. + /// + /// This is effectively equivalent to the iterator returned by + /// [`AhoCorasick::try_find_overlapping_iter`], with the only difference + /// being that the iterator checks for errors before construction and + /// absolves the caller of needing to check for errors on every search + /// call. (Indeed, if the first `try_find_overlapping` call succeeds and + /// the same `Input` is given to subsequent calls, then all subsequent + /// calls are guaranteed to succeed.) + /// + /// ``` + /// use aho_corasick::{ + /// automaton::OverlappingState, + /// AhoCorasick, Input, Match, + /// }; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let mut state = OverlappingState::start(); + /// let mut matches = vec![]; + /// + /// loop { + /// ac.try_find_overlapping(haystack, &mut state)?; + /// let mat = match state.get_match() { + /// None => break, + /// Some(mat) => mat, + /// }; + /// matches.push(mat); + /// } + /// let expected = vec![ + /// Match::must(2, 0..3), + /// Match::must(0, 0..6), + /// Match::must(2, 11..14), + /// Match::must(2, 22..25), + /// Match::must(0, 22..28), + /// Match::must(1, 22..31), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// # Example: anchored iteration + /// + /// The previous example can also be adapted to implement + /// iteration over all anchored matches. In particular, + /// [`AhoCorasick::try_find_overlapping_iter`] does not support this + /// because it isn't totally clear what the match semantics ought to be. + /// + /// In this example, we will find all overlapping matches that start at + /// the beginning of our search. + /// + /// ``` + /// use aho_corasick::{ + /// automaton::OverlappingState, + /// AhoCorasick, Anchored, Input, Match, StartKind, + /// }; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .start_kind(StartKind::Anchored) + /// .build(patterns) + /// .unwrap(); + /// let input = Input::new(haystack).anchored(Anchored::Yes); + /// let mut state = OverlappingState::start(); + /// let mut matches = vec![]; + /// + /// loop { + /// ac.try_find_overlapping(input.clone(), &mut state)?; + /// let mat = match state.get_match() { + /// None => break, + /// Some(mat) => mat, + /// }; + /// matches.push(mat); + /// } + /// let expected = vec![ + /// Match::must(2, 0..3), + /// Match::must(0, 0..6), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn try_find_overlapping<'h, I: Into>>( + &self, + input: I, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + let input = input.into(); + enforce_anchored_consistency(self.start_kind, input.get_anchored())?; + self.aut.try_find_overlapping(&input, state) + } + + /// Returns an iterator of non-overlapping matches, using the match + /// semantics that this automaton was constructed with. + /// + /// This is the fallible version of [`AhoCorasick::find_iter`]. + /// + /// Note that the error returned by this method occurs during construction + /// of the iterator. The iterator itself yields `Match` values. That is, + /// once the iterator is constructed, the iteration itself will never + /// report an error. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the given `Input` configuration. + /// + /// For example, if the Aho-Corasick searcher only supports anchored + /// searches or only supports unanchored searches, then providing an + /// `Input` that requests an anchored (or unanchored) search when it isn't + /// supported would result in an error. + /// + /// # Example: leftmost-first searching + /// + /// Basic usage with leftmost-first semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Input, MatchKind, PatternID}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let matches: Vec = ac + /// .try_find_iter(Input::new(haystack))? + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![ + /// PatternID::must(0), + /// PatternID::must(2), + /// PatternID::must(0), + /// ], matches); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// # Example: anchored leftmost-first searching + /// + /// This shows how to anchor the search, such that all matches must begin + /// at the starting location of the search. For an iterator, an anchored + /// search implies that all matches are adjacent. + /// + /// ``` + /// use aho_corasick::{ + /// AhoCorasick, Anchored, Input, MatchKind, PatternID, StartKind, + /// }; + /// + /// let patterns = &["foo", "bar", "quux"]; + /// let haystack = "fooquuxbar foo"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .start_kind(StartKind::Anchored) + /// .build(patterns) + /// .unwrap(); + /// let matches: Vec = ac + /// .try_find_iter(Input::new(haystack).anchored(Anchored::Yes))? + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![ + /// PatternID::must(0), + /// PatternID::must(2), + /// PatternID::must(1), + /// // The final 'foo' is not found because it is not adjacent to the + /// // 'bar' match. It needs to be adjacent because our search is + /// // anchored. + /// ], matches); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn try_find_iter<'a, 'h, I: Into>>( + &'a self, + input: I, + ) -> Result, MatchError> { + let input = input.into(); + enforce_anchored_consistency(self.start_kind, input.get_anchored())?; + Ok(FindIter(self.aut.try_find_iter(input)?)) + } + + /// Returns an iterator of overlapping matches. + /// + /// This is the fallible version of [`AhoCorasick::find_overlapping_iter`]. + /// + /// Note that the error returned by this method occurs during construction + /// of the iterator. The iterator itself yields `Match` values. That is, + /// once the iterator is constructed, the iteration itself will never + /// report an error. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the given `Input` configuration or does not support overlapping + /// searches. + /// + /// One example is that only Aho-Corasicker searchers built with + /// [`MatchKind::Standard`] semantics support overlapping searches. Using + /// any other match semantics will result in this returning an error. + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Input, PatternID}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let matches: Vec = ac + /// .try_find_overlapping_iter(Input::new(haystack))? + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![ + /// PatternID::must(2), + /// PatternID::must(0), + /// PatternID::must(2), + /// PatternID::must(2), + /// PatternID::must(0), + /// PatternID::must(1), + /// ], matches); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// # Example: anchored overlapping search returns an error + /// + /// It isn't clear what the match semantics for anchored overlapping + /// iterators *ought* to be, so currently an error is returned. Callers + /// may use [`AhoCorasick::try_find_overlapping`] to implement their own + /// semantics if desired. + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Anchored, Input, StartKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "appendappendage app"; + /// + /// let ac = AhoCorasick::builder() + /// .start_kind(StartKind::Anchored) + /// .build(patterns) + /// .unwrap(); + /// let input = Input::new(haystack).anchored(Anchored::Yes); + /// assert!(ac.try_find_overlapping_iter(input).is_err()); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn try_find_overlapping_iter<'a, 'h, I: Into>>( + &'a self, + input: I, + ) -> Result, MatchError> { + let input = input.into(); + enforce_anchored_consistency(self.start_kind, input.get_anchored())?; + Ok(FindOverlappingIter(self.aut.try_find_overlapping_iter(input)?)) + } + + /// Replace all matches with a corresponding value in the `replace_with` + /// slice given. Matches correspond to the same matches as reported by + /// [`AhoCorasick::try_find_iter`]. + /// + /// Replacements are determined by the index of the matching pattern. + /// For example, if the pattern with index `2` is found, then it is + /// replaced by `replace_with[2]`. + /// + /// # Panics + /// + /// This panics when `replace_with.len()` does not equal + /// [`AhoCorasick::patterns_len`]. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the default `Input` configuration. More specifically, this occurs only + /// when the Aho-Corasick searcher does not support unanchored searches + /// since this replacement routine always does an unanchored search. + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let result = ac.try_replace_all(haystack, &["x", "y", "z"])?; + /// assert_eq!("x the z to the xage", result); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn try_replace_all( + &self, + haystack: &str, + replace_with: &[B], + ) -> Result + where + B: AsRef, + { + enforce_anchored_consistency(self.start_kind, Anchored::No)?; + self.aut.try_replace_all(haystack, replace_with) + } + + /// Replace all matches using raw bytes with a corresponding value in the + /// `replace_with` slice given. Matches correspond to the same matches as + /// reported by [`AhoCorasick::try_find_iter`]. + /// + /// Replacements are determined by the index of the matching pattern. + /// For example, if the pattern with index `2` is found, then it is + /// replaced by `replace_with[2]`. + /// + /// This is the fallible version of [`AhoCorasick::replace_all_bytes`]. + /// + /// # Panics + /// + /// This panics when `replace_with.len()` does not equal + /// [`AhoCorasick::patterns_len`]. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the default `Input` configuration. More specifically, this occurs only + /// when the Aho-Corasick searcher does not support unanchored searches + /// since this replacement routine always does an unanchored search. + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = b"append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let result = ac.try_replace_all_bytes(haystack, &["x", "y", "z"])?; + /// assert_eq!(b"x the z to the xage".to_vec(), result); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn try_replace_all_bytes( + &self, + haystack: &[u8], + replace_with: &[B], + ) -> Result, MatchError> + where + B: AsRef<[u8]>, + { + enforce_anchored_consistency(self.start_kind, Anchored::No)?; + self.aut.try_replace_all_bytes(haystack, replace_with) + } + + /// Replace all matches using a closure called on each match. + /// Matches correspond to the same matches as reported by + /// [`AhoCorasick::try_find_iter`]. + /// + /// The closure accepts three parameters: the match found, the text of + /// the match and a string buffer with which to write the replaced text + /// (if any). If the closure returns `true`, then it continues to the next + /// match. If the closure returns `false`, then searching is stopped. + /// + /// Note that any matches with boundaries that don't fall on a valid UTF-8 + /// boundary are silently skipped. + /// + /// This is the fallible version of [`AhoCorasick::replace_all_with`]. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the default `Input` configuration. More specifically, this occurs only + /// when the Aho-Corasick searcher does not support unanchored searches + /// since this replacement routine always does an unanchored search. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let mut result = String::new(); + /// ac.try_replace_all_with(haystack, &mut result, |mat, _, dst| { + /// dst.push_str(&mat.pattern().as_usize().to_string()); + /// true + /// })?; + /// assert_eq!("0 the 2 to the 0age", result); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// Stopping the replacement by returning `false` (continued from the + /// example above): + /// + /// ``` + /// # use aho_corasick::{AhoCorasick, MatchKind, PatternID}; + /// # let patterns = &["append", "appendage", "app"]; + /// # let haystack = "append the app to the appendage"; + /// # let ac = AhoCorasick::builder() + /// # .match_kind(MatchKind::LeftmostFirst) + /// # .build(patterns) + /// # .unwrap(); + /// let mut result = String::new(); + /// ac.try_replace_all_with(haystack, &mut result, |mat, _, dst| { + /// dst.push_str(&mat.pattern().as_usize().to_string()); + /// mat.pattern() != PatternID::must(2) + /// })?; + /// assert_eq!("0 the 2 to the appendage", result); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn try_replace_all_with( + &self, + haystack: &str, + dst: &mut String, + replace_with: F, + ) -> Result<(), MatchError> + where + F: FnMut(&Match, &str, &mut String) -> bool, + { + enforce_anchored_consistency(self.start_kind, Anchored::No)?; + self.aut.try_replace_all_with(haystack, dst, replace_with) + } + + /// Replace all matches using raw bytes with a closure called on each + /// match. Matches correspond to the same matches as reported by + /// [`AhoCorasick::try_find_iter`]. + /// + /// The closure accepts three parameters: the match found, the text of + /// the match and a byte buffer with which to write the replaced text + /// (if any). If the closure returns `true`, then it continues to the next + /// match. If the closure returns `false`, then searching is stopped. + /// + /// This is the fallible version of + /// [`AhoCorasick::replace_all_with_bytes`]. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the default `Input` configuration. More specifically, this occurs only + /// when the Aho-Corasick searcher does not support unanchored searches + /// since this replacement routine always does an unanchored search. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = b"append the app to the appendage"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let mut result = vec![]; + /// ac.try_replace_all_with_bytes(haystack, &mut result, |mat, _, dst| { + /// dst.extend(mat.pattern().as_usize().to_string().bytes()); + /// true + /// })?; + /// assert_eq!(b"0 the 2 to the 0age".to_vec(), result); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// Stopping the replacement by returning `false` (continued from the + /// example above): + /// + /// ``` + /// # use aho_corasick::{AhoCorasick, MatchKind, PatternID}; + /// # let patterns = &["append", "appendage", "app"]; + /// # let haystack = b"append the app to the appendage"; + /// # let ac = AhoCorasick::builder() + /// # .match_kind(MatchKind::LeftmostFirst) + /// # .build(patterns) + /// # .unwrap(); + /// let mut result = vec![]; + /// ac.try_replace_all_with_bytes(haystack, &mut result, |mat, _, dst| { + /// dst.extend(mat.pattern().as_usize().to_string().bytes()); + /// mat.pattern() != PatternID::must(2) + /// })?; + /// assert_eq!(b"0 the 2 to the appendage".to_vec(), result); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn try_replace_all_with_bytes( + &self, + haystack: &[u8], + dst: &mut Vec, + replace_with: F, + ) -> Result<(), MatchError> + where + F: FnMut(&Match, &[u8], &mut Vec) -> bool, + { + enforce_anchored_consistency(self.start_kind, Anchored::No)?; + self.aut.try_replace_all_with_bytes(haystack, dst, replace_with) + } + + /// Returns an iterator of non-overlapping matches in the given + /// stream. Matches correspond to the same matches as reported by + /// [`AhoCorasick::try_find_iter`]. + /// + /// The matches yielded by this iterator use absolute position offsets in + /// the stream given, where the first byte has index `0`. Matches are + /// yieled until the stream is exhausted. + /// + /// Each item yielded by the iterator is an `Result`, where an error is yielded if there was a problem + /// reading from the reader given. + /// + /// When searching a stream, an internal buffer is used. Therefore, callers + /// should avoiding providing a buffered reader, if possible. + /// + /// This is the fallible version of [`AhoCorasick::stream_find_iter`]. + /// Note that both methods return iterators that produce `Result` values. + /// The difference is that this routine returns an error if _construction_ + /// of the iterator failed. The `Result` values yield by the iterator + /// come from whether the given reader returns an error or not during the + /// search. + /// + /// # Memory usage + /// + /// In general, searching streams will use a constant amount of memory for + /// its internal buffer. The one requirement is that the internal buffer + /// must be at least the size of the longest possible match. In most use + /// cases, the default buffer size will be much larger than any individual + /// match. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the default `Input` configuration. More specifically, this occurs only + /// when the Aho-Corasick searcher does not support unanchored searches + /// since this stream searching routine always does an unanchored search. + /// + /// This also returns an error if the searcher does not support stream + /// searches. Only searchers built with [`MatchKind::Standard`] semantics + /// support stream searches. + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::{AhoCorasick, PatternID}; + /// + /// let patterns = &["append", "appendage", "app"]; + /// let haystack = "append the app to the appendage"; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let mut matches = vec![]; + /// for result in ac.try_stream_find_iter(haystack.as_bytes())? { + /// let mat = result?; + /// matches.push(mat.pattern()); + /// } + /// assert_eq!(vec![ + /// PatternID::must(2), + /// PatternID::must(2), + /// PatternID::must(2), + /// ], matches); + /// + /// # Ok::<(), Box>(()) + /// ``` + #[cfg(feature = "std")] + pub fn try_stream_find_iter<'a, R: std::io::Read>( + &'a self, + rdr: R, + ) -> Result, MatchError> { + enforce_anchored_consistency(self.start_kind, Anchored::No)?; + self.aut.try_stream_find_iter(rdr).map(StreamFindIter) + } + + /// Search for and replace all matches of this automaton in + /// the given reader, and write the replacements to the given + /// writer. Matches correspond to the same matches as reported by + /// [`AhoCorasick::try_find_iter`]. + /// + /// Replacements are determined by the index of the matching pattern. For + /// example, if the pattern with index `2` is found, then it is replaced by + /// `replace_with[2]`. + /// + /// After all matches are replaced, the writer is _not_ flushed. + /// + /// If there was a problem reading from the given reader or writing to the + /// given writer, then the corresponding `io::Error` is returned and all + /// replacement is stopped. + /// + /// When searching a stream, an internal buffer is used. Therefore, callers + /// should avoiding providing a buffered reader, if possible. However, + /// callers may want to provide a buffered writer. + /// + /// Note that there is currently no infallible version of this routine. + /// + /// # Memory usage + /// + /// In general, searching streams will use a constant amount of memory for + /// its internal buffer. The one requirement is that the internal buffer + /// must be at least the size of the longest possible match. In most use + /// cases, the default buffer size will be much larger than any individual + /// match. + /// + /// # Panics + /// + /// This panics when `replace_with.len()` does not equal + /// [`AhoCorasick::patterns_len`]. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the default `Input` configuration. More specifically, this occurs only + /// when the Aho-Corasick searcher does not support unanchored searches + /// since this stream searching routine always does an unanchored search. + /// + /// This also returns an error if the searcher does not support stream + /// searches. Only searchers built with [`MatchKind::Standard`] semantics + /// support stream searches. + /// + /// # Example: basic usage + /// + /// ``` + /// use aho_corasick::AhoCorasick; + /// + /// let patterns = &["fox", "brown", "quick"]; + /// let haystack = "The quick brown fox."; + /// let replace_with = &["sloth", "grey", "slow"]; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let mut result = vec![]; + /// ac.try_stream_replace_all( + /// haystack.as_bytes(), + /// &mut result, + /// replace_with, + /// )?; + /// assert_eq!(b"The slow grey sloth.".to_vec(), result); + /// + /// # Ok::<(), Box>(()) + /// ``` + #[cfg(feature = "std")] + pub fn try_stream_replace_all( + &self, + rdr: R, + wtr: W, + replace_with: &[B], + ) -> Result<(), std::io::Error> + where + R: std::io::Read, + W: std::io::Write, + B: AsRef<[u8]>, + { + enforce_anchored_consistency(self.start_kind, Anchored::No) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + self.aut.try_stream_replace_all(rdr, wtr, replace_with) + } + + /// Search the given reader and replace all matches of this automaton + /// using the given closure. The result is written to the given + /// writer. Matches correspond to the same matches as reported by + /// [`AhoCorasick::try_find_iter`]. + /// + /// The closure accepts three parameters: the match found, the text of + /// the match and the writer with which to write the replaced text (if any). + /// + /// After all matches are replaced, the writer is _not_ flushed. + /// + /// If there was a problem reading from the given reader or writing to the + /// given writer, then the corresponding `io::Error` is returned and all + /// replacement is stopped. + /// + /// When searching a stream, an internal buffer is used. Therefore, callers + /// should avoiding providing a buffered reader, if possible. However, + /// callers may want to provide a buffered writer. + /// + /// Note that there is currently no infallible version of this routine. + /// + /// # Memory usage + /// + /// In general, searching streams will use a constant amount of memory for + /// its internal buffer. The one requirement is that the internal buffer + /// must be at least the size of the longest possible match. In most use + /// cases, the default buffer size will be much larger than any individual + /// match. + /// + /// # Errors + /// + /// This returns an error when this Aho-Corasick searcher does not support + /// the default `Input` configuration. More specifically, this occurs only + /// when the Aho-Corasick searcher does not support unanchored searches + /// since this stream searching routine always does an unanchored search. + /// + /// This also returns an error if the searcher does not support stream + /// searches. Only searchers built with [`MatchKind::Standard`] semantics + /// support stream searches. + /// + /// # Example: basic usage + /// + /// ``` + /// use std::io::Write; + /// use aho_corasick::AhoCorasick; + /// + /// let patterns = &["fox", "brown", "quick"]; + /// let haystack = "The quick brown fox."; + /// + /// let ac = AhoCorasick::new(patterns).unwrap(); + /// let mut result = vec![]; + /// ac.try_stream_replace_all_with( + /// haystack.as_bytes(), + /// &mut result, + /// |mat, _, wtr| { + /// wtr.write_all(mat.pattern().as_usize().to_string().as_bytes()) + /// }, + /// )?; + /// assert_eq!(b"The 2 1 0.".to_vec(), result); + /// + /// # Ok::<(), Box>(()) + /// ``` + #[cfg(feature = "std")] + pub fn try_stream_replace_all_with( + &self, + rdr: R, + wtr: W, + replace_with: F, + ) -> Result<(), std::io::Error> + where + R: std::io::Read, + W: std::io::Write, + F: FnMut(&Match, &[u8], &mut W) -> Result<(), std::io::Error>, + { + enforce_anchored_consistency(self.start_kind, Anchored::No) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + self.aut.try_stream_replace_all_with(rdr, wtr, replace_with) + } +} + +/// Routines for querying information about the Aho-Corasick automaton. +impl AhoCorasick { + /// Returns the kind of the Aho-Corasick automaton used by this searcher. + /// + /// Knowing the Aho-Corasick kind is principally useful for diagnostic + /// purposes. In particular, if no specific kind was given to + /// [`AhoCorasickBuilder::kind`], then one is automatically chosen and + /// this routine will report which one. + /// + /// Note that the heuristics used for choosing which `AhoCorasickKind` + /// may be changed in a semver compatible release. + /// + /// # Examples + /// + /// ``` + /// use aho_corasick::{AhoCorasick, AhoCorasickKind}; + /// + /// let ac = AhoCorasick::new(&["foo", "bar", "quux", "baz"]).unwrap(); + /// // The specific Aho-Corasick kind chosen is not guaranteed! + /// assert_eq!(AhoCorasickKind::DFA, ac.kind()); + /// ``` + pub fn kind(&self) -> AhoCorasickKind { + self.kind + } + + /// Returns the type of starting search configuration supported by this + /// Aho-Corasick automaton. + /// + /// # Examples + /// + /// ``` + /// use aho_corasick::{AhoCorasick, StartKind}; + /// + /// let ac = AhoCorasick::new(&["foo", "bar", "quux", "baz"]).unwrap(); + /// assert_eq!(StartKind::Unanchored, ac.start_kind()); + /// ``` + pub fn start_kind(&self) -> StartKind { + self.start_kind + } + + /// Returns the match kind used by this automaton. + /// + /// The match kind is important because it determines what kinds of + /// matches are returned. Also, some operations (such as overlapping + /// search and stream searching) are only supported when using the + /// [`MatchKind::Standard`] match kind. + /// + /// # Examples + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let ac = AhoCorasick::new(&["foo", "bar", "quux", "baz"]).unwrap(); + /// assert_eq!(MatchKind::Standard, ac.match_kind()); + /// ``` + pub fn match_kind(&self) -> MatchKind { + self.aut.match_kind() + } + + /// Returns the length of the shortest pattern matched by this automaton. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::AhoCorasick; + /// + /// let ac = AhoCorasick::new(&["foo", "bar", "quux", "baz"]).unwrap(); + /// assert_eq!(3, ac.min_pattern_len()); + /// ``` + /// + /// Note that an `AhoCorasick` automaton has a minimum length of `0` if + /// and only if it can match the empty string: + /// + /// ``` + /// use aho_corasick::AhoCorasick; + /// + /// let ac = AhoCorasick::new(&["foo", "", "quux", "baz"]).unwrap(); + /// assert_eq!(0, ac.min_pattern_len()); + /// ``` + pub fn min_pattern_len(&self) -> usize { + self.aut.min_pattern_len() + } + + /// Returns the length of the longest pattern matched by this automaton. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::AhoCorasick; + /// + /// let ac = AhoCorasick::new(&["foo", "bar", "quux", "baz"]).unwrap(); + /// assert_eq!(4, ac.max_pattern_len()); + /// ``` + pub fn max_pattern_len(&self) -> usize { + self.aut.max_pattern_len() + } + + /// Return the total number of patterns matched by this automaton. + /// + /// This includes patterns that may never participate in a match. For + /// example, if [`MatchKind::LeftmostFirst`] match semantics are used, and + /// the patterns `Sam` and `Samwise` were used to build the automaton (in + /// that order), then `Samwise` can never participate in a match because + /// `Sam` will always take priority. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::AhoCorasick; + /// + /// let ac = AhoCorasick::new(&["foo", "bar", "baz"]).unwrap(); + /// assert_eq!(3, ac.patterns_len()); + /// ``` + pub fn patterns_len(&self) -> usize { + self.aut.patterns_len() + } + + /// Returns the approximate total amount of heap used by this automaton, in + /// units of bytes. + /// + /// # Examples + /// + /// This example shows the difference in heap usage between a few + /// configurations: + /// + /// ``` + /// # if !cfg!(target_pointer_width = "64") { return; } + /// use aho_corasick::{AhoCorasick, AhoCorasickKind, MatchKind}; + /// + /// let ac = AhoCorasick::builder() + /// .kind(None) // default + /// .build(&["foobar", "bruce", "triskaidekaphobia", "springsteen"]) + /// .unwrap(); + /// assert_eq!(5_632, ac.memory_usage()); + /// + /// let ac = AhoCorasick::builder() + /// .kind(None) // default + /// .ascii_case_insensitive(true) + /// .build(&["foobar", "bruce", "triskaidekaphobia", "springsteen"]) + /// .unwrap(); + /// assert_eq!(11_136, ac.memory_usage()); + /// + /// let ac = AhoCorasick::builder() + /// .kind(Some(AhoCorasickKind::NoncontiguousNFA)) + /// .ascii_case_insensitive(true) + /// .build(&["foobar", "bruce", "triskaidekaphobia", "springsteen"]) + /// .unwrap(); + /// assert_eq!(10_879, ac.memory_usage()); + /// + /// let ac = AhoCorasick::builder() + /// .kind(Some(AhoCorasickKind::ContiguousNFA)) + /// .ascii_case_insensitive(true) + /// .build(&["foobar", "bruce", "triskaidekaphobia", "springsteen"]) + /// .unwrap(); + /// assert_eq!(2_584, ac.memory_usage()); + /// + /// let ac = AhoCorasick::builder() + /// .kind(Some(AhoCorasickKind::DFA)) + /// .ascii_case_insensitive(true) + /// .build(&["foobar", "bruce", "triskaidekaphobia", "springsteen"]) + /// .unwrap(); + /// // While this shows the DFA being the biggest here by a small margin, + /// // don't let the difference fool you. With such a small number of + /// // patterns, the difference is small, but a bigger number of patterns + /// // will reveal that the rate of growth of the DFA is far bigger than + /// // the NFAs above. For a large number of patterns, it is easy for the + /// // DFA to take an order of magnitude more heap space (or more!). + /// assert_eq!(11_136, ac.memory_usage()); + /// ``` + pub fn memory_usage(&self) -> usize { + self.aut.memory_usage() + } +} + +// We provide a manual debug impl so that we don't include the 'start_kind', +// principally because it's kind of weird to do so and because it screws with +// the carefully curated debug output for the underlying automaton. +impl core::fmt::Debug for AhoCorasick { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_tuple("AhoCorasick").field(&self.aut).finish() + } +} + +/// An iterator of non-overlapping matches in a particular haystack. +/// +/// This iterator yields matches according to the [`MatchKind`] used by this +/// automaton. +/// +/// This iterator is constructed via the [`AhoCorasick::find_iter`] and +/// [`AhoCorasick::try_find_iter`] methods. +/// +/// The lifetime `'a` refers to the lifetime of the `AhoCorasick` automaton. +/// +/// The lifetime `'h` refers to the lifetime of the haystack being searched. +#[derive(Debug)] +pub struct FindIter<'a, 'h>(automaton::FindIter<'a, 'h, Arc>); + +impl<'a, 'h> Iterator for FindIter<'a, 'h> { + type Item = Match; + + #[inline] + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// An iterator of overlapping matches in a particular haystack. +/// +/// This iterator will report all possible matches in a particular haystack, +/// even when the matches overlap. +/// +/// This iterator is constructed via the [`AhoCorasick::find_overlapping_iter`] +/// and [`AhoCorasick::try_find_overlapping_iter`] methods. +/// +/// The lifetime `'a` refers to the lifetime of the `AhoCorasick` automaton. +/// +/// The lifetime `'h` refers to the lifetime of the haystack being searched. +#[derive(Debug)] +pub struct FindOverlappingIter<'a, 'h>( + automaton::FindOverlappingIter<'a, 'h, Arc>, +); + +impl<'a, 'h> Iterator for FindOverlappingIter<'a, 'h> { + type Item = Match; + + #[inline] + fn next(&mut self) -> Option { + self.0.next() + } +} + +/// An iterator that reports Aho-Corasick matches in a stream. +/// +/// This iterator yields elements of type `Result`, +/// where an error is reported if there was a problem reading from the +/// underlying stream. The iterator terminates only when the underlying stream +/// reaches `EOF`. +/// +/// This iterator is constructed via the [`AhoCorasick::stream_find_iter`] and +/// [`AhoCorasick::try_stream_find_iter`] methods. +/// +/// The type variable `R` refers to the `io::Read` stream that is being read +/// from. +/// +/// The lifetime `'a` refers to the lifetime of the corresponding +/// [`AhoCorasick`] searcher. +#[cfg(feature = "std")] +#[derive(Debug)] +pub struct StreamFindIter<'a, R>( + automaton::StreamFindIter<'a, Arc, R>, +); + +#[cfg(feature = "std")] +impl<'a, R: std::io::Read> Iterator for StreamFindIter<'a, R> { + type Item = Result; + + fn next(&mut self) -> Option> { + self.0.next() + } +} + +/// A builder for configuring an Aho-Corasick automaton. +/// +/// # Quick advice +/// +/// * Use [`AhoCorasickBuilder::match_kind`] to configure your searcher +/// with [`MatchKind::LeftmostFirst`] if you want to match how backtracking +/// regex engines execute searches for `pat1|pat2|..|patN`. Use +/// [`MatchKind::LeftmostLongest`] if you want to match how POSIX regex engines +/// do it. +/// * If you need an anchored search, use [`AhoCorasickBuilder::start_kind`] to +/// set the [`StartKind::Anchored`] mode since [`StartKind::Unanchored`] is the +/// default. Or just use [`StartKind::Both`] to support both types of searches. +/// * You might want to use [`AhoCorasickBuilder::kind`] to set your searcher +/// to always use a [`AhoCorasickKind::DFA`] if search speed is critical and +/// memory usage isn't a concern. Otherwise, not setting a kind will probably +/// make the right choice for you. Beware that if you use [`StartKind::Both`] +/// to build a searcher that supports both unanchored and anchored searches +/// _and_ you set [`AhoCorasickKind::DFA`], then the DFA will essentially be +/// duplicated to support both simultaneously. This results in very high memory +/// usage. +/// * For all other options, their defaults are almost certainly what you want. +#[derive(Clone, Debug, Default)] +pub struct AhoCorasickBuilder { + nfa_noncontiguous: noncontiguous::Builder, + nfa_contiguous: contiguous::Builder, + dfa: dfa::Builder, + kind: Option, + start_kind: StartKind, +} + +impl AhoCorasickBuilder { + /// Create a new builder for configuring an Aho-Corasick automaton. + /// + /// The builder provides a way to configure a number of things, including + /// ASCII case insensitivity and what kind of match semantics are used. + pub fn new() -> AhoCorasickBuilder { + AhoCorasickBuilder::default() + } + + /// Build an Aho-Corasick automaton using the configuration set on this + /// builder. + /// + /// A builder may be reused to create more automatons. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{AhoCorasickBuilder, PatternID}; + /// + /// let patterns = &["foo", "bar", "baz"]; + /// let ac = AhoCorasickBuilder::new().build(patterns).unwrap(); + /// assert_eq!( + /// Some(PatternID::must(1)), + /// ac.find("xxx bar xxx").map(|m| m.pattern()), + /// ); + /// ``` + pub fn build(&self, patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + let nfa = self.nfa_noncontiguous.build(patterns)?; + let (aut, kind): (Arc, AhoCorasickKind) = + match self.kind { + None => { + debug!( + "asked for automatic Aho-Corasick implementation, \ + criteria: ", + nfa.patterns_len(), + nfa.max_pattern_len(), + self.start_kind, + ); + self.build_auto(nfa) + } + Some(AhoCorasickKind::NoncontiguousNFA) => { + debug!("forcefully chose noncontiguous NFA"); + (Arc::new(nfa), AhoCorasickKind::NoncontiguousNFA) + } + Some(AhoCorasickKind::ContiguousNFA) => { + debug!("forcefully chose contiguous NFA"); + let cnfa = + self.nfa_contiguous.build_from_noncontiguous(&nfa)?; + (Arc::new(cnfa), AhoCorasickKind::ContiguousNFA) + } + Some(AhoCorasickKind::DFA) => { + debug!("forcefully chose DFA"); + let dfa = self.dfa.build_from_noncontiguous(&nfa)?; + (Arc::new(dfa), AhoCorasickKind::DFA) + } + }; + Ok(AhoCorasick { aut, kind, start_kind: self.start_kind }) + } + + /// Implements the automatic selection logic for the Aho-Corasick + /// implementation to use. Since all Aho-Corasick automatons are built + /// from a non-contiguous NFA, the caller is responsible for building + /// that first. + fn build_auto( + &self, + nfa: noncontiguous::NFA, + ) -> (Arc, AhoCorasickKind) { + // We try to build a DFA if we have a very small number of patterns, + // otherwise the memory usage just gets too crazy. We also only do it + // when the start kind is unanchored or anchored, but not both, because + // both implies two full copies of the transition table. + let try_dfa = !matches!(self.start_kind, StartKind::Both) + && nfa.patterns_len() <= 100; + if try_dfa { + match self.dfa.build_from_noncontiguous(&nfa) { + Ok(dfa) => { + debug!("chose a DFA"); + return (Arc::new(dfa), AhoCorasickKind::DFA); + } + Err(_err) => { + debug!( + "failed to build DFA, trying something else: {}", + _err + ); + } + } + } + // We basically always want a contiguous NFA if the limited + // circumstances in which we use a DFA are not true. It is quite fast + // and has excellent memory usage. The only way we don't use it is if + // there are so many states that it can't fit in a contiguous NFA. + // And the only way to know that is to try to build it. Building a + // contiguous NFA is mostly just reshuffling data from a noncontiguous + // NFA, so it isn't too expensive, especially relative to building a + // noncontiguous NFA in the first place. + match self.nfa_contiguous.build_from_noncontiguous(&nfa) { + Ok(nfa) => { + debug!("chose contiguous NFA"); + return (Arc::new(nfa), AhoCorasickKind::ContiguousNFA); + } + #[allow(unused_variables)] // unused when 'logging' is disabled + Err(_err) => { + debug!( + "failed to build contiguous NFA, \ + trying something else: {}", + _err + ); + } + } + debug!("chose non-contiguous NFA"); + (Arc::new(nfa), AhoCorasickKind::NoncontiguousNFA) + } + + /// Set the desired match semantics. + /// + /// The default is [`MatchKind::Standard`], which corresponds to the match + /// semantics supported by the standard textbook description of the + /// Aho-Corasick algorithm. Namely, matches are reported as soon as they + /// are found. Moreover, this is the only way to get overlapping matches + /// or do stream searching. + /// + /// The other kinds of match semantics that are supported are + /// [`MatchKind::LeftmostFirst`] and [`MatchKind::LeftmostLongest`]. The + /// former corresponds to the match you would get if you were to try to + /// match each pattern at each position in the haystack in the same order + /// that you give to the automaton. That is, it returns the leftmost match + /// corresponding to the earliest pattern given to the automaton. The + /// latter corresponds to finding the longest possible match among all + /// leftmost matches. + /// + /// For more details on match semantics, see the [documentation for + /// `MatchKind`](MatchKind). + /// + /// Note that setting this to [`MatchKind::LeftmostFirst`] or + /// [`MatchKind::LeftmostLongest`] will cause some search routines on + /// [`AhoCorasick`] to return an error (or panic if you're using the + /// infallible API). Notably, this includes stream and overlapping + /// searches. + /// + /// # Examples + /// + /// In these examples, we demonstrate the differences between match + /// semantics for a particular set of patterns in a specific order: + /// `b`, `abc`, `abcd`. + /// + /// Standard semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::Standard) // default, not necessary + /// .build(patterns) + /// .unwrap(); + /// let mat = ac.find(haystack).expect("should have a match"); + /// assert_eq!("b", &haystack[mat.start()..mat.end()]); + /// ``` + /// + /// Leftmost-first semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let mat = ac.find(haystack).expect("should have a match"); + /// assert_eq!("abc", &haystack[mat.start()..mat.end()]); + /// ``` + /// + /// Leftmost-longest semantics: + /// + /// ``` + /// use aho_corasick::{AhoCorasick, MatchKind}; + /// + /// let patterns = &["b", "abc", "abcd"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostLongest) + /// .build(patterns) + /// .unwrap(); + /// let mat = ac.find(haystack).expect("should have a match"); + /// assert_eq!("abcd", &haystack[mat.start()..mat.end()]); + /// ``` + pub fn match_kind(&mut self, kind: MatchKind) -> &mut AhoCorasickBuilder { + self.nfa_noncontiguous.match_kind(kind); + self.nfa_contiguous.match_kind(kind); + self.dfa.match_kind(kind); + self + } + + /// Sets the starting state configuration for the automaton. + /// + /// Every Aho-Corasick automaton is capable of having two start states: one + /// that is used for unanchored searches and one that is used for anchored + /// searches. Some automatons, like the NFAs, support this with almost zero + /// additional cost. Other automatons, like the DFA, require two copies of + /// the underlying transition table to support both simultaneously. + /// + /// Because there may be an added non-trivial cost to supporting both, it + /// is possible to configure which starting state configuration is needed. + /// + /// Indeed, since anchored searches tend to be somewhat more rare, + /// _only_ unanchored searches are supported by default. Thus, + /// [`StartKind::Unanchored`] is the default. + /// + /// Note that when this is set to [`StartKind::Unanchored`], then + /// running an anchored search will result in an error (or a panic + /// if using the infallible APIs). Similarly, when this is set to + /// [`StartKind::Anchored`], then running an unanchored search will + /// result in an error (or a panic if using the infallible APIs). When + /// [`StartKind::Both`] is used, then both unanchored and anchored searches + /// are always supported. + /// + /// Also note that even if an `AhoCorasick` searcher is using an NFA + /// internally (which always supports both unanchored and anchored + /// searches), an error will still be reported for a search that isn't + /// supported by the configuration set via this method. This means, + /// for example, that an error is never dependent on which internal + /// implementation of Aho-Corasick is used. + /// + /// # Example: anchored search + /// + /// This shows how to build a searcher that only supports anchored + /// searches: + /// + /// ``` + /// use aho_corasick::{ + /// AhoCorasick, Anchored, Input, Match, MatchKind, StartKind, + /// }; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .start_kind(StartKind::Anchored) + /// .build(&["b", "abc", "abcd"]) + /// .unwrap(); + /// + /// // An unanchored search is not supported! An error here is guaranteed + /// // given the configuration above regardless of which kind of + /// // Aho-Corasick implementation ends up being used internally. + /// let input = Input::new("foo abcd").anchored(Anchored::No); + /// assert!(ac.try_find(input).is_err()); + /// + /// let input = Input::new("foo abcd").anchored(Anchored::Yes); + /// assert_eq!(None, ac.try_find(input)?); + /// + /// let input = Input::new("abcd").anchored(Anchored::Yes); + /// assert_eq!(Some(Match::must(1, 0..3)), ac.try_find(input)?); + /// + /// # Ok::<(), Box>(()) + /// ``` + /// + /// # Example: unanchored and anchored searches + /// + /// This shows how to build a searcher that supports both unanchored and + /// anchored searches: + /// + /// ``` + /// use aho_corasick::{ + /// AhoCorasick, Anchored, Input, Match, MatchKind, StartKind, + /// }; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .start_kind(StartKind::Both) + /// .build(&["b", "abc", "abcd"]) + /// .unwrap(); + /// + /// let input = Input::new("foo abcd").anchored(Anchored::No); + /// assert_eq!(Some(Match::must(1, 4..7)), ac.try_find(input)?); + /// + /// let input = Input::new("foo abcd").anchored(Anchored::Yes); + /// assert_eq!(None, ac.try_find(input)?); + /// + /// let input = Input::new("abcd").anchored(Anchored::Yes); + /// assert_eq!(Some(Match::must(1, 0..3)), ac.try_find(input)?); + /// + /// # Ok::<(), Box>(()) + /// ``` + pub fn start_kind(&mut self, kind: StartKind) -> &mut AhoCorasickBuilder { + self.dfa.start_kind(kind); + self.start_kind = kind; + self + } + + /// Enable ASCII-aware case insensitive matching. + /// + /// When this option is enabled, searching will be performed without + /// respect to case for ASCII letters (`a-z` and `A-Z`) only. + /// + /// Enabling this option does not change the search algorithm, but it may + /// increase the size of the automaton. + /// + /// **NOTE:** It is unlikely that support for Unicode case folding will + /// be added in the future. The ASCII case works via a simple hack to the + /// underlying automaton, but full Unicode handling requires a fair bit of + /// sophistication. If you do need Unicode handling, you might consider + /// using the [`regex` crate](https://docs.rs/regex) or the lower level + /// [`regex-automata` crate](https://docs.rs/regex-automata). + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::AhoCorasick; + /// + /// let patterns = &["FOO", "bAr", "BaZ"]; + /// let haystack = "foo bar baz"; + /// + /// let ac = AhoCorasick::builder() + /// .ascii_case_insensitive(true) + /// .build(patterns) + /// .unwrap(); + /// assert_eq!(3, ac.find_iter(haystack).count()); + /// ``` + pub fn ascii_case_insensitive( + &mut self, + yes: bool, + ) -> &mut AhoCorasickBuilder { + self.nfa_noncontiguous.ascii_case_insensitive(yes); + self.nfa_contiguous.ascii_case_insensitive(yes); + self.dfa.ascii_case_insensitive(yes); + self + } + + /// Choose the type of underlying automaton to use. + /// + /// Currently, there are four choices: + /// + /// * [`AhoCorasickKind::NoncontiguousNFA`] instructs the searcher to + /// use a [`noncontiguous::NFA`]. A noncontiguous NFA is the fastest to + /// be built, has moderate memory usage and is typically the slowest to + /// execute a search. + /// * [`AhoCorasickKind::ContiguousNFA`] instructs the searcher to use a + /// [`contiguous::NFA`]. A contiguous NFA is a little slower to build than + /// a noncontiguous NFA, has excellent memory usage and is typically a + /// little slower than a DFA for a search. + /// * [`AhoCorasickKind::DFA`] instructs the searcher to use a + /// [`dfa::DFA`]. A DFA is very slow to build, uses exorbitant amounts of + /// memory, but will typically execute searches the fastest. + /// * `None` (the default) instructs the searcher to choose the "best" + /// Aho-Corasick implementation. This choice is typically based primarily + /// on the number of patterns. + /// + /// Setting this configuration does not change the time complexity for + /// constructing the Aho-Corasick automaton (which is `O(p)` where `p` + /// is the total number of patterns being compiled). Setting this to + /// [`AhoCorasickKind::DFA`] does however reduce the time complexity of + /// non-overlapping searches from `O(n + p)` to `O(n)`, where `n` is the + /// length of the haystack. + /// + /// In general, you should probably stick to the default unless you have + /// some kind of reason to use a specific Aho-Corasick implementation. For + /// example, you might choose `AhoCorasickKind::DFA` if you don't care + /// about memory usage and want the fastest possible search times. + /// + /// Setting this guarantees that the searcher returned uses the chosen + /// implementation. If that implementation could not be constructed, then + /// an error will be returned. In contrast, when `None` is used, it is + /// possible for it to attempt to construct, for example, a contiguous + /// NFA and have it fail. In which case, it will fall back to using a + /// noncontiguous NFA. + /// + /// If `None` is given, then one may use [`AhoCorasick::kind`] to determine + /// which Aho-Corasick implementation was chosen. + /// + /// Note that the heuristics used for choosing which `AhoCorasickKind` + /// may be changed in a semver compatible release. + pub fn kind( + &mut self, + kind: Option, + ) -> &mut AhoCorasickBuilder { + self.kind = kind; + self + } + + /// Enable heuristic prefilter optimizations. + /// + /// When enabled, searching will attempt to quickly skip to match + /// candidates using specialized literal search routines. A prefilter + /// cannot always be used, and is generally treated as a heuristic. It + /// can be useful to disable this if the prefilter is observed to be + /// sub-optimal for a particular workload. + /// + /// Currently, prefilters are typically only active when building searchers + /// with a small (less than 100) number of patterns. + /// + /// This is enabled by default. + pub fn prefilter(&mut self, yes: bool) -> &mut AhoCorasickBuilder { + self.nfa_noncontiguous.prefilter(yes); + self.nfa_contiguous.prefilter(yes); + self.dfa.prefilter(yes); + self + } + + /// Set the limit on how many states use a dense representation for their + /// transitions. Other states will generally use a sparse representation. + /// + /// A dense representation uses more memory but is generally faster, since + /// the next transition in a dense representation can be computed in a + /// constant number of instructions. A sparse representation uses less + /// memory but is generally slower, since the next transition in a sparse + /// representation requires executing a variable number of instructions. + /// + /// This setting is only used when an Aho-Corasick implementation is used + /// that supports the dense versus sparse representation trade off. Not all + /// do. + /// + /// This limit is expressed in terms of the depth of a state, i.e., the + /// number of transitions from the starting state of the automaton. The + /// idea is that most of the time searching will be spent near the starting + /// state of the automaton, so states near the start state should use a + /// dense representation. States further away from the start state would + /// then use a sparse representation. + /// + /// By default, this is set to a low but non-zero number. Setting this to + /// `0` is almost never what you want, since it is likely to make searches + /// very slow due to the start state itself being forced to use a sparse + /// representation. However, it is unlikely that increasing this number + /// will help things much, since the most active states have a small depth. + /// More to the point, the memory usage increases superlinearly as this + /// number increases. + pub fn dense_depth(&mut self, depth: usize) -> &mut AhoCorasickBuilder { + self.nfa_noncontiguous.dense_depth(depth); + self.nfa_contiguous.dense_depth(depth); + self + } + + /// A debug settting for whether to attempt to shrink the size of the + /// automaton's alphabet or not. + /// + /// This option is enabled by default and should never be disabled unless + /// one is debugging the underlying automaton. + /// + /// When enabled, some (but not all) Aho-Corasick automatons will use a map + /// from all possible bytes to their corresponding equivalence class. Each + /// equivalence class represents a set of bytes that does not discriminate + /// between a match and a non-match in the automaton. + /// + /// The advantage of this map is that the size of the transition table can + /// be reduced drastically from `#states * 256 * sizeof(u32)` to + /// `#states * k * sizeof(u32)` where `k` is the number of equivalence + /// classes (rounded up to the nearest power of 2). As a result, total + /// space usage can decrease substantially. Moreover, since a smaller + /// alphabet is used, automaton compilation becomes faster as well. + /// + /// **WARNING:** This is only useful for debugging automatons. Disabling + /// this does not yield any speed advantages. Namely, even when this is + /// disabled, a byte class map is still used while searching. The only + /// difference is that every byte will be forced into its own distinct + /// equivalence class. This is useful for debugging the actual generated + /// transitions because it lets one see the transitions defined on actual + /// bytes instead of the equivalence classes. + pub fn byte_classes(&mut self, yes: bool) -> &mut AhoCorasickBuilder { + self.nfa_contiguous.byte_classes(yes); + self.dfa.byte_classes(yes); + self + } +} + +/// The type of Aho-Corasick implementation to use in an [`AhoCorasick`] +/// searcher. +/// +/// This is principally used as an input to the +/// [`AhoCorasickBuilder::start_kind`] method. Its documentation goes into more +/// detail about each choice. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AhoCorasickKind { + /// Use a noncontiguous NFA. + NoncontiguousNFA, + /// Use a contiguous NFA. + ContiguousNFA, + /// Use a DFA. Warning: DFAs typically use a large amount of memory. + DFA, +} + +/// A trait that effectively gives us practical dynamic dispatch over anything +/// that impls `Automaton`, but without needing to add a bunch of bounds to +/// the core `Automaton` trait. Basically, we provide all of the marker traits +/// that our automatons have, in addition to `Debug` impls and requiring that +/// there is no borrowed data. Without these, the main `AhoCorasick` type would +/// not be able to meaningfully impl `Debug` or the marker traits without also +/// requiring that all impls of `Automaton` do so, which would be not great. +trait AcAutomaton: + Automaton + Debug + Send + Sync + UnwindSafe + RefUnwindSafe + 'static +{ +} + +impl AcAutomaton for A where + A: Automaton + Debug + Send + Sync + UnwindSafe + RefUnwindSafe + 'static +{ +} + +impl crate::automaton::private::Sealed for Arc {} + +// I'm not sure why this trait impl shows up in the docs, as the AcAutomaton +// trait is not exported. So we forcefully hide it. +// +// SAFETY: This just defers to the underlying 'AcAutomaton' and thus inherits +// its safety properties. +#[doc(hidden)] +unsafe impl Automaton for Arc { + #[inline(always)] + fn start_state(&self, anchored: Anchored) -> Result { + (**self).start_state(anchored) + } + + #[inline(always)] + fn next_state( + &self, + anchored: Anchored, + sid: StateID, + byte: u8, + ) -> StateID { + (**self).next_state(anchored, sid, byte) + } + + #[inline(always)] + fn is_special(&self, sid: StateID) -> bool { + (**self).is_special(sid) + } + + #[inline(always)] + fn is_dead(&self, sid: StateID) -> bool { + (**self).is_dead(sid) + } + + #[inline(always)] + fn is_match(&self, sid: StateID) -> bool { + (**self).is_match(sid) + } + + #[inline(always)] + fn is_start(&self, sid: StateID) -> bool { + (**self).is_start(sid) + } + + #[inline(always)] + fn match_kind(&self) -> MatchKind { + (**self).match_kind() + } + + #[inline(always)] + fn match_len(&self, sid: StateID) -> usize { + (**self).match_len(sid) + } + + #[inline(always)] + fn match_pattern(&self, sid: StateID, index: usize) -> PatternID { + (**self).match_pattern(sid, index) + } + + #[inline(always)] + fn patterns_len(&self) -> usize { + (**self).patterns_len() + } + + #[inline(always)] + fn pattern_len(&self, pid: PatternID) -> usize { + (**self).pattern_len(pid) + } + + #[inline(always)] + fn min_pattern_len(&self) -> usize { + (**self).min_pattern_len() + } + + #[inline(always)] + fn max_pattern_len(&self) -> usize { + (**self).max_pattern_len() + } + + #[inline(always)] + fn memory_usage(&self) -> usize { + (**self).memory_usage() + } + + #[inline(always)] + fn prefilter(&self) -> Option<&Prefilter> { + (**self).prefilter() + } + + // Even though 'try_find' and 'try_find_overlapping' each have their + // own default impls, we explicitly define them here to fix a perf bug. + // Without these explicit definitions, the default impl will wind up using + // dynamic dispatch for all 'Automaton' method calls, including things like + // 'next_state' that absolutely must get inlined or else perf is trashed. + // Defining them explicitly here like this still requires dynamic dispatch + // to call 'try_find' itself, but all uses of 'Automaton' within 'try_find' + // are monomorphized. + // + // We don't need to explicitly impl any other methods, I think, because + // they are all implemented themselves in terms of 'try_find' and + // 'try_find_overlapping'. We still might wind up with an extra virtual + // call here or there, but that's okay since it's outside of any perf + // critical areas. + + #[inline(always)] + fn try_find( + &self, + input: &Input<'_>, + ) -> Result, MatchError> { + (**self).try_find(input) + } + + #[inline(always)] + fn try_find_overlapping( + &self, + input: &Input<'_>, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + (**self).try_find_overlapping(input, state) + } +} + +/// Returns an error if the start state configuration does not support the +/// desired search configuration. See the internal 'AhoCorasick::start_kind' +/// field docs for more details. +fn enforce_anchored_consistency( + have: StartKind, + want: Anchored, +) -> Result<(), MatchError> { + match have { + StartKind::Both => Ok(()), + StartKind::Unanchored if !want.is_anchored() => Ok(()), + StartKind::Unanchored => Err(MatchError::invalid_input_anchored()), + StartKind::Anchored if want.is_anchored() => Ok(()), + StartKind::Anchored => Err(MatchError::invalid_input_unanchored()), + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/automaton.rs b/third_party/crates/aho-corasick-1.1.5/src/automaton.rs new file mode 100644 index 0000000..c41dc6e --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/automaton.rs @@ -0,0 +1,1608 @@ +/*! +Provides [`Automaton`] trait for abstracting over Aho-Corasick automata. + +The `Automaton` trait provides a way to write generic code over any +Aho-Corasick automaton. It also provides access to lower level APIs that +permit walking the state transitions of an Aho-Corasick automaton manually. +*/ + +use alloc::{string::String, vec::Vec}; + +use crate::util::{ + error::MatchError, + primitives::PatternID, + search::{Anchored, Input, Match, MatchKind, Span}, +}; + +pub use crate::util::{ + prefilter::{Candidate, Prefilter}, + primitives::{StateID, StateIDError}, +}; + +/// We seal the `Automaton` trait for now. It's a big trait, and it's +/// conceivable that I might want to add new required methods, and sealing the +/// trait permits doing that in a backwards compatible fashion. On other the +/// hand, if you have a solid use case for implementing the trait yourself, +/// please file an issue and we can discuss it. This was *mostly* done as a +/// conservative step. +pub(crate) mod private { + pub trait Sealed {} +} +impl private::Sealed for crate::nfa::noncontiguous::NFA {} +impl private::Sealed for crate::nfa::contiguous::NFA {} +impl private::Sealed for crate::dfa::DFA {} + +impl<'a, T: private::Sealed + ?Sized> private::Sealed for &'a T {} + +/// A trait that abstracts over Aho-Corasick automata. +/// +/// This trait primarily exists for niche use cases such as: +/// +/// * Using an NFA or DFA directly, bypassing the top-level +/// [`AhoCorasick`](crate::AhoCorasick) searcher. Currently, these include +/// [`noncontiguous::NFA`](crate::nfa::noncontiguous::NFA), +/// [`contiguous::NFA`](crate::nfa::contiguous::NFA) and +/// [`dfa::DFA`](crate::dfa::DFA). +/// * Implementing your own custom search routine by walking the automaton +/// yourself. This might be useful for implementing search on non-contiguous +/// strings or streams. +/// +/// For most use cases, it is not expected that users will need +/// to use or even know about this trait. Indeed, the top level +/// [`AhoCorasick`](crate::AhoCorasick) searcher does not expose any details +/// about this trait, nor does it implement it itself. +/// +/// Note that this trait defines a number of default methods, such as +/// [`Automaton::try_find`] and [`Automaton::try_find_iter`], which implement +/// higher level search routines in terms of the lower level automata API. +/// +/// # Sealed +/// +/// Currently, this trait is sealed. That means users of this crate can write +/// generic routines over this trait but cannot implement it themselves. This +/// restriction may be lifted in the future, but sealing the trait permits +/// adding new required methods in a backwards compatible fashion. +/// +/// # Special states +/// +/// This trait encodes a notion of "special" states in an automaton. Namely, +/// a state is treated as special if it is a dead, match or start state: +/// +/// * A dead state is a state that cannot be left once entered. All transitions +/// on a dead state lead back to itself. The dead state is meant to be treated +/// as a sentinel indicating that the search should stop and return a match if +/// one has been found, and nothing otherwise. +/// * A match state is a state that indicates one or more patterns have +/// matched. Depending on the [`MatchKind`] of the automaton, a search may +/// stop once a match is seen, or it may continue looking for matches until +/// it enters a dead state or sees the end of the haystack. +/// * A start state is a state that a search begins in. It is useful to know +/// when a search enters a start state because it may mean that a prefilter can +/// be used to skip ahead and quickly look for candidate matches. Unlike dead +/// and match states, it is never necessary to explicitly handle start states +/// for correctness. Indeed, in this crate, implementations of `Automaton` +/// will only treat start states as "special" when a prefilter is enabled and +/// active. Otherwise, treating it as special has no purpose and winds up +/// slowing down the overall search because it results in ping-ponging between +/// the main state transition and the "special" state logic. +/// +/// Since checking whether a state is special by doing three different +/// checks would be too expensive inside a fast search loop, the +/// [`Automaton::is_special`] method is provided for quickly checking whether +/// the state is special. The `Automaton::is_dead`, `Automaton::is_match` and +/// `Automaton::is_start` predicates can then be used to determine which kind +/// of special state it is. +/// +/// # Panics +/// +/// Most of the APIs on this trait should panic or give incorrect results +/// if invalid inputs are given to it. For example, `Automaton::next_state` +/// has unspecified behavior if the state ID given to it is not a valid +/// state ID for the underlying automaton. Valid state IDs can only be +/// retrieved in one of two ways: calling `Automaton::start_state` or calling +/// `Automaton::next_state` with a valid state ID. +/// +/// # Safety +/// +/// This trait is not safe to implement so that code may rely on the +/// correctness of implementations of this trait to avoid undefined behavior. +/// The primary correctness guarantees are: +/// +/// * `Automaton::start_state` always returns a valid state ID or an error or +/// panics. +/// * `Automaton::next_state`, when given a valid state ID, always returns +/// a valid state ID for all values of `anchored` and `byte`, or otherwise +/// panics. +/// +/// In general, the rest of the methods on `Automaton` need to uphold their +/// contracts as well. For example, `Automaton::is_dead` should only returns +/// true if the given state ID is actually a dead state. +/// +/// Note that currently this crate does not rely on the safety property defined +/// here to avoid undefined behavior. Instead, this was done to make it +/// _possible_ to do in the future. +/// +/// # Example +/// +/// This example shows how one might implement a basic but correct search +/// routine. We keep things simple by not using prefilters or worrying about +/// anchored searches, but do make sure our search is correct for all possible +/// [`MatchKind`] semantics. (The comments in the code below note the parts +/// that are needed to support certain `MatchKind` semantics.) +/// +/// ``` +/// use aho_corasick::{ +/// automaton::Automaton, +/// nfa::noncontiguous::NFA, +/// Anchored, Match, MatchError, MatchKind, +/// }; +/// +/// // Run an unanchored search for 'aut' in 'haystack'. Return the first match +/// // seen according to the automaton's match semantics. This returns an error +/// // if the given automaton does not support unanchored searches. +/// fn find( +/// aut: A, +/// haystack: &[u8], +/// ) -> Result, MatchError> { +/// let mut sid = aut.start_state(Anchored::No)?; +/// let mut at = 0; +/// let mut mat = None; +/// let get_match = |sid, at| { +/// let pid = aut.match_pattern(sid, 0); +/// let len = aut.pattern_len(pid); +/// Match::new(pid, (at - len)..at) +/// }; +/// // Start states can be match states! +/// if aut.is_match(sid) { +/// mat = Some(get_match(sid, at)); +/// // Standard semantics require matches to be reported as soon as +/// // they're seen. Otherwise, we continue until we see a dead state +/// // or the end of the haystack. +/// if matches!(aut.match_kind(), MatchKind::Standard) { +/// return Ok(mat); +/// } +/// } +/// while at < haystack.len() { +/// sid = aut.next_state(Anchored::No, sid, haystack[at]); +/// if aut.is_special(sid) { +/// if aut.is_dead(sid) { +/// return Ok(mat); +/// } else if aut.is_match(sid) { +/// mat = Some(get_match(sid, at + 1)); +/// // As above, standard semantics require that we return +/// // immediately once a match is found. +/// if matches!(aut.match_kind(), MatchKind::Standard) { +/// return Ok(mat); +/// } +/// } +/// } +/// at += 1; +/// } +/// Ok(mat) +/// } +/// +/// // Show that it works for standard searches. +/// let nfa = NFA::new(&["samwise", "sam"]).unwrap(); +/// assert_eq!(Some(Match::must(1, 0..3)), find(&nfa, b"samwise")?); +/// +/// // But also works when using leftmost-first. Notice how the match result +/// // has changed! +/// let nfa = NFA::builder() +/// .match_kind(MatchKind::LeftmostFirst) +/// .build(&["samwise", "sam"]) +/// .unwrap(); +/// assert_eq!(Some(Match::must(0, 0..7)), find(&nfa, b"samwise")?); +/// +/// # Ok::<(), Box>(()) +/// ``` +pub unsafe trait Automaton: private::Sealed { + /// Returns the starting state for the given anchor mode. + /// + /// Upon success, the state ID returned is guaranteed to be valid for + /// this automaton. + /// + /// # Errors + /// + /// This returns an error when the given search configuration is not + /// supported by the underlying automaton. For example, if the underlying + /// automaton only supports unanchored searches but the given configuration + /// was set to an anchored search, then this must return an error. + fn start_state(&self, anchored: Anchored) -> Result; + + /// Performs a state transition from `sid` for `byte` and returns the next + /// state. + /// + /// `anchored` should be [`Anchored::Yes`] when executing an anchored + /// search and [`Anchored::No`] otherwise. For some implementations of + /// `Automaton`, it is required to know whether the search is anchored + /// or not in order to avoid following failure transitions. Other + /// implementations may ignore `anchored` altogether and depend on + /// `Automaton::start_state` returning a state that walks a different path + /// through the automaton depending on whether the search is anchored or + /// not. + /// + /// # Panics + /// + /// This routine may panic or return incorrect results when the given state + /// ID is invalid. A state ID is valid if and only if: + /// + /// 1. It came from a call to `Automaton::start_state`, or + /// 2. It came from a previous call to `Automaton::next_state` with a + /// valid state ID. + /// + /// Implementations must treat all possible values of `byte` as valid. + /// + /// Implementations may panic on unsupported values of `anchored`, but are + /// not required to do so. + fn next_state( + &self, + anchored: Anchored, + sid: StateID, + byte: u8, + ) -> StateID; + + /// Returns true if the given ID represents a "special" state. A special + /// state is a dead, match or start state. + /// + /// Note that implementations may choose to return false when the given ID + /// corresponds to a start state. Namely, it always correct to treat start + /// states as non-special. Implementations must return true for states that + /// are dead or contain matches. + /// + /// This has unspecified behavior when given an invalid state ID. + fn is_special(&self, sid: StateID) -> bool; + + /// Returns true if the given ID represents a dead state. + /// + /// A dead state is a type of "sink" in a finite state machine. It + /// corresponds to a state whose transitions all loop back to itself. That + /// is, once entered, it can never be left. In practice, it serves as a + /// sentinel indicating that the search should terminate. + /// + /// This has unspecified behavior when given an invalid state ID. + fn is_dead(&self, sid: StateID) -> bool; + + /// Returns true if the given ID represents a match state. + /// + /// A match state is always associated with one or more pattern IDs that + /// matched at the position in the haystack when the match state was + /// entered. When a match state is entered, the match semantics dictate + /// whether it should be returned immediately (for `MatchKind::Standard`) + /// or if the search should continue (for `MatchKind::LeftmostFirst` and + /// `MatchKind::LeftmostLongest`) until a dead state is seen or the end of + /// the haystack has been reached. + /// + /// This has unspecified behavior when given an invalid state ID. + fn is_match(&self, sid: StateID) -> bool; + + /// Returns true if the given ID represents a start state. + /// + /// While it is never incorrect to ignore start states during a search + /// (except for the start of the search of course), knowing whether one has + /// entered a start state can be useful for certain classes of performance + /// optimizations. For example, if one is in a start state, it may be legal + /// to try to skip ahead and look for match candidates more quickly than + /// would otherwise be accomplished by walking the automaton. + /// + /// Implementations of `Automaton` in this crate "unspecialize" start + /// states when a prefilter is not active or enabled. In this case, it + /// is possible for `Automaton::is_special(sid)` to return false while + /// `Automaton::is_start(sid)` returns true. + /// + /// This has unspecified behavior when given an invalid state ID. + fn is_start(&self, sid: StateID) -> bool; + + /// Returns the match semantics that this automaton was built with. + fn match_kind(&self) -> MatchKind; + + /// Returns the total number of matches for the given state ID. + /// + /// This has unspecified behavior if the given ID does not refer to a match + /// state. + fn match_len(&self, sid: StateID) -> usize; + + /// Returns the pattern ID for the match state given by `sid` at the + /// `index` given. + /// + /// Typically, `index` is only ever greater than `0` when implementing an + /// overlapping search. Otherwise, it's likely that your search only cares + /// about reporting the first pattern ID in a match state. + /// + /// This has unspecified behavior if the given ID does not refer to a match + /// state, or if the index is greater than or equal to the total number of + /// matches in this match state. + fn match_pattern(&self, sid: StateID, index: usize) -> PatternID; + + /// Returns the total number of patterns compiled into this automaton. + fn patterns_len(&self) -> usize; + + /// Returns the length of the pattern for the given ID. + /// + /// This has unspecified behavior when given an invalid pattern + /// ID. A pattern ID is valid if and only if it is less than + /// `Automaton::patterns_len`. + fn pattern_len(&self, pid: PatternID) -> usize; + + /// Returns the length, in bytes, of the shortest pattern in this + /// automaton. + fn min_pattern_len(&self) -> usize; + + /// Returns the length, in bytes, of the longest pattern in this automaton. + fn max_pattern_len(&self) -> usize; + + /// Returns the heap memory usage, in bytes, used by this automaton. + fn memory_usage(&self) -> usize; + + /// Returns a prefilter, if available, that can be used to accelerate + /// searches for this automaton. + /// + /// The typical way this is used is when the start state is entered during + /// a search. When that happens, one can use a prefilter to skip ahead and + /// look for candidate matches without having to walk the automaton on the + /// bytes between candidates. + /// + /// Typically a prefilter is only available when there are a small (<100) + /// number of patterns built into the automaton. + fn prefilter(&self) -> Option<&Prefilter>; + + /// Executes a non-overlapping search with this automaton using the given + /// configuration. + /// + /// See + /// [`AhoCorasick::try_find`](crate::AhoCorasick::try_find) + /// for more documentation and examples. + fn try_find( + &self, + input: &Input<'_>, + ) -> Result, MatchError> { + try_find_fwd(&self, input) + } + + /// Executes a overlapping search with this automaton using the given + /// configuration. + /// + /// See + /// [`AhoCorasick::try_find_overlapping`](crate::AhoCorasick::try_find_overlapping) + /// for more documentation and examples. + fn try_find_overlapping( + &self, + input: &Input<'_>, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + try_find_overlapping_fwd(&self, input, state) + } + + /// Returns an iterator of non-overlapping matches with this automaton + /// using the given configuration. + /// + /// See + /// [`AhoCorasick::try_find_iter`](crate::AhoCorasick::try_find_iter) + /// for more documentation and examples. + fn try_find_iter<'a, 'h>( + &'a self, + input: Input<'h>, + ) -> Result, MatchError> + where + Self: Sized, + { + FindIter::new(self, input) + } + + /// Returns an iterator of overlapping matches with this automaton + /// using the given configuration. + /// + /// See + /// [`AhoCorasick::try_find_overlapping_iter`](crate::AhoCorasick::try_find_overlapping_iter) + /// for more documentation and examples. + fn try_find_overlapping_iter<'a, 'h>( + &'a self, + input: Input<'h>, + ) -> Result, MatchError> + where + Self: Sized, + { + if !self.match_kind().is_standard() { + return Err(MatchError::unsupported_overlapping( + self.match_kind(), + )); + } + // We might consider lifting this restriction. The reason why I added + // it was to ban the combination of "anchored search" and "overlapping + // iteration." The match semantics aren't totally clear in that case. + // Should we allow *any* matches that are adjacent to *any* previous + // match? Or only following the most recent one? Or only matches + // that start at the beginning of the search? We might also elect to + // just keep this restriction in place, as callers should be able to + // implement it themselves if they want to. + if input.get_anchored().is_anchored() { + return Err(MatchError::invalid_input_anchored()); + } + let _ = self.start_state(input.get_anchored())?; + let state = OverlappingState::start(); + Ok(FindOverlappingIter { aut: self, input, state }) + } + + /// Replaces all non-overlapping matches in `haystack` with + /// strings from `replace_with` depending on the pattern that + /// matched. The `replace_with` slice must have length equal to + /// `Automaton::patterns_len`. + /// + /// See + /// [`AhoCorasick::try_replace_all`](crate::AhoCorasick::try_replace_all) + /// for more documentation and examples. + fn try_replace_all( + &self, + haystack: &str, + replace_with: &[B], + ) -> Result + where + Self: Sized, + B: AsRef, + { + assert_eq!( + replace_with.len(), + self.patterns_len(), + "replace_all requires a replacement for every pattern \ + in the automaton" + ); + let mut dst = String::with_capacity(haystack.len()); + self.try_replace_all_with(haystack, &mut dst, |mat, _, dst| { + dst.push_str(replace_with[mat.pattern()].as_ref()); + true + })?; + Ok(dst) + } + + /// Replaces all non-overlapping matches in `haystack` with + /// strings from `replace_with` depending on the pattern that + /// matched. The `replace_with` slice must have length equal to + /// `Automaton::patterns_len`. + /// + /// See + /// [`AhoCorasick::try_replace_all_bytes`](crate::AhoCorasick::try_replace_all_bytes) + /// for more documentation and examples. + fn try_replace_all_bytes( + &self, + haystack: &[u8], + replace_with: &[B], + ) -> Result, MatchError> + where + Self: Sized, + B: AsRef<[u8]>, + { + assert_eq!( + replace_with.len(), + self.patterns_len(), + "replace_all requires a replacement for every pattern \ + in the automaton" + ); + let mut dst = Vec::with_capacity(haystack.len()); + self.try_replace_all_with_bytes(haystack, &mut dst, |mat, _, dst| { + dst.extend(replace_with[mat.pattern()].as_ref()); + true + })?; + Ok(dst) + } + + /// Replaces all non-overlapping matches in `haystack` by calling the + /// `replace_with` closure given. + /// + /// See + /// [`AhoCorasick::try_replace_all_with`](crate::AhoCorasick::try_replace_all_with) + /// for more documentation and examples. + fn try_replace_all_with( + &self, + haystack: &str, + dst: &mut String, + mut replace_with: F, + ) -> Result<(), MatchError> + where + Self: Sized, + F: FnMut(&Match, &str, &mut String) -> bool, + { + let mut last_match = 0; + for m in self.try_find_iter(Input::new(haystack))? { + // Since there are no restrictions on what kinds of patterns are + // in an Aho-Corasick automaton, we might get matches that split + // a codepoint, or even matches of a partial codepoint. When that + // happens, we just skip the match. + if !haystack.is_char_boundary(m.start()) + || !haystack.is_char_boundary(m.end()) + { + continue; + } + dst.push_str(&haystack[last_match..m.start()]); + last_match = m.end(); + if !replace_with(&m, &haystack[m.start()..m.end()], dst) { + break; + }; + } + dst.push_str(&haystack[last_match..]); + Ok(()) + } + + /// Replaces all non-overlapping matches in `haystack` by calling the + /// `replace_with` closure given. + /// + /// See + /// [`AhoCorasick::try_replace_all_with_bytes`](crate::AhoCorasick::try_replace_all_with_bytes) + /// for more documentation and examples. + fn try_replace_all_with_bytes( + &self, + haystack: &[u8], + dst: &mut Vec, + mut replace_with: F, + ) -> Result<(), MatchError> + where + Self: Sized, + F: FnMut(&Match, &[u8], &mut Vec) -> bool, + { + let mut last_match = 0; + for m in self.try_find_iter(Input::new(haystack))? { + dst.extend(&haystack[last_match..m.start()]); + last_match = m.end(); + if !replace_with(&m, &haystack[m.start()..m.end()], dst) { + break; + }; + } + dst.extend(&haystack[last_match..]); + Ok(()) + } + + /// Returns an iterator of non-overlapping matches with this automaton + /// from the stream given. + /// + /// See + /// [`AhoCorasick::try_stream_find_iter`](crate::AhoCorasick::try_stream_find_iter) + /// for more documentation and examples. + #[cfg(feature = "std")] + fn try_stream_find_iter<'a, R: std::io::Read>( + &'a self, + rdr: R, + ) -> Result, MatchError> + where + Self: Sized, + { + Ok(StreamFindIter { it: StreamChunkIter::new(self, rdr)? }) + } + + /// Replaces all non-overlapping matches in `rdr` with strings from + /// `replace_with` depending on the pattern that matched, and writes the + /// result to `wtr`. The `replace_with` slice must have length equal to + /// `Automaton::patterns_len`. + /// + /// See + /// [`AhoCorasick::try_stream_replace_all`](crate::AhoCorasick::try_stream_replace_all) + /// for more documentation and examples. + #[cfg(feature = "std")] + fn try_stream_replace_all( + &self, + rdr: R, + wtr: W, + replace_with: &[B], + ) -> std::io::Result<()> + where + Self: Sized, + R: std::io::Read, + W: std::io::Write, + B: AsRef<[u8]>, + { + assert_eq!( + replace_with.len(), + self.patterns_len(), + "streaming replace_all requires a replacement for every pattern \ + in the automaton", + ); + self.try_stream_replace_all_with(rdr, wtr, |mat, _, wtr| { + wtr.write_all(replace_with[mat.pattern()].as_ref()) + }) + } + + /// Replaces all non-overlapping matches in `rdr` by calling the + /// `replace_with` closure given and writing the result to `wtr`. + /// + /// See + /// [`AhoCorasick::try_stream_replace_all_with`](crate::AhoCorasick::try_stream_replace_all_with) + /// for more documentation and examples. + #[cfg(feature = "std")] + fn try_stream_replace_all_with( + &self, + rdr: R, + mut wtr: W, + mut replace_with: F, + ) -> std::io::Result<()> + where + Self: Sized, + R: std::io::Read, + W: std::io::Write, + F: FnMut(&Match, &[u8], &mut W) -> std::io::Result<()>, + { + let mut it = StreamChunkIter::new(self, rdr).map_err(|e| { + let kind = std::io::ErrorKind::Other; + std::io::Error::new(kind, e) + })?; + while let Some(result) = it.next() { + let chunk = result?; + match chunk { + StreamChunk::NonMatch { bytes, .. } => { + wtr.write_all(bytes)?; + } + StreamChunk::Match { bytes, mat } => { + replace_with(&mat, bytes, &mut wtr)?; + } + } + } + Ok(()) + } +} + +// SAFETY: This just defers to the underlying 'AcAutomaton' and thus inherits +// its safety properties. +unsafe impl<'a, A: Automaton + ?Sized> Automaton for &'a A { + #[inline(always)] + fn start_state(&self, anchored: Anchored) -> Result { + (**self).start_state(anchored) + } + + #[inline(always)] + fn next_state( + &self, + anchored: Anchored, + sid: StateID, + byte: u8, + ) -> StateID { + (**self).next_state(anchored, sid, byte) + } + + #[inline(always)] + fn is_special(&self, sid: StateID) -> bool { + (**self).is_special(sid) + } + + #[inline(always)] + fn is_dead(&self, sid: StateID) -> bool { + (**self).is_dead(sid) + } + + #[inline(always)] + fn is_match(&self, sid: StateID) -> bool { + (**self).is_match(sid) + } + + #[inline(always)] + fn is_start(&self, sid: StateID) -> bool { + (**self).is_start(sid) + } + + #[inline(always)] + fn match_kind(&self) -> MatchKind { + (**self).match_kind() + } + + #[inline(always)] + fn match_len(&self, sid: StateID) -> usize { + (**self).match_len(sid) + } + + #[inline(always)] + fn match_pattern(&self, sid: StateID, index: usize) -> PatternID { + (**self).match_pattern(sid, index) + } + + #[inline(always)] + fn patterns_len(&self) -> usize { + (**self).patterns_len() + } + + #[inline(always)] + fn pattern_len(&self, pid: PatternID) -> usize { + (**self).pattern_len(pid) + } + + #[inline(always)] + fn min_pattern_len(&self) -> usize { + (**self).min_pattern_len() + } + + #[inline(always)] + fn max_pattern_len(&self) -> usize { + (**self).max_pattern_len() + } + + #[inline(always)] + fn memory_usage(&self) -> usize { + (**self).memory_usage() + } + + #[inline(always)] + fn prefilter(&self) -> Option<&Prefilter> { + (**self).prefilter() + } +} + +/// Represents the current state of an overlapping search. +/// +/// This is used for overlapping searches since they need to know something +/// about the previous search. For example, when multiple patterns match at the +/// same position, this state tracks the last reported pattern so that the next +/// search knows whether to report another matching pattern or continue with +/// the search at the next position. Additionally, it also tracks which state +/// the last search call terminated in and the current offset of the search +/// in the haystack. +/// +/// This type provides limited introspection capabilities. The only thing a +/// caller can do is construct it and pass it around to permit search routines +/// to use it to track state, and to ask whether a match has been found. +/// +/// Callers should always provide a fresh state constructed via +/// [`OverlappingState::start`] when starting a new search. That same state +/// should be reused for subsequent searches on the same `Input`. The state +/// given will advance through the haystack itself. Callers can detect the end +/// of a search when neither an error nor a match is returned. +/// +/// # Example +/// +/// This example shows how to manually iterate over all overlapping matches. If +/// you need this, you might consider using +/// [`AhoCorasick::find_overlapping_iter`](crate::AhoCorasick::find_overlapping_iter) +/// instead, but this shows how to correctly use an `OverlappingState`. +/// +/// ``` +/// use aho_corasick::{ +/// automaton::OverlappingState, +/// AhoCorasick, Input, Match, +/// }; +/// +/// let patterns = &["append", "appendage", "app"]; +/// let haystack = "append the app to the appendage"; +/// +/// let ac = AhoCorasick::new(patterns).unwrap(); +/// let mut state = OverlappingState::start(); +/// let mut matches = vec![]; +/// +/// loop { +/// ac.find_overlapping(haystack, &mut state); +/// let mat = match state.get_match() { +/// None => break, +/// Some(mat) => mat, +/// }; +/// matches.push(mat); +/// } +/// let expected = vec![ +/// Match::must(2, 0..3), +/// Match::must(0, 0..6), +/// Match::must(2, 11..14), +/// Match::must(2, 22..25), +/// Match::must(0, 22..28), +/// Match::must(1, 22..31), +/// ]; +/// assert_eq!(expected, matches); +/// ``` +#[derive(Clone, Debug)] +pub struct OverlappingState { + /// The match reported by the most recent overlapping search to use this + /// state. + /// + /// If a search does not find any matches, then it is expected to clear + /// this value. + mat: Option, + /// The state ID of the state at which the search was in when the call + /// terminated. When this is a match state, `last_match` must be set to a + /// non-None value. + /// + /// A `None` value indicates the start state of the corresponding + /// automaton. We cannot use the actual ID, since any one automaton may + /// have many start states, and which one is in use depends on search-time + /// factors (such as whether the search is anchored or not). + id: Option, + /// The position of the search. + /// + /// When `id` is None (i.e., we are starting a search), this is set to + /// the beginning of the search as given by the caller regardless of its + /// current value. Subsequent calls to an overlapping search pick up at + /// this offset. + at: usize, + /// The index into the matching patterns of the next match to report if the + /// current state is a match state. Note that this may be 1 greater than + /// the total number of matches to report for the current match state. (In + /// which case, no more matches should be reported at the current position + /// and the search should advance to the next position.) + next_match_index: Option, +} + +impl OverlappingState { + /// Create a new overlapping state that begins at the start state. + pub fn start() -> OverlappingState { + OverlappingState { mat: None, id: None, at: 0, next_match_index: None } + } + + /// Return the match result of the most recent search to execute with this + /// state. + /// + /// Every search will clear this result automatically, such that if no + /// match is found, this will always correctly report `None`. + pub fn get_match(&self) -> Option { + self.mat + } +} + +/// An iterator of non-overlapping matches in a particular haystack. +/// +/// This iterator yields matches according to the [`MatchKind`] used by this +/// automaton. +/// +/// This iterator is constructed via the [`Automaton::try_find_iter`] method. +/// +/// The type variable `A` refers to the implementation of the [`Automaton`] +/// trait used to execute the search. +/// +/// The lifetime `'a` refers to the lifetime of the [`Automaton`] +/// implementation. +/// +/// The lifetime `'h` refers to the lifetime of the haystack being searched. +#[derive(Debug)] +pub struct FindIter<'a, 'h, A> { + /// The automaton used to drive the search. + aut: &'a A, + /// The input parameters to give to each search call. + /// + /// The start position of the search is mutated during iteration. + input: Input<'h>, + /// Records the end offset of the most recent match. This is necessary to + /// handle a corner case for preventing empty matches from overlapping with + /// the ending bounds of a prior match. + last_match_end: Option, +} + +impl<'a, 'h, A: Automaton> FindIter<'a, 'h, A> { + /// Creates a new non-overlapping iterator. If the given automaton would + /// return an error on a search with the given input configuration, then + /// that error is returned here. + fn new( + aut: &'a A, + input: Input<'h>, + ) -> Result, MatchError> { + // The only way this search can fail is if we cannot retrieve the start + // state. e.g., Asking for an anchored search when only unanchored + // searches are supported. + let _ = aut.start_state(input.get_anchored())?; + Ok(FindIter { aut, input, last_match_end: None }) + } + + /// Executes a search and returns a match if one is found. + /// + /// This does not advance the input forward. It just executes a search + /// based on the current configuration/offsets. + fn search(&self) -> Option { + // The unwrap is OK here because we check at iterator construction time + // that no subsequent search call (using the same configuration) will + // ever return an error. + self.aut + .try_find(&self.input) + .expect("already checked that no match error can occur") + } + + /// Handles the special case of an empty match by ensuring that 1) the + /// iterator always advances and 2) empty matches never overlap with other + /// matches. + /// + /// (1) is necessary because we principally make progress by setting the + /// starting location of the next search to the ending location of the last + /// match. But if a match is empty, then this results in a search that does + /// not advance and thus does not terminate. + /// + /// (2) is not strictly necessary, but makes intuitive sense and matches + /// the presiding behavior of most general purpose regex engines. + /// (Obviously this crate isn't a regex engine, but we choose to match + /// their semantics.) The "intuitive sense" here is that we want to report + /// NON-overlapping matches. So for example, given the patterns 'a' and + /// '' (an empty string) against the haystack 'a', without the special + /// handling, you'd get the matches [0, 1) and [1, 1), where the latter + /// overlaps with the end bounds of the former. + /// + /// Note that we mark this cold and forcefully prevent inlining because + /// handling empty matches like this is extremely rare and does require + /// quite a bit of code, comparatively. Keeping this code out of the main + /// iterator function keeps it smaller and more amenable to inlining + /// itself. + #[cold] + #[inline(never)] + fn handle_overlapping_empty_match( + &mut self, + mut m: Match, + ) -> Option { + assert!(m.is_empty()); + if Some(m.end()) == self.last_match_end { + self.input.set_start(self.input.start().checked_add(1).unwrap()); + m = self.search()?; + } + Some(m) + } +} + +impl<'a, 'h, A: Automaton> Iterator for FindIter<'a, 'h, A> { + type Item = Match; + + #[inline(always)] + fn next(&mut self) -> Option { + let mut m = self.search()?; + if m.is_empty() { + m = self.handle_overlapping_empty_match(m)?; + } + self.input.set_start(m.end()); + self.last_match_end = Some(m.end()); + Some(m) + } +} + +/// An iterator of overlapping matches in a particular haystack. +/// +/// This iterator will report all possible matches in a particular haystack, +/// even when the matches overlap. +/// +/// This iterator is constructed via the +/// [`Automaton::try_find_overlapping_iter`] method. +/// +/// The type variable `A` refers to the implementation of the [`Automaton`] +/// trait used to execute the search. +/// +/// The lifetime `'a` refers to the lifetime of the [`Automaton`] +/// implementation. +/// +/// The lifetime `'h` refers to the lifetime of the haystack being searched. +#[derive(Debug)] +pub struct FindOverlappingIter<'a, 'h, A> { + aut: &'a A, + input: Input<'h>, + state: OverlappingState, +} + +impl<'a, 'h, A: Automaton> Iterator for FindOverlappingIter<'a, 'h, A> { + type Item = Match; + + #[inline(always)] + fn next(&mut self) -> Option { + self.aut + .try_find_overlapping(&self.input, &mut self.state) + .expect("already checked that no match error can occur here"); + self.state.get_match() + } +} + +/// An iterator that reports matches in a stream. +/// +/// This iterator yields elements of type `io::Result`, where an error +/// is reported if there was a problem reading from the underlying stream. +/// The iterator terminates only when the underlying stream reaches `EOF`. +/// +/// This iterator is constructed via the [`Automaton::try_stream_find_iter`] +/// method. +/// +/// The type variable `A` refers to the implementation of the [`Automaton`] +/// trait used to execute the search. +/// +/// The type variable `R` refers to the `io::Read` stream that is being read +/// from. +/// +/// The lifetime `'a` refers to the lifetime of the [`Automaton`] +/// implementation. +#[cfg(feature = "std")] +#[derive(Debug)] +pub struct StreamFindIter<'a, A, R> { + it: StreamChunkIter<'a, A, R>, +} + +#[cfg(feature = "std")] +impl<'a, A: Automaton, R: std::io::Read> Iterator + for StreamFindIter<'a, A, R> +{ + type Item = std::io::Result; + + fn next(&mut self) -> Option> { + loop { + match self.it.next() { + None => return None, + Some(Err(err)) => return Some(Err(err)), + Some(Ok(StreamChunk::NonMatch { .. })) => {} + Some(Ok(StreamChunk::Match { mat, .. })) => { + return Some(Ok(mat)); + } + } + } + } +} + +/// An iterator that reports matches in a stream. +/// +/// (This doesn't actually implement the `Iterator` trait because it returns +/// something with a lifetime attached to a buffer it owns, but that's OK. It +/// still has a `next` method and is iterator-like enough to be fine.) +/// +/// This iterator yields elements of type `io::Result`, where +/// an error is reported if there was a problem reading from the underlying +/// stream. The iterator terminates only when the underlying stream reaches +/// `EOF`. +/// +/// The idea here is that each chunk represents either a match or a non-match, +/// and if you concatenated all of the chunks together, you'd reproduce the +/// entire contents of the stream, byte-for-byte. +/// +/// This chunk machinery is a bit complicated and it isn't strictly required +/// for a stream searcher that just reports matches. But we do need something +/// like this to deal with the "replacement" API, which needs to know which +/// chunks it can copy and which it needs to replace. +#[cfg(feature = "std")] +#[derive(Debug)] +struct StreamChunkIter<'a, A, R> { + /// The underlying automaton to do the search. + aut: &'a A, + /// The source of bytes we read from. + rdr: R, + /// A roll buffer for managing bytes from `rdr`. Basically, this is used + /// to handle the case of a match that is split by two different + /// calls to `rdr.read()`. This isn't strictly needed if all we needed to + /// do was report matches, but here we are reporting chunks of non-matches + /// and matches and in order to do that, we really just cannot treat our + /// stream as non-overlapping blocks of bytes. We need to permit some + /// overlap while we retain bytes from a previous `read` call in memory. + buf: crate::util::buffer::Buffer, + /// The unanchored starting state of this automaton. + start: StateID, + /// The state of the automaton. + sid: StateID, + /// The absolute position over the entire stream. + absolute_pos: usize, + /// The position we're currently at within `buf`. + buffer_pos: usize, + /// The buffer position of the end of the bytes that we last returned + /// to the caller. Basically, whenever we find a match, we look to see if + /// there is a difference between where the match started and the position + /// of the last byte we returned to the caller. If there's a difference, + /// then we need to return a 'NonMatch' chunk. + buffer_reported_pos: usize, +} + +#[cfg(feature = "std")] +impl<'a, A: Automaton, R: std::io::Read> StreamChunkIter<'a, A, R> { + fn new( + aut: &'a A, + rdr: R, + ) -> Result, MatchError> { + // This restriction is a carry-over from older versions of this crate. + // I didn't have the bandwidth to think through how to handle, say, + // leftmost-first or leftmost-longest matching, but... it should be + // possible? The main problem is that once you see a match state in + // leftmost-first semantics, you can't just stop at that point and + // report a match. You have to keep going until you either hit a dead + // state or EOF. So how do you know when you'll hit a dead state? Well, + // you don't. With Aho-Corasick, I believe you can put a bound on it + // and say, "once a match has been seen, you'll need to scan forward at + // most N bytes" where N=aut.max_pattern_len(). + // + // Which is fine, but it does mean that state about whether we're still + // looking for a dead state or not needs to persist across buffer + // refills. Which this code doesn't really handle. It does preserve + // *some* state across buffer refills, basically ensuring that a match + // span is always in memory. + if !aut.match_kind().is_standard() { + return Err(MatchError::unsupported_stream(aut.match_kind())); + } + // This is kind of a cop-out, but empty matches are SUPER annoying. + // If we know they can't happen (which is what we enforce here), then + // it makes a lot of logic much simpler. With that said, I'm open to + // supporting this case, but we need to define proper semantics for it + // first. It wasn't totally clear to me what it should do at the time + // of writing, so I decided to just be conservative. + // + // It also seems like a very weird case to support anyway. Why search a + // stream if you're just going to get a match at every position? + // + // ¯\_(ツ)_/¯ + if aut.min_pattern_len() == 0 { + return Err(MatchError::unsupported_empty()); + } + let start = aut.start_state(Anchored::No)?; + Ok(StreamChunkIter { + aut, + rdr, + buf: crate::util::buffer::Buffer::new(aut.max_pattern_len()), + start, + sid: start, + absolute_pos: 0, + buffer_pos: 0, + buffer_reported_pos: 0, + }) + } + + fn next(&mut self) -> Option> { + // This code is pretty gnarly. It IS simpler than the equivalent code + // in the previous aho-corasick release, in part because we inline + // automaton traversal here and also in part because we have abdicated + // support for automatons that contain an empty pattern. + // + // I suspect this code could be made a bit simpler by designing a + // better buffer abstraction. + // + // But in general, this code is basically write-only. So you'll need + // to go through it step-by-step to grok it. One of the key bits of + // complexity is tracking a few different offsets. 'buffer_pos' is + // where we are in the buffer for search. 'buffer_reported_pos' is the + // position immediately following the last byte in the buffer that + // we've returned to the caller. And 'absolute_pos' is the overall + // current absolute position of the search in the entire stream, and + // this is what match spans are reported in terms of. + loop { + if self.aut.is_match(self.sid) { + let mat = self.get_match(); + if let Some(r) = self.get_non_match_chunk(mat) { + self.buffer_reported_pos += r.len(); + let bytes = &self.buf.buffer()[r]; + return Some(Ok(StreamChunk::NonMatch { bytes })); + } + self.sid = self.start; + let r = self.get_match_chunk(mat); + self.buffer_reported_pos += r.len(); + let bytes = &self.buf.buffer()[r]; + return Some(Ok(StreamChunk::Match { bytes, mat })); + } + if self.buffer_pos >= self.buf.buffer().len() { + if let Some(r) = self.get_pre_roll_non_match_chunk() { + self.buffer_reported_pos += r.len(); + let bytes = &self.buf.buffer()[r]; + return Some(Ok(StreamChunk::NonMatch { bytes })); + } + if self.buf.buffer().len() >= self.buf.min_buffer_len() { + self.buffer_pos = self.buf.min_buffer_len(); + self.buffer_reported_pos -= + self.buf.buffer().len() - self.buf.min_buffer_len(); + self.buf.roll(); + } + match self.buf.fill(&mut self.rdr) { + Err(err) => return Some(Err(err)), + Ok(true) => {} + Ok(false) => { + // We've hit EOF, but if there are still some + // unreported bytes remaining, return them now. + if let Some(r) = self.get_eof_non_match_chunk() { + self.buffer_reported_pos += r.len(); + let bytes = &self.buf.buffer()[r]; + return Some(Ok(StreamChunk::NonMatch { bytes })); + } + // We've reported everything! + return None; + } + } + } + let start = self.absolute_pos; + for &byte in self.buf.buffer()[self.buffer_pos..].iter() { + self.sid = self.aut.next_state(Anchored::No, self.sid, byte); + self.absolute_pos += 1; + if self.aut.is_match(self.sid) { + break; + } + } + self.buffer_pos += self.absolute_pos - start; + } + } + + /// Return a match chunk for the given match. It is assumed that the match + /// ends at the current `buffer_pos`. + fn get_match_chunk(&self, mat: Match) -> core::ops::Range { + let start = self.buffer_pos - mat.len(); + let end = self.buffer_pos; + start..end + } + + /// Return a non-match chunk, if necessary, just before reporting a match. + /// This returns `None` if there is nothing to report. Otherwise, this + /// assumes that the given match ends at the current `buffer_pos`. + fn get_non_match_chunk( + &self, + mat: Match, + ) -> Option> { + let buffer_mat_start = self.buffer_pos - mat.len(); + if buffer_mat_start > self.buffer_reported_pos { + let start = self.buffer_reported_pos; + let end = buffer_mat_start; + return Some(start..end); + } + None + } + + /// Look for any bytes that should be reported as a non-match just before + /// rolling the buffer. + /// + /// Note that this only reports bytes up to `buffer.len() - + /// min_buffer_len`, as it's not possible to know whether the bytes + /// following that will participate in a match or not. + fn get_pre_roll_non_match_chunk(&self) -> Option> { + let end = + self.buf.buffer().len().saturating_sub(self.buf.min_buffer_len()); + if self.buffer_reported_pos < end { + return Some(self.buffer_reported_pos..end); + } + None + } + + /// Return any unreported bytes as a non-match up to the end of the buffer. + /// + /// This should only be called when the entire contents of the buffer have + /// been searched and EOF has been hit when trying to fill the buffer. + fn get_eof_non_match_chunk(&self) -> Option> { + if self.buffer_reported_pos < self.buf.buffer().len() { + return Some(self.buffer_reported_pos..self.buf.buffer().len()); + } + None + } + + /// Return the match at the current position for the current state. + /// + /// This panics if `self.aut.is_match(self.sid)` isn't true. + fn get_match(&self) -> Match { + get_match(self.aut, self.sid, 0, self.absolute_pos) + } +} + +/// A single chunk yielded by the stream chunk iterator. +/// +/// The `'r` lifetime refers to the lifetime of the stream chunk iterator. +#[cfg(feature = "std")] +#[derive(Debug)] +enum StreamChunk<'r> { + /// A chunk that does not contain any matches. + NonMatch { bytes: &'r [u8] }, + /// A chunk that precisely contains a match. + Match { bytes: &'r [u8], mat: Match }, +} + +#[inline(never)] +pub(crate) fn try_find_fwd( + aut: &A, + input: &Input<'_>, +) -> Result, MatchError> { + if input.is_done() { + return Ok(None); + } + let earliest = aut.match_kind().is_standard() || input.get_earliest(); + if input.get_anchored().is_anchored() { + try_find_fwd_imp(aut, input, None, Anchored::Yes, earliest) + } else if let Some(pre) = aut.prefilter() { + if earliest { + try_find_fwd_imp(aut, input, Some(pre), Anchored::No, true) + } else { + try_find_fwd_imp(aut, input, Some(pre), Anchored::No, false) + } + } else { + if earliest { + try_find_fwd_imp(aut, input, None, Anchored::No, true) + } else { + try_find_fwd_imp(aut, input, None, Anchored::No, false) + } + } +} + +#[inline(always)] +fn try_find_fwd_imp( + aut: &A, + input: &Input<'_>, + pre: Option<&Prefilter>, + anchored: Anchored, + earliest: bool, +) -> Result, MatchError> { + let mut sid = aut.start_state(input.get_anchored())?; + let mut at = input.start(); + let mut mat = None; + if aut.is_match(sid) { + mat = Some(get_match(aut, sid, 0, at)); + if earliest { + return Ok(mat); + } + } + if let Some(pre) = pre { + match pre.find_in(input.haystack(), input.get_span()) { + Candidate::None => return Ok(None), + Candidate::Match(m) => return Ok(Some(m)), + Candidate::PossibleStartOfMatch(i) => { + at = i; + } + } + } + while at < input.end() { + // I've tried unrolling this loop and eliding bounds checks, but no + // matter what I did, I could not observe a consistent improvement on + // any benchmark I could devise. (If someone wants to re-litigate this, + // the way to do it is to add an 'next_state_unchecked' method to the + // 'Automaton' trait with a default impl that uses 'next_state'. Then + // use 'aut.next_state_unchecked' here and implement it on DFA using + // unchecked slice index acces.) + sid = aut.next_state(anchored, sid, input.haystack()[at]); + if aut.is_special(sid) { + if aut.is_dead(sid) { + return Ok(mat); + } else if aut.is_match(sid) { + // We use 'at + 1' here because the match state is entered + // at the last byte of the pattern. Since we use half-open + // intervals, the end of the range of the match is one past the + // last byte. + let m = get_match(aut, sid, 0, at + 1); + // For the automata in this crate, we make a size trade off + // where we reuse the same automaton for both anchored and + // unanchored searches. We achieve this, principally, by simply + // not following failure transitions while computing the next + // state. Instead, if we fail to find the next state, we return + // a dead state, which instructs the search to stop. (This + // is why 'next_state' needs to know whether the search is + // anchored or not.) In addition, we have different start + // states for anchored and unanchored searches. The latter has + // a self-loop where as the former does not. + // + // In this way, we can use the same trie to execute both + // anchored and unanchored searches. There is a catch though. + // When building an Aho-Corasick automaton for unanchored + // searches, we copy matches from match states to other states + // (which would otherwise not be match states) if they are + // reachable via a failure transition. In the case of an + // anchored search, we *specifically* do not want to report + // these matches because they represent matches that start past + // the beginning of the search. + // + // Now we could tweak the automaton somehow to differentiate + // anchored from unanchored match states, but this would make + // 'aut.is_match' and potentially 'aut.is_special' slower. And + // also make the automaton itself more complex. + // + // Instead, we insert a special hack: if the search is + // anchored, we simply ignore matches that don't begin at + // the start of the search. This is not quite ideal, but we + // do specialize this function in such a way that unanchored + // searches don't pay for this additional branch. While this + // might cause a search to continue on for more than it + // otherwise optimally would, it will be no more than the + // longest pattern in the automaton. The reason for this is + // that we ensure we don't follow failure transitions during + // an anchored search. Combined with using a different anchored + // starting state with no self-loop, we guarantee that we'll + // at worst move through a number of transitions equal to the + // longest pattern. + // + // Now for DFAs, the whole point of them is to eliminate + // failure transitions entirely. So there is no way to say "if + // it's an anchored search don't follow failure transitions." + // Instead, we actually have to build two entirely separate + // automatons into the transition table. One with failure + // transitions built into it and another that is effectively + // just an encoding of the base trie into a transition table. + // DFAs still need this check though, because the match states + // still carry matches only reachable via a failure transition. + // Why? Because removing them seems difficult, although I + // haven't given it a lot of thought. + if !(anchored.is_anchored() && m.start() > input.start()) { + mat = Some(m); + if earliest { + return Ok(mat); + } + } + } else if let Some(pre) = pre { + // If we're here, we know it's a special state that is not a + // dead or a match state AND that a prefilter is active. Thus, + // it must be a start state. + debug_assert!(aut.is_start(sid)); + // We don't care about 'Candidate::Match' here because if such + // a match were possible, it would have been returned above + // when we run the prefilter before walking the automaton. + let span = Span::from(at..input.end()); + match pre.find_in(input.haystack(), span).into_option() { + None => return Ok(None), + Some(i) => { + if i > at { + at = i; + continue; + } + } + } + } else { + // When pre.is_none(), then starting states should not be + // treated as special. That is, without a prefilter, is_special + // should only return true when the state is a dead or a match + // state. + // + // It is possible to execute a search without a prefilter even + // when the underlying searcher has one: an anchored search. + // But in this case, the automaton makes it impossible to move + // back to the start state by construction, and thus, we should + // never reach this branch. + debug_assert!(false, "unreachable"); + } + } + at += 1; + } + Ok(mat) +} + +#[inline(never)] +fn try_find_overlapping_fwd( + aut: &A, + input: &Input<'_>, + state: &mut OverlappingState, +) -> Result<(), MatchError> { + state.mat = None; + if input.is_done() { + return Ok(()); + } + // Searching with a pattern ID is always anchored, so we should only ever + // use a prefilter when no pattern ID is given. + if aut.prefilter().is_some() && !input.get_anchored().is_anchored() { + let pre = aut.prefilter().unwrap(); + try_find_overlapping_fwd_imp(aut, input, Some(pre), state) + } else { + try_find_overlapping_fwd_imp(aut, input, None, state) + } +} + +#[inline(always)] +fn try_find_overlapping_fwd_imp( + aut: &A, + input: &Input<'_>, + pre: Option<&Prefilter>, + state: &mut OverlappingState, +) -> Result<(), MatchError> { + let mut sid = match state.id { + None => { + let sid = aut.start_state(input.get_anchored())?; + // Handle the case where the start state is a match state. That is, + // the empty string is in our automaton. We report every match we + // can here before moving on and updating 'state.at' and 'state.id' + // to find more matches in other parts of the haystack. + if aut.is_match(sid) { + let i = state.next_match_index.unwrap_or(0); + let len = aut.match_len(sid); + if i < len { + state.next_match_index = Some(i + 1); + state.mat = Some(get_match(aut, sid, i, input.start())); + return Ok(()); + } + } + state.at = input.start(); + state.id = Some(sid); + state.next_match_index = None; + state.mat = None; + sid + } + Some(sid) => { + // If we still have matches left to report in this state then + // report them until we've exhausted them. Only after that do we + // advance to the next offset in the haystack. + if let Some(i) = state.next_match_index { + let len = aut.match_len(sid); + if i < len { + state.next_match_index = Some(i + 1); + state.mat = Some(get_match(aut, sid, i, state.at + 1)); + return Ok(()); + } + // Once we've reported all matches at a given position, we need + // to advance the search to the next position. + state.at += 1; + state.next_match_index = None; + state.mat = None; + } + sid + } + }; + while state.at < input.end() { + sid = aut.next_state( + input.get_anchored(), + sid, + input.haystack()[state.at], + ); + if aut.is_special(sid) { + state.id = Some(sid); + if aut.is_dead(sid) { + return Ok(()); + } else if aut.is_match(sid) { + state.next_match_index = Some(1); + state.mat = Some(get_match(aut, sid, 0, state.at + 1)); + return Ok(()); + } else if let Some(pre) = pre { + // If we're here, we know it's a special state that is not a + // dead or a match state AND that a prefilter is active. Thus, + // it must be a start state. + debug_assert!(aut.is_start(sid)); + let span = Span::from(state.at..input.end()); + match pre.find_in(input.haystack(), span).into_option() { + None => return Ok(()), + Some(i) => { + if i > state.at { + state.at = i; + continue; + } + } + } + } else { + // When pre.is_none(), then starting states should not be + // treated as special. That is, without a prefilter, is_special + // should only return true when the state is a dead or a match + // state. + // + // ... except for one special case: in stream searching, we + // currently call overlapping search with a 'None' prefilter, + // regardless of whether one exists or not, because stream + // searching can't currently deal with prefilters correctly in + // all cases. + } + } + state.at += 1; + } + state.id = Some(sid); + Ok(()) +} + +#[inline(always)] +fn get_match( + aut: &A, + sid: StateID, + index: usize, + at: usize, +) -> Match { + let pid = aut.match_pattern(sid, index); + let len = aut.pattern_len(pid); + Match::new(pid, (at - len)..at) +} + +/// Write a prefix "state" indicator for fmt::Debug impls. It always writes +/// exactly two printable bytes to the given formatter. +/// +/// Specifically, this tries to succinctly distinguish the different types of +/// states: dead states, start states and match states. It even accounts for +/// the possible overlappings of different state types. (The only possible +/// overlapping is that of match and start states.) +pub(crate) fn fmt_state_indicator( + f: &mut core::fmt::Formatter<'_>, + aut: A, + id: StateID, +) -> core::fmt::Result { + if aut.is_dead(id) { + write!(f, "D ")?; + } else if aut.is_match(id) { + if aut.is_start(id) { + write!(f, "*>")?; + } else { + write!(f, "* ")?; + } + } else if aut.is_start(id) { + write!(f, " >")?; + } else { + write!(f, " ")?; + } + Ok(()) +} + +/// Return an iterator of transitions in a sparse format given an iterator +/// of all explicitly defined transitions. The iterator yields ranges of +/// transitions, such that any adjacent transitions mapped to the same +/// state are combined into a single range. +pub(crate) fn sparse_transitions<'a>( + mut it: impl Iterator + 'a, +) -> impl Iterator + 'a { + let mut cur: Option<(u8, u8, StateID)> = None; + core::iter::from_fn(move || { + while let Some((class, next)) = it.next() { + let (prev_start, prev_end, prev_next) = match cur { + Some(x) => x, + None => { + cur = Some((class, class, next)); + continue; + } + }; + if prev_next == next { + cur = Some((prev_start, class, prev_next)); + } else { + cur = Some((class, class, next)); + return Some((prev_start, prev_end, prev_next)); + } + } + if let Some((start, end, next)) = cur.take() { + return Some((start, end, next)); + } + None + }) +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/dfa.rs b/third_party/crates/aho-corasick-1.1.5/src/dfa.rs new file mode 100644 index 0000000..1aa4f0e --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/dfa.rs @@ -0,0 +1,835 @@ +/*! +Provides direct access to a DFA implementation of Aho-Corasick. + +This is a low-level API that generally only needs to be used in niche +circumstances. When possible, prefer using [`AhoCorasick`](crate::AhoCorasick) +instead of a DFA directly. Using an `DFA` directly is typically only necessary +when one needs access to the [`Automaton`] trait implementation. +*/ + +use alloc::{vec, vec::Vec}; + +use crate::{ + automaton::Automaton, + nfa::noncontiguous, + util::{ + alphabet::ByteClasses, + error::{BuildError, MatchError}, + int::{Usize, U32}, + prefilter::Prefilter, + primitives::{IteratorIndexExt, PatternID, SmallIndex, StateID}, + search::{Anchored, MatchKind, StartKind}, + special::Special, + }, +}; + +/// A DFA implementation of Aho-Corasick. +/// +/// When possible, prefer using [`AhoCorasick`](crate::AhoCorasick) instead of +/// this type directly. Using a `DFA` directly is typically only necessary when +/// one needs access to the [`Automaton`] trait implementation. +/// +/// This DFA can only be built by first constructing a [`noncontiguous::NFA`]. +/// Both [`DFA::new`] and [`Builder::build`] do this for you automatically, but +/// [`Builder::build_from_noncontiguous`] permits doing it explicitly. +/// +/// A DFA provides the best possible search performance (in this crate) via two +/// mechanisms: +/// +/// * All states use a dense representation for their transitions. +/// * All failure transitions are pre-computed such that they are never +/// explicitly handled at search time. +/// +/// These two facts combined mean that every state transition is performed +/// using a constant number of instructions. However, this comes at +/// great cost. The memory usage of a DFA can be quite exorbitant. +/// It is potentially multiple orders of magnitude greater than a +/// [`contiguous::NFA`](crate::nfa::contiguous::NFA) for example. In exchange, +/// a DFA will typically have better search speed than a `contiguous::NFA`, but +/// not by orders of magnitude. +/// +/// Unless you have a small number of patterns or memory usage is not a concern +/// and search performance is critical, a DFA is usually not the best choice. +/// +/// Moreover, unlike the NFAs in this crate, it is costly for a DFA to +/// support for anchored and unanchored search configurations. Namely, +/// since failure transitions are pre-computed, supporting both anchored +/// and unanchored searches requires a duplication of the transition table, +/// making the memory usage of such a DFA ever bigger. (The NFAs in this crate +/// unconditionally support both anchored and unanchored searches because there +/// is essentially no added cost for doing so.) It is for this reason that +/// a DFA's support for anchored and unanchored searches can be configured +/// via [`Builder::start_kind`]. By default, a DFA only supports unanchored +/// searches. +/// +/// # Example +/// +/// This example shows how to build an `DFA` directly and use it to execute +/// [`Automaton::try_find`]: +/// +/// ``` +/// use aho_corasick::{ +/// automaton::Automaton, +/// dfa::DFA, +/// Input, Match, +/// }; +/// +/// let patterns = &["b", "abc", "abcd"]; +/// let haystack = "abcd"; +/// +/// let nfa = DFA::new(patterns).unwrap(); +/// assert_eq!( +/// Some(Match::must(0, 1..2)), +/// nfa.try_find(&Input::new(haystack))?, +/// ); +/// # Ok::<(), Box>(()) +/// ``` +/// +/// It is also possible to implement your own version of `try_find`. See the +/// [`Automaton`] documentation for an example. +#[derive(Clone)] +pub struct DFA { + /// The DFA transition table. IDs in this table are pre-multiplied. So + /// instead of the IDs being 0, 1, 2, 3, ..., they are 0*stride, 1*stride, + /// 2*stride, 3*stride, ... + trans: Vec, + /// The matches for every match state in this DFA. This is first indexed by + /// state index (so that's `sid >> stride2`) and then by order in which the + /// matches are meant to occur. + matches: Vec>, + /// The amount of heap memory used, in bytes, by the inner Vecs of + /// 'matches'. + matches_memory_usage: usize, + /// The length of each pattern. This is used to compute the start offset + /// of a match. + pattern_lens: Vec, + /// A prefilter for accelerating searches, if one exists. + prefilter: Option, + /// The match semantics built into this DFA. + match_kind: MatchKind, + /// The total number of states in this DFA. + state_len: usize, + /// The alphabet size, or total number of equivalence classes, for this + /// DFA. Note that the actual number of transitions in each state is + /// stride=2^stride2, where stride is the smallest power of 2 greater than + /// or equal to alphabet_len. We do things this way so that we can use + /// bitshifting to go from a state ID to an index into 'matches'. + alphabet_len: usize, + /// The exponent with a base 2, such that stride=2^stride2. Given a state + /// index 'i', its state identifier is 'i << stride2'. Given a state + /// identifier 'sid', its state index is 'sid >> stride2'. + stride2: usize, + /// The equivalence classes for this DFA. All transitions are defined on + /// equivalence classes and not on the 256 distinct byte values. + byte_classes: ByteClasses, + /// The length of the shortest pattern in this automaton. + min_pattern_len: usize, + /// The length of the longest pattern in this automaton. + max_pattern_len: usize, + /// The information required to deduce which states are "special" in this + /// DFA. + special: Special, +} + +impl DFA { + /// Create a new Aho-Corasick DFA using the default configuration. + /// + /// Use a [`Builder`] if you want to change the configuration. + pub fn new(patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + DFA::builder().build(patterns) + } + + /// A convenience method for returning a new Aho-Corasick DFA builder. + /// + /// This usually permits one to just import the `DFA` type. + pub fn builder() -> Builder { + Builder::new() + } +} + +impl DFA { + /// A sentinel state ID indicating that a search should stop once it has + /// entered this state. When a search stops, it returns a match if one has + /// been found, otherwise no match. A DFA always has an actual dead state + /// at this ID. + /// + /// N.B. DFAs, unlike NFAs, do not have any notion of a FAIL state. + /// Namely, the whole point of a DFA is that the FAIL state is completely + /// compiled away. That is, DFA construction involves pre-computing the + /// failure transitions everywhere, such that failure transitions are no + /// longer used at search time. This, combined with its uniformly dense + /// representation, are the two most important factors in why it's faster + /// than the NFAs in this crate. + const DEAD: StateID = StateID::new_unchecked(0); + + /// Adds the given pattern IDs as matches to the given state and also + /// records the added memory usage. + fn set_matches( + &mut self, + sid: StateID, + pids: impl Iterator, + ) { + let index = (sid.as_usize() >> self.stride2).checked_sub(2).unwrap(); + let mut at_least_one = false; + for pid in pids { + self.matches[index].push(pid); + self.matches_memory_usage += PatternID::SIZE; + at_least_one = true; + } + assert!(at_least_one, "match state must have non-empty pids"); + } +} + +// SAFETY: 'start_state' always returns a valid state ID, 'next_state' always +// returns a valid state ID given a valid state ID. We otherwise claim that +// all other methods are correct as well. +unsafe impl Automaton for DFA { + #[inline(always)] + fn start_state(&self, anchored: Anchored) -> Result { + // Either of the start state IDs can be DEAD, in which case, support + // for that type of search is not provided by this DFA. Which start + // state IDs are inactive depends on the 'StartKind' configuration at + // DFA construction time. + match anchored { + Anchored::No => { + let start = self.special.start_unanchored_id; + if start == DFA::DEAD { + Err(MatchError::invalid_input_unanchored()) + } else { + Ok(start) + } + } + Anchored::Yes => { + let start = self.special.start_anchored_id; + if start == DFA::DEAD { + Err(MatchError::invalid_input_anchored()) + } else { + Ok(start) + } + } + } + } + + #[inline(always)] + fn next_state( + &self, + _anchored: Anchored, + sid: StateID, + byte: u8, + ) -> StateID { + let class = self.byte_classes.get(byte); + self.trans[(sid.as_u32() + u32::from(class)).as_usize()] + } + + #[inline(always)] + fn is_special(&self, sid: StateID) -> bool { + sid <= self.special.max_special_id + } + + #[inline(always)] + fn is_dead(&self, sid: StateID) -> bool { + sid == DFA::DEAD + } + + #[inline(always)] + fn is_match(&self, sid: StateID) -> bool { + !self.is_dead(sid) && sid <= self.special.max_match_id + } + + #[inline(always)] + fn is_start(&self, sid: StateID) -> bool { + sid == self.special.start_unanchored_id + || sid == self.special.start_anchored_id + } + + #[inline(always)] + fn match_kind(&self) -> MatchKind { + self.match_kind + } + + #[inline(always)] + fn patterns_len(&self) -> usize { + self.pattern_lens.len() + } + + #[inline(always)] + fn pattern_len(&self, pid: PatternID) -> usize { + self.pattern_lens[pid].as_usize() + } + + #[inline(always)] + fn min_pattern_len(&self) -> usize { + self.min_pattern_len + } + + #[inline(always)] + fn max_pattern_len(&self) -> usize { + self.max_pattern_len + } + + #[inline(always)] + fn match_len(&self, sid: StateID) -> usize { + debug_assert!(self.is_match(sid)); + let offset = (sid.as_usize() >> self.stride2) - 2; + self.matches[offset].len() + } + + #[inline(always)] + fn match_pattern(&self, sid: StateID, index: usize) -> PatternID { + debug_assert!(self.is_match(sid)); + let offset = (sid.as_usize() >> self.stride2) - 2; + self.matches[offset][index] + } + + #[inline(always)] + fn memory_usage(&self) -> usize { + use core::mem::size_of; + + (self.trans.len() * size_of::()) + + (self.matches.len() * size_of::>()) + + self.matches_memory_usage + + (self.pattern_lens.len() * size_of::()) + + self.prefilter.as_ref().map_or(0, |p| p.memory_usage()) + } + + #[inline(always)] + fn prefilter(&self) -> Option<&Prefilter> { + self.prefilter.as_ref() + } +} + +impl core::fmt::Debug for DFA { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + use crate::{ + automaton::{fmt_state_indicator, sparse_transitions}, + util::debug::DebugByte, + }; + + writeln!(f, "dfa::DFA(")?; + for index in 0..self.state_len { + let sid = StateID::new_unchecked(index << self.stride2); + // While we do currently include the FAIL state in the transition + // table (to simplify construction), it is never actually used. It + // poses problems with the code below because it gets treated as + // a match state incidentally when it is, of course, not. So we + // special case it. The fail state is always the first state after + // the dead state. + // + // If the construction is changed to remove the fail state (it + // probably should be), then this special case should be updated. + if index == 1 { + writeln!(f, "F {:06}:", sid.as_usize())?; + continue; + } + fmt_state_indicator(f, self, sid)?; + write!(f, "{:06}: ", sid.as_usize())?; + + let it = (0..self.byte_classes.alphabet_len()).map(|class| { + (class.as_u8(), self.trans[sid.as_usize() + class]) + }); + for (i, (start, end, next)) in sparse_transitions(it).enumerate() { + if i > 0 { + write!(f, ", ")?; + } + if start == end { + write!( + f, + "{:?} => {:?}", + DebugByte(start), + next.as_usize() + )?; + } else { + write!( + f, + "{:?}-{:?} => {:?}", + DebugByte(start), + DebugByte(end), + next.as_usize() + )?; + } + } + write!(f, "\n")?; + if self.is_match(sid) { + write!(f, " matches: ")?; + for i in 0..self.match_len(sid) { + if i > 0 { + write!(f, ", ")?; + } + let pid = self.match_pattern(sid, i); + write!(f, "{}", pid.as_usize())?; + } + write!(f, "\n")?; + } + } + writeln!(f, "match kind: {:?}", self.match_kind)?; + writeln!(f, "prefilter: {:?}", self.prefilter.is_some())?; + writeln!(f, "state length: {:?}", self.state_len)?; + writeln!(f, "pattern length: {:?}", self.patterns_len())?; + writeln!(f, "shortest pattern length: {:?}", self.min_pattern_len)?; + writeln!(f, "longest pattern length: {:?}", self.max_pattern_len)?; + writeln!(f, "alphabet length: {:?}", self.alphabet_len)?; + writeln!(f, "stride: {:?}", 1 << self.stride2)?; + writeln!(f, "byte classes: {:?}", self.byte_classes)?; + writeln!(f, "memory usage: {:?}", self.memory_usage())?; + writeln!(f, ")")?; + Ok(()) + } +} + +/// A builder for configuring an Aho-Corasick DFA. +/// +/// This builder has a subset of the options available to a +/// [`AhoCorasickBuilder`](crate::AhoCorasickBuilder). Of the shared options, +/// their behavior is identical. +#[derive(Clone, Debug)] +pub struct Builder { + noncontiguous: noncontiguous::Builder, + start_kind: StartKind, + byte_classes: bool, +} + +impl Default for Builder { + fn default() -> Builder { + Builder { + noncontiguous: noncontiguous::Builder::new(), + start_kind: StartKind::Unanchored, + byte_classes: true, + } + } +} + +impl Builder { + /// Create a new builder for configuring an Aho-Corasick DFA. + pub fn new() -> Builder { + Builder::default() + } + + /// Build an Aho-Corasick DFA from the given iterator of patterns. + /// + /// A builder may be reused to create more DFAs. + pub fn build(&self, patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + let nnfa = self.noncontiguous.build(patterns)?; + self.build_from_noncontiguous(&nnfa) + } + + /// Build an Aho-Corasick DFA from the given noncontiguous NFA. + /// + /// Note that when this method is used, only the `start_kind` and + /// `byte_classes` settings on this builder are respected. The other + /// settings only apply to the initial construction of the Aho-Corasick + /// automaton. Since using this method requires that initial construction + /// has already completed, all settings impacting only initial construction + /// are no longer relevant. + pub fn build_from_noncontiguous( + &self, + nnfa: &noncontiguous::NFA, + ) -> Result { + debug!("building DFA"); + let byte_classes = if self.byte_classes { + nnfa.byte_classes().clone() + } else { + ByteClasses::singletons() + }; + let state_len = match self.start_kind { + StartKind::Unanchored | StartKind::Anchored => nnfa.states().len(), + StartKind::Both => { + // These unwraps are OK because we know that the number of + // NFA states is < StateID::LIMIT which is in turn less than + // i32::MAX. Thus, there is always room to multiply by 2. + // Finally, the number of states is always at least 4 in the + // NFA (DEAD, FAIL, START-UNANCHORED, START-ANCHORED), so the + // subtraction of 4 is okay. + // + // Note that we subtract 4 because the "anchored" part of + // the DFA duplicates the unanchored part (without failure + // transitions), but reuses the DEAD, FAIL and START states. + nnfa.states() + .len() + .checked_mul(2) + .unwrap() + .checked_sub(4) + .unwrap() + } + }; + let trans_len = + match state_len.checked_shl(byte_classes.stride2().as_u32()) { + Some(trans_len) => trans_len, + None => { + return Err(BuildError::state_id_overflow( + StateID::MAX.as_u64(), + usize::MAX.as_u64(), + )) + } + }; + StateID::new(trans_len.checked_sub(byte_classes.stride()).unwrap()) + .map_err(|e| { + BuildError::state_id_overflow( + StateID::MAX.as_u64(), + e.attempted(), + ) + })?; + let num_match_states = match self.start_kind { + StartKind::Unanchored | StartKind::Anchored => { + nnfa.special().max_match_id.as_usize().checked_sub(1).unwrap() + } + StartKind::Both => nnfa + .special() + .max_match_id + .as_usize() + .checked_sub(1) + .unwrap() + .checked_mul(2) + .unwrap(), + }; + let mut dfa = DFA { + trans: vec![DFA::DEAD; trans_len], + matches: vec![vec![]; num_match_states], + matches_memory_usage: 0, + pattern_lens: nnfa.pattern_lens_raw().to_vec(), + prefilter: nnfa.prefilter().cloned(), + match_kind: nnfa.match_kind(), + state_len, + alphabet_len: byte_classes.alphabet_len(), + stride2: byte_classes.stride2(), + byte_classes, + min_pattern_len: nnfa.min_pattern_len(), + max_pattern_len: nnfa.max_pattern_len(), + // The special state IDs are set later. + special: Special::zero(), + }; + match self.start_kind { + StartKind::Both => { + self.finish_build_both_starts(nnfa, &mut dfa); + } + StartKind::Unanchored => { + self.finish_build_one_start(Anchored::No, nnfa, &mut dfa); + } + StartKind::Anchored => { + self.finish_build_one_start(Anchored::Yes, nnfa, &mut dfa) + } + } + debug!( + "DFA built, ", + dfa.state_len, + dfa.memory_usage(), + dfa.byte_classes.alphabet_len(), + dfa.byte_classes.stride(), + ); + // The vectors can grow ~twice as big during construction because a + // Vec amortizes growth. But here, let's shrink things back down to + // what we actually need since we're never going to add more to it. + dfa.trans.shrink_to_fit(); + dfa.pattern_lens.shrink_to_fit(); + dfa.matches.shrink_to_fit(); + // TODO: We might also want to shrink each Vec inside of `dfa.matches`, + // or even better, convert it to one contiguous allocation. But I think + // I went with nested allocs for good reason (can't remember), so this + // may be tricky to do. I decided not to shrink them here because it + // might require a fair bit of work to do. It's unclear whether it's + // worth it. + Ok(dfa) + } + + /// Finishes building a DFA for either unanchored or anchored searches, + /// but NOT both. + fn finish_build_one_start( + &self, + anchored: Anchored, + nnfa: &noncontiguous::NFA, + dfa: &mut DFA, + ) { + // This function always succeeds because we check above that all of the + // states in the NFA can be mapped to DFA state IDs. + let stride2 = dfa.stride2; + let old2new = |oldsid: StateID| { + StateID::new_unchecked(oldsid.as_usize() << stride2) + }; + for (oldsid, state) in nnfa.states().iter().with_state_ids() { + let newsid = old2new(oldsid); + if state.is_match() { + dfa.set_matches(newsid, nnfa.iter_matches(oldsid)); + } + sparse_iter( + nnfa, + oldsid, + &dfa.byte_classes, + |byte, class, mut oldnextsid| { + if oldnextsid == noncontiguous::NFA::FAIL { + if anchored.is_anchored() { + oldnextsid = noncontiguous::NFA::DEAD; + } else if state.fail() == noncontiguous::NFA::DEAD { + // This is a special case that avoids following + // DEAD transitions in a non-contiguous NFA. + // Following these transitions is pretty slow + // because the non-contiguous NFA will always use + // a sparse representation for it (because the + // DEAD state is usually treated as a sentinel). + // The *vast* majority of failure states are DEAD + // states, so this winds up being pretty slow if + // we go through the non-contiguous NFA state + // transition logic. Instead, just do it ourselves. + oldnextsid = noncontiguous::NFA::DEAD; + } else { + oldnextsid = nnfa.next_state( + Anchored::No, + state.fail(), + byte, + ); + } + } + dfa.trans[newsid.as_usize() + usize::from(class)] = + old2new(oldnextsid); + }, + ); + } + // Now that we've remapped all the IDs in our states, all that's left + // is remapping the special state IDs. + let old = nnfa.special(); + let new = &mut dfa.special; + new.max_special_id = old2new(old.max_special_id); + new.max_match_id = old2new(old.max_match_id); + if anchored.is_anchored() { + new.start_unanchored_id = DFA::DEAD; + new.start_anchored_id = old2new(old.start_anchored_id); + } else { + new.start_unanchored_id = old2new(old.start_unanchored_id); + new.start_anchored_id = DFA::DEAD; + } + } + + /// Finishes building a DFA that supports BOTH unanchored and anchored + /// searches. It works by inter-leaving unanchored states with anchored + /// states in the same transition table. This way, we avoid needing to + /// re-shuffle states afterward to ensure that our states still look like + /// DEAD, MATCH, ..., START-UNANCHORED, START-ANCHORED, NON-MATCH, ... + /// + /// Honestly this is pretty inscrutable... Simplifications are most + /// welcome. + fn finish_build_both_starts( + &self, + nnfa: &noncontiguous::NFA, + dfa: &mut DFA, + ) { + let stride2 = dfa.stride2; + let stride = 1 << stride2; + let mut remap_unanchored = vec![DFA::DEAD; nnfa.states().len()]; + let mut remap_anchored = vec![DFA::DEAD; nnfa.states().len()]; + let mut is_anchored = vec![false; dfa.state_len]; + let mut newsid = DFA::DEAD; + let next_dfa_id = + |sid: StateID| StateID::new_unchecked(sid.as_usize() + stride); + for (oldsid, state) in nnfa.states().iter().with_state_ids() { + if oldsid == noncontiguous::NFA::DEAD + || oldsid == noncontiguous::NFA::FAIL + { + remap_unanchored[oldsid] = newsid; + remap_anchored[oldsid] = newsid; + newsid = next_dfa_id(newsid); + } else if oldsid == nnfa.special().start_unanchored_id + || oldsid == nnfa.special().start_anchored_id + { + if oldsid == nnfa.special().start_unanchored_id { + remap_unanchored[oldsid] = newsid; + remap_anchored[oldsid] = DFA::DEAD; + } else { + remap_unanchored[oldsid] = DFA::DEAD; + remap_anchored[oldsid] = newsid; + is_anchored[newsid.as_usize() >> stride2] = true; + } + if state.is_match() { + dfa.set_matches(newsid, nnfa.iter_matches(oldsid)); + } + sparse_iter( + nnfa, + oldsid, + &dfa.byte_classes, + |_, class, oldnextsid| { + let class = usize::from(class); + if oldnextsid == noncontiguous::NFA::FAIL { + dfa.trans[newsid.as_usize() + class] = DFA::DEAD; + } else { + dfa.trans[newsid.as_usize() + class] = oldnextsid; + } + }, + ); + newsid = next_dfa_id(newsid); + } else { + let unewsid = newsid; + newsid = next_dfa_id(newsid); + let anewsid = newsid; + newsid = next_dfa_id(newsid); + + remap_unanchored[oldsid] = unewsid; + remap_anchored[oldsid] = anewsid; + is_anchored[anewsid.as_usize() >> stride2] = true; + if state.is_match() { + dfa.set_matches(unewsid, nnfa.iter_matches(oldsid)); + dfa.set_matches(anewsid, nnfa.iter_matches(oldsid)); + } + sparse_iter( + nnfa, + oldsid, + &dfa.byte_classes, + |byte, class, oldnextsid| { + let class = usize::from(class); + if oldnextsid == noncontiguous::NFA::FAIL { + let oldnextsid = + if state.fail() == noncontiguous::NFA::DEAD { + noncontiguous::NFA::DEAD + } else { + nnfa.next_state( + Anchored::No, + state.fail(), + byte, + ) + }; + dfa.trans[unewsid.as_usize() + class] = oldnextsid; + } else { + dfa.trans[unewsid.as_usize() + class] = oldnextsid; + dfa.trans[anewsid.as_usize() + class] = oldnextsid; + } + }, + ); + } + } + for i in 0..dfa.state_len { + let sid = i << stride2; + if is_anchored[i] { + for next in dfa.trans[sid..][..stride].iter_mut() { + *next = remap_anchored[*next]; + } + } else { + for next in dfa.trans[sid..][..stride].iter_mut() { + *next = remap_unanchored[*next]; + } + } + } + // Now that we've remapped all the IDs in our states, all that's left + // is remapping the special state IDs. + let old = nnfa.special(); + let new = &mut dfa.special; + new.max_special_id = remap_anchored[old.max_special_id]; + new.max_match_id = remap_anchored[old.max_match_id]; + new.start_unanchored_id = remap_unanchored[old.start_unanchored_id]; + new.start_anchored_id = remap_anchored[old.start_anchored_id]; + } + + /// Set the desired match semantics. + /// + /// This only applies when using [`Builder::build`] and not + /// [`Builder::build_from_noncontiguous`]. + /// + /// See + /// [`AhoCorasickBuilder::match_kind`](crate::AhoCorasickBuilder::match_kind) + /// for more documentation and examples. + pub fn match_kind(&mut self, kind: MatchKind) -> &mut Builder { + self.noncontiguous.match_kind(kind); + self + } + + /// Enable ASCII-aware case insensitive matching. + /// + /// This only applies when using [`Builder::build`] and not + /// [`Builder::build_from_noncontiguous`]. + /// + /// See + /// [`AhoCorasickBuilder::ascii_case_insensitive`](crate::AhoCorasickBuilder::ascii_case_insensitive) + /// for more documentation and examples. + pub fn ascii_case_insensitive(&mut self, yes: bool) -> &mut Builder { + self.noncontiguous.ascii_case_insensitive(yes); + self + } + + /// Enable heuristic prefilter optimizations. + /// + /// This only applies when using [`Builder::build`] and not + /// [`Builder::build_from_noncontiguous`]. + /// + /// See + /// [`AhoCorasickBuilder::prefilter`](crate::AhoCorasickBuilder::prefilter) + /// for more documentation and examples. + pub fn prefilter(&mut self, yes: bool) -> &mut Builder { + self.noncontiguous.prefilter(yes); + self + } + + /// Sets the starting state configuration for the automaton. + /// + /// See + /// [`AhoCorasickBuilder::start_kind`](crate::AhoCorasickBuilder::start_kind) + /// for more documentation and examples. + pub fn start_kind(&mut self, kind: StartKind) -> &mut Builder { + self.start_kind = kind; + self + } + + /// A debug setting for whether to attempt to shrink the size of the + /// automaton's alphabet or not. + /// + /// This should never be enabled unless you're debugging an automaton. + /// Namely, disabling byte classes makes transitions easier to reason + /// about, since they use the actual bytes instead of equivalence classes. + /// Disabling this confers no performance benefit at search time. + /// + /// See + /// [`AhoCorasickBuilder::byte_classes`](crate::AhoCorasickBuilder::byte_classes) + /// for more documentation and examples. + pub fn byte_classes(&mut self, yes: bool) -> &mut Builder { + self.byte_classes = yes; + self + } +} + +/// Iterate over all possible equivalence class transitions in this state. +/// The closure is called for all transitions with a distinct equivalence +/// class, even those not explicitly represented in this sparse state. For +/// any implicitly defined transitions, the given closure is called with +/// the fail state ID. +/// +/// The closure is guaranteed to be called precisely +/// `byte_classes.alphabet_len()` times, once for every possible class in +/// ascending order. +fn sparse_iter( + nnfa: &noncontiguous::NFA, + oldsid: StateID, + classes: &ByteClasses, + mut f: F, +) { + let mut prev_class = None; + let mut byte = 0usize; + for t in nnfa.iter_trans(oldsid) { + while byte < usize::from(t.byte()) { + let rep = byte.as_u8(); + let class = classes.get(rep); + byte += 1; + if prev_class != Some(class) { + f(rep, class, noncontiguous::NFA::FAIL); + prev_class = Some(class); + } + } + let rep = t.byte(); + let class = classes.get(rep); + byte += 1; + if prev_class != Some(class) { + f(rep, class, t.next()); + prev_class = Some(class); + } + } + for b in byte..=255 { + let rep = b.as_u8(); + let class = classes.get(rep); + if prev_class != Some(class) { + f(rep, class, noncontiguous::NFA::FAIL); + prev_class = Some(class); + } + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/lib.rs b/third_party/crates/aho-corasick-1.1.5/src/lib.rs new file mode 100644 index 0000000..273dd3b --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/lib.rs @@ -0,0 +1,326 @@ +/*! +A library for finding occurrences of many patterns at once. This library +provides multiple pattern search principally through an implementation of the +[Aho-Corasick algorithm](https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm), +which builds a fast finite state machine for executing searches in linear time. + +Additionally, this library provides a number of configuration options for +building the automaton that permit controlling the space versus time trade +off. Other features include simple ASCII case insensitive matching, finding +overlapping matches, replacements, searching streams and even searching and +replacing text in streams. + +Finally, unlike most other Aho-Corasick implementations, this one +supports enabling [leftmost-first](MatchKind::LeftmostFirst) or +[leftmost-longest](MatchKind::LeftmostLongest) match semantics, using a +(seemingly) novel alternative construction algorithm. For more details on what +match semantics means, see the [`MatchKind`] type. + +# Overview + +This section gives a brief overview of the primary types in this crate: + +* [`AhoCorasick`] is the primary type and represents an Aho-Corasick automaton. +This is the type you use to execute searches. +* [`AhoCorasickBuilder`] can be used to build an Aho-Corasick automaton, and +supports configuring a number of options. +* [`Match`] represents a single match reported by an Aho-Corasick automaton. +Each match has two pieces of information: the pattern that matched and the +start and end byte offsets corresponding to the position in the haystack at +which it matched. + +# Example: basic searching + +This example shows how to search for occurrences of multiple patterns +simultaneously. Each match includes the pattern that matched along with the +byte offsets of the match. + +``` +use aho_corasick::{AhoCorasick, PatternID}; + +let patterns = &["apple", "maple", "Snapple"]; +let haystack = "Nobody likes maple in their apple flavored Snapple."; + +let ac = AhoCorasick::new(patterns).unwrap(); +let mut matches = vec![]; +for mat in ac.find_iter(haystack) { + matches.push((mat.pattern(), mat.start(), mat.end())); +} +assert_eq!(matches, vec![ + (PatternID::must(1), 13, 18), + (PatternID::must(0), 28, 33), + (PatternID::must(2), 43, 50), +]); +``` + +# Example: case insensitivity + +This is like the previous example, but matches `Snapple` case insensitively +using `AhoCorasickBuilder`: + +``` +use aho_corasick::{AhoCorasick, PatternID}; + +let patterns = &["apple", "maple", "snapple"]; +let haystack = "Nobody likes maple in their apple flavored Snapple."; + +let ac = AhoCorasick::builder() + .ascii_case_insensitive(true) + .build(patterns) + .unwrap(); +let mut matches = vec![]; +for mat in ac.find_iter(haystack) { + matches.push((mat.pattern(), mat.start(), mat.end())); +} +assert_eq!(matches, vec![ + (PatternID::must(1), 13, 18), + (PatternID::must(0), 28, 33), + (PatternID::must(2), 43, 50), +]); +``` + +# Example: replacing matches in a stream + +This example shows how to execute a search and replace on a stream without +loading the entire stream into memory first. + +``` +# #[cfg(feature = "std")] { +use aho_corasick::AhoCorasick; + +# fn example() -> Result<(), std::io::Error> { +let patterns = &["fox", "brown", "quick"]; +let replace_with = &["sloth", "grey", "slow"]; + +// In a real example, these might be `std::fs::File`s instead. All you need to +// do is supply a pair of `std::io::Read` and `std::io::Write` implementations. +let rdr = "The quick brown fox."; +let mut wtr = vec![]; + +let ac = AhoCorasick::new(patterns).unwrap(); +ac.try_stream_replace_all(rdr.as_bytes(), &mut wtr, replace_with)?; +assert_eq!(b"The slow grey sloth.".to_vec(), wtr); +# Ok(()) }; example().unwrap() +# } +``` + +# Example: finding the leftmost first match + +In the textbook description of Aho-Corasick, its formulation is typically +structured such that it reports all possible matches, even when they overlap +with another. In many cases, overlapping matches may not be desired, such as +the case of finding all successive non-overlapping matches like you might with +a standard regular expression. + +Unfortunately the "obvious" way to modify the Aho-Corasick algorithm to do +this doesn't always work in the expected way, since it will report matches as +soon as they are seen. For example, consider matching the regex `Samwise|Sam` +against the text `Samwise`. Most regex engines (that are Perl-like, or +non-POSIX) will report `Samwise` as a match, but the standard Aho-Corasick +algorithm modified for reporting non-overlapping matches will report `Sam`. + +A novel contribution of this library is the ability to change the match +semantics of Aho-Corasick (without additional search time overhead) such that +`Samwise` is reported instead. For example, here's the standard approach: + +``` +use aho_corasick::AhoCorasick; + +let patterns = &["Samwise", "Sam"]; +let haystack = "Samwise"; + +let ac = AhoCorasick::new(patterns).unwrap(); +let mat = ac.find(haystack).expect("should have a match"); +assert_eq!("Sam", &haystack[mat.start()..mat.end()]); +``` + +And now here's the leftmost-first version, which matches how a Perl-like +regex will work: + +``` +use aho_corasick::{AhoCorasick, MatchKind}; + +let patterns = &["Samwise", "Sam"]; +let haystack = "Samwise"; + +let ac = AhoCorasick::builder() + .match_kind(MatchKind::LeftmostFirst) + .build(patterns) + .unwrap(); +let mat = ac.find(haystack).expect("should have a match"); +assert_eq!("Samwise", &haystack[mat.start()..mat.end()]); +``` + +In addition to leftmost-first semantics, this library also supports +leftmost-longest semantics, which match the POSIX behavior of a regular +expression alternation. See [`MatchKind`] for more details. + +# Prefilters + +While an Aho-Corasick automaton can perform admirably when compared to more +naive solutions, it is generally slower than more specialized algorithms that +are accelerated using vector instructions such as SIMD. + +For that reason, this library will internally use a "prefilter" to attempt +to accelerate searches when possible. Currently, this library has several +different algorithms it might use depending on the patterns provided. Once the +number of patterns gets too big, prefilters are no longer used. + +While a prefilter is generally good to have on by default since it works +well in the common case, it can lead to less predictable or even sub-optimal +performance in some cases. For that reason, prefilters can be explicitly +disabled via [`AhoCorasickBuilder::prefilter`]. + +# Lower level APIs + +This crate also provides several sub-modules that collectively expose many of +the implementation details of the main [`AhoCorasick`] type. Most users of this +library can completely ignore the submodules and their contents, but if you +needed finer grained control, some parts of them may be useful to you. Here is +a brief overview of each and why you might want to use them: + +* The [`packed`] sub-module contains a lower level API for using fast +vectorized routines for finding a small number of patterns in a haystack. +You might want to use this API when you want to completely side-step using +Aho-Corasick automata. Otherwise, the fast vectorized routines are used +automatically as prefilters for `AhoCorasick` searches whenever possible. +* The [`automaton`] sub-module provides a lower level finite state +machine interface that the various Aho-Corasick implementations in +this crate implement. This sub-module's main contribution is the +[`Automaton`](automaton::Automaton) trait, which permits manually walking the +state transitions of an Aho-Corasick automaton. +* The [`dfa`] and [`nfa`] sub-modules provide DFA and NFA implementations of +the aforementioned `Automaton` trait. The main reason one might want to use +these sub-modules is to get access to a type that implements the `Automaton` +trait. (The top-level `AhoCorasick` type does not implement the `Automaton` +trait.) + +As mentioned above, if you aren't sure whether you need these sub-modules, +you should be able to safely ignore them and just focus on the [`AhoCorasick`] +type. + +# Crate features + +This crate exposes a few features for controlling dependency usage and whether +this crate can be used without the standard library. + +* **std** - + Enables support for the standard library. This feature is enabled by + default. When disabled, only `core` and `alloc` are used. At an API + level, enabling `std` enables `std::error::Error` trait impls for the + various error types, and higher level stream search routines such as + [`AhoCorasick::try_stream_find_iter`]. But the `std` feature is also required + to enable vectorized prefilters. Prefilters can greatly accelerate searches, + but generally only apply when the number of patterns is small (less than + ~100). +* **perf-literal** - + Enables support for literal prefilters that use vectorized routines from + external crates. This feature is enabled by default. If you're only using + Aho-Corasick for large numbers of patterns or otherwise can abide lower + throughput when searching with a small number of patterns, then it is + reasonable to disable this feature. +* **logging** - + Enables a dependency on the `log` crate and emits messages to aide in + diagnostics. This feature is disabled by default. +*/ + +#![no_std] +#![deny(missing_docs)] +#![deny(rustdoc::broken_intra_doc_links)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +extern crate alloc; +#[cfg(any(test, feature = "std"))] +extern crate std; + +#[cfg(doctest)] +doc_comment::doctest!("../README.md"); + +#[cfg(feature = "std")] +pub use crate::ahocorasick::StreamFindIter; +pub use crate::{ + ahocorasick::{ + AhoCorasick, AhoCorasickBuilder, AhoCorasickKind, FindIter, + FindOverlappingIter, + }, + util::{ + error::{BuildError, MatchError, MatchErrorKind}, + primitives::{PatternID, PatternIDError}, + search::{Anchored, Input, Match, MatchKind, Span, StartKind}, + }, +}; + +#[macro_use] +mod macros; + +mod ahocorasick; +pub mod automaton; +pub mod dfa; +pub mod nfa; +pub mod packed; +#[cfg(test)] +mod tests; +// I wrote out the module for implementing fst::Automaton only to later realize +// that this would make fst a public dependency and fst is not at 1.0 yet. I +// decided to just keep the code in tree, but build it only during tests. +// +// TODO: I think I've changed my mind again. I'm considering pushing it out +// into either a separate crate or into 'fst' directly as an optional feature. +// #[cfg(test)] +// #[allow(dead_code)] +// mod transducer; +pub(crate) mod util; + +#[cfg(test)] +mod testoibits { + use std::panic::{RefUnwindSafe, UnwindSafe}; + + use super::*; + + fn assert_all() {} + + #[test] + fn oibits_main() { + assert_all::(); + assert_all::(); + assert_all::(); + assert_all::(); + assert_all::(); + + assert_all::(); + assert_all::(); + assert_all::(); + + assert_all::(); + assert_all::(); + assert_all::(); + assert_all::(); + assert_all::(); + assert_all::(); + } + + #[test] + fn oibits_automaton() { + use crate::{automaton, dfa::DFA}; + + assert_all::>(); + assert_all::>(); + #[cfg(feature = "std")] + assert_all::>(); + assert_all::(); + + assert_all::(); + assert_all::(); + } + + #[test] + fn oibits_packed() { + use crate::packed; + + assert_all::(); + assert_all::(); + assert_all::(); + assert_all::(); + assert_all::(); + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/macros.rs b/third_party/crates/aho-corasick-1.1.5/src/macros.rs new file mode 100644 index 0000000..fc73e6e --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/macros.rs @@ -0,0 +1,18 @@ +#![allow(unused_macros)] + +macro_rules! log { + ($($tt:tt)*) => { + #[cfg(feature = "logging")] + { + $($tt)* + } + } +} + +macro_rules! debug { + ($($tt:tt)*) => { log!(log::debug!($($tt)*)) } +} + +macro_rules! trace { + ($($tt:tt)*) => { log!(log::trace!($($tt)*)) } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/nfa/contiguous.rs b/third_party/crates/aho-corasick-1.1.5/src/nfa/contiguous.rs new file mode 100644 index 0000000..6ea2a47 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/nfa/contiguous.rs @@ -0,0 +1,1141 @@ +/*! +Provides a contiguous NFA implementation of Aho-Corasick. + +This is a low-level API that generally only needs to be used in niche +circumstances. When possible, prefer using [`AhoCorasick`](crate::AhoCorasick) +instead of a contiguous NFA directly. Using an `NFA` directly is typically only +necessary when one needs access to the [`Automaton`] trait implementation. +*/ + +use alloc::{vec, vec::Vec}; + +use crate::{ + automaton::Automaton, + nfa::noncontiguous, + util::{ + alphabet::ByteClasses, + error::{BuildError, MatchError}, + int::{Usize, U16, U32}, + prefilter::Prefilter, + primitives::{IteratorIndexExt, PatternID, SmallIndex, StateID}, + search::{Anchored, MatchKind}, + special::Special, + }, +}; + +/// A contiguous NFA implementation of Aho-Corasick. +/// +/// When possible, prefer using [`AhoCorasick`](crate::AhoCorasick) instead of +/// this type directly. Using an `NFA` directly is typically only necessary +/// when one needs access to the [`Automaton`] trait implementation. +/// +/// This NFA can only be built by first constructing a [`noncontiguous::NFA`]. +/// Both [`NFA::new`] and [`Builder::build`] do this for you automatically, but +/// [`Builder::build_from_noncontiguous`] permits doing it explicitly. +/// +/// The main difference between a noncontiguous NFA and a contiguous NFA is +/// that the latter represents all of its states and transitions in a single +/// allocation, where as the former uses a separate allocation for each state. +/// Doing this at construction time while keeping a low memory footprint isn't +/// feasible, which is primarily why there are two different NFA types: one +/// that does the least amount of work possible to build itself, and another +/// that does a little extra work to compact itself and make state transitions +/// faster by making some states use a dense representation. +/// +/// Because a contiguous NFA uses a single allocation, there is a lot more +/// opportunity for compression tricks to reduce the heap memory used. Indeed, +/// it is not uncommon for a contiguous NFA to use an order of magnitude less +/// heap memory than a noncontiguous NFA. Since building a contiguous NFA +/// usually only takes a fraction of the time it takes to build a noncontiguous +/// NFA, the overall build time is not much slower. Thus, in most cases, a +/// contiguous NFA is the best choice. +/// +/// Since a contiguous NFA uses various tricks for compression and to achieve +/// faster state transitions, currently, its limit on the number of states +/// is somewhat smaller than what a noncontiguous NFA can achieve. Generally +/// speaking, you shouldn't expect to run into this limit if the number of +/// patterns is under 1 million. It is plausible that this limit will be +/// increased in the future. If the limit is reached, building a contiguous NFA +/// will return an error. Often, since building a contiguous NFA is relatively +/// cheap, it can make sense to always try it even if you aren't sure if it +/// will fail or not. If it does, you can always fall back to a noncontiguous +/// NFA. (Indeed, the main [`AhoCorasick`](crate::AhoCorasick) type employs a +/// strategy similar to this at construction time.) +/// +/// # Example +/// +/// This example shows how to build an `NFA` directly and use it to execute +/// [`Automaton::try_find`]: +/// +/// ``` +/// use aho_corasick::{ +/// automaton::Automaton, +/// nfa::contiguous::NFA, +/// Input, Match, +/// }; +/// +/// let patterns = &["b", "abc", "abcd"]; +/// let haystack = "abcd"; +/// +/// let nfa = NFA::new(patterns).unwrap(); +/// assert_eq!( +/// Some(Match::must(0, 1..2)), +/// nfa.try_find(&Input::new(haystack))?, +/// ); +/// # Ok::<(), Box>(()) +/// ``` +/// +/// It is also possible to implement your own version of `try_find`. See the +/// [`Automaton`] documentation for an example. +#[derive(Clone)] +pub struct NFA { + /// The raw NFA representation. Each state is packed with a header + /// (containing the format of the state, the failure transition and, for + /// a sparse state, the number of transitions), its transitions and any + /// matching pattern IDs for match states. + repr: Vec, + /// The length of each pattern. This is used to compute the start offset + /// of a match. + pattern_lens: Vec, + /// The total number of states in this NFA. + state_len: usize, + /// A prefilter for accelerating searches, if one exists. + prefilter: Option, + /// The match semantics built into this NFA. + match_kind: MatchKind, + /// The alphabet size, or total number of equivalence classes, for this + /// NFA. Dense states always have this many transitions. + alphabet_len: usize, + /// The equivalence classes for this NFA. All transitions, dense and + /// sparse, are defined on equivalence classes and not on the 256 distinct + /// byte values. + byte_classes: ByteClasses, + /// The length of the shortest pattern in this automaton. + min_pattern_len: usize, + /// The length of the longest pattern in this automaton. + max_pattern_len: usize, + /// The information required to deduce which states are "special" in this + /// NFA. + special: Special, +} + +impl NFA { + /// Create a new Aho-Corasick contiguous NFA using the default + /// configuration. + /// + /// Use a [`Builder`] if you want to change the configuration. + pub fn new(patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + NFA::builder().build(patterns) + } + + /// A convenience method for returning a new Aho-Corasick contiguous NFA + /// builder. + /// + /// This usually permits one to just import the `NFA` type. + pub fn builder() -> Builder { + Builder::new() + } +} + +impl NFA { + /// A sentinel state ID indicating that a search should stop once it has + /// entered this state. When a search stops, it returns a match if one + /// has been found, otherwise no match. A contiguous NFA always has an + /// actual dead state at this ID. + const DEAD: StateID = StateID::new_unchecked(0); + /// Another sentinel state ID indicating that a search should move through + /// current state's failure transition. + /// + /// Note that unlike DEAD, this does not actually point to a valid state + /// in a contiguous NFA. (noncontiguous::NFA::FAIL does point to a valid + /// state.) Instead, this points to the position that is guaranteed to + /// never be a valid state ID (by making sure it points to a place in the + /// middle of the encoding of the DEAD state). Since we never need to + /// actually look at the FAIL state itself, this works out. + /// + /// By why do it this way? So that FAIL is a constant. I don't have any + /// concrete evidence that this materially helps matters, but it's easy to + /// do. The alternative would be making the FAIL ID point to the second + /// state, which could be made a constant but is a little trickier to do. + /// The easiest path is to just make the FAIL state a runtime value, but + /// since comparisons with FAIL occur in perf critical parts of the search, + /// we want it to be as tight as possible and not waste any registers. + /// + /// Very hand wavy... But the code complexity that results from this is + /// very mild. + const FAIL: StateID = StateID::new_unchecked(1); +} + +// SAFETY: 'start_state' always returns a valid state ID, 'next_state' always +// returns a valid state ID given a valid state ID. We otherwise claim that +// all other methods are correct as well. +unsafe impl Automaton for NFA { + #[inline(always)] + fn start_state(&self, anchored: Anchored) -> Result { + match anchored { + Anchored::No => Ok(self.special.start_unanchored_id), + Anchored::Yes => Ok(self.special.start_anchored_id), + } + } + + #[inline(always)] + fn next_state( + &self, + anchored: Anchored, + mut sid: StateID, + byte: u8, + ) -> StateID { + let repr = &self.repr; + let class = self.byte_classes.get(byte); + let u32tosid = StateID::from_u32_unchecked; + loop { + let o = sid.as_usize(); + let kind = repr[o] & 0xFF; + // I tried to encapsulate the "next transition" logic into its own + // function, but it seemed to always result in sub-optimal codegen + // that led to real and significant slowdowns. So we just inline + // the logic here. + // + // I've also tried a lot of different ways to speed up this + // routine, and most of them have failed. + if kind == State::KIND_DENSE { + let next = u32tosid(repr[o + 2 + usize::from(class)]); + if next != NFA::FAIL { + return next; + } + } else if kind == State::KIND_ONE { + if class == repr[o].low_u16().high_u8() { + return u32tosid(repr[o + 2]); + } + } else { + // NOTE: I tried a SWAR technique in the loop below, but found + // it slower. See the 'swar' test in the tests for this module. + let trans_len = kind.as_usize(); + let classes_len = u32_len(trans_len); + let trans_offset = o + 2 + classes_len; + for (i, &chunk) in + repr[o + 2..][..classes_len].iter().enumerate() + { + let classes = chunk.to_ne_bytes(); + if classes[0] == class { + return u32tosid(repr[trans_offset + i * 4]); + } + if classes[1] == class { + return u32tosid(repr[trans_offset + i * 4 + 1]); + } + if classes[2] == class { + return u32tosid(repr[trans_offset + i * 4 + 2]); + } + if classes[3] == class { + return u32tosid(repr[trans_offset + i * 4 + 3]); + } + } + } + // For an anchored search, we never follow failure transitions + // because failure transitions lead us down a path to matching + // a *proper* suffix of the path we were on. Thus, it can only + // produce matches that appear after the beginning of the search. + if anchored.is_anchored() { + return NFA::DEAD; + } + sid = u32tosid(repr[o + 1]); + } + } + + #[inline(always)] + fn is_special(&self, sid: StateID) -> bool { + sid <= self.special.max_special_id + } + + #[inline(always)] + fn is_dead(&self, sid: StateID) -> bool { + sid == NFA::DEAD + } + + #[inline(always)] + fn is_match(&self, sid: StateID) -> bool { + !self.is_dead(sid) && sid <= self.special.max_match_id + } + + #[inline(always)] + fn is_start(&self, sid: StateID) -> bool { + sid == self.special.start_unanchored_id + || sid == self.special.start_anchored_id + } + + #[inline(always)] + fn match_kind(&self) -> MatchKind { + self.match_kind + } + + #[inline(always)] + fn patterns_len(&self) -> usize { + self.pattern_lens.len() + } + + #[inline(always)] + fn pattern_len(&self, pid: PatternID) -> usize { + self.pattern_lens[pid].as_usize() + } + + #[inline(always)] + fn min_pattern_len(&self) -> usize { + self.min_pattern_len + } + + #[inline(always)] + fn max_pattern_len(&self) -> usize { + self.max_pattern_len + } + + #[inline(always)] + fn match_len(&self, sid: StateID) -> usize { + State::match_len(self.alphabet_len, &self.repr[sid.as_usize()..]) + } + + #[inline(always)] + fn match_pattern(&self, sid: StateID, index: usize) -> PatternID { + State::match_pattern( + self.alphabet_len, + &self.repr[sid.as_usize()..], + index, + ) + } + + #[inline(always)] + fn memory_usage(&self) -> usize { + use core::mem::size_of; + + (self.repr.len() * size_of::()) + + (self.pattern_lens.len() * size_of::()) + + self.prefilter.as_ref().map_or(0, |p| p.memory_usage()) + } + + #[inline(always)] + fn prefilter(&self) -> Option<&Prefilter> { + self.prefilter.as_ref() + } +} + +impl core::fmt::Debug for NFA { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + use crate::automaton::fmt_state_indicator; + + writeln!(f, "contiguous::NFA(")?; + let mut sid = NFA::DEAD; // always the first state and always present + loop { + let raw = &self.repr[sid.as_usize()..]; + if raw.is_empty() { + break; + } + let is_match = self.is_match(sid); + let state = State::read(self.alphabet_len, is_match, raw); + fmt_state_indicator(f, self, sid)?; + write!( + f, + "{:06}({:06}): ", + sid.as_usize(), + state.fail.as_usize() + )?; + state.fmt(f)?; + write!(f, "\n")?; + if self.is_match(sid) { + write!(f, " matches: ")?; + for i in 0..state.match_len { + let pid = State::match_pattern(self.alphabet_len, raw, i); + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}", pid.as_usize())?; + } + write!(f, "\n")?; + } + // The FAIL state doesn't actually have space for a state allocated + // for it, so we have to treat it as a special case. write below + // the DEAD state. + if sid == NFA::DEAD { + writeln!(f, "F {:06}:", NFA::FAIL.as_usize())?; + } + let len = State::len(self.alphabet_len, is_match, raw); + sid = StateID::new(sid.as_usize().checked_add(len).unwrap()) + .unwrap(); + } + writeln!(f, "match kind: {:?}", self.match_kind)?; + writeln!(f, "prefilter: {:?}", self.prefilter.is_some())?; + writeln!(f, "state length: {:?}", self.state_len)?; + writeln!(f, "pattern length: {:?}", self.patterns_len())?; + writeln!(f, "shortest pattern length: {:?}", self.min_pattern_len)?; + writeln!(f, "longest pattern length: {:?}", self.max_pattern_len)?; + writeln!(f, "alphabet length: {:?}", self.alphabet_len)?; + writeln!(f, "byte classes: {:?}", self.byte_classes)?; + writeln!(f, "memory usage: {:?}", self.memory_usage())?; + writeln!(f, ")")?; + + Ok(()) + } +} + +/// The "in memory" representation a single dense or sparse state. +/// +/// A `State`'s in memory representation is not ever actually materialized +/// during a search with a contiguous NFA. Doing so would be too slow. (Indeed, +/// the only time a `State` is actually constructed is in `Debug` impls.) +/// Instead, a `State` exposes a number of static methods for reading certain +/// things from the raw binary encoding of the state. +#[derive(Clone)] +struct State<'a> { + /// The state to transition to when 'class_to_next' yields a transition + /// to the FAIL state. + fail: StateID, + /// The number of pattern IDs in this state. For a non-match state, this is + /// always zero. Otherwise it is always bigger than zero. + match_len: usize, + /// The sparse or dense representation of the transitions for this state. + trans: StateTrans<'a>, +} + +/// The underlying representation of sparse or dense transitions for a state. +/// +/// Note that like `State`, we don't typically construct values of this type +/// during a search since we don't always need all values and thus would +/// represent a lot of wasteful work. +#[derive(Clone)] +enum StateTrans<'a> { + /// A sparse representation of transitions for a state, where only non-FAIL + /// transitions are explicitly represented. + Sparse { + classes: &'a [u32], + /// The transitions for this state, where each transition is packed + /// into a u32. The low 8 bits correspond to the byte class for the + /// transition, and the high 24 bits correspond to the next state ID. + /// + /// This packing is why the max state ID allowed for a contiguous + /// NFA is 2^24-1. + nexts: &'a [u32], + }, + /// A "one transition" state that is never a match state. + /// + /// These are by far the most common state, so we use a specialized and + /// very compact representation for them. + One { + /// The element of this NFA's alphabet that this transition is + /// defined for. + class: u8, + /// The state this should transition to if the current symbol is + /// equal to 'class'. + next: u32, + }, + /// A dense representation of transitions for a state, where all + /// transitions are explicitly represented, including transitions to the + /// FAIL state. + Dense { + /// A dense set of transitions to other states. The transitions may + /// point to a FAIL state, in which case, the search should try the + /// same transition lookup at 'fail'. + /// + /// Note that this is indexed by byte equivalence classes and not + /// byte values. That means 'class_to_next[byte]' is wrong and + /// 'class_to_next[classes.get(byte)]' is correct. The number of + /// transitions is always equivalent to 'classes.alphabet_len()'. + class_to_next: &'a [u32], + }, +} + +impl<'a> State<'a> { + /// The offset of where the "kind" of a state is stored. If it isn't one + /// of the sentinel values below, then it's a sparse state and the kind + /// corresponds to the number of transitions in the state. + const KIND: usize = 0; + + /// A sentinel value indicating that the state uses a dense representation. + const KIND_DENSE: u32 = 0xFF; + /// A sentinel value indicating that the state uses a special "one + /// transition" encoding. In practice, non-match states with one transition + /// make up the overwhelming majority of all states in any given + /// Aho-Corasick automaton, so we can specialize them using a very compact + /// representation. + const KIND_ONE: u32 = 0xFE; + + /// The maximum number of transitions to encode as a sparse state. Usually + /// states with a lot of transitions are either very rare, or occur near + /// the start state. In the latter case, they are probably dense already + /// anyway. In the former case, making them dense is fine because they're + /// rare. + /// + /// This needs to be small enough to permit each of the sentinel values for + /// 'KIND' above. Namely, a sparse state embeds the number of transitions + /// into the 'KIND'. Basically, "sparse" is a state kind too, but it's the + /// "else" branch. + /// + /// N.B. There isn't anything particularly magical about 127 here. I + /// just picked it because I figured any sparse state with this many + /// transitions is going to be exceptionally rare, and if it did have this + /// many transitions, then it would be quite slow to do a linear scan on + /// the transitions during a search anyway. + const MAX_SPARSE_TRANSITIONS: usize = 127; + + /// Remap state IDs in-place. + /// + /// `state` should be the the raw binary encoding of a state. (The start + /// of the slice must correspond to the start of the state, but the slice + /// may extend past the end of the encoding of the state.) + fn remap( + alphabet_len: usize, + old_to_new: &[StateID], + state: &mut [u32], + ) -> Result<(), BuildError> { + let kind = State::kind(state); + if kind == State::KIND_DENSE { + state[1] = old_to_new[state[1].as_usize()].as_u32(); + for next in state[2..][..alphabet_len].iter_mut() { + *next = old_to_new[next.as_usize()].as_u32(); + } + } else if kind == State::KIND_ONE { + state[1] = old_to_new[state[1].as_usize()].as_u32(); + state[2] = old_to_new[state[2].as_usize()].as_u32(); + } else { + let trans_len = State::sparse_trans_len(state); + let classes_len = u32_len(trans_len); + state[1] = old_to_new[state[1].as_usize()].as_u32(); + for next in state[2 + classes_len..][..trans_len].iter_mut() { + *next = old_to_new[next.as_usize()].as_u32(); + } + } + Ok(()) + } + + /// Returns the length, in number of u32s, of this state. + /// + /// This is useful for reading states consecutively, e.g., in the Debug + /// impl without needing to store a separate map from state index to state + /// identifier. + /// + /// `state` should be the the raw binary encoding of a state. (The start + /// of the slice must correspond to the start of the state, but the slice + /// may extend past the end of the encoding of the state.) + fn len(alphabet_len: usize, is_match: bool, state: &[u32]) -> usize { + let kind_len = 1; + let fail_len = 1; + let kind = State::kind(state); + let (classes_len, trans_len) = if kind == State::KIND_DENSE { + (0, alphabet_len) + } else if kind == State::KIND_ONE { + (0, 1) + } else { + let trans_len = State::sparse_trans_len(state); + let classes_len = u32_len(trans_len); + (classes_len, trans_len) + }; + let match_len = if !is_match { + 0 + } else if State::match_len(alphabet_len, state) == 1 { + // This is a special case because when there is one pattern ID for + // a match state, it is represented by a single u32 with its high + // bit set (which is impossible for a valid pattern ID). + 1 + } else { + // We add 1 to include the u32 that indicates the number of + // pattern IDs that follow. + 1 + State::match_len(alphabet_len, state) + }; + kind_len + fail_len + classes_len + trans_len + match_len + } + + /// Returns the kind of this state. + /// + /// This only includes the low byte. + #[inline(always)] + fn kind(state: &[u32]) -> u32 { + state[State::KIND] & 0xFF + } + + /// Get the number of sparse transitions in this state. This can never + /// be more than State::MAX_SPARSE_TRANSITIONS, as all states with more + /// transitions are encoded as dense states. + /// + /// `state` should be the the raw binary encoding of a sparse state. (The + /// start of the slice must correspond to the start of the state, but the + /// slice may extend past the end of the encoding of the state.) If this + /// isn't a sparse state, then the return value is unspecified. + /// + /// Do note that this is only legal to call on a sparse state. So for + /// example, "one transition" state is not a sparse state, so it would not + /// be legal to call this method on such a state. + #[inline(always)] + fn sparse_trans_len(state: &[u32]) -> usize { + (state[State::KIND] & 0xFF).as_usize() + } + + /// Returns the total number of matching pattern IDs in this state. Calling + /// this on a state that isn't a match results in unspecified behavior. + /// Thus, the returned number is never 0 for all correct calls. + /// + /// `state` should be the the raw binary encoding of a state. (The start + /// of the slice must correspond to the start of the state, but the slice + /// may extend past the end of the encoding of the state.) + #[inline(always)] + fn match_len(alphabet_len: usize, state: &[u32]) -> usize { + // We don't need to handle KIND_ONE here because it can never be a + // match state. + let packed = if State::kind(state) == State::KIND_DENSE { + let start = 2 + alphabet_len; + state[start].as_usize() + } else { + let trans_len = State::sparse_trans_len(state); + let classes_len = u32_len(trans_len); + let start = 2 + classes_len + trans_len; + state[start].as_usize() + }; + if packed & (1 << 31) == 0 { + packed + } else { + 1 + } + } + + /// Returns the pattern ID corresponding to the given index for the state + /// given. The `index` provided must be less than the number of pattern IDs + /// in this state. + /// + /// `state` should be the the raw binary encoding of a state. (The start of + /// the slice must correspond to the start of the state, but the slice may + /// extend past the end of the encoding of the state.) + /// + /// If the given state is not a match state or if the index is out of + /// bounds, then this has unspecified behavior. + #[inline(always)] + fn match_pattern( + alphabet_len: usize, + state: &[u32], + index: usize, + ) -> PatternID { + // We don't need to handle KIND_ONE here because it can never be a + // match state. + let start = if State::kind(state) == State::KIND_DENSE { + 2 + alphabet_len + } else { + let trans_len = State::sparse_trans_len(state); + let classes_len = u32_len(trans_len); + 2 + classes_len + trans_len + }; + let packed = state[start]; + let pid = if packed & (1 << 31) == 0 { + state[start + 1 + index] + } else { + assert_eq!(0, index); + packed & !(1 << 31) + }; + PatternID::from_u32_unchecked(pid) + } + + /// Read a state's binary encoding to its in-memory representation. + /// + /// `alphabet_len` should be the total number of transitions defined for + /// dense states. + /// + /// `is_match` should be true if this state is a match state and false + /// otherwise. + /// + /// `state` should be the the raw binary encoding of a state. (The start + /// of the slice must correspond to the start of the state, but the slice + /// may extend past the end of the encoding of the state.) + fn read( + alphabet_len: usize, + is_match: bool, + state: &'a [u32], + ) -> State<'a> { + let kind = State::kind(state); + let match_len = + if !is_match { 0 } else { State::match_len(alphabet_len, state) }; + let (trans, fail) = if kind == State::KIND_DENSE { + let fail = StateID::from_u32_unchecked(state[1]); + let class_to_next = &state[2..][..alphabet_len]; + (StateTrans::Dense { class_to_next }, fail) + } else if kind == State::KIND_ONE { + let fail = StateID::from_u32_unchecked(state[1]); + let class = state[State::KIND].low_u16().high_u8(); + let next = state[2]; + (StateTrans::One { class, next }, fail) + } else { + let fail = StateID::from_u32_unchecked(state[1]); + let trans_len = State::sparse_trans_len(state); + let classes_len = u32_len(trans_len); + let classes = &state[2..][..classes_len]; + let nexts = &state[2 + classes_len..][..trans_len]; + (StateTrans::Sparse { classes, nexts }, fail) + }; + State { fail, match_len, trans } + } + + /// Encode the "old" state from a noncontiguous NFA to its binary + /// representation to the given `dst` slice. `classes` should be the byte + /// classes computed for the noncontiguous NFA that the given state came + /// from. + /// + /// This returns an error if `dst` became so big that `StateID`s can no + /// longer be created for new states. Otherwise, it returns the state ID of + /// the new state created. + /// + /// When `force_dense` is true, then the encoded state will always use a + /// dense format. Otherwise, the choice between dense and sparse will be + /// automatically chosen based on the old state. + fn write( + nnfa: &noncontiguous::NFA, + oldsid: StateID, + old: &noncontiguous::State, + classes: &ByteClasses, + dst: &mut Vec, + force_dense: bool, + ) -> Result { + let sid = StateID::new(dst.len()).map_err(|e| { + BuildError::state_id_overflow(StateID::MAX.as_u64(), e.attempted()) + })?; + let old_len = nnfa.iter_trans(oldsid).count(); + // For states with a lot of transitions, we might as well just make + // them dense. These kinds of hot states tend to be very rare, so we're + // okay with it. This also gives us more sentinels in the state's + // 'kind', which lets us create different state kinds to save on + // space. + let kind = if force_dense || old_len > State::MAX_SPARSE_TRANSITIONS { + State::KIND_DENSE + } else if old_len == 1 && !old.is_match() { + State::KIND_ONE + } else { + // For a sparse state, the kind is just the number of transitions. + u32::try_from(old_len).unwrap() + }; + if kind == State::KIND_DENSE { + dst.push(kind); + dst.push(old.fail().as_u32()); + State::write_dense_trans(nnfa, oldsid, classes, dst)?; + } else if kind == State::KIND_ONE { + let t = nnfa.iter_trans(oldsid).next().unwrap(); + let class = u32::from(classes.get(t.byte())); + dst.push(kind | (class << 8)); + dst.push(old.fail().as_u32()); + dst.push(t.next().as_u32()); + } else { + dst.push(kind); + dst.push(old.fail().as_u32()); + State::write_sparse_trans(nnfa, oldsid, classes, dst)?; + } + // Now finally write the number of matches and the matches themselves. + if old.is_match() { + let matches_len = nnfa.iter_matches(oldsid).count(); + if matches_len == 1 { + let pid = nnfa.iter_matches(oldsid).next().unwrap().as_u32(); + assert_eq!(0, pid & (1 << 31)); + dst.push((1 << 31) | pid); + } else { + assert_eq!(0, matches_len & (1 << 31)); + dst.push(matches_len.as_u32()); + dst.extend(nnfa.iter_matches(oldsid).map(|pid| pid.as_u32())); + } + } + Ok(sid) + } + + /// Encode the "old" state transitions from a noncontiguous NFA to its + /// binary sparse representation to the given `dst` slice. `classes` should + /// be the byte classes computed for the noncontiguous NFA that the given + /// state came from. + /// + /// This returns an error if `dst` became so big that `StateID`s can no + /// longer be created for new states. + fn write_sparse_trans( + nnfa: &noncontiguous::NFA, + oldsid: StateID, + classes: &ByteClasses, + dst: &mut Vec, + ) -> Result<(), BuildError> { + let (mut chunk, mut len) = ([0; 4], 0); + for t in nnfa.iter_trans(oldsid) { + chunk[len] = classes.get(t.byte()); + len += 1; + if len == 4 { + dst.push(u32::from_ne_bytes(chunk)); + chunk = [0; 4]; + len = 0; + } + } + if len > 0 { + // In the case where the number of transitions isn't divisible + // by 4, the last u32 chunk will have some left over room. In + // this case, we "just" repeat the last equivalence class. By + // doing this, we know the leftover faux transitions will never + // be followed because if they were, it would have been followed + // prior to it in the last equivalence class. This saves us some + // branching in the search time state transition code. + let repeat = chunk[len - 1]; + while len < 4 { + chunk[len] = repeat; + len += 1; + } + dst.push(u32::from_ne_bytes(chunk)); + } + for t in nnfa.iter_trans(oldsid) { + dst.push(t.next().as_u32()); + } + Ok(()) + } + + /// Encode the "old" state transitions from a noncontiguous NFA to its + /// binary dense representation to the given `dst` slice. `classes` should + /// be the byte classes computed for the noncontiguous NFA that the given + /// state came from. + /// + /// This returns an error if `dst` became so big that `StateID`s can no + /// longer be created for new states. + fn write_dense_trans( + nnfa: &noncontiguous::NFA, + oldsid: StateID, + classes: &ByteClasses, + dst: &mut Vec, + ) -> Result<(), BuildError> { + // Our byte classes let us shrink the size of our dense states to the + // number of equivalence classes instead of just fixing it to 256. + // Any non-explicitly defined transition is just a transition to the + // FAIL state, so we fill that in first and then overwrite them with + // explicitly defined transitions. (Most states probably only have one + // or two explicitly defined transitions.) + // + // N.B. Remember that while building the contiguous NFA, we use state + // IDs from the noncontiguous NFA. It isn't until we've added all + // states that we go back and map noncontiguous IDs to contiguous IDs. + let start = dst.len(); + dst.extend( + core::iter::repeat(noncontiguous::NFA::FAIL.as_u32()) + .take(classes.alphabet_len()), + ); + assert!(start < dst.len(), "equivalence classes are never empty"); + for t in nnfa.iter_trans(oldsid) { + dst[start + usize::from(classes.get(t.byte()))] = + t.next().as_u32(); + } + Ok(()) + } + + /// Return an iterator over every explicitly defined transition in this + /// state. + fn transitions(&self) -> impl Iterator + '_ { + let mut i = 0; + core::iter::from_fn(move || match self.trans { + StateTrans::Sparse { classes, nexts } => { + if i >= nexts.len() { + return None; + } + let chunk = classes[i / 4]; + let class = chunk.to_ne_bytes()[i % 4]; + let next = StateID::from_u32_unchecked(nexts[i]); + i += 1; + Some((class, next)) + } + StateTrans::One { class, next } => { + if i == 0 { + i += 1; + Some((class, StateID::from_u32_unchecked(next))) + } else { + None + } + } + StateTrans::Dense { class_to_next } => { + if i >= class_to_next.len() { + return None; + } + let class = i.as_u8(); + let next = StateID::from_u32_unchecked(class_to_next[i]); + i += 1; + Some((class, next)) + } + }) + } +} + +impl<'a> core::fmt::Debug for State<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + use crate::{automaton::sparse_transitions, util::debug::DebugByte}; + + let it = sparse_transitions(self.transitions()) + // Writing out all FAIL transitions is quite noisy. Instead, we + // just require readers of the output to assume anything absent + // maps to the FAIL transition. + .filter(|&(_, _, sid)| sid != NFA::FAIL) + .enumerate(); + for (i, (start, end, sid)) in it { + if i > 0 { + write!(f, ", ")?; + } + if start == end { + write!(f, "{:?} => {:?}", DebugByte(start), sid.as_usize())?; + } else { + write!( + f, + "{:?}-{:?} => {:?}", + DebugByte(start), + DebugByte(end), + sid.as_usize() + )?; + } + } + Ok(()) + } +} + +/// A builder for configuring an Aho-Corasick contiguous NFA. +/// +/// This builder has a subset of the options available to a +/// [`AhoCorasickBuilder`](crate::AhoCorasickBuilder). Of the shared options, +/// their behavior is identical. +#[derive(Clone, Debug)] +pub struct Builder { + noncontiguous: noncontiguous::Builder, + dense_depth: usize, + byte_classes: bool, +} + +impl Default for Builder { + fn default() -> Builder { + Builder { + noncontiguous: noncontiguous::Builder::new(), + dense_depth: 2, + byte_classes: true, + } + } +} + +impl Builder { + /// Create a new builder for configuring an Aho-Corasick contiguous NFA. + pub fn new() -> Builder { + Builder::default() + } + + /// Build an Aho-Corasick contiguous NFA from the given iterator of + /// patterns. + /// + /// A builder may be reused to create more NFAs. + pub fn build(&self, patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + let nnfa = self.noncontiguous.build(patterns)?; + self.build_from_noncontiguous(&nnfa) + } + + /// Build an Aho-Corasick contiguous NFA from the given noncontiguous NFA. + /// + /// Note that when this method is used, only the `dense_depth` and + /// `byte_classes` settings on this builder are respected. The other + /// settings only apply to the initial construction of the Aho-Corasick + /// automaton. Since using this method requires that initial construction + /// has already completed, all settings impacting only initial construction + /// are no longer relevant. + pub fn build_from_noncontiguous( + &self, + nnfa: &noncontiguous::NFA, + ) -> Result { + debug!("building contiguous NFA"); + let byte_classes = if self.byte_classes { + nnfa.byte_classes().clone() + } else { + ByteClasses::singletons() + }; + let mut index_to_state_id = vec![NFA::DEAD; nnfa.states().len()]; + let mut nfa = NFA { + repr: vec![], + pattern_lens: nnfa.pattern_lens_raw().to_vec(), + state_len: nnfa.states().len(), + prefilter: nnfa.prefilter().map(|p| p.clone()), + match_kind: nnfa.match_kind(), + alphabet_len: byte_classes.alphabet_len(), + byte_classes, + min_pattern_len: nnfa.min_pattern_len(), + max_pattern_len: nnfa.max_pattern_len(), + // The special state IDs are set later. + special: Special::zero(), + }; + for (oldsid, state) in nnfa.states().iter().with_state_ids() { + // We don't actually encode a fail state since it isn't necessary. + // But we still want to make sure any FAIL ids are mapped + // correctly. + if oldsid == noncontiguous::NFA::FAIL { + index_to_state_id[oldsid] = NFA::FAIL; + continue; + } + let force_dense = state.depth().as_usize() < self.dense_depth; + let newsid = State::write( + nnfa, + oldsid, + state, + &nfa.byte_classes, + &mut nfa.repr, + force_dense, + )?; + index_to_state_id[oldsid] = newsid; + } + for &newsid in index_to_state_id.iter() { + if newsid == NFA::FAIL { + continue; + } + let state = &mut nfa.repr[newsid.as_usize()..]; + State::remap(nfa.alphabet_len, &index_to_state_id, state)?; + } + // Now that we've remapped all the IDs in our states, all that's left + // is remapping the special state IDs. + let remap = &index_to_state_id; + let old = nnfa.special(); + let new = &mut nfa.special; + new.max_special_id = remap[old.max_special_id]; + new.max_match_id = remap[old.max_match_id]; + new.start_unanchored_id = remap[old.start_unanchored_id]; + new.start_anchored_id = remap[old.start_anchored_id]; + debug!( + "contiguous NFA built, ", + nfa.state_len, + nfa.memory_usage(), + nfa.byte_classes.alphabet_len(), + ); + // The vectors can grow ~twice as big during construction because a + // Vec amortizes growth. But here, let's shrink things back down to + // what we actually need since we're never going to add more to it. + nfa.repr.shrink_to_fit(); + nfa.pattern_lens.shrink_to_fit(); + Ok(nfa) + } + + /// Set the desired match semantics. + /// + /// This only applies when using [`Builder::build`] and not + /// [`Builder::build_from_noncontiguous`]. + /// + /// See + /// [`AhoCorasickBuilder::match_kind`](crate::AhoCorasickBuilder::match_kind) + /// for more documentation and examples. + pub fn match_kind(&mut self, kind: MatchKind) -> &mut Builder { + self.noncontiguous.match_kind(kind); + self + } + + /// Enable ASCII-aware case insensitive matching. + /// + /// This only applies when using [`Builder::build`] and not + /// [`Builder::build_from_noncontiguous`]. + /// + /// See + /// [`AhoCorasickBuilder::ascii_case_insensitive`](crate::AhoCorasickBuilder::ascii_case_insensitive) + /// for more documentation and examples. + pub fn ascii_case_insensitive(&mut self, yes: bool) -> &mut Builder { + self.noncontiguous.ascii_case_insensitive(yes); + self + } + + /// Enable heuristic prefilter optimizations. + /// + /// This only applies when using [`Builder::build`] and not + /// [`Builder::build_from_noncontiguous`]. + /// + /// See + /// [`AhoCorasickBuilder::prefilter`](crate::AhoCorasickBuilder::prefilter) + /// for more documentation and examples. + pub fn prefilter(&mut self, yes: bool) -> &mut Builder { + self.noncontiguous.prefilter(yes); + self + } + + /// Set the limit on how many states use a dense representation for their + /// transitions. Other states will generally use a sparse representation. + /// + /// See + /// [`AhoCorasickBuilder::dense_depth`](crate::AhoCorasickBuilder::dense_depth) + /// for more documentation and examples. + pub fn dense_depth(&mut self, depth: usize) -> &mut Builder { + self.dense_depth = depth; + self + } + + /// A debug setting for whether to attempt to shrink the size of the + /// automaton's alphabet or not. + /// + /// This should never be enabled unless you're debugging an automaton. + /// Namely, disabling byte classes makes transitions easier to reason + /// about, since they use the actual bytes instead of equivalence classes. + /// Disabling this confers no performance benefit at search time. + /// + /// See + /// [`AhoCorasickBuilder::byte_classes`](crate::AhoCorasickBuilder::byte_classes) + /// for more documentation and examples. + pub fn byte_classes(&mut self, yes: bool) -> &mut Builder { + self.byte_classes = yes; + self + } +} + +/// Computes the number of u32 values needed to represent one byte per the +/// number of transitions given. +fn u32_len(ntrans: usize) -> usize { + if ntrans % 4 == 0 { + ntrans >> 2 + } else { + (ntrans >> 2) + 1 + } +} + +#[cfg(test)] +mod tests { + // This test demonstrates a SWAR technique I tried in the sparse transition + // code inside of 'next_state'. Namely, sparse transitions work by + // iterating over u32 chunks, with each chunk containing up to 4 classes + // corresponding to 4 transitions. This SWAR technique lets us find a + // matching transition without converting the u32 to a [u8; 4]. + // + // It turned out to be a little slower unfortunately, which isn't too + // surprising, since this is likely a throughput oriented optimization. + // Loop unrolling doesn't really help us because the vast majority of + // states have very few transitions. + // + // Anyway, this code was a little tricky to write, so I converted it to a + // test in case someone figures out how to use it more effectively than + // I could. + // + // (This also only works on little endian. So big endian would need to be + // accounted for if we ever decided to use this I think.) + #[cfg(target_endian = "little")] + #[test] + fn swar() { + use super::*; + + fn has_zero_byte(x: u32) -> u32 { + const LO_U32: u32 = 0x01010101; + const HI_U32: u32 = 0x80808080; + + x.wrapping_sub(LO_U32) & !x & HI_U32 + } + + fn broadcast(b: u8) -> u32 { + (u32::from(b)) * (u32::MAX / 255) + } + + fn index_of(x: u32) -> usize { + let o = + (((x - 1) & 0x01010101).wrapping_mul(0x01010101) >> 24) - 1; + o.as_usize() + } + + let bytes: [u8; 4] = [b'1', b'A', b'a', b'z']; + let chunk = u32::from_ne_bytes(bytes); + + let needle = broadcast(b'1'); + assert_eq!(0, index_of(has_zero_byte(needle ^ chunk))); + let needle = broadcast(b'A'); + assert_eq!(1, index_of(has_zero_byte(needle ^ chunk))); + let needle = broadcast(b'a'); + assert_eq!(2, index_of(has_zero_byte(needle ^ chunk))); + let needle = broadcast(b'z'); + assert_eq!(3, index_of(has_zero_byte(needle ^ chunk))); + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/nfa/mod.rs b/third_party/crates/aho-corasick-1.1.5/src/nfa/mod.rs new file mode 100644 index 0000000..93f4dc2 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/nfa/mod.rs @@ -0,0 +1,40 @@ +/*! +Provides direct access to NFA implementations of Aho-Corasick. + +The principle characteristic of an NFA in this crate is that it may +transition through multiple states per byte of haystack. In Aho-Corasick +parlance, NFAs follow failure transitions during a search. In contrast, +a [`DFA`](crate::dfa::DFA) pre-computes all failure transitions during +compilation at the expense of a much bigger memory footprint. + +Currently, there are two NFA implementations provided: noncontiguous and +contiguous. The names reflect their internal representation, and consequently, +the trade offs associated with them: + +* A [`noncontiguous::NFA`] uses a separate allocation for every NFA state to +represent its transitions in a sparse format. This is ideal for building an +NFA, since it cheaply permits different states to have a different number of +transitions. A noncontiguous NFA is where the main Aho-Corasick construction +algorithm is implemented. All other Aho-Corasick implementations are built by +first constructing a noncontiguous NFA. +* A [`contiguous::NFA`] is uses a single allocation to represent all states, +while still encoding most states as sparse states but permitting states near +the starting state to have a dense representation. The dense representation +uses more memory, but permits computing transitions during a search more +quickly. By only making the most active states dense (the states near the +starting state), a contiguous NFA better balances memory usage with search +speed. The single contiguous allocation also uses less overhead per state and +enables compression tricks where most states only use 8 bytes of heap memory. + +When given the choice between these two, you almost always want to pick a +contiguous NFA. It takes only a little longer to build, but both its memory +usage and search speed are typically much better than a noncontiguous NFA. A +noncontiguous NFA is useful when prioritizing build times, or when there are +so many patterns that a contiguous NFA could not be built. (Currently, because +of both memory and search speed improvements, a contiguous NFA has a smaller +internal limit on the total number of NFA states it can represent. But you +would likely need to have hundreds of thousands or even millions of patterns +before you hit this limit.) +*/ +pub mod contiguous; +pub mod noncontiguous; diff --git a/third_party/crates/aho-corasick-1.1.5/src/nfa/noncontiguous.rs b/third_party/crates/aho-corasick-1.1.5/src/nfa/noncontiguous.rs new file mode 100644 index 0000000..af32617 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/nfa/noncontiguous.rs @@ -0,0 +1,1762 @@ +/*! +Provides a noncontiguous NFA implementation of Aho-Corasick. + +This is a low-level API that generally only needs to be used in niche +circumstances. When possible, prefer using [`AhoCorasick`](crate::AhoCorasick) +instead of a noncontiguous NFA directly. Using an `NFA` directly is typically +only necessary when one needs access to the [`Automaton`] trait implementation. +*/ + +use alloc::{ + collections::{BTreeSet, VecDeque}, + vec, + vec::Vec, +}; + +use crate::{ + automaton::Automaton, + util::{ + alphabet::{ByteClassSet, ByteClasses}, + error::{BuildError, MatchError}, + prefilter::{self, opposite_ascii_case, Prefilter}, + primitives::{IteratorIndexExt, PatternID, SmallIndex, StateID}, + remapper::Remapper, + search::{Anchored, MatchKind}, + special::Special, + }, +}; + +/// A noncontiguous NFA implementation of Aho-Corasick. +/// +/// When possible, prefer using [`AhoCorasick`](crate::AhoCorasick) instead of +/// this type directly. Using an `NFA` directly is typically only necessary +/// when one needs access to the [`Automaton`] trait implementation. +/// +/// This NFA represents the "core" implementation of Aho-Corasick in this +/// crate. Namely, constructing this NFA involving building a trie and then +/// filling in the failure transitions between states, similar to what is +/// described in any standard textbook description of Aho-Corasick. +/// +/// In order to minimize heap usage and to avoid additional construction costs, +/// this implementation represents the transitions of all states as distinct +/// sparse memory allocations. This is where it gets its name from. That is, +/// this NFA has no contiguous memory allocation for its transition table. Each +/// state gets its own allocation. +/// +/// While the sparse representation keeps memory usage to somewhat reasonable +/// levels, it is still quite large and also results in somewhat mediocre +/// search performance. For this reason, it is almost always a good idea to +/// use a [`contiguous::NFA`](crate::nfa::contiguous::NFA) instead. It is +/// marginally slower to build, but has higher throughput and can sometimes use +/// an order of magnitude less memory. The main reason to use a noncontiguous +/// NFA is when you need the fastest possible construction time, or when a +/// contiguous NFA does not have the desired capacity. (The total number of NFA +/// states it can have is fewer than a noncontiguous NFA.) +/// +/// # Example +/// +/// This example shows how to build an `NFA` directly and use it to execute +/// [`Automaton::try_find`]: +/// +/// ``` +/// use aho_corasick::{ +/// automaton::Automaton, +/// nfa::noncontiguous::NFA, +/// Input, Match, +/// }; +/// +/// let patterns = &["b", "abc", "abcd"]; +/// let haystack = "abcd"; +/// +/// let nfa = NFA::new(patterns).unwrap(); +/// assert_eq!( +/// Some(Match::must(0, 1..2)), +/// nfa.try_find(&Input::new(haystack))?, +/// ); +/// # Ok::<(), Box>(()) +/// ``` +/// +/// It is also possible to implement your own version of `try_find`. See the +/// [`Automaton`] documentation for an example. +#[derive(Clone)] +pub struct NFA { + /// The match semantics built into this NFA. + match_kind: MatchKind, + /// A set of states. Each state defines its own transitions, a fail + /// transition and a set of indices corresponding to matches. + /// + /// The first state is always the fail state, which is used only as a + /// sentinel. Namely, in the final NFA, no transition into the fail state + /// exists. (Well, they do, but they aren't followed. Instead, the state's + /// failure transition is followed.) + /// + /// The second state (index 1) is always the dead state. Dead states are + /// in every automaton, but only used when leftmost-{first,longest} match + /// semantics are enabled. Specifically, they instruct search to stop + /// at specific points in order to report the correct match location. In + /// the standard Aho-Corasick construction, there are no transitions to + /// the dead state. + /// + /// The third state (index 2) is generally intended to be the starting or + /// "root" state. + states: Vec, + /// Transitions stored in a sparse representation via a linked list. + /// + /// Each transition contains three pieces of information: the byte it + /// is defined for, the state it transitions to and a link to the next + /// transition in the same state (or `StateID::ZERO` if it is the last + /// transition). + /// + /// The first transition for each state is determined by `State::sparse`. + /// + /// Note that this contains a complete set of all transitions in this NFA, + /// including states that have a dense representation for transitions. + /// (Adding dense transitions for a state doesn't remove its sparse + /// transitions, since deleting transitions from this particular sparse + /// representation would be fairly expensive.) + sparse: Vec, + /// Transitions stored in a dense representation. + /// + /// A state has a row in this table if and only if `State::dense` is + /// not equal to `StateID::ZERO`. When not zero, there are precisely + /// `NFA::byte_classes::alphabet_len()` entries beginning at `State::dense` + /// in this table. + /// + /// Generally a very small minority of states have a dense representation + /// since it uses so much memory. + dense: Vec, + /// Matches stored in linked list for each state. + /// + /// Like sparse transitions, each match has a link to the next match in the + /// state. + /// + /// The first match for each state is determined by `State::matches`. + matches: Vec, + /// The length, in bytes, of each pattern in this NFA. This slice is + /// indexed by `PatternID`. + /// + /// The number of entries in this vector corresponds to the total number of + /// patterns in this automaton. + pattern_lens: Vec, + /// A prefilter for quickly skipping to candidate matches, if pertinent. + prefilter: Option, + /// A set of equivalence classes in terms of bytes. We compute this while + /// building the NFA, but don't use it in the NFA's states. Instead, we + /// use this for building the DFA. We store it on the NFA since it's easy + /// to compute while visiting the patterns. + byte_classes: ByteClasses, + /// The length, in bytes, of the shortest pattern in this automaton. This + /// information is useful for detecting whether an automaton matches the + /// empty string or not. + min_pattern_len: usize, + /// The length, in bytes, of the longest pattern in this automaton. This + /// information is useful for keeping correct buffer sizes when searching + /// on streams. + max_pattern_len: usize, + /// The information required to deduce which states are "special" in this + /// NFA. + /// + /// Since the DEAD and FAIL states are always the first two states and + /// there are only ever two start states (which follow all of the match + /// states), it follows that we can determine whether a state is a fail, + /// dead, match or start with just a few comparisons on the ID itself: + /// + /// is_dead(sid): sid == NFA::DEAD + /// is_fail(sid): sid == NFA::FAIL + /// is_match(sid): NFA::FAIL < sid && sid <= max_match_id + /// is_start(sid): sid == start_unanchored_id || sid == start_anchored_id + /// + /// Note that this only applies to the NFA after it has been constructed. + /// During construction, the start states are the first ones added and the + /// match states are inter-leaved with non-match states. Once all of the + /// states have been added, the states are shuffled such that the above + /// predicates hold. + special: Special, +} + +impl NFA { + /// Create a new Aho-Corasick noncontiguous NFA using the default + /// configuration. + /// + /// Use a [`Builder`] if you want to change the configuration. + pub fn new(patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + NFA::builder().build(patterns) + } + + /// A convenience method for returning a new Aho-Corasick noncontiguous NFA + /// builder. + /// + /// This usually permits one to just import the `NFA` type. + pub fn builder() -> Builder { + Builder::new() + } +} + +impl NFA { + /// The DEAD state is a sentinel state like the FAIL state. The DEAD state + /// instructs any search to stop and return any currently recorded match, + /// or no match otherwise. Generally speaking, it is impossible for an + /// unanchored standard search to enter a DEAD state. But an anchored + /// search can, and so to can a leftmost search. + /// + /// We put DEAD before FAIL so that DEAD is always 0. We repeat this + /// decision across the other Aho-Corasicm automata, so that DEAD + /// states there are always 0 too. It's not that we need all of the + /// implementations to agree, but rather, the contiguous NFA and the DFA + /// use a sort of "premultiplied" state identifier where the only state + /// whose ID is always known and constant is the first state. Subsequent + /// state IDs depend on how much space has already been used in the + /// transition table. + pub(crate) const DEAD: StateID = StateID::new_unchecked(0); + /// The FAIL state mostly just corresponds to the ID of any transition on a + /// state that isn't explicitly defined. When one transitions into the FAIL + /// state, one must follow the previous state's failure transition before + /// doing the next state lookup. In this way, FAIL is more of a sentinel + /// than a state that one actually transitions into. In particular, it is + /// never exposed in the `Automaton` interface. + pub(crate) const FAIL: StateID = StateID::new_unchecked(1); + + /// Returns the equivalence classes of bytes found while constructing + /// this NFA. + /// + /// Note that the NFA doesn't actually make use of these equivalence + /// classes. Instead, these are useful for building the DFA when desired. + pub(crate) fn byte_classes(&self) -> &ByteClasses { + &self.byte_classes + } + + /// Returns a slice containing the length of each pattern in this searcher. + /// It is indexed by `PatternID` and has length `NFA::patterns_len`. + /// + /// This is exposed for convenience when building a contiguous NFA. But it + /// can be reconstructed from the `Automaton` API if necessary. + pub(crate) fn pattern_lens_raw(&self) -> &[SmallIndex] { + &self.pattern_lens + } + + /// Returns a slice of all states in this non-contiguous NFA. + pub(crate) fn states(&self) -> &[State] { + &self.states + } + + /// Returns the underlying "special" state information for this NFA. + pub(crate) fn special(&self) -> &Special { + &self.special + } + + /// Swaps the states at `id1` and `id2`. + /// + /// This does not update the transitions of any state to account for the + /// state swap. + pub(crate) fn swap_states(&mut self, id1: StateID, id2: StateID) { + self.states.swap(id1.as_usize(), id2.as_usize()); + } + + /// Re-maps all state IDs in this NFA according to the `map` function + /// given. + pub(crate) fn remap(&mut self, map: impl Fn(StateID) -> StateID) { + let alphabet_len = self.byte_classes.alphabet_len(); + for state in self.states.iter_mut() { + state.fail = map(state.fail); + let mut link = state.sparse; + while link != StateID::ZERO { + let t = &mut self.sparse[link]; + t.next = map(t.next); + link = t.link; + } + if state.dense != StateID::ZERO { + let start = state.dense.as_usize(); + for next in self.dense[start..][..alphabet_len].iter_mut() { + *next = map(*next); + } + } + } + } + + /// Iterate over all of the transitions for the given state ID. + pub(crate) fn iter_trans( + &self, + sid: StateID, + ) -> impl Iterator + '_ { + let mut link = self.states[sid].sparse; + core::iter::from_fn(move || { + if link == StateID::ZERO { + return None; + } + let t = self.sparse[link]; + link = t.link; + Some(t) + }) + } + + /// Iterate over all of the matches for the given state ID. + pub(crate) fn iter_matches( + &self, + sid: StateID, + ) -> impl Iterator + '_ { + let mut link = self.states[sid].matches; + core::iter::from_fn(move || { + if link == StateID::ZERO { + return None; + } + let m = self.matches[link]; + link = m.link; + Some(m.pid) + }) + } + + /// Return the link following the one given. If the one given is the last + /// link for the given state, then return `None`. + /// + /// If no previous link is given, then this returns the first link in the + /// state, if one exists. + /// + /// This is useful for manually iterating over the transitions in a single + /// state without borrowing the NFA. This permits mutating other parts of + /// the NFA during iteration. Namely, one can access the transition pointed + /// to by the link via `self.sparse[link]`. + fn next_link( + &self, + sid: StateID, + prev: Option, + ) -> Option { + let link = + prev.map_or(self.states[sid].sparse, |p| self.sparse[p].link); + if link == StateID::ZERO { + None + } else { + Some(link) + } + } + + /// Follow the transition for the given byte in the given state. If no such + /// transition exists, then the FAIL state ID is returned. + #[inline(always)] + fn follow_transition(&self, sid: StateID, byte: u8) -> StateID { + let s = &self.states[sid]; + // This is a special case that targets starting states and states + // near a start state. Namely, after the initial trie is constructed, + // we look for states close to the start state to convert to a dense + // representation for their transitions. This winds up using a lot more + // memory per state in exchange for faster transition lookups. But + // since we only do this for a small number of states (by default), the + // memory usage is usually minimal. + // + // This has *massive* benefit when executing searches because the + // unanchored starting state is by far the hottest state and is + // frequently visited. Moreover, the 'for' loop below that works + // decently on an actually sparse state is disastrous on a state that + // is nearly or completely dense. + if s.dense == StateID::ZERO { + self.follow_transition_sparse(sid, byte) + } else { + let class = usize::from(self.byte_classes.get(byte)); + self.dense[s.dense.as_usize() + class] + } + } + + /// Like `follow_transition`, but always uses the sparse representation. + #[inline(always)] + fn follow_transition_sparse(&self, sid: StateID, byte: u8) -> StateID { + for t in self.iter_trans(sid) { + if byte <= t.byte { + if byte == t.byte { + return t.next; + } + break; + } + } + NFA::FAIL + } + + /// Set the transition for the given byte to the state ID given. + /// + /// Note that one should not set transitions to the FAIL state. It is not + /// technically incorrect, but it wastes space. If a transition is not + /// defined, then it is automatically assumed to lead to the FAIL state. + fn add_transition( + &mut self, + prev: StateID, + byte: u8, + next: StateID, + ) -> Result<(), BuildError> { + if self.states[prev].dense != StateID::ZERO { + let dense = self.states[prev].dense; + let class = usize::from(self.byte_classes.get(byte)); + self.dense[dense.as_usize() + class] = next; + } + + let head = self.states[prev].sparse; + if head == StateID::ZERO || byte < self.sparse[head].byte { + let new_link = self.alloc_transition()?; + self.sparse[new_link] = Transition { byte, next, link: head }; + self.states[prev].sparse = new_link; + return Ok(()); + } else if byte == self.sparse[head].byte { + self.sparse[head].next = next; + return Ok(()); + } + + // We handled the only cases where the beginning of the transition + // chain needs to change. At this point, we now know that there is + // at least one entry in the transition chain and the byte for that + // transition is less than the byte for the transition we're adding. + let (mut link_prev, mut link_next) = (head, self.sparse[head].link); + while link_next != StateID::ZERO && byte > self.sparse[link_next].byte + { + link_prev = link_next; + link_next = self.sparse[link_next].link; + } + if link_next == StateID::ZERO || byte < self.sparse[link_next].byte { + let link = self.alloc_transition()?; + self.sparse[link] = Transition { byte, next, link: link_next }; + self.sparse[link_prev].link = link; + } else { + assert_eq!(byte, self.sparse[link_next].byte); + self.sparse[link_next].next = next; + } + Ok(()) + } + + /// This sets every possible transition (all 255 of them) for the given + /// state to the name `next` value. + /// + /// This is useful for efficiently initializing start/dead states. + /// + /// # Panics + /// + /// This requires that the state has no transitions added to it already. + /// If it has any transitions, then this panics. It will also panic if + /// the state has been densified prior to calling this. + fn init_full_state( + &mut self, + prev: StateID, + next: StateID, + ) -> Result<(), BuildError> { + assert_eq!( + StateID::ZERO, + self.states[prev].dense, + "state must not be dense yet" + ); + assert_eq!( + StateID::ZERO, + self.states[prev].sparse, + "state must have zero transitions" + ); + let mut prev_link = StateID::ZERO; + for byte in 0..=255 { + let new_link = self.alloc_transition()?; + self.sparse[new_link] = + Transition { byte, next, link: StateID::ZERO }; + if prev_link == StateID::ZERO { + self.states[prev].sparse = new_link; + } else { + self.sparse[prev_link].link = new_link; + } + prev_link = new_link; + } + Ok(()) + } + + /// Add a match for the given pattern ID to the state for the given ID. + fn add_match( + &mut self, + sid: StateID, + pid: PatternID, + ) -> Result<(), BuildError> { + let head = self.states[sid].matches; + let mut link = head; + while self.matches[link].link != StateID::ZERO { + link = self.matches[link].link; + } + let new_match_link = self.alloc_match()?; + self.matches[new_match_link].pid = pid; + if link == StateID::ZERO { + self.states[sid].matches = new_match_link; + } else { + self.matches[link].link = new_match_link; + } + Ok(()) + } + + /// Copy matches from the `src` state to the `dst` state. This is useful + /// when a match state can be reached via a failure transition. In which + /// case, you'll want to copy the matches (if any) from the state reached + /// by the failure transition to the original state you were at. + fn copy_matches( + &mut self, + src: StateID, + dst: StateID, + ) -> Result<(), BuildError> { + let head_dst = self.states[dst].matches; + let mut link_dst = head_dst; + while self.matches[link_dst].link != StateID::ZERO { + link_dst = self.matches[link_dst].link; + } + let mut link_src = self.states[src].matches; + while link_src != StateID::ZERO { + let new_match_link = + StateID::new(self.matches.len()).map_err(|e| { + BuildError::state_id_overflow( + StateID::MAX.as_u64(), + e.attempted(), + ) + })?; + self.matches.push(Match { + pid: self.matches[link_src].pid, + link: StateID::ZERO, + }); + if link_dst == StateID::ZERO { + self.states[dst].matches = new_match_link; + } else { + self.matches[link_dst].link = new_match_link; + } + + link_dst = new_match_link; + link_src = self.matches[link_src].link; + } + Ok(()) + } + + /// Create a new entry in `NFA::trans`, if there's room, and return that + /// entry's ID. If there's no room, then an error is returned. + fn alloc_transition(&mut self) -> Result { + let id = StateID::new(self.sparse.len()).map_err(|e| { + BuildError::state_id_overflow(StateID::MAX.as_u64(), e.attempted()) + })?; + self.sparse.push(Transition::default()); + Ok(id) + } + + /// Create a new entry in `NFA::matches`, if there's room, and return that + /// entry's ID. If there's no room, then an error is returned. + fn alloc_match(&mut self) -> Result { + let id = StateID::new(self.matches.len()).map_err(|e| { + BuildError::state_id_overflow(StateID::MAX.as_u64(), e.attempted()) + })?; + self.matches.push(Match::default()); + Ok(id) + } + + /// Create a new set of `N` transitions in this NFA's dense transition + /// table. The ID return corresponds to the index at which the `N` + /// transitions begin. So `id+0` is the first transition and `id+(N-1)` is + /// the last. + /// + /// `N` is determined via `NFA::byte_classes::alphabet_len`. + fn alloc_dense_state(&mut self) -> Result { + let id = StateID::new(self.dense.len()).map_err(|e| { + BuildError::state_id_overflow(StateID::MAX.as_u64(), e.attempted()) + })?; + // We use FAIL because it's the correct default. If a state doesn't + // have a transition defined for every possible byte value, then the + // transition function should return NFA::FAIL. + self.dense.extend( + core::iter::repeat(NFA::FAIL) + .take(self.byte_classes.alphabet_len()), + ); + Ok(id) + } + + /// Allocate and add a fresh state to the underlying NFA and return its + /// ID (guaranteed to be one more than the ID of the previously allocated + /// state). If the ID would overflow `StateID`, then this returns an error. + fn alloc_state(&mut self, depth: usize) -> Result { + // This is OK because we error when building the trie if we see a + // pattern whose length cannot fit into a 'SmallIndex', and the longest + // possible depth corresponds to the length of the longest pattern. + let depth = SmallIndex::new(depth) + .expect("patterns longer than SmallIndex::MAX are not allowed"); + let id = StateID::new(self.states.len()).map_err(|e| { + BuildError::state_id_overflow(StateID::MAX.as_u64(), e.attempted()) + })?; + self.states.push(State { + sparse: StateID::ZERO, + dense: StateID::ZERO, + matches: StateID::ZERO, + fail: self.special.start_unanchored_id, + depth, + }); + Ok(id) + } +} + +// SAFETY: 'start_state' always returns a valid state ID, 'next_state' always +// returns a valid state ID given a valid state ID. We otherwise claim that +// all other methods are correct as well. +unsafe impl Automaton for NFA { + #[inline(always)] + fn start_state(&self, anchored: Anchored) -> Result { + match anchored { + Anchored::No => Ok(self.special.start_unanchored_id), + Anchored::Yes => Ok(self.special.start_anchored_id), + } + } + + #[inline(always)] + fn next_state( + &self, + anchored: Anchored, + mut sid: StateID, + byte: u8, + ) -> StateID { + // This terminates since: + // + // 1. state.fail never points to the FAIL state. + // 2. All state.fail values point to a state closer to the start state. + // 3. The start state has no transitions to the FAIL state. + loop { + let next = self.follow_transition(sid, byte); + if next != NFA::FAIL { + return next; + } + // For an anchored search, we never follow failure transitions + // because failure transitions lead us down a path to matching + // a *proper* suffix of the path we were on. Thus, it can only + // produce matches that appear after the beginning of the search. + if anchored.is_anchored() { + return NFA::DEAD; + } + sid = self.states[sid].fail(); + } + } + + #[inline(always)] + fn is_special(&self, sid: StateID) -> bool { + sid <= self.special.max_special_id + } + + #[inline(always)] + fn is_dead(&self, sid: StateID) -> bool { + sid == NFA::DEAD + } + + #[inline(always)] + fn is_match(&self, sid: StateID) -> bool { + // N.B. This returns true when sid==NFA::FAIL but that's okay because + // NFA::FAIL is not actually a valid state ID from the perspective of + // the Automaton trait. Namely, it is never returned by 'start_state' + // or by 'next_state'. So we don't need to care about it here. + !self.is_dead(sid) && sid <= self.special.max_match_id + } + + #[inline(always)] + fn is_start(&self, sid: StateID) -> bool { + sid == self.special.start_unanchored_id + || sid == self.special.start_anchored_id + } + + #[inline(always)] + fn match_kind(&self) -> MatchKind { + self.match_kind + } + + #[inline(always)] + fn patterns_len(&self) -> usize { + self.pattern_lens.len() + } + + #[inline(always)] + fn pattern_len(&self, pid: PatternID) -> usize { + self.pattern_lens[pid].as_usize() + } + + #[inline(always)] + fn min_pattern_len(&self) -> usize { + self.min_pattern_len + } + + #[inline(always)] + fn max_pattern_len(&self) -> usize { + self.max_pattern_len + } + + #[inline(always)] + fn match_len(&self, sid: StateID) -> usize { + self.iter_matches(sid).count() + } + + #[inline(always)] + fn match_pattern(&self, sid: StateID, index: usize) -> PatternID { + self.iter_matches(sid).nth(index).unwrap() + } + + #[inline(always)] + fn memory_usage(&self) -> usize { + self.states.len() * core::mem::size_of::() + + self.sparse.len() * core::mem::size_of::() + + self.matches.len() * core::mem::size_of::() + + self.dense.len() * StateID::SIZE + + self.pattern_lens.len() * SmallIndex::SIZE + + self.prefilter.as_ref().map_or(0, |p| p.memory_usage()) + } + + #[inline(always)] + fn prefilter(&self) -> Option<&Prefilter> { + self.prefilter.as_ref() + } +} + +/// A representation of a sparse NFA state for an Aho-Corasick automaton. +/// +/// It contains the transitions to the next state, a failure transition for +/// cases where there exists no other transition for the current input byte +/// and the matches implied by visiting this state (if any). +#[derive(Clone, Debug)] +pub(crate) struct State { + /// A pointer to `NFA::trans` corresponding to the head of a linked list + /// containing all of the transitions for this state. + /// + /// This is `StateID::ZERO` if and only if this state has zero transitions. + sparse: StateID, + /// A pointer to a row of `N` transitions in `NFA::dense`. These + /// transitions correspond precisely to what is obtained by traversing + /// `sparse`, but permits constant time lookup. + /// + /// When this is zero (which is true for most states in the default + /// configuration), then this state has no dense representation. + /// + /// Note that `N` is equal to `NFA::byte_classes::alphabet_len()`. This is + /// typically much less than 256 (the maximum value). + dense: StateID, + /// A pointer to `NFA::matches` corresponding to the head of a linked list + /// containing all of the matches for this state. + /// + /// This is `StateID::ZERO` if and only if this state is not a match state. + matches: StateID, + /// The state that should be transitioned to if the current byte in the + /// haystack does not have a corresponding transition defined in this + /// state. + fail: StateID, + /// The depth of this state. Specifically, this is the distance from this + /// state to the starting state. (For the special sentinel states DEAD and + /// FAIL, their depth is always 0.) The depth of a starting state is 0. + /// + /// Note that depth is currently not used in this non-contiguous NFA. It + /// may in the future, but it is used in the contiguous NFA. Namely, it + /// permits an optimization where states near the starting state have their + /// transitions stored in a dense fashion, but all other states have their + /// transitions stored in a sparse fashion. (This non-contiguous NFA uses + /// a sparse representation for all states unconditionally.) In any case, + /// this is really the only convenient place to compute and store this + /// information, which we need when building the contiguous NFA. + depth: SmallIndex, +} + +impl State { + /// Return true if and only if this state is a match state. + pub(crate) fn is_match(&self) -> bool { + self.matches != StateID::ZERO + } + + /// Returns the failure transition for this state. + pub(crate) fn fail(&self) -> StateID { + self.fail + } + + /// Returns the depth of this state. That is, the number of transitions + /// this state is from the start state of the NFA. + pub(crate) fn depth(&self) -> SmallIndex { + self.depth + } +} + +/// A single transition in a non-contiguous NFA. +#[derive(Clone, Copy, Default)] +#[repr(packed)] +pub(crate) struct Transition { + byte: u8, + next: StateID, + link: StateID, +} + +impl Transition { + /// Return the byte for which this transition is defined. + pub(crate) fn byte(&self) -> u8 { + self.byte + } + + /// Return the ID of the state that this transition points to. + pub(crate) fn next(&self) -> StateID { + self.next + } + + /// Return the ID of the next transition. + fn link(&self) -> StateID { + self.link + } +} + +impl core::fmt::Debug for Transition { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "Transition(byte: {:X?}, next: {:?}, link: {:?})", + self.byte, + self.next().as_usize(), + self.link().as_usize() + ) + } +} + +/// A single match in a non-contiguous NFA. +#[derive(Clone, Copy, Default)] +struct Match { + pid: PatternID, + link: StateID, +} + +impl Match { + /// Return the pattern ID for this match. + pub(crate) fn pattern(&self) -> PatternID { + self.pid + } + + /// Return the ID of the next match. + fn link(&self) -> StateID { + self.link + } +} + +impl core::fmt::Debug for Match { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "Match(pid: {:?}, link: {:?})", + self.pattern().as_usize(), + self.link().as_usize() + ) + } +} + +/// A builder for configuring an Aho-Corasick noncontiguous NFA. +/// +/// This builder has a subset of the options available to a +/// [`AhoCorasickBuilder`](crate::AhoCorasickBuilder). Of the shared options, +/// their behavior is identical. +#[derive(Clone, Debug)] +pub struct Builder { + match_kind: MatchKind, + prefilter: bool, + ascii_case_insensitive: bool, + dense_depth: usize, +} + +impl Default for Builder { + fn default() -> Builder { + Builder { + match_kind: MatchKind::default(), + prefilter: true, + ascii_case_insensitive: false, + dense_depth: 3, + } + } +} + +impl Builder { + /// Create a new builder for configuring an Aho-Corasick noncontiguous NFA. + pub fn new() -> Builder { + Builder::default() + } + + /// Build an Aho-Corasick noncontiguous NFA from the given iterator of + /// patterns. + /// + /// A builder may be reused to create more NFAs. + pub fn build(&self, patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + debug!("building non-contiguous NFA"); + let nfa = Compiler::new(self)?.compile(patterns)?; + debug!( + "non-contiguous NFA built, ", + nfa.states.len(), + nfa.memory_usage() + ); + Ok(nfa) + } + + /// Set the desired match semantics. + /// + /// See + /// [`AhoCorasickBuilder::match_kind`](crate::AhoCorasickBuilder::match_kind) + /// for more documentation and examples. + pub fn match_kind(&mut self, kind: MatchKind) -> &mut Builder { + self.match_kind = kind; + self + } + + /// Enable ASCII-aware case insensitive matching. + /// + /// See + /// [`AhoCorasickBuilder::ascii_case_insensitive`](crate::AhoCorasickBuilder::ascii_case_insensitive) + /// for more documentation and examples. + pub fn ascii_case_insensitive(&mut self, yes: bool) -> &mut Builder { + self.ascii_case_insensitive = yes; + self + } + + /// Set the limit on how many states use a dense representation for their + /// transitions. Other states will generally use a sparse representation. + /// + /// See + /// [`AhoCorasickBuilder::dense_depth`](crate::AhoCorasickBuilder::dense_depth) + /// for more documentation and examples. + pub fn dense_depth(&mut self, depth: usize) -> &mut Builder { + self.dense_depth = depth; + self + } + + /// Enable heuristic prefilter optimizations. + /// + /// See + /// [`AhoCorasickBuilder::prefilter`](crate::AhoCorasickBuilder::prefilter) + /// for more documentation and examples. + pub fn prefilter(&mut self, yes: bool) -> &mut Builder { + self.prefilter = yes; + self + } +} + +/// A compiler uses a builder configuration and builds up the NFA formulation +/// of an Aho-Corasick automaton. This roughly corresponds to the standard +/// formulation described in textbooks, with some tweaks to support leftmost +/// searching. +#[derive(Debug)] +struct Compiler<'a> { + builder: &'a Builder, + prefilter: prefilter::Builder, + nfa: NFA, + byteset: ByteClassSet, +} + +impl<'a> Compiler<'a> { + fn new(builder: &'a Builder) -> Result, BuildError> { + let prefilter = prefilter::Builder::new(builder.match_kind) + .ascii_case_insensitive(builder.ascii_case_insensitive); + Ok(Compiler { + builder, + prefilter, + nfa: NFA { + match_kind: builder.match_kind, + states: vec![], + sparse: vec![], + dense: vec![], + matches: vec![], + pattern_lens: vec![], + prefilter: None, + byte_classes: ByteClasses::singletons(), + min_pattern_len: usize::MAX, + max_pattern_len: 0, + special: Special::zero(), + }, + byteset: ByteClassSet::empty(), + }) + } + + fn compile(mut self, patterns: I) -> Result + where + I: IntoIterator, + P: AsRef<[u8]>, + { + // Add dummy transition/match links, so that no valid link will point + // to another link at index 0. + self.nfa.sparse.push(Transition::default()); + self.nfa.matches.push(Match::default()); + // Add a dummy dense transition so that no states can have dense==0 + // represent a valid pointer to dense transitions. This permits + // dense==0 to be a sentinel indicating "no dense transitions." + self.nfa.dense.push(NFA::DEAD); + // the dead state, only used for leftmost and fixed to id==0 + self.nfa.alloc_state(0)?; + // the fail state, which is never entered and fixed to id==1 + self.nfa.alloc_state(0)?; + // unanchored start state, initially fixed to id==2 but later shuffled + // to appear after all non-start match states. + self.nfa.special.start_unanchored_id = self.nfa.alloc_state(0)?; + // anchored start state, initially fixed to id==3 but later shuffled + // to appear after unanchored start state. + self.nfa.special.start_anchored_id = self.nfa.alloc_state(0)?; + // Initialize the unanchored starting state in order to make it dense, + // and thus make transition lookups on this state faster. + self.init_unanchored_start_state()?; + // Set all transitions on the DEAD state to point to itself. This way, + // the DEAD state can never be escaped. It MUST be used as a sentinel + // in any correct search. + self.add_dead_state_loop()?; + // Build the base trie from the given patterns. + self.build_trie(patterns)?; + self.nfa.states.shrink_to_fit(); + // Turn our set of bytes into equivalent classes. This NFA + // implementation uses byte classes only for states that use a dense + // representation of transitions. (And that's why this comes before + // `self.densify()`, as the byte classes need to be set first.) + self.nfa.byte_classes = self.byteset.byte_classes(); + // Add transitions (and maybe matches) to the anchored starting state. + // The anchored starting state is used for anchored searches. The only + // mechanical difference between it and the unanchored start state is + // that missing transitions map to the DEAD state instead of the FAIL + // state. + self.set_anchored_start_state()?; + // Rewrite transitions to the FAIL state on the unanchored start state + // as self-transitions. This keeps the start state active at all times. + self.add_unanchored_start_state_loop(); + // Make some (possibly zero) states use a dense representation for + // transitions. It's important to do this right after the states + // and non-failure transitions are solidified. That way, subsequent + // accesses (particularly `fill_failure_transitions`) will benefit from + // the faster transition lookup in densified states. + self.densify()?; + // The meat of the Aho-Corasick algorithm: compute and write failure + // transitions. i.e., the state to move to when a transition isn't + // defined in the current state. These are epsilon transitions and thus + // make this formulation an NFA. + self.fill_failure_transitions()?; + // Handle a special case under leftmost semantics when at least one + // of the patterns is the empty string. + self.close_start_state_loop_for_leftmost(); + // Shuffle states so that we have DEAD, FAIL, MATCH, ..., START, START, + // NON-MATCH, ... This permits us to very quickly query the type of + // the state we're currently in during a search. + self.shuffle(); + self.nfa.prefilter = self.prefilter.build(); + // Store the maximum ID of all *relevant* special states. Start states + // are only relevant when we have a prefilter, otherwise, there is zero + // reason to care about whether a state is a start state or not during + // a search. Indeed, without a prefilter, we are careful to explicitly + // NOT care about start states, otherwise the search can ping pong + // between the unrolled loop and the handling of special-status states + // and destroy perf. + self.nfa.special.max_special_id = if self.nfa.prefilter.is_some() { + // Why the anchored starting state? Because we always put it + // after the unanchored starting state and it is therefore the + // maximum. Why put unanchored followed by anchored? No particular + // reason, but that's how the states are logically organized in the + // Thompson NFA implementation found in regex-automata. ¯\_(ツ)_/¯ + self.nfa.special.start_anchored_id + } else { + self.nfa.special.max_match_id + }; + self.nfa.sparse.shrink_to_fit(); + self.nfa.dense.shrink_to_fit(); + self.nfa.matches.shrink_to_fit(); + self.nfa.pattern_lens.shrink_to_fit(); + Ok(self.nfa) + } + + /// This sets up the initial prefix trie that makes up the Aho-Corasick + /// automaton. Effectively, it creates the basic structure of the + /// automaton, where every pattern given has a path from the start state to + /// the end of the pattern. + fn build_trie(&mut self, patterns: I) -> Result<(), BuildError> + where + I: IntoIterator, + P: AsRef<[u8]>, + { + 'PATTERNS: for (i, pat) in patterns.into_iter().enumerate() { + let pid = PatternID::new(i).map_err(|e| { + BuildError::pattern_id_overflow( + PatternID::MAX.as_u64(), + e.attempted(), + ) + })?; + let pat = pat.as_ref(); + let patlen = SmallIndex::new(pat.len()) + .map_err(|_| BuildError::pattern_too_long(pid, pat.len()))?; + self.nfa.min_pattern_len = + core::cmp::min(self.nfa.min_pattern_len, pat.len()); + self.nfa.max_pattern_len = + core::cmp::max(self.nfa.max_pattern_len, pat.len()); + assert_eq!( + i, + self.nfa.pattern_lens.len(), + "expected number of patterns to match pattern ID" + ); + self.nfa.pattern_lens.push(patlen); + // We add the pattern to the prefilter here because the pattern + // ID in the prefilter is determined with respect to the patterns + // added to the prefilter. That is, it isn't the ID we have here, + // but the one determined by its own accounting of patterns. + // To ensure they line up, we add every pattern we see to the + // prefilter, even if some patterns ultimately are impossible to + // match (in leftmost-first semantics specifically). + // + // Another way of doing this would be to expose an API in the + // prefilter to permit setting your own pattern IDs. Or to just use + // our own map and go between them. But this case is sufficiently + // rare that we don't bother and just make sure they're in sync. + if self.builder.prefilter { + self.prefilter.add(pat); + } + + let mut prev = self.nfa.special.start_unanchored_id; + let mut saw_match = false; + for (depth, &b) in pat.iter().enumerate() { + // When leftmost-first match semantics are requested, we + // specifically stop adding patterns when a previously added + // pattern is a prefix of it. We avoid adding it because + // leftmost-first semantics imply that the pattern can never + // match. This is not just an optimization to save space! It + // is necessary for correctness. In fact, this is the only + // difference in the automaton between the implementations for + // leftmost-first and leftmost-longest. + saw_match = saw_match || self.nfa.states[prev].is_match(); + if self.builder.match_kind.is_leftmost_first() && saw_match { + // Skip to the next pattern immediately. This avoids + // incorrectly adding a match after this loop terminates. + continue 'PATTERNS; + } + + // Add this byte to our equivalence classes. These don't + // get used while building the trie, but other Aho-Corasick + // implementations may use them. + self.byteset.set_range(b, b); + if self.builder.ascii_case_insensitive { + let b = opposite_ascii_case(b); + self.byteset.set_range(b, b); + } + + // If the transition from prev using the current byte already + // exists, then just move through it. Otherwise, add a new + // state. We track the depth here so that we can determine + // how to represent transitions. States near the start state + // use a dense representation that uses more memory but is + // faster. Other states use a sparse representation that uses + // less memory but is slower. + let next = self.nfa.follow_transition(prev, b); + if next != NFA::FAIL { + prev = next; + } else { + let next = self.nfa.alloc_state(depth)?; + self.nfa.add_transition(prev, b, next)?; + if self.builder.ascii_case_insensitive { + let b = opposite_ascii_case(b); + self.nfa.add_transition(prev, b, next)?; + } + prev = next; + } + } + // Once the pattern has been added, log the match in the final + // state that it reached. + self.nfa.add_match(prev, pid)?; + } + Ok(()) + } + + /// This routine creates failure transitions according to the standard + /// textbook formulation of the Aho-Corasick algorithm, with a couple small + /// tweaks to support "leftmost" semantics. + /// + /// Building failure transitions is the most interesting part of building + /// the Aho-Corasick automaton, because they are what allow searches to + /// be performed in linear time. Specifically, a failure transition is + /// a single transition associated with each state that points back to + /// the longest proper suffix of the pattern being searched. The failure + /// transition is followed whenever there exists no transition on the + /// current state for the current input byte. If there is no other proper + /// suffix, then the failure transition points back to the starting state. + /// + /// For example, let's say we built an Aho-Corasick automaton with the + /// following patterns: 'abcd' and 'cef'. The trie looks like this: + /// + /// ```ignore + /// a - S1 - b - S2 - c - S3 - d - S4* + /// / + /// S0 - c - S5 - e - S6 - f - S7* + /// ``` + /// + /// At this point, it should be fairly straight-forward to see how this + /// trie can be used in a simplistic way. At any given position in the + /// text we're searching (called the "subject" string), all we need to do + /// is follow the transitions in the trie by consuming one transition for + /// each byte in the subject string. If we reach a match state, then we can + /// report that location as a match. + /// + /// The trick comes when searching a subject string like 'abcef'. We'll + /// initially follow the transition from S0 to S1 and wind up in S3 after + /// observng the 'c' byte. At this point, the next byte is 'e' but state + /// S3 has no transition for 'e', so the search fails. We then would need + /// to restart the search at the next position in 'abcef', which + /// corresponds to 'b'. The match would fail, but the next search starting + /// at 'c' would finally succeed. The problem with this approach is that + /// we wind up searching the subject string potentially many times. In + /// effect, this makes the algorithm have worst case `O(n * m)` complexity, + /// where `n ~ len(subject)` and `m ~ len(all patterns)`. We would instead + /// like to achieve a `O(n + m)` worst case complexity. + /// + /// This is where failure transitions come in. Instead of dying at S3 in + /// the first search, the automaton can instruct the search to move to + /// another part of the automaton that corresponds to a suffix of what + /// we've seen so far. Recall that we've seen 'abc' in the subject string, + /// and the automaton does indeed have a non-empty suffix, 'c', that could + /// potentially lead to another match. Thus, the actual Aho-Corasick + /// automaton for our patterns in this case looks like this: + /// + /// ```ignore + /// a - S1 - b - S2 - c - S3 - d - S4* + /// / / + /// / ---------------- + /// / / + /// S0 - c - S5 - e - S6 - f - S7* + /// ``` + /// + /// That is, we have a failure transition from S3 to S5, which is followed + /// exactly in cases when we are in state S3 but see any byte other than + /// 'd' (that is, we've "failed" to find a match in this portion of our + /// trie). We know we can transition back to S5 because we've already seen + /// a 'c' byte, so we don't need to re-scan it. We can then pick back up + /// with the search starting at S5 and complete our match. + /// + /// Adding failure transitions to a trie is fairly simple, but subtle. The + /// key issue is that you might have multiple failure transition that you + /// need to follow. For example, look at the trie for the patterns + /// 'abcd', 'b', 'bcd' and 'cd': + /// + /// ```ignore + /// - a - S1 - b - S2* - c - S3 - d - S4* + /// / / / + /// / ------- ------- + /// / / / + /// S0 --- b - S5* - c - S6 - d - S7* + /// \ / + /// \ -------- + /// \ / + /// - c - S8 - d - S9* + /// ``` + /// + /// The failure transitions for this trie are defined from S2 to S5, + /// S3 to S6 and S6 to S8. Moreover, state S2 needs to track that it + /// corresponds to a match, since its failure transition to S5 is itself + /// a match state. + /// + /// Perhaps simplest way to think about adding these failure transitions + /// is recursively. That is, if you know the failure transitions for every + /// possible previous state that could be visited (e.g., when computing the + /// failure transition for S3, you already know the failure transitions + /// for S0, S1 and S2), then you can simply follow the failure transition + /// of the previous state and check whether the incoming transition is + /// defined after following the failure transition. + /// + /// For example, when determining the failure state for S3, by our + /// assumptions, we already know that there is a failure transition from + /// S2 (the previous state) to S5. So we follow that transition and check + /// whether the transition connecting S2 to S3 is defined. Indeed, it is, + /// as there is a transition from S5 to S6 for the byte 'c'. If no such + /// transition existed, we could keep following the failure transitions + /// until we reach the start state, which is the failure transition for + /// every state that has no corresponding proper suffix. + /// + /// We don't actually use recursion to implement this, but instead, use a + /// breadth first search of the automaton. Our base case is the start + /// state, whose failure transition is just a transition to itself. + /// + /// When building a leftmost automaton, we proceed as above, but only + /// include a subset of failure transitions. Namely, we omit any failure + /// transitions that appear after a match state in the trie. This is + /// because failure transitions always point back to a proper suffix of + /// what has been seen so far. Thus, following a failure transition after + /// a match implies looking for a match that starts after the one that has + /// already been seen, which is of course therefore not the leftmost match. + /// + /// N.B. I came up with this algorithm on my own, and after scouring all of + /// the other AC implementations I know of (Perl, Snort, many on GitHub). + /// I couldn't find any that implement leftmost semantics like this. + /// Perl of course needs leftmost-first semantics, but they implement it + /// with a seeming hack at *search* time instead of encoding it into the + /// automaton. There are also a couple Java libraries that support leftmost + /// longest semantics, but they do it by building a queue of matches at + /// search time, which is even worse than what Perl is doing. ---AG + fn fill_failure_transitions(&mut self) -> Result<(), BuildError> { + let is_leftmost = self.builder.match_kind.is_leftmost(); + let start_uid = self.nfa.special.start_unanchored_id; + // Initialize the queue for breadth first search with all transitions + // out of the start state. We handle the start state specially because + // we only want to follow non-self transitions. If we followed self + // transitions, then this would never terminate. + let mut queue = VecDeque::new(); + let mut seen = self.queued_set(); + let mut prev_link = None; + while let Some(link) = self.nfa.next_link(start_uid, prev_link) { + prev_link = Some(link); + let t = self.nfa.sparse[link]; + + // Skip anything we've seen before and any self-transitions on the + // start state. + if start_uid == t.next() || seen.contains(t.next) { + continue; + } + queue.push_back(t.next); + seen.insert(t.next); + // Under leftmost semantics, if a state immediately following + // the start state is a match state, then we never want to + // follow its failure transition since the failure transition + // necessarily leads back to the start state, which we never + // want to do for leftmost matching after a match has been + // found. + // + // We apply the same logic to non-start states below as well. + if is_leftmost && self.nfa.states[t.next].is_match() { + self.nfa.states[t.next].fail = NFA::DEAD; + } + } + while let Some(id) = queue.pop_front() { + let mut prev_link = None; + while let Some(link) = self.nfa.next_link(id, prev_link) { + prev_link = Some(link); + let t = self.nfa.sparse[link]; + + if seen.contains(t.next) { + // The only way to visit a duplicate state in a transition + // list is when ASCII case insensitivity is enabled. In + // this case, we want to skip it since it's redundant work. + // But it would also end up duplicating matches, which + // results in reporting duplicate matches in some cases. + // See the 'acasei010' regression test. + continue; + } + queue.push_back(t.next); + seen.insert(t.next); + + // As above for start states, under leftmost semantics, once + // we see a match all subsequent states should have no failure + // transitions because failure transitions always imply looking + // for a match that is a suffix of what has been seen so far + // (where "seen so far" corresponds to the string formed by + // following the transitions from the start state to the + // current state). Under leftmost semantics, we specifically do + // not want to allow this to happen because we always want to + // report the match found at the leftmost position. + // + // The difference between leftmost-first and leftmost-longest + // occurs previously while we build the trie. For + // leftmost-first, we simply omit any entries that would + // otherwise require passing through a match state. + // + // Note that for correctness, the failure transition has to be + // set to the dead state for ALL states following a match, not + // just the match state itself. However, by setting the failure + // transition to the dead state on all match states, the dead + // state will automatically propagate to all subsequent states + // via the failure state computation below. + if is_leftmost && self.nfa.states[t.next].is_match() { + self.nfa.states[t.next].fail = NFA::DEAD; + continue; + } + let mut fail = self.nfa.states[id].fail; + while self.nfa.follow_transition(fail, t.byte) == NFA::FAIL { + fail = self.nfa.states[fail].fail; + } + fail = self.nfa.follow_transition(fail, t.byte); + self.nfa.states[t.next].fail = fail; + self.nfa.copy_matches(fail, t.next)?; + } + // If the start state is a match state, then this automaton can + // match the empty string. This implies all states are match states + // since every position matches the empty string, so copy the + // matches from the start state to every state. Strictly speaking, + // this is only necessary for overlapping matches since each + // non-empty non-start match state needs to report empty matches + // in addition to its own. For the non-overlapping case, such + // states only report the first match, which is never empty since + // it isn't a start state. + if !is_leftmost { + self.nfa + .copy_matches(self.nfa.special.start_unanchored_id, id)?; + } + } + Ok(()) + } + + /// Shuffle the states so that they appear in this sequence: + /// + /// DEAD, FAIL, MATCH..., START, START, NON-MATCH... + /// + /// The idea here is that if we know how special states are laid out in our + /// transition table, then we can determine what "kind" of state we're in + /// just by comparing our current state ID with a particular value. In this + /// way, we avoid doing extra memory lookups. + /// + /// Before shuffling begins, our states look something like this: + /// + /// DEAD, FAIL, START, START, (MATCH | NON-MATCH)... + /// + /// So all we need to do is move all of the MATCH states so that they + /// all appear before any NON-MATCH state, like so: + /// + /// DEAD, FAIL, START, START, MATCH... NON-MATCH... + /// + /// Then it's just a simple matter of swapping the two START states with + /// the last two MATCH states. + /// + /// (This is the same technique used for fully compiled DFAs in + /// regex-automata.) + fn shuffle(&mut self) { + let old_start_uid = self.nfa.special.start_unanchored_id; + let old_start_aid = self.nfa.special.start_anchored_id; + assert!(old_start_uid < old_start_aid); + assert_eq!( + 3, + old_start_aid.as_usize(), + "anchored start state should be at index 3" + ); + // We implement shuffling by a sequence of pairwise swaps of states. + // Since we have a number of things referencing states via their + // IDs and swapping them changes their IDs, we need to record every + // swap we make so that we can remap IDs. The remapper handles this + // book-keeping for us. + let mut remapper = Remapper::new(&self.nfa, 0); + // The way we proceed here is by moving all match states so that + // they directly follow the start states. So it will go: DEAD, FAIL, + // START-UNANCHORED, START-ANCHORED, MATCH, ..., NON-MATCH, ... + // + // To do that, we proceed forward through all states after + // START-ANCHORED and swap match states so that they appear before all + // non-match states. + let mut next_avail = StateID::from(4u8); + for i in next_avail.as_usize()..self.nfa.states.len() { + let sid = StateID::new(i).unwrap(); + if !self.nfa.states[sid].is_match() { + continue; + } + remapper.swap(&mut self.nfa, sid, next_avail); + // The key invariant here is that only non-match states exist + // between 'next_avail' and 'sid' (with them being potentially + // equivalent). Thus, incrementing 'next_avail' by 1 is guaranteed + // to land on the leftmost non-match state. (Unless 'next_avail' + // and 'sid' are equivalent, in which case, a swap will occur but + // it is a no-op.) + next_avail = StateID::new(next_avail.one_more()).unwrap(); + } + // Now we'd like to move the start states to immediately following the + // match states. (The start states may themselves be match states, but + // we'll handle that later.) We arrange the states this way so that we + // don't necessarily need to check whether a state is a start state or + // not before checking whether a state is a match state. For example, + // we'd like to be able to write this as our state machine loop: + // + // sid = start() + // for byte in haystack: + // sid = next(sid, byte) + // if sid <= nfa.max_start_id: + // if sid <= nfa.max_dead_id: + // # search complete + // elif sid <= nfa.max_match_id: + // # found match + // + // The important context here is that we might not want to look for + // start states at all. Namely, if a searcher doesn't have a prefilter, + // then there is no reason to care about whether we're in a start state + // or not. And indeed, if we did check for it, this very hot loop would + // ping pong between the special state handling and the main state + // transition logic. This in turn stalls the CPU by killing branch + // prediction. + // + // So essentially, we really want to be able to "forget" that start + // states even exist and this is why we put them at the end. + let new_start_aid = + StateID::new(next_avail.as_usize().checked_sub(1).unwrap()) + .unwrap(); + remapper.swap(&mut self.nfa, old_start_aid, new_start_aid); + let new_start_uid = + StateID::new(next_avail.as_usize().checked_sub(2).unwrap()) + .unwrap(); + remapper.swap(&mut self.nfa, old_start_uid, new_start_uid); + let new_max_match_id = + StateID::new(next_avail.as_usize().checked_sub(3).unwrap()) + .unwrap(); + self.nfa.special.max_match_id = new_max_match_id; + self.nfa.special.start_unanchored_id = new_start_uid; + self.nfa.special.start_anchored_id = new_start_aid; + // If one start state is a match state, then they both are. + if self.nfa.states[self.nfa.special.start_anchored_id].is_match() { + self.nfa.special.max_match_id = self.nfa.special.start_anchored_id; + } + remapper.remap(&mut self.nfa); + } + + /// Attempts to convert the transition representation of a subset of states + /// in this NFA from sparse to dense. This can greatly improve search + /// performance since states with a higher number of transitions tend to + /// correlate with very active states. + /// + /// We generally only densify states that are close to the start state. + /// These tend to be the most active states and thus benefit from a dense + /// representation more than other states. + /// + /// This tends to best balance between memory usage and performance. In + /// particular, the *vast majority* of all states in a typical Aho-Corasick + /// automaton have only 1 transition and are usually farther from the start + /// state and thus don't get densified. + /// + /// Note that this doesn't remove the sparse representation of transitions + /// for states that are densified. It could be done, but actually removing + /// entries from `NFA::sparse` is likely more expensive than it's worth. + fn densify(&mut self) -> Result<(), BuildError> { + for i in 0..self.nfa.states.len() { + let sid = StateID::new(i).unwrap(); + // Don't bother densifying states that are only used as sentinels. + if sid == NFA::DEAD || sid == NFA::FAIL { + continue; + } + // Only densify states that are "close enough" to the start state. + if self.nfa.states[sid].depth.as_usize() + >= self.builder.dense_depth + { + continue; + } + let dense = self.nfa.alloc_dense_state()?; + let mut prev_link = None; + while let Some(link) = self.nfa.next_link(sid, prev_link) { + prev_link = Some(link); + let t = self.nfa.sparse[link]; + + let class = usize::from(self.nfa.byte_classes.get(t.byte)); + let index = dense.as_usize() + class; + self.nfa.dense[index] = t.next; + } + self.nfa.states[sid].dense = dense; + } + Ok(()) + } + + /// Returns a set that tracked queued states. + /// + /// This is only necessary when ASCII case insensitivity is enabled, since + /// it is the only way to visit the same state twice. Otherwise, this + /// returns an inert set that nevers adds anything and always reports + /// `false` for every member test. + fn queued_set(&self) -> QueuedSet { + if self.builder.ascii_case_insensitive { + QueuedSet::active() + } else { + QueuedSet::inert() + } + } + + /// Initializes the unanchored start state by making it dense. This is + /// achieved by explicitly setting every transition to the FAIL state. + /// This isn't necessary for correctness, since any missing transition is + /// automatically assumed to be mapped to the FAIL state. We do this to + /// make the unanchored starting state dense, and thus in turn make + /// transition lookups on it faster. (Which is worth doing because it's + /// the most active state.) + fn init_unanchored_start_state(&mut self) -> Result<(), BuildError> { + let start_uid = self.nfa.special.start_unanchored_id; + let start_aid = self.nfa.special.start_anchored_id; + self.nfa.init_full_state(start_uid, NFA::FAIL)?; + self.nfa.init_full_state(start_aid, NFA::FAIL)?; + Ok(()) + } + + /// Setup the anchored start state by copying all of the transitions and + /// matches from the unanchored starting state with one change: the failure + /// transition is changed to the DEAD state, so that for any undefined + /// transitions, the search will stop. + fn set_anchored_start_state(&mut self) -> Result<(), BuildError> { + let start_uid = self.nfa.special.start_unanchored_id; + let start_aid = self.nfa.special.start_anchored_id; + let (mut uprev_link, mut aprev_link) = (None, None); + loop { + let unext = self.nfa.next_link(start_uid, uprev_link); + let anext = self.nfa.next_link(start_aid, aprev_link); + let (ulink, alink) = match (unext, anext) { + (Some(ulink), Some(alink)) => (ulink, alink), + (None, None) => break, + _ => unreachable!(), + }; + uprev_link = Some(ulink); + aprev_link = Some(alink); + self.nfa.sparse[alink].next = self.nfa.sparse[ulink].next; + } + self.nfa.copy_matches(start_uid, start_aid)?; + // This is the main difference between the unanchored and anchored + // starting states. If a lookup on an anchored starting state fails, + // then the search should stop. + // + // N.B. This assumes that the loop on the unanchored starting state + // hasn't been created yet. + self.nfa.states[start_aid].fail = NFA::DEAD; + Ok(()) + } + + /// Set the failure transitions on the start state to loop back to the + /// start state. This effectively permits the Aho-Corasick automaton to + /// match at any position. This is also required for finding the next + /// state to terminate, namely, finding the next state should never return + /// a fail_id. + /// + /// This must be done after building the initial trie, since trie + /// construction depends on transitions to `fail_id` to determine whether a + /// state already exists or not. + fn add_unanchored_start_state_loop(&mut self) { + let start_uid = self.nfa.special.start_unanchored_id; + let mut prev_link = None; + while let Some(link) = self.nfa.next_link(start_uid, prev_link) { + prev_link = Some(link); + if self.nfa.sparse[link].next() == NFA::FAIL { + self.nfa.sparse[link].next = start_uid; + } + } + } + + /// Remove the start state loop by rewriting any transitions on the start + /// state back to the start state with transitions to the dead state. + /// + /// The loop is only closed when two conditions are met: the start state + /// is a match state and the match kind is leftmost-first or + /// leftmost-longest. + /// + /// The reason for this is that under leftmost semantics, a start state + /// that is also a match implies that we should never restart the search + /// process. We allow normal transitions out of the start state, but if + /// none exist, we transition to the dead state, which signals that + /// searching should stop. + fn close_start_state_loop_for_leftmost(&mut self) { + let start_uid = self.nfa.special.start_unanchored_id; + let start = &mut self.nfa.states[start_uid]; + let dense = start.dense; + if self.builder.match_kind.is_leftmost() && start.is_match() { + let mut prev_link = None; + while let Some(link) = self.nfa.next_link(start_uid, prev_link) { + prev_link = Some(link); + if self.nfa.sparse[link].next() == start_uid { + self.nfa.sparse[link].next = NFA::DEAD; + if dense != StateID::ZERO { + let b = self.nfa.sparse[link].byte; + let class = usize::from(self.nfa.byte_classes.get(b)); + self.nfa.dense[dense.as_usize() + class] = NFA::DEAD; + } + } + } + } + } + + /// Sets all transitions on the dead state to point back to the dead state. + /// Normally, missing transitions map back to the failure state, but the + /// point of the dead state is to act as a sink that can never be escaped. + fn add_dead_state_loop(&mut self) -> Result<(), BuildError> { + self.nfa.init_full_state(NFA::DEAD, NFA::DEAD)?; + Ok(()) + } +} + +/// A set of state identifiers used to avoid revisiting the same state multiple +/// times when filling in failure transitions. +/// +/// This set has an "inert" and an "active" mode. When inert, the set never +/// stores anything and always returns `false` for every member test. This is +/// useful to avoid the performance and memory overhead of maintaining this +/// set when it is not needed. +#[derive(Debug)] +struct QueuedSet { + set: Option>, +} + +impl QueuedSet { + /// Return an inert set that returns `false` for every state ID membership + /// test. + fn inert() -> QueuedSet { + QueuedSet { set: None } + } + + /// Return an active set that tracks state ID membership. + fn active() -> QueuedSet { + QueuedSet { set: Some(BTreeSet::new()) } + } + + /// Inserts the given state ID into this set. (If the set is inert, then + /// this is a no-op.) + fn insert(&mut self, state_id: StateID) { + if let Some(ref mut set) = self.set { + set.insert(state_id); + } + } + + /// Returns true if and only if the given state ID is in this set. If the + /// set is inert, this always returns false. + fn contains(&self, state_id: StateID) -> bool { + match self.set { + None => false, + Some(ref set) => set.contains(&state_id), + } + } +} + +impl core::fmt::Debug for NFA { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + use crate::{ + automaton::{fmt_state_indicator, sparse_transitions}, + util::debug::DebugByte, + }; + + writeln!(f, "noncontiguous::NFA(")?; + for (sid, state) in self.states.iter().with_state_ids() { + // The FAIL state doesn't actually have space for a state allocated + // for it, so we have to treat it as a special case. + if sid == NFA::FAIL { + writeln!(f, "F {:06}:", sid.as_usize())?; + continue; + } + fmt_state_indicator(f, self, sid)?; + write!( + f, + "{:06}({:06}): ", + sid.as_usize(), + state.fail.as_usize() + )?; + + let it = sparse_transitions( + self.iter_trans(sid).map(|t| (t.byte, t.next)), + ) + .enumerate(); + for (i, (start, end, sid)) in it { + if i > 0 { + write!(f, ", ")?; + } + if start == end { + write!( + f, + "{:?} => {:?}", + DebugByte(start), + sid.as_usize() + )?; + } else { + write!( + f, + "{:?}-{:?} => {:?}", + DebugByte(start), + DebugByte(end), + sid.as_usize() + )?; + } + } + + write!(f, "\n")?; + if self.is_match(sid) { + write!(f, " matches: ")?; + for (i, pid) in self.iter_matches(sid).enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}", pid.as_usize())?; + } + write!(f, "\n")?; + } + } + writeln!(f, "match kind: {:?}", self.match_kind)?; + writeln!(f, "prefilter: {:?}", self.prefilter.is_some())?; + writeln!(f, "state length: {:?}", self.states.len())?; + writeln!(f, "pattern length: {:?}", self.patterns_len())?; + writeln!(f, "shortest pattern length: {:?}", self.min_pattern_len)?; + writeln!(f, "longest pattern length: {:?}", self.max_pattern_len)?; + writeln!(f, "memory usage: {:?}", self.memory_usage())?; + writeln!(f, ")")?; + Ok(()) + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/api.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/api.rs new file mode 100644 index 0000000..6bea99e --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/api.rs @@ -0,0 +1,691 @@ +use alloc::sync::Arc; + +use crate::{ + packed::{pattern::Patterns, rabinkarp::RabinKarp, teddy}, + util::search::{Match, Span}, +}; + +/// This is a limit placed on the total number of patterns we're willing to try +/// and match at once. As more sophisticated algorithms are added, this number +/// may be increased. +const PATTERN_LIMIT: usize = 128; + +/// A knob for controlling the match semantics of a packed multiple string +/// searcher. +/// +/// This differs from the [`MatchKind`](crate::MatchKind) type in the top-level +/// crate module in that it doesn't support "standard" match semantics, +/// and instead only supports leftmost-first or leftmost-longest. Namely, +/// "standard" semantics cannot be easily supported by packed searchers. +/// +/// For more information on the distinction between leftmost-first and +/// leftmost-longest, see the docs on the top-level `MatchKind` type. +/// +/// Unlike the top-level `MatchKind` type, the default match semantics for this +/// type are leftmost-first. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum MatchKind { + /// Use leftmost-first match semantics, which reports leftmost matches. + /// When there are multiple possible leftmost matches, the match + /// corresponding to the pattern that appeared earlier when constructing + /// the automaton is reported. + /// + /// This is the default. + LeftmostFirst, + /// Use leftmost-longest match semantics, which reports leftmost matches. + /// When there are multiple possible leftmost matches, the longest match + /// is chosen. + LeftmostLongest, +} + +impl Default for MatchKind { + fn default() -> MatchKind { + MatchKind::LeftmostFirst + } +} + +/// The configuration for a packed multiple pattern searcher. +/// +/// The configuration is currently limited only to being able to select the +/// match semantics (leftmost-first or leftmost-longest) of a searcher. In the +/// future, more knobs may be made available. +/// +/// A configuration produces a [`packed::Builder`](Builder), which in turn can +/// be used to construct a [`packed::Searcher`](Searcher) for searching. +/// +/// # Example +/// +/// This example shows how to use leftmost-longest semantics instead of the +/// default (leftmost-first). +/// +/// ``` +/// use aho_corasick::{packed::{Config, MatchKind}, PatternID}; +/// +/// # fn example() -> Option<()> { +/// let searcher = Config::new() +/// .match_kind(MatchKind::LeftmostLongest) +/// .builder() +/// .add("foo") +/// .add("foobar") +/// .build()?; +/// let matches: Vec = searcher +/// .find_iter("foobar") +/// .map(|mat| mat.pattern()) +/// .collect(); +/// assert_eq!(vec![PatternID::must(1)], matches); +/// # Some(()) } +/// # if cfg!(all(feature = "std", any( +/// # target_arch = "x86_64", target_arch = "aarch64", +/// # ))) { +/// # example().unwrap() +/// # } else { +/// # assert!(example().is_none()); +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct Config { + kind: MatchKind, + force: Option, + only_teddy_fat: Option, + only_teddy_256bit: Option, + heuristic_pattern_limits: bool, +} + +/// An internal option for forcing the use of a particular packed algorithm. +/// +/// When an algorithm is forced, if a searcher could not be constructed for it, +/// then no searcher will be returned even if an alternative algorithm would +/// work. +#[derive(Clone, Debug)] +enum ForceAlgorithm { + Teddy, + RabinKarp, +} + +impl Default for Config { + fn default() -> Config { + Config::new() + } +} + +impl Config { + /// Create a new default configuration. A default configuration uses + /// leftmost-first match semantics. + pub fn new() -> Config { + Config { + kind: MatchKind::LeftmostFirst, + force: None, + only_teddy_fat: None, + only_teddy_256bit: None, + heuristic_pattern_limits: true, + } + } + + /// Create a packed builder from this configuration. The builder can be + /// used to accumulate patterns and create a [`Searcher`] from them. + pub fn builder(&self) -> Builder { + Builder::from_config(self.clone()) + } + + /// Set the match semantics for this configuration. + pub fn match_kind(&mut self, kind: MatchKind) -> &mut Config { + self.kind = kind; + self + } + + /// An undocumented method for forcing the use of the Teddy algorithm. + /// + /// This is only exposed for more precise testing and benchmarks. Callers + /// should not use it as it is not part of the API stability guarantees of + /// this crate. + #[doc(hidden)] + pub fn only_teddy(&mut self, yes: bool) -> &mut Config { + if yes { + self.force = Some(ForceAlgorithm::Teddy); + } else { + self.force = None; + } + self + } + + /// An undocumented method for forcing the use of the Fat Teddy algorithm. + /// + /// This is only exposed for more precise testing and benchmarks. Callers + /// should not use it as it is not part of the API stability guarantees of + /// this crate. + #[doc(hidden)] + pub fn only_teddy_fat(&mut self, yes: Option) -> &mut Config { + self.only_teddy_fat = yes; + self + } + + /// An undocumented method for forcing the use of SSE (`Some(false)`) or + /// AVX (`Some(true)`) algorithms. + /// + /// This is only exposed for more precise testing and benchmarks. Callers + /// should not use it as it is not part of the API stability guarantees of + /// this crate. + #[doc(hidden)] + pub fn only_teddy_256bit(&mut self, yes: Option) -> &mut Config { + self.only_teddy_256bit = yes; + self + } + + /// An undocumented method for forcing the use of the Rabin-Karp algorithm. + /// + /// This is only exposed for more precise testing and benchmarks. Callers + /// should not use it as it is not part of the API stability guarantees of + /// this crate. + #[doc(hidden)] + pub fn only_rabin_karp(&mut self, yes: bool) -> &mut Config { + if yes { + self.force = Some(ForceAlgorithm::RabinKarp); + } else { + self.force = None; + } + self + } + + /// Request that heuristic limitations on the number of patterns be + /// employed. This useful to disable for benchmarking where one wants to + /// explore how Teddy performs on large number of patterns even if the + /// heuristics would otherwise refuse construction. + /// + /// This is enabled by default. + pub fn heuristic_pattern_limits(&mut self, yes: bool) -> &mut Config { + self.heuristic_pattern_limits = yes; + self + } +} + +/// A builder for constructing a packed searcher from a collection of patterns. +/// +/// # Example +/// +/// This example shows how to use a builder to construct a searcher. By +/// default, leftmost-first match semantics are used. +/// +/// ``` +/// use aho_corasick::{packed::{Builder, MatchKind}, PatternID}; +/// +/// # fn example() -> Option<()> { +/// let searcher = Builder::new() +/// .add("foobar") +/// .add("foo") +/// .build()?; +/// let matches: Vec = searcher +/// .find_iter("foobar") +/// .map(|mat| mat.pattern()) +/// .collect(); +/// assert_eq!(vec![PatternID::ZERO], matches); +/// # Some(()) } +/// # if cfg!(all(feature = "std", any( +/// # target_arch = "x86_64", target_arch = "aarch64", +/// # ))) { +/// # example().unwrap() +/// # } else { +/// # assert!(example().is_none()); +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct Builder { + /// The configuration of this builder and subsequent matcher. + config: Config, + /// Set to true if the builder detects that a matcher cannot be built. + inert: bool, + /// The patterns provided by the caller. + patterns: Patterns, +} + +impl Builder { + /// Create a new builder for constructing a multi-pattern searcher. This + /// constructor uses the default configuration. + pub fn new() -> Builder { + Builder::from_config(Config::new()) + } + + fn from_config(config: Config) -> Builder { + Builder { config, inert: false, patterns: Patterns::new() } + } + + /// Build a searcher from the patterns added to this builder so far. + pub fn build(&self) -> Option { + if self.inert || self.patterns.is_empty() { + return None; + } + let mut patterns = self.patterns.clone(); + patterns.set_match_kind(self.config.kind); + let patterns = Arc::new(patterns); + let rabinkarp = RabinKarp::new(&patterns); + // Effectively, we only want to return a searcher if we can use Teddy, + // since Teddy is our only fast packed searcher at the moment. + // Rabin-Karp is only used when searching haystacks smaller than what + // Teddy can support. Thus, the only way to get a Rabin-Karp searcher + // is to force it using undocumented APIs (for tests/benchmarks). + let (search_kind, minimum_len) = match self.config.force { + None | Some(ForceAlgorithm::Teddy) => { + debug!("trying to build Teddy packed matcher"); + let teddy = match self.build_teddy(Arc::clone(&patterns)) { + None => return None, + Some(teddy) => teddy, + }; + let minimum_len = teddy.minimum_len(); + (SearchKind::Teddy(teddy), minimum_len) + } + Some(ForceAlgorithm::RabinKarp) => { + debug!("using Rabin-Karp packed matcher"); + (SearchKind::RabinKarp, 0) + } + }; + Some(Searcher { patterns, rabinkarp, search_kind, minimum_len }) + } + + fn build_teddy(&self, patterns: Arc) -> Option { + teddy::Builder::new() + .only_256bit(self.config.only_teddy_256bit) + .only_fat(self.config.only_teddy_fat) + .heuristic_pattern_limits(self.config.heuristic_pattern_limits) + .build(patterns) + } + + /// Add the given pattern to this set to match. + /// + /// The order in which patterns are added is significant. Namely, when + /// using leftmost-first match semantics, then when multiple patterns can + /// match at a particular location, the pattern that was added first is + /// used as the match. + /// + /// If the number of patterns added exceeds the amount supported by packed + /// searchers, then the builder will stop accumulating patterns and render + /// itself inert. At this point, constructing a searcher will always return + /// `None`. + pub fn add>(&mut self, pattern: P) -> &mut Builder { + if self.inert { + return self; + } else if self.patterns.len() >= PATTERN_LIMIT { + self.inert = true; + self.patterns.reset(); + return self; + } + // Just in case PATTERN_LIMIT increases beyond u16::MAX. + assert!(self.patterns.len() <= core::u16::MAX as usize); + + let pattern = pattern.as_ref(); + if pattern.is_empty() { + self.inert = true; + self.patterns.reset(); + return self; + } + self.patterns.add(pattern); + self + } + + /// Add the given iterator of patterns to this set to match. + /// + /// The iterator must yield elements that can be converted into a `&[u8]`. + /// + /// The order in which patterns are added is significant. Namely, when + /// using leftmost-first match semantics, then when multiple patterns can + /// match at a particular location, the pattern that was added first is + /// used as the match. + /// + /// If the number of patterns added exceeds the amount supported by packed + /// searchers, then the builder will stop accumulating patterns and render + /// itself inert. At this point, constructing a searcher will always return + /// `None`. + pub fn extend(&mut self, patterns: I) -> &mut Builder + where + I: IntoIterator, + P: AsRef<[u8]>, + { + for p in patterns { + self.add(p); + } + self + } + + /// Returns the number of patterns added to this builder. + pub fn len(&self) -> usize { + self.patterns.len() + } + + /// Returns the length, in bytes, of the shortest pattern added. + pub fn minimum_len(&self) -> usize { + self.patterns.minimum_len() + } +} + +impl Default for Builder { + fn default() -> Builder { + Builder::new() + } +} + +/// A packed searcher for quickly finding occurrences of multiple patterns. +/// +/// If callers need more flexible construction, or if one wants to change the +/// match semantics (either leftmost-first or leftmost-longest), then one can +/// use the [`Config`] and/or [`Builder`] types for more fine grained control. +/// +/// # Example +/// +/// This example shows how to create a searcher from an iterator of patterns. +/// By default, leftmost-first match semantics are used. +/// +/// ``` +/// use aho_corasick::{packed::{MatchKind, Searcher}, PatternID}; +/// +/// # fn example() -> Option<()> { +/// let searcher = Searcher::new(["foobar", "foo"].iter().cloned())?; +/// let matches: Vec = searcher +/// .find_iter("foobar") +/// .map(|mat| mat.pattern()) +/// .collect(); +/// assert_eq!(vec![PatternID::ZERO], matches); +/// # Some(()) } +/// # if cfg!(all(feature = "std", any( +/// # target_arch = "x86_64", target_arch = "aarch64", +/// # ))) { +/// # example().unwrap() +/// # } else { +/// # assert!(example().is_none()); +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct Searcher { + patterns: Arc, + rabinkarp: RabinKarp, + search_kind: SearchKind, + minimum_len: usize, +} + +#[derive(Clone, Debug)] +enum SearchKind { + Teddy(teddy::Searcher), + RabinKarp, +} + +impl Searcher { + /// A convenience function for constructing a searcher from an iterator + /// of things that can be converted to a `&[u8]`. + /// + /// If a searcher could not be constructed (either because of an + /// unsupported CPU or because there are too many patterns), then `None` + /// is returned. + /// + /// # Example + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{packed::{MatchKind, Searcher}, PatternID}; + /// + /// # fn example() -> Option<()> { + /// let searcher = Searcher::new(["foobar", "foo"].iter().cloned())?; + /// let matches: Vec = searcher + /// .find_iter("foobar") + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![PatternID::ZERO], matches); + /// # Some(()) } + /// # if cfg!(all(feature = "std", any( + /// # target_arch = "x86_64", target_arch = "aarch64", + /// # ))) { + /// # example().unwrap() + /// # } else { + /// # assert!(example().is_none()); + /// # } + /// ``` + pub fn new(patterns: I) -> Option + where + I: IntoIterator, + P: AsRef<[u8]>, + { + Builder::new().extend(patterns).build() + } + + /// A convenience function for calling `Config::new()`. + /// + /// This is useful for avoiding an additional import. + pub fn config() -> Config { + Config::new() + } + + /// A convenience function for calling `Builder::new()`. + /// + /// This is useful for avoiding an additional import. + pub fn builder() -> Builder { + Builder::new() + } + + /// Return the first occurrence of any of the patterns in this searcher, + /// according to its match semantics, in the given haystack. The `Match` + /// returned will include the identifier of the pattern that matched, which + /// corresponds to the index of the pattern (starting from `0`) in which it + /// was added. + /// + /// # Example + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{packed::{MatchKind, Searcher}, PatternID}; + /// + /// # fn example() -> Option<()> { + /// let searcher = Searcher::new(["foobar", "foo"].iter().cloned())?; + /// let mat = searcher.find("foobar")?; + /// assert_eq!(PatternID::ZERO, mat.pattern()); + /// assert_eq!(0, mat.start()); + /// assert_eq!(6, mat.end()); + /// # Some(()) } + /// # if cfg!(all(feature = "std", any( + /// # target_arch = "x86_64", target_arch = "aarch64", + /// # ))) { + /// # example().unwrap() + /// # } else { + /// # assert!(example().is_none()); + /// # } + /// ``` + #[inline] + pub fn find>(&self, haystack: B) -> Option { + let haystack = haystack.as_ref(); + self.find_in(haystack, Span::from(0..haystack.len())) + } + + /// Return the first occurrence of any of the patterns in this searcher, + /// according to its match semantics, in the given haystack starting from + /// the given position. + /// + /// The `Match` returned will include the identifier of the pattern that + /// matched, which corresponds to the index of the pattern (starting from + /// `0`) in which it was added. The offsets in the `Match` will be relative + /// to the start of `haystack` (and not `at`). + /// + /// # Panics + /// + /// When `span` does not correspond to a valid range in `haystack`. + /// + /// # Example + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{packed::{MatchKind, Searcher}, PatternID, Span}; + /// + /// # fn example() -> Option<()> { + /// let haystack = "foofoobar"; + /// let searcher = Searcher::new(["foobar", "foo"].iter().cloned())?; + /// let mat = searcher.find_in(haystack, Span::from(3..haystack.len()))?; + /// assert_eq!(PatternID::ZERO, mat.pattern()); + /// assert_eq!(3, mat.start()); + /// assert_eq!(9, mat.end()); + /// # Some(()) } + /// # if cfg!(all(feature = "std", any( + /// # target_arch = "x86_64", target_arch = "aarch64", + /// # ))) { + /// # example().unwrap() + /// # } else { + /// # assert!(example().is_none()); + /// # } + /// ``` + #[inline] + pub fn find_in>( + &self, + haystack: B, + span: Span, + ) -> Option { + let haystack = haystack.as_ref(); + match self.search_kind { + SearchKind::Teddy(ref teddy) => { + if haystack[span].len() < teddy.minimum_len() { + return self.find_in_slow(haystack, span); + } + teddy.find(&haystack[..span.end], span.start) + } + SearchKind::RabinKarp => { + self.rabinkarp.find_at(&haystack[..span.end], span.start) + } + } + } + + /// Return an iterator of non-overlapping occurrences of the patterns in + /// this searcher, according to its match semantics, in the given haystack. + /// + /// # Example + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::{packed::{MatchKind, Searcher}, PatternID}; + /// + /// # fn example() -> Option<()> { + /// let searcher = Searcher::new(["foobar", "foo"].iter().cloned())?; + /// let matches: Vec = searcher + /// .find_iter("foobar fooba foofoo") + /// .map(|mat| mat.pattern()) + /// .collect(); + /// assert_eq!(vec![ + /// PatternID::must(0), + /// PatternID::must(1), + /// PatternID::must(1), + /// PatternID::must(1), + /// ], matches); + /// # Some(()) } + /// # if cfg!(all(feature = "std", any( + /// # target_arch = "x86_64", target_arch = "aarch64", + /// # ))) { + /// # example().unwrap() + /// # } else { + /// # assert!(example().is_none()); + /// # } + /// ``` + #[inline] + pub fn find_iter<'a, 'b, B: ?Sized + AsRef<[u8]>>( + &'a self, + haystack: &'b B, + ) -> FindIter<'a, 'b> { + let haystack = haystack.as_ref(); + let span = Span::from(0..haystack.len()); + FindIter { searcher: self, haystack, span } + } + + /// Returns the match kind used by this packed searcher. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// use aho_corasick::packed::{MatchKind, Searcher}; + /// + /// # fn example() -> Option<()> { + /// let searcher = Searcher::new(["foobar", "foo"].iter().cloned())?; + /// // leftmost-first is the default. + /// assert_eq!(&MatchKind::LeftmostFirst, searcher.match_kind()); + /// # Some(()) } + /// # if cfg!(all(feature = "std", any( + /// # target_arch = "x86_64", target_arch = "aarch64", + /// # ))) { + /// # example().unwrap() + /// # } else { + /// # assert!(example().is_none()); + /// # } + /// ``` + #[inline] + pub fn match_kind(&self) -> &MatchKind { + self.patterns.match_kind() + } + + /// Returns the minimum length of a haystack that is required in order for + /// packed searching to be effective. + /// + /// In some cases, the underlying packed searcher may not be able to search + /// very short haystacks. When that occurs, the implementation will defer + /// to a slower non-packed searcher (which is still generally faster than + /// Aho-Corasick for a small number of patterns). However, callers may + /// want to avoid ever using the slower variant, which one can do by + /// never passing a haystack shorter than the minimum length returned by + /// this method. + #[inline] + pub fn minimum_len(&self) -> usize { + self.minimum_len + } + + /// Returns the approximate total amount of heap used by this searcher, in + /// units of bytes. + #[inline] + pub fn memory_usage(&self) -> usize { + self.patterns.memory_usage() + + self.rabinkarp.memory_usage() + + self.search_kind.memory_usage() + } + + /// Use a slow (non-packed) searcher. + /// + /// This is useful when a packed searcher could be constructed, but could + /// not be used to search a specific haystack. For example, if Teddy was + /// built but the haystack is smaller than ~34 bytes, then Teddy might not + /// be able to run. + fn find_in_slow(&self, haystack: &[u8], span: Span) -> Option { + self.rabinkarp.find_at(&haystack[..span.end], span.start) + } +} + +impl SearchKind { + fn memory_usage(&self) -> usize { + match *self { + SearchKind::Teddy(ref ted) => ted.memory_usage(), + SearchKind::RabinKarp => 0, + } + } +} + +/// An iterator over non-overlapping matches from a packed searcher. +/// +/// The lifetime `'s` refers to the lifetime of the underlying [`Searcher`], +/// while the lifetime `'h` refers to the lifetime of the haystack being +/// searched. +#[derive(Debug)] +pub struct FindIter<'s, 'h> { + searcher: &'s Searcher, + haystack: &'h [u8], + span: Span, +} + +impl<'s, 'h> Iterator for FindIter<'s, 'h> { + type Item = Match; + + fn next(&mut self) -> Option { + if self.span.start > self.span.end { + return None; + } + match self.searcher.find_in(self.haystack, self.span) { + None => None, + Some(m) => { + self.span.start = m.end(); + Some(m) + } + } + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/ext.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/ext.rs new file mode 100644 index 0000000..b689642 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/ext.rs @@ -0,0 +1,39 @@ +/// A trait for adding some helper routines to pointers. +pub(crate) trait Pointer { + /// Returns the distance, in units of `T`, between `self` and `origin`. + /// + /// # Safety + /// + /// Same as `ptr::offset_from` in addition to `self >= origin`. + unsafe fn distance(self, origin: Self) -> usize; + + /// Casts this pointer to `usize`. + /// + /// Callers should not convert the `usize` back to a pointer if at all + /// possible. (And if you believe it's necessary, open an issue to discuss + /// why. Otherwise, it has the potential to violate pointer provenance.) + /// The purpose of this function is just to be able to do arithmetic, i.e., + /// computing offsets or alignments. + fn as_usize(self) -> usize; +} + +impl Pointer for *const T { + unsafe fn distance(self, origin: *const T) -> usize { + // TODO: Replace with `ptr::sub_ptr` once stabilized. + usize::try_from(self.offset_from(origin)).unwrap_unchecked() + } + + fn as_usize(self) -> usize { + self as usize + } +} + +impl Pointer for *mut T { + unsafe fn distance(self, origin: *mut T) -> usize { + (self as *const T).distance(origin as *const T) + } + + fn as_usize(self) -> usize { + (self as *const T).as_usize() + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/mod.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/mod.rs new file mode 100644 index 0000000..3990bc9 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/mod.rs @@ -0,0 +1,120 @@ +/*! +Provides packed multiple substring search, principally for a small number of +patterns. + +This sub-module provides vectorized routines for quickly finding +matches of a small number of patterns. In general, users of this crate +shouldn't need to interface with this module directly, as the primary +[`AhoCorasick`](crate::AhoCorasick) searcher will use these routines +automatically as a prefilter when applicable. However, in some cases, callers +may want to bypass the Aho-Corasick machinery entirely and use this vectorized +searcher directly. + +# Overview + +The primary types in this sub-module are: + +* [`Searcher`] executes the actual search algorithm to report matches in a +haystack. +* [`Builder`] accumulates patterns incrementally and can construct a +`Searcher`. +* [`Config`] permits tuning the searcher, and itself will produce a `Builder` +(which can then be used to build a `Searcher`). Currently, the only tuneable +knob are the match semantics, but this may be expanded in the future. + +# Examples + +This example shows how to create a searcher from an iterator of patterns. +By default, leftmost-first match semantics are used. (See the top-level +[`MatchKind`] type for more details about match semantics, which apply +similarly to packed substring search.) + +``` +use aho_corasick::{packed::{MatchKind, Searcher}, PatternID}; + +# fn example() -> Option<()> { +let searcher = Searcher::new(["foobar", "foo"].iter().cloned())?; +let matches: Vec = searcher + .find_iter("foobar") + .map(|mat| mat.pattern()) + .collect(); +assert_eq!(vec![PatternID::ZERO], matches); +# Some(()) } +# if cfg!(all(feature = "std", any( +# target_arch = "x86_64", target_arch = "aarch64", +# ))) { +# example().unwrap() +# } else { +# assert!(example().is_none()); +# } +``` + +This example shows how to use [`Config`] to change the match semantics to +leftmost-longest: + +``` +use aho_corasick::{packed::{Config, MatchKind}, PatternID}; + +# fn example() -> Option<()> { +let searcher = Config::new() + .match_kind(MatchKind::LeftmostLongest) + .builder() + .add("foo") + .add("foobar") + .build()?; +let matches: Vec = searcher + .find_iter("foobar") + .map(|mat| mat.pattern()) + .collect(); +assert_eq!(vec![PatternID::must(1)], matches); +# Some(()) } +# if cfg!(all(feature = "std", any( +# target_arch = "x86_64", target_arch = "aarch64", +# ))) { +# example().unwrap() +# } else { +# assert!(example().is_none()); +# } +``` + +# Packed substring searching + +Packed substring searching refers to the use of SIMD (Single Instruction, +Multiple Data) to accelerate the detection of matches in a haystack. Unlike +conventional algorithms, such as Aho-Corasick, SIMD algorithms for substring +search tend to do better with a small number of patterns, where as Aho-Corasick +generally maintains reasonably consistent performance regardless of the number +of patterns you give it. Because of this, the vectorized searcher in this +sub-module cannot be used as a general purpose searcher, since building the +searcher may fail even when given a small number of patterns. However, in +exchange, when searching for a small number of patterns, searching can be quite +a bit faster than Aho-Corasick (sometimes by an order of magnitude). + +The key take away here is that constructing a searcher from a list of patterns +is a fallible operation with no clear rules for when it will fail. While the +precise conditions under which building a searcher can fail is specifically an +implementation detail, here are some common reasons: + +* Too many patterns were given. Typically, the limit is on the order of 100 or + so, but this limit may fluctuate based on available CPU features. +* The available packed algorithms require CPU features that aren't available. + For example, currently, this crate only provides packed algorithms for + `x86_64` and `aarch64`. Therefore, constructing a packed searcher on any + other target will always fail. +* Zero patterns were given, or one of the patterns given was empty. Packed + searchers require at least one pattern and that all patterns are non-empty. +* Something else about the nature of the patterns (typically based on + heuristics) suggests that a packed searcher would perform very poorly, so + no searcher is built. +*/ + +pub use crate::packed::api::{Builder, Config, FindIter, MatchKind, Searcher}; + +mod api; +mod ext; +mod pattern; +mod rabinkarp; +mod teddy; +#[cfg(all(feature = "std", test))] +mod tests; +mod vector; diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/pattern.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/pattern.rs new file mode 100644 index 0000000..14da87a --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/pattern.rs @@ -0,0 +1,480 @@ +use core::{cmp, fmt, mem, u16, usize}; + +use alloc::{boxed::Box, string::String, vec, vec::Vec}; + +use crate::{ + packed::{api::MatchKind, ext::Pointer}, + PatternID, +}; + +/// A non-empty collection of non-empty patterns to search for. +/// +/// This collection of patterns is what is passed around to both execute +/// searches and to construct the searchers themselves. Namely, this permits +/// searches to avoid copying all of the patterns, and allows us to keep only +/// one copy throughout all packed searchers. +/// +/// Note that this collection is not a set. The same pattern can appear more +/// than once. +#[derive(Clone, Debug)] +pub(crate) struct Patterns { + /// The match semantics supported by this collection of patterns. + /// + /// The match semantics determines the order of the iterator over patterns. + /// For leftmost-first, patterns are provided in the same order as were + /// provided by the caller. For leftmost-longest, patterns are provided in + /// descending order of length, with ties broken by the order in which they + /// were provided by the caller. + kind: MatchKind, + /// The collection of patterns, indexed by their identifier. + by_id: Vec>, + /// The order of patterns defined for iteration, given by pattern + /// identifiers. The order of `by_id` and `order` is always the same for + /// leftmost-first semantics, but may be different for leftmost-longest + /// semantics. + order: Vec, + /// The length of the smallest pattern, in bytes. + minimum_len: usize, + /// The total number of pattern bytes across the entire collection. This + /// is used for reporting total heap usage in constant time. + total_pattern_bytes: usize, +} + +// BREADCRUMBS: I think we want to experiment with a different bucket +// representation. Basically, each bucket is just a Range to a single +// contiguous allocation? Maybe length-prefixed patterns or something? The +// idea is to try to get rid of the pointer chasing in verification. I don't +// know that that is the issue, but I suspect it is. + +impl Patterns { + /// Create a new collection of patterns for the given match semantics. The + /// ID of each pattern is the index of the pattern at which it occurs in + /// the `by_id` slice. + /// + /// If any of the patterns in the slice given are empty, then this panics. + /// Similarly, if the number of patterns given is zero, then this also + /// panics. + pub(crate) fn new() -> Patterns { + Patterns { + kind: MatchKind::default(), + by_id: vec![], + order: vec![], + minimum_len: usize::MAX, + total_pattern_bytes: 0, + } + } + + /// Add a pattern to this collection. + /// + /// This panics if the pattern given is empty. + pub(crate) fn add(&mut self, bytes: &[u8]) { + assert!(!bytes.is_empty()); + assert!(self.by_id.len() <= u16::MAX as usize); + + let id = PatternID::new(self.by_id.len()).unwrap(); + self.order.push(id); + self.by_id.push(bytes.to_vec()); + self.minimum_len = cmp::min(self.minimum_len, bytes.len()); + self.total_pattern_bytes += bytes.len(); + } + + /// Set the match kind semantics for this collection of patterns. + /// + /// If the kind is not set, then the default is leftmost-first. + pub(crate) fn set_match_kind(&mut self, kind: MatchKind) { + self.kind = kind; + match self.kind { + MatchKind::LeftmostFirst => { + self.order.sort(); + } + MatchKind::LeftmostLongest => { + let (order, by_id) = (&mut self.order, &mut self.by_id); + order.sort_by(|&id1, &id2| { + by_id[id1].len().cmp(&by_id[id2].len()).reverse() + }); + } + } + } + + /// Return the number of patterns in this collection. + /// + /// This is guaranteed to be greater than zero. + pub(crate) fn len(&self) -> usize { + self.by_id.len() + } + + /// Returns true if and only if this collection of patterns is empty. + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the approximate total amount of heap used by these patterns, in + /// units of bytes. + pub(crate) fn memory_usage(&self) -> usize { + self.order.len() * mem::size_of::() + + self.by_id.len() * mem::size_of::>() + + self.total_pattern_bytes + } + + /// Clears all heap memory associated with this collection of patterns and + /// resets all state such that it is a valid empty collection. + pub(crate) fn reset(&mut self) { + self.kind = MatchKind::default(); + self.by_id.clear(); + self.order.clear(); + self.minimum_len = usize::MAX; + } + + /// Returns the length, in bytes, of the smallest pattern. + /// + /// This is guaranteed to be at least one. + pub(crate) fn minimum_len(&self) -> usize { + self.minimum_len + } + + /// Returns the match semantics used by these patterns. + pub(crate) fn match_kind(&self) -> &MatchKind { + &self.kind + } + + /// Return the pattern with the given identifier. If such a pattern does + /// not exist, then this panics. + pub(crate) fn get(&self, id: PatternID) -> Pattern<'_> { + Pattern(&self.by_id[id]) + } + + /// Return the pattern with the given identifier without performing bounds + /// checks. + /// + /// # Safety + /// + /// Callers must ensure that a pattern with the given identifier exists + /// before using this method. + pub(crate) unsafe fn get_unchecked(&self, id: PatternID) -> Pattern<'_> { + Pattern(self.by_id.get_unchecked(id.as_usize())) + } + + /// Return an iterator over all the patterns in this collection, in the + /// order in which they should be matched. + /// + /// Specifically, in a naive multi-pattern matcher, the following is + /// guaranteed to satisfy the match semantics of this collection of + /// patterns: + /// + /// ```ignore + /// for i in 0..haystack.len(): + /// for p in patterns.iter(): + /// if haystack[i..].starts_with(p.bytes()): + /// return Match(p.id(), i, i + p.bytes().len()) + /// ``` + /// + /// Namely, among the patterns in a collection, if they are matched in + /// the order provided by this iterator, then the result is guaranteed + /// to satisfy the correct match semantics. (Either leftmost-first or + /// leftmost-longest.) + pub(crate) fn iter(&self) -> PatternIter<'_> { + PatternIter { patterns: self, i: 0 } + } +} + +/// An iterator over the patterns in the `Patterns` collection. +/// +/// The order of the patterns provided by this iterator is consistent with the +/// match semantics of the originating collection of patterns. +/// +/// The lifetime `'p` corresponds to the lifetime of the collection of patterns +/// this is iterating over. +#[derive(Debug)] +pub(crate) struct PatternIter<'p> { + patterns: &'p Patterns, + i: usize, +} + +impl<'p> Iterator for PatternIter<'p> { + type Item = (PatternID, Pattern<'p>); + + fn next(&mut self) -> Option<(PatternID, Pattern<'p>)> { + if self.i >= self.patterns.len() { + return None; + } + let id = self.patterns.order[self.i]; + let p = self.patterns.get(id); + self.i += 1; + Some((id, p)) + } +} + +/// A pattern that is used in packed searching. +#[derive(Clone)] +pub(crate) struct Pattern<'a>(&'a [u8]); + +impl<'a> fmt::Debug for Pattern<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Pattern") + .field("lit", &String::from_utf8_lossy(self.0)) + .finish() + } +} + +impl<'p> Pattern<'p> { + /// Returns the length of this pattern, in bytes. + pub(crate) fn len(&self) -> usize { + self.0.len() + } + + /// Returns the bytes of this pattern. + pub(crate) fn bytes(&self) -> &[u8] { + self.0 + } + + /// Returns the first `len` low nybbles from this pattern. If this pattern + /// is shorter than `len`, then this panics. + pub(crate) fn low_nybbles(&self, len: usize) -> Box<[u8]> { + let mut nybs = vec![0; len].into_boxed_slice(); + for (i, byte) in self.bytes().iter().take(len).enumerate() { + nybs[i] = byte & 0xF; + } + nybs + } + + /// Returns true if this pattern is a prefix of the given bytes. + #[inline(always)] + pub(crate) fn is_prefix(&self, bytes: &[u8]) -> bool { + is_prefix(bytes, self.bytes()) + } + + /// Returns true if this pattern is a prefix of the haystack given by the + /// raw `start` and `end` pointers. + /// + /// # Safety + /// + /// * It must be the case that `start < end` and that the distance between + /// them is at least equal to `V::BYTES`. That is, it must always be valid + /// to do at least an unaligned load of `V` at `start`. + /// * Both `start` and `end` must be valid for reads. + /// * Both `start` and `end` must point to an initialized value. + /// * Both `start` and `end` must point to the same allocated object and + /// must either be in bounds or at most one byte past the end of the + /// allocated object. + /// * Both `start` and `end` must be _derived from_ a pointer to the same + /// object. + /// * The distance between `start` and `end` must not overflow `isize`. + /// * The distance being in bounds must not rely on "wrapping around" the + /// address space. + #[inline(always)] + pub(crate) unsafe fn is_prefix_raw( + &self, + start: *const u8, + end: *const u8, + ) -> bool { + let patlen = self.bytes().len(); + let haylen = end.distance(start); + if patlen > haylen { + return false; + } + // SAFETY: We've checked that the haystack has length at least equal + // to this pattern. All other safety concerns are the responsibility + // of the caller. + is_equal_raw(start, self.bytes().as_ptr(), patlen) + } +} + +/// Returns true if and only if `needle` is a prefix of `haystack`. +/// +/// This uses a latency optimized variant of `memcmp` internally which *might* +/// make this faster for very short strings. +/// +/// # Inlining +/// +/// This routine is marked `inline(always)`. If you want to call this function +/// in a way that is not always inlined, you'll need to wrap a call to it in +/// another function that is marked as `inline(never)` or just `inline`. +#[inline(always)] +fn is_prefix(haystack: &[u8], needle: &[u8]) -> bool { + if needle.len() > haystack.len() { + return false; + } + // SAFETY: Our pointers are derived directly from borrowed slices which + // uphold all of our safety guarantees except for length. We account for + // length with the check above. + unsafe { is_equal_raw(haystack.as_ptr(), needle.as_ptr(), needle.len()) } +} + +/// Compare corresponding bytes in `x` and `y` for equality. +/// +/// That is, this returns true if and only if `x.len() == y.len()` and +/// `x[i] == y[i]` for all `0 <= i < x.len()`. +/// +/// Note that this isn't used. We only use it in tests as a convenient way +/// of testing `is_equal_raw`. +/// +/// # Inlining +/// +/// This routine is marked `inline(always)`. If you want to call this function +/// in a way that is not always inlined, you'll need to wrap a call to it in +/// another function that is marked as `inline(never)` or just `inline`. +/// +/// # Motivation +/// +/// Why not use slice equality instead? Well, slice equality usually results in +/// a call out to the current platform's `libc` which might not be inlineable +/// or have other overhead. This routine isn't guaranteed to be a win, but it +/// might be in some cases. +#[cfg(test)] +#[inline(always)] +fn is_equal(x: &[u8], y: &[u8]) -> bool { + if x.len() != y.len() { + return false; + } + // SAFETY: Our pointers are derived directly from borrowed slices which + // uphold all of our safety guarantees except for length. We account for + // length with the check above. + unsafe { is_equal_raw(x.as_ptr(), y.as_ptr(), x.len()) } +} + +/// Compare `n` bytes at the given pointers for equality. +/// +/// This returns true if and only if `*x.add(i) == *y.add(i)` for all +/// `0 <= i < n`. +/// +/// # Inlining +/// +/// This routine is marked `inline(always)`. If you want to call this function +/// in a way that is not always inlined, you'll need to wrap a call to it in +/// another function that is marked as `inline(never)` or just `inline`. +/// +/// # Motivation +/// +/// Why not use slice equality instead? Well, slice equality usually results in +/// a call out to the current platform's `libc` which might not be inlineable +/// or have other overhead. This routine isn't guaranteed to be a win, but it +/// might be in some cases. +/// +/// # Safety +/// +/// * Both `x` and `y` must be valid for reads of up to `n` bytes. +/// * Both `x` and `y` must point to an initialized value. +/// * Both `x` and `y` must each point to an allocated object and +/// must either be in bounds or at most one byte past the end of the +/// allocated object. `x` and `y` do not need to point to the same allocated +/// object, but they may. +/// * Both `x` and `y` must be _derived from_ a pointer to their respective +/// allocated objects. +/// * The distance between `x` and `x+n` must not overflow `isize`. Similarly +/// for `y` and `y+n`. +/// * The distance being in bounds must not rely on "wrapping around" the +/// address space. +#[inline(always)] +unsafe fn is_equal_raw(mut x: *const u8, mut y: *const u8, n: usize) -> bool { + // If we don't have enough bytes to do 4-byte at a time loads, then + // handle each possible length specially. Note that I used to have a + // byte-at-a-time loop here and that turned out to be quite a bit slower + // for the memmem/pathological/defeat-simple-vector-alphabet benchmark. + if n < 4 { + return match n { + 0 => true, + 1 => x.read() == y.read(), + 2 => { + x.cast::().read_unaligned() + == y.cast::().read_unaligned() + } + // I also tried copy_nonoverlapping here and it looks like the + // codegen is the same. + 3 => x.cast::<[u8; 3]>().read() == y.cast::<[u8; 3]>().read(), + _ => unreachable!(), + }; + } + // When we have 4 or more bytes to compare, then proceed in chunks of 4 at + // a time using unaligned loads. + // + // Also, why do 4 byte loads instead of, say, 8 byte loads? The reason is + // that this particular version of memcmp is likely to be called with tiny + // needles. That means that if we do 8 byte loads, then a higher proportion + // of memcmp calls will use the slower variant above. With that said, this + // is a hypothesis and is only loosely supported by benchmarks. There's + // likely some improvement that could be made here. The main thing here + // though is to optimize for latency, not throughput. + + // SAFETY: The caller is responsible for ensuring the pointers we get are + // valid and readable for at least `n` bytes. We also do unaligned loads, + // so there's no need to ensure we're aligned. (This is justified by this + // routine being specifically for short strings.) + let xend = x.add(n.wrapping_sub(4)); + let yend = y.add(n.wrapping_sub(4)); + while x < xend { + let vx = x.cast::().read_unaligned(); + let vy = y.cast::().read_unaligned(); + if vx != vy { + return false; + } + x = x.add(4); + y = y.add(4); + } + let vx = xend.cast::().read_unaligned(); + let vy = yend.cast::().read_unaligned(); + vx == vy +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn equals_different_lengths() { + assert!(!is_equal(b"", b"a")); + assert!(!is_equal(b"a", b"")); + assert!(!is_equal(b"ab", b"a")); + assert!(!is_equal(b"a", b"ab")); + } + + #[test] + fn equals_mismatch() { + let one_mismatch = [ + (&b"a"[..], &b"x"[..]), + (&b"ab"[..], &b"ax"[..]), + (&b"abc"[..], &b"abx"[..]), + (&b"abcd"[..], &b"abcx"[..]), + (&b"abcde"[..], &b"abcdx"[..]), + (&b"abcdef"[..], &b"abcdex"[..]), + (&b"abcdefg"[..], &b"abcdefx"[..]), + (&b"abcdefgh"[..], &b"abcdefgx"[..]), + (&b"abcdefghi"[..], &b"abcdefghx"[..]), + (&b"abcdefghij"[..], &b"abcdefghix"[..]), + (&b"abcdefghijk"[..], &b"abcdefghijx"[..]), + (&b"abcdefghijkl"[..], &b"abcdefghijkx"[..]), + (&b"abcdefghijklm"[..], &b"abcdefghijklx"[..]), + (&b"abcdefghijklmn"[..], &b"abcdefghijklmx"[..]), + ]; + for (x, y) in one_mismatch { + assert_eq!(x.len(), y.len(), "lengths should match"); + assert!(!is_equal(x, y)); + assert!(!is_equal(y, x)); + } + } + + #[test] + fn equals_yes() { + assert!(is_equal(b"", b"")); + assert!(is_equal(b"a", b"a")); + assert!(is_equal(b"ab", b"ab")); + assert!(is_equal(b"abc", b"abc")); + assert!(is_equal(b"abcd", b"abcd")); + assert!(is_equal(b"abcde", b"abcde")); + assert!(is_equal(b"abcdef", b"abcdef")); + assert!(is_equal(b"abcdefg", b"abcdefg")); + assert!(is_equal(b"abcdefgh", b"abcdefgh")); + assert!(is_equal(b"abcdefghi", b"abcdefghi")); + } + + #[test] + fn prefix() { + assert!(is_prefix(b"", b"")); + assert!(is_prefix(b"a", b"")); + assert!(is_prefix(b"ab", b"")); + assert!(is_prefix(b"foo", b"foo")); + assert!(is_prefix(b"foobar", b"foo")); + + assert!(!is_prefix(b"foo", b"fob")); + assert!(!is_prefix(b"foobar", b"fob")); + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/rabinkarp.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/rabinkarp.rs new file mode 100644 index 0000000..fdd8a6f --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/rabinkarp.rs @@ -0,0 +1,168 @@ +use alloc::{sync::Arc, vec, vec::Vec}; + +use crate::{packed::pattern::Patterns, util::search::Match, PatternID}; + +/// The type of the rolling hash used in the Rabin-Karp algorithm. +type Hash = usize; + +/// The number of buckets to store our patterns in. We don't want this to be +/// too big in order to avoid wasting memory, but we don't want it to be too +/// small either to avoid spending too much time confirming literals. +/// +/// The number of buckets MUST be a power of two. Otherwise, determining the +/// bucket from a hash will slow down the code considerably. Using a power +/// of two means `hash % NUM_BUCKETS` can compile down to a simple `and` +/// instruction. +const NUM_BUCKETS: usize = 64; + +/// An implementation of the Rabin-Karp algorithm. The main idea of this +/// algorithm is to maintain a rolling hash as it moves through the input, and +/// then check whether that hash corresponds to the same hash for any of the +/// patterns we're looking for. +/// +/// A draw back of naively scaling Rabin-Karp to multiple patterns is that +/// it requires all of the patterns to be the same length, which in turn +/// corresponds to the number of bytes to hash. We adapt this to work for +/// multiple patterns of varying size by fixing the number of bytes to hash +/// to be the length of the smallest pattern. We also split the patterns into +/// several buckets to hopefully make the confirmation step faster. +/// +/// Wikipedia has a decent explanation, if a bit heavy on the theory: +/// https://en.wikipedia.org/wiki/Rabin%E2%80%93Karp_algorithm +/// +/// But ESMAJ provides something a bit more concrete: +/// https://www-igm.univ-mlv.fr/~lecroq/string/node5.html +#[derive(Clone, Debug)] +pub(crate) struct RabinKarp { + /// The patterns we're searching for. + patterns: Arc, + /// The order of patterns in each bucket is significant. Namely, they are + /// arranged such that the first one to match is the correct match. This + /// may not necessarily correspond to the order provided by the caller. + /// For example, if leftmost-longest semantics are used, then the patterns + /// are sorted by their length in descending order. If leftmost-first + /// semantics are used, then the patterns are sorted by their pattern ID + /// in ascending order (which corresponds to the caller's order). + buckets: Vec>, + /// The length of the hashing window. Generally, this corresponds to the + /// length of the smallest pattern. + hash_len: usize, + /// The factor to subtract out of a hash before updating it with a new + /// byte. + hash_2pow: usize, +} + +impl RabinKarp { + /// Compile a new Rabin-Karp matcher from the patterns given. + /// + /// This panics if any of the patterns in the collection are empty, or if + /// the collection is itself empty. + pub(crate) fn new(patterns: &Arc) -> RabinKarp { + assert!(patterns.len() >= 1); + let hash_len = patterns.minimum_len(); + assert!(hash_len >= 1); + + let mut hash_2pow = 1usize; + for _ in 1..hash_len { + hash_2pow = hash_2pow.wrapping_shl(1); + } + + let mut rk = RabinKarp { + patterns: Arc::clone(patterns), + buckets: vec![vec![]; NUM_BUCKETS], + hash_len, + hash_2pow, + }; + for (id, pat) in patterns.iter() { + let hash = rk.hash(&pat.bytes()[..rk.hash_len]); + let bucket = hash % NUM_BUCKETS; + rk.buckets[bucket].push((hash, id)); + } + rk + } + + /// Return the first matching pattern in the given haystack, begining the + /// search at `at`. + pub(crate) fn find_at( + &self, + haystack: &[u8], + mut at: usize, + ) -> Option { + assert_eq!(NUM_BUCKETS, self.buckets.len()); + + if at + self.hash_len > haystack.len() { + return None; + } + let mut hash = self.hash(&haystack[at..at + self.hash_len]); + loop { + let bucket = &self.buckets[hash % NUM_BUCKETS]; + for &(phash, pid) in bucket { + if phash == hash { + if let Some(c) = self.verify(pid, haystack, at) { + return Some(c); + } + } + } + if at + self.hash_len >= haystack.len() { + return None; + } + hash = self.update_hash( + hash, + haystack[at], + haystack[at + self.hash_len], + ); + at += 1; + } + } + + /// Returns the approximate total amount of heap used by this searcher, in + /// units of bytes. + pub(crate) fn memory_usage(&self) -> usize { + self.buckets.len() * core::mem::size_of::>() + + self.patterns.len() * core::mem::size_of::<(Hash, PatternID)>() + } + + /// Verify whether the pattern with the given id matches at + /// `haystack[at..]`. + /// + /// We tag this function as `cold` because it helps improve codegen. + /// Intuitively, it would seem like inlining it would be better. However, + /// the only time this is called and a match is not found is when there + /// there is a hash collision, or when a prefix of a pattern matches but + /// the entire pattern doesn't match. This is hopefully fairly rare, and + /// if it does occur a lot, it's going to be slow no matter what we do. + #[cold] + fn verify( + &self, + id: PatternID, + haystack: &[u8], + at: usize, + ) -> Option { + let pat = self.patterns.get(id); + if pat.is_prefix(&haystack[at..]) { + Some(Match::new(id, at..at + pat.len())) + } else { + None + } + } + + /// Hash the given bytes. + fn hash(&self, bytes: &[u8]) -> Hash { + assert_eq!(self.hash_len, bytes.len()); + + let mut hash = 0usize; + for &b in bytes { + hash = hash.wrapping_shl(1).wrapping_add(b as usize); + } + hash + } + + /// Update the hash given based on removing `old_byte` at the beginning + /// of some byte string, and appending `new_byte` to the end of that same + /// byte string. + fn update_hash(&self, prev: Hash, old_byte: u8, new_byte: u8) -> Hash { + prev.wrapping_sub((old_byte as usize).wrapping_mul(self.hash_2pow)) + .wrapping_shl(1) + .wrapping_add(new_byte as usize) + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/README.md b/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/README.md new file mode 100644 index 0000000..f0928cb --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/README.md @@ -0,0 +1,386 @@ +Teddy is a SIMD accelerated multiple substring matching algorithm. The name +and the core ideas in the algorithm were learned from the [Hyperscan][1_u] +project. The implementation in this repository was mostly motivated for use in +accelerating regex searches by searching for small sets of required literals +extracted from the regex. + + +# Background + +The key idea of Teddy is to do *packed* substring matching. In the literature, +packed substring matching is the idea of examining multiple bytes in a haystack +at a time to detect matches. Implementations of, for example, memchr (which +detects matches of a single byte) have been doing this for years. Only +recently, with the introduction of various SIMD instructions, has this been +extended to substring matching. The PCMPESTRI instruction (and its relatives), +for example, implements substring matching in hardware. It is, however, limited +to substrings of length 16 bytes or fewer, but this restriction is fine in a +regex engine, since we rarely care about the performance difference between +searching for a 16 byte literal and a 16 + N literal; 16 is already long +enough. The key downside of the PCMPESTRI instruction, on current (2016) CPUs +at least, is its latency and throughput. As a result, it is often faster to +do substring search with a Boyer-Moore (or Two-Way) variant and a well placed +memchr to quickly skip through the haystack. + +There are fewer results from the literature on packed substring matching, +and even fewer for packed multiple substring matching. Ben-Kiki et al. [2] +describes use of PCMPESTRI for substring matching, but is mostly theoretical +and hand-waves performance. There is other theoretical work done by Bille [3] +as well. + +The rest of the work in the field, as far as I'm aware, is by Faro and Kulekci +and is generally focused on multiple pattern search. Their first paper [4a] +introduces the concept of a fingerprint, which is computed for every block of +N bytes in every pattern. The haystack is then scanned N bytes at a time and +a fingerprint is computed in the same way it was computed for blocks in the +patterns. If the fingerprint corresponds to one that was found in a pattern, +then a verification step follows to confirm that one of the substrings with the +corresponding fingerprint actually matches at the current location. Various +implementation tricks are employed to make sure the fingerprint lookup is fast; +typically by truncating the fingerprint. (This may, of course, provoke more +steps in the verification process, so a balance must be struck.) + +The main downside of [4a] is that the minimum substring length is 32 bytes, +presumably because of how the algorithm uses certain SIMD instructions. This +essentially makes it useless for general purpose regex matching, where a small +number of short patterns is far more likely. + +Faro and Kulekci published another paper [4b] that is conceptually very similar +to [4a]. The key difference is that it uses the CRC32 instruction (introduced +as part of SSE 4.2) to compute fingerprint values. This also enables the +algorithm to work effectively on substrings as short as 7 bytes with 4 byte +windows. 7 bytes is unfortunately still too long. The window could be +technically shrunk to 2 bytes, thereby reducing minimum length to 3, but the +small window size ends up negating most performance benefits—and it's likely +the common case in a general purpose regex engine. + +Faro and Kulekci also published [4c] that appears to be intended as a +replacement to using PCMPESTRI. In particular, it is specifically motivated by +the high throughput/latency time of PCMPESTRI and therefore chooses other SIMD +instructions that are faster. While this approach works for short substrings, +I personally couldn't see a way to generalize it to multiple substring search. + +Faro and Kulekci have another paper [4d] that I haven't been able to read +because it is behind a paywall. + + +# Teddy + +Finally, we get to Teddy. If the above literature review is complete, then it +appears that Teddy is a novel algorithm. More than that, in my experience, it +completely blows away the competition for short substrings, which is exactly +what we want in a general purpose regex engine. Again, the algorithm appears +to be developed by the authors of [Hyperscan][1_u]. Hyperscan was open sourced +late 2015, and no earlier history could be found. Therefore, tracking the exact +provenance of the algorithm with respect to the published literature seems +difficult. + +At a high level, Teddy works somewhat similarly to the fingerprint algorithms +published by Faro and Kulekci, but Teddy does it in a way that scales a bit +better. Namely: + +1. Teddy's core algorithm scans the haystack in 16 (for SSE, or 32 for AVX) + byte chunks. 16 (or 32) is significant because it corresponds to the number + of bytes in a SIMD vector. +2. Bitwise operations are performed on each chunk to discover if any region of + it matches a set of precomputed fingerprints from the patterns. If there are + matches, then a verification step is performed. In this implementation, our + verification step is naive. This can be improved upon. + +The details to make this work are quite clever. First, we must choose how to +pick our fingerprints. In Hyperscan's implementation, I *believe* they use the +last N bytes of each substring, where N must be at least the minimum length of +any substring in the set being searched. In this implementation, we use the +first N bytes of each substring. (The tradeoffs between these choices aren't +yet clear to me.) We then must figure out how to quickly test whether an +occurrence of any fingerprint from the set of patterns appears in a 16 byte +block from the haystack. To keep things simple, let's assume N = 1 and examine +some examples to motivate the approach. Here are our patterns: + +```ignore +foo +bar +baz +``` + +The corresponding fingerprints, for N = 1, are `f`, `b` and `b`. Now let's set +our 16 byte block to: + +```ignore +bat cat foo bump +xxxxxxxxxxxxxxxx +``` + +To cut to the chase, Teddy works by using bitsets. In particular, Teddy creates +a mask that allows us to quickly compute membership of a fingerprint in a 16 +byte block that also tells which pattern the fingerprint corresponds to. In +this case, our fingerprint is a single byte, so an appropriate abstraction is +a map from a single byte to a list of patterns that contain that fingerprint: + +```ignore +f |--> foo +b |--> bar, baz +``` + +Now, all we need to do is figure out how to represent this map in vector space +and use normal SIMD operations to perform a lookup. The first simplification +we can make is to represent our patterns as bit fields occupying a single +byte. This is important, because a single SIMD vector can store 16 bytes. + +```ignore +f |--> 00000001 +b |--> 00000010, 00000100 +``` + +How do we perform lookup though? It turns out that SSSE3 introduced a very cool +instruction called PSHUFB. The instruction takes two SIMD vectors, `A` and `B`, +and returns a third vector `C`. All vectors are treated as 16 8-bit integers. +`C` is formed by `C[i] = A[B[i]]`. (This is a bit of a simplification, but true +for the purposes of this algorithm. For full details, see [Intel's Intrinsics +Guide][5_u].) This essentially lets us use the values in `B` to lookup values +in `A`. + +If we could somehow cause `B` to contain our 16 byte block from the haystack, +and if `A` could contain our bitmasks, then we'd end up with something like +this for `A`: + +```ignore + 0x00 0x01 ... 0x62 ... 0x66 ... 0xFF +A = 0 0 00000110 00000001 0 +``` + +And if `B` contains our window from our haystack, we could use shuffle to take +the values from `B` and use them to look up our bitsets in `A`. But of course, +we can't do this because `A` in the above example contains 256 bytes, which +is much larger than the size of a SIMD vector. + +Nybbles to the rescue! A nybble is 4 bits. Instead of one mask to hold all of +our bitsets, we can use two masks, where one mask corresponds to the lower four +bits of our fingerprint and the other mask corresponds to the upper four bits. +So our map now looks like: + +```ignore +'f' & 0xF = 0x6 |--> 00000001 +'f' >> 4 = 0x6 |--> 00000111 +'b' & 0xF = 0x2 |--> 00000110 +'b' >> 4 = 0x6 |--> 00000111 +``` + +Notice that the bitsets for each nybble correspond to the union of all +fingerprints that contain that nybble. For example, both `f` and `b` have the +same upper 4 bits but differ on the lower 4 bits. Putting this together, we +have `A0`, `A1` and `B`, where `A0` is our mask for the lower nybble, `A1` is +our mask for the upper nybble and `B` is our 16 byte block from the haystack: + +```ignore + 0x00 0x01 0x02 0x03 ... 0x06 ... 0xF +A0 = 0 0 00000110 0 00000001 0 +A1 = 0 0 0 0 00000111 0 +B = b a t _ t p +B = 0x62 0x61 0x74 0x20 0x74 0x70 +``` + +But of course, we can't use `B` with `PSHUFB` yet, since its values are 8 bits, +and we need indexes that are at most 4 bits (corresponding to one of 16 +values). We can apply the same transformation to split `B` into lower and upper +nybbles as we did `A`. As before, `B0` corresponds to the lower nybbles and +`B1` corresponds to the upper nybbles: + +```ignore + b a t _ c a t _ f o o _ b u m p +B0 = 0x2 0x1 0x4 0x0 0x3 0x1 0x4 0x0 0x6 0xF 0xF 0x0 0x2 0x5 0xD 0x0 +B1 = 0x6 0x6 0x7 0x2 0x6 0x6 0x7 0x2 0x6 0x6 0x6 0x2 0x6 0x7 0x6 0x7 +``` + +And now we have a nice correspondence. `B0` can index `A0` and `B1` can index +`A1`. Here's what we get when we apply `C0 = PSHUFB(A0, B0)`: + +```ignore + b a ... f o ... p + A0[0x2] A0[0x1] A0[0x6] A0[0xF] A0[0x0] +C0 = 00000110 0 00000001 0 0 +``` + +And `C1 = PSHUFB(A1, B1)`: + +```ignore + b a ... f o ... p + A1[0x6] A1[0x6] A1[0x6] A1[0x6] A1[0x7] +C1 = 00000111 00000111 00000111 00000111 0 +``` + +Notice how neither one of `C0` or `C1` is guaranteed to report fully correct +results all on its own. For example, `C1` claims that `b` is a fingerprint for +the pattern `foo` (since `A1[0x6] = 00000111`), and that `o` is a fingerprint +for all of our patterns. But if we combined `C0` and `C1` with an `AND` +operation: + +```ignore + b a ... f o ... p +C = 00000110 0 00000001 0 0 +``` + +Then we now have that `C[i]` contains a bitset corresponding to the matching +fingerprints in a haystack's 16 byte block, where `i` is the `ith` byte in that +block. + +Once we have that, we can look for the position of the least significant bit +in `C`. (Least significant because we only target little endian here. Thus, +the least significant bytes correspond to bytes in our haystack at a lower +address.) That position, modulo `8`, gives us the pattern that the fingerprint +matches. That position, integer divided by `8`, also gives us the byte offset +that the fingerprint occurs in inside the 16 byte haystack block. Using those +two pieces of information, we can run a verification procedure that tries +to match all substrings containing that fingerprint at that position in the +haystack. + + +# Implementation notes + +The problem with the algorithm as described above is that it uses a single byte +for a fingerprint. This will work well if the fingerprints are rare in the +haystack (e.g., capital letters or special characters in normal English text), +but if the fingerprints are common, you'll wind up spending too much time in +the verification step, which effectively negates the performance benefits of +scanning 16 bytes at a time. Remember, the key to the performance of this +algorithm is to do as little work as possible per 16 (or 32) bytes. + +This algorithm can be extrapolated in a relatively straight-forward way to use +larger fingerprints. That is, instead of a single byte prefix, we might use a +two or three byte prefix. The implementation here implements N = {1, 2, 3} +and always picks the largest N possible. The rationale is that the bigger the +fingerprint, the fewer verification steps we'll do. Of course, if N is too +large, then we'll end up doing too much on each step. + +The way to extend it is: + +1. Add a mask for each byte in the fingerprint. (Remember that each mask is + composed of two SIMD vectors.) This results in a value of `C` for each byte + in the fingerprint while searching. +2. When testing each 16 (or 32) byte block, each value of `C` must be shifted + so that they are aligned. Once aligned, they should all be `AND`'d together. + This will give you only the bitsets corresponding to the full match of the + fingerprint. To do this, one needs to save the last byte (for N=2) or last + two bytes (for N=3) from the previous iteration, and then line them up with + the first one or two bytes of the next iteration. + +## Verification + +Verification generally follows the procedure outlined above. The tricky parts +are in the right formulation of operations to get our bits out of our vectors. +We have a limited set of operations available to us on SIMD vectors as 128-bit +or 256-bit numbers, so we wind up needing to rip out 2 (or 4) 64-bit integers +from our vectors, and then run our verification step on each of those. The +verification step looks at the least significant bit set, and from its +position, we can derive the byte offset and bucket. (Again, as described +above.) Once we know the bucket, we do a fairly naive exhaustive search for +every literal in that bucket. (Hyperscan is a bit smarter here and uses a hash +table, but I haven't had time to thoroughly explore that. A few initial +half-hearted attempts resulted in worse performance.) + +## AVX + +The AVX version of Teddy extrapolates almost perfectly from the SSE version. +The only hickup is that PALIGNR is used to align chunks in the 16-bit version, +and there is no equivalent instruction in AVX. AVX does have VPALIGNR, but it +only works within 128-bit lanes. So there's a bit of tomfoolery to get around +this by shuffling the vectors before calling VPALIGNR. + +The only other aspect to AVX is that since our masks are still fundamentally +16-bytes (0x0-0xF), they are duplicated to 32-bytes, so that they can apply to +32-byte chunks. + +## Fat Teddy + +In the version of Teddy described above, 8 buckets are used to group patterns +that we want to search for. However, when AVX is available, we can extend the +number of buckets to 16 by permitting each byte in our masks to use 16-bits +instead of 8-bits to represent the buckets it belongs to. (This variant is also +in Hyperscan.) However, what we give up is the ability to scan 32 bytes at a +time, even though we're using AVX. Instead, we have to scan 16 bytes at a time. +What we gain, though, is (hopefully) less work in our verification routine. +It patterns are more spread out across more buckets, then there should overall +be fewer false positives. In general, Fat Teddy permits us to grow our capacity +a bit and search for more literals before Teddy gets overwhelmed. + +The tricky part of Fat Teddy is in how we adjust our masks and our verification +procedure. For the masks, we simply represent the first 8 buckets in each of +the low 16 bytes, and then the second 8 buckets in each of the high 16 bytes. +Then, in the search loop, instead of loading 32 bytes from the haystack, we +load the same 16 bytes from the haystack into both the low and high 16 byte +portions of our 256-bit vector. So for example, a mask might look like this: + + bits: 00100001 00000000 ... 11000000 00000000 00000001 ... 00000000 + byte: 31 30 16 15 14 0 + offset: 15 14 0 15 14 0 + buckets: 8-15 8-15 8-15 0-7 0-7 0-7 + +Where `byte` is the position in the vector (higher numbers corresponding to +more significant bits), `offset` is the corresponding position in the haystack +chunk, and `buckets` corresponds to the bucket assignments for that particular +byte. + +In particular, notice that the bucket assignments for offset `0` are spread +out between bytes `0` and `16`. This works well for the chunk-by-chunk search +procedure, but verification really wants to process all bucket assignments for +each offset at once. Otherwise, we might wind up finding a match at offset +`1` in one the first 8 buckets, when we really should have reported a match +at offset `0` in one of the second 8 buckets. (Because we want the leftmost +match.) + +Thus, for verification, we rearrange the above vector such that it is a +sequence of 16-bit integers, where the least significant 16-bit integer +corresponds to all of the bucket assignments for offset `0`. So with the +above vector, the least significant 16-bit integer would be + + 11000000 000000 + +which was taken from bytes `16` and `0`. Then the verification step pretty much +runs as described, except with 16 buckets instead of 8. + + +# References + +- **[1]** [Hyperscan on GitHub](https://github.com/intel/hyperscan), + [webpage](https://www.hyperscan.io/) +- **[2a]** Ben-Kiki, O., Bille, P., Breslauer, D., Gasieniec, L., Grossi, R., + & Weimann, O. (2011). + _Optimal packed string matching_. + In LIPIcs-Leibniz International Proceedings in Informatics (Vol. 13). + Schloss Dagstuhl-Leibniz-Zentrum fuer Informatik. + DOI: 10.4230/LIPIcs.FSTTCS.2011.423. + [PDF](https://drops.dagstuhl.de/opus/volltexte/2011/3355/pdf/37.pdf). +- **[2b]** Ben-Kiki, O., Bille, P., Breslauer, D., Ga̧sieniec, L., Grossi, R., + & Weimann, O. (2014). + _Towards optimal packed string matching_. + Theoretical Computer Science, 525, 111-129. + DOI: 10.1016/j.tcs.2013.06.013. + [PDF](https://www.cs.haifa.ac.il/~oren/Publications/bpsm.pdf). +- **[3]** Bille, P. (2011). + _Fast searching in packed strings_. + Journal of Discrete Algorithms, 9(1), 49-56. + DOI: 10.1016/j.jda.2010.09.003. + [PDF](https://www.sciencedirect.com/science/article/pii/S1570866710000353). +- **[4a]** Faro, S., & Külekci, M. O. (2012, October). + _Fast multiple string matching using streaming SIMD extensions technology_. + In String Processing and Information Retrieval (pp. 217-228). + Springer Berlin Heidelberg. + DOI: 10.1007/978-3-642-34109-0_23. + [PDF](https://www.dmi.unict.it/faro/papers/conference/faro32.pdf). +- **[4b]** Faro, S., & Külekci, M. O. (2013, September). + _Towards a Very Fast Multiple String Matching Algorithm for Short Patterns_. + In Stringology (pp. 78-91). + [PDF](https://www.dmi.unict.it/faro/papers/conference/faro36.pdf). +- **[4c]** Faro, S., & Külekci, M. O. (2013, January). + _Fast packed string matching for short patterns_. + In Proceedings of the Meeting on Algorithm Engineering & Expermiments + (pp. 113-121). + Society for Industrial and Applied Mathematics. + [PDF](https://arxiv.org/pdf/1209.6449.pdf). +- **[4d]** Faro, S., & Külekci, M. O. (2014). + _Fast and flexible packed string matching_. + Journal of Discrete Algorithms, 28, 61-72. + DOI: 10.1016/j.jda.2014.07.003. + +[1_u]: https://github.com/intel/hyperscan +[5_u]: https://software.intel.com/sites/landingpage/IntrinsicsGuide diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/builder.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/builder.rs new file mode 100644 index 0000000..e9bb68b --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/builder.rs @@ -0,0 +1,792 @@ +use core::{ + fmt::Debug, + panic::{RefUnwindSafe, UnwindSafe}, +}; + +use alloc::sync::Arc; + +use crate::packed::{ext::Pointer, pattern::Patterns, teddy::generic::Match}; + +/// A builder for constructing a Teddy matcher. +/// +/// The builder primarily permits fine grained configuration of the Teddy +/// matcher. Most options are made only available for testing/benchmarking +/// purposes. In reality, options are automatically determined by the nature +/// and number of patterns given to the builder. +#[derive(Clone, Debug)] +pub(crate) struct Builder { + /// When none, this is automatically determined. Otherwise, `false` means + /// slim Teddy is used (8 buckets) and `true` means fat Teddy is used + /// (16 buckets). Fat Teddy requires AVX2, so if that CPU feature isn't + /// available and Fat Teddy was requested, no matcher will be built. + only_fat: Option, + /// When none, this is automatically determined. Otherwise, `false` means + /// that 128-bit vectors will be used (up to SSSE3 instructions) where as + /// `true` means that 256-bit vectors will be used. As with `fat`, if + /// 256-bit vectors are requested and they aren't available, then a + /// searcher will not be built. + only_256bit: Option, + /// When true (the default), the number of patterns will be used as a + /// heuristic for refusing construction of a Teddy searcher. The point here + /// is that too many patterns can overwhelm Teddy. But this can be disabled + /// in cases where the caller knows better. + heuristic_pattern_limits: bool, +} + +impl Default for Builder { + fn default() -> Builder { + Builder::new() + } +} + +impl Builder { + /// Create a new builder for configuring a Teddy matcher. + pub(crate) fn new() -> Builder { + Builder { + only_fat: None, + only_256bit: None, + heuristic_pattern_limits: true, + } + } + + /// Build a matcher for the set of patterns given. If a matcher could not + /// be built, then `None` is returned. + /// + /// Generally, a matcher isn't built if the necessary CPU features aren't + /// available, an unsupported target or if the searcher is believed to be + /// slower than standard techniques (i.e., if there are too many literals). + pub(crate) fn build(&self, patterns: Arc) -> Option { + self.build_imp(patterns) + } + + /// Require the use of Fat (true) or Slim (false) Teddy. Fat Teddy uses + /// 16 buckets where as Slim Teddy uses 8 buckets. More buckets are useful + /// for a larger set of literals. + /// + /// `None` is the default, which results in an automatic selection based + /// on the number of literals and available CPU features. + pub(crate) fn only_fat(&mut self, yes: Option) -> &mut Builder { + self.only_fat = yes; + self + } + + /// Request the use of 256-bit vectors (true) or 128-bit vectors (false). + /// Generally, a larger vector size is better since it either permits + /// matching more patterns or matching more bytes in the haystack at once. + /// + /// `None` is the default, which results in an automatic selection based on + /// the number of literals and available CPU features. + pub(crate) fn only_256bit(&mut self, yes: Option) -> &mut Builder { + self.only_256bit = yes; + self + } + + /// Request that heuristic limitations on the number of patterns be + /// employed. This useful to disable for benchmarking where one wants to + /// explore how Teddy performs on large number of patterns even if the + /// heuristics would otherwise refuse construction. + /// + /// This is enabled by default. + pub(crate) fn heuristic_pattern_limits( + &mut self, + yes: bool, + ) -> &mut Builder { + self.heuristic_pattern_limits = yes; + self + } + + fn build_imp(&self, patterns: Arc) -> Option { + let patlimit = self.heuristic_pattern_limits; + // There's no particular reason why we limit ourselves to little endian + // here, but it seems likely that some parts of Teddy as they are + // currently written (e.g., the uses of `trailing_zeros`) are likely + // wrong on non-little-endian targets. Such things are likely easy to + // fix, but at the time of writing (2023/09/18), I actually do not know + // how to test this code on a big-endian target. So for now, we're + // conservative and just bail out. + if !cfg!(target_endian = "little") { + debug!("skipping Teddy because target isn't little endian"); + return None; + } + // Too many patterns will overwhelm Teddy and likely lead to slow + // downs, typically in the verification step. + if patlimit && patterns.len() > 64 { + debug!("skipping Teddy because of too many patterns"); + return None; + } + + #[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] + { + use self::x86_64::{FatAVX2, SlimAVX2, SlimSSSE3}; + + let mask_len = core::cmp::min(4, patterns.minimum_len()); + let beefy = patterns.len() > 32; + let has_avx2 = self::x86_64::is_available_avx2(); + let has_ssse3 = has_avx2 || self::x86_64::is_available_ssse3(); + let use_avx2 = if self.only_256bit == Some(true) { + if !has_avx2 { + debug!( + "skipping Teddy because avx2 was demanded but unavailable" + ); + return None; + } + true + } else if self.only_256bit == Some(false) { + if !has_ssse3 { + debug!( + "skipping Teddy because ssse3 was demanded but unavailable" + ); + return None; + } + false + } else if !has_ssse3 && !has_avx2 { + debug!( + "skipping Teddy because ssse3 and avx2 are unavailable" + ); + return None; + } else { + has_avx2 + }; + let fat = match self.only_fat { + None => use_avx2 && beefy, + Some(false) => false, + Some(true) if !use_avx2 => { + debug!( + "skipping Teddy because fat was demanded, but fat \ + Teddy requires avx2 which is unavailable" + ); + return None; + } + Some(true) => true, + }; + // Just like for aarch64, it's possible that too many patterns will + // overhwelm Teddy. Unlike aarch64 though, we have Fat teddy which + // helps things scale a bit more by spreading patterns over more + // buckets. + // + // These thresholds were determined by looking at the measurements + // for the rust/aho-corasick/packed/leftmost-first and + // rust/aho-corasick/dfa/leftmost-first engines on the `teddy/` + // benchmarks. + if patlimit && mask_len == 1 && patterns.len() > 16 { + debug!( + "skipping Teddy (mask len: 1) because there are \ + too many patterns", + ); + return None; + } + match (mask_len, use_avx2, fat) { + (1, false, _) => { + debug!("Teddy choice: 128-bit slim, 1 byte"); + SlimSSSE3::<1>::new(&patterns) + } + (1, true, false) => { + debug!("Teddy choice: 256-bit slim, 1 byte"); + SlimAVX2::<1>::new(&patterns) + } + (1, true, true) => { + debug!("Teddy choice: 256-bit fat, 1 byte"); + FatAVX2::<1>::new(&patterns) + } + (2, false, _) => { + debug!("Teddy choice: 128-bit slim, 2 bytes"); + SlimSSSE3::<2>::new(&patterns) + } + (2, true, false) => { + debug!("Teddy choice: 256-bit slim, 2 bytes"); + SlimAVX2::<2>::new(&patterns) + } + (2, true, true) => { + debug!("Teddy choice: 256-bit fat, 2 bytes"); + FatAVX2::<2>::new(&patterns) + } + (3, false, _) => { + debug!("Teddy choice: 128-bit slim, 3 bytes"); + SlimSSSE3::<3>::new(&patterns) + } + (3, true, false) => { + debug!("Teddy choice: 256-bit slim, 3 bytes"); + SlimAVX2::<3>::new(&patterns) + } + (3, true, true) => { + debug!("Teddy choice: 256-bit fat, 3 bytes"); + FatAVX2::<3>::new(&patterns) + } + (4, false, _) => { + debug!("Teddy choice: 128-bit slim, 4 bytes"); + SlimSSSE3::<4>::new(&patterns) + } + (4, true, false) => { + debug!("Teddy choice: 256-bit slim, 4 bytes"); + SlimAVX2::<4>::new(&patterns) + } + (4, true, true) => { + debug!("Teddy choice: 256-bit fat, 4 bytes"); + FatAVX2::<4>::new(&patterns) + } + _ => { + debug!("no supported Teddy configuration found"); + None + } + } + } + #[cfg(all( + target_arch = "aarch64", + target_feature = "neon", + target_endian = "little" + ))] + { + use self::aarch64::SlimNeon; + + let mask_len = core::cmp::min(4, patterns.minimum_len()); + if self.only_256bit == Some(true) { + debug!( + "skipping Teddy because 256-bits were demanded \ + but unavailable" + ); + return None; + } + if self.only_fat == Some(true) { + debug!( + "skipping Teddy because fat was demanded but unavailable" + ); + } + // Since we don't have Fat teddy in aarch64 (I think we'd want at + // least 256-bit vectors for that), we need to be careful not to + // allow too many patterns as it might overwhelm Teddy. Generally + // speaking, as the mask length goes up, the more patterns we can + // handle because the mask length results in fewer candidates + // generated. + // + // These thresholds were determined by looking at the measurements + // for the rust/aho-corasick/packed/leftmost-first and + // rust/aho-corasick/dfa/leftmost-first engines on the `teddy/` + // benchmarks. + match mask_len { + 1 => { + if patlimit && patterns.len() > 16 { + debug!( + "skipping Teddy (mask len: 1) because there are \ + too many patterns", + ); + } + debug!("Teddy choice: 128-bit slim, 1 byte"); + SlimNeon::<1>::new(&patterns) + } + 2 => { + if patlimit && patterns.len() > 32 { + debug!( + "skipping Teddy (mask len: 2) because there are \ + too many patterns", + ); + } + debug!("Teddy choice: 128-bit slim, 2 bytes"); + SlimNeon::<2>::new(&patterns) + } + 3 => { + if patlimit && patterns.len() > 48 { + debug!( + "skipping Teddy (mask len: 3) because there are \ + too many patterns", + ); + } + debug!("Teddy choice: 128-bit slim, 3 bytes"); + SlimNeon::<3>::new(&patterns) + } + 4 => { + debug!("Teddy choice: 128-bit slim, 4 bytes"); + SlimNeon::<4>::new(&patterns) + } + _ => { + debug!("no supported Teddy configuration found"); + None + } + } + } + #[cfg(not(any( + all(target_arch = "x86_64", target_feature = "sse2"), + all( + target_arch = "aarch64", + target_feature = "neon", + target_endian = "little" + ) + )))] + { + None + } + } +} + +/// A searcher that dispatches to one of several possible Teddy variants. +#[derive(Clone, Debug)] +pub(crate) struct Searcher { + /// The Teddy variant we use. We use dynamic dispatch under the theory that + /// it results in better codegen then a enum, although this is a specious + /// claim. + /// + /// This `Searcher` is essentially a wrapper for a `SearcherT` trait + /// object. We just make `memory_usage` and `minimum_len` available without + /// going through dynamic dispatch. + imp: Arc, + /// Total heap memory used by the Teddy variant. + memory_usage: usize, + /// The minimum haystack length this searcher can handle. It is intended + /// for callers to use some other search routine (such as Rabin-Karp) in + /// cases where the haystack (or remainer of the haystack) is too short. + minimum_len: usize, +} + +impl Searcher { + /// Look for the leftmost occurrence of any pattern in this search in the + /// given haystack starting at the given position. + /// + /// # Panics + /// + /// This panics when `haystack[at..].len()` is less than the minimum length + /// for this haystack. + #[inline(always)] + pub(crate) fn find( + &self, + haystack: &[u8], + at: usize, + ) -> Option { + // SAFETY: The Teddy implementations all require a minimum haystack + // length, and this is required for safety. Therefore, we assert it + // here in order to make this method sound. + assert!(haystack[at..].len() >= self.minimum_len); + let hayptr = haystack.as_ptr(); + // SAFETY: Construction of the searcher guarantees that we are able + // to run it in the current environment (i.e., we won't get an AVX2 + // searcher on a x86-64 CPU without AVX2 support). Also, the pointers + // are valid as they are derived directly from a borrowed slice. + let teddym = unsafe { + self.imp.find(hayptr.add(at), hayptr.add(haystack.len()))? + }; + let start = teddym.start().as_usize().wrapping_sub(hayptr.as_usize()); + let end = teddym.end().as_usize().wrapping_sub(hayptr.as_usize()); + let span = crate::Span { start, end }; + // OK because we won't permit the construction of a searcher that + // could report a pattern ID bigger than what can fit in the crate-wide + // PatternID type. + let pid = crate::PatternID::new_unchecked(teddym.pattern().as_usize()); + let m = crate::Match::new(pid, span); + Some(m) + } + + /// Returns the approximate total amount of heap used by this type, in + /// units of bytes. + #[inline(always)] + pub(crate) fn memory_usage(&self) -> usize { + self.memory_usage + } + + /// Returns the minimum length, in bytes, that a haystack must be in order + /// to use it with this searcher. + #[inline(always)] + pub(crate) fn minimum_len(&self) -> usize { + self.minimum_len + } +} + +/// A trait that provides dynamic dispatch over the different possible Teddy +/// variants on the same algorithm. +/// +/// On `x86_64` for example, it isn't known until runtime which of 12 possible +/// variants will be used. One might use one of the four slim 128-bit vector +/// variants, or one of the four 256-bit vector variants or even one of the +/// four fat 256-bit vector variants. +/// +/// Since this choice is generally made when the Teddy searcher is constructed +/// and this choice is based on the patterns given and what the current CPU +/// supports, it follows that there must be some kind of indirection at search +/// time that "selects" the variant chosen at build time. +/// +/// There are a few different ways to go about this. One approach is to use an +/// enum. It works fine, but in my experiments, this generally results in worse +/// codegen. Another approach, which is what we use here, is dynamic dispatch +/// via a trait object. We basically implement this trait for each possible +/// variant, select the variant we want at build time and convert it to a +/// trait object for use at search time. +/// +/// Another approach is to use function pointers and stick each of the possible +/// variants into a union. This is essentially isomorphic to the dynamic +/// dispatch approach, but doesn't require any allocations. Since this crate +/// requires `alloc`, there's no real reason (AFAIK) to go down this path. (The +/// `memchr` crate does this.) +trait SearcherT: + Debug + Send + Sync + UnwindSafe + RefUnwindSafe + 'static +{ + /// Execute a search on the given haystack (identified by `start` and `end` + /// raw pointers). + /// + /// # Safety + /// + /// Essentially, the `start` and `end` pointers must be valid and point + /// to a haystack one can read. As long as you derive them from, for + /// example, a `&[u8]`, they should automatically satisfy all of the safety + /// obligations: + /// + /// * Both `start` and `end` must be valid for reads. + /// * Both `start` and `end` must point to an initialized value. + /// * Both `start` and `end` must point to the same allocated object and + /// must either be in bounds or at most one byte past the end of the + /// allocated object. + /// * Both `start` and `end` must be _derived from_ a pointer to the same + /// object. + /// * The distance between `start` and `end` must not overflow `isize`. + /// * The distance being in bounds must not rely on "wrapping around" the + /// address space. + /// * It must be the case that `start <= end`. + /// * `end - start` must be greater than the minimum length for this + /// searcher. + /// + /// Also, it is expected that implementations of this trait will tag this + /// method with a `target_feature` attribute. Callers must ensure that + /// they are executing this method in an environment where that attribute + /// is valid. + unsafe fn find(&self, start: *const u8, end: *const u8) -> Option; +} + +#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] +mod x86_64 { + use core::arch::x86_64::{__m128i, __m256i}; + + use alloc::sync::Arc; + + use crate::packed::{ + ext::Pointer, + pattern::Patterns, + teddy::generic::{self, Match}, + }; + + use super::{Searcher, SearcherT}; + + #[derive(Clone, Debug)] + pub(super) struct SlimSSSE3 { + slim128: generic::Slim<__m128i, BYTES>, + } + + // Defines SlimSSSE3 wrapper functions for 1, 2, 3 and 4 bytes. + macro_rules! slim_ssse3 { + ($len:expr) => { + impl SlimSSSE3<$len> { + /// Creates a new searcher using "slim" Teddy with 128-bit + /// vectors. If SSSE3 is not available in the current + /// environment, then this returns `None`. + pub(super) fn new( + patterns: &Arc, + ) -> Option { + if !is_available_ssse3() { + return None; + } + Some(unsafe { SlimSSSE3::<$len>::new_unchecked(patterns) }) + } + + /// Creates a new searcher using "slim" Teddy with 256-bit + /// vectors without checking whether SSSE3 is available or not. + /// + /// # Safety + /// + /// Callers must ensure that SSSE3 is available in the current + /// environment. + #[target_feature(enable = "ssse3")] + unsafe fn new_unchecked(patterns: &Arc) -> Searcher { + let slim128 = generic::Slim::<__m128i, $len>::new( + Arc::clone(patterns), + ); + let memory_usage = slim128.memory_usage(); + let minimum_len = slim128.minimum_len(); + let imp = Arc::new(SlimSSSE3 { slim128 }); + Searcher { imp, memory_usage, minimum_len } + } + } + + impl SearcherT for SlimSSSE3<$len> { + #[target_feature(enable = "ssse3")] + #[inline] + unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + // SAFETY: All obligations except for `target_feature` are + // passed to the caller. Our use of `target_feature` is + // safe because construction of this type requires that the + // requisite target features are available. + self.slim128.find(start, end) + } + } + }; + } + + slim_ssse3!(1); + slim_ssse3!(2); + slim_ssse3!(3); + slim_ssse3!(4); + + #[derive(Clone, Debug)] + pub(super) struct SlimAVX2 { + slim128: generic::Slim<__m128i, BYTES>, + slim256: generic::Slim<__m256i, BYTES>, + } + + // Defines SlimAVX2 wrapper functions for 1, 2, 3 and 4 bytes. + macro_rules! slim_avx2 { + ($len:expr) => { + impl SlimAVX2<$len> { + /// Creates a new searcher using "slim" Teddy with 256-bit + /// vectors. If AVX2 is not available in the current + /// environment, then this returns `None`. + pub(super) fn new( + patterns: &Arc, + ) -> Option { + if !is_available_avx2() { + return None; + } + Some(unsafe { SlimAVX2::<$len>::new_unchecked(patterns) }) + } + + /// Creates a new searcher using "slim" Teddy with 256-bit + /// vectors without checking whether AVX2 is available or not. + /// + /// # Safety + /// + /// Callers must ensure that AVX2 is available in the current + /// environment. + #[target_feature(enable = "avx2")] + unsafe fn new_unchecked(patterns: &Arc) -> Searcher { + let slim128 = generic::Slim::<__m128i, $len>::new( + Arc::clone(&patterns), + ); + let slim256 = generic::Slim::<__m256i, $len>::new( + Arc::clone(&patterns), + ); + let memory_usage = + slim128.memory_usage() + slim256.memory_usage(); + let minimum_len = slim128.minimum_len(); + let imp = Arc::new(SlimAVX2 { slim128, slim256 }); + Searcher { imp, memory_usage, minimum_len } + } + } + + impl SearcherT for SlimAVX2<$len> { + #[target_feature(enable = "avx2")] + #[inline] + unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + // SAFETY: All obligations except for `target_feature` are + // passed to the caller. Our use of `target_feature` is + // safe because construction of this type requires that the + // requisite target features are available. + let len = end.distance(start); + if len < self.slim256.minimum_len() { + self.slim128.find(start, end) + } else { + self.slim256.find(start, end) + } + } + } + }; + } + + slim_avx2!(1); + slim_avx2!(2); + slim_avx2!(3); + slim_avx2!(4); + + #[derive(Clone, Debug)] + pub(super) struct FatAVX2 { + fat256: generic::Fat<__m256i, BYTES>, + } + + // Defines SlimAVX2 wrapper functions for 1, 2, 3 and 4 bytes. + macro_rules! fat_avx2 { + ($len:expr) => { + impl FatAVX2<$len> { + /// Creates a new searcher using "slim" Teddy with 256-bit + /// vectors. If AVX2 is not available in the current + /// environment, then this returns `None`. + pub(super) fn new( + patterns: &Arc, + ) -> Option { + if !is_available_avx2() { + return None; + } + Some(unsafe { FatAVX2::<$len>::new_unchecked(patterns) }) + } + + /// Creates a new searcher using "slim" Teddy with 256-bit + /// vectors without checking whether AVX2 is available or not. + /// + /// # Safety + /// + /// Callers must ensure that AVX2 is available in the current + /// environment. + #[target_feature(enable = "avx2")] + unsafe fn new_unchecked(patterns: &Arc) -> Searcher { + let fat256 = generic::Fat::<__m256i, $len>::new( + Arc::clone(&patterns), + ); + let memory_usage = fat256.memory_usage(); + let minimum_len = fat256.minimum_len(); + let imp = Arc::new(FatAVX2 { fat256 }); + Searcher { imp, memory_usage, minimum_len } + } + } + + impl SearcherT for FatAVX2<$len> { + #[target_feature(enable = "avx2")] + #[inline] + unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + // SAFETY: All obligations except for `target_feature` are + // passed to the caller. Our use of `target_feature` is + // safe because construction of this type requires that the + // requisite target features are available. + self.fat256.find(start, end) + } + } + }; + } + + fat_avx2!(1); + fat_avx2!(2); + fat_avx2!(3); + fat_avx2!(4); + + #[inline] + pub(super) fn is_available_ssse3() -> bool { + #[cfg(not(target_feature = "sse2"))] + { + false + } + #[cfg(target_feature = "sse2")] + { + #[cfg(target_feature = "ssse3")] + { + true + } + #[cfg(not(target_feature = "ssse3"))] + { + #[cfg(feature = "std")] + { + std::is_x86_feature_detected!("ssse3") + } + #[cfg(not(feature = "std"))] + { + false + } + } + } + } + + #[inline] + pub(super) fn is_available_avx2() -> bool { + #[cfg(not(target_feature = "sse2"))] + { + false + } + #[cfg(target_feature = "sse2")] + { + #[cfg(target_feature = "avx2")] + { + true + } + #[cfg(not(target_feature = "avx2"))] + { + #[cfg(feature = "std")] + { + std::is_x86_feature_detected!("avx2") + } + #[cfg(not(feature = "std"))] + { + false + } + } + } + } +} + +#[cfg(all( + target_arch = "aarch64", + target_feature = "neon", + target_endian = "little" +))] +mod aarch64 { + use core::arch::aarch64::uint8x16_t; + + use alloc::sync::Arc; + + use crate::packed::{ + pattern::Patterns, + teddy::generic::{self, Match}, + }; + + use super::{Searcher, SearcherT}; + + #[derive(Clone, Debug)] + pub(super) struct SlimNeon { + slim128: generic::Slim, + } + + // Defines SlimSSSE3 wrapper functions for 1, 2, 3 and 4 bytes. + macro_rules! slim_neon { + ($len:expr) => { + impl SlimNeon<$len> { + /// Creates a new searcher using "slim" Teddy with 128-bit + /// vectors. If SSSE3 is not available in the current + /// environment, then this returns `None`. + pub(super) fn new( + patterns: &Arc, + ) -> Option { + Some(unsafe { SlimNeon::<$len>::new_unchecked(patterns) }) + } + + /// Creates a new searcher using "slim" Teddy with 256-bit + /// vectors without checking whether SSSE3 is available or not. + /// + /// # Safety + /// + /// Callers must ensure that SSSE3 is available in the current + /// environment. + #[target_feature(enable = "neon")] + unsafe fn new_unchecked(patterns: &Arc) -> Searcher { + let slim128 = generic::Slim::::new( + Arc::clone(patterns), + ); + let memory_usage = slim128.memory_usage(); + let minimum_len = slim128.minimum_len(); + let imp = Arc::new(SlimNeon { slim128 }); + Searcher { imp, memory_usage, minimum_len } + } + } + + impl SearcherT for SlimNeon<$len> { + #[target_feature(enable = "neon")] + #[inline] + unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + // SAFETY: All obligations except for `target_feature` are + // passed to the caller. Our use of `target_feature` is + // safe because construction of this type requires that the + // requisite target features are available. + self.slim128.find(start, end) + } + } + }; + } + + slim_neon!(1); + slim_neon!(2); + slim_neon!(3); + slim_neon!(4); +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/generic.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/generic.rs new file mode 100644 index 0000000..2aacd00 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/generic.rs @@ -0,0 +1,1382 @@ +use core::fmt::Debug; + +use alloc::{ + boxed::Box, collections::BTreeMap, format, sync::Arc, vec, vec::Vec, +}; + +use crate::{ + packed::{ + ext::Pointer, + pattern::Patterns, + vector::{FatVector, Vector}, + }, + util::int::U32, + PatternID, +}; + +/// A match type specialized to the Teddy implementations below. +/// +/// Essentially, instead of representing a match at byte offsets, we use +/// raw pointers. This is because the implementations below operate on raw +/// pointers, and so this is a more natural return type based on how the +/// implementation works. +/// +/// Also, the `PatternID` used here is a `u16`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Match { + pid: PatternID, + start: *const u8, + end: *const u8, +} + +impl Match { + /// Returns the ID of the pattern that matched. + pub(crate) fn pattern(&self) -> PatternID { + self.pid + } + + /// Returns a pointer into the haystack at which the match starts. + pub(crate) fn start(&self) -> *const u8 { + self.start + } + + /// Returns a pointer into the haystack at which the match ends. + pub(crate) fn end(&self) -> *const u8 { + self.end + } +} + +/// A "slim" Teddy implementation that is generic over both the vector type +/// and the minimum length of the patterns being searched for. +/// +/// Only 1, 2, 3 and 4 bytes are supported as minimum lengths. +#[derive(Clone, Debug)] +pub(crate) struct Slim { + /// A generic data structure for doing "slim" Teddy verification. + teddy: Teddy<8>, + /// The masks used as inputs to the shuffle operation to generate + /// candidates (which are fed into the verification routines). + masks: [Mask; BYTES], +} + +impl Slim { + /// Create a new "slim" Teddy searcher for the given patterns. + /// + /// # Panics + /// + /// This panics when `BYTES` is any value other than 1, 2, 3 or 4. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + pub(crate) unsafe fn new(patterns: Arc) -> Slim { + assert!( + 1 <= BYTES && BYTES <= 4, + "only 1, 2, 3 or 4 bytes are supported" + ); + let teddy = Teddy::new(patterns); + let masks = SlimMaskBuilder::from_teddy(&teddy); + Slim { teddy, masks } + } + + /// Returns the approximate total amount of heap used by this type, in + /// units of bytes. + #[inline(always)] + pub(crate) fn memory_usage(&self) -> usize { + self.teddy.memory_usage() + } + + /// Returns the minimum length, in bytes, that a haystack must be in order + /// to use it with this searcher. + #[inline(always)] + pub(crate) fn minimum_len(&self) -> usize { + V::BYTES + (BYTES - 1) + } +} + +impl Slim { + /// Look for an occurrences of the patterns in this finder in the haystack + /// given by the `start` and `end` pointers. + /// + /// If no match could be found, then `None` is returned. + /// + /// # Safety + /// + /// The given pointers representing the haystack must be valid to read + /// from. They must also point to a region of memory that is at least the + /// minimum length required by this searcher. + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + pub(crate) unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + let len = end.distance(start); + debug_assert!(len >= self.minimum_len()); + let mut cur = start; + while cur <= end.sub(V::BYTES) { + if let Some(m) = self.find_one(cur, end) { + return Some(m); + } + cur = cur.add(V::BYTES); + } + if cur < end { + cur = end.sub(V::BYTES); + if let Some(m) = self.find_one(cur, end) { + return Some(m); + } + } + None + } + + /// Look for a match starting at the `V::BYTES` at and after `cur`. If + /// there isn't one, then `None` is returned. + /// + /// # Safety + /// + /// The given pointers representing the haystack must be valid to read + /// from. They must also point to a region of memory that is at least the + /// minimum length required by this searcher. + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + unsafe fn find_one( + &self, + cur: *const u8, + end: *const u8, + ) -> Option { + let c = self.candidate(cur); + if !c.is_zero() { + if let Some(m) = self.teddy.verify(cur, end, c) { + return Some(m); + } + } + None + } + + /// Look for a candidate match (represented as a vector) starting at the + /// `V::BYTES` at and after `cur`. If there isn't one, then a vector with + /// all bits set to zero is returned. + /// + /// # Safety + /// + /// The given pointer representing the haystack must be valid to read + /// from. + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + unsafe fn candidate(&self, cur: *const u8) -> V { + let chunk = V::load_unaligned(cur); + Mask::members1(chunk, self.masks) + } +} + +impl Slim { + /// See Slim::find. + #[inline(always)] + pub(crate) unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + let len = end.distance(start); + debug_assert!(len >= self.minimum_len()); + let mut cur = start.add(1); + let mut prev0 = V::splat(0xFF); + while cur <= end.sub(V::BYTES) { + if let Some(m) = self.find_one(cur, end, &mut prev0) { + return Some(m); + } + cur = cur.add(V::BYTES); + } + if cur < end { + cur = end.sub(V::BYTES); + prev0 = V::splat(0xFF); + if let Some(m) = self.find_one(cur, end, &mut prev0) { + return Some(m); + } + } + None + } + + /// See Slim::find_one. + #[inline(always)] + unsafe fn find_one( + &self, + cur: *const u8, + end: *const u8, + prev0: &mut V, + ) -> Option { + let c = self.candidate(cur, prev0); + if !c.is_zero() { + if let Some(m) = self.teddy.verify(cur.sub(1), end, c) { + return Some(m); + } + } + None + } + + /// See Slim::candidate. + #[inline(always)] + unsafe fn candidate(&self, cur: *const u8, prev0: &mut V) -> V { + let chunk = V::load_unaligned(cur); + let (res0, res1) = Mask::members2(chunk, self.masks); + let res0prev0 = res0.shift_in_one_byte(*prev0); + let res = res0prev0.and(res1); + *prev0 = res0; + res + } +} + +impl Slim { + /// See Slim::find. + #[inline(always)] + pub(crate) unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + let len = end.distance(start); + debug_assert!(len >= self.minimum_len()); + let mut cur = start.add(2); + let mut prev0 = V::splat(0xFF); + let mut prev1 = V::splat(0xFF); + while cur <= end.sub(V::BYTES) { + if let Some(m) = self.find_one(cur, end, &mut prev0, &mut prev1) { + return Some(m); + } + cur = cur.add(V::BYTES); + } + if cur < end { + cur = end.sub(V::BYTES); + prev0 = V::splat(0xFF); + prev1 = V::splat(0xFF); + if let Some(m) = self.find_one(cur, end, &mut prev0, &mut prev1) { + return Some(m); + } + } + None + } + + /// See Slim::find_one. + #[inline(always)] + unsafe fn find_one( + &self, + cur: *const u8, + end: *const u8, + prev0: &mut V, + prev1: &mut V, + ) -> Option { + let c = self.candidate(cur, prev0, prev1); + if !c.is_zero() { + if let Some(m) = self.teddy.verify(cur.sub(2), end, c) { + return Some(m); + } + } + None + } + + /// See Slim::candidate. + #[inline(always)] + unsafe fn candidate( + &self, + cur: *const u8, + prev0: &mut V, + prev1: &mut V, + ) -> V { + let chunk = V::load_unaligned(cur); + let (res0, res1, res2) = Mask::members3(chunk, self.masks); + let res0prev0 = res0.shift_in_two_bytes(*prev0); + let res1prev1 = res1.shift_in_one_byte(*prev1); + let res = res0prev0.and(res1prev1).and(res2); + *prev0 = res0; + *prev1 = res1; + res + } +} + +impl Slim { + /// See Slim::find. + #[inline(always)] + pub(crate) unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + let len = end.distance(start); + debug_assert!(len >= self.minimum_len()); + let mut cur = start.add(3); + let mut prev0 = V::splat(0xFF); + let mut prev1 = V::splat(0xFF); + let mut prev2 = V::splat(0xFF); + while cur <= end.sub(V::BYTES) { + if let Some(m) = + self.find_one(cur, end, &mut prev0, &mut prev1, &mut prev2) + { + return Some(m); + } + cur = cur.add(V::BYTES); + } + if cur < end { + cur = end.sub(V::BYTES); + prev0 = V::splat(0xFF); + prev1 = V::splat(0xFF); + prev2 = V::splat(0xFF); + if let Some(m) = + self.find_one(cur, end, &mut prev0, &mut prev1, &mut prev2) + { + return Some(m); + } + } + None + } + + /// See Slim::find_one. + #[inline(always)] + unsafe fn find_one( + &self, + cur: *const u8, + end: *const u8, + prev0: &mut V, + prev1: &mut V, + prev2: &mut V, + ) -> Option { + let c = self.candidate(cur, prev0, prev1, prev2); + if !c.is_zero() { + if let Some(m) = self.teddy.verify(cur.sub(3), end, c) { + return Some(m); + } + } + None + } + + /// See Slim::candidate. + #[inline(always)] + unsafe fn candidate( + &self, + cur: *const u8, + prev0: &mut V, + prev1: &mut V, + prev2: &mut V, + ) -> V { + let chunk = V::load_unaligned(cur); + let (res0, res1, res2, res3) = Mask::members4(chunk, self.masks); + let res0prev0 = res0.shift_in_three_bytes(*prev0); + let res1prev1 = res1.shift_in_two_bytes(*prev1); + let res2prev2 = res2.shift_in_one_byte(*prev2); + let res = res0prev0.and(res1prev1).and(res2prev2).and(res3); + *prev0 = res0; + *prev1 = res1; + *prev2 = res2; + res + } +} + +/// A "fat" Teddy implementation that is generic over both the vector type +/// and the minimum length of the patterns being searched for. +/// +/// Only 1, 2, 3 and 4 bytes are supported as minimum lengths. +#[derive(Clone, Debug)] +pub(crate) struct Fat { + /// A generic data structure for doing "fat" Teddy verification. + teddy: Teddy<16>, + /// The masks used as inputs to the shuffle operation to generate + /// candidates (which are fed into the verification routines). + masks: [Mask; BYTES], +} + +impl Fat { + /// Create a new "fat" Teddy searcher for the given patterns. + /// + /// # Panics + /// + /// This panics when `BYTES` is any value other than 1, 2, 3 or 4. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + pub(crate) unsafe fn new(patterns: Arc) -> Fat { + assert!( + 1 <= BYTES && BYTES <= 4, + "only 1, 2, 3 or 4 bytes are supported" + ); + let teddy = Teddy::new(patterns); + let masks = FatMaskBuilder::from_teddy(&teddy); + Fat { teddy, masks } + } + + /// Returns the approximate total amount of heap used by this type, in + /// units of bytes. + #[inline(always)] + pub(crate) fn memory_usage(&self) -> usize { + self.teddy.memory_usage() + } + + /// Returns the minimum length, in bytes, that a haystack must be in order + /// to use it with this searcher. + #[inline(always)] + pub(crate) fn minimum_len(&self) -> usize { + V::Half::BYTES + (BYTES - 1) + } +} + +impl Fat { + /// Look for an occurrences of the patterns in this finder in the haystack + /// given by the `start` and `end` pointers. + /// + /// If no match could be found, then `None` is returned. + /// + /// # Safety + /// + /// The given pointers representing the haystack must be valid to read + /// from. They must also point to a region of memory that is at least the + /// minimum length required by this searcher. + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + pub(crate) unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + let len = end.distance(start); + debug_assert!(len >= self.minimum_len()); + let mut cur = start; + while cur <= end.sub(V::Half::BYTES) { + if let Some(m) = self.find_one(cur, end) { + return Some(m); + } + cur = cur.add(V::Half::BYTES); + } + if cur < end { + cur = end.sub(V::Half::BYTES); + if let Some(m) = self.find_one(cur, end) { + return Some(m); + } + } + None + } + + /// Look for a match starting at the `V::BYTES` at and after `cur`. If + /// there isn't one, then `None` is returned. + /// + /// # Safety + /// + /// The given pointers representing the haystack must be valid to read + /// from. They must also point to a region of memory that is at least the + /// minimum length required by this searcher. + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + unsafe fn find_one( + &self, + cur: *const u8, + end: *const u8, + ) -> Option { + let c = self.candidate(cur); + if !c.is_zero() { + if let Some(m) = self.teddy.verify(cur, end, c) { + return Some(m); + } + } + None + } + + /// Look for a candidate match (represented as a vector) starting at the + /// `V::BYTES` at and after `cur`. If there isn't one, then a vector with + /// all bits set to zero is returned. + /// + /// # Safety + /// + /// The given pointer representing the haystack must be valid to read + /// from. + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + unsafe fn candidate(&self, cur: *const u8) -> V { + let chunk = V::load_half_unaligned(cur); + Mask::members1(chunk, self.masks) + } +} + +impl Fat { + /// See `Fat::find`. + #[inline(always)] + pub(crate) unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + let len = end.distance(start); + debug_assert!(len >= self.minimum_len()); + let mut cur = start.add(1); + let mut prev0 = V::splat(0xFF); + while cur <= end.sub(V::Half::BYTES) { + if let Some(m) = self.find_one(cur, end, &mut prev0) { + return Some(m); + } + cur = cur.add(V::Half::BYTES); + } + if cur < end { + cur = end.sub(V::Half::BYTES); + prev0 = V::splat(0xFF); + if let Some(m) = self.find_one(cur, end, &mut prev0) { + return Some(m); + } + } + None + } + + /// See `Fat::find_one`. + #[inline(always)] + unsafe fn find_one( + &self, + cur: *const u8, + end: *const u8, + prev0: &mut V, + ) -> Option { + let c = self.candidate(cur, prev0); + if !c.is_zero() { + if let Some(m) = self.teddy.verify(cur.sub(1), end, c) { + return Some(m); + } + } + None + } + + /// See `Fat::candidate`. + #[inline(always)] + unsafe fn candidate(&self, cur: *const u8, prev0: &mut V) -> V { + let chunk = V::load_half_unaligned(cur); + let (res0, res1) = Mask::members2(chunk, self.masks); + let res0prev0 = res0.half_shift_in_one_byte(*prev0); + let res = res0prev0.and(res1); + *prev0 = res0; + res + } +} + +impl Fat { + /// See `Fat::find`. + #[inline(always)] + pub(crate) unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + let len = end.distance(start); + debug_assert!(len >= self.minimum_len()); + let mut cur = start.add(2); + let mut prev0 = V::splat(0xFF); + let mut prev1 = V::splat(0xFF); + while cur <= end.sub(V::Half::BYTES) { + if let Some(m) = self.find_one(cur, end, &mut prev0, &mut prev1) { + return Some(m); + } + cur = cur.add(V::Half::BYTES); + } + if cur < end { + cur = end.sub(V::Half::BYTES); + prev0 = V::splat(0xFF); + prev1 = V::splat(0xFF); + if let Some(m) = self.find_one(cur, end, &mut prev0, &mut prev1) { + return Some(m); + } + } + None + } + + /// See `Fat::find_one`. + #[inline(always)] + unsafe fn find_one( + &self, + cur: *const u8, + end: *const u8, + prev0: &mut V, + prev1: &mut V, + ) -> Option { + let c = self.candidate(cur, prev0, prev1); + if !c.is_zero() { + if let Some(m) = self.teddy.verify(cur.sub(2), end, c) { + return Some(m); + } + } + None + } + + /// See `Fat::candidate`. + #[inline(always)] + unsafe fn candidate( + &self, + cur: *const u8, + prev0: &mut V, + prev1: &mut V, + ) -> V { + let chunk = V::load_half_unaligned(cur); + let (res0, res1, res2) = Mask::members3(chunk, self.masks); + let res0prev0 = res0.half_shift_in_two_bytes(*prev0); + let res1prev1 = res1.half_shift_in_one_byte(*prev1); + let res = res0prev0.and(res1prev1).and(res2); + *prev0 = res0; + *prev1 = res1; + res + } +} + +impl Fat { + /// See `Fat::find`. + #[inline(always)] + pub(crate) unsafe fn find( + &self, + start: *const u8, + end: *const u8, + ) -> Option { + let len = end.distance(start); + debug_assert!(len >= self.minimum_len()); + let mut cur = start.add(3); + let mut prev0 = V::splat(0xFF); + let mut prev1 = V::splat(0xFF); + let mut prev2 = V::splat(0xFF); + while cur <= end.sub(V::Half::BYTES) { + if let Some(m) = + self.find_one(cur, end, &mut prev0, &mut prev1, &mut prev2) + { + return Some(m); + } + cur = cur.add(V::Half::BYTES); + } + if cur < end { + cur = end.sub(V::Half::BYTES); + prev0 = V::splat(0xFF); + prev1 = V::splat(0xFF); + prev2 = V::splat(0xFF); + if let Some(m) = + self.find_one(cur, end, &mut prev0, &mut prev1, &mut prev2) + { + return Some(m); + } + } + None + } + + /// See `Fat::find_one`. + #[inline(always)] + unsafe fn find_one( + &self, + cur: *const u8, + end: *const u8, + prev0: &mut V, + prev1: &mut V, + prev2: &mut V, + ) -> Option { + let c = self.candidate(cur, prev0, prev1, prev2); + if !c.is_zero() { + if let Some(m) = self.teddy.verify(cur.sub(3), end, c) { + return Some(m); + } + } + None + } + + /// See `Fat::candidate`. + #[inline(always)] + unsafe fn candidate( + &self, + cur: *const u8, + prev0: &mut V, + prev1: &mut V, + prev2: &mut V, + ) -> V { + let chunk = V::load_half_unaligned(cur); + let (res0, res1, res2, res3) = Mask::members4(chunk, self.masks); + let res0prev0 = res0.half_shift_in_three_bytes(*prev0); + let res1prev1 = res1.half_shift_in_two_bytes(*prev1); + let res2prev2 = res2.half_shift_in_one_byte(*prev2); + let res = res0prev0.and(res1prev1).and(res2prev2).and(res3); + *prev0 = res0; + *prev1 = res1; + *prev2 = res2; + res + } +} + +/// The common elements of all "slim" and "fat" Teddy search implementations. +/// +/// Essentially, this contains the patterns and the buckets. Namely, it +/// contains enough to implement the verification step after candidates are +/// identified via the shuffle masks. +/// +/// It is generic over the number of buckets used. In general, the number of +/// buckets is either 8 (for "slim" Teddy) or 16 (for "fat" Teddy). The generic +/// parameter isn't really meant to be instantiated for any value other than +/// 8 or 16, although it is technically possible. The main hiccup is that there +/// is some bit-shifting done in the critical part of verification that could +/// be quite expensive if `N` is not a multiple of 2. +#[derive(Clone, Debug)] +struct Teddy { + /// The patterns we are searching for. + /// + /// A pattern string can be found by its `PatternID`. + patterns: Arc, + /// The allocation of patterns in buckets. This only contains the IDs of + /// patterns. In order to do full verification, callers must provide the + /// actual patterns when using Teddy. + buckets: [Vec; BUCKETS], + // N.B. The above representation is very simple, but it definitely results + // in ping-ponging between different allocations during verification. I've + // tried experimenting with other representations that flatten the pattern + // strings into a single allocation, but it doesn't seem to help much. + // Probably everything is small enough to fit into cache anyway, and so the + // pointer chasing isn't a big deal? + // + // One other avenue I haven't explored is some kind of hashing trick + // that let's us do another high-confidence check before launching into + // `memcmp`. +} + +impl Teddy { + /// Create a new generic data structure for Teddy verification. + fn new(patterns: Arc) -> Teddy { + assert_ne!(0, patterns.len(), "Teddy requires at least one pattern"); + assert_ne!( + 0, + patterns.minimum_len(), + "Teddy does not support zero-length patterns" + ); + assert!( + BUCKETS == 8 || BUCKETS == 16, + "Teddy only supports 8 or 16 buckets" + ); + // MSRV(1.63): Use core::array::from_fn below instead of allocating a + // superfluous outer Vec. Not a big deal (especially given the BTreeMap + // allocation below), but nice to not do it. + let buckets = + <[Vec; BUCKETS]>::try_from(vec![vec![]; BUCKETS]) + .unwrap(); + let mut t = Teddy { patterns, buckets }; + + let mut map: BTreeMap, usize> = BTreeMap::new(); + for (id, pattern) in t.patterns.iter() { + // We try to be slightly clever in how we assign patterns into + // buckets. Generally speaking, we want patterns with the same + // prefix to be in the same bucket, since it minimizes the amount + // of time we spend churning through buckets in the verification + // step. + // + // So we could assign patterns with the same N-prefix (where N is + // the size of the mask, which is one of {1, 2, 3}) to the same + // bucket. However, case insensitive searches are fairly common, so + // we'd for example, ideally want to treat `abc` and `ABC` as if + // they shared the same prefix. ASCII has the nice property that + // the lower 4 bits of A and a are the same, so we therefore group + // patterns with the same low-nybble-N-prefix into the same bucket. + // + // MOREOVER, this is actually necessary for correctness! In + // particular, by grouping patterns with the same prefix into the + // same bucket, we ensure that we preserve correct leftmost-first + // and leftmost-longest match semantics. In addition to the fact + // that `patterns.iter()` iterates in the correct order, this + // guarantees that all possible ambiguous matches will occur in + // the same bucket. The verification routine could be adjusted to + // support correct leftmost match semantics regardless of bucket + // allocation, but that results in a performance hit. It's much + // nicer to be able to just stop as soon as a match is found. + let lonybs = pattern.low_nybbles(t.mask_len()); + if let Some(&bucket) = map.get(&lonybs) { + t.buckets[bucket].push(id); + } else { + // N.B. We assign buckets in reverse because it shouldn't have + // any influence on performance, but it does make it harder to + // get leftmost match semantics accidentally correct. + let bucket = (BUCKETS - 1) - (id.as_usize() % BUCKETS); + t.buckets[bucket].push(id); + map.insert(lonybs, bucket); + } + } + t + } + + /// Verify whether there are any matches starting at or after `cur` in the + /// haystack. The candidate chunk given should correspond to 8-bit bitsets + /// for N buckets. + /// + /// # Safety + /// + /// The given pointers representing the haystack must be valid to read + /// from. + #[inline(always)] + unsafe fn verify64( + &self, + cur: *const u8, + end: *const u8, + mut candidate_chunk: u64, + ) -> Option { + while candidate_chunk != 0 { + let bit = candidate_chunk.trailing_zeros().as_usize(); + candidate_chunk &= !(1 << bit); + + let cur = cur.add(bit / BUCKETS); + let bucket = bit % BUCKETS; + if let Some(m) = self.verify_bucket(cur, end, bucket) { + return Some(m); + } + } + None + } + + /// Verify whether there are any matches starting at `at` in the given + /// `haystack` corresponding only to patterns in the given bucket. + /// + /// # Safety + /// + /// The given pointers representing the haystack must be valid to read + /// from. + /// + /// The bucket index must be less than or equal to `self.buckets.len()`. + #[inline(always)] + unsafe fn verify_bucket( + &self, + cur: *const u8, + end: *const u8, + bucket: usize, + ) -> Option { + debug_assert!(bucket < self.buckets.len()); + // SAFETY: The caller must ensure that the bucket index is correct. + for pid in self.buckets.get_unchecked(bucket).iter().copied() { + // SAFETY: This is safe because we are guaranteed that every + // index in a Teddy bucket is a valid index into `pats`, by + // construction. + debug_assert!(pid.as_usize() < self.patterns.len()); + let pat = self.patterns.get_unchecked(pid); + if pat.is_prefix_raw(cur, end) { + let start = cur; + let end = start.add(pat.len()); + return Some(Match { pid, start, end }); + } + } + None + } + + /// Returns the total number of masks required by the patterns in this + /// Teddy searcher. + /// + /// Basically, the mask length corresponds to the type of Teddy searcher + /// to use: a 1-byte, 2-byte, 3-byte or 4-byte searcher. The bigger the + /// better, typically, since searching for longer substrings usually + /// decreases the rate of false positives. Therefore, the number of masks + /// needed is the length of the shortest pattern in this searcher. If the + /// length of the shortest pattern (in bytes) is bigger than 4, then the + /// mask length is 4 since there are no Teddy searchers for more than 4 + /// bytes. + fn mask_len(&self) -> usize { + core::cmp::min(4, self.patterns.minimum_len()) + } + + /// Returns the approximate total amount of heap used by this type, in + /// units of bytes. + fn memory_usage(&self) -> usize { + // This is an upper bound rather than a precise accounting. No + // particular reason, other than it's probably very close to actual + // memory usage in practice. + self.patterns.len() * core::mem::size_of::() + } +} + +impl Teddy<8> { + /// Runs the verification routine for "slim" Teddy. + /// + /// The candidate given should be a collection of 8-bit bitsets (one bitset + /// per lane), where the ith bit is set in the jth lane if and only if the + /// byte occurring at `at + j` in `cur` is in the bucket `i`. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + /// + /// The given pointers must be valid to read from. + #[inline(always)] + unsafe fn verify( + &self, + mut cur: *const u8, + end: *const u8, + candidate: V, + ) -> Option { + debug_assert!(!candidate.is_zero()); + // Convert the candidate into 64-bit chunks, and then verify each of + // those chunks. + candidate.for_each_64bit_lane( + #[inline(always)] + |_, chunk| { + let result = self.verify64(cur, end, chunk); + cur = cur.add(8); + result + }, + ) + } +} + +impl Teddy<16> { + /// Runs the verification routine for "fat" Teddy. + /// + /// The candidate given should be a collection of 8-bit bitsets (one bitset + /// per lane), where the ith bit is set in the jth lane if and only if the + /// byte occurring at `at + (j < 16 ? j : j - 16)` in `cur` is in the + /// bucket `j < 16 ? i : i + 8`. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + /// + /// The given pointers must be valid to read from. + #[inline(always)] + unsafe fn verify( + &self, + mut cur: *const u8, + end: *const u8, + candidate: V, + ) -> Option { + // This is a bit tricky, but we basically want to convert our + // candidate, which looks like this (assuming a 256-bit vector): + // + // a31 a30 ... a17 a16 a15 a14 ... a01 a00 + // + // where each a(i) is an 8-bit bitset corresponding to the activated + // buckets, to this + // + // a31 a15 a30 a14 a29 a13 ... a18 a02 a17 a01 a16 a00 + // + // Namely, for Fat Teddy, the high 128-bits of the candidate correspond + // to the same bytes in the haystack in the low 128-bits (so we only + // scan 16 bytes at a time), but are for buckets 8-15 instead of 0-7. + // + // The verification routine wants to look at all potentially matching + // buckets before moving on to the next lane. So for example, both + // a16 and a00 both correspond to the first byte in our window; a00 + // contains buckets 0-7 and a16 contains buckets 8-15. Specifically, + // a16 should be checked before a01. So the transformation shown above + // allows us to use our normal verification procedure with one small + // change: we treat each bitset as 16 bits instead of 8 bits. + debug_assert!(!candidate.is_zero()); + + // Swap the 128-bit lanes in the candidate vector. + let swapped = candidate.swap_halves(); + // Interleave the bytes from the low 128-bit lanes, starting with + // cand first. + let r1 = candidate.interleave_low_8bit_lanes(swapped); + // Interleave the bytes from the high 128-bit lanes, starting with + // cand first. + let r2 = candidate.interleave_high_8bit_lanes(swapped); + // Now just take the 2 low 64-bit integers from both r1 and r2. We + // can drop the high 64-bit integers because they are a mirror image + // of the low 64-bit integers. All we care about are the low 128-bit + // lanes of r1 and r2. Combined, they contain all our 16-bit bitsets + // laid out in the desired order, as described above. + r1.for_each_low_64bit_lane( + r2, + #[inline(always)] + |_, chunk| { + let result = self.verify64(cur, end, chunk); + cur = cur.add(4); + result + }, + ) + } +} + +/// A vector generic mask for the low and high nybbles in a set of patterns. +/// Each 8-bit lane `j` in a vector corresponds to a bitset where the `i`th bit +/// is set if and only if the nybble `j` is in the bucket `i` at a particular +/// position. +/// +/// This is slightly tweaked dependending on whether Slim or Fat Teddy is being +/// used. For Slim Teddy, the bitsets in the lower half are the same as the +/// bitsets in the higher half, so that we can search `V::BYTES` bytes at a +/// time. (Remember, the nybbles in the haystack are used as indices into these +/// masks, and 256-bit shuffles only operate on 128-bit lanes.) +/// +/// For Fat Teddy, the bitsets are not repeated, but instead, the high half +/// bits correspond to an addition 8 buckets. So that a bitset `00100010` has +/// buckets 1 and 5 set if it's in the lower half, but has buckets 9 and 13 set +/// if it's in the higher half. +#[derive(Clone, Copy, Debug)] +struct Mask { + lo: V, + hi: V, +} + +impl Mask { + /// Return a candidate for Teddy (fat or slim) that is searching for 1-byte + /// candidates. + /// + /// If a candidate is returned, it will be a collection of 8-bit bitsets + /// (one bitset per lane), where the ith bit is set in the jth lane if and + /// only if the byte occurring at the jth lane in `chunk` is in the bucket + /// `i`. If no candidate is found, then the vector returned will have all + /// lanes set to zero. + /// + /// `chunk` should correspond to a `V::BYTES` window of the haystack (where + /// the least significant byte corresponds to the start of the window). For + /// fat Teddy, the haystack window length should be `V::BYTES / 2`, with + /// the window repeated in each half of the vector. + /// + /// `mask1` should correspond to a low/high mask for the first byte of all + /// patterns that are being searched. + #[inline(always)] + unsafe fn members1(chunk: V, masks: [Mask; 1]) -> V { + let lomask = V::splat(0xF); + let hlo = chunk.and(lomask); + let hhi = chunk.shift_8bit_lane_right::<4>().and(lomask); + let locand = masks[0].lo.shuffle_bytes(hlo); + let hicand = masks[0].hi.shuffle_bytes(hhi); + locand.and(hicand) + } + + /// Return a candidate for Teddy (fat or slim) that is searching for 2-byte + /// candidates. + /// + /// If candidates are returned, each will be a collection of 8-bit bitsets + /// (one bitset per lane), where the ith bit is set in the jth lane if and + /// only if the byte occurring at the jth lane in `chunk` is in the bucket + /// `i`. Each candidate returned corresponds to the first and second bytes + /// of the patterns being searched. If no candidate is found, then all of + /// the lanes will be set to zero in at least one of the vectors returned. + /// + /// `chunk` should correspond to a `V::BYTES` window of the haystack (where + /// the least significant byte corresponds to the start of the window). For + /// fat Teddy, the haystack window length should be `V::BYTES / 2`, with + /// the window repeated in each half of the vector. + /// + /// The masks should correspond to the masks computed for the first and + /// second bytes of all patterns that are being searched. + #[inline(always)] + unsafe fn members2(chunk: V, masks: [Mask; 2]) -> (V, V) { + let lomask = V::splat(0xF); + let hlo = chunk.and(lomask); + let hhi = chunk.shift_8bit_lane_right::<4>().and(lomask); + + let locand1 = masks[0].lo.shuffle_bytes(hlo); + let hicand1 = masks[0].hi.shuffle_bytes(hhi); + let cand1 = locand1.and(hicand1); + + let locand2 = masks[1].lo.shuffle_bytes(hlo); + let hicand2 = masks[1].hi.shuffle_bytes(hhi); + let cand2 = locand2.and(hicand2); + + (cand1, cand2) + } + + /// Return a candidate for Teddy (fat or slim) that is searching for 3-byte + /// candidates. + /// + /// If candidates are returned, each will be a collection of 8-bit bitsets + /// (one bitset per lane), where the ith bit is set in the jth lane if and + /// only if the byte occurring at the jth lane in `chunk` is in the bucket + /// `i`. Each candidate returned corresponds to the first, second and third + /// bytes of the patterns being searched. If no candidate is found, then + /// all of the lanes will be set to zero in at least one of the vectors + /// returned. + /// + /// `chunk` should correspond to a `V::BYTES` window of the haystack (where + /// the least significant byte corresponds to the start of the window). For + /// fat Teddy, the haystack window length should be `V::BYTES / 2`, with + /// the window repeated in each half of the vector. + /// + /// The masks should correspond to the masks computed for the first, second + /// and third bytes of all patterns that are being searched. + #[inline(always)] + unsafe fn members3(chunk: V, masks: [Mask; 3]) -> (V, V, V) { + let lomask = V::splat(0xF); + let hlo = chunk.and(lomask); + let hhi = chunk.shift_8bit_lane_right::<4>().and(lomask); + + let locand1 = masks[0].lo.shuffle_bytes(hlo); + let hicand1 = masks[0].hi.shuffle_bytes(hhi); + let cand1 = locand1.and(hicand1); + + let locand2 = masks[1].lo.shuffle_bytes(hlo); + let hicand2 = masks[1].hi.shuffle_bytes(hhi); + let cand2 = locand2.and(hicand2); + + let locand3 = masks[2].lo.shuffle_bytes(hlo); + let hicand3 = masks[2].hi.shuffle_bytes(hhi); + let cand3 = locand3.and(hicand3); + + (cand1, cand2, cand3) + } + + /// Return a candidate for Teddy (fat or slim) that is searching for 4-byte + /// candidates. + /// + /// If candidates are returned, each will be a collection of 8-bit bitsets + /// (one bitset per lane), where the ith bit is set in the jth lane if and + /// only if the byte occurring at the jth lane in `chunk` is in the bucket + /// `i`. Each candidate returned corresponds to the first, second, third + /// and fourth bytes of the patterns being searched. If no candidate is + /// found, then all of the lanes will be set to zero in at least one of the + /// vectors returned. + /// + /// `chunk` should correspond to a `V::BYTES` window of the haystack (where + /// the least significant byte corresponds to the start of the window). For + /// fat Teddy, the haystack window length should be `V::BYTES / 2`, with + /// the window repeated in each half of the vector. + /// + /// The masks should correspond to the masks computed for the first, + /// second, third and fourth bytes of all patterns that are being searched. + #[inline(always)] + unsafe fn members4(chunk: V, masks: [Mask; 4]) -> (V, V, V, V) { + let lomask = V::splat(0xF); + let hlo = chunk.and(lomask); + let hhi = chunk.shift_8bit_lane_right::<4>().and(lomask); + + let locand1 = masks[0].lo.shuffle_bytes(hlo); + let hicand1 = masks[0].hi.shuffle_bytes(hhi); + let cand1 = locand1.and(hicand1); + + let locand2 = masks[1].lo.shuffle_bytes(hlo); + let hicand2 = masks[1].hi.shuffle_bytes(hhi); + let cand2 = locand2.and(hicand2); + + let locand3 = masks[2].lo.shuffle_bytes(hlo); + let hicand3 = masks[2].hi.shuffle_bytes(hhi); + let cand3 = locand3.and(hicand3); + + let locand4 = masks[3].lo.shuffle_bytes(hlo); + let hicand4 = masks[3].hi.shuffle_bytes(hhi); + let cand4 = locand4.and(hicand4); + + (cand1, cand2, cand3, cand4) + } +} + +/// Represents the low and high nybble masks that will be used during +/// search. Each mask is 32 bytes wide, although only the first 16 bytes are +/// used for 128-bit vectors. +/// +/// Each byte in the mask corresponds to a 8-bit bitset, where bit `i` is set +/// if and only if the corresponding nybble is in the ith bucket. The index of +/// the byte (0-15, inclusive) corresponds to the nybble. +/// +/// Each mask is used as the target of a shuffle, where the indices for the +/// shuffle are taken from the haystack. AND'ing the shuffles for both the +/// low and high masks together also results in 8-bit bitsets, but where bit +/// `i` is set if and only if the correspond *byte* is in the ith bucket. +#[derive(Clone, Default)] +struct SlimMaskBuilder { + lo: [u8; 32], + hi: [u8; 32], +} + +impl SlimMaskBuilder { + /// Update this mask by adding the given byte to the given bucket. The + /// given bucket must be in the range 0-7. + /// + /// # Panics + /// + /// When `bucket >= 8`. + fn add(&mut self, bucket: usize, byte: u8) { + assert!(bucket < 8); + + let bucket = u8::try_from(bucket).unwrap(); + let byte_lo = usize::from(byte & 0xF); + let byte_hi = usize::from((byte >> 4) & 0xF); + // When using 256-bit vectors, we need to set this bucket assignment in + // the low and high 128-bit portions of the mask. This allows us to + // process 32 bytes at a time. Namely, AVX2 shuffles operate on each + // of the 128-bit lanes, rather than the full 256-bit vector at once. + self.lo[byte_lo] |= 1 << bucket; + self.lo[byte_lo + 16] |= 1 << bucket; + self.hi[byte_hi] |= 1 << bucket; + self.hi[byte_hi + 16] |= 1 << bucket; + } + + /// Turn this builder into a vector mask. + /// + /// # Panics + /// + /// When `V` represents a vector bigger than what `MaskBytes` can contain. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + unsafe fn build(&self) -> Mask { + assert!(V::BYTES <= self.lo.len()); + assert!(V::BYTES <= self.hi.len()); + Mask { + lo: V::load_unaligned(self.lo[..].as_ptr()), + hi: V::load_unaligned(self.hi[..].as_ptr()), + } + } + + /// A convenience function for building `N` vector masks from a slim + /// `Teddy` value. + /// + /// # Panics + /// + /// When `V` represents a vector bigger than what `MaskBytes` can contain. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + unsafe fn from_teddy( + teddy: &Teddy<8>, + ) -> [Mask; BYTES] { + // MSRV(1.63): Use core::array::from_fn to just build the array here + // instead of creating a vector and turning it into an array. + let mut mask_builders = vec![SlimMaskBuilder::default(); BYTES]; + for (bucket_index, bucket) in teddy.buckets.iter().enumerate() { + for pid in bucket.iter().copied() { + let pat = teddy.patterns.get(pid); + for (i, builder) in mask_builders.iter_mut().enumerate() { + builder.add(bucket_index, pat.bytes()[i]); + } + } + } + let array = + <[SlimMaskBuilder; BYTES]>::try_from(mask_builders).unwrap(); + array.map(|builder| builder.build()) + } +} + +impl Debug for SlimMaskBuilder { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let (mut parts_lo, mut parts_hi) = (vec![], vec![]); + for i in 0..32 { + parts_lo.push(format!("{:02}: {:08b}", i, self.lo[i])); + parts_hi.push(format!("{:02}: {:08b}", i, self.hi[i])); + } + f.debug_struct("SlimMaskBuilder") + .field("lo", &parts_lo) + .field("hi", &parts_hi) + .finish() + } +} + +/// Represents the low and high nybble masks that will be used during "fat" +/// Teddy search. +/// +/// Each mask is 32 bytes wide, and at the time of writing, only 256-bit vectors +/// support fat Teddy. +/// +/// A fat Teddy mask is like a slim Teddy mask, except that instead of +/// repeating the bitsets in the high and low 128-bits in 256-bit vectors, the +/// high and low 128-bit halves each represent distinct buckets. (Bringing the +/// total to 16 instead of 8.) This permits spreading the patterns out a bit +/// more and thus putting less pressure on verification to be fast. +/// +/// Each byte in the mask corresponds to a 8-bit bitset, where bit `i` is set +/// if and only if the corresponding nybble is in the ith bucket. The index of +/// the byte (0-15, inclusive) corresponds to the nybble. +#[derive(Clone, Copy, Default)] +struct FatMaskBuilder { + lo: [u8; 32], + hi: [u8; 32], +} + +impl FatMaskBuilder { + /// Update this mask by adding the given byte to the given bucket. The + /// given bucket must be in the range 0-15. + /// + /// # Panics + /// + /// When `bucket >= 16`. + fn add(&mut self, bucket: usize, byte: u8) { + assert!(bucket < 16); + + let bucket = u8::try_from(bucket).unwrap(); + let byte_lo = usize::from(byte & 0xF); + let byte_hi = usize::from((byte >> 4) & 0xF); + // Unlike slim teddy, fat teddy only works with AVX2. For fat teddy, + // the high 128 bits of our mask correspond to buckets 8-15, while the + // low 128 bits correspond to buckets 0-7. + if bucket < 8 { + self.lo[byte_lo] |= 1 << bucket; + self.hi[byte_hi] |= 1 << bucket; + } else { + self.lo[byte_lo + 16] |= 1 << (bucket % 8); + self.hi[byte_hi + 16] |= 1 << (bucket % 8); + } + } + + /// Turn this builder into a vector mask. + /// + /// # Panics + /// + /// When `V` represents a vector bigger than what `MaskBytes` can contain. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + unsafe fn build(&self) -> Mask { + assert!(V::BYTES <= self.lo.len()); + assert!(V::BYTES <= self.hi.len()); + Mask { + lo: V::load_unaligned(self.lo[..].as_ptr()), + hi: V::load_unaligned(self.hi[..].as_ptr()), + } + } + + /// A convenience function for building `N` vector masks from a fat + /// `Teddy` value. + /// + /// # Panics + /// + /// When `V` represents a vector bigger than what `MaskBytes` can contain. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[inline(always)] + unsafe fn from_teddy( + teddy: &Teddy<16>, + ) -> [Mask; BYTES] { + // MSRV(1.63): Use core::array::from_fn to just build the array here + // instead of creating a vector and turning it into an array. + let mut mask_builders = vec![FatMaskBuilder::default(); BYTES]; + for (bucket_index, bucket) in teddy.buckets.iter().enumerate() { + for pid in bucket.iter().copied() { + let pat = teddy.patterns.get(pid); + for (i, builder) in mask_builders.iter_mut().enumerate() { + builder.add(bucket_index, pat.bytes()[i]); + } + } + } + let array = + <[FatMaskBuilder; BYTES]>::try_from(mask_builders).unwrap(); + array.map(|builder| builder.build()) + } +} + +impl Debug for FatMaskBuilder { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let (mut parts_lo, mut parts_hi) = (vec![], vec![]); + for i in 0..32 { + parts_lo.push(format!("{:02}: {:08b}", i, self.lo[i])); + parts_hi.push(format!("{:02}: {:08b}", i, self.hi[i])); + } + f.debug_struct("FatMaskBuilder") + .field("lo", &parts_lo) + .field("hi", &parts_hi) + .finish() + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/mod.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/mod.rs new file mode 100644 index 0000000..26cfcdc --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/teddy/mod.rs @@ -0,0 +1,9 @@ +// Regrettable, but Teddy stuff just isn't used on all targets. And for some +// targets, like aarch64, only "slim" Teddy is used and so "fat" Teddy gets a +// bunch of dead-code warnings. Just not worth trying to squash them. Blech. +#![allow(dead_code)] + +pub(crate) use self::builder::{Builder, Searcher}; + +mod builder; +mod generic; diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/tests.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/tests.rs new file mode 100644 index 0000000..2b0d44e --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/tests.rs @@ -0,0 +1,583 @@ +use std::collections::HashMap; + +use alloc::{ + format, + string::{String, ToString}, + vec, + vec::Vec, +}; + +use crate::{ + packed::{Config, MatchKind}, + util::search::Match, +}; + +/// A description of a single test against a multi-pattern searcher. +/// +/// A single test may not necessarily pass on every configuration of a +/// searcher. The tests are categorized and grouped appropriately below. +#[derive(Clone, Debug, Eq, PartialEq)] +struct SearchTest { + /// The name of this test, for debugging. + name: &'static str, + /// The patterns to search for. + patterns: &'static [&'static str], + /// The text to search. + haystack: &'static str, + /// Each match is a triple of (pattern_index, start, end), where + /// pattern_index is an index into `patterns` and `start`/`end` are indices + /// into `haystack`. + matches: &'static [(usize, usize, usize)], +} + +struct SearchTestOwned { + offset: usize, + name: String, + patterns: Vec, + haystack: String, + matches: Vec<(usize, usize, usize)>, +} + +impl SearchTest { + fn variations(&self) -> Vec { + let count = if cfg!(miri) { 1 } else { 261 }; + let mut tests = vec![]; + for i in 0..count { + tests.push(self.offset_prefix(i)); + tests.push(self.offset_suffix(i)); + tests.push(self.offset_both(i)); + } + tests + } + + fn offset_both(&self, off: usize) -> SearchTestOwned { + SearchTestOwned { + offset: off, + name: self.name.to_string(), + patterns: self.patterns.iter().map(|s| s.to_string()).collect(), + haystack: format!( + "{}{}{}", + "Z".repeat(off), + self.haystack, + "Z".repeat(off) + ), + matches: self + .matches + .iter() + .map(|&(id, s, e)| (id, s + off, e + off)) + .collect(), + } + } + + fn offset_prefix(&self, off: usize) -> SearchTestOwned { + SearchTestOwned { + offset: off, + name: self.name.to_string(), + patterns: self.patterns.iter().map(|s| s.to_string()).collect(), + haystack: format!("{}{}", "Z".repeat(off), self.haystack), + matches: self + .matches + .iter() + .map(|&(id, s, e)| (id, s + off, e + off)) + .collect(), + } + } + + fn offset_suffix(&self, off: usize) -> SearchTestOwned { + SearchTestOwned { + offset: off, + name: self.name.to_string(), + patterns: self.patterns.iter().map(|s| s.to_string()).collect(), + haystack: format!("{}{}", self.haystack, "Z".repeat(off)), + matches: self.matches.to_vec(), + } + } +} + +/// Short-hand constructor for SearchTest. We use it a lot below. +macro_rules! t { + ($name:ident, $patterns:expr, $haystack:expr, $matches:expr) => { + SearchTest { + name: stringify!($name), + patterns: $patterns, + haystack: $haystack, + matches: $matches, + } + }; +} + +/// A collection of test groups. +type TestCollection = &'static [&'static [SearchTest]]; + +// Define several collections corresponding to the different type of match +// semantics supported. These collections have some overlap, but each +// collection should have some tests that no other collection has. + +/// Tests for leftmost-first match semantics. +const PACKED_LEFTMOST_FIRST: TestCollection = + &[BASICS, LEFTMOST, LEFTMOST_FIRST, REGRESSION, TEDDY]; + +/// Tests for leftmost-longest match semantics. +const PACKED_LEFTMOST_LONGEST: TestCollection = + &[BASICS, LEFTMOST, LEFTMOST_LONGEST, REGRESSION, TEDDY]; + +// Now define the individual tests that make up the collections above. + +/// A collection of tests for the that should always be true regardless of +/// match semantics. That is, all combinations of leftmost-{first, longest} +/// should produce the same answer. +const BASICS: &'static [SearchTest] = &[ + t!(basic001, &["a"], "", &[]), + t!(basic010, &["a"], "a", &[(0, 0, 1)]), + t!(basic020, &["a"], "aa", &[(0, 0, 1), (0, 1, 2)]), + t!(basic030, &["a"], "aaa", &[(0, 0, 1), (0, 1, 2), (0, 2, 3)]), + t!(basic040, &["a"], "aba", &[(0, 0, 1), (0, 2, 3)]), + t!(basic050, &["a"], "bba", &[(0, 2, 3)]), + t!(basic060, &["a"], "bbb", &[]), + t!(basic070, &["a"], "bababbbba", &[(0, 1, 2), (0, 3, 4), (0, 8, 9)]), + t!(basic100, &["aa"], "", &[]), + t!(basic110, &["aa"], "aa", &[(0, 0, 2)]), + t!(basic120, &["aa"], "aabbaa", &[(0, 0, 2), (0, 4, 6)]), + t!(basic130, &["aa"], "abbab", &[]), + t!(basic140, &["aa"], "abbabaa", &[(0, 5, 7)]), + t!(basic150, &["aaa"], "aaa", &[(0, 0, 3)]), + t!(basic200, &["abc"], "abc", &[(0, 0, 3)]), + t!(basic210, &["abc"], "zazabzabcz", &[(0, 6, 9)]), + t!(basic220, &["abc"], "zazabczabcz", &[(0, 3, 6), (0, 7, 10)]), + t!(basic230, &["abcd"], "abcd", &[(0, 0, 4)]), + t!(basic240, &["abcd"], "zazabzabcdz", &[(0, 6, 10)]), + t!(basic250, &["abcd"], "zazabcdzabcdz", &[(0, 3, 7), (0, 8, 12)]), + t!(basic300, &["a", "b"], "", &[]), + t!(basic310, &["a", "b"], "z", &[]), + t!(basic320, &["a", "b"], "b", &[(1, 0, 1)]), + t!(basic330, &["a", "b"], "a", &[(0, 0, 1)]), + t!( + basic340, + &["a", "b"], + "abba", + &[(0, 0, 1), (1, 1, 2), (1, 2, 3), (0, 3, 4),] + ), + t!( + basic350, + &["b", "a"], + "abba", + &[(1, 0, 1), (0, 1, 2), (0, 2, 3), (1, 3, 4),] + ), + t!(basic360, &["abc", "bc"], "xbc", &[(1, 1, 3),]), + t!(basic400, &["foo", "bar"], "", &[]), + t!(basic410, &["foo", "bar"], "foobar", &[(0, 0, 3), (1, 3, 6),]), + t!(basic420, &["foo", "bar"], "barfoo", &[(1, 0, 3), (0, 3, 6),]), + t!(basic430, &["foo", "bar"], "foofoo", &[(0, 0, 3), (0, 3, 6),]), + t!(basic440, &["foo", "bar"], "barbar", &[(1, 0, 3), (1, 3, 6),]), + t!(basic450, &["foo", "bar"], "bafofoo", &[(0, 4, 7),]), + t!(basic460, &["bar", "foo"], "bafofoo", &[(1, 4, 7),]), + t!(basic470, &["foo", "bar"], "fobabar", &[(1, 4, 7),]), + t!(basic480, &["bar", "foo"], "fobabar", &[(0, 4, 7),]), + t!(basic700, &["yabcdef", "abcdezghi"], "yabcdefghi", &[(0, 0, 7),]), + t!(basic710, &["yabcdef", "abcdezghi"], "yabcdezghi", &[(1, 1, 10),]), + t!( + basic720, + &["yabcdef", "bcdeyabc", "abcdezghi"], + "yabcdezghi", + &[(2, 1, 10),] + ), + t!(basic810, &["abcd", "bcd", "cd"], "abcd", &[(0, 0, 4),]), + t!(basic820, &["bcd", "cd", "abcd"], "abcd", &[(2, 0, 4),]), + t!(basic830, &["abc", "bc"], "zazabcz", &[(0, 3, 6),]), + t!( + basic840, + &["ab", "ba"], + "abababa", + &[(0, 0, 2), (0, 2, 4), (0, 4, 6),] + ), + t!(basic850, &["foo", "foo"], "foobarfoo", &[(0, 0, 3), (0, 6, 9),]), +]; + +/// Tests for leftmost match semantics. These should pass for both +/// leftmost-first and leftmost-longest match kinds. Stated differently, among +/// ambiguous matches, the longest match and the match that appeared first when +/// constructing the automaton should always be the same. +const LEFTMOST: &'static [SearchTest] = &[ + t!(leftmost000, &["ab", "ab"], "abcd", &[(0, 0, 2)]), + t!(leftmost030, &["a", "ab"], "aa", &[(0, 0, 1), (0, 1, 2)]), + t!(leftmost031, &["ab", "a"], "aa", &[(1, 0, 1), (1, 1, 2)]), + t!(leftmost032, &["ab", "a"], "xayabbbz", &[(1, 1, 2), (0, 3, 5)]), + t!(leftmost300, &["abcd", "bce", "b"], "abce", &[(1, 1, 4)]), + t!(leftmost310, &["abcd", "ce", "bc"], "abce", &[(2, 1, 3)]), + t!(leftmost320, &["abcd", "bce", "ce", "b"], "abce", &[(1, 1, 4)]), + t!(leftmost330, &["abcd", "bce", "cz", "bc"], "abcz", &[(3, 1, 3)]), + t!(leftmost340, &["bce", "cz", "bc"], "bcz", &[(2, 0, 2)]), + t!(leftmost350, &["abc", "bd", "ab"], "abd", &[(2, 0, 2)]), + t!( + leftmost360, + &["abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(2, 0, 8),] + ), + t!( + leftmost370, + &["abcdefghi", "cde", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + leftmost380, + &["abcdefghi", "hz", "abcdefgh", "a"], + "abcdefghz", + &[(2, 0, 8),] + ), + t!( + leftmost390, + &["b", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + leftmost400, + &["h", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + leftmost410, + &["z", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8), (0, 8, 9),] + ), +]; + +/// Tests for non-overlapping leftmost-first match semantics. These tests +/// should generally be specific to leftmost-first, which means they should +/// generally fail under leftmost-longest semantics. +const LEFTMOST_FIRST: &'static [SearchTest] = &[ + t!(leftfirst000, &["ab", "abcd"], "abcd", &[(0, 0, 2)]), + t!(leftfirst020, &["abcd", "ab"], "abcd", &[(0, 0, 4)]), + t!(leftfirst030, &["ab", "ab"], "abcd", &[(0, 0, 2)]), + t!(leftfirst040, &["a", "ab"], "xayabbbz", &[(0, 1, 2), (0, 3, 4)]), + t!(leftfirst100, &["abcdefg", "bcde", "bcdef"], "abcdef", &[(1, 1, 5)]), + t!(leftfirst110, &["abcdefg", "bcdef", "bcde"], "abcdef", &[(1, 1, 6)]), + t!(leftfirst300, &["abcd", "b", "bce"], "abce", &[(1, 1, 2)]), + t!( + leftfirst310, + &["abcd", "b", "bce", "ce"], + "abce", + &[(1, 1, 2), (3, 2, 4),] + ), + t!( + leftfirst320, + &["a", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(0, 0, 1), (2, 7, 9),] + ), + t!(leftfirst330, &["a", "abab"], "abab", &[(0, 0, 1), (0, 2, 3)]), + t!( + leftfirst340, + &["abcdef", "x", "x", "x", "x", "x", "x", "abcde"], + "abcdef", + &[(0, 0, 6)] + ), +]; + +/// Tests for non-overlapping leftmost-longest match semantics. These tests +/// should generally be specific to leftmost-longest, which means they should +/// generally fail under leftmost-first semantics. +const LEFTMOST_LONGEST: &'static [SearchTest] = &[ + t!(leftlong000, &["ab", "abcd"], "abcd", &[(1, 0, 4)]), + t!(leftlong010, &["abcd", "bcd", "cd", "b"], "abcd", &[(0, 0, 4),]), + t!(leftlong040, &["a", "ab"], "a", &[(0, 0, 1)]), + t!(leftlong050, &["a", "ab"], "ab", &[(1, 0, 2)]), + t!(leftlong060, &["ab", "a"], "a", &[(1, 0, 1)]), + t!(leftlong070, &["ab", "a"], "ab", &[(0, 0, 2)]), + t!(leftlong100, &["abcdefg", "bcde", "bcdef"], "abcdef", &[(2, 1, 6)]), + t!(leftlong110, &["abcdefg", "bcdef", "bcde"], "abcdef", &[(1, 1, 6)]), + t!(leftlong300, &["abcd", "b", "bce"], "abce", &[(2, 1, 4)]), + t!( + leftlong310, + &["a", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!(leftlong320, &["a", "abab"], "abab", &[(1, 0, 4)]), + t!(leftlong330, &["abcd", "b", "ce"], "abce", &[(1, 1, 2), (2, 2, 4),]), + t!(leftlong340, &["a", "ab"], "xayabbbz", &[(0, 1, 2), (1, 3, 5)]), +]; + +/// Regression tests that are applied to all combinations. +/// +/// If regression tests are needed for specific match semantics, then add them +/// to the appropriate group above. +const REGRESSION: &'static [SearchTest] = &[ + t!(regression010, &["inf", "ind"], "infind", &[(0, 0, 3), (1, 3, 6),]), + t!(regression020, &["ind", "inf"], "infind", &[(1, 0, 3), (0, 3, 6),]), + t!( + regression030, + &["libcore/", "libstd/"], + "libcore/char/methods.rs", + &[(0, 0, 8),] + ), + t!( + regression040, + &["libstd/", "libcore/"], + "libcore/char/methods.rs", + &[(1, 0, 8),] + ), + t!( + regression050, + &["\x00\x00\x01", "\x00\x00\x00"], + "\x00\x00\x00", + &[(1, 0, 3),] + ), + t!( + regression060, + &["\x00\x00\x00", "\x00\x00\x01"], + "\x00\x00\x00", + &[(0, 0, 3),] + ), +]; + +const TEDDY: &'static [SearchTest] = &[ + t!( + teddy010, + &["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"], + "abcdefghijk", + &[ + (0, 0, 1), + (1, 1, 2), + (2, 2, 3), + (3, 3, 4), + (4, 4, 5), + (5, 5, 6), + (6, 6, 7), + (7, 7, 8), + (8, 8, 9), + (9, 9, 10), + (10, 10, 11) + ] + ), + t!( + teddy020, + &["ab", "bc", "cd", "de", "ef", "fg", "gh", "hi", "ij", "jk", "kl"], + "abcdefghijk", + &[(0, 0, 2), (2, 2, 4), (4, 4, 6), (6, 6, 8), (8, 8, 10),] + ), + t!( + teddy030, + &["abc"], + "abcdefghijklmnopqrstuvwxyzabcdefghijk", + &[(0, 0, 3), (0, 26, 29)] + ), +]; + +// Now define a test for each combination of things above that we want to run. +// Since there are a few different combinations for each collection of tests, +// we define a couple of macros to avoid repetition drudgery. The testconfig +// macro constructs the automaton from a given match kind, and runs the search +// tests one-by-one over the given collection. The `with` parameter allows one +// to configure the config with additional parameters. The testcombo macro +// invokes testconfig in precisely this way: it sets up several tests where +// each one turns a different knob on Config. + +macro_rules! testconfig { + ($name:ident, $collection:expr, $with:expr) => { + #[test] + fn $name() { + run_search_tests($collection, |test| { + let mut config = Config::new(); + $with(&mut config); + let mut builder = config.builder(); + builder.extend(test.patterns.iter().map(|p| p.as_bytes())); + let searcher = match builder.build() { + Some(searcher) => searcher, + None => { + // For x86-64 and aarch64, not building a searcher is + // probably a bug, so be loud. + if cfg!(any( + target_arch = "x86_64", + target_arch = "aarch64" + )) { + panic!("failed to build packed searcher") + } + return None; + } + }; + Some(searcher.find_iter(&test.haystack).collect()) + }); + } + }; +} + +testconfig!( + search_default_leftmost_first, + PACKED_LEFTMOST_FIRST, + |_: &mut Config| {} +); + +testconfig!( + search_default_leftmost_longest, + PACKED_LEFTMOST_LONGEST, + |c: &mut Config| { + c.match_kind(MatchKind::LeftmostLongest); + } +); + +testconfig!( + search_teddy_leftmost_first, + PACKED_LEFTMOST_FIRST, + |c: &mut Config| { + c.only_teddy(true); + } +); + +testconfig!( + search_teddy_leftmost_longest, + PACKED_LEFTMOST_LONGEST, + |c: &mut Config| { + c.only_teddy(true).match_kind(MatchKind::LeftmostLongest); + } +); + +testconfig!( + search_teddy_ssse3_leftmost_first, + PACKED_LEFTMOST_FIRST, + |c: &mut Config| { + c.only_teddy(true); + #[cfg(target_arch = "x86_64")] + if std::is_x86_feature_detected!("ssse3") { + c.only_teddy_256bit(Some(false)); + } + } +); + +testconfig!( + search_teddy_ssse3_leftmost_longest, + PACKED_LEFTMOST_LONGEST, + |c: &mut Config| { + c.only_teddy(true).match_kind(MatchKind::LeftmostLongest); + #[cfg(target_arch = "x86_64")] + if std::is_x86_feature_detected!("ssse3") { + c.only_teddy_256bit(Some(false)); + } + } +); + +testconfig!( + search_teddy_avx2_leftmost_first, + PACKED_LEFTMOST_FIRST, + |c: &mut Config| { + c.only_teddy(true); + #[cfg(target_arch = "x86_64")] + if std::is_x86_feature_detected!("avx2") { + c.only_teddy_256bit(Some(true)); + } + } +); + +testconfig!( + search_teddy_avx2_leftmost_longest, + PACKED_LEFTMOST_LONGEST, + |c: &mut Config| { + c.only_teddy(true).match_kind(MatchKind::LeftmostLongest); + #[cfg(target_arch = "x86_64")] + if std::is_x86_feature_detected!("avx2") { + c.only_teddy_256bit(Some(true)); + } + } +); + +testconfig!( + search_teddy_fat_leftmost_first, + PACKED_LEFTMOST_FIRST, + |c: &mut Config| { + c.only_teddy(true); + #[cfg(target_arch = "x86_64")] + if std::is_x86_feature_detected!("avx2") { + c.only_teddy_fat(Some(true)); + } + } +); + +testconfig!( + search_teddy_fat_leftmost_longest, + PACKED_LEFTMOST_LONGEST, + |c: &mut Config| { + c.only_teddy(true).match_kind(MatchKind::LeftmostLongest); + #[cfg(target_arch = "x86_64")] + if std::is_x86_feature_detected!("avx2") { + c.only_teddy_fat(Some(true)); + } + } +); + +testconfig!( + search_rabinkarp_leftmost_first, + PACKED_LEFTMOST_FIRST, + |c: &mut Config| { + c.only_rabin_karp(true); + } +); + +testconfig!( + search_rabinkarp_leftmost_longest, + PACKED_LEFTMOST_LONGEST, + |c: &mut Config| { + c.only_rabin_karp(true).match_kind(MatchKind::LeftmostLongest); + } +); + +#[test] +fn search_tests_have_unique_names() { + let assert = |constname, tests: &[SearchTest]| { + let mut seen = HashMap::new(); // map from test name to position + for (i, test) in tests.iter().enumerate() { + if !seen.contains_key(test.name) { + seen.insert(test.name, i); + } else { + let last = seen[test.name]; + panic!( + "{} tests have duplicate names at positions {} and {}", + constname, last, i + ); + } + } + }; + assert("BASICS", BASICS); + assert("LEFTMOST", LEFTMOST); + assert("LEFTMOST_FIRST", LEFTMOST_FIRST); + assert("LEFTMOST_LONGEST", LEFTMOST_LONGEST); + assert("REGRESSION", REGRESSION); + assert("TEDDY", TEDDY); +} + +fn run_search_tests Option>>( + which: TestCollection, + mut f: F, +) { + let get_match_triples = + |matches: Vec| -> Vec<(usize, usize, usize)> { + matches + .into_iter() + .map(|m| (m.pattern().as_usize(), m.start(), m.end())) + .collect() + }; + for &tests in which { + for spec in tests { + for test in spec.variations() { + let results = match f(&test) { + None => continue, + Some(results) => results, + }; + assert_eq!( + test.matches, + get_match_triples(results).as_slice(), + "test: {}, patterns: {:?}, haystack(len={:?}): {:?}, \ + offset: {:?}", + test.name, + test.patterns, + test.haystack.len(), + test.haystack, + test.offset, + ); + } + } + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/packed/vector.rs b/third_party/crates/aho-corasick-1.1.5/src/packed/vector.rs new file mode 100644 index 0000000..57c02cc --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/packed/vector.rs @@ -0,0 +1,1757 @@ +// NOTE: The descriptions for each of the vector methods on the traits below +// are pretty inscrutable. For this reason, there are tests for every method +// on for every trait impl below. If you're confused about what an op does, +// consult its test. (They probably should be doc tests, but I couldn't figure +// out how to write them in a non-annoying way.) + +use core::{ + fmt::Debug, + panic::{RefUnwindSafe, UnwindSafe}, +}; + +/// A trait for describing vector operations used by vectorized searchers. +/// +/// The trait is highly constrained to low level vector operations needed for +/// the specific algorithms used in this crate. In general, it was invented +/// mostly to be generic over x86's __m128i and __m256i types. At time of +/// writing, it also supports wasm and aarch64 128-bit vector types as well. +/// +/// # Safety +/// +/// All methods are not safe since they are intended to be implemented using +/// vendor intrinsics, which are also not safe. Callers must ensure that +/// the appropriate target features are enabled in the calling function, +/// and that the current CPU supports them. All implementations should +/// avoid marking the routines with `#[target_feature]` and instead mark +/// them as `#[inline(always)]` to ensure they get appropriately inlined. +/// (`inline(always)` cannot be used with target_feature.) +pub(crate) trait Vector: + Copy + Debug + Send + Sync + UnwindSafe + RefUnwindSafe +{ + /// The number of bits in the vector. + const BITS: usize; + /// The number of bytes in the vector. That is, this is the size of the + /// vector in memory. + const BYTES: usize; + + /// Create a vector with 8-bit lanes with the given byte repeated into each + /// lane. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn splat(byte: u8) -> Self; + + /// Read a vector-size number of bytes from the given pointer. The pointer + /// does not need to be aligned. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + /// + /// Callers must guarantee that at least `BYTES` bytes are readable from + /// `data`. + unsafe fn load_unaligned(data: *const u8) -> Self; + + /// Returns true if and only if this vector has zero in all of its lanes. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn is_zero(self) -> bool; + + /// Do an 8-bit pairwise equality check. If lane `i` is equal in this + /// vector and the one given, then lane `i` in the resulting vector is set + /// to `0xFF`. Otherwise, it is set to `0x00`. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn cmpeq(self, vector2: Self) -> Self; + + /// Perform a bitwise 'and' of this vector and the one given and return + /// the result. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn and(self, vector2: Self) -> Self; + + /// Perform a bitwise 'or' of this vector and the one given and return + /// the result. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + #[allow(dead_code)] // unused, but useful enough to keep around? + unsafe fn or(self, vector2: Self) -> Self; + + /// Shift each 8-bit lane in this vector to the right by the number of + /// bits indictated by the `BITS` type parameter. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn shift_8bit_lane_right(self) -> Self; + + /// Shift this vector to the left by one byte and shift the most + /// significant byte of `vector2` into the least significant position of + /// this vector. + /// + /// Stated differently, this behaves as if `self` and `vector2` were + /// concatenated into a `2 * Self::BITS` temporary buffer and then shifted + /// right by `Self::BYTES - 1` bytes. + /// + /// With respect to the Teddy algorithm, `vector2` is usually a previous + /// `Self::BYTES` chunk from the haystack and `self` is the chunk + /// immediately following it. This permits combining the last two bytes + /// from the previous chunk (`vector2`) with the first `Self::BYTES - 1` + /// bytes from the current chunk. This permits aligning the result of + /// various shuffles so that they can be and-ed together and a possible + /// candidate discovered. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn shift_in_one_byte(self, vector2: Self) -> Self; + + /// Shift this vector to the left by two bytes and shift the two most + /// significant bytes of `vector2` into the least significant position of + /// this vector. + /// + /// Stated differently, this behaves as if `self` and `vector2` were + /// concatenated into a `2 * Self::BITS` temporary buffer and then shifted + /// right by `Self::BYTES - 2` bytes. + /// + /// With respect to the Teddy algorithm, `vector2` is usually a previous + /// `Self::BYTES` chunk from the haystack and `self` is the chunk + /// immediately following it. This permits combining the last two bytes + /// from the previous chunk (`vector2`) with the first `Self::BYTES - 2` + /// bytes from the current chunk. This permits aligning the result of + /// various shuffles so that they can be and-ed together and a possible + /// candidate discovered. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn shift_in_two_bytes(self, vector2: Self) -> Self; + + /// Shift this vector to the left by three bytes and shift the three most + /// significant bytes of `vector2` into the least significant position of + /// this vector. + /// + /// Stated differently, this behaves as if `self` and `vector2` were + /// concatenated into a `2 * Self::BITS` temporary buffer and then shifted + /// right by `Self::BYTES - 3` bytes. + /// + /// With respect to the Teddy algorithm, `vector2` is usually a previous + /// `Self::BYTES` chunk from the haystack and `self` is the chunk + /// immediately following it. This permits combining the last three bytes + /// from the previous chunk (`vector2`) with the first `Self::BYTES - 3` + /// bytes from the current chunk. This permits aligning the result of + /// various shuffles so that they can be and-ed together and a possible + /// candidate discovered. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn shift_in_three_bytes(self, vector2: Self) -> Self; + + /// Shuffles the bytes in this vector according to the indices in each of + /// the corresponding lanes in `indices`. + /// + /// If `i` is the index of corresponding lanes, `A` is this vector, `B` is + /// indices and `C` is the resulting vector, then `C = A[B[i]]`. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn shuffle_bytes(self, indices: Self) -> Self; + + /// Call the provided function for each 64-bit lane in this vector. The + /// given function is provided the lane index and lane value as a `u64`. + /// + /// If `f` returns `Some`, then iteration over the lanes is stopped and the + /// value is returned. Otherwise, this returns `None`. + /// + /// # Notes + /// + /// Conceptually it would be nice if we could have a + /// `unpack64(self) -> [u64; BITS / 64]` method, but defining that is + /// tricky given Rust's [current support for const generics][support]. + /// And even if we could, it would be tricky to write generic code over + /// it. (Not impossible. We could introduce another layer that requires + /// `AsRef<[u64]>` or something.) + /// + /// [support]: https://github.com/rust-lang/rust/issues/60551 + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn for_each_64bit_lane( + self, + f: impl FnMut(usize, u64) -> Option, + ) -> Option; +} + +/// This trait extends the `Vector` trait with additional operations to support +/// Fat Teddy. +/// +/// Fat Teddy uses 16 buckets instead of 8, but reads half as many bytes (as +/// the vector size) instead of the full size of a vector per iteration. For +/// example, when using a 256-bit vector, Slim Teddy reads 32 bytes at a timr +/// but Fat Teddy reads 16 bytes at a time. +/// +/// Fat Teddy is useful when searching for a large number of literals. +/// The extra number of buckets spreads the literals out more and reduces +/// verification time. +/// +/// Currently we only implement this for AVX on x86_64. It would be nice to +/// implement this for SSE on x86_64 and NEON on aarch64, with the latter two +/// only reading 8 bytes at a time. It's not clear how well it would work, but +/// there are some tricky things to figure out in terms of implementation. The +/// `half_shift_in_{one,two,three}_bytes` methods in particular are probably +/// the trickiest of the bunch. For AVX2, these are implemented by taking +/// advantage of the fact that `_mm256_alignr_epi8` operates on each 128-bit +/// half instead of the full 256-bit vector. (Where as `_mm_alignr_epi8` +/// operates on the full 128-bit vector and not on each 64-bit half.) I didn't +/// do a careful survey of NEON to see if it could easily support these +/// operations. +pub(crate) trait FatVector: Vector { + type Half: Vector; + + /// Read a half-vector-size number of bytes from the given pointer, and + /// broadcast it across both halfs of a full vector. The pointer does not + /// need to be aligned. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + /// + /// Callers must guarantee that at least `Self::HALF::BYTES` bytes are + /// readable from `data`. + unsafe fn load_half_unaligned(data: *const u8) -> Self; + + /// Like `Vector::shift_in_one_byte`, except this is done for each half + /// of the vector instead. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn half_shift_in_one_byte(self, vector2: Self) -> Self; + + /// Like `Vector::shift_in_two_bytes`, except this is done for each half + /// of the vector instead. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn half_shift_in_two_bytes(self, vector2: Self) -> Self; + + /// Like `Vector::shift_in_two_bytes`, except this is done for each half + /// of the vector instead. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn half_shift_in_three_bytes(self, vector2: Self) -> Self; + + /// Swap the 128-bit lanes in this vector. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn swap_halves(self) -> Self; + + /// Unpack and interleave the 8-bit lanes from the low 128 bits of each + /// vector and return the result. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn interleave_low_8bit_lanes(self, vector2: Self) -> Self; + + /// Unpack and interleave the 8-bit lanes from the high 128 bits of each + /// vector and return the result. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn interleave_high_8bit_lanes(self, vector2: Self) -> Self; + + /// Call the provided function for each 64-bit lane in the lower half + /// of this vector and then in the other vector. The given function is + /// provided the lane index and lane value as a `u64`. (The high 128-bits + /// of each vector are ignored.) + /// + /// If `f` returns `Some`, then iteration over the lanes is stopped and the + /// value is returned. Otherwise, this returns `None`. + /// + /// # Safety + /// + /// Callers must ensure that this is okay to call in the current target for + /// the current CPU. + unsafe fn for_each_low_64bit_lane( + self, + vector2: Self, + f: impl FnMut(usize, u64) -> Option, + ) -> Option; +} + +#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] +mod x86_64_ssse3 { + use core::arch::x86_64::*; + + use crate::util::int::{I32, I8}; + + use super::Vector; + + impl Vector for __m128i { + const BITS: usize = 128; + const BYTES: usize = 16; + + #[inline(always)] + unsafe fn splat(byte: u8) -> __m128i { + _mm_set1_epi8(i8::from_bits(byte)) + } + + #[inline(always)] + unsafe fn load_unaligned(data: *const u8) -> __m128i { + _mm_loadu_si128(data.cast::<__m128i>()) + } + + #[inline(always)] + unsafe fn is_zero(self) -> bool { + let cmp = self.cmpeq(Self::splat(0)); + _mm_movemask_epi8(cmp).to_bits() == 0xFFFF + } + + #[inline(always)] + unsafe fn cmpeq(self, vector2: Self) -> __m128i { + _mm_cmpeq_epi8(self, vector2) + } + + #[inline(always)] + unsafe fn and(self, vector2: Self) -> __m128i { + _mm_and_si128(self, vector2) + } + + #[inline(always)] + unsafe fn or(self, vector2: Self) -> __m128i { + _mm_or_si128(self, vector2) + } + + #[inline(always)] + unsafe fn shift_8bit_lane_right(self) -> Self { + // Apparently there is no _mm_srli_epi8, so we emulate it by + // shifting 16-bit integers and masking out the high nybble of each + // 8-bit lane (since that nybble will contain bits from the low + // nybble of the previous lane). + let lomask = Self::splat(0xF); + _mm_srli_epi16(self, BITS).and(lomask) + } + + #[inline(always)] + unsafe fn shift_in_one_byte(self, vector2: Self) -> Self { + _mm_alignr_epi8(self, vector2, 15) + } + + #[inline(always)] + unsafe fn shift_in_two_bytes(self, vector2: Self) -> Self { + _mm_alignr_epi8(self, vector2, 14) + } + + #[inline(always)] + unsafe fn shift_in_three_bytes(self, vector2: Self) -> Self { + _mm_alignr_epi8(self, vector2, 13) + } + + #[inline(always)] + unsafe fn shuffle_bytes(self, indices: Self) -> Self { + _mm_shuffle_epi8(self, indices) + } + + #[inline(always)] + unsafe fn for_each_64bit_lane( + self, + mut f: impl FnMut(usize, u64) -> Option, + ) -> Option { + // We could just use _mm_extract_epi64 here, but that requires + // SSE 4.1. It isn't necessarily a problem to just require SSE 4.1, + // but everything else works with SSSE3 so we stick to that subset. + let lanes: [u64; 2] = core::mem::transmute(self); + if let Some(t) = f(0, lanes[0]) { + return Some(t); + } + if let Some(t) = f(1, lanes[1]) { + return Some(t); + } + None + } + } +} + +#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))] +mod x86_64_avx2 { + use core::arch::x86_64::*; + + use crate::util::int::{I32, I64, I8}; + + use super::{FatVector, Vector}; + + impl Vector for __m256i { + const BITS: usize = 256; + const BYTES: usize = 32; + + #[inline(always)] + unsafe fn splat(byte: u8) -> __m256i { + _mm256_set1_epi8(i8::from_bits(byte)) + } + + #[inline(always)] + unsafe fn load_unaligned(data: *const u8) -> __m256i { + _mm256_loadu_si256(data.cast::<__m256i>()) + } + + #[inline(always)] + unsafe fn is_zero(self) -> bool { + let cmp = self.cmpeq(Self::splat(0)); + _mm256_movemask_epi8(cmp).to_bits() == 0xFFFFFFFF + } + + #[inline(always)] + unsafe fn cmpeq(self, vector2: Self) -> __m256i { + _mm256_cmpeq_epi8(self, vector2) + } + + #[inline(always)] + unsafe fn and(self, vector2: Self) -> __m256i { + _mm256_and_si256(self, vector2) + } + + #[inline(always)] + unsafe fn or(self, vector2: Self) -> __m256i { + _mm256_or_si256(self, vector2) + } + + #[inline(always)] + unsafe fn shift_8bit_lane_right(self) -> Self { + let lomask = Self::splat(0xF); + _mm256_srli_epi16(self, BITS).and(lomask) + } + + #[inline(always)] + unsafe fn shift_in_one_byte(self, vector2: Self) -> Self { + // Credit goes to jneem for figuring this out: + // https://github.com/jneem/teddy/blob/9ab5e899ad6ef6911aecd3cf1033f1abe6e1f66c/src/x86/teddy_simd.rs#L145-L184 + // + // TL;DR avx2's PALIGNR instruction is actually just two 128-bit + // PALIGNR instructions, which is not what we want, so we need to + // do some extra shuffling. + let v = _mm256_permute2x128_si256(vector2, self, 0x21); + _mm256_alignr_epi8(self, v, 15) + } + + #[inline(always)] + unsafe fn shift_in_two_bytes(self, vector2: Self) -> Self { + // Credit goes to jneem for figuring this out: + // https://github.com/jneem/teddy/blob/9ab5e899ad6ef6911aecd3cf1033f1abe6e1f66c/src/x86/teddy_simd.rs#L145-L184 + // + // TL;DR avx2's PALIGNR instruction is actually just two 128-bit + // PALIGNR instructions, which is not what we want, so we need to + // do some extra shuffling. + let v = _mm256_permute2x128_si256(vector2, self, 0x21); + _mm256_alignr_epi8(self, v, 14) + } + + #[inline(always)] + unsafe fn shift_in_three_bytes(self, vector2: Self) -> Self { + // Credit goes to jneem for figuring this out: + // https://github.com/jneem/teddy/blob/9ab5e899ad6ef6911aecd3cf1033f1abe6e1f66c/src/x86/teddy_simd.rs#L145-L184 + // + // TL;DR avx2's PALIGNR instruction is actually just two 128-bit + // PALIGNR instructions, which is not what we want, so we need to + // do some extra shuffling. + let v = _mm256_permute2x128_si256(vector2, self, 0x21); + _mm256_alignr_epi8(self, v, 13) + } + + #[inline(always)] + unsafe fn shuffle_bytes(self, indices: Self) -> Self { + _mm256_shuffle_epi8(self, indices) + } + + #[inline(always)] + unsafe fn for_each_64bit_lane( + self, + mut f: impl FnMut(usize, u64) -> Option, + ) -> Option { + // NOTE: At one point in the past, I used transmute to this to + // get a [u64; 4], but it turned out to lead to worse codegen IIRC. + // I've tried it more recently, and it looks like that's no longer + // the case. But since there's no difference, we stick with the + // slightly more complicated but transmute-free version. + let lane = _mm256_extract_epi64(self, 0).to_bits(); + if let Some(t) = f(0, lane) { + return Some(t); + } + let lane = _mm256_extract_epi64(self, 1).to_bits(); + if let Some(t) = f(1, lane) { + return Some(t); + } + let lane = _mm256_extract_epi64(self, 2).to_bits(); + if let Some(t) = f(2, lane) { + return Some(t); + } + let lane = _mm256_extract_epi64(self, 3).to_bits(); + if let Some(t) = f(3, lane) { + return Some(t); + } + None + } + } + + impl FatVector for __m256i { + type Half = __m128i; + + #[inline(always)] + unsafe fn load_half_unaligned(data: *const u8) -> Self { + let half = Self::Half::load_unaligned(data); + _mm256_broadcastsi128_si256(half) + } + + #[inline(always)] + unsafe fn half_shift_in_one_byte(self, vector2: Self) -> Self { + _mm256_alignr_epi8(self, vector2, 15) + } + + #[inline(always)] + unsafe fn half_shift_in_two_bytes(self, vector2: Self) -> Self { + _mm256_alignr_epi8(self, vector2, 14) + } + + #[inline(always)] + unsafe fn half_shift_in_three_bytes(self, vector2: Self) -> Self { + _mm256_alignr_epi8(self, vector2, 13) + } + + #[inline(always)] + unsafe fn swap_halves(self) -> Self { + _mm256_permute4x64_epi64(self, 0x4E) + } + + #[inline(always)] + unsafe fn interleave_low_8bit_lanes(self, vector2: Self) -> Self { + _mm256_unpacklo_epi8(self, vector2) + } + + #[inline(always)] + unsafe fn interleave_high_8bit_lanes(self, vector2: Self) -> Self { + _mm256_unpackhi_epi8(self, vector2) + } + + #[inline(always)] + unsafe fn for_each_low_64bit_lane( + self, + vector2: Self, + mut f: impl FnMut(usize, u64) -> Option, + ) -> Option { + let lane = _mm256_extract_epi64(self, 0).to_bits(); + if let Some(t) = f(0, lane) { + return Some(t); + } + let lane = _mm256_extract_epi64(self, 1).to_bits(); + if let Some(t) = f(1, lane) { + return Some(t); + } + let lane = _mm256_extract_epi64(vector2, 0).to_bits(); + if let Some(t) = f(2, lane) { + return Some(t); + } + let lane = _mm256_extract_epi64(vector2, 1).to_bits(); + if let Some(t) = f(3, lane) { + return Some(t); + } + None + } + } +} + +#[cfg(all( + target_arch = "aarch64", + target_feature = "neon", + target_endian = "little" +))] +mod aarch64_neon { + use core::arch::aarch64::*; + + use super::Vector; + + impl Vector for uint8x16_t { + const BITS: usize = 128; + const BYTES: usize = 16; + + #[inline(always)] + unsafe fn splat(byte: u8) -> uint8x16_t { + vdupq_n_u8(byte) + } + + #[inline(always)] + unsafe fn load_unaligned(data: *const u8) -> uint8x16_t { + vld1q_u8(data) + } + + #[inline(always)] + unsafe fn is_zero(self) -> bool { + // Could also use vmaxvq_u8. + // ... I tried that and couldn't observe any meaningful difference + // in benchmarks. + let maxes = vreinterpretq_u64_u8(vpmaxq_u8(self, self)); + vgetq_lane_u64(maxes, 0) == 0 + } + + #[inline(always)] + unsafe fn cmpeq(self, vector2: Self) -> uint8x16_t { + vceqq_u8(self, vector2) + } + + #[inline(always)] + unsafe fn and(self, vector2: Self) -> uint8x16_t { + vandq_u8(self, vector2) + } + + #[inline(always)] + unsafe fn or(self, vector2: Self) -> uint8x16_t { + vorrq_u8(self, vector2) + } + + #[inline(always)] + unsafe fn shift_8bit_lane_right(self) -> Self { + debug_assert!(BITS <= 7); + vshrq_n_u8(self, BITS) + } + + #[inline(always)] + unsafe fn shift_in_one_byte(self, vector2: Self) -> Self { + vextq_u8(vector2, self, 15) + } + + #[inline(always)] + unsafe fn shift_in_two_bytes(self, vector2: Self) -> Self { + vextq_u8(vector2, self, 14) + } + + #[inline(always)] + unsafe fn shift_in_three_bytes(self, vector2: Self) -> Self { + vextq_u8(vector2, self, 13) + } + + #[inline(always)] + unsafe fn shuffle_bytes(self, indices: Self) -> Self { + vqtbl1q_u8(self, indices) + } + + #[inline(always)] + unsafe fn for_each_64bit_lane( + self, + mut f: impl FnMut(usize, u64) -> Option, + ) -> Option { + let this = vreinterpretq_u64_u8(self); + let lane = vgetq_lane_u64(this, 0); + if let Some(t) = f(0, lane) { + return Some(t); + } + let lane = vgetq_lane_u64(this, 1); + if let Some(t) = f(1, lane) { + return Some(t); + } + None + } + } +} + +#[cfg(all(test, target_arch = "x86_64", target_feature = "sse2"))] +mod tests_x86_64_ssse3 { + use core::arch::x86_64::*; + + use crate::util::int::{I32, U32}; + + use super::*; + + fn is_runnable() -> bool { + std::is_x86_feature_detected!("ssse3") + } + + #[target_feature(enable = "ssse3")] + unsafe fn load(lanes: [u8; 16]) -> __m128i { + __m128i::load_unaligned(&lanes as *const u8) + } + + #[target_feature(enable = "ssse3")] + unsafe fn unload(v: __m128i) -> [u8; 16] { + [ + _mm_extract_epi8(v, 0).to_bits().low_u8(), + _mm_extract_epi8(v, 1).to_bits().low_u8(), + _mm_extract_epi8(v, 2).to_bits().low_u8(), + _mm_extract_epi8(v, 3).to_bits().low_u8(), + _mm_extract_epi8(v, 4).to_bits().low_u8(), + _mm_extract_epi8(v, 5).to_bits().low_u8(), + _mm_extract_epi8(v, 6).to_bits().low_u8(), + _mm_extract_epi8(v, 7).to_bits().low_u8(), + _mm_extract_epi8(v, 8).to_bits().low_u8(), + _mm_extract_epi8(v, 9).to_bits().low_u8(), + _mm_extract_epi8(v, 10).to_bits().low_u8(), + _mm_extract_epi8(v, 11).to_bits().low_u8(), + _mm_extract_epi8(v, 12).to_bits().low_u8(), + _mm_extract_epi8(v, 13).to_bits().low_u8(), + _mm_extract_epi8(v, 14).to_bits().low_u8(), + _mm_extract_epi8(v, 15).to_bits().low_u8(), + ] + } + + #[test] + fn vector_splat() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v = __m128i::splat(0xAF); + assert_eq!( + unload(v), + [ + 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, + 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF + ] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_is_zero() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v = load([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert!(!v.is_zero()); + let v = load([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert!(v.is_zero()); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_cmpeq() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 1]); + let v2 = + load([16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]); + assert_eq!( + unload(v1.cmpeq(v2)), + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_and() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v1 = + load([0, 0, 0, 0, 0, 0b1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + let v2 = + load([0, 0, 0, 0, 0, 0b1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!( + unload(v1.and(v2)), + [0, 0, 0, 0, 0, 0b1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_or() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v1 = + load([0, 0, 0, 0, 0, 0b1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + let v2 = + load([0, 0, 0, 0, 0, 0b1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!( + unload(v1.or(v2)), + [0, 0, 0, 0, 0, 0b1011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shift_8bit_lane_right() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v = load([ + 0, 0, 0, 0, 0b1011, 0b0101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + assert_eq!( + unload(v.shift_8bit_lane_right::<2>()), + [0, 0, 0, 0, 0b0010, 0b0001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_one_byte() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = load([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.shift_in_one_byte(v2)), + [32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_two_bytes() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = load([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.shift_in_two_bytes(v2)), + [31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_three_bytes() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = load([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.shift_in_three_bytes(v2)), + [30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shuffle_bytes() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = + load([0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12]); + assert_eq!( + unload(v1.shuffle_bytes(v2)), + [1, 1, 1, 1, 5, 5, 5, 5, 9, 9, 9, 9, 13, 13, 13, 13], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_for_each_64bit_lane() { + #[target_feature(enable = "ssse3")] + unsafe fn test() { + let v = load([ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, + 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + ]); + let mut lanes = [0u64; 2]; + v.for_each_64bit_lane(|i, lane| { + lanes[i] = lane; + None::<()> + }); + assert_eq!(lanes, [0x0807060504030201, 0x100F0E0D0C0B0A09],); + } + if !is_runnable() { + return; + } + unsafe { test() } + } +} + +#[cfg(all(test, target_arch = "x86_64", target_feature = "sse2"))] +mod tests_x86_64_avx2 { + use core::arch::x86_64::*; + + use crate::util::int::{I32, U32}; + + use super::*; + + fn is_runnable() -> bool { + std::is_x86_feature_detected!("avx2") + } + + #[target_feature(enable = "avx2")] + unsafe fn load(lanes: [u8; 32]) -> __m256i { + __m256i::load_unaligned(&lanes as *const u8) + } + + #[target_feature(enable = "avx2")] + unsafe fn load_half(lanes: [u8; 16]) -> __m256i { + __m256i::load_half_unaligned(&lanes as *const u8) + } + + #[target_feature(enable = "avx2")] + unsafe fn unload(v: __m256i) -> [u8; 32] { + [ + _mm256_extract_epi8(v, 0).to_bits().low_u8(), + _mm256_extract_epi8(v, 1).to_bits().low_u8(), + _mm256_extract_epi8(v, 2).to_bits().low_u8(), + _mm256_extract_epi8(v, 3).to_bits().low_u8(), + _mm256_extract_epi8(v, 4).to_bits().low_u8(), + _mm256_extract_epi8(v, 5).to_bits().low_u8(), + _mm256_extract_epi8(v, 6).to_bits().low_u8(), + _mm256_extract_epi8(v, 7).to_bits().low_u8(), + _mm256_extract_epi8(v, 8).to_bits().low_u8(), + _mm256_extract_epi8(v, 9).to_bits().low_u8(), + _mm256_extract_epi8(v, 10).to_bits().low_u8(), + _mm256_extract_epi8(v, 11).to_bits().low_u8(), + _mm256_extract_epi8(v, 12).to_bits().low_u8(), + _mm256_extract_epi8(v, 13).to_bits().low_u8(), + _mm256_extract_epi8(v, 14).to_bits().low_u8(), + _mm256_extract_epi8(v, 15).to_bits().low_u8(), + _mm256_extract_epi8(v, 16).to_bits().low_u8(), + _mm256_extract_epi8(v, 17).to_bits().low_u8(), + _mm256_extract_epi8(v, 18).to_bits().low_u8(), + _mm256_extract_epi8(v, 19).to_bits().low_u8(), + _mm256_extract_epi8(v, 20).to_bits().low_u8(), + _mm256_extract_epi8(v, 21).to_bits().low_u8(), + _mm256_extract_epi8(v, 22).to_bits().low_u8(), + _mm256_extract_epi8(v, 23).to_bits().low_u8(), + _mm256_extract_epi8(v, 24).to_bits().low_u8(), + _mm256_extract_epi8(v, 25).to_bits().low_u8(), + _mm256_extract_epi8(v, 26).to_bits().low_u8(), + _mm256_extract_epi8(v, 27).to_bits().low_u8(), + _mm256_extract_epi8(v, 28).to_bits().low_u8(), + _mm256_extract_epi8(v, 29).to_bits().low_u8(), + _mm256_extract_epi8(v, 30).to_bits().low_u8(), + _mm256_extract_epi8(v, 31).to_bits().low_u8(), + ] + } + + #[test] + fn vector_splat() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v = __m256i::splat(0xAF); + assert_eq!( + unload(v), + [ + 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, + 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, + 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, + 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, + ] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_is_zero() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v = load([ + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + assert!(!v.is_zero()); + let v = load([ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + assert!(v.is_zero()); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_cmpeq() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 1, + ]); + let v2 = load([ + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, + ]); + assert_eq!( + unload(v1.cmpeq(v2)), + [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF + ] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_and() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 0, 0, 0, 0, 0, 0b1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + let v2 = load([ + 0, 0, 0, 0, 0, 0b1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + assert_eq!( + unload(v1.and(v2)), + [ + 0, 0, 0, 0, 0, 0b1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_or() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 0, 0, 0, 0, 0, 0b1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + let v2 = load([ + 0, 0, 0, 0, 0, 0b1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + assert_eq!( + unload(v1.or(v2)), + [ + 0, 0, 0, 0, 0, 0b1011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shift_8bit_lane_right() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v = load([ + 0, 0, 0, 0, 0b1011, 0b0101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + assert_eq!( + unload(v.shift_8bit_lane_right::<2>()), + [ + 0, 0, 0, 0, 0b0010, 0b0001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_one_byte() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + let v2 = load([ + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, + ]); + assert_eq!( + unload(v1.shift_in_one_byte(v2)), + [ + 64, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_two_bytes() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + let v2 = load([ + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, + ]); + assert_eq!( + unload(v1.shift_in_two_bytes(v2)), + [ + 63, 64, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_three_bytes() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + let v2 = load([ + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, + ]); + assert_eq!( + unload(v1.shift_in_three_bytes(v2)), + [ + 62, 63, 64, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_shuffle_bytes() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + let v2 = load([ + 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12, 16, 16, + 16, 16, 20, 20, 20, 20, 24, 24, 24, 24, 28, 28, 28, 28, + ]); + assert_eq!( + unload(v1.shuffle_bytes(v2)), + [ + 1, 1, 1, 1, 5, 5, 5, 5, 9, 9, 9, 9, 13, 13, 13, 13, 17, + 17, 17, 17, 21, 21, 21, 21, 25, 25, 25, 25, 29, 29, 29, + 29 + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn vector_for_each_64bit_lane() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v = load([ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, + 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, + 0x1F, 0x20, + ]); + let mut lanes = [0u64; 4]; + v.for_each_64bit_lane(|i, lane| { + lanes[i] = lane; + None::<()> + }); + assert_eq!( + lanes, + [ + 0x0807060504030201, + 0x100F0E0D0C0B0A09, + 0x1817161514131211, + 0x201F1E1D1C1B1A19 + ] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn fat_vector_half_shift_in_one_byte() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load_half([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ]); + let v2 = load_half([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.half_shift_in_one_byte(v2)), + [ + 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 32, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn fat_vector_half_shift_in_two_bytes() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load_half([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ]); + let v2 = load_half([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.half_shift_in_two_bytes(v2)), + [ + 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 31, + 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn fat_vector_half_shift_in_three_bytes() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load_half([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + ]); + let v2 = load_half([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.half_shift_in_three_bytes(v2)), + [ + 30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 30, + 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn fat_vector_swap_halves() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v = load([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v.swap_halves()), + [ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn fat_vector_interleave_low_8bit_lanes() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + let v2 = load([ + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, + ]); + assert_eq!( + unload(v1.interleave_low_8bit_lanes(v2)), + [ + 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, + 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, + 24, 56, + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn fat_vector_interleave_high_8bit_lanes() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + let v2 = load([ + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, + ]); + assert_eq!( + unload(v1.interleave_high_8bit_lanes(v2)), + [ + 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, 16, + 48, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, + 63, 32, 64, + ], + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } + + #[test] + fn fat_vector_for_each_low_64bit_lane() { + #[target_feature(enable = "avx2")] + unsafe fn test() { + let v1 = load([ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, + 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, + 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, + 0x1F, 0x20, + ]); + let v2 = load([ + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, + 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34, + 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, + 0x3F, 0x40, + ]); + let mut lanes = [0u64; 4]; + v1.for_each_low_64bit_lane(v2, |i, lane| { + lanes[i] = lane; + None::<()> + }); + assert_eq!( + lanes, + [ + 0x0807060504030201, + 0x100F0E0D0C0B0A09, + 0x2827262524232221, + 0x302F2E2D2C2B2A29 + ] + ); + } + if !is_runnable() { + return; + } + unsafe { test() } + } +} + +#[cfg(all(test, target_arch = "aarch64", target_feature = "neon"))] +mod tests_aarch64_neon { + use core::arch::aarch64::*; + + use super::*; + + #[target_feature(enable = "neon")] + unsafe fn load(lanes: [u8; 16]) -> uint8x16_t { + uint8x16_t::load_unaligned(&lanes as *const u8) + } + + #[target_feature(enable = "neon")] + unsafe fn unload(v: uint8x16_t) -> [u8; 16] { + [ + vgetq_lane_u8(v, 0), + vgetq_lane_u8(v, 1), + vgetq_lane_u8(v, 2), + vgetq_lane_u8(v, 3), + vgetq_lane_u8(v, 4), + vgetq_lane_u8(v, 5), + vgetq_lane_u8(v, 6), + vgetq_lane_u8(v, 7), + vgetq_lane_u8(v, 8), + vgetq_lane_u8(v, 9), + vgetq_lane_u8(v, 10), + vgetq_lane_u8(v, 11), + vgetq_lane_u8(v, 12), + vgetq_lane_u8(v, 13), + vgetq_lane_u8(v, 14), + vgetq_lane_u8(v, 15), + ] + } + + // Example functions. These don't test the Vector traits, but rather, + // specific NEON instructions. They are basically little experiments I + // wrote to figure out what an instruction does since their descriptions + // are so dense. I decided to keep the experiments around as example tests + // in case there' useful. + + #[test] + fn example_vmaxvq_u8_non_zero() { + #[target_feature(enable = "neon")] + unsafe fn example() { + let v = load([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(vmaxvq_u8(v), 1); + } + unsafe { example() } + } + + #[test] + fn example_vmaxvq_u8_zero() { + #[target_feature(enable = "neon")] + unsafe fn example() { + let v = load([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(vmaxvq_u8(v), 0); + } + unsafe { example() } + } + + #[test] + fn example_vpmaxq_u8_non_zero() { + #[target_feature(enable = "neon")] + unsafe fn example() { + let v = load([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + let r = vpmaxq_u8(v, v); + assert_eq!( + unload(r), + [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] + ); + } + unsafe { example() } + } + + #[test] + fn example_vpmaxq_u8_self() { + #[target_feature(enable = "neon")] + unsafe fn example() { + let v = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let r = vpmaxq_u8(v, v); + assert_eq!( + unload(r), + [2, 4, 6, 8, 10, 12, 14, 16, 2, 4, 6, 8, 10, 12, 14, 16] + ); + } + unsafe { example() } + } + + #[test] + fn example_vpmaxq_u8_other() { + #[target_feature(enable = "neon")] + unsafe fn example() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = load([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + let r = vpmaxq_u8(v1, v2); + assert_eq!( + unload(r), + [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] + ); + } + unsafe { example() } + } + + // Now we test the actual methods on the Vector trait. + + #[test] + fn vector_splat() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v = uint8x16_t::splat(0xAF); + assert_eq!( + unload(v), + [ + 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, + 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF, 0xAF + ] + ); + } + unsafe { test() } + } + + #[test] + fn vector_is_zero() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v = load([0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert!(!v.is_zero()); + let v = load([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert!(v.is_zero()); + } + unsafe { test() } + } + + #[test] + fn vector_cmpeq() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 1]); + let v2 = + load([16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]); + assert_eq!( + unload(v1.cmpeq(v2)), + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF] + ); + } + unsafe { test() } + } + + #[test] + fn vector_and() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v1 = + load([0, 0, 0, 0, 0, 0b1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + let v2 = + load([0, 0, 0, 0, 0, 0b1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!( + unload(v1.and(v2)), + [0, 0, 0, 0, 0, 0b1000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + unsafe { test() } + } + + #[test] + fn vector_or() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v1 = + load([0, 0, 0, 0, 0, 0b1001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + let v2 = + load([0, 0, 0, 0, 0, 0b1010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!( + unload(v1.or(v2)), + [0, 0, 0, 0, 0, 0b1011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + unsafe { test() } + } + + #[test] + fn vector_shift_8bit_lane_right() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v = load([ + 0, 0, 0, 0, 0b1011, 0b0101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + assert_eq!( + unload(v.shift_8bit_lane_right::<2>()), + [0, 0, 0, 0, 0b0010, 0b0001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + ); + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_one_byte() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = load([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.shift_in_one_byte(v2)), + [32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + ); + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_two_bytes() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = load([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.shift_in_two_bytes(v2)), + [31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], + ); + } + unsafe { test() } + } + + #[test] + fn vector_shift_in_three_bytes() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = load([ + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + ]); + assert_eq!( + unload(v1.shift_in_three_bytes(v2)), + [30, 31, 32, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], + ); + } + unsafe { test() } + } + + #[test] + fn vector_shuffle_bytes() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v1 = + load([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + let v2 = + load([0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12]); + assert_eq!( + unload(v1.shuffle_bytes(v2)), + [1, 1, 1, 1, 5, 5, 5, 5, 9, 9, 9, 9, 13, 13, 13, 13], + ); + } + unsafe { test() } + } + + #[test] + fn vector_for_each_64bit_lane() { + #[target_feature(enable = "neon")] + unsafe fn test() { + let v = load([ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, + 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + ]); + let mut lanes = [0u64; 2]; + v.for_each_64bit_lane(|i, lane| { + lanes[i] = lane; + None::<()> + }); + assert_eq!(lanes, [0x0807060504030201, 0x100F0E0D0C0B0A09],); + } + unsafe { test() } + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/tests.rs b/third_party/crates/aho-corasick-1.1.5/src/tests.rs new file mode 100644 index 0000000..a5276f8 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/tests.rs @@ -0,0 +1,1664 @@ +use std::{collections::HashMap, format, string::String, vec::Vec}; + +use crate::{ + AhoCorasick, AhoCorasickBuilder, AhoCorasickKind, Anchored, Input, Match, + MatchKind, StartKind, +}; + +/// A description of a single test against an Aho-Corasick automaton. +/// +/// A single test may not necessarily pass on every configuration of an +/// Aho-Corasick automaton. The tests are categorized and grouped appropriately +/// below. +#[derive(Clone, Debug, Eq, PartialEq)] +struct SearchTest { + /// The name of this test, for debugging. + name: &'static str, + /// The patterns to search for. + patterns: &'static [&'static str], + /// The text to search. + haystack: &'static str, + /// Each match is a triple of (pattern_index, start, end), where + /// pattern_index is an index into `patterns` and `start`/`end` are indices + /// into `haystack`. + matches: &'static [(usize, usize, usize)], +} + +/// Short-hand constructor for SearchTest. We use it a lot below. +macro_rules! t { + ($name:ident, $patterns:expr, $haystack:expr, $matches:expr) => { + SearchTest { + name: stringify!($name), + patterns: $patterns, + haystack: $haystack, + matches: $matches, + } + }; +} + +/// A collection of test groups. +type TestCollection = &'static [&'static [SearchTest]]; + +// Define several collections corresponding to the different type of match +// semantics supported by Aho-Corasick. These collections have some overlap, +// but each collection should have some tests that no other collection has. + +/// Tests for Aho-Corasick's standard non-overlapping match semantics. +const AC_STANDARD_NON_OVERLAPPING: TestCollection = + &[BASICS, NON_OVERLAPPING, STANDARD, REGRESSION]; + +/// Tests for Aho-Corasick's anchored standard non-overlapping match semantics. +const AC_STANDARD_ANCHORED_NON_OVERLAPPING: TestCollection = + &[ANCHORED_BASICS, ANCHORED_NON_OVERLAPPING, STANDARD_ANCHORED]; + +/// Tests for Aho-Corasick's standard overlapping match semantics. +const AC_STANDARD_OVERLAPPING: TestCollection = + &[BASICS, OVERLAPPING, REGRESSION]; + +/* +Iterators of anchored overlapping searches were removed from the API in +after 0.7, but we leave the tests commented out for posterity. +/// Tests for Aho-Corasick's anchored standard overlapping match semantics. +const AC_STANDARD_ANCHORED_OVERLAPPING: TestCollection = + &[ANCHORED_BASICS, ANCHORED_OVERLAPPING]; +*/ + +/// Tests for Aho-Corasick's leftmost-first match semantics. +const AC_LEFTMOST_FIRST: TestCollection = + &[BASICS, NON_OVERLAPPING, LEFTMOST, LEFTMOST_FIRST, REGRESSION]; + +/// Tests for Aho-Corasick's anchored leftmost-first match semantics. +const AC_LEFTMOST_FIRST_ANCHORED: TestCollection = &[ + ANCHORED_BASICS, + ANCHORED_NON_OVERLAPPING, + ANCHORED_LEFTMOST, + ANCHORED_LEFTMOST_FIRST, +]; + +/// Tests for Aho-Corasick's leftmost-longest match semantics. +const AC_LEFTMOST_LONGEST: TestCollection = + &[BASICS, NON_OVERLAPPING, LEFTMOST, LEFTMOST_LONGEST, REGRESSION]; + +/// Tests for Aho-Corasick's anchored leftmost-longest match semantics. +const AC_LEFTMOST_LONGEST_ANCHORED: TestCollection = &[ + ANCHORED_BASICS, + ANCHORED_NON_OVERLAPPING, + ANCHORED_LEFTMOST, + ANCHORED_LEFTMOST_LONGEST, +]; + +// Now define the individual tests that make up the collections above. + +/// A collection of tests for the Aho-Corasick algorithm that should always be +/// true regardless of match semantics. That is, all combinations of +/// leftmost-{shortest, first, longest} x {overlapping, non-overlapping} +/// should produce the same answer. +const BASICS: &'static [SearchTest] = &[ + t!(basic000, &[], "", &[]), + t!(basic001, &[""], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(basic002, &["a"], "", &[]), + t!(basic010, &["a"], "a", &[(0, 0, 1)]), + t!(basic020, &["a"], "aa", &[(0, 0, 1), (0, 1, 2)]), + t!(basic030, &["a"], "aaa", &[(0, 0, 1), (0, 1, 2), (0, 2, 3)]), + t!(basic040, &["a"], "aba", &[(0, 0, 1), (0, 2, 3)]), + t!(basic050, &["a"], "bba", &[(0, 2, 3)]), + t!(basic060, &["a"], "bbb", &[]), + t!(basic070, &["a"], "bababbbba", &[(0, 1, 2), (0, 3, 4), (0, 8, 9)]), + t!(basic100, &["aa"], "", &[]), + t!(basic110, &["aa"], "aa", &[(0, 0, 2)]), + t!(basic120, &["aa"], "aabbaa", &[(0, 0, 2), (0, 4, 6)]), + t!(basic130, &["aa"], "abbab", &[]), + t!(basic140, &["aa"], "abbabaa", &[(0, 5, 7)]), + t!(basic200, &["abc"], "abc", &[(0, 0, 3)]), + t!(basic210, &["abc"], "zazabzabcz", &[(0, 6, 9)]), + t!(basic220, &["abc"], "zazabczabcz", &[(0, 3, 6), (0, 7, 10)]), + t!(basic300, &["a", "b"], "", &[]), + t!(basic310, &["a", "b"], "z", &[]), + t!(basic320, &["a", "b"], "b", &[(1, 0, 1)]), + t!(basic330, &["a", "b"], "a", &[(0, 0, 1)]), + t!( + basic340, + &["a", "b"], + "abba", + &[(0, 0, 1), (1, 1, 2), (1, 2, 3), (0, 3, 4),] + ), + t!( + basic350, + &["b", "a"], + "abba", + &[(1, 0, 1), (0, 1, 2), (0, 2, 3), (1, 3, 4),] + ), + t!(basic360, &["abc", "bc"], "xbc", &[(1, 1, 3),]), + t!(basic400, &["foo", "bar"], "", &[]), + t!(basic410, &["foo", "bar"], "foobar", &[(0, 0, 3), (1, 3, 6),]), + t!(basic420, &["foo", "bar"], "barfoo", &[(1, 0, 3), (0, 3, 6),]), + t!(basic430, &["foo", "bar"], "foofoo", &[(0, 0, 3), (0, 3, 6),]), + t!(basic440, &["foo", "bar"], "barbar", &[(1, 0, 3), (1, 3, 6),]), + t!(basic450, &["foo", "bar"], "bafofoo", &[(0, 4, 7),]), + t!(basic460, &["bar", "foo"], "bafofoo", &[(1, 4, 7),]), + t!(basic470, &["foo", "bar"], "fobabar", &[(1, 4, 7),]), + t!(basic480, &["bar", "foo"], "fobabar", &[(0, 4, 7),]), + t!(basic600, &[""], "", &[(0, 0, 0)]), + t!(basic610, &[""], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(basic620, &[""], "abc", &[(0, 0, 0), (0, 1, 1), (0, 2, 2), (0, 3, 3)]), + t!(basic700, &["yabcdef", "abcdezghi"], "yabcdefghi", &[(0, 0, 7),]), + t!(basic710, &["yabcdef", "abcdezghi"], "yabcdezghi", &[(1, 1, 10),]), + t!( + basic720, + &["yabcdef", "bcdeyabc", "abcdezghi"], + "yabcdezghi", + &[(2, 1, 10),] + ), +]; + +/// A collection of *anchored* tests for the Aho-Corasick algorithm that should +/// always be true regardless of match semantics. That is, all combinations of +/// leftmost-{shortest, first, longest} x {overlapping, non-overlapping} should +/// produce the same answer. +const ANCHORED_BASICS: &'static [SearchTest] = &[ + t!(abasic000, &[], "", &[]), + t!(abasic001, &[], "a", &[]), + t!(abasic002, &[], "abc", &[]), + t!(abasic010, &[""], "", &[(0, 0, 0)]), + t!(abasic020, &[""], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(abasic030, &[""], "abc", &[(0, 0, 0), (0, 1, 1), (0, 2, 2), (0, 3, 3)]), + t!(abasic100, &["a"], "a", &[(0, 0, 1)]), + t!(abasic110, &["a"], "aa", &[(0, 0, 1), (0, 1, 2)]), + t!(abasic120, &["a", "b"], "ab", &[(0, 0, 1), (1, 1, 2)]), + t!(abasic130, &["a", "b"], "ba", &[(1, 0, 1), (0, 1, 2)]), + t!(abasic140, &["foo", "foofoo"], "foo", &[(0, 0, 3)]), + t!(abasic150, &["foofoo", "foo"], "foo", &[(1, 0, 3)]), + t!(abasic200, &["foo"], "foofoo foo", &[(0, 0, 3), (0, 3, 6)]), +]; + +/// Tests for non-overlapping standard match semantics. +/// +/// These tests generally shouldn't pass for leftmost-{first,longest}, although +/// some do in order to write clearer tests. For example, standard000 will +/// pass with leftmost-first semantics, but standard010 will not. We write +/// both to emphasize how the match semantics work. +const STANDARD: &'static [SearchTest] = &[ + t!(standard000, &["ab", "abcd"], "abcd", &[(0, 0, 2)]), + t!(standard010, &["abcd", "ab"], "abcd", &[(1, 0, 2)]), + t!(standard020, &["abcd", "ab", "abc"], "abcd", &[(1, 0, 2)]), + t!(standard030, &["abcd", "abc", "ab"], "abcd", &[(2, 0, 2)]), + t!(standard040, &["a", ""], "a", &[(1, 0, 0), (1, 1, 1)]), + t!( + standard400, + &["abcd", "bcd", "cd", "b"], + "abcd", + &[(3, 1, 2), (2, 2, 4),] + ), + t!(standard410, &["", "a"], "a", &[(0, 0, 0), (0, 1, 1),]), + t!(standard420, &["", "a"], "aa", &[(0, 0, 0), (0, 1, 1), (0, 2, 2),]), + t!(standard430, &["", "a", ""], "a", &[(0, 0, 0), (0, 1, 1),]), + t!(standard440, &["a", "", ""], "a", &[(1, 0, 0), (1, 1, 1),]), + t!(standard450, &["", "", "a"], "a", &[(0, 0, 0), (0, 1, 1),]), +]; + +/// Like STANDARD, but for anchored searches. +const STANDARD_ANCHORED: &'static [SearchTest] = &[ + t!(astandard000, &["ab", "abcd"], "abcd", &[(0, 0, 2)]), + t!(astandard010, &["abcd", "ab"], "abcd", &[(1, 0, 2)]), + t!(astandard020, &["abcd", "ab", "abc"], "abcd", &[(1, 0, 2)]), + t!(astandard030, &["abcd", "abc", "ab"], "abcd", &[(2, 0, 2)]), + t!(astandard040, &["a", ""], "a", &[(1, 0, 0), (1, 1, 1)]), + t!(astandard050, &["abcd", "bcd", "cd", "b"], "abcd", &[(0, 0, 4)]), + t!(astandard410, &["", "a"], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(astandard420, &["", "a"], "aa", &[(0, 0, 0), (0, 1, 1), (0, 2, 2)]), + t!(astandard430, &["", "a", ""], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(astandard440, &["a", "", ""], "a", &[(1, 0, 0), (1, 1, 1)]), + t!(astandard450, &["", "", "a"], "a", &[(0, 0, 0), (0, 1, 1)]), +]; + +/// Tests for non-overlapping leftmost match semantics. These should pass for +/// both leftmost-first and leftmost-longest match kinds. Stated differently, +/// among ambiguous matches, the longest match and the match that appeared +/// first when constructing the automaton should always be the same. +const LEFTMOST: &'static [SearchTest] = &[ + t!(leftmost000, &["ab", "ab"], "abcd", &[(0, 0, 2)]), + t!(leftmost010, &["a", ""], "a", &[(0, 0, 1)]), + t!(leftmost011, &["a", ""], "ab", &[(0, 0, 1), (1, 2, 2)]), + t!(leftmost020, &["", ""], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(leftmost030, &["a", "ab"], "aa", &[(0, 0, 1), (0, 1, 2)]), + t!(leftmost031, &["ab", "a"], "aa", &[(1, 0, 1), (1, 1, 2)]), + t!(leftmost032, &["ab", "a"], "xayabbbz", &[(1, 1, 2), (0, 3, 5)]), + t!(leftmost300, &["abcd", "bce", "b"], "abce", &[(1, 1, 4)]), + t!(leftmost310, &["abcd", "ce", "bc"], "abce", &[(2, 1, 3)]), + t!(leftmost320, &["abcd", "bce", "ce", "b"], "abce", &[(1, 1, 4)]), + t!(leftmost330, &["abcd", "bce", "cz", "bc"], "abcz", &[(3, 1, 3)]), + t!(leftmost340, &["bce", "cz", "bc"], "bcz", &[(2, 0, 2)]), + t!(leftmost350, &["abc", "bd", "ab"], "abd", &[(2, 0, 2)]), + t!( + leftmost360, + &["abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(2, 0, 8),] + ), + t!( + leftmost370, + &["abcdefghi", "cde", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + leftmost380, + &["abcdefghi", "hz", "abcdefgh", "a"], + "abcdefghz", + &[(2, 0, 8),] + ), + t!( + leftmost390, + &["b", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + leftmost400, + &["h", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + leftmost410, + &["z", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8), (0, 8, 9),] + ), +]; + +/// Like LEFTMOST, but for anchored searches. +const ANCHORED_LEFTMOST: &'static [SearchTest] = &[ + t!(aleftmost000, &["ab", "ab"], "abcd", &[(0, 0, 2)]), + // We shouldn't allow an empty match immediately following a match, right? + t!(aleftmost010, &["a", ""], "a", &[(0, 0, 1)]), + t!(aleftmost020, &["", ""], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(aleftmost030, &["a", "ab"], "aa", &[(0, 0, 1), (0, 1, 2)]), + t!(aleftmost031, &["ab", "a"], "aa", &[(1, 0, 1), (1, 1, 2)]), + t!(aleftmost032, &["ab", "a"], "xayabbbz", &[]), + t!(aleftmost300, &["abcd", "bce", "b"], "abce", &[]), + t!(aleftmost301, &["abcd", "bcd", "cd", "b"], "abcd", &[(0, 0, 4)]), + t!(aleftmost310, &["abcd", "ce", "bc"], "abce", &[]), + t!(aleftmost320, &["abcd", "bce", "ce", "b"], "abce", &[]), + t!(aleftmost330, &["abcd", "bce", "cz", "bc"], "abcz", &[]), + t!(aleftmost340, &["bce", "cz", "bc"], "bcz", &[(2, 0, 2)]), + t!(aleftmost350, &["abc", "bd", "ab"], "abd", &[(2, 0, 2)]), + t!( + aleftmost360, + &["abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(2, 0, 8),] + ), + t!( + aleftmost370, + &["abcdefghi", "cde", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + aleftmost380, + &["abcdefghi", "hz", "abcdefgh", "a"], + "abcdefghz", + &[(2, 0, 8),] + ), + t!( + aleftmost390, + &["b", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + aleftmost400, + &["h", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!( + aleftmost410, + &["z", "abcdefghi", "hz", "abcdefgh"], + "abcdefghzyz", + &[(3, 0, 8), (0, 8, 9)] + ), +]; + +/// Tests for non-overlapping leftmost-first match semantics. These tests +/// should generally be specific to leftmost-first, which means they should +/// generally fail under leftmost-longest semantics. +const LEFTMOST_FIRST: &'static [SearchTest] = &[ + t!(leftfirst000, &["ab", "abcd"], "abcd", &[(0, 0, 2)]), + t!(leftfirst010, &["", "a"], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(leftfirst011, &["", "a", ""], "a", &[(0, 0, 0), (0, 1, 1),]), + t!(leftfirst012, &["a", "", ""], "a", &[(0, 0, 1)]), + t!(leftfirst013, &["", "", "a"], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(leftfirst014, &["a", ""], "a", &[(0, 0, 1)]), + t!(leftfirst015, &["a", ""], "ab", &[(0, 0, 1), (1, 2, 2)]), + t!(leftfirst020, &["abcd", "ab"], "abcd", &[(0, 0, 4)]), + t!(leftfirst030, &["ab", "ab"], "abcd", &[(0, 0, 2)]), + t!(leftfirst040, &["a", "ab"], "xayabbbz", &[(0, 1, 2), (0, 3, 4)]), + t!(leftfirst100, &["abcdefg", "bcde", "bcdef"], "abcdef", &[(1, 1, 5)]), + t!(leftfirst110, &["abcdefg", "bcdef", "bcde"], "abcdef", &[(1, 1, 6)]), + t!(leftfirst300, &["abcd", "b", "bce"], "abce", &[(1, 1, 2)]), + t!( + leftfirst310, + &["abcd", "b", "bce", "ce"], + "abce", + &[(1, 1, 2), (3, 2, 4),] + ), + t!( + leftfirst320, + &["a", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(0, 0, 1), (2, 7, 9),] + ), + t!(leftfirst330, &["a", "abab"], "abab", &[(0, 0, 1), (0, 2, 3)]), + t!(leftfirst400, &["amwix", "samwise", "sam"], "Zsamwix", &[(2, 1, 4)]), +]; + +/// Like LEFTMOST_FIRST, but for anchored searches. +const ANCHORED_LEFTMOST_FIRST: &'static [SearchTest] = &[ + t!(aleftfirst000, &["ab", "abcd"], "abcd", &[(0, 0, 2)]), + t!(aleftfirst010, &["", "a"], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(aleftfirst011, &["", "a", ""], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(aleftfirst012, &["a", "", ""], "a", &[(0, 0, 1)]), + t!(aleftfirst013, &["", "", "a"], "a", &[(0, 0, 0), (0, 1, 1)]), + t!(aleftfirst020, &["abcd", "ab"], "abcd", &[(0, 0, 4)]), + t!(aleftfirst030, &["ab", "ab"], "abcd", &[(0, 0, 2)]), + t!(aleftfirst040, &["a", "ab"], "xayabbbz", &[]), + t!(aleftfirst100, &["abcdefg", "bcde", "bcdef"], "abcdef", &[]), + t!(aleftfirst110, &["abcdefg", "bcdef", "bcde"], "abcdef", &[]), + t!(aleftfirst300, &["abcd", "b", "bce"], "abce", &[]), + t!(aleftfirst310, &["abcd", "b", "bce", "ce"], "abce", &[]), + t!( + aleftfirst320, + &["a", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(0, 0, 1)] + ), + t!(aleftfirst330, &["a", "abab"], "abab", &[(0, 0, 1)]), + t!(aleftfirst400, &["wise", "samwise", "sam"], "samwix", &[(2, 0, 3)]), +]; + +/// Tests for non-overlapping leftmost-longest match semantics. These tests +/// should generally be specific to leftmost-longest, which means they should +/// generally fail under leftmost-first semantics. +const LEFTMOST_LONGEST: &'static [SearchTest] = &[ + t!(leftlong000, &["ab", "abcd"], "abcd", &[(1, 0, 4)]), + t!(leftlong010, &["abcd", "bcd", "cd", "b"], "abcd", &[(0, 0, 4),]), + t!(leftlong020, &["", "a"], "a", &[(1, 0, 1)]), + t!(leftlong021, &["", "a", ""], "a", &[(1, 0, 1)]), + t!(leftlong022, &["a", "", ""], "a", &[(0, 0, 1)]), + t!(leftlong023, &["", "", "a"], "a", &[(2, 0, 1)]), + t!(leftlong024, &["", "a"], "ab", &[(1, 0, 1), (0, 2, 2)]), + t!(leftlong030, &["", "a"], "aa", &[(1, 0, 1), (1, 1, 2)]), + t!(leftlong040, &["a", "ab"], "a", &[(0, 0, 1)]), + t!(leftlong050, &["a", "ab"], "ab", &[(1, 0, 2)]), + t!(leftlong060, &["ab", "a"], "a", &[(1, 0, 1)]), + t!(leftlong070, &["ab", "a"], "ab", &[(0, 0, 2)]), + t!(leftlong100, &["abcdefg", "bcde", "bcdef"], "abcdef", &[(2, 1, 6)]), + t!(leftlong110, &["abcdefg", "bcdef", "bcde"], "abcdef", &[(1, 1, 6)]), + t!(leftlong300, &["abcd", "b", "bce"], "abce", &[(2, 1, 4)]), + t!( + leftlong310, + &["a", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!(leftlong320, &["a", "abab"], "abab", &[(1, 0, 4)]), + t!(leftlong330, &["abcd", "b", "ce"], "abce", &[(1, 1, 2), (2, 2, 4),]), + t!(leftlong340, &["a", "ab"], "xayabbbz", &[(0, 1, 2), (1, 3, 5)]), +]; + +/// Like LEFTMOST_LONGEST, but for anchored searches. +const ANCHORED_LEFTMOST_LONGEST: &'static [SearchTest] = &[ + t!(aleftlong000, &["ab", "abcd"], "abcd", &[(1, 0, 4)]), + t!(aleftlong010, &["abcd", "bcd", "cd", "b"], "abcd", &[(0, 0, 4),]), + t!(aleftlong020, &["", "a"], "a", &[(1, 0, 1)]), + t!(aleftlong021, &["", "a", ""], "a", &[(1, 0, 1)]), + t!(aleftlong022, &["a", "", ""], "a", &[(0, 0, 1)]), + t!(aleftlong023, &["", "", "a"], "a", &[(2, 0, 1)]), + t!(aleftlong030, &["", "a"], "aa", &[(1, 0, 1), (1, 1, 2)]), + t!(aleftlong040, &["a", "ab"], "a", &[(0, 0, 1)]), + t!(aleftlong050, &["a", "ab"], "ab", &[(1, 0, 2)]), + t!(aleftlong060, &["ab", "a"], "a", &[(1, 0, 1)]), + t!(aleftlong070, &["ab", "a"], "ab", &[(0, 0, 2)]), + t!(aleftlong100, &["abcdefg", "bcde", "bcdef"], "abcdef", &[]), + t!(aleftlong110, &["abcdefg", "bcdef", "bcde"], "abcdef", &[]), + t!(aleftlong300, &["abcd", "b", "bce"], "abce", &[]), + t!( + aleftlong310, + &["a", "abcdefghi", "hz", "abcdefgh"], + "abcdefghz", + &[(3, 0, 8),] + ), + t!(aleftlong320, &["a", "abab"], "abab", &[(1, 0, 4)]), + t!(aleftlong330, &["abcd", "b", "ce"], "abce", &[]), + t!(aleftlong340, &["a", "ab"], "xayabbbz", &[]), +]; + +/// Tests for non-overlapping match semantics. +/// +/// Generally these tests shouldn't pass when using overlapping semantics. +/// These should pass for both standard and leftmost match semantics. +const NON_OVERLAPPING: &'static [SearchTest] = &[ + t!(nover010, &["abcd", "bcd", "cd"], "abcd", &[(0, 0, 4),]), + t!(nover020, &["bcd", "cd", "abcd"], "abcd", &[(2, 0, 4),]), + t!(nover030, &["abc", "bc"], "zazabcz", &[(0, 3, 6),]), + t!( + nover100, + &["ab", "ba"], + "abababa", + &[(0, 0, 2), (0, 2, 4), (0, 4, 6),] + ), + t!(nover200, &["foo", "foo"], "foobarfoo", &[(0, 0, 3), (0, 6, 9),]), + t!(nover300, &["", ""], "", &[(0, 0, 0),]), + t!(nover310, &["", ""], "a", &[(0, 0, 0), (0, 1, 1),]), +]; + +/// Like NON_OVERLAPPING, but for anchored searches. +const ANCHORED_NON_OVERLAPPING: &'static [SearchTest] = &[ + t!(anover010, &["abcd", "bcd", "cd"], "abcd", &[(0, 0, 4),]), + t!(anover020, &["bcd", "cd", "abcd"], "abcd", &[(2, 0, 4),]), + t!(anover030, &["abc", "bc"], "zazabcz", &[]), + t!( + anover100, + &["ab", "ba"], + "abababa", + &[(0, 0, 2), (0, 2, 4), (0, 4, 6)] + ), + t!(anover200, &["foo", "foo"], "foobarfoo", &[(0, 0, 3)]), + t!(anover300, &["", ""], "", &[(0, 0, 0)]), + t!(anover310, &["", ""], "a", &[(0, 0, 0), (0, 1, 1)]), +]; + +/// Tests for overlapping match semantics. +/// +/// This only supports standard match semantics, since leftmost-{first,longest} +/// do not support overlapping matches. +const OVERLAPPING: &'static [SearchTest] = &[ + t!( + over000, + &["abcd", "bcd", "cd", "b"], + "abcd", + &[(3, 1, 2), (0, 0, 4), (1, 1, 4), (2, 2, 4),] + ), + t!( + over010, + &["bcd", "cd", "b", "abcd"], + "abcd", + &[(2, 1, 2), (3, 0, 4), (0, 1, 4), (1, 2, 4),] + ), + t!( + over020, + &["abcd", "bcd", "cd"], + "abcd", + &[(0, 0, 4), (1, 1, 4), (2, 2, 4),] + ), + t!( + over030, + &["bcd", "abcd", "cd"], + "abcd", + &[(1, 0, 4), (0, 1, 4), (2, 2, 4),] + ), + t!( + over040, + &["bcd", "cd", "abcd"], + "abcd", + &[(2, 0, 4), (0, 1, 4), (1, 2, 4),] + ), + t!(over050, &["abc", "bc"], "zazabcz", &[(0, 3, 6), (1, 4, 6),]), + t!( + over100, + &["ab", "ba"], + "abababa", + &[(0, 0, 2), (1, 1, 3), (0, 2, 4), (1, 3, 5), (0, 4, 6), (1, 5, 7),] + ), + t!( + over200, + &["foo", "foo"], + "foobarfoo", + &[(0, 0, 3), (1, 0, 3), (0, 6, 9), (1, 6, 9),] + ), + t!(over300, &["", ""], "", &[(0, 0, 0), (1, 0, 0),]), + t!( + over310, + &["", ""], + "a", + &[(0, 0, 0), (1, 0, 0), (0, 1, 1), (1, 1, 1),] + ), + t!(over320, &["", "a"], "a", &[(0, 0, 0), (1, 0, 1), (0, 1, 1),]), + t!( + over330, + &["", "a", ""], + "a", + &[(0, 0, 0), (2, 0, 0), (1, 0, 1), (0, 1, 1), (2, 1, 1),] + ), + t!( + over340, + &["a", "", ""], + "a", + &[(1, 0, 0), (2, 0, 0), (0, 0, 1), (1, 1, 1), (2, 1, 1),] + ), + t!( + over350, + &["", "", "a"], + "a", + &[(0, 0, 0), (1, 0, 0), (2, 0, 1), (0, 1, 1), (1, 1, 1),] + ), + t!( + over360, + &["foo", "foofoo"], + "foofoo", + &[(0, 0, 3), (1, 0, 6), (0, 3, 6)] + ), +]; + +/* +Iterators of anchored overlapping searches were removed from the API in +after 0.7, but we leave the tests commented out for posterity. +/// Like OVERLAPPING, but for anchored searches. +const ANCHORED_OVERLAPPING: &'static [SearchTest] = &[ + t!(aover000, &["abcd", "bcd", "cd", "b"], "abcd", &[(0, 0, 4)]), + t!(aover010, &["bcd", "cd", "b", "abcd"], "abcd", &[(3, 0, 4)]), + t!(aover020, &["abcd", "bcd", "cd"], "abcd", &[(0, 0, 4)]), + t!(aover030, &["bcd", "abcd", "cd"], "abcd", &[(1, 0, 4)]), + t!(aover040, &["bcd", "cd", "abcd"], "abcd", &[(2, 0, 4)]), + t!(aover050, &["abc", "bc"], "zazabcz", &[]), + t!(aover100, &["ab", "ba"], "abababa", &[(0, 0, 2)]), + t!(aover200, &["foo", "foo"], "foobarfoo", &[(0, 0, 3), (1, 0, 3)]), + t!(aover300, &["", ""], "", &[(0, 0, 0), (1, 0, 0),]), + t!(aover310, &["", ""], "a", &[(0, 0, 0), (1, 0, 0)]), + t!(aover320, &["", "a"], "a", &[(0, 0, 0), (1, 0, 1)]), + t!(aover330, &["", "a", ""], "a", &[(0, 0, 0), (2, 0, 0), (1, 0, 1)]), + t!(aover340, &["a", "", ""], "a", &[(1, 0, 0), (2, 0, 0), (0, 0, 1)]), + t!(aover350, &["", "", "a"], "a", &[(0, 0, 0), (1, 0, 0), (2, 0, 1)]), + t!(aover360, &["foo", "foofoo"], "foofoo", &[(0, 0, 3), (1, 0, 6)]), +]; +*/ + +/// Tests for ASCII case insensitivity. +/// +/// These tests should all have the same behavior regardless of match semantics +/// or whether the search is overlapping. +const ASCII_CASE_INSENSITIVE: &'static [SearchTest] = &[ + t!(acasei000, &["a"], "A", &[(0, 0, 1)]), + t!(acasei010, &["Samwise"], "SAMWISE", &[(0, 0, 7)]), + t!(acasei011, &["Samwise"], "SAMWISE.abcd", &[(0, 0, 7)]), + t!(acasei020, &["fOoBaR"], "quux foobar baz", &[(0, 5, 11)]), +]; + +/// Like ASCII_CASE_INSENSITIVE, but specifically for non-overlapping tests. +const ASCII_CASE_INSENSITIVE_NON_OVERLAPPING: &'static [SearchTest] = &[ + t!(acasei000, &["foo", "FOO"], "fOo", &[(0, 0, 3)]), + t!(acasei000, &["FOO", "foo"], "fOo", &[(0, 0, 3)]), + t!(acasei010, &["abc", "def"], "abcdef", &[(0, 0, 3), (1, 3, 6)]), +]; + +/// Like ASCII_CASE_INSENSITIVE, but specifically for overlapping tests. +const ASCII_CASE_INSENSITIVE_OVERLAPPING: &'static [SearchTest] = &[ + t!(acasei000, &["foo", "FOO"], "fOo", &[(0, 0, 3), (1, 0, 3)]), + t!(acasei001, &["FOO", "foo"], "fOo", &[(0, 0, 3), (1, 0, 3)]), + // This is a regression test from: + // https://github.com/BurntSushi/aho-corasick/issues/68 + // Previously, it was reporting a duplicate (1, 3, 6) match. + t!( + acasei010, + &["abc", "def", "abcdef"], + "abcdef", + &[(0, 0, 3), (2, 0, 6), (1, 3, 6)] + ), +]; + +/// Regression tests that are applied to all Aho-Corasick combinations. +/// +/// If regression tests are needed for specific match semantics, then add them +/// to the appropriate group above. +const REGRESSION: &'static [SearchTest] = &[ + t!(regression010, &["inf", "ind"], "infind", &[(0, 0, 3), (1, 3, 6),]), + t!(regression020, &["ind", "inf"], "infind", &[(1, 0, 3), (0, 3, 6),]), + t!( + regression030, + &["libcore/", "libstd/"], + "libcore/char/methods.rs", + &[(0, 0, 8),] + ), + t!( + regression040, + &["libstd/", "libcore/"], + "libcore/char/methods.rs", + &[(1, 0, 8),] + ), + t!( + regression050, + &["\x00\x00\x01", "\x00\x00\x00"], + "\x00\x00\x00", + &[(1, 0, 3),] + ), + t!( + regression060, + &["\x00\x00\x00", "\x00\x00\x01"], + "\x00\x00\x00", + &[(0, 0, 3),] + ), +]; + +// Now define a test for each combination of things above that we want to run. +// Since there are a few different combinations for each collection of tests, +// we define a couple of macros to avoid repetition drudgery. The testconfig +// macro constructs the automaton from a given match kind, and runs the search +// tests one-by-one over the given collection. The `with` parameter allows one +// to configure the builder with additional parameters. The testcombo macro +// invokes testconfig in precisely this way: it sets up several tests where +// each one turns a different knob on AhoCorasickBuilder. + +macro_rules! testconfig { + (anchored, $name:ident, $collection:expr, $kind:ident, $with:expr) => { + #[test] + fn $name() { + run_search_tests($collection, |test| { + let mut builder = AhoCorasick::builder(); + $with(&mut builder); + let input = Input::new(test.haystack).anchored(Anchored::Yes); + builder + .match_kind(MatchKind::$kind) + .build(test.patterns) + .unwrap() + .try_find_iter(input) + .unwrap() + .collect() + }); + } + }; + (overlapping, $name:ident, $collection:expr, $kind:ident, $with:expr) => { + #[test] + fn $name() { + run_search_tests($collection, |test| { + let mut builder = AhoCorasick::builder(); + $with(&mut builder); + builder + .match_kind(MatchKind::$kind) + .build(test.patterns) + .unwrap() + .find_overlapping_iter(test.haystack) + .collect() + }); + } + }; + (stream, $name:ident, $collection:expr, $kind:ident, $with:expr) => { + #[test] + fn $name() { + run_stream_search_tests($collection, |test| { + let buf = std::io::BufReader::with_capacity( + 1, + test.haystack.as_bytes(), + ); + let mut builder = AhoCorasick::builder(); + $with(&mut builder); + builder + .match_kind(MatchKind::$kind) + .build(test.patterns) + .unwrap() + .stream_find_iter(buf) + .map(|result| result.unwrap()) + .collect() + }); + } + }; + ($name:ident, $collection:expr, $kind:ident, $with:expr) => { + #[test] + fn $name() { + run_search_tests($collection, |test| { + let mut builder = AhoCorasick::builder(); + $with(&mut builder); + builder + .match_kind(MatchKind::$kind) + .build(test.patterns) + .unwrap() + .find_iter(test.haystack) + .collect() + }); + } + }; +} + +macro_rules! testcombo { + ($name:ident, $collection:expr, $kind:ident) => { + mod $name { + use super::*; + + testconfig!(default, $collection, $kind, |_| ()); + testconfig!( + nfa_default, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)); + } + ); + testconfig!( + nfa_noncontig_no_prefilter, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .prefilter(false); + } + ); + testconfig!( + nfa_noncontig_all_sparse, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .dense_depth(0); + } + ); + testconfig!( + nfa_noncontig_all_dense, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .dense_depth(usize::MAX); + } + ); + testconfig!( + nfa_contig_default, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)); + } + ); + testconfig!( + nfa_contig_no_prefilter, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)) + .prefilter(false); + } + ); + testconfig!( + nfa_contig_all_sparse, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)) + .dense_depth(0); + } + ); + testconfig!( + nfa_contig_all_dense, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)) + .dense_depth(usize::MAX); + } + ); + testconfig!( + nfa_contig_no_byte_class, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)) + .byte_classes(false); + } + ); + testconfig!( + dfa_default, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)); + } + ); + testconfig!( + dfa_start_both, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)) + .start_kind(StartKind::Both); + } + ); + testconfig!( + dfa_no_prefilter, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).prefilter(false); + } + ); + testconfig!( + dfa_start_both_no_prefilter, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)) + .start_kind(StartKind::Both) + .prefilter(false); + } + ); + testconfig!( + dfa_no_byte_class, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).byte_classes(false); + } + ); + testconfig!( + dfa_start_both_no_byte_class, + $collection, + $kind, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)) + .start_kind(StartKind::Both) + .byte_classes(false); + } + ); + } + }; +} + +// Write out the various combinations of match semantics given the variety of +// configurations tested by 'testcombo!'. +testcombo!(search_leftmost_longest, AC_LEFTMOST_LONGEST, LeftmostLongest); +testcombo!(search_leftmost_first, AC_LEFTMOST_FIRST, LeftmostFirst); +testcombo!( + search_standard_nonoverlapping, + AC_STANDARD_NON_OVERLAPPING, + Standard +); + +// Write out the overlapping combo by hand since there is only one of them. +testconfig!( + overlapping, + search_standard_overlapping_default, + AC_STANDARD_OVERLAPPING, + Standard, + |_| () +); +testconfig!( + overlapping, + search_standard_overlapping_nfa_noncontig_default, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)); + } +); +testconfig!( + overlapping, + search_standard_overlapping_nfa_noncontig_no_prefilter, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)).prefilter(false); + } +); +testconfig!( + overlapping, + search_standard_overlapping_nfa_contig_default, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)); + } +); +testconfig!( + overlapping, + search_standard_overlapping_nfa_contig_no_prefilter, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)).prefilter(false); + } +); +testconfig!( + overlapping, + search_standard_overlapping_nfa_contig_all_sparse, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)).dense_depth(0); + } +); +testconfig!( + overlapping, + search_standard_overlapping_nfa_contig_all_dense, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)).dense_depth(usize::MAX); + } +); +testconfig!( + overlapping, + search_standard_overlapping_dfa_default, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)); + } +); +testconfig!( + overlapping, + search_standard_overlapping_dfa_start_both, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).start_kind(StartKind::Both); + } +); +testconfig!( + overlapping, + search_standard_overlapping_dfa_no_prefilter, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).prefilter(false); + } +); +testconfig!( + overlapping, + search_standard_overlapping_dfa_start_both_no_prefilter, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)) + .start_kind(StartKind::Both) + .prefilter(false); + } +); +testconfig!( + overlapping, + search_standard_overlapping_dfa_no_byte_class, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).byte_classes(false); + } +); +testconfig!( + overlapping, + search_standard_overlapping_dfa_start_both_no_byte_class, + AC_STANDARD_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)) + .start_kind(StartKind::Both) + .byte_classes(false); + } +); + +// Also write out tests manually for streams, since we only test the standard +// match semantics. We also don't bother testing different automaton +// configurations, since those are well covered by tests above. +#[cfg(feature = "std")] +testconfig!( + stream, + search_standard_stream_default, + AC_STANDARD_NON_OVERLAPPING, + Standard, + |_| () +); +#[cfg(feature = "std")] +testconfig!( + stream, + search_standard_stream_nfa_noncontig_default, + AC_STANDARD_NON_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)); + } +); +#[cfg(feature = "std")] +testconfig!( + stream, + search_standard_stream_nfa_contig_default, + AC_STANDARD_NON_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)); + } +); +#[cfg(feature = "std")] +testconfig!( + stream, + search_standard_stream_dfa_default, + AC_STANDARD_NON_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)); + } +); + +// Same thing for anchored searches. Write them out manually. +testconfig!( + anchored, + search_standard_anchored_default, + AC_STANDARD_ANCHORED_NON_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored); + } +); +testconfig!( + anchored, + search_standard_anchored_nfa_noncontig_default, + AC_STANDARD_ANCHORED_NON_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored) + .kind(Some(AhoCorasickKind::NoncontiguousNFA)); + } +); +testconfig!( + anchored, + search_standard_anchored_nfa_contig_default, + AC_STANDARD_ANCHORED_NON_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored) + .kind(Some(AhoCorasickKind::ContiguousNFA)); + } +); +testconfig!( + anchored, + search_standard_anchored_dfa_default, + AC_STANDARD_ANCHORED_NON_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored).kind(Some(AhoCorasickKind::DFA)); + } +); +testconfig!( + anchored, + search_standard_anchored_dfa_start_both, + AC_STANDARD_ANCHORED_NON_OVERLAPPING, + Standard, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Both).kind(Some(AhoCorasickKind::DFA)); + } +); +testconfig!( + anchored, + search_leftmost_first_anchored_default, + AC_LEFTMOST_FIRST_ANCHORED, + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored); + } +); +testconfig!( + anchored, + search_leftmost_first_anchored_nfa_noncontig_default, + AC_LEFTMOST_FIRST_ANCHORED, + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored) + .kind(Some(AhoCorasickKind::NoncontiguousNFA)); + } +); +testconfig!( + anchored, + search_leftmost_first_anchored_nfa_contig_default, + AC_LEFTMOST_FIRST_ANCHORED, + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored) + .kind(Some(AhoCorasickKind::ContiguousNFA)); + } +); +testconfig!( + anchored, + search_leftmost_first_anchored_dfa_default, + AC_LEFTMOST_FIRST_ANCHORED, + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored).kind(Some(AhoCorasickKind::DFA)); + } +); +testconfig!( + anchored, + search_leftmost_first_anchored_dfa_start_both, + AC_LEFTMOST_FIRST_ANCHORED, + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Both).kind(Some(AhoCorasickKind::DFA)); + } +); +testconfig!( + anchored, + search_leftmost_longest_anchored_default, + AC_LEFTMOST_LONGEST_ANCHORED, + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored); + } +); +testconfig!( + anchored, + search_leftmost_longest_anchored_nfa_noncontig_default, + AC_LEFTMOST_LONGEST_ANCHORED, + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored) + .kind(Some(AhoCorasickKind::NoncontiguousNFA)); + } +); +testconfig!( + anchored, + search_leftmost_longest_anchored_nfa_contig_default, + AC_LEFTMOST_LONGEST_ANCHORED, + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored) + .kind(Some(AhoCorasickKind::ContiguousNFA)); + } +); +testconfig!( + anchored, + search_leftmost_longest_anchored_dfa_default, + AC_LEFTMOST_LONGEST_ANCHORED, + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Anchored).kind(Some(AhoCorasickKind::DFA)); + } +); +testconfig!( + anchored, + search_leftmost_longest_anchored_dfa_start_both, + AC_LEFTMOST_LONGEST_ANCHORED, + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.start_kind(StartKind::Both).kind(Some(AhoCorasickKind::DFA)); + } +); + +// And also write out the test combinations for ASCII case insensitivity. +testconfig!( + acasei_standard_default, + &[ASCII_CASE_INSENSITIVE], + Standard, + |b: &mut AhoCorasickBuilder| { + b.prefilter(false).ascii_case_insensitive(true); + } +); +testconfig!( + acasei_standard_nfa_noncontig_default, + &[ASCII_CASE_INSENSITIVE], + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .prefilter(false) + .ascii_case_insensitive(true); + } +); +testconfig!( + acasei_standard_nfa_contig_default, + &[ASCII_CASE_INSENSITIVE], + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)) + .prefilter(false) + .ascii_case_insensitive(true); + } +); +testconfig!( + acasei_standard_dfa_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).ascii_case_insensitive(true); + } +); +testconfig!( + overlapping, + acasei_standard_overlapping_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_OVERLAPPING], + Standard, + |b: &mut AhoCorasickBuilder| { + b.ascii_case_insensitive(true); + } +); +testconfig!( + overlapping, + acasei_standard_overlapping_nfa_noncontig_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_OVERLAPPING], + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .ascii_case_insensitive(true); + } +); +testconfig!( + overlapping, + acasei_standard_overlapping_nfa_contig_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_OVERLAPPING], + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)) + .ascii_case_insensitive(true); + } +); +testconfig!( + overlapping, + acasei_standard_overlapping_dfa_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_OVERLAPPING], + Standard, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).ascii_case_insensitive(true); + } +); +testconfig!( + acasei_leftmost_first_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.ascii_case_insensitive(true); + } +); +testconfig!( + acasei_leftmost_first_nfa_noncontig_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .ascii_case_insensitive(true); + } +); +testconfig!( + acasei_leftmost_first_nfa_contig_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)) + .ascii_case_insensitive(true); + } +); +testconfig!( + acasei_leftmost_first_dfa_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + LeftmostFirst, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).ascii_case_insensitive(true); + } +); +testconfig!( + acasei_leftmost_longest_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.ascii_case_insensitive(true); + } +); +testconfig!( + acasei_leftmost_longest_nfa_noncontig_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .ascii_case_insensitive(true); + } +); +testconfig!( + acasei_leftmost_longest_nfa_contig_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::ContiguousNFA)) + .ascii_case_insensitive(true); + } +); +testconfig!( + acasei_leftmost_longest_dfa_default, + &[ASCII_CASE_INSENSITIVE, ASCII_CASE_INSENSITIVE_NON_OVERLAPPING], + LeftmostLongest, + |b: &mut AhoCorasickBuilder| { + b.kind(Some(AhoCorasickKind::DFA)).ascii_case_insensitive(true); + } +); + +fn run_search_tests Vec>( + which: TestCollection, + mut f: F, +) { + let get_match_triples = + |matches: Vec| -> Vec<(usize, usize, usize)> { + matches + .into_iter() + .map(|m| (m.pattern().as_usize(), m.start(), m.end())) + .collect() + }; + for &tests in which { + for test in tests { + assert_eq!( + test.matches, + get_match_triples(f(&test)).as_slice(), + "test: {}, patterns: {:?}, haystack: {:?}", + test.name, + test.patterns, + test.haystack + ); + } + } +} + +// Like 'run_search_tests', but we skip any tests that contain the empty +// pattern because stream searching doesn't support it. +#[cfg(feature = "std")] +fn run_stream_search_tests Vec>( + which: TestCollection, + mut f: F, +) { + let get_match_triples = + |matches: Vec| -> Vec<(usize, usize, usize)> { + matches + .into_iter() + .map(|m| (m.pattern().as_usize(), m.start(), m.end())) + .collect() + }; + for &tests in which { + for test in tests { + if test.patterns.iter().any(|p| p.is_empty()) { + continue; + } + assert_eq!( + test.matches, + get_match_triples(f(&test)).as_slice(), + "test: {}, patterns: {:?}, haystack: {:?}", + test.name, + test.patterns, + test.haystack + ); + } + } +} + +#[test] +fn search_tests_have_unique_names() { + let assert = |constname, tests: &[SearchTest]| { + let mut seen = HashMap::new(); // map from test name to position + for (i, test) in tests.iter().enumerate() { + if !seen.contains_key(test.name) { + seen.insert(test.name, i); + } else { + let last = seen[test.name]; + panic!( + "{} tests have duplicate names at positions {} and {}", + constname, last, i + ); + } + } + }; + assert("BASICS", BASICS); + assert("STANDARD", STANDARD); + assert("LEFTMOST", LEFTMOST); + assert("LEFTMOST_FIRST", LEFTMOST_FIRST); + assert("LEFTMOST_LONGEST", LEFTMOST_LONGEST); + assert("NON_OVERLAPPING", NON_OVERLAPPING); + assert("OVERLAPPING", OVERLAPPING); + assert("REGRESSION", REGRESSION); +} + +#[cfg(feature = "std")] +#[test] +#[should_panic] +fn stream_not_allowed_leftmost_first() { + let fsm = AhoCorasick::builder() + .match_kind(MatchKind::LeftmostFirst) + .build(None::) + .unwrap(); + assert_eq!(fsm.stream_find_iter(&b""[..]).count(), 0); +} + +#[cfg(feature = "std")] +#[test] +#[should_panic] +fn stream_not_allowed_leftmost_longest() { + let fsm = AhoCorasick::builder() + .match_kind(MatchKind::LeftmostLongest) + .build(None::) + .unwrap(); + assert_eq!(fsm.stream_find_iter(&b""[..]).count(), 0); +} + +#[test] +#[should_panic] +fn overlapping_not_allowed_leftmost_first() { + let fsm = AhoCorasick::builder() + .match_kind(MatchKind::LeftmostFirst) + .build(None::) + .unwrap(); + assert_eq!(fsm.find_overlapping_iter("").count(), 0); +} + +#[test] +#[should_panic] +fn overlapping_not_allowed_leftmost_longest() { + let fsm = AhoCorasick::builder() + .match_kind(MatchKind::LeftmostLongest) + .build(None::) + .unwrap(); + assert_eq!(fsm.find_overlapping_iter("").count(), 0); +} + +// This tests that if we build an AC matcher with an "unanchored" start kind, +// then we can't run an anchored search even if the underlying searcher +// supports it. +// +// The key bit here is that both of the NFAs in this crate unconditionally +// support both unanchored and anchored searches, but the DFA does not because +// of the added cost of doing so. To avoid the top-level AC matcher sometimes +// supporting anchored and sometimes not (depending on which searcher it +// chooses to use internally), we ensure that the given 'StartKind' is always +// respected. +#[test] +fn anchored_not_allowed_even_if_technically_available() { + let ac = AhoCorasick::builder() + .kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .start_kind(StartKind::Unanchored) + .build(&["foo"]) + .unwrap(); + assert!(ac.try_find(Input::new("foo").anchored(Anchored::Yes)).is_err()); + + let ac = AhoCorasick::builder() + .kind(Some(AhoCorasickKind::ContiguousNFA)) + .start_kind(StartKind::Unanchored) + .build(&["foo"]) + .unwrap(); + assert!(ac.try_find(Input::new("foo").anchored(Anchored::Yes)).is_err()); + + // For completeness, check that the DFA returns an error too. + let ac = AhoCorasick::builder() + .kind(Some(AhoCorasickKind::DFA)) + .start_kind(StartKind::Unanchored) + .build(&["foo"]) + .unwrap(); + assert!(ac.try_find(Input::new("foo").anchored(Anchored::Yes)).is_err()); +} + +// This is like the test aboved, but with unanchored and anchored flipped. That +// is, we asked for an AC searcher with anchored support and we check that +// unanchored searches return an error even if the underlying searcher would +// technically support it. +#[test] +fn unanchored_not_allowed_even_if_technically_available() { + let ac = AhoCorasick::builder() + .kind(Some(AhoCorasickKind::NoncontiguousNFA)) + .start_kind(StartKind::Anchored) + .build(&["foo"]) + .unwrap(); + assert!(ac.try_find(Input::new("foo").anchored(Anchored::No)).is_err()); + + let ac = AhoCorasick::builder() + .kind(Some(AhoCorasickKind::ContiguousNFA)) + .start_kind(StartKind::Anchored) + .build(&["foo"]) + .unwrap(); + assert!(ac.try_find(Input::new("foo").anchored(Anchored::No)).is_err()); + + // For completeness, check that the DFA returns an error too. + let ac = AhoCorasick::builder() + .kind(Some(AhoCorasickKind::DFA)) + .start_kind(StartKind::Anchored) + .build(&["foo"]) + .unwrap(); + assert!(ac.try_find(Input::new("foo").anchored(Anchored::No)).is_err()); +} + +// This tests that a prefilter does not cause a search to report a match +// outside the bounds provided by the caller. +// +// This is a regression test for a bug I introduced during the rewrite of most +// of the crate after 0.7. It was never released. The tricky part here is +// ensuring we get a prefilter that can report matches on its own (such as the +// packed searcher). Otherwise, prefilters that report false positives might +// have searched past the bounds provided by the caller, but confirming the +// match would subsequently fail. +#[test] +fn prefilter_stays_in_bounds() { + let ac = AhoCorasick::builder() + .match_kind(MatchKind::LeftmostFirst) + .build(&["sam", "frodo", "pippin", "merry", "gandalf", "sauron"]) + .unwrap(); + let haystack = "foo gandalf"; + assert_eq!(None, ac.find(Input::new(haystack).range(0..10))); +} + +// See: https://github.com/BurntSushi/aho-corasick/issues/44 +// +// In short, this test ensures that enabling ASCII case insensitivity does not +// visit an exponential number of states when filling in failure transitions. +#[test] +fn regression_ascii_case_insensitive_no_exponential() { + let ac = AhoCorasick::builder() + .ascii_case_insensitive(true) + .build(&["Tsubaki House-Triple Shot Vol01校花三姐妹"]) + .unwrap(); + assert!(ac.find("").is_none()); +} + +// See: https://github.com/BurntSushi/aho-corasick/issues/53 +// +// This test ensures that the rare byte prefilter works in a particular corner +// case. In particular, the shift offset detected for '/' in the patterns below +// was incorrect, leading to a false negative. +#[test] +fn regression_rare_byte_prefilter() { + use crate::AhoCorasick; + + let ac = AhoCorasick::new(&["ab/j/", "x/"]).unwrap(); + assert!(ac.is_match("ab/j/")); +} + +#[test] +fn regression_case_insensitive_prefilter() { + for c in b'a'..b'z' { + for c2 in b'a'..b'z' { + let c = c as char; + let c2 = c2 as char; + let needle = format!("{}{}", c, c2).to_lowercase(); + let haystack = needle.to_uppercase(); + let ac = AhoCorasick::builder() + .ascii_case_insensitive(true) + .prefilter(true) + .build(&[&needle]) + .unwrap(); + assert_eq!( + 1, + ac.find_iter(&haystack).count(), + "failed to find {:?} in {:?}\n\nautomaton:\n{:?}", + needle, + haystack, + ac, + ); + } + } +} + +// See: https://github.com/BurntSushi/aho-corasick/issues/64 +// +// This occurs when the rare byte prefilter is active. +#[cfg(feature = "std")] +#[test] +fn regression_stream_rare_byte_prefilter() { + use std::io::Read; + + // NOTE: The test only fails if this ends with j. + const MAGIC: [u8; 5] = *b"1234j"; + + // NOTE: The test fails for value in 8188..=8191 These value put the string + // to search accross two call to read because the buffer size is 64KB by + // default. + const BEGIN: usize = 65_535; + + /// This is just a structure that implements Reader. The reader + /// implementation will simulate a file filled with 0, except for the MAGIC + /// string at offset BEGIN. + #[derive(Default)] + struct R { + read: usize, + } + + impl Read for R { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.read > 100000 { + return Ok(0); + } + let mut from = 0; + if self.read < BEGIN { + from = buf.len().min(BEGIN - self.read); + for x in 0..from { + buf[x] = 0; + } + self.read += from; + } + if self.read >= BEGIN && self.read <= BEGIN + MAGIC.len() { + let to = buf.len().min(BEGIN + MAGIC.len() - self.read + from); + if to > from { + buf[from..to].copy_from_slice( + &MAGIC + [self.read - BEGIN..self.read - BEGIN + to - from], + ); + self.read += to - from; + from = to; + } + } + for x in from..buf.len() { + buf[x] = 0; + self.read += 1; + } + Ok(buf.len()) + } + } + + fn run() -> std::io::Result<()> { + let aut = AhoCorasick::builder() + // Enable byte classes to make debugging the automaton easier. It + // should have no effect on the test result. + .byte_classes(false) + .build(&[&MAGIC]) + .unwrap(); + + // While reading from a vector, it works: + let mut buf = alloc::vec![]; + R::default().read_to_end(&mut buf)?; + let from_whole = aut.find_iter(&buf).next().unwrap().start(); + + // But using stream_find_iter fails! + let mut file = std::io::BufReader::new(R::default()); + let begin = aut + .stream_find_iter(&mut file) + .next() + .expect("NOT FOUND!!!!")? // Panic here + .start(); + assert_eq!(from_whole, begin); + Ok(()) + } + + run().unwrap() +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/transducer.rs b/third_party/crates/aho-corasick-1.1.5/src/transducer.rs new file mode 100644 index 0000000..39bb240 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/transducer.rs @@ -0,0 +1,270 @@ +/*! +Provides implementations of `fst::Automaton` for Aho-Corasick automata. + +This works by providing two wrapper types, [`Anchored`] and [`Unanchored`]. +The former executes an anchored search on an FST while the latter executes +an unanchored search. Building these wrappers is fallible and will fail if +the underlying Aho-Corasick automaton does not support the type of search it +represents. +*/ + +use crate::{ + automaton::{Automaton, StateID}, + Anchored as AcAnchored, Input, MatchError, +}; + +/// Represents an unanchored Aho-Corasick search of a finite state transducer. +/// +/// Wrapping an Aho-Corasick automaton in `Unanchored` will fail if the +/// underlying automaton does not support unanchored searches. +/// +/// # Example +/// +/// This shows how to build an FST of keys and then run an unanchored search on +/// those keys using an Aho-Corasick automaton. +/// +/// ``` +/// use aho_corasick::{nfa::contiguous::NFA, transducer::Unanchored}; +/// use fst::{Automaton, IntoStreamer, Set, Streamer}; +/// +/// let set = Set::from_iter(&["abcd", "bc", "bcd", "xyz"]).unwrap(); +/// let nfa = NFA::new(&["bcd", "x"]).unwrap(); +/// // NFAs always support both unanchored and anchored searches. +/// let searcher = Unanchored::new(&nfa).unwrap(); +/// +/// let mut stream = set.search(searcher).into_stream(); +/// let mut results = vec![]; +/// while let Some(key) = stream.next() { +/// results.push(std::str::from_utf8(key).unwrap().to_string()); +/// } +/// assert_eq!(vec!["abcd", "bcd", "xyz"], results); +/// ``` +#[derive(Clone, Debug)] +pub struct Unanchored(A); + +impl Unanchored { + /// Create a new `Unanchored` implementation of the `fst::Automaton` trait. + /// + /// If the given Aho-Corasick automaton does not support unanchored + /// searches, then this returns an error. + pub fn new(aut: A) -> Result, MatchError> { + let input = Input::new("").anchored(AcAnchored::No); + let _ = aut.start_state(&input)?; + Ok(Unanchored(aut)) + } + + /// Returns a borrow to the underlying automaton. + pub fn as_ref(&self) -> &A { + &self.0 + } + + /// Unwrap this value and return the inner automaton. + pub fn into_inner(self) -> A { + self.0 + } +} + +impl fst::Automaton for Unanchored { + type State = StateID; + + #[inline] + fn start(&self) -> StateID { + let input = Input::new("").anchored(AcAnchored::No); + self.0.start_state(&input).expect("support for unanchored searches") + } + + #[inline] + fn is_match(&self, state: &StateID) -> bool { + self.0.is_match(*state) + } + + #[inline] + fn accept(&self, state: &StateID, byte: u8) -> StateID { + if fst::Automaton::is_match(self, state) { + return *state; + } + self.0.next_state(AcAnchored::No, *state, byte) + } + + #[inline] + fn can_match(&self, state: &StateID) -> bool { + !self.0.is_dead(*state) + } +} + +/// Represents an anchored Aho-Corasick search of a finite state transducer. +/// +/// Wrapping an Aho-Corasick automaton in `Unanchored` will fail if the +/// underlying automaton does not support unanchored searches. +/// +/// # Example +/// +/// This shows how to build an FST of keys and then run an anchored search on +/// those keys using an Aho-Corasick automaton. +/// +/// ``` +/// use aho_corasick::{nfa::contiguous::NFA, transducer::Anchored}; +/// use fst::{Automaton, IntoStreamer, Set, Streamer}; +/// +/// let set = Set::from_iter(&["abcd", "bc", "bcd", "xyz"]).unwrap(); +/// let nfa = NFA::new(&["bcd", "x"]).unwrap(); +/// // NFAs always support both unanchored and anchored searches. +/// let searcher = Anchored::new(&nfa).unwrap(); +/// +/// let mut stream = set.search(searcher).into_stream(); +/// let mut results = vec![]; +/// while let Some(key) = stream.next() { +/// results.push(std::str::from_utf8(key).unwrap().to_string()); +/// } +/// assert_eq!(vec!["bcd", "xyz"], results); +/// ``` +/// +/// This is like the example above, except we use an Aho-Corasick DFA, which +/// requires explicitly configuring it to support anchored searches. (NFAs +/// unconditionally support both unanchored and anchored searches.) +/// +/// ``` +/// use aho_corasick::{dfa::DFA, transducer::Anchored, StartKind}; +/// use fst::{Automaton, IntoStreamer, Set, Streamer}; +/// +/// let set = Set::from_iter(&["abcd", "bc", "bcd", "xyz"]).unwrap(); +/// let dfa = DFA::builder() +/// .start_kind(StartKind::Anchored) +/// .build(&["bcd", "x"]) +/// .unwrap(); +/// // We've explicitly configured our DFA to support anchored searches. +/// let searcher = Anchored::new(&dfa).unwrap(); +/// +/// let mut stream = set.search(searcher).into_stream(); +/// let mut results = vec![]; +/// while let Some(key) = stream.next() { +/// results.push(std::str::from_utf8(key).unwrap().to_string()); +/// } +/// assert_eq!(vec!["bcd", "xyz"], results); +/// ``` +#[derive(Clone, Debug)] +pub struct Anchored(A); + +impl Anchored { + /// Create a new `Anchored` implementation of the `fst::Automaton` trait. + /// + /// If the given Aho-Corasick automaton does not support anchored searches, + /// then this returns an error. + pub fn new(aut: A) -> Result, MatchError> { + let input = Input::new("").anchored(AcAnchored::Yes); + let _ = aut.start_state(&input)?; + Ok(Anchored(aut)) + } + + /// Returns a borrow to the underlying automaton. + pub fn as_ref(&self) -> &A { + &self.0 + } + + /// Unwrap this value and return the inner automaton. + pub fn into_inner(self) -> A { + self.0 + } +} + +impl fst::Automaton for Anchored { + type State = StateID; + + #[inline] + fn start(&self) -> StateID { + let input = Input::new("").anchored(AcAnchored::Yes); + self.0.start_state(&input).expect("support for unanchored searches") + } + + #[inline] + fn is_match(&self, state: &StateID) -> bool { + self.0.is_match(*state) + } + + #[inline] + fn accept(&self, state: &StateID, byte: u8) -> StateID { + if fst::Automaton::is_match(self, state) { + return *state; + } + self.0.next_state(AcAnchored::Yes, *state, byte) + } + + #[inline] + fn can_match(&self, state: &StateID) -> bool { + !self.0.is_dead(*state) + } +} + +#[cfg(test)] +mod tests { + use alloc::{string::String, vec, vec::Vec}; + + use fst::{Automaton, IntoStreamer, Set, Streamer}; + + use crate::{ + dfa::DFA, + nfa::{contiguous, noncontiguous}, + StartKind, + }; + + use super::*; + + fn search>( + set: &Set, + aut: A, + ) -> Vec { + let mut stream = set.search(aut).into_stream(); + let mut results = vec![]; + while let Some(key) = stream.next() { + results.push(String::from(core::str::from_utf8(key).unwrap())); + } + results + } + + #[test] + fn unanchored() { + let set = + Set::from_iter(&["a", "bar", "baz", "wat", "xba", "xbax", "z"]) + .unwrap(); + let patterns = vec!["baz", "bax"]; + let expected = vec!["baz", "xbax"]; + + let aut = Unanchored(noncontiguous::NFA::new(&patterns).unwrap()); + let got = search(&set, &aut); + assert_eq!(got, expected); + + let aut = Unanchored(contiguous::NFA::new(&patterns).unwrap()); + let got = search(&set, &aut); + assert_eq!(got, expected); + + let aut = Unanchored(DFA::new(&patterns).unwrap()); + let got = search(&set, &aut); + assert_eq!(got, expected); + } + + #[test] + fn anchored() { + let set = + Set::from_iter(&["a", "bar", "baz", "wat", "xba", "xbax", "z"]) + .unwrap(); + let patterns = vec!["baz", "bax"]; + let expected = vec!["baz"]; + + let aut = Anchored(noncontiguous::NFA::new(&patterns).unwrap()); + let got = search(&set, &aut); + assert_eq!(got, expected); + + let aut = Anchored(contiguous::NFA::new(&patterns).unwrap()); + let got = search(&set, &aut); + assert_eq!(got, expected); + + let aut = Anchored( + DFA::builder() + .start_kind(StartKind::Anchored) + .build(&patterns) + .unwrap(), + ); + let got = search(&set, &aut); + assert_eq!(got, expected); + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/alphabet.rs b/third_party/crates/aho-corasick-1.1.5/src/util/alphabet.rs new file mode 100644 index 0000000..69724fa --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/alphabet.rs @@ -0,0 +1,409 @@ +use crate::util::int::Usize; + +/// A representation of byte oriented equivalence classes. +/// +/// This is used in finite state machines to reduce the size of the transition +/// table. This can have a particularly large impact not only on the total size +/// of an FSM, but also on FSM build times because it reduces the number of +/// transitions that need to be visited/set. +#[derive(Clone, Copy)] +pub(crate) struct ByteClasses([u8; 256]); + +impl ByteClasses { + /// Creates a new set of equivalence classes where all bytes are mapped to + /// the same class. + pub(crate) fn empty() -> ByteClasses { + ByteClasses([0; 256]) + } + + /// Creates a new set of equivalence classes where each byte belongs to + /// its own equivalence class. + pub(crate) fn singletons() -> ByteClasses { + let mut classes = ByteClasses::empty(); + for b in 0..=255 { + classes.set(b, b); + } + classes + } + + /// Set the equivalence class for the given byte. + #[inline] + pub(crate) fn set(&mut self, byte: u8, class: u8) { + self.0[usize::from(byte)] = class; + } + + /// Get the equivalence class for the given byte. + #[inline] + pub(crate) fn get(&self, byte: u8) -> u8 { + self.0[usize::from(byte)] + } + + /// Return the total number of elements in the alphabet represented by + /// these equivalence classes. Equivalently, this returns the total number + /// of equivalence classes. + #[inline] + pub(crate) fn alphabet_len(&self) -> usize { + // Add one since the number of equivalence classes is one bigger than + // the last one. + usize::from(self.0[255]) + 1 + } + + /// Returns the stride, as a base-2 exponent, required for these + /// equivalence classes. + /// + /// The stride is always the smallest power of 2 that is greater than or + /// equal to the alphabet length. This is done so that converting between + /// state IDs and indices can be done with shifts alone, which is much + /// faster than integer division. The "stride2" is the exponent. i.e., + /// `2^stride2 = stride`. + pub(crate) fn stride2(&self) -> usize { + let zeros = self.alphabet_len().next_power_of_two().trailing_zeros(); + usize::try_from(zeros).unwrap() + } + + /// Returns the stride for these equivalence classes, which corresponds + /// to the smallest power of 2 greater than or equal to the number of + /// equivalence classes. + pub(crate) fn stride(&self) -> usize { + 1 << self.stride2() + } + + /// Returns true if and only if every byte in this class maps to its own + /// equivalence class. Equivalently, there are 257 equivalence classes + /// and each class contains exactly one byte (plus the special EOI class). + #[inline] + pub(crate) fn is_singleton(&self) -> bool { + self.alphabet_len() == 256 + } + + /// Returns an iterator over all equivalence classes in this set. + pub(crate) fn iter(&self) -> ByteClassIter { + ByteClassIter { it: 0..self.alphabet_len() } + } + + /// Returns an iterator of the bytes in the given equivalence class. + pub(crate) fn elements(&self, class: u8) -> ByteClassElements { + ByteClassElements { classes: self, class, bytes: 0..=255 } + } + + /// Returns an iterator of byte ranges in the given equivalence class. + /// + /// That is, a sequence of contiguous ranges are returned. Typically, every + /// class maps to a single contiguous range. + fn element_ranges(&self, class: u8) -> ByteClassElementRanges { + ByteClassElementRanges { elements: self.elements(class), range: None } + } +} + +impl core::fmt::Debug for ByteClasses { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + if self.is_singleton() { + write!(f, "ByteClasses()") + } else { + write!(f, "ByteClasses(")?; + for (i, class) in self.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{:?} => [", class)?; + for (start, end) in self.element_ranges(class) { + if start == end { + write!(f, "{:?}", start)?; + } else { + write!(f, "{:?}-{:?}", start, end)?; + } + } + write!(f, "]")?; + } + write!(f, ")") + } + } +} + +/// An iterator over each equivalence class. +#[derive(Debug)] +pub(crate) struct ByteClassIter { + it: core::ops::Range, +} + +impl Iterator for ByteClassIter { + type Item = u8; + + fn next(&mut self) -> Option { + self.it.next().map(|class| class.as_u8()) + } +} + +/// An iterator over all elements in a specific equivalence class. +#[derive(Debug)] +pub(crate) struct ByteClassElements<'a> { + classes: &'a ByteClasses, + class: u8, + bytes: core::ops::RangeInclusive, +} + +impl<'a> Iterator for ByteClassElements<'a> { + type Item = u8; + + fn next(&mut self) -> Option { + while let Some(byte) = self.bytes.next() { + if self.class == self.classes.get(byte) { + return Some(byte); + } + } + None + } +} + +/// An iterator over all elements in an equivalence class expressed as a +/// sequence of contiguous ranges. +#[derive(Debug)] +pub(crate) struct ByteClassElementRanges<'a> { + elements: ByteClassElements<'a>, + range: Option<(u8, u8)>, +} + +impl<'a> Iterator for ByteClassElementRanges<'a> { + type Item = (u8, u8); + + fn next(&mut self) -> Option<(u8, u8)> { + loop { + let element = match self.elements.next() { + None => return self.range.take(), + Some(element) => element, + }; + match self.range.take() { + None => { + self.range = Some((element, element)); + } + Some((start, end)) => { + if usize::from(end) + 1 != usize::from(element) { + self.range = Some((element, element)); + return Some((start, end)); + } + self.range = Some((start, element)); + } + } + } + } +} + +/// A partitioning of bytes into equivalence classes. +/// +/// A byte class set keeps track of an *approximation* of equivalence classes +/// of bytes during NFA construction. That is, every byte in an equivalence +/// class cannot discriminate between a match and a non-match. +/// +/// Note that this may not compute the minimal set of equivalence classes. +/// Basically, any byte in a pattern given to the noncontiguous NFA builder +/// will automatically be treated as its own equivalence class. All other +/// bytes---any byte not in any pattern---will be treated as their own +/// equivalence classes. In theory, all bytes not in any pattern should +/// be part of a single equivalence class, but in practice, we only treat +/// contiguous ranges of bytes as an equivalence class. So the number of +/// classes computed may be bigger than necessary. This usually doesn't make +/// much of a difference, and keeps the implementation simple. +#[derive(Clone, Debug)] +pub(crate) struct ByteClassSet(ByteSet); + +impl Default for ByteClassSet { + fn default() -> ByteClassSet { + ByteClassSet::empty() + } +} + +impl ByteClassSet { + /// Create a new set of byte classes where all bytes are part of the same + /// equivalence class. + pub(crate) fn empty() -> Self { + ByteClassSet(ByteSet::empty()) + } + + /// Indicate the the range of byte given (inclusive) can discriminate a + /// match between it and all other bytes outside of the range. + pub(crate) fn set_range(&mut self, start: u8, end: u8) { + debug_assert!(start <= end); + if start > 0 { + self.0.add(start - 1); + } + self.0.add(end); + } + + /// Convert this boolean set to a map that maps all byte values to their + /// corresponding equivalence class. The last mapping indicates the largest + /// equivalence class identifier (which is never bigger than 255). + pub(crate) fn byte_classes(&self) -> ByteClasses { + let mut classes = ByteClasses::empty(); + let mut class = 0u8; + let mut b = 0u8; + loop { + classes.set(b, class); + if b == 255 { + break; + } + if self.0.contains(b) { + class = class.checked_add(1).unwrap(); + } + b = b.checked_add(1).unwrap(); + } + classes + } +} + +/// A simple set of bytes that is reasonably cheap to copy and allocation free. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct ByteSet { + bits: BitSet, +} + +/// The representation of a byte set. Split out so that we can define a +/// convenient Debug impl for it while keeping "ByteSet" in the output. +#[derive(Clone, Copy, Default, Eq, PartialEq)] +struct BitSet([u128; 2]); + +impl ByteSet { + /// Create an empty set of bytes. + pub(crate) fn empty() -> ByteSet { + ByteSet { bits: BitSet([0; 2]) } + } + + /// Add a byte to this set. + /// + /// If the given byte already belongs to this set, then this is a no-op. + pub(crate) fn add(&mut self, byte: u8) { + let bucket = byte / 128; + let bit = byte % 128; + self.bits.0[usize::from(bucket)] |= 1 << bit; + } + + /// Return true if and only if the given byte is in this set. + pub(crate) fn contains(&self, byte: u8) -> bool { + let bucket = byte / 128; + let bit = byte % 128; + self.bits.0[usize::from(bucket)] & (1 << bit) > 0 + } +} + +impl core::fmt::Debug for BitSet { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut fmtd = f.debug_set(); + for b in 0u8..=255 { + if (ByteSet { bits: *self }).contains(b) { + fmtd.entry(&b); + } + } + fmtd.finish() + } +} + +#[cfg(test)] +mod tests { + use alloc::{vec, vec::Vec}; + + use super::*; + + #[test] + fn byte_classes() { + let mut set = ByteClassSet::empty(); + set.set_range(b'a', b'z'); + + let classes = set.byte_classes(); + assert_eq!(classes.get(0), 0); + assert_eq!(classes.get(1), 0); + assert_eq!(classes.get(2), 0); + assert_eq!(classes.get(b'a' - 1), 0); + assert_eq!(classes.get(b'a'), 1); + assert_eq!(classes.get(b'm'), 1); + assert_eq!(classes.get(b'z'), 1); + assert_eq!(classes.get(b'z' + 1), 2); + assert_eq!(classes.get(254), 2); + assert_eq!(classes.get(255), 2); + + let mut set = ByteClassSet::empty(); + set.set_range(0, 2); + set.set_range(4, 6); + let classes = set.byte_classes(); + assert_eq!(classes.get(0), 0); + assert_eq!(classes.get(1), 0); + assert_eq!(classes.get(2), 0); + assert_eq!(classes.get(3), 1); + assert_eq!(classes.get(4), 2); + assert_eq!(classes.get(5), 2); + assert_eq!(classes.get(6), 2); + assert_eq!(classes.get(7), 3); + assert_eq!(classes.get(255), 3); + } + + #[test] + fn full_byte_classes() { + let mut set = ByteClassSet::empty(); + for b in 0u8..=255 { + set.set_range(b, b); + } + assert_eq!(set.byte_classes().alphabet_len(), 256); + } + + #[test] + fn elements_typical() { + let mut set = ByteClassSet::empty(); + set.set_range(b'b', b'd'); + set.set_range(b'g', b'm'); + set.set_range(b'z', b'z'); + let classes = set.byte_classes(); + // class 0: \x00-a + // class 1: b-d + // class 2: e-f + // class 3: g-m + // class 4: n-y + // class 5: z-z + // class 6: \x7B-\xFF + assert_eq!(classes.alphabet_len(), 7); + + let elements = classes.elements(0).collect::>(); + assert_eq!(elements.len(), 98); + assert_eq!(elements[0], b'\x00'); + assert_eq!(elements[97], b'a'); + + let elements = classes.elements(1).collect::>(); + assert_eq!(elements, vec![b'b', b'c', b'd'],); + + let elements = classes.elements(2).collect::>(); + assert_eq!(elements, vec![b'e', b'f'],); + + let elements = classes.elements(3).collect::>(); + assert_eq!(elements, vec![b'g', b'h', b'i', b'j', b'k', b'l', b'm',],); + + let elements = classes.elements(4).collect::>(); + assert_eq!(elements.len(), 12); + assert_eq!(elements[0], b'n'); + assert_eq!(elements[11], b'y'); + + let elements = classes.elements(5).collect::>(); + assert_eq!(elements, vec![b'z']); + + let elements = classes.elements(6).collect::>(); + assert_eq!(elements.len(), 133); + assert_eq!(elements[0], b'\x7B'); + assert_eq!(elements[132], b'\xFF'); + } + + #[test] + fn elements_singletons() { + let classes = ByteClasses::singletons(); + assert_eq!(classes.alphabet_len(), 256); + + let elements = classes.elements(b'a').collect::>(); + assert_eq!(elements, vec![b'a']); + } + + #[test] + fn elements_empty() { + let classes = ByteClasses::empty(); + assert_eq!(classes.alphabet_len(), 1); + + let elements = classes.elements(0).collect::>(); + assert_eq!(elements.len(), 256); + assert_eq!(elements[0], b'\x00'); + assert_eq!(elements[255], b'\xFF'); + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/buffer.rs b/third_party/crates/aho-corasick-1.1.5/src/util/buffer.rs new file mode 100644 index 0000000..e9e982a --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/buffer.rs @@ -0,0 +1,124 @@ +use alloc::{vec, vec::Vec}; + +/// The default buffer capacity that we use for the stream buffer. +const DEFAULT_BUFFER_CAPACITY: usize = 64 * (1 << 10); // 64 KB + +/// A fairly simple roll buffer for supporting stream searches. +/// +/// This buffer acts as a temporary place to store a fixed amount of data when +/// reading from a stream. Its central purpose is to allow "rolling" some +/// suffix of the data to the beginning of the buffer before refilling it with +/// more data from the stream. For example, let's say we are trying to match +/// "foobar" on a stream. When we report the match, we'd like to not only +/// report the correct offsets at which the match occurs, but also the matching +/// bytes themselves. So let's say our stream is a file with the following +/// contents: `test test foobar test test`. Now assume that we happen to read +/// the aforementioned file in two chunks: `test test foo` and `bar test test`. +/// Naively, it would not be possible to report a single contiguous `foobar` +/// match, but this roll buffer allows us to do that. Namely, after the second +/// read, the contents of the buffer should be `st foobar test test`, where the +/// search should ultimately resume immediately after `foo`. (The prefix `st ` +/// is included because the roll buffer saves N bytes at the end of the buffer, +/// where N is the maximum possible length of a match.) +/// +/// A lot of the logic for dealing with this is unfortunately split out between +/// this roll buffer and the `StreamChunkIter`. +/// +/// Note also that this buffer is not actually required to just report matches. +/// Because a `Match` is just some offsets. But it *is* required for supporting +/// things like `try_stream_replace_all` because that needs some mechanism for +/// knowing which bytes in the stream correspond to a match and which don't. So +/// when a match occurs across two `read` calls, *something* needs to retain +/// the bytes from the previous `read` call because you don't know before the +/// second read call whether a match exists or not. +#[derive(Debug)] +pub(crate) struct Buffer { + /// The raw buffer contents. This has a fixed size and never increases. + buf: Vec, + /// The minimum size of the buffer, which is equivalent to the maximum + /// possible length of a match. This corresponds to the amount that we + /// roll + min: usize, + /// The end of the contents of this buffer. + end: usize, +} + +impl Buffer { + /// Create a new buffer for stream searching. The minimum buffer length + /// given should be the size of the maximum possible match length. + pub(crate) fn new(min_buffer_len: usize) -> Buffer { + let min = core::cmp::max(1, min_buffer_len); + // The minimum buffer amount is also the amount that we roll our + // buffer in order to support incremental searching. To this end, + // our actual capacity needs to be at least 1 byte bigger than our + // minimum amount, otherwise we won't have any overlap. In actuality, + // we want our buffer to be a bit bigger than that for performance + // reasons, so we set a lower bound of `8 * min`. + // + // TODO: It would be good to find a way to test the streaming + // implementation with the minimal buffer size. For now, we just + // uncomment out the next line and comment out the subsequent line. + // let capacity = 1 + min; + let capacity = core::cmp::max(min * 8, DEFAULT_BUFFER_CAPACITY); + Buffer { buf: vec![0; capacity], min, end: 0 } + } + + /// Return the contents of this buffer. + #[inline] + pub(crate) fn buffer(&self) -> &[u8] { + &self.buf[..self.end] + } + + /// Return the minimum size of the buffer. The only way a buffer may be + /// smaller than this is if the stream itself contains less than the + /// minimum buffer amount. + #[inline] + pub(crate) fn min_buffer_len(&self) -> usize { + self.min + } + + /// Return all free capacity in this buffer. + fn free_buffer(&mut self) -> &mut [u8] { + &mut self.buf[self.end..] + } + + /// Refill the contents of this buffer by reading as much as possible into + /// this buffer's free capacity. If no more bytes could be read, then this + /// returns false. Otherwise, this reads until it has filled the buffer + /// past the minimum amount. + pub(crate) fn fill( + &mut self, + mut rdr: R, + ) -> std::io::Result { + let mut readany = false; + loop { + let readlen = rdr.read(self.free_buffer())?; + if readlen == 0 { + return Ok(readany); + } + readany = true; + self.end += readlen; + if self.buffer().len() >= self.min { + return Ok(true); + } + } + } + + /// Roll the contents of the buffer so that the suffix of this buffer is + /// moved to the front and all other contents are dropped. The size of the + /// suffix corresponds precisely to the minimum buffer length. + /// + /// This should only be called when the entire contents of this buffer have + /// been searched. + pub(crate) fn roll(&mut self) { + let roll_start = self + .end + .checked_sub(self.min) + .expect("buffer capacity should be bigger than minimum amount"); + let roll_end = roll_start + self.min; + + assert!(roll_end <= self.end); + self.buf.copy_within(roll_start..roll_end, 0); + self.end = self.min; + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/byte_frequencies.rs b/third_party/crates/aho-corasick-1.1.5/src/util/byte_frequencies.rs new file mode 100644 index 0000000..c313b62 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/byte_frequencies.rs @@ -0,0 +1,258 @@ +pub const BYTE_FREQUENCIES: [u8; 256] = [ + 55, // '\x00' + 52, // '\x01' + 51, // '\x02' + 50, // '\x03' + 49, // '\x04' + 48, // '\x05' + 47, // '\x06' + 46, // '\x07' + 45, // '\x08' + 103, // '\t' + 242, // '\n' + 66, // '\x0b' + 67, // '\x0c' + 229, // '\r' + 44, // '\x0e' + 43, // '\x0f' + 42, // '\x10' + 41, // '\x11' + 40, // '\x12' + 39, // '\x13' + 38, // '\x14' + 37, // '\x15' + 36, // '\x16' + 35, // '\x17' + 34, // '\x18' + 33, // '\x19' + 56, // '\x1a' + 32, // '\x1b' + 31, // '\x1c' + 30, // '\x1d' + 29, // '\x1e' + 28, // '\x1f' + 255, // ' ' + 148, // '!' + 164, // '"' + 149, // '#' + 136, // '$' + 160, // '%' + 155, // '&' + 173, // "'" + 221, // '(' + 222, // ')' + 134, // '*' + 122, // '+' + 232, // ',' + 202, // '-' + 215, // '.' + 224, // '/' + 208, // '0' + 220, // '1' + 204, // '2' + 187, // '3' + 183, // '4' + 179, // '5' + 177, // '6' + 168, // '7' + 178, // '8' + 200, // '9' + 226, // ':' + 195, // ';' + 154, // '<' + 184, // '=' + 174, // '>' + 126, // '?' + 120, // '@' + 191, // 'A' + 157, // 'B' + 194, // 'C' + 170, // 'D' + 189, // 'E' + 162, // 'F' + 161, // 'G' + 150, // 'H' + 193, // 'I' + 142, // 'J' + 137, // 'K' + 171, // 'L' + 176, // 'M' + 185, // 'N' + 167, // 'O' + 186, // 'P' + 112, // 'Q' + 175, // 'R' + 192, // 'S' + 188, // 'T' + 156, // 'U' + 140, // 'V' + 143, // 'W' + 123, // 'X' + 133, // 'Y' + 128, // 'Z' + 147, // '[' + 138, // '\\' + 146, // ']' + 114, // '^' + 223, // '_' + 151, // '`' + 249, // 'a' + 216, // 'b' + 238, // 'c' + 236, // 'd' + 253, // 'e' + 227, // 'f' + 218, // 'g' + 230, // 'h' + 247, // 'i' + 135, // 'j' + 180, // 'k' + 241, // 'l' + 233, // 'm' + 246, // 'n' + 244, // 'o' + 231, // 'p' + 139, // 'q' + 245, // 'r' + 243, // 's' + 251, // 't' + 235, // 'u' + 201, // 'v' + 196, // 'w' + 240, // 'x' + 214, // 'y' + 152, // 'z' + 182, // '{' + 205, // '|' + 181, // '}' + 127, // '~' + 27, // '\x7f' + 212, // '\x80' + 211, // '\x81' + 210, // '\x82' + 213, // '\x83' + 228, // '\x84' + 197, // '\x85' + 169, // '\x86' + 159, // '\x87' + 131, // '\x88' + 172, // '\x89' + 105, // '\x8a' + 80, // '\x8b' + 98, // '\x8c' + 96, // '\x8d' + 97, // '\x8e' + 81, // '\x8f' + 207, // '\x90' + 145, // '\x91' + 116, // '\x92' + 115, // '\x93' + 144, // '\x94' + 130, // '\x95' + 153, // '\x96' + 121, // '\x97' + 107, // '\x98' + 132, // '\x99' + 109, // '\x9a' + 110, // '\x9b' + 124, // '\x9c' + 111, // '\x9d' + 82, // '\x9e' + 108, // '\x9f' + 118, // '\xa0' + 141, // '¡' + 113, // '¢' + 129, // '£' + 119, // '¤' + 125, // '¥' + 165, // '¦' + 117, // '§' + 92, // '¨' + 106, // '©' + 83, // 'ª' + 72, // '«' + 99, // '¬' + 93, // '\xad' + 65, // '®' + 79, // '¯' + 166, // '°' + 237, // '±' + 163, // '²' + 199, // '³' + 190, // '´' + 225, // 'µ' + 209, // '¶' + 203, // '·' + 198, // '¸' + 217, // '¹' + 219, // 'º' + 206, // '»' + 234, // '¼' + 248, // '½' + 158, // '¾' + 239, // '¿' + 255, // 'À' + 255, // 'Á' + 255, // 'Â' + 255, // 'Ã' + 255, // 'Ä' + 255, // 'Å' + 255, // 'Æ' + 255, // 'Ç' + 255, // 'È' + 255, // 'É' + 255, // 'Ê' + 255, // 'Ë' + 255, // 'Ì' + 255, // 'Í' + 255, // 'Î' + 255, // 'Ï' + 255, // 'Ð' + 255, // 'Ñ' + 255, // 'Ò' + 255, // 'Ó' + 255, // 'Ô' + 255, // 'Õ' + 255, // 'Ö' + 255, // '×' + 255, // 'Ø' + 255, // 'Ù' + 255, // 'Ú' + 255, // 'Û' + 255, // 'Ü' + 255, // 'Ý' + 255, // 'Þ' + 255, // 'ß' + 255, // 'à' + 255, // 'á' + 255, // 'â' + 255, // 'ã' + 255, // 'ä' + 255, // 'å' + 255, // 'æ' + 255, // 'ç' + 255, // 'è' + 255, // 'é' + 255, // 'ê' + 255, // 'ë' + 255, // 'ì' + 255, // 'í' + 255, // 'î' + 255, // 'ï' + 255, // 'ð' + 255, // 'ñ' + 255, // 'ò' + 255, // 'ó' + 255, // 'ô' + 255, // 'õ' + 255, // 'ö' + 255, // '÷' + 255, // 'ø' + 255, // 'ù' + 255, // 'ú' + 255, // 'û' + 255, // 'ü' + 255, // 'ý' + 255, // 'þ' + 255, // 'ÿ' +]; diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/debug.rs b/third_party/crates/aho-corasick-1.1.5/src/util/debug.rs new file mode 100644 index 0000000..22b5f22 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/debug.rs @@ -0,0 +1,26 @@ +/// A type that wraps a single byte with a convenient fmt::Debug impl that +/// escapes the byte. +pub(crate) struct DebugByte(pub(crate) u8); + +impl core::fmt::Debug for DebugByte { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + // Special case ASCII space. It's too hard to read otherwise, so + // put quotes around it. I sometimes wonder whether just '\x20' would + // be better... + if self.0 == b' ' { + return write!(f, "' '"); + } + // 10 bytes is enough to cover any output from ascii::escape_default. + let mut bytes = [0u8; 10]; + let mut len = 0; + for (i, mut b) in core::ascii::escape_default(self.0).enumerate() { + // capitalize \xab to \xAB + if i >= 2 && b'a' <= b && b <= b'f' { + b -= 32; + } + bytes[len] = b; + len += 1; + } + write!(f, "{}", core::str::from_utf8(&bytes[..len]).unwrap()) + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/error.rs b/third_party/crates/aho-corasick-1.1.5/src/util/error.rs new file mode 100644 index 0000000..326d046 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/error.rs @@ -0,0 +1,259 @@ +use crate::util::{ + primitives::{PatternID, SmallIndex}, + search::MatchKind, +}; + +/// An error that occurred during the construction of an Aho-Corasick +/// automaton. +/// +/// Build errors occur when some kind of limit has been exceeded, either in the +/// number of states, the number of patterns of the length of a pattern. These +/// limits aren't part of the public API, but they should generally be large +/// enough to handle most use cases. +/// +/// When the `std` feature is enabled, this implements the `std::error::Error` +/// trait. +#[derive(Clone, Debug)] +pub struct BuildError { + kind: ErrorKind, +} + +/// The kind of error that occurred. +#[derive(Clone, Debug)] +enum ErrorKind { + /// An error that occurs when allocating a new state would result in an + /// identifier that exceeds the capacity of a `StateID`. + StateIDOverflow { + /// The maximum possible id. + max: u64, + /// The maximum ID requested. + requested_max: u64, + }, + /// An error that occurs when adding a pattern to an Aho-Corasick + /// automaton would result in an identifier that exceeds the capacity of a + /// `PatternID`. + PatternIDOverflow { + /// The maximum possible id. + max: u64, + /// The maximum ID requested. + requested_max: u64, + }, + /// Occurs when a pattern string is given to the Aho-Corasick constructor + /// that is too long. + PatternTooLong { + /// The ID of the pattern that was too long. + pattern: PatternID, + /// The length that was too long. + len: usize, + }, +} + +impl BuildError { + pub(crate) fn state_id_overflow( + max: u64, + requested_max: u64, + ) -> BuildError { + BuildError { kind: ErrorKind::StateIDOverflow { max, requested_max } } + } + + pub(crate) fn pattern_id_overflow( + max: u64, + requested_max: u64, + ) -> BuildError { + BuildError { + kind: ErrorKind::PatternIDOverflow { max, requested_max }, + } + } + + pub(crate) fn pattern_too_long( + pattern: PatternID, + len: usize, + ) -> BuildError { + BuildError { kind: ErrorKind::PatternTooLong { pattern, len } } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for BuildError {} + +impl core::fmt::Display for BuildError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.kind { + ErrorKind::StateIDOverflow { max, requested_max } => { + write!( + f, + "state identifier overflow: failed to create state ID \ + from {}, which exceeds the max of {}", + requested_max, max, + ) + } + ErrorKind::PatternIDOverflow { max, requested_max } => { + write!( + f, + "pattern identifier overflow: failed to create pattern ID \ + from {}, which exceeds the max of {}", + requested_max, max, + ) + } + ErrorKind::PatternTooLong { pattern, len } => { + write!( + f, + "pattern {} with length {} exceeds \ + the maximum pattern length of {}", + pattern.as_usize(), + len, + SmallIndex::MAX.as_usize(), + ) + } + } + } +} + +/// An error that occurred during an Aho-Corasick search. +/// +/// An error that occurs during a search is limited to some kind of +/// misconfiguration that resulted in an illegal call. Stated differently, +/// whether an error occurs is not dependent on the specific bytes in the +/// haystack. +/// +/// Examples of misconfiguration: +/// +/// * Executing a stream or overlapping search on a searcher that was built was +/// something other than [`MatchKind::Standard`](crate::MatchKind::Standard) +/// semantics. +/// * Requested an anchored or an unanchored search on a searcher that doesn't +/// support unanchored or anchored searches, respectively. +/// +/// When the `std` feature is enabled, this implements the `std::error::Error` +/// trait. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MatchError(alloc::boxed::Box); + +impl MatchError { + /// Create a new error value with the given kind. + /// + /// This is a more verbose version of the kind-specific constructors, e.g., + /// `MatchError::unsupported_stream`. + pub fn new(kind: MatchErrorKind) -> MatchError { + MatchError(alloc::boxed::Box::new(kind)) + } + + /// Returns a reference to the underlying error kind. + pub fn kind(&self) -> &MatchErrorKind { + &self.0 + } + + /// Create a new "invalid anchored search" error. This occurs when the + /// caller requests an anchored search but where anchored searches aren't + /// supported. + /// + /// This is the same as calling `MatchError::new` with a + /// [`MatchErrorKind::InvalidInputAnchored`] kind. + pub fn invalid_input_anchored() -> MatchError { + MatchError::new(MatchErrorKind::InvalidInputAnchored) + } + + /// Create a new "invalid unanchored search" error. This occurs when the + /// caller requests an unanchored search but where unanchored searches + /// aren't supported. + /// + /// This is the same as calling `MatchError::new` with a + /// [`MatchErrorKind::InvalidInputUnanchored`] kind. + pub fn invalid_input_unanchored() -> MatchError { + MatchError::new(MatchErrorKind::InvalidInputUnanchored) + } + + /// Create a new "unsupported stream search" error. This occurs when the + /// caller requests a stream search while using an Aho-Corasick automaton + /// with a match kind other than [`MatchKind::Standard`]. + /// + /// The match kind given should be the match kind of the automaton. It + /// should never be `MatchKind::Standard`. + pub fn unsupported_stream(got: MatchKind) -> MatchError { + MatchError::new(MatchErrorKind::UnsupportedStream { got }) + } + + /// Create a new "unsupported overlapping search" error. This occurs when + /// the caller requests an overlapping search while using an Aho-Corasick + /// automaton with a match kind other than [`MatchKind::Standard`]. + /// + /// The match kind given should be the match kind of the automaton. It + /// should never be `MatchKind::Standard`. + pub fn unsupported_overlapping(got: MatchKind) -> MatchError { + MatchError::new(MatchErrorKind::UnsupportedOverlapping { got }) + } + + /// Create a new "unsupported empty pattern" error. This occurs when the + /// caller requests a search for which matching an automaton that contains + /// an empty pattern string is not supported. + pub fn unsupported_empty() -> MatchError { + MatchError::new(MatchErrorKind::UnsupportedEmpty) + } +} + +/// The underlying kind of a [`MatchError`]. +/// +/// This is a **non-exhaustive** enum. That means new variants may be added in +/// a semver-compatible release. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum MatchErrorKind { + /// An error indicating that an anchored search was requested, but from a + /// searcher that was built without anchored support. + InvalidInputAnchored, + /// An error indicating that an unanchored search was requested, but from a + /// searcher that was built without unanchored support. + InvalidInputUnanchored, + /// An error indicating that a stream search was attempted on an + /// Aho-Corasick automaton with an unsupported `MatchKind`. + UnsupportedStream { + /// The match semantics for the automaton that was used. + got: MatchKind, + }, + /// An error indicating that an overlapping search was attempted on an + /// Aho-Corasick automaton with an unsupported `MatchKind`. + UnsupportedOverlapping { + /// The match semantics for the automaton that was used. + got: MatchKind, + }, + /// An error indicating that the operation requested doesn't support + /// automatons that contain an empty pattern string. + UnsupportedEmpty, +} + +#[cfg(feature = "std")] +impl std::error::Error for MatchError {} + +impl core::fmt::Display for MatchError { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + match *self.kind() { + MatchErrorKind::InvalidInputAnchored => { + write!(f, "anchored searches are not supported or enabled") + } + MatchErrorKind::InvalidInputUnanchored => { + write!(f, "unanchored searches are not supported or enabled") + } + MatchErrorKind::UnsupportedStream { got } => { + write!( + f, + "match kind {:?} does not support stream searching", + got, + ) + } + MatchErrorKind::UnsupportedOverlapping { got } => { + write!( + f, + "match kind {:?} does not support overlapping searches", + got, + ) + } + MatchErrorKind::UnsupportedEmpty => { + write!( + f, + "matching with an empty pattern string is not \ + supported for this operation", + ) + } + } + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/int.rs b/third_party/crates/aho-corasick-1.1.5/src/util/int.rs new file mode 100644 index 0000000..54762b6 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/int.rs @@ -0,0 +1,278 @@ +/*! +This module provides several integer oriented traits for converting between +both fixed size integers and integers whose size varies based on the target +(like `usize`). + +The main design principle for this module is to centralize all uses of `as`. +The thinking here is that `as` makes it very easy to perform accidental lossy +conversions, and if we centralize all its uses here under more descriptive +higher level operations, its use and correctness becomes easier to audit. + +This was copied mostly wholesale from `regex-automata`. + +NOTE: for simplicity, we don't take target pointer width into account here for +`usize` conversions. Since we currently only panic in debug mode, skipping the +check when it can be proven it isn't needed at compile time doesn't really +matter. Now, if we wind up wanting to do as many checks as possible in release +mode, then we would want to skip those when we know the conversions are always +non-lossy. +*/ + +// We define a little more than what we need, but I'd rather just have +// everything via a consistent and uniform API then have holes. +#![allow(dead_code)] + +pub(crate) trait U8 { + fn as_usize(self) -> usize; +} + +impl U8 for u8 { + fn as_usize(self) -> usize { + usize::from(self) + } +} + +pub(crate) trait U16 { + fn as_usize(self) -> usize; + fn low_u8(self) -> u8; + fn high_u8(self) -> u8; +} + +impl U16 for u16 { + fn as_usize(self) -> usize { + usize::from(self) + } + + fn low_u8(self) -> u8 { + self as u8 + } + + fn high_u8(self) -> u8 { + (self >> 8) as u8 + } +} + +pub(crate) trait U32 { + fn as_usize(self) -> usize; + fn low_u8(self) -> u8; + fn low_u16(self) -> u16; + fn high_u16(self) -> u16; +} + +impl U32 for u32 { + #[inline] + fn as_usize(self) -> usize { + #[cfg(debug_assertions)] + { + usize::try_from(self).expect("u32 overflowed usize") + } + #[cfg(not(debug_assertions))] + { + self as usize + } + } + + fn low_u8(self) -> u8 { + self as u8 + } + + fn low_u16(self) -> u16 { + self as u16 + } + + fn high_u16(self) -> u16 { + (self >> 16) as u16 + } +} + +pub(crate) trait U64 { + fn as_usize(self) -> usize; + fn low_u8(self) -> u8; + fn low_u16(self) -> u16; + fn low_u32(self) -> u32; + fn high_u32(self) -> u32; +} + +impl U64 for u64 { + fn as_usize(self) -> usize { + #[cfg(debug_assertions)] + { + usize::try_from(self).expect("u64 overflowed usize") + } + #[cfg(not(debug_assertions))] + { + self as usize + } + } + + fn low_u8(self) -> u8 { + self as u8 + } + + fn low_u16(self) -> u16 { + self as u16 + } + + fn low_u32(self) -> u32 { + self as u32 + } + + fn high_u32(self) -> u32 { + (self >> 32) as u32 + } +} + +pub(crate) trait I8 { + fn as_usize(self) -> usize; + fn to_bits(self) -> u8; + fn from_bits(n: u8) -> i8; +} + +impl I8 for i8 { + fn as_usize(self) -> usize { + #[cfg(debug_assertions)] + { + usize::try_from(self).expect("i8 overflowed usize") + } + #[cfg(not(debug_assertions))] + { + self as usize + } + } + + fn to_bits(self) -> u8 { + self as u8 + } + + fn from_bits(n: u8) -> i8 { + n as i8 + } +} + +pub(crate) trait I32 { + fn as_usize(self) -> usize; + fn to_bits(self) -> u32; + fn from_bits(n: u32) -> i32; +} + +impl I32 for i32 { + fn as_usize(self) -> usize { + #[cfg(debug_assertions)] + { + usize::try_from(self).expect("i32 overflowed usize") + } + #[cfg(not(debug_assertions))] + { + self as usize + } + } + + fn to_bits(self) -> u32 { + self as u32 + } + + fn from_bits(n: u32) -> i32 { + n as i32 + } +} + +pub(crate) trait I64 { + fn as_usize(self) -> usize; + fn to_bits(self) -> u64; + fn from_bits(n: u64) -> i64; +} + +impl I64 for i64 { + fn as_usize(self) -> usize { + #[cfg(debug_assertions)] + { + usize::try_from(self).expect("i64 overflowed usize") + } + #[cfg(not(debug_assertions))] + { + self as usize + } + } + + fn to_bits(self) -> u64 { + self as u64 + } + + fn from_bits(n: u64) -> i64 { + n as i64 + } +} + +pub(crate) trait Usize { + fn as_u8(self) -> u8; + fn as_u16(self) -> u16; + fn as_u32(self) -> u32; + fn as_u64(self) -> u64; +} + +impl Usize for usize { + fn as_u8(self) -> u8 { + #[cfg(debug_assertions)] + { + u8::try_from(self).expect("usize overflowed u8") + } + #[cfg(not(debug_assertions))] + { + self as u8 + } + } + + fn as_u16(self) -> u16 { + #[cfg(debug_assertions)] + { + u16::try_from(self).expect("usize overflowed u16") + } + #[cfg(not(debug_assertions))] + { + self as u16 + } + } + + fn as_u32(self) -> u32 { + #[cfg(debug_assertions)] + { + u32::try_from(self).expect("usize overflowed u32") + } + #[cfg(not(debug_assertions))] + { + self as u32 + } + } + + fn as_u64(self) -> u64 { + #[cfg(debug_assertions)] + { + u64::try_from(self).expect("usize overflowed u64") + } + #[cfg(not(debug_assertions))] + { + self as u64 + } + } +} + +// Pointers aren't integers, but we convert pointers to integers to perform +// offset arithmetic in some places. (And no, we don't convert the integers +// back to pointers.) So add 'as_usize' conversions here too for completeness. +// +// These 'as' casts are actually okay because they're always non-lossy. But the +// idea here is to just try and remove as much 'as' as possible, particularly +// in this crate where we are being really paranoid about offsets and making +// sure we don't panic on inputs that might be untrusted. This way, the 'as' +// casts become easier to audit if they're all in one place, even when some of +// them are actually okay 100% of the time. + +pub(crate) trait Pointer { + fn as_usize(self) -> usize; +} + +impl Pointer for *const T { + fn as_usize(self) -> usize { + self as usize + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/mod.rs b/third_party/crates/aho-corasick-1.1.5/src/util/mod.rs new file mode 100644 index 0000000..f7a1ddd --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/mod.rs @@ -0,0 +1,12 @@ +pub(crate) mod alphabet; +#[cfg(feature = "std")] +pub(crate) mod buffer; +pub(crate) mod byte_frequencies; +pub(crate) mod debug; +pub(crate) mod error; +pub(crate) mod int; +pub(crate) mod prefilter; +pub(crate) mod primitives; +pub(crate) mod remapper; +pub(crate) mod search; +pub(crate) mod special; diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/prefilter.rs b/third_party/crates/aho-corasick-1.1.5/src/util/prefilter.rs new file mode 100644 index 0000000..ec31716 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/prefilter.rs @@ -0,0 +1,924 @@ +use core::{ + cmp, + fmt::Debug, + panic::{RefUnwindSafe, UnwindSafe}, + u8, +}; + +use alloc::{sync::Arc, vec, vec::Vec}; + +use crate::{ + packed, + util::{ + alphabet::ByteSet, + search::{Match, MatchKind, Span}, + }, +}; + +/// A prefilter for accelerating a search. +/// +/// This crate uses prefilters in the core search implementations to accelerate +/// common cases. They typically only apply to cases where there are a small +/// number of patterns (less than 100 or so), but when they do, thoughput can +/// be boosted considerably, perhaps by an order of magnitude. When a prefilter +/// is active, it is used whenever a search enters an automaton's start state. +/// +/// Currently, prefilters cannot be constructed by +/// callers. A `Prefilter` can only be accessed via the +/// [`Automaton::prefilter`](crate::automaton::Automaton::prefilter) +/// method and used to execute a search. In other words, a prefilter can be +/// used to optimize your own search implementation if necessary, but cannot do +/// much else. If you have a use case for more APIs, please submit an issue. +#[derive(Clone, Debug)] +pub struct Prefilter { + finder: Arc, + memory_usage: usize, +} + +impl Prefilter { + /// Execute a search in the haystack within the span given. If a match or + /// a possible match is returned, then it is guaranteed to occur within + /// the bounds of the span. + /// + /// If the span provided is invalid for the given haystack, then behavior + /// is unspecified. + #[inline] + pub fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + self.finder.find_in(haystack, span) + } + + #[inline] + pub(crate) fn memory_usage(&self) -> usize { + self.memory_usage + } +} + +/// A candidate is the result of running a prefilter on a haystack at a +/// particular position. +/// +/// The result is either no match, a confirmed match or a possible match. +/// +/// When no match is returned, the prefilter is guaranteeing that no possible +/// match can be found in the haystack, and the caller may trust this. That is, +/// all correct prefilters must never report false negatives. +/// +/// In some cases, a prefilter can confirm a match very quickly, in which case, +/// the caller may use this to stop what it's doing and report the match. In +/// this case, prefilter implementations must never report a false positive. +/// In other cases, the prefilter can only report a potential match, in which +/// case the callers must attempt to confirm the match. In this case, prefilter +/// implementations are permitted to return false positives. +#[derive(Clone, Debug)] +pub enum Candidate { + /// No match was found. Since false negatives are not possible, this means + /// the search can quit as it is guaranteed not to find another match. + None, + /// A confirmed match was found. Callers do not need to confirm it. + Match(Match), + /// The start of a possible match was found. Callers must confirm it before + /// reporting it as a match. + PossibleStartOfMatch(usize), +} + +impl Candidate { + /// Convert this candidate into an option. This is useful when callers + /// do not distinguish between true positives and false positives (i.e., + /// the caller must always confirm the match). + pub fn into_option(self) -> Option { + match self { + Candidate::None => None, + Candidate::Match(ref m) => Some(m.start()), + Candidate::PossibleStartOfMatch(start) => Some(start), + } + } +} + +/// A prefilter describes the behavior of fast literal scanners for quickly +/// skipping past bytes in the haystack that we know cannot possibly +/// participate in a match. +trait PrefilterI: + Send + Sync + RefUnwindSafe + UnwindSafe + Debug + 'static +{ + /// Returns the next possible match candidate. This may yield false + /// positives, so callers must confirm a match starting at the position + /// returned. This, however, must never produce false negatives. That is, + /// this must, at minimum, return the starting position of the next match + /// in the given haystack after or at the given position. + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate; +} + +impl PrefilterI for Arc

{ + #[inline(always)] + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + (**self).find_in(haystack, span) + } +} + +/// A builder for constructing the best possible prefilter. When constructed, +/// this builder will heuristically select the best prefilter it can build, +/// if any, and discard the rest. +#[derive(Debug)] +pub(crate) struct Builder { + count: usize, + ascii_case_insensitive: bool, + start_bytes: StartBytesBuilder, + rare_bytes: RareBytesBuilder, + memmem: MemmemBuilder, + packed: Option, + // If we run across a condition that suggests we shouldn't use a prefilter + // at all (like an empty pattern), then disable prefilters entirely. + enabled: bool, +} + +impl Builder { + /// Create a new builder for constructing the best possible prefilter. + pub(crate) fn new(kind: MatchKind) -> Builder { + let pbuilder = kind + .as_packed() + .map(|kind| packed::Config::new().match_kind(kind).builder()); + Builder { + count: 0, + ascii_case_insensitive: false, + start_bytes: StartBytesBuilder::new(), + rare_bytes: RareBytesBuilder::new(), + memmem: MemmemBuilder::default(), + packed: pbuilder, + enabled: true, + } + } + + /// Enable ASCII case insensitivity. When set, byte strings added to this + /// builder will be interpreted without respect to ASCII case. + pub(crate) fn ascii_case_insensitive(mut self, yes: bool) -> Builder { + self.ascii_case_insensitive = yes; + self.start_bytes = self.start_bytes.ascii_case_insensitive(yes); + self.rare_bytes = self.rare_bytes.ascii_case_insensitive(yes); + self + } + + /// Return a prefilter suitable for quickly finding potential matches. + /// + /// All patterns added to an Aho-Corasick automaton should be added to this + /// builder before attempting to construct the prefilter. + pub(crate) fn build(&self) -> Option { + if !self.enabled { + debug!("prefilter not enabled, skipping"); + return None; + } + // If we only have one pattern, then deferring to memmem is always + // the best choice. This is kind of a weird case, because, well, why + // use Aho-Corasick if you only have one pattern? But maybe you don't + // know exactly how many patterns you'll get up front, and you need to + // support the option of multiple patterns. So instead of relying on + // the caller to branch and use memmem explicitly, we just do it for + // them. + if !self.ascii_case_insensitive { + if let Some(pre) = self.memmem.build() { + debug!("using memmem prefilter"); + return Some(pre); + } + } + let (packed, patlen, minlen) = if self.ascii_case_insensitive { + (None, usize::MAX, 0) + } else { + let patlen = self.packed.as_ref().map_or(usize::MAX, |p| p.len()); + let minlen = self.packed.as_ref().map_or(0, |p| p.minimum_len()); + let packed = + self.packed.as_ref().and_then(|b| b.build()).map(|s| { + let memory_usage = s.memory_usage(); + debug!( + "built packed prefilter (len: {}, \ + minimum pattern len: {}, memory usage: {}) \ + for consideration", + patlen, minlen, memory_usage, + ); + Prefilter { finder: Arc::new(Packed(s)), memory_usage } + }); + (packed, patlen, minlen) + }; + match (self.start_bytes.build(), self.rare_bytes.build()) { + // If we could build both start and rare prefilters, then there are + // a few cases in which we'd want to use the start-byte prefilter + // over the rare-byte prefilter, since the former has lower + // overhead. + (prestart @ Some(_), prerare @ Some(_)) => { + debug!( + "both start (len={}, rank={}) and \ + rare (len={}, rank={}) byte prefilters \ + are available", + self.start_bytes.count, + self.start_bytes.rank_sum, + self.rare_bytes.count, + self.rare_bytes.rank_sum, + ); + if patlen <= 16 + && minlen >= 2 + && self.start_bytes.count >= 3 + && self.rare_bytes.count >= 3 + { + debug!( + "start and rare byte prefilters available, but \ + they're probably slower than packed so using \ + packed" + ); + return packed; + } + // If the start-byte prefilter can scan for a smaller number + // of bytes than the rare-byte prefilter, then it's probably + // faster. + let has_fewer_bytes = + self.start_bytes.count < self.rare_bytes.count; + // Otherwise, if the combined frequency rank of the detected + // bytes in the start-byte prefilter is "close" to the combined + // frequency rank of the rare-byte prefilter, then we pick + // the start-byte prefilter even if the rare-byte prefilter + // heuristically searches for rare bytes. This is because the + // rare-byte prefilter has higher constant costs, so we tend to + // prefer the start-byte prefilter when we can. + let has_rarer_bytes = + self.start_bytes.rank_sum <= self.rare_bytes.rank_sum + 50; + if has_fewer_bytes { + debug!( + "using start byte prefilter because it has fewer + bytes to search for than the rare byte prefilter", + ); + prestart + } else if has_rarer_bytes { + debug!( + "using start byte prefilter because its byte \ + frequency rank was determined to be \ + \"good enough\" relative to the rare byte prefilter \ + byte frequency rank", + ); + prestart + } else { + debug!("using rare byte prefilter"); + prerare + } + } + (prestart @ Some(_), None) => { + if patlen <= 16 && minlen >= 2 && self.start_bytes.count >= 3 { + debug!( + "start byte prefilter available, but \ + it's probably slower than packed so using \ + packed" + ); + return packed; + } + debug!( + "have start byte prefilter but not rare byte prefilter, \ + so using start byte prefilter", + ); + prestart + } + (None, prerare @ Some(_)) => { + if patlen <= 16 && minlen >= 2 && self.rare_bytes.count >= 3 { + debug!( + "rare byte prefilter available, but \ + it's probably slower than packed so using \ + packed" + ); + return packed; + } + debug!( + "have rare byte prefilter but not start byte prefilter, \ + so using rare byte prefilter", + ); + prerare + } + (None, None) if self.ascii_case_insensitive => { + debug!( + "no start or rare byte prefilter and ASCII case \ + insensitivity was enabled, so skipping prefilter", + ); + None + } + (None, None) => { + if packed.is_some() { + debug!("falling back to packed prefilter"); + } else { + debug!("no prefilter available"); + } + packed + } + } + } + + /// Add a literal string to this prefilter builder. + pub(crate) fn add(&mut self, bytes: &[u8]) { + if bytes.is_empty() { + self.enabled = false; + } + if !self.enabled { + return; + } + self.count += 1; + self.start_bytes.add(bytes); + self.rare_bytes.add(bytes); + self.memmem.add(bytes); + if let Some(ref mut pbuilder) = self.packed { + pbuilder.add(bytes); + } + } +} + +/// A type that wraps a packed searcher and implements the `Prefilter` +/// interface. +#[derive(Clone, Debug)] +struct Packed(packed::Searcher); + +impl PrefilterI for Packed { + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + self.0 + .find_in(haystack, span) + .map_or(Candidate::None, Candidate::Match) + } +} + +/// A builder for constructing a prefilter that uses memmem. +#[derive(Debug, Default)] +struct MemmemBuilder { + /// The number of patterns that have been added. + count: usize, + /// The singular pattern to search for. This is only set when count==1. + one: Option>, +} + +impl MemmemBuilder { + fn build(&self) -> Option { + #[cfg(all(feature = "std", feature = "perf-literal"))] + fn imp(builder: &MemmemBuilder) -> Option { + let pattern = builder.one.as_ref()?; + assert_eq!(1, builder.count); + let finder = Arc::new(Memmem( + memchr::memmem::Finder::new(pattern).into_owned(), + )); + let memory_usage = pattern.len(); + Some(Prefilter { finder, memory_usage }) + } + + #[cfg(not(all(feature = "std", feature = "perf-literal")))] + fn imp(_: &MemmemBuilder) -> Option { + None + } + + imp(self) + } + + fn add(&mut self, bytes: &[u8]) { + self.count += 1; + if self.count == 1 { + self.one = Some(bytes.to_vec()); + } else { + self.one = None; + } + } +} + +/// A type that wraps a SIMD accelerated single substring search from the +/// `memchr` crate for use as a prefilter. +/// +/// Currently, this prefilter is only active for Aho-Corasick searchers with +/// a single pattern. In theory, this could be extended to support searchers +/// that have a common prefix of more than one byte (for one byte, we would use +/// memchr), but it's not clear if it's worth it or not. +/// +/// Also, unfortunately, this currently also requires the 'std' feature to +/// be enabled. That's because memchr doesn't have a no-std-but-with-alloc +/// mode, and so APIs like Finder::into_owned aren't available when 'std' is +/// disabled. But there should be an 'alloc' feature that brings in APIs like +/// Finder::into_owned but doesn't use std-only features like runtime CPU +/// feature detection. +#[cfg(all(feature = "std", feature = "perf-literal"))] +#[derive(Clone, Debug)] +struct Memmem(memchr::memmem::Finder<'static>); + +#[cfg(all(feature = "std", feature = "perf-literal"))] +impl PrefilterI for Memmem { + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + use crate::util::primitives::PatternID; + + self.0.find(&haystack[span]).map_or(Candidate::None, |i| { + let start = span.start + i; + let end = start + self.0.needle().len(); + // N.B. We can declare a match and use a fixed pattern ID here + // because a Memmem prefilter is only ever created for searchers + // with exactly one pattern. Thus, every match is always a match + // and it is always for the first and only pattern. + Candidate::Match(Match::new(PatternID::ZERO, start..end)) + }) + } +} + +/// A builder for constructing a rare byte prefilter. +/// +/// A rare byte prefilter attempts to pick out a small set of rare bytes that +/// occurr in the patterns, and then quickly scan to matches of those rare +/// bytes. +#[derive(Clone, Debug)] +struct RareBytesBuilder { + /// Whether this prefilter should account for ASCII case insensitivity or + /// not. + ascii_case_insensitive: bool, + /// A set of rare bytes, indexed by byte value. + rare_set: ByteSet, + /// A set of byte offsets associated with bytes in a pattern. An entry + /// corresponds to a particular bytes (its index) and is only non-zero if + /// the byte occurred at an offset greater than 0 in at least one pattern. + /// + /// If a byte's offset is not representable in 8 bits, then the rare bytes + /// prefilter becomes inert. + byte_offsets: RareByteOffsets, + /// Whether this is available as a prefilter or not. This can be set to + /// false during construction if a condition is seen that invalidates the + /// use of the rare-byte prefilter. + available: bool, + /// The number of bytes set to an active value in `byte_offsets`. + count: usize, + /// The sum of frequency ranks for the rare bytes detected. This is + /// intended to give a heuristic notion of how rare the bytes are. + rank_sum: u16, +} + +/// A set of byte offsets, keyed by byte. +#[derive(Clone, Copy)] +struct RareByteOffsets { + /// Each entry corresponds to the maximum offset of the corresponding + /// byte across all patterns seen. + set: [RareByteOffset; 256], +} + +impl RareByteOffsets { + /// Create a new empty set of rare byte offsets. + pub(crate) fn empty() -> RareByteOffsets { + RareByteOffsets { set: [RareByteOffset::default(); 256] } + } + + /// Add the given offset for the given byte to this set. If the offset is + /// greater than the existing offset, then it overwrites the previous + /// value and returns false. If there is no previous value set, then this + /// sets it and returns true. + pub(crate) fn set(&mut self, byte: u8, off: RareByteOffset) { + self.set[byte as usize].max = + cmp::max(self.set[byte as usize].max, off.max); + } +} + +impl core::fmt::Debug for RareByteOffsets { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut offsets = vec![]; + for off in self.set.iter() { + if off.max > 0 { + offsets.push(off); + } + } + f.debug_struct("RareByteOffsets").field("set", &offsets).finish() + } +} + +/// Offsets associated with an occurrence of a "rare" byte in any of the +/// patterns used to construct a single Aho-Corasick automaton. +#[derive(Clone, Copy, Debug)] +struct RareByteOffset { + /// The maximum offset at which a particular byte occurs from the start + /// of any pattern. This is used as a shift amount. That is, when an + /// occurrence of this byte is found, the candidate position reported by + /// the prefilter is `position_of_byte - max`, such that the automaton + /// will begin its search at a position that is guaranteed to observe a + /// match. + /// + /// To avoid accidentally quadratic behavior, a prefilter is considered + /// ineffective when it is asked to start scanning from a position that it + /// has already scanned past. + /// + /// Using a `u8` here means that if we ever see a pattern that's longer + /// than 255 bytes, then the entire rare byte prefilter is disabled. + max: u8, +} + +impl Default for RareByteOffset { + fn default() -> RareByteOffset { + RareByteOffset { max: 0 } + } +} + +impl RareByteOffset { + /// Create a new rare byte offset. If the given offset is too big, then + /// None is returned. In that case, callers should render the rare bytes + /// prefilter inert. + fn new(max: usize) -> Option { + if max > u8::MAX as usize { + None + } else { + Some(RareByteOffset { max: max as u8 }) + } + } +} + +impl RareBytesBuilder { + /// Create a new builder for constructing a rare byte prefilter. + fn new() -> RareBytesBuilder { + RareBytesBuilder { + ascii_case_insensitive: false, + rare_set: ByteSet::empty(), + byte_offsets: RareByteOffsets::empty(), + available: true, + count: 0, + rank_sum: 0, + } + } + + /// Enable ASCII case insensitivity. When set, byte strings added to this + /// builder will be interpreted without respect to ASCII case. + fn ascii_case_insensitive(mut self, yes: bool) -> RareBytesBuilder { + self.ascii_case_insensitive = yes; + self + } + + /// Build the rare bytes prefilter. + /// + /// If there are more than 3 distinct rare bytes found, or if heuristics + /// otherwise determine that this prefilter should not be used, then `None` + /// is returned. + fn build(&self) -> Option { + #[cfg(feature = "perf-literal")] + fn imp(builder: &RareBytesBuilder) -> Option { + if !builder.available || builder.count > 3 { + return None; + } + let (mut bytes, mut len) = ([0; 3], 0); + for b in 0..=255 { + if builder.rare_set.contains(b) { + bytes[len] = b; + len += 1; + } + } + let finder: Arc = match len { + 0 => return None, + 1 => Arc::new(RareBytesOne { + byte1: bytes[0], + offset: builder.byte_offsets.set[bytes[0] as usize], + }), + 2 => Arc::new(RareBytesTwo { + offsets: builder.byte_offsets, + byte1: bytes[0], + byte2: bytes[1], + }), + 3 => Arc::new(RareBytesThree { + offsets: builder.byte_offsets, + byte1: bytes[0], + byte2: bytes[1], + byte3: bytes[2], + }), + _ => unreachable!(), + }; + Some(Prefilter { finder, memory_usage: 0 }) + } + + #[cfg(not(feature = "perf-literal"))] + fn imp(_: &RareBytesBuilder) -> Option { + None + } + + imp(self) + } + + /// Add a byte string to this builder. + /// + /// All patterns added to an Aho-Corasick automaton should be added to this + /// builder before attempting to construct the prefilter. + fn add(&mut self, bytes: &[u8]) { + // If we've already given up, then do nothing. + if !self.available { + return; + } + // If we've already blown our budget, then don't waste time looking + // for more rare bytes. + if self.count > 3 { + self.available = false; + return; + } + // If the pattern is too long, then our offset table is bunk, so + // give up. + if bytes.len() >= 256 { + self.available = false; + return; + } + let mut rarest = match bytes.first() { + None => return, + Some(&b) => (b, freq_rank(b)), + }; + // The idea here is to look for the rarest byte in each pattern, and + // add that to our set. As a special exception, if we see a byte that + // we've already added, then we immediately stop and choose that byte, + // even if there's another rare byte in the pattern. This helps us + // apply the rare byte optimization in more cases by attempting to pick + // bytes that are in common between patterns. So for example, if we + // were searching for `Sherlock` and `lockjaw`, then this would pick + // `k` for both patterns, resulting in the use of `memchr` instead of + // `memchr2` for `k` and `j`. + let mut found = false; + for (pos, &b) in bytes.iter().enumerate() { + self.set_offset(pos, b); + if found { + continue; + } + if self.rare_set.contains(b) { + found = true; + continue; + } + let rank = freq_rank(b); + if rank < rarest.1 { + rarest = (b, rank); + } + } + if !found { + self.add_rare_byte(rarest.0); + } + } + + fn set_offset(&mut self, pos: usize, byte: u8) { + // This unwrap is OK because pos is never bigger than our max. + let offset = RareByteOffset::new(pos).unwrap(); + self.byte_offsets.set(byte, offset); + if self.ascii_case_insensitive { + self.byte_offsets.set(opposite_ascii_case(byte), offset); + } + } + + fn add_rare_byte(&mut self, byte: u8) { + self.add_one_rare_byte(byte); + if self.ascii_case_insensitive { + self.add_one_rare_byte(opposite_ascii_case(byte)); + } + } + + fn add_one_rare_byte(&mut self, byte: u8) { + if !self.rare_set.contains(byte) { + self.rare_set.add(byte); + self.count += 1; + self.rank_sum += freq_rank(byte) as u16; + } + } +} + +/// A prefilter for scanning for a single "rare" byte. +#[cfg(feature = "perf-literal")] +#[derive(Clone, Debug)] +struct RareBytesOne { + byte1: u8, + offset: RareByteOffset, +} + +#[cfg(feature = "perf-literal")] +impl PrefilterI for RareBytesOne { + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + memchr::memchr(self.byte1, &haystack[span]) + .map(|i| { + let pos = span.start + i; + cmp::max( + span.start, + pos.saturating_sub(usize::from(self.offset.max)), + ) + }) + .map_or(Candidate::None, Candidate::PossibleStartOfMatch) + } +} + +/// A prefilter for scanning for two "rare" bytes. +#[cfg(feature = "perf-literal")] +#[derive(Clone, Debug)] +struct RareBytesTwo { + offsets: RareByteOffsets, + byte1: u8, + byte2: u8, +} + +#[cfg(feature = "perf-literal")] +impl PrefilterI for RareBytesTwo { + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + memchr::memchr2(self.byte1, self.byte2, &haystack[span]) + .map(|i| { + let pos = span.start + i; + let offset = self.offsets.set[usize::from(haystack[pos])].max; + cmp::max(span.start, pos.saturating_sub(usize::from(offset))) + }) + .map_or(Candidate::None, Candidate::PossibleStartOfMatch) + } +} + +/// A prefilter for scanning for three "rare" bytes. +#[cfg(feature = "perf-literal")] +#[derive(Clone, Debug)] +struct RareBytesThree { + offsets: RareByteOffsets, + byte1: u8, + byte2: u8, + byte3: u8, +} + +#[cfg(feature = "perf-literal")] +impl PrefilterI for RareBytesThree { + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + memchr::memchr3(self.byte1, self.byte2, self.byte3, &haystack[span]) + .map(|i| { + let pos = span.start + i; + let offset = self.offsets.set[usize::from(haystack[pos])].max; + cmp::max(span.start, pos.saturating_sub(usize::from(offset))) + }) + .map_or(Candidate::None, Candidate::PossibleStartOfMatch) + } +} + +/// A builder for constructing a starting byte prefilter. +/// +/// A starting byte prefilter is a simplistic prefilter that looks for possible +/// matches by reporting all positions corresponding to a particular byte. This +/// generally only takes affect when there are at most 3 distinct possible +/// starting bytes. e.g., the patterns `foo`, `bar`, and `baz` have two +/// distinct starting bytes (`f` and `b`), and this prefilter returns all +/// occurrences of either `f` or `b`. +/// +/// In some cases, a heuristic frequency analysis may determine that it would +/// be better not to use this prefilter even when there are 3 or fewer distinct +/// starting bytes. +#[derive(Clone, Debug)] +struct StartBytesBuilder { + /// Whether this prefilter should account for ASCII case insensitivity or + /// not. + ascii_case_insensitive: bool, + /// The set of starting bytes observed. + byteset: Vec, + /// The number of bytes set to true in `byteset`. + count: usize, + /// The sum of frequency ranks for the rare bytes detected. This is + /// intended to give a heuristic notion of how rare the bytes are. + rank_sum: u16, +} + +impl StartBytesBuilder { + /// Create a new builder for constructing a start byte prefilter. + fn new() -> StartBytesBuilder { + StartBytesBuilder { + ascii_case_insensitive: false, + byteset: vec![false; 256], + count: 0, + rank_sum: 0, + } + } + + /// Enable ASCII case insensitivity. When set, byte strings added to this + /// builder will be interpreted without respect to ASCII case. + fn ascii_case_insensitive(mut self, yes: bool) -> StartBytesBuilder { + self.ascii_case_insensitive = yes; + self + } + + /// Build the starting bytes prefilter. + /// + /// If there are more than 3 distinct starting bytes, or if heuristics + /// otherwise determine that this prefilter should not be used, then `None` + /// is returned. + fn build(&self) -> Option { + #[cfg(feature = "perf-literal")] + fn imp(builder: &StartBytesBuilder) -> Option { + if builder.count > 3 { + return None; + } + let (mut bytes, mut len) = ([0; 3], 0); + for b in 0..256 { + if !builder.byteset[b] { + continue; + } + // We don't handle non-ASCII bytes for now. Getting non-ASCII + // bytes right is trickier, since we generally don't want to put + // a leading UTF-8 code unit into a prefilter that isn't ASCII, + // since they can frequently. Instead, it would be better to use a + // continuation byte, but this requires more sophisticated analysis + // of the automaton and a richer prefilter API. + if b > 0x7F { + return None; + } + bytes[len] = b as u8; + len += 1; + } + let finder: Arc = match len { + 0 => return None, + 1 => Arc::new(StartBytesOne { byte1: bytes[0] }), + 2 => Arc::new(StartBytesTwo { + byte1: bytes[0], + byte2: bytes[1], + }), + 3 => Arc::new(StartBytesThree { + byte1: bytes[0], + byte2: bytes[1], + byte3: bytes[2], + }), + _ => unreachable!(), + }; + Some(Prefilter { finder, memory_usage: 0 }) + } + + #[cfg(not(feature = "perf-literal"))] + fn imp(_: &StartBytesBuilder) -> Option { + None + } + + imp(self) + } + + /// Add a byte string to this builder. + /// + /// All patterns added to an Aho-Corasick automaton should be added to this + /// builder before attempting to construct the prefilter. + fn add(&mut self, bytes: &[u8]) { + if self.count > 3 { + return; + } + if let Some(&byte) = bytes.first() { + self.add_one_byte(byte); + if self.ascii_case_insensitive { + self.add_one_byte(opposite_ascii_case(byte)); + } + } + } + + fn add_one_byte(&mut self, byte: u8) { + if !self.byteset[byte as usize] { + self.byteset[byte as usize] = true; + self.count += 1; + self.rank_sum += freq_rank(byte) as u16; + } + } +} + +/// A prefilter for scanning for a single starting byte. +#[cfg(feature = "perf-literal")] +#[derive(Clone, Debug)] +struct StartBytesOne { + byte1: u8, +} + +#[cfg(feature = "perf-literal")] +impl PrefilterI for StartBytesOne { + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + memchr::memchr(self.byte1, &haystack[span]) + .map(|i| span.start + i) + .map_or(Candidate::None, Candidate::PossibleStartOfMatch) + } +} + +/// A prefilter for scanning for two starting bytes. +#[cfg(feature = "perf-literal")] +#[derive(Clone, Debug)] +struct StartBytesTwo { + byte1: u8, + byte2: u8, +} + +#[cfg(feature = "perf-literal")] +impl PrefilterI for StartBytesTwo { + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + memchr::memchr2(self.byte1, self.byte2, &haystack[span]) + .map(|i| span.start + i) + .map_or(Candidate::None, Candidate::PossibleStartOfMatch) + } +} + +/// A prefilter for scanning for three starting bytes. +#[cfg(feature = "perf-literal")] +#[derive(Clone, Debug)] +struct StartBytesThree { + byte1: u8, + byte2: u8, + byte3: u8, +} + +#[cfg(feature = "perf-literal")] +impl PrefilterI for StartBytesThree { + fn find_in(&self, haystack: &[u8], span: Span) -> Candidate { + memchr::memchr3(self.byte1, self.byte2, self.byte3, &haystack[span]) + .map(|i| span.start + i) + .map_or(Candidate::None, Candidate::PossibleStartOfMatch) + } +} + +/// If the given byte is an ASCII letter, then return it in the opposite case. +/// e.g., Given `b'A'`, this returns `b'a'`, and given `b'a'`, this returns +/// `b'A'`. If a non-ASCII letter is given, then the given byte is returned. +pub(crate) fn opposite_ascii_case(b: u8) -> u8 { + if b'A' <= b && b <= b'Z' { + b.to_ascii_lowercase() + } else if b'a' <= b && b <= b'z' { + b.to_ascii_uppercase() + } else { + b + } +} + +/// Return the frequency rank of the given byte. The higher the rank, the more +/// common the byte (heuristically speaking). +fn freq_rank(b: u8) -> u8 { + use crate::util::byte_frequencies::BYTE_FREQUENCIES; + BYTE_FREQUENCIES[b as usize] +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/primitives.rs b/third_party/crates/aho-corasick-1.1.5/src/util/primitives.rs new file mode 100644 index 0000000..784d397 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/primitives.rs @@ -0,0 +1,759 @@ +/*! +Lower level primitive types that are useful in a variety of circumstances. + +# Overview + +This list represents the principle types in this module and briefly describes +when you might want to use them. + +* [`PatternID`] - A type that represents the identifier of a regex pattern. +This is probably the most widely used type in this module (which is why it's +also re-exported in the crate root). +* [`StateID`] - A type the represents the identifier of a finite automaton +state. This is used for both NFAs and DFAs, with the notable exception of +the hybrid NFA/DFA. (The hybrid NFA/DFA uses a special purpose "lazy" state +identifier.) +* [`SmallIndex`] - The internal representation of both a `PatternID` and a +`StateID`. Its purpose is to serve as a type that can index memory without +being as big as a `usize` on 64-bit targets. The main idea behind this type +is that there are many things in regex engines that will, in practice, never +overflow a 32-bit integer. (For example, like the number of patterns in a regex +or the number of states in an NFA.) Thus, a `SmallIndex` can be used to index +memory without peppering `as` casts everywhere. Moreover, it forces callers +to handle errors in the case where, somehow, the value would otherwise overflow +either a 32-bit integer or a `usize` (e.g., on 16-bit targets). +*/ + +// The macro we use to define some types below adds methods that we don't +// use on some of the types. There isn't much, so we just squash the warning. +#![allow(dead_code)] + +use alloc::vec::Vec; + +use crate::util::int::{Usize, U16, U32, U64}; + +/// A type that represents a "small" index. +/// +/// The main idea of this type is to provide something that can index memory, +/// but uses less memory than `usize` on 64-bit systems. Specifically, its +/// representation is always a `u32` and has `repr(transparent)` enabled. (So +/// it is safe to transmute between a `u32` and a `SmallIndex`.) +/// +/// A small index is typically useful in cases where there is no practical way +/// that the index will overflow a 32-bit integer. A good example of this is +/// an NFA state. If you could somehow build an NFA with `2^30` states, its +/// memory usage would be exorbitant and its runtime execution would be so +/// slow as to be completely worthless. Therefore, this crate generally deems +/// it acceptable to return an error if it would otherwise build an NFA that +/// requires a slice longer than what a 32-bit integer can index. In exchange, +/// we can use 32-bit indices instead of 64-bit indices in various places. +/// +/// This type ensures this by providing a constructor that will return an error +/// if its argument cannot fit into the type. This makes it much easier to +/// handle these sorts of boundary cases that are otherwise extremely subtle. +/// +/// On all targets, this type guarantees that its value will fit in a `u32`, +/// `i32`, `usize` and an `isize`. This means that on 16-bit targets, for +/// example, this type's maximum value will never overflow an `isize`, +/// which means it will never overflow a `i16` even though its internal +/// representation is still a `u32`. +/// +/// The purpose for making the type fit into even signed integer types like +/// `isize` is to guarantee that the difference between any two small indices +/// is itself also a small index. This is useful in certain contexts, e.g., +/// for delta encoding. +/// +/// # Other types +/// +/// The following types wrap `SmallIndex` to provide a more focused use case: +/// +/// * [`PatternID`] is for representing the identifiers of patterns. +/// * [`StateID`] is for representing the identifiers of states in finite +/// automata. It is used for both NFAs and DFAs. +/// +/// # Representation +/// +/// This type is always represented internally by a `u32` and is marked as +/// `repr(transparent)`. Thus, this type always has the same representation as +/// a `u32`. It is thus safe to transmute between a `u32` and a `SmallIndex`. +/// +/// # Indexing +/// +/// For convenience, callers may use a `SmallIndex` to index slices. +/// +/// # Safety +/// +/// While a `SmallIndex` is meant to guarantee that its value fits into `usize` +/// without using as much space as a `usize` on all targets, callers must +/// not rely on this property for safety. Callers may choose to rely on this +/// property for correctness however. For example, creating a `SmallIndex` with +/// an invalid value can be done in entirely safe code. This may in turn result +/// in panics or silent logical errors. +#[derive( + Clone, Copy, Debug, Default, Eq, Hash, PartialEq, PartialOrd, Ord, +)] +#[repr(transparent)] +pub(crate) struct SmallIndex(u32); + +impl SmallIndex { + /// The maximum index value. + #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))] + pub const MAX: SmallIndex = + // FIXME: Use as_usize() once const functions in traits are stable. + SmallIndex::new_unchecked(core::i32::MAX as usize - 1); + + /// The maximum index value. + #[cfg(target_pointer_width = "16")] + pub const MAX: SmallIndex = + SmallIndex::new_unchecked(core::isize::MAX - 1); + + /// The total number of values that can be represented as a small index. + pub const LIMIT: usize = SmallIndex::MAX.as_usize() + 1; + + /// The zero index value. + pub const ZERO: SmallIndex = SmallIndex::new_unchecked(0); + + /// The number of bytes that a single small index uses in memory. + pub const SIZE: usize = core::mem::size_of::(); + + /// Create a new small index. + /// + /// If the given index exceeds [`SmallIndex::MAX`], then this returns + /// an error. + #[inline] + pub fn new(index: usize) -> Result { + SmallIndex::try_from(index) + } + + /// Create a new small index without checking whether the given value + /// exceeds [`SmallIndex::MAX`]. + /// + /// Using this routine with an invalid index value will result in + /// unspecified behavior, but *not* undefined behavior. In particular, an + /// invalid index value is likely to cause panics or possibly even silent + /// logical errors. + /// + /// Callers must never rely on a `SmallIndex` to be within a certain range + /// for memory safety. + #[inline] + pub const fn new_unchecked(index: usize) -> SmallIndex { + // FIXME: Use as_u32() once const functions in traits are stable. + SmallIndex::from_u32_unchecked(index as u32) + } + + /// Create a new small index from a `u32` without checking whether the + /// given value exceeds [`SmallIndex::MAX`]. + /// + /// Using this routine with an invalid index value will result in + /// unspecified behavior, but *not* undefined behavior. In particular, an + /// invalid index value is likely to cause panics or possibly even silent + /// logical errors. + /// + /// Callers must never rely on a `SmallIndex` to be within a certain range + /// for memory safety. + #[inline] + pub const fn from_u32_unchecked(index: u32) -> SmallIndex { + SmallIndex(index) + } + + /// Like [`SmallIndex::new`], but panics if the given index is not valid. + #[inline] + pub fn must(index: usize) -> SmallIndex { + SmallIndex::new(index).expect("invalid small index") + } + + /// Return this small index as a `usize`. This is guaranteed to never + /// overflow `usize`. + #[inline] + pub const fn as_usize(&self) -> usize { + // FIXME: Use as_usize() once const functions in traits are stable. + self.0 as usize + } + + /// Return this small index as a `u64`. This is guaranteed to never + /// overflow. + #[inline] + pub const fn as_u64(&self) -> u64 { + // FIXME: Use u64::from() once const functions in traits are stable. + self.0 as u64 + } + + /// Return the internal `u32` of this small index. This is guaranteed to + /// never overflow `u32`. + #[inline] + pub const fn as_u32(&self) -> u32 { + self.0 + } + + /// Return the internal `u32` of this small index represented as an `i32`. + /// This is guaranteed to never overflow an `i32`. + #[inline] + pub const fn as_i32(&self) -> i32 { + // This is OK because we guarantee that our max value is <= i32::MAX. + self.0 as i32 + } + + /// Returns one more than this small index as a usize. + /// + /// Since a small index has constraints on its maximum value, adding `1` to + /// it will always fit in a `usize`, `isize`, `u32` and a `i32`. + #[inline] + pub fn one_more(&self) -> usize { + self.as_usize() + 1 + } + + /// Decode this small index from the bytes given using the native endian + /// byte order for the current target. + /// + /// If the decoded integer is not representable as a small index for the + /// current target, then this returns an error. + #[inline] + pub fn from_ne_bytes( + bytes: [u8; 4], + ) -> Result { + let id = u32::from_ne_bytes(bytes); + if id > SmallIndex::MAX.as_u32() { + return Err(SmallIndexError { attempted: u64::from(id) }); + } + Ok(SmallIndex::new_unchecked(id.as_usize())) + } + + /// Decode this small index from the bytes given using the native endian + /// byte order for the current target. + /// + /// This is analogous to [`SmallIndex::new_unchecked`] in that is does not + /// check whether the decoded integer is representable as a small index. + #[inline] + pub fn from_ne_bytes_unchecked(bytes: [u8; 4]) -> SmallIndex { + SmallIndex::new_unchecked(u32::from_ne_bytes(bytes).as_usize()) + } + + /// Return the underlying small index integer as raw bytes in native endian + /// format. + #[inline] + pub fn to_ne_bytes(&self) -> [u8; 4] { + self.0.to_ne_bytes() + } +} + +impl core::ops::Index for [T] { + type Output = T; + + #[inline] + fn index(&self, index: SmallIndex) -> &T { + &self[index.as_usize()] + } +} + +impl core::ops::IndexMut for [T] { + #[inline] + fn index_mut(&mut self, index: SmallIndex) -> &mut T { + &mut self[index.as_usize()] + } +} + +impl core::ops::Index for Vec { + type Output = T; + + #[inline] + fn index(&self, index: SmallIndex) -> &T { + &self[index.as_usize()] + } +} + +impl core::ops::IndexMut for Vec { + #[inline] + fn index_mut(&mut self, index: SmallIndex) -> &mut T { + &mut self[index.as_usize()] + } +} + +impl From for SmallIndex { + fn from(sid: StateID) -> SmallIndex { + sid.0 + } +} + +impl From for SmallIndex { + fn from(pid: PatternID) -> SmallIndex { + pid.0 + } +} + +impl From for SmallIndex { + fn from(index: u8) -> SmallIndex { + SmallIndex::new_unchecked(usize::from(index)) + } +} + +impl TryFrom for SmallIndex { + type Error = SmallIndexError; + + fn try_from(index: u16) -> Result { + if u32::from(index) > SmallIndex::MAX.as_u32() { + return Err(SmallIndexError { attempted: u64::from(index) }); + } + Ok(SmallIndex::new_unchecked(index.as_usize())) + } +} + +impl TryFrom for SmallIndex { + type Error = SmallIndexError; + + fn try_from(index: u32) -> Result { + if index > SmallIndex::MAX.as_u32() { + return Err(SmallIndexError { attempted: u64::from(index) }); + } + Ok(SmallIndex::new_unchecked(index.as_usize())) + } +} + +impl TryFrom for SmallIndex { + type Error = SmallIndexError; + + fn try_from(index: u64) -> Result { + if index > SmallIndex::MAX.as_u64() { + return Err(SmallIndexError { attempted: index }); + } + Ok(SmallIndex::new_unchecked(index.as_usize())) + } +} + +impl TryFrom for SmallIndex { + type Error = SmallIndexError; + + fn try_from(index: usize) -> Result { + if index > SmallIndex::MAX.as_usize() { + return Err(SmallIndexError { attempted: index.as_u64() }); + } + Ok(SmallIndex::new_unchecked(index)) + } +} + +/// This error occurs when a small index could not be constructed. +/// +/// This occurs when given an integer exceeding the maximum small index value. +/// +/// When the `std` feature is enabled, this implements the `Error` trait. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SmallIndexError { + attempted: u64, +} + +impl SmallIndexError { + /// Returns the value that could not be converted to a small index. + pub fn attempted(&self) -> u64 { + self.attempted + } +} + +#[cfg(feature = "std")] +impl std::error::Error for SmallIndexError {} + +impl core::fmt::Display for SmallIndexError { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "failed to create small index from {:?}, which exceeds {:?}", + self.attempted(), + SmallIndex::MAX, + ) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct SmallIndexIter { + rng: core::ops::Range, +} + +impl Iterator for SmallIndexIter { + type Item = SmallIndex; + + fn next(&mut self) -> Option { + if self.rng.start >= self.rng.end { + return None; + } + let next_id = self.rng.start + 1; + let id = core::mem::replace(&mut self.rng.start, next_id); + // new_unchecked is OK since we asserted that the number of + // elements in this iterator will fit in an ID at construction. + Some(SmallIndex::new_unchecked(id)) + } +} + +macro_rules! index_type_impls { + ($name:ident, $err:ident, $iter:ident, $withiter:ident) => { + impl $name { + /// The maximum value. + pub const MAX: $name = $name(SmallIndex::MAX); + + /// The total number of values that can be represented. + pub const LIMIT: usize = SmallIndex::LIMIT; + + /// The zero value. + pub const ZERO: $name = $name(SmallIndex::ZERO); + + /// The number of bytes that a single value uses in memory. + pub const SIZE: usize = SmallIndex::SIZE; + + /// Create a new value that is represented by a "small index." + /// + /// If the given index exceeds the maximum allowed value, then this + /// returns an error. + #[inline] + pub fn new(value: usize) -> Result<$name, $err> { + SmallIndex::new(value).map($name).map_err($err) + } + + /// Create a new value without checking whether the given argument + /// exceeds the maximum. + /// + /// Using this routine with an invalid value will result in + /// unspecified behavior, but *not* undefined behavior. In + /// particular, an invalid ID value is likely to cause panics or + /// possibly even silent logical errors. + /// + /// Callers must never rely on this type to be within a certain + /// range for memory safety. + #[inline] + pub const fn new_unchecked(value: usize) -> $name { + $name(SmallIndex::new_unchecked(value)) + } + + /// Create a new value from a `u32` without checking whether the + /// given value exceeds the maximum. + /// + /// Using this routine with an invalid value will result in + /// unspecified behavior, but *not* undefined behavior. In + /// particular, an invalid ID value is likely to cause panics or + /// possibly even silent logical errors. + /// + /// Callers must never rely on this type to be within a certain + /// range for memory safety. + #[inline] + pub const fn from_u32_unchecked(index: u32) -> $name { + $name(SmallIndex::from_u32_unchecked(index)) + } + + /// Like `new`, but panics if the given value is not valid. + #[inline] + pub fn must(value: usize) -> $name { + $name::new(value).expect(concat!( + "invalid ", + stringify!($name), + " value" + )) + } + + /// Return the internal value as a `usize`. This is guaranteed to + /// never overflow `usize`. + #[inline] + pub const fn as_usize(&self) -> usize { + self.0.as_usize() + } + + /// Return the internal value as a `u64`. This is guaranteed to + /// never overflow. + #[inline] + pub const fn as_u64(&self) -> u64 { + self.0.as_u64() + } + + /// Return the internal value as a `u32`. This is guaranteed to + /// never overflow `u32`. + #[inline] + pub const fn as_u32(&self) -> u32 { + self.0.as_u32() + } + + /// Return the internal value as a `i32`. This is guaranteed to + /// never overflow an `i32`. + #[inline] + pub const fn as_i32(&self) -> i32 { + self.0.as_i32() + } + + /// Returns one more than this value as a usize. + /// + /// Since values represented by a "small index" have constraints + /// on their maximum value, adding `1` to it will always fit in a + /// `usize`, `u32` and a `i32`. + #[inline] + pub fn one_more(&self) -> usize { + self.0.one_more() + } + + /// Decode this value from the bytes given using the native endian + /// byte order for the current target. + /// + /// If the decoded integer is not representable as a small index + /// for the current target, then this returns an error. + #[inline] + pub fn from_ne_bytes(bytes: [u8; 4]) -> Result<$name, $err> { + SmallIndex::from_ne_bytes(bytes).map($name).map_err($err) + } + + /// Decode this value from the bytes given using the native endian + /// byte order for the current target. + /// + /// This is analogous to `new_unchecked` in that is does not check + /// whether the decoded integer is representable as a small index. + #[inline] + pub fn from_ne_bytes_unchecked(bytes: [u8; 4]) -> $name { + $name(SmallIndex::from_ne_bytes_unchecked(bytes)) + } + + /// Return the underlying integer as raw bytes in native endian + /// format. + #[inline] + pub fn to_ne_bytes(&self) -> [u8; 4] { + self.0.to_ne_bytes() + } + + /// Returns an iterator over all values from 0 up to and not + /// including the given length. + /// + /// If the given length exceeds this type's limit, then this + /// panics. + pub(crate) fn iter(len: usize) -> $iter { + $iter::new(len) + } + } + + // We write our own Debug impl so that we get things like PatternID(5) + // instead of PatternID(SmallIndex(5)). + impl core::fmt::Debug for $name { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_tuple(stringify!($name)).field(&self.as_u32()).finish() + } + } + + impl core::ops::Index<$name> for [T] { + type Output = T; + + #[inline] + fn index(&self, index: $name) -> &T { + &self[index.as_usize()] + } + } + + impl core::ops::IndexMut<$name> for [T] { + #[inline] + fn index_mut(&mut self, index: $name) -> &mut T { + &mut self[index.as_usize()] + } + } + + impl core::ops::Index<$name> for Vec { + type Output = T; + + #[inline] + fn index(&self, index: $name) -> &T { + &self[index.as_usize()] + } + } + + impl core::ops::IndexMut<$name> for Vec { + #[inline] + fn index_mut(&mut self, index: $name) -> &mut T { + &mut self[index.as_usize()] + } + } + + impl From for $name { + fn from(index: SmallIndex) -> $name { + $name(index) + } + } + + impl From for $name { + fn from(value: u8) -> $name { + $name(SmallIndex::from(value)) + } + } + + impl TryFrom for $name { + type Error = $err; + + fn try_from(value: u16) -> Result<$name, $err> { + SmallIndex::try_from(value).map($name).map_err($err) + } + } + + impl TryFrom for $name { + type Error = $err; + + fn try_from(value: u32) -> Result<$name, $err> { + SmallIndex::try_from(value).map($name).map_err($err) + } + } + + impl TryFrom for $name { + type Error = $err; + + fn try_from(value: u64) -> Result<$name, $err> { + SmallIndex::try_from(value).map($name).map_err($err) + } + } + + impl TryFrom for $name { + type Error = $err; + + fn try_from(value: usize) -> Result<$name, $err> { + SmallIndex::try_from(value).map($name).map_err($err) + } + } + + /// This error occurs when an ID could not be constructed. + /// + /// This occurs when given an integer exceeding the maximum allowed + /// value. + /// + /// When the `std` feature is enabled, this implements the `Error` + /// trait. + #[derive(Clone, Debug, Eq, PartialEq)] + pub struct $err(SmallIndexError); + + impl $err { + /// Returns the value that could not be converted to an ID. + pub fn attempted(&self) -> u64 { + self.0.attempted() + } + } + + #[cfg(feature = "std")] + impl std::error::Error for $err {} + + impl core::fmt::Display for $err { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "failed to create {} from {:?}, which exceeds {:?}", + stringify!($name), + self.attempted(), + $name::MAX, + ) + } + } + + #[derive(Clone, Debug)] + pub(crate) struct $iter(SmallIndexIter); + + impl $iter { + fn new(len: usize) -> $iter { + assert!( + len <= $name::LIMIT, + "cannot create iterator for {} when number of \ + elements exceed {:?}", + stringify!($name), + $name::LIMIT, + ); + $iter(SmallIndexIter { rng: 0..len }) + } + } + + impl Iterator for $iter { + type Item = $name; + + fn next(&mut self) -> Option<$name> { + self.0.next().map($name) + } + } + + /// An iterator adapter that is like std::iter::Enumerate, but attaches + /// small index values instead. It requires `ExactSizeIterator`. At + /// construction, it ensures that the index of each element in the + /// iterator is representable in the corresponding small index type. + #[derive(Clone, Debug)] + pub(crate) struct $withiter { + it: I, + ids: $iter, + } + + impl $withiter { + fn new(it: I) -> $withiter { + let ids = $name::iter(it.len()); + $withiter { it, ids } + } + } + + impl Iterator for $withiter { + type Item = ($name, I::Item); + + fn next(&mut self) -> Option<($name, I::Item)> { + let item = self.it.next()?; + // Number of elements in this iterator must match, according + // to contract of ExactSizeIterator. + let id = self.ids.next().unwrap(); + Some((id, item)) + } + } + }; +} + +/// The identifier of a pattern in an Aho-Corasick automaton. +/// +/// It is represented by a `u32` even on 64-bit systems in order to conserve +/// space. Namely, on all targets, this type guarantees that its value will +/// fit in a `u32`, `i32`, `usize` and an `isize`. This means that on 16-bit +/// targets, for example, this type's maximum value will never overflow an +/// `isize`, which means it will never overflow a `i16` even though its +/// internal representation is still a `u32`. +/// +/// # Safety +/// +/// While a `PatternID` is meant to guarantee that its value fits into `usize` +/// without using as much space as a `usize` on all targets, callers must +/// not rely on this property for safety. Callers may choose to rely on this +/// property for correctness however. For example, creating a `StateID` with an +/// invalid value can be done in entirely safe code. This may in turn result in +/// panics or silent logical errors. +#[derive(Clone, Copy, Default, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct PatternID(SmallIndex); + +/// The identifier of a finite automaton state. +/// +/// It is represented by a `u32` even on 64-bit systems in order to conserve +/// space. Namely, on all targets, this type guarantees that its value will +/// fit in a `u32`, `i32`, `usize` and an `isize`. This means that on 16-bit +/// targets, for example, this type's maximum value will never overflow an +/// `isize`, which means it will never overflow a `i16` even though its +/// internal representation is still a `u32`. +/// +/// # Safety +/// +/// While a `StateID` is meant to guarantee that its value fits into `usize` +/// without using as much space as a `usize` on all targets, callers must +/// not rely on this property for safety. Callers may choose to rely on this +/// property for correctness however. For example, creating a `StateID` with an +/// invalid value can be done in entirely safe code. This may in turn result in +/// panics or silent logical errors. +#[derive(Clone, Copy, Default, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct StateID(SmallIndex); + +index_type_impls!(PatternID, PatternIDError, PatternIDIter, WithPatternIDIter); +index_type_impls!(StateID, StateIDError, StateIDIter, WithStateIDIter); + +/// A utility trait that defines a couple of adapters for making it convenient +/// to access indices as "small index" types. We require ExactSizeIterator so +/// that iterator construction can do a single check to make sure the index of +/// each element is representable by its small index type. +pub(crate) trait IteratorIndexExt: Iterator { + fn with_pattern_ids(self) -> WithPatternIDIter + where + Self: Sized + ExactSizeIterator, + { + WithPatternIDIter::new(self) + } + + fn with_state_ids(self) -> WithStateIDIter + where + Self: Sized + ExactSizeIterator, + { + WithStateIDIter::new(self) + } +} + +impl IteratorIndexExt for I {} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/remapper.rs b/third_party/crates/aho-corasick-1.1.5/src/util/remapper.rs new file mode 100644 index 0000000..7c47a08 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/remapper.rs @@ -0,0 +1,214 @@ +use alloc::vec::Vec; + +use crate::{nfa::noncontiguous, util::primitives::StateID}; + +/// Remappable is a tightly coupled abstraction that facilitates remapping +/// state identifiers in DFAs. +/// +/// The main idea behind remapping state IDs is that DFAs often need to check +/// if a certain state is a "special" state of some kind (like a match state) +/// during a search. Since this is extremely perf critical code, we want this +/// check to be as fast as possible. Partitioning state IDs into, for example, +/// into "non-match" and "match" states means one can tell if a state is a +/// match state via a simple comparison of the state ID. +/// +/// The issue is that during the DFA construction process, it's not +/// particularly easy to partition the states. Instead, the simplest thing is +/// to often just do a pass over all of the states and shuffle them into their +/// desired partitionings. To do that, we need a mechanism for swapping states. +/// Hence, this abstraction. +/// +/// Normally, for such little code, I would just duplicate it. But this is a +/// key optimization and the implementation is a bit subtle. So the abstraction +/// is basically a ham-fisted attempt at DRY. The only place we use this is in +/// the dense and one-pass DFAs. +/// +/// See also src/dfa/special.rs for a more detailed explanation of how dense +/// DFAs are partitioned. +pub(crate) trait Remappable: core::fmt::Debug { + /// Return the total number of states. + fn state_len(&self) -> usize; + + /// Swap the states pointed to by the given IDs. The underlying finite + /// state machine should be mutated such that all of the transitions in + /// `id1` are now in the memory region where the transitions for `id2` + /// were, and all of the transitions in `id2` are now in the memory region + /// where the transitions for `id1` were. + /// + /// Essentially, this "moves" `id1` to `id2` and `id2` to `id1`. + /// + /// It is expected that, after calling this, the underlying state machine + /// will be left in an inconsistent state, since any other transitions + /// pointing to, e.g., `id1` need to be updated to point to `id2`, since + /// that's where `id1` moved to. + /// + /// In order to "fix" the underlying inconsistent state, a `Remapper` + /// should be used to guarantee that `remap` is called at the appropriate + /// time. + fn swap_states(&mut self, id1: StateID, id2: StateID); + + /// This must remap every single state ID in the underlying value according + /// to the function given. For example, in a DFA, this should remap every + /// transition and every starting state ID. + fn remap(&mut self, map: impl Fn(StateID) -> StateID); +} + +/// Remapper is an abstraction the manages the remapping of state IDs in a +/// finite state machine. This is useful when one wants to shuffle states into +/// different positions in the machine. +/// +/// One of the key complexities this manages is the ability to correctly move +/// one state multiple times. +/// +/// Once shuffling is complete, `remap` must be called, which will rewrite +/// all pertinent transitions to updated state IDs. Neglecting to call `remap` +/// will almost certainly result in a corrupt machine. +#[derive(Debug)] +pub(crate) struct Remapper { + /// A map from the index of a state to its pre-multiplied identifier. + /// + /// When a state is swapped with another, then their corresponding + /// locations in this map are also swapped. Thus, its new position will + /// still point to its old pre-multiplied StateID. + /// + /// While there is a bit more to it, this then allows us to rewrite the + /// state IDs in a DFA's transition table in a single pass. This is done + /// by iterating over every ID in this map, then iterating over each + /// transition for the state at that ID and re-mapping the transition from + /// `old_id` to `map[dfa.to_index(old_id)]`. That is, we find the position + /// in this map where `old_id` *started*, and set it to where it ended up + /// after all swaps have been completed. + map: Vec, + /// A way to map indices to state IDs (and back). + idx: IndexMapper, +} + +impl Remapper { + /// Create a new remapper from the given remappable implementation. The + /// remapper can then be used to swap states. The remappable value given + /// here must the same one given to `swap` and `remap`. + /// + /// The given stride should be the stride of the transition table expressed + /// as a power of 2. This stride is used to map between state IDs and state + /// indices. If state IDs and state indices are equivalent, then provide + /// a `stride2` of `0`, which acts as an identity. + pub(crate) fn new(r: &impl Remappable, stride2: usize) -> Remapper { + let idx = IndexMapper { stride2 }; + let map = (0..r.state_len()).map(|i| idx.to_state_id(i)).collect(); + Remapper { map, idx } + } + + /// Swap two states. Once this is called, callers must follow through to + /// call `remap`, or else it's possible for the underlying remappable + /// value to be in a corrupt state. + pub(crate) fn swap( + &mut self, + r: &mut impl Remappable, + id1: StateID, + id2: StateID, + ) { + if id1 == id2 { + return; + } + r.swap_states(id1, id2); + self.map.swap(self.idx.to_index(id1), self.idx.to_index(id2)); + } + + /// Complete the remapping process by rewriting all state IDs in the + /// remappable value according to the swaps performed. + pub(crate) fn remap(mut self, r: &mut impl Remappable) { + // Update the map to account for states that have been swapped + // multiple times. For example, if (A, C) and (C, G) are swapped, then + // transitions previously pointing to A should now point to G. But if + // we don't update our map, they will erroneously be set to C. All we + // do is follow the swaps in our map until we see our original state + // ID. + // + // The intuition here is to think about how changes are made to the + // map: only through pairwise swaps. That means that starting at any + // given state, it is always possible to find the loop back to that + // state by following the swaps represented in the map (which might be + // 0 swaps). + // + // We are also careful to clone the map before starting in order to + // freeze it. We use the frozen map to find our loops, since we need to + // update our map as well. Without freezing it, our updates could break + // the loops referenced above and produce incorrect results. + let oldmap = self.map.clone(); + for i in 0..r.state_len() { + let cur_id = self.idx.to_state_id(i); + let mut new_id = oldmap[i]; + if cur_id == new_id { + continue; + } + loop { + let id = oldmap[self.idx.to_index(new_id)]; + if cur_id == id { + self.map[i] = new_id; + break; + } + new_id = id; + } + } + r.remap(|sid| self.map[self.idx.to_index(sid)]); + } +} + +/// A simple type for mapping between state indices and state IDs. +/// +/// The reason why this exists is because state IDs are "premultiplied" in a +/// DFA. That is, in order to get to the transitions for a particular state, +/// one need only use the state ID as-is, instead of having to multiply it by +/// transition table's stride. +/// +/// The downside of this is that it's inconvenient to map between state IDs +/// using a dense map, e.g., Vec. That's because state IDs look like +/// `0`, `stride`, `2*stride`, `3*stride`, etc., instead of `0`, `1`, `2`, `3`, +/// etc. +/// +/// Since our state IDs are premultiplied, we can convert back-and-forth +/// between IDs and indices by simply unmultiplying the IDs and multiplying the +/// indices. +/// +/// Note that for a sparse NFA, state IDs and indices are equivalent. In this +/// case, we set the stride of the index mapped to be `0`, which acts as an +/// identity. +#[derive(Debug)] +struct IndexMapper { + /// The power of 2 corresponding to the stride of the corresponding + /// transition table. 'id >> stride2' de-multiplies an ID while 'index << + /// stride2' pre-multiplies an index to an ID. + stride2: usize, +} + +impl IndexMapper { + /// Convert a state ID to a state index. + fn to_index(&self, id: StateID) -> usize { + id.as_usize() >> self.stride2 + } + + /// Convert a state index to a state ID. + fn to_state_id(&self, index: usize) -> StateID { + // CORRECTNESS: If the given index is not valid, then it is not + // required for this to panic or return a valid state ID. We'll "just" + // wind up with panics or silent logic errors at some other point. But + // this is OK because if Remappable::state_len is correct and so is + // 'to_index', then all inputs to 'to_state_id' should be valid indices + // and thus transform into valid state IDs. + StateID::new_unchecked(index << self.stride2) + } +} + +impl Remappable for noncontiguous::NFA { + fn state_len(&self) -> usize { + noncontiguous::NFA::states(self).len() + } + + fn swap_states(&mut self, id1: StateID, id2: StateID) { + noncontiguous::NFA::swap_states(self, id1, id2) + } + + fn remap(&mut self, map: impl Fn(StateID) -> StateID) { + noncontiguous::NFA::remap(self, map) + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/search.rs b/third_party/crates/aho-corasick-1.1.5/src/util/search.rs new file mode 100644 index 0000000..6eb24f6 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/search.rs @@ -0,0 +1,1158 @@ +use core::ops::{Range, RangeBounds}; + +use crate::util::primitives::PatternID; + +/// The configuration and the haystack to use for an Aho-Corasick search. +/// +/// When executing a search, there are a few parameters one might want to +/// configure: +/// +/// * The haystack to search, provided to the [`Input::new`] constructor. This +/// is the only required parameter. +/// * The span _within_ the haystack to limit a search to. (The default +/// is the entire haystack.) This is configured via [`Input::span`] or +/// [`Input::range`]. +/// * Whether to run an unanchored (matches can occur anywhere after the +/// start of the search) or anchored (matches can only occur beginning at +/// the start of the search) search. Unanchored search is the default. This is +/// configured via [`Input::anchored`]. +/// * Whether to quit the search as soon as a match has been found, regardless +/// of the [`MatchKind`] that the searcher was built with. This is configured +/// via [`Input::earliest`]. +/// +/// For most cases, the defaults for all optional parameters are appropriate. +/// The utility of this type is that it keeps the default or common case simple +/// while permitting tweaking parameters in more niche use cases while reusing +/// the same search APIs. +/// +/// # Valid bounds and search termination +/// +/// An `Input` permits setting the bounds of a search via either +/// [`Input::span`] or [`Input::range`]. The bounds set must be valid, or +/// else a panic will occur. Bounds are valid if and only if: +/// +/// * The bounds represent a valid range into the input's haystack. +/// * **or** the end bound is a valid ending bound for the haystack *and* +/// the start bound is exactly one greater than the end bound. +/// +/// In the latter case, [`Input::is_done`] will return true and indicates any +/// search receiving such an input should immediately return with no match. +/// +/// Other than representing "search is complete," the `Input::span` and +/// `Input::range` APIs are never necessary. Instead, callers can slice the +/// haystack instead, e.g., with `&haystack[start..end]`. With that said, they +/// can be more convenient than slicing because the match positions reported +/// when using `Input::span` or `Input::range` are in terms of the original +/// haystack. If you instead use `&haystack[start..end]`, then you'll need to +/// add `start` to any match position returned in order for it to be a correct +/// index into `haystack`. +/// +/// # Example: `&str` and `&[u8]` automatically convert to an `Input` +/// +/// There is a `From<&T> for Input` implementation for all `T: AsRef<[u8]>`. +/// Additionally, the [`AhoCorasick`](crate::AhoCorasick) search APIs accept +/// a `Into`. These two things combined together mean you can provide +/// things like `&str` and `&[u8]` to search APIs when the defaults are +/// suitable, but also an `Input` when they're not. For example: +/// +/// ``` +/// use aho_corasick::{AhoCorasick, Anchored, Input, Match, StartKind}; +/// +/// // Build a searcher that supports both unanchored and anchored modes. +/// let ac = AhoCorasick::builder() +/// .start_kind(StartKind::Both) +/// .build(&["abcd", "b"]) +/// .unwrap(); +/// let haystack = "abcd"; +/// +/// // A search using default parameters is unanchored. With standard +/// // semantics, this finds `b` first. +/// assert_eq!( +/// Some(Match::must(1, 1..2)), +/// ac.find(haystack), +/// ); +/// // Using the same 'find' routine, we can provide an 'Input' explicitly +/// // that is configured to do an anchored search. Since 'b' doesn't start +/// // at the beginning of the search, it is not reported as a match. +/// assert_eq!( +/// Some(Match::must(0, 0..4)), +/// ac.find(Input::new(haystack).anchored(Anchored::Yes)), +/// ); +/// ``` +#[derive(Clone)] +pub struct Input<'h> { + haystack: &'h [u8], + span: Span, + anchored: Anchored, + earliest: bool, +} + +impl<'h> Input<'h> { + /// Create a new search configuration for the given haystack. + #[inline] + pub fn new>(haystack: &'h H) -> Input<'h> { + Input { + haystack: haystack.as_ref(), + span: Span { start: 0, end: haystack.as_ref().len() }, + anchored: Anchored::No, + earliest: false, + } + } + + /// Set the span for this search. + /// + /// This routine is generic over how a span is provided. While + /// a [`Span`] may be given directly, one may also provide a + /// `std::ops::Range`. To provide anything supported by range + /// syntax, use the [`Input::range`] method. + /// + /// The default span is the entire haystack. + /// + /// Note that [`Input::range`] overrides this method and vice versa. + /// + /// # Panics + /// + /// This panics if the given span does not correspond to valid bounds in + /// the haystack or the termination of a search. + /// + /// # Example + /// + /// This example shows how the span of the search can impact whether a + /// match is reported or not. + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Input, MatchKind}; + /// + /// let patterns = &["b", "abcd", "abc"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// let input = Input::new(haystack).span(0..3); + /// let mat = ac.try_find(input)?.expect("should have a match"); + /// // Without the span stopping the search early, 'abcd' would be reported + /// // because it is the correct leftmost-first match. + /// assert_eq!("abc", &haystack[mat.span()]); + /// + /// # Ok::<(), Box>(()) + /// ``` + #[inline] + pub fn span>(mut self, span: S) -> Input<'h> { + self.set_span(span); + self + } + + /// Like `Input::span`, but accepts any range instead. + /// + /// The default range is the entire haystack. + /// + /// Note that [`Input::span`] overrides this method and vice versa. + /// + /// # Panics + /// + /// This routine will panic if the given range could not be converted + /// to a valid [`Range`]. For example, this would panic when given + /// `0..=usize::MAX` since it cannot be represented using a half-open + /// interval in terms of `usize`. + /// + /// This routine also panics if the given range does not correspond to + /// valid bounds in the haystack or the termination of a search. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// + /// let input = Input::new("foobar").range(2..=4); + /// assert_eq!(2..5, input.get_range()); + /// ``` + #[inline] + pub fn range>(mut self, range: R) -> Input<'h> { + self.set_range(range); + self + } + + /// Sets the anchor mode of a search. + /// + /// When a search is anchored (via [`Anchored::Yes`]), a match must begin + /// at the start of a search. When a search is not anchored (that's + /// [`Anchored::No`]), searchers will look for a match anywhere in the + /// haystack. + /// + /// By default, the anchored mode is [`Anchored::No`]. + /// + /// # Support for anchored searches + /// + /// Anchored or unanchored searches might not always be available, + /// depending on the type of searcher used and its configuration: + /// + /// * [`noncontiguous::NFA`](crate::nfa::noncontiguous::NFA) always + /// supports both unanchored and anchored searches. + /// * [`contiguous::NFA`](crate::nfa::contiguous::NFA) always supports both + /// unanchored and anchored searches. + /// * [`dfa::DFA`](crate::dfa::DFA) supports only unanchored + /// searches by default. + /// [`dfa::Builder::start_kind`](crate::dfa::Builder::start_kind) can + /// be used to change the default to supporting both kinds of searches + /// or even just anchored searches. + /// * [`AhoCorasick`](crate::AhoCorasick) inherits the same setup as a + /// `DFA`. Namely, it only supports unanchored searches by default, but + /// [`AhoCorasickBuilder::start_kind`](crate::AhoCorasickBuilder::start_kind) + /// can change this. + /// + /// If you try to execute a search using a `try_` ("fallible") method with + /// an unsupported anchor mode, then an error will be returned. For calls + /// to infallible search methods, a panic will result. + /// + /// # Example + /// + /// This demonstrates the differences between an anchored search and + /// an unanchored search. Notice that we build our `AhoCorasick` searcher + /// with [`StartKind::Both`] so that it supports both unanchored and + /// anchored searches simultaneously. + /// + /// ``` + /// use aho_corasick::{ + /// AhoCorasick, Anchored, Input, MatchKind, StartKind, + /// }; + /// + /// let patterns = &["bcd"]; + /// let haystack = "abcd"; + /// + /// let ac = AhoCorasick::builder() + /// .start_kind(StartKind::Both) + /// .build(patterns) + /// .unwrap(); + /// + /// // Note that 'Anchored::No' is the default, so it doesn't need to + /// // be explicitly specified here. + /// let input = Input::new(haystack); + /// let mat = ac.try_find(input)?.expect("should have a match"); + /// assert_eq!("bcd", &haystack[mat.span()]); + /// + /// // While 'bcd' occurs in the haystack, it does not begin where our + /// // search begins, so no match is found. + /// let input = Input::new(haystack).anchored(Anchored::Yes); + /// assert_eq!(None, ac.try_find(input)?); + /// + /// // However, if we start our search where 'bcd' starts, then we will + /// // find a match. + /// let input = Input::new(haystack).range(1..).anchored(Anchored::Yes); + /// let mat = ac.try_find(input)?.expect("should have a match"); + /// assert_eq!("bcd", &haystack[mat.span()]); + /// + /// # Ok::<(), Box>(()) + /// ``` + #[inline] + pub fn anchored(mut self, mode: Anchored) -> Input<'h> { + self.set_anchored(mode); + self + } + + /// Whether to execute an "earliest" search or not. + /// + /// When running a non-overlapping search, an "earliest" search will + /// return the match location as early as possible. For example, given + /// the patterns `abc` and `b`, and a haystack of `abc`, a normal + /// leftmost-first search will return `abc` as a match. But an "earliest" + /// search will return as soon as it is known that a match occurs, which + /// happens once `b` is seen. + /// + /// Note that when using [`MatchKind::Standard`], the "earliest" option + /// has no effect since standard semantics are already "earliest." Note + /// also that this has no effect in overlapping searches, since overlapping + /// searches also use standard semantics and report all possible matches. + /// + /// This is disabled by default. + /// + /// # Example + /// + /// This example shows the difference between "earliest" searching and + /// normal leftmost searching. + /// + /// ``` + /// use aho_corasick::{AhoCorasick, Anchored, Input, MatchKind, StartKind}; + /// + /// let patterns = &["abc", "b"]; + /// let haystack = "abc"; + /// + /// let ac = AhoCorasick::builder() + /// .match_kind(MatchKind::LeftmostFirst) + /// .build(patterns) + /// .unwrap(); + /// + /// // The normal leftmost-first match. + /// let input = Input::new(haystack); + /// let mat = ac.try_find(input)?.expect("should have a match"); + /// assert_eq!("abc", &haystack[mat.span()]); + /// + /// // The "earliest" possible match, even if it isn't leftmost-first. + /// let input = Input::new(haystack).earliest(true); + /// let mat = ac.try_find(input)?.expect("should have a match"); + /// assert_eq!("b", &haystack[mat.span()]); + /// + /// # Ok::<(), Box>(()) + /// ``` + #[inline] + pub fn earliest(mut self, yes: bool) -> Input<'h> { + self.set_earliest(yes); + self + } + + /// Set the span for this search configuration. + /// + /// This is like the [`Input::span`] method, except this mutates the + /// span in place. + /// + /// This routine is generic over how a span is provided. While + /// a [`Span`] may be given directly, one may also provide a + /// `std::ops::Range`. + /// + /// # Panics + /// + /// This panics if the given span does not correspond to valid bounds in + /// the haystack or the termination of a search. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// input.set_span(2..4); + /// assert_eq!(2..4, input.get_range()); + /// ``` + #[inline] + pub fn set_span>(&mut self, span: S) { + let span = span.into(); + assert!( + span.end <= self.haystack.len() + && span.start <= span.end.wrapping_add(1), + "invalid span {:?} for haystack of length {}", + span, + self.haystack.len(), + ); + self.span = span; + } + + /// Set the span for this search configuration given any range. + /// + /// This is like the [`Input::range`] method, except this mutates the + /// span in place. + /// + /// # Panics + /// + /// This routine will panic if the given range could not be converted + /// to a valid [`Range`]. For example, this would panic when given + /// `0..=usize::MAX` since it cannot be represented using a half-open + /// interval in terms of `usize`. + /// + /// This routine also panics if the given range does not correspond to + /// valid bounds in the haystack or the termination of a search. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// input.set_range(2..=4); + /// assert_eq!(2..5, input.get_range()); + /// ``` + #[inline] + pub fn set_range>(&mut self, range: R) { + use core::ops::Bound; + + // It's a little weird to convert ranges into spans, and then spans + // back into ranges when we actually slice the haystack. Because + // of that process, we always represent everything as a half-open + // internal. Therefore, handling things like m..=n is a little awkward. + let start = match range.start_bound() { + Bound::Included(&i) => i, + // Can this case ever happen? Range syntax doesn't support it... + Bound::Excluded(&i) => i.checked_add(1).unwrap(), + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(&i) => i.checked_add(1).unwrap(), + Bound::Excluded(&i) => i, + Bound::Unbounded => self.haystack().len(), + }; + self.set_span(Span { start, end }); + } + + /// Set the starting offset for the span for this search configuration. + /// + /// This is a convenience routine for only mutating the start of a span + /// without having to set the entire span. + /// + /// # Panics + /// + /// This panics if the given span does not correspond to valid bounds in + /// the haystack or the termination of a search. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// input.set_start(5); + /// assert_eq!(5..6, input.get_range()); + /// ``` + #[inline] + pub fn set_start(&mut self, start: usize) { + self.set_span(Span { start, ..self.get_span() }); + } + + /// Set the ending offset for the span for this search configuration. + /// + /// This is a convenience routine for only mutating the end of a span + /// without having to set the entire span. + /// + /// # Panics + /// + /// This panics if the given span does not correspond to valid bounds in + /// the haystack or the termination of a search. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// input.set_end(5); + /// assert_eq!(0..5, input.get_range()); + /// ``` + #[inline] + pub fn set_end(&mut self, end: usize) { + self.set_span(Span { end, ..self.get_span() }); + } + + /// Set the anchor mode of a search. + /// + /// This is like [`Input::anchored`], except it mutates the search + /// configuration in place. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::{Anchored, Input}; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(Anchored::No, input.get_anchored()); + /// + /// input.set_anchored(Anchored::Yes); + /// assert_eq!(Anchored::Yes, input.get_anchored()); + /// ``` + #[inline] + pub fn set_anchored(&mut self, mode: Anchored) { + self.anchored = mode; + } + + /// Set whether the search should execute in "earliest" mode or not. + /// + /// This is like [`Input::earliest`], except it mutates the search + /// configuration in place. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert!(!input.get_earliest()); + /// input.set_earliest(true); + /// assert!(input.get_earliest()); + /// ``` + #[inline] + pub fn set_earliest(&mut self, yes: bool) { + self.earliest = yes; + } + + /// Return a borrow of the underlying haystack as a slice of bytes. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(b"foobar", input.haystack()); + /// ``` + #[inline] + pub fn haystack(&self) -> &[u8] { + self.haystack + } + + /// Return the start position of this search. + /// + /// This is a convenience routine for `search.get_span().start()`. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(0, input.start()); + /// + /// let input = Input::new("foobar").span(2..4); + /// assert_eq!(2, input.start()); + /// ``` + #[inline] + pub fn start(&self) -> usize { + self.get_span().start + } + + /// Return the end position of this search. + /// + /// This is a convenience routine for `search.get_span().end()`. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(6, input.end()); + /// + /// let input = Input::new("foobar").span(2..4); + /// assert_eq!(4, input.end()); + /// ``` + #[inline] + pub fn end(&self) -> usize { + self.get_span().end + } + + /// Return the span for this search configuration. + /// + /// If one was not explicitly set, then the span corresponds to the entire + /// range of the haystack. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::{Input, Span}; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(Span { start: 0, end: 6 }, input.get_span()); + /// ``` + #[inline] + pub fn get_span(&self) -> Span { + self.span + } + + /// Return the span as a range for this search configuration. + /// + /// If one was not explicitly set, then the span corresponds to the entire + /// range of the haystack. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// ``` + #[inline] + pub fn get_range(&self) -> Range { + self.get_span().range() + } + + /// Return the anchored mode for this search configuration. + /// + /// If no anchored mode was set, then it defaults to [`Anchored::No`]. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::{Anchored, Input}; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(Anchored::No, input.get_anchored()); + /// + /// input.set_anchored(Anchored::Yes); + /// assert_eq!(Anchored::Yes, input.get_anchored()); + /// ``` + #[inline] + pub fn get_anchored(&self) -> Anchored { + self.anchored + } + + /// Return whether this search should execute in "earliest" mode. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let input = Input::new("foobar"); + /// assert!(!input.get_earliest()); + /// ``` + #[inline] + pub fn get_earliest(&self) -> bool { + self.earliest + } + + /// Return true if this input has been exhausted, which in turn means all + /// subsequent searches will return no matches. + /// + /// This occurs precisely when the start position of this search is greater + /// than the end position of the search. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert!(!input.is_done()); + /// input.set_start(6); + /// assert!(!input.is_done()); + /// input.set_start(7); + /// assert!(input.is_done()); + /// ``` + #[inline] + pub fn is_done(&self) -> bool { + self.get_span().start > self.get_span().end + } +} + +impl<'h> core::fmt::Debug for Input<'h> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut fmter = f.debug_struct("Input"); + match core::str::from_utf8(self.haystack()) { + Ok(nice) => fmter.field("haystack", &nice), + Err(_) => fmter.field("haystack", &self.haystack()), + } + .field("span", &self.span) + .field("anchored", &self.anchored) + .field("earliest", &self.earliest) + .finish() + } +} + +impl<'h, H: ?Sized + AsRef<[u8]>> From<&'h H> for Input<'h> { + #[inline] + fn from(haystack: &'h H) -> Input<'h> { + Input::new(haystack) + } +} + +/// A representation of a range in a haystack. +/// +/// A span corresponds to the starting and ending _byte offsets_ of a +/// contiguous region of bytes. The starting offset is inclusive while the +/// ending offset is exclusive. That is, a span is a half-open interval. +/// +/// A span is used to report the offsets of a match, but it is also used to +/// convey which region of a haystack should be searched via routines like +/// [`Input::span`]. +/// +/// This is basically equivalent to a `std::ops::Range`, except this +/// type implements `Copy` which makes it more ergonomic to use in the context +/// of this crate. Indeed, `Span` exists only because `Range` does +/// not implement `Copy`. Like a range, this implements `Index` for `[u8]` +/// and `str`, and `IndexMut` for `[u8]`. For convenience, this also impls +/// `From`, which means things like `Span::from(5..10)` work. +/// +/// There are no constraints on the values of a span. It is, for example, legal +/// to create a span where `start > end`. +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub struct Span { + /// The start offset of the span, inclusive. + pub start: usize, + /// The end offset of the span, exclusive. + pub end: usize, +} + +impl Span { + /// Returns this span as a range. + #[inline] + pub fn range(&self) -> Range { + Range::from(*self) + } + + /// Returns true when this span is empty. That is, when `start >= end`. + #[inline] + pub fn is_empty(&self) -> bool { + self.start >= self.end + } + + /// Returns the length of this span. + /// + /// This returns `0` in precisely the cases that `is_empty` returns `true`. + #[inline] + pub fn len(&self) -> usize { + self.end.saturating_sub(self.start) + } + + /// Returns true when the given offset is contained within this span. + /// + /// Note that an empty span contains no offsets and will always return + /// false. + #[inline] + pub fn contains(&self, offset: usize) -> bool { + !self.is_empty() && self.start <= offset && offset <= self.end + } + + /// Returns a new span with `offset` added to this span's `start` and `end` + /// values. + /// + /// # Panics + /// + /// This panics if adding `offset` to either part of this `Span` would + /// result in overflow. + #[inline] + pub fn offset(&self, offset: usize) -> Span { + Span { + start: self + .start + .checked_add(offset) + .expect("invalid start+offset"), + end: self.end.checked_add(offset).expect("invalid end+offset"), + } + } +} + +impl core::fmt::Debug for Span { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{}..{}", self.start, self.end) + } +} + +impl core::ops::Index for [u8] { + type Output = [u8]; + + #[inline] + fn index(&self, index: Span) -> &[u8] { + &self[index.range()] + } +} + +impl core::ops::IndexMut for [u8] { + #[inline] + fn index_mut(&mut self, index: Span) -> &mut [u8] { + &mut self[index.range()] + } +} + +impl core::ops::Index for str { + type Output = str; + + #[inline] + fn index(&self, index: Span) -> &str { + &self[index.range()] + } +} + +impl From> for Span { + #[inline] + fn from(range: Range) -> Span { + Span { start: range.start, end: range.end } + } +} + +impl From for Range { + #[inline] + fn from(span: Span) -> Range { + Range { start: span.start, end: span.end } + } +} + +impl PartialEq> for Span { + #[inline] + fn eq(&self, range: &Range) -> bool { + self.start == range.start && self.end == range.end + } +} + +impl PartialEq for Range { + #[inline] + fn eq(&self, span: &Span) -> bool { + self.start == span.start && self.end == span.end + } +} + +/// The type of anchored search to perform. +/// +/// If an Aho-Corasick searcher does not support the anchored mode selected, +/// then the search will return an error or panic, depending on whether a +/// fallible or an infallible routine was called. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Anchored { + /// Run an unanchored search. This means a match may occur anywhere at or + /// after the start position of the search up until the end position of the + /// search. + No, + /// Run an anchored search. This means that a match must begin at the start + /// position of the search and end before the end position of the search. + Yes, +} + +impl Anchored { + /// Returns true if and only if this anchor mode corresponds to an anchored + /// search. + /// + /// # Example + /// + /// ``` + /// use aho_corasick::Anchored; + /// + /// assert!(!Anchored::No.is_anchored()); + /// assert!(Anchored::Yes.is_anchored()); + /// ``` + #[inline] + pub fn is_anchored(&self) -> bool { + matches!(*self, Anchored::Yes) + } +} + +/// A representation of a match reported by an Aho-Corasick searcher. +/// +/// A match has two essential pieces of information: the [`PatternID`] that +/// matches, and the [`Span`] of the match in a haystack. +/// +/// The pattern is identified by an ID, which corresponds to its position +/// (starting from `0`) relative to other patterns used to construct the +/// corresponding searcher. If only a single pattern is provided, then all +/// matches are guaranteed to have a pattern ID of `0`. +/// +/// Every match reported by a searcher guarantees that its span has its start +/// offset as less than or equal to its end offset. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct Match { + /// The pattern ID. + pattern: PatternID, + /// The underlying match span. + span: Span, +} + +impl Match { + /// Create a new match from a pattern ID and a span. + /// + /// This constructor is generic over how a span is provided. While + /// a [`Span`] may be given directly, one may also provide a + /// `std::ops::Range`. + /// + /// # Panics + /// + /// This panics if `end < start`. + /// + /// # Example + /// + /// This shows how to create a match for the first pattern in an + /// Aho-Corasick searcher using convenient range syntax. + /// + /// ``` + /// use aho_corasick::{Match, PatternID}; + /// + /// let m = Match::new(PatternID::ZERO, 5..10); + /// assert_eq!(0, m.pattern().as_usize()); + /// assert_eq!(5, m.start()); + /// assert_eq!(10, m.end()); + /// ``` + #[inline] + pub fn new>(pattern: PatternID, span: S) -> Match { + let span = span.into(); + assert!(span.start <= span.end, "invalid match span"); + Match { pattern, span } + } + + /// Create a new match from a pattern ID and a byte offset span. + /// + /// This constructor is generic over how a span is provided. While + /// a [`Span`] may be given directly, one may also provide a + /// `std::ops::Range`. + /// + /// This is like [`Match::new`], but accepts a `usize` instead of a + /// [`PatternID`]. This panics if the given `usize` is not representable + /// as a `PatternID`. + /// + /// # Panics + /// + /// This panics if `end < start` or if `pattern > PatternID::MAX`. + /// + /// # Example + /// + /// This shows how to create a match for the third pattern in an + /// Aho-Corasick searcher using convenient range syntax. + /// + /// ``` + /// use aho_corasick::Match; + /// + /// let m = Match::must(3, 5..10); + /// assert_eq!(3, m.pattern().as_usize()); + /// assert_eq!(5, m.start()); + /// assert_eq!(10, m.end()); + /// ``` + #[inline] + pub fn must>(pattern: usize, span: S) -> Match { + Match::new(PatternID::must(pattern), span) + } + + /// Returns the ID of the pattern that matched. + /// + /// The ID of a pattern is derived from the position in which it was + /// originally inserted into the corresponding searcher. The first pattern + /// has identifier `0`, and each subsequent pattern is `1`, `2` and so on. + #[inline] + pub fn pattern(&self) -> PatternID { + self.pattern + } + + /// The starting position of the match. + /// + /// This is a convenience routine for `Match::span().start`. + #[inline] + pub fn start(&self) -> usize { + self.span().start + } + + /// The ending position of the match. + /// + /// This is a convenience routine for `Match::span().end`. + #[inline] + pub fn end(&self) -> usize { + self.span().end + } + + /// Returns the match span as a range. + /// + /// This is a convenience routine for `Match::span().range()`. + #[inline] + pub fn range(&self) -> core::ops::Range { + self.span().range() + } + + /// Returns the span for this match. + #[inline] + pub fn span(&self) -> Span { + self.span + } + + /// Returns true when the span in this match is empty. + /// + /// An empty match can only be returned when empty pattern is in the + /// Aho-Corasick searcher. + #[inline] + pub fn is_empty(&self) -> bool { + self.span().is_empty() + } + + /// Returns the length of this match. + /// + /// This returns `0` in precisely the cases that `is_empty` returns `true`. + #[inline] + pub fn len(&self) -> usize { + self.span().len() + } + + /// Returns a new match with `offset` added to its span's `start` and `end` + /// values. + /// + /// # Panics + /// + /// This panics if adding `offset` to either part of this match's `Span` + /// would result in overflow. + #[inline] + pub fn offset(&self, offset: usize) -> Match { + Match { pattern: self.pattern, span: self.span.offset(offset) } + } +} + +/// A knob for controlling the match semantics of an Aho-Corasick automaton. +/// +/// There are two generally different ways that Aho-Corasick automatons can +/// report matches. The first way is the "standard" approach that results from +/// implementing most textbook explanations of Aho-Corasick. The second way is +/// to report only the leftmost non-overlapping matches. The leftmost approach +/// is in turn split into two different ways of resolving ambiguous matches: +/// leftmost-first and leftmost-longest. +/// +/// The `Standard` match kind is the default and is the only one that supports +/// overlapping matches and stream searching. (Trying to find overlapping or +/// streaming matches using leftmost match semantics will result in an error in +/// fallible APIs and a panic when using infallibe APIs.) The `Standard` match +/// kind will report matches as they are seen. When searching for overlapping +/// matches, then all possible matches are reported. When searching for +/// non-overlapping matches, the first match seen is reported. For example, for +/// non-overlapping matches, given the patterns `abcd` and `b` and the haystack +/// `abcdef`, only a match for `b` is reported since it is detected first. The +/// `abcd` match is never reported since it overlaps with the `b` match. +/// +/// In contrast, the leftmost match kind always prefers the leftmost match +/// among all possible matches. Given the same example as above with `abcd` and +/// `b` as patterns and `abcdef` as the haystack, the leftmost match is `abcd` +/// since it begins before the `b` match, even though the `b` match is detected +/// before the `abcd` match. In this case, the `b` match is not reported at all +/// since it overlaps with the `abcd` match. +/// +/// The difference between leftmost-first and leftmost-longest is in how they +/// resolve ambiguous matches when there are multiple leftmost matches to +/// choose from. Leftmost-first always chooses the pattern that was provided +/// earliest, where as leftmost-longest always chooses the longest matching +/// pattern. For example, given the patterns `a` and `ab` and the subject +/// string `ab`, the leftmost-first match is `a` but the leftmost-longest match +/// is `ab`. Conversely, if the patterns were given in reverse order, i.e., +/// `ab` and `a`, then both the leftmost-first and leftmost-longest matches +/// would be `ab`. Stated differently, the leftmost-first match depends on the +/// order in which the patterns were given to the Aho-Corasick automaton. +/// Because of that, when leftmost-first matching is used, if a pattern `A` +/// that appears before a pattern `B` is a prefix of `B`, then it is impossible +/// to ever observe a match of `B`. +/// +/// If you're not sure which match kind to pick, then stick with the standard +/// kind, which is the default. In particular, if you need overlapping or +/// streaming matches, then you _must_ use the standard kind. The leftmost +/// kinds are useful in specific circumstances. For example, leftmost-first can +/// be very useful as a way to implement match priority based on the order of +/// patterns given and leftmost-longest can be useful for dictionary searching +/// such that only the longest matching words are reported. +/// +/// # Relationship with regular expression alternations +/// +/// Understanding match semantics can be a little tricky, and one easy way +/// to conceptualize non-overlapping matches from an Aho-Corasick automaton +/// is to think about them as a simple alternation of literals in a regular +/// expression. For example, let's say we wanted to match the strings +/// `Sam` and `Samwise`, which would turn into the regex `Sam|Samwise`. It +/// turns out that regular expression engines have two different ways of +/// matching this alternation. The first way, leftmost-longest, is commonly +/// found in POSIX compatible implementations of regular expressions (such as +/// `grep`). The second way, leftmost-first, is commonly found in backtracking +/// implementations such as Perl. (Some regex engines, such as RE2 and Rust's +/// regex engine do not use backtracking, but still implement leftmost-first +/// semantics in an effort to match the behavior of dominant backtracking +/// regex engines such as those found in Perl, Ruby, Python, Javascript and +/// PHP.) +/// +/// That is, when matching `Sam|Samwise` against `Samwise`, a POSIX regex +/// will match `Samwise` because it is the longest possible match, but a +/// Perl-like regex will match `Sam` since it appears earlier in the +/// alternation. Indeed, the regex `Sam|Samwise` in a Perl-like regex engine +/// will never match `Samwise` since `Sam` will always have higher priority. +/// Conversely, matching the regex `Samwise|Sam` against `Samwise` will lead to +/// a match of `Samwise` in both POSIX and Perl-like regexes since `Samwise` is +/// still longest match, but it also appears earlier than `Sam`. +/// +/// The "standard" match semantics of Aho-Corasick generally don't correspond +/// to the match semantics of any large group of regex implementations, so +/// there's no direct analogy that can be made here. Standard match semantics +/// are generally useful for overlapping matches, or if you just want to see +/// matches as they are detected. +/// +/// The main conclusion to draw from this section is that the match semantics +/// can be tweaked to precisely match either Perl-like regex alternations or +/// POSIX regex alternations. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MatchKind { + /// Use standard match semantics, which support overlapping matches. When + /// used with non-overlapping matches, matches are reported as they are + /// seen. + Standard, + /// Use leftmost-first match semantics, which reports leftmost matches. + /// When there are multiple possible leftmost matches, the match + /// corresponding to the pattern that appeared earlier when constructing + /// the automaton is reported. + /// + /// This does **not** support overlapping matches or stream searching. If + /// this match kind is used, attempting to find overlapping matches or + /// stream matches will fail. + LeftmostFirst, + /// Use leftmost-longest match semantics, which reports leftmost matches. + /// When there are multiple possible leftmost matches, the longest match + /// is chosen. + /// + /// This does **not** support overlapping matches or stream searching. If + /// this match kind is used, attempting to find overlapping matches or + /// stream matches will fail. + LeftmostLongest, +} + +/// The default match kind is `MatchKind::Standard`. +impl Default for MatchKind { + fn default() -> MatchKind { + MatchKind::Standard + } +} + +impl MatchKind { + #[inline] + pub(crate) fn is_standard(&self) -> bool { + matches!(*self, MatchKind::Standard) + } + + #[inline] + pub(crate) fn is_leftmost(&self) -> bool { + matches!(*self, MatchKind::LeftmostFirst | MatchKind::LeftmostLongest) + } + + #[inline] + pub(crate) fn is_leftmost_first(&self) -> bool { + matches!(*self, MatchKind::LeftmostFirst) + } + + /// Convert this match kind into a packed match kind. If this match kind + /// corresponds to standard semantics, then this returns None, since + /// packed searching does not support standard semantics. + #[inline] + pub(crate) fn as_packed(&self) -> Option { + match *self { + MatchKind::Standard => None, + MatchKind::LeftmostFirst => { + Some(crate::packed::MatchKind::LeftmostFirst) + } + MatchKind::LeftmostLongest => { + Some(crate::packed::MatchKind::LeftmostLongest) + } + } + } +} + +/// The kind of anchored starting configurations to support in an Aho-Corasick +/// searcher. +/// +/// Depending on which searcher is used internally by +/// [`AhoCorasick`](crate::AhoCorasick), supporting both unanchored +/// and anchored searches can be quite costly. For this reason, +/// [`AhoCorasickBuilder::start_kind`](crate::AhoCorasickBuilder::start_kind) +/// can be used to configure whether your searcher supports unanchored, +/// anchored or both kinds of searches. +/// +/// This searcher configuration knob works in concert with the search time +/// configuration [`Input::anchored`]. Namely, if one requests an unsupported +/// anchored mode, then the search will either panic or return an error, +/// depending on whether you're using infallible or fallibe APIs, respectively. +/// +/// `AhoCorasick` by default only supports unanchored searches. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StartKind { + /// Support both anchored and unanchored searches. + Both, + /// Support only unanchored searches. Requesting an anchored search will + /// return an error in fallible APIs and panic in infallible APIs. + Unanchored, + /// Support only anchored searches. Requesting an unanchored search will + /// return an error in fallible APIs and panic in infallible APIs. + Anchored, +} + +impl Default for StartKind { + fn default() -> StartKind { + StartKind::Unanchored + } +} diff --git a/third_party/crates/aho-corasick-1.1.5/src/util/special.rs b/third_party/crates/aho-corasick-1.1.5/src/util/special.rs new file mode 100644 index 0000000..beeba40 --- /dev/null +++ b/third_party/crates/aho-corasick-1.1.5/src/util/special.rs @@ -0,0 +1,42 @@ +use crate::util::primitives::StateID; + +/// A collection of sentinel state IDs for Aho-Corasick automata. +/// +/// This specifically enables the technique by which we determine which states +/// are dead, matches or start states. Namely, by arranging states in a +/// particular order, we can determine the type of a state simply by looking at +/// its ID. +#[derive(Clone, Debug)] +pub(crate) struct Special { + /// The maximum ID of all the "special" states. This corresponds either to + /// start_anchored_id when a prefilter is active and max_match_id when a + /// prefilter is not active. The idea here is that if there is no prefilter, + /// then there is no point in treating start states as special. + pub(crate) max_special_id: StateID, + /// The maximum ID of all the match states. Any state ID bigger than this + /// is guaranteed to be a non-match ID. + /// + /// It is possible and legal for max_match_id to be equal to + /// start_anchored_id, which occurs precisely in the case where the empty + /// string is a pattern that was added to the underlying automaton. + pub(crate) max_match_id: StateID, + /// The state ID of the start state used for unanchored searches. + pub(crate) start_unanchored_id: StateID, + /// The state ID of the start state used for anchored searches. This is + /// always start_unanchored_id+1. + pub(crate) start_anchored_id: StateID, +} + +impl Special { + /// Create a new set of "special" state IDs with all IDs initialized to + /// zero. The general idea here is that they will be updated and set to + /// correct values later. + pub(crate) fn zero() -> Special { + Special { + max_special_id: StateID::ZERO, + max_match_id: StateID::ZERO, + start_unanchored_id: StateID::ZERO, + start_anchored_id: StateID::ZERO, + } + } +} diff --git a/third_party/crates/autocfg-1.5.1/.cargo-checksum.json b/third_party/crates/autocfg-1.5.1/.cargo-checksum.json new file mode 100644 index 0000000..4b62cd1 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"5713fb5e9d7124071a4fb0d0f8f80535560c7cef1995c42fe168af1c2627a050","Cargo.lock":"dc8d767ba9ab277c030476ca736afce4ecb4999a01bd3db8c8210a12ef79e81c","Cargo.toml":"ed7c832b4250c3be736e040d68688c2b83a5b164a93ef533e61532cbe0d44e99","Cargo.toml.orig":"57d2aec0d4157bd9446f97705aa2acff86cc3d1883f470526fed089e7cf9754e","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"27995d58ad5c1145c1a8cd86244ce844886958a35eb2b78c6b772748669999ac","README.md":"1c6c43f1d22ab1c6411b82862c356c550947f3cecbb4c7eede5edefc8890ff32","examples/integers.rs":"589ff4271566dfa322becddf3e2c7b592e6e0bc97b02892ce75619b7e452e930","examples/nightly.rs":"ac8b5a9aa1e04465e44f5053b3c899b635e07af058c73aa8b45176bf4b5912f9","examples/paths.rs":"1b30e466b824ce8df7ad0a55334424131d9d2573d6cf9f7d5d50c09c8901d526","examples/traits.rs":"cbee6a3e1f7db60b02ae25b714926517144a77cb492021f492774cf0e1865a9e","examples/versions.rs":"38535e6d9f5bfae0de474a3db79a40e8f5da8ba9334c5ff4c363de9bc99d4d12","src/error.rs":"fd8ff67c64f7cd1b9f81325a81de4baa34c39d6ae298bdb33f9829cc91acac39","src/lib.rs":"772630d0e09d06f343fd72284c8330eacde87d2a4ee22f70af62a0e2119fbf05","src/rustc.rs":"a8a213ddb64a05c1a1af933bcb331a98879e942b167c33d8f94f9f60ebb14e29","src/tests.rs":"b39f4d880ad343e65307a9e0c381954ea27adce4732f825516ce7952e2e5a91d","src/version.rs":"4f7d23b36f01c7be1871be86c038d6cb4689e145d67c82d3793690e9aa05b133","tests/no_std.rs":"18859dc4992fe1769887bde05f03d28f1ce524eafd17646d3fbcb4379422761a","tests/rustflags.rs":"e8ded4d57ba25379a38ab48456d67df14f82abbbb5f6bb66221c6decbcb517a3","tests/support/mod.rs":"32087d365b438ac3f62df9bb066d8d648b80cb130a5c777afcb2f21fbb68d88e","tests/tests.rs":"39149fe69a45f577eeb531587a5e600d8bc9f8d937480172dac9402939d856f0","tests/wrappers.rs":"96aee2217fbfc7a3aff16e70609c885620a4704dfd708de8b2d0bad4a8c6247d"},"package":"f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"} \ No newline at end of file diff --git a/third_party/crates/autocfg-1.5.1/.cargo_vcs_info.json b/third_party/crates/autocfg-1.5.1/.cargo_vcs_info.json new file mode 100644 index 0000000..df94611 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "2799b09c24e6632f8e653c5cd8fc303e85a906ba" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/third_party/crates/autocfg-1.5.1/BUILD.bazel b/third_party/crates/autocfg-1.5.1/BUILD.bazel new file mode 100644 index 0000000..bac4e9a --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/BUILD.bazel @@ -0,0 +1,110 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_library( + name = "autocfg", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_root = "src/lib.rs", + edition = "2015", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=autocfg", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "1.5.1", +) diff --git a/third_party/crates/autocfg-1.5.1/Cargo.lock b/third_party/crates/autocfg-1.5.1/Cargo.lock new file mode 100644 index 0000000..706e399 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/Cargo.lock @@ -0,0 +1,6 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "autocfg" +version = "1.5.1" + diff --git a/third_party/crates/autocfg-1.5.1/Cargo.toml b/third_party/crates/autocfg-1.5.1/Cargo.toml new file mode 100644 index 0000000..2327046 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/Cargo.toml @@ -0,0 +1,75 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +rust-version = "1.0" +name = "autocfg" +version = "1.5.1" +authors = ["Josh Stone "] +build = false +exclude = ["/.github/**"] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Automatic cfg for Rust compiler features" +documentation = "https://docs.rs/autocfg/" +readme = "README.md" +keywords = [ + "rustc", + "build", + "autoconf", +] +categories = ["development-tools::build-utils"] +license = "Apache-2.0 OR MIT" +repository = "https://github.com/cuviper/autocfg" + +[lib] +name = "autocfg" +path = "src/lib.rs" + +[[example]] +name = "integers" +path = "examples/integers.rs" + +[[example]] +name = "nightly" +path = "examples/nightly.rs" + +[[example]] +name = "paths" +path = "examples/paths.rs" + +[[example]] +name = "traits" +path = "examples/traits.rs" + +[[example]] +name = "versions" +path = "examples/versions.rs" + +[[test]] +name = "no_std" +path = "tests/no_std.rs" + +[[test]] +name = "rustflags" +path = "tests/rustflags.rs" + +[[test]] +name = "tests" +path = "tests/tests.rs" + +[[test]] +name = "wrappers" +path = "tests/wrappers.rs" +harness = false diff --git a/third_party/crates/autocfg-1.5.1/Cargo.toml.orig b/third_party/crates/autocfg-1.5.1/Cargo.toml.orig new file mode 100644 index 0000000..8e9bf95 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/Cargo.toml.orig @@ -0,0 +1,26 @@ +[package] +name = "autocfg" +version = "1.5.1" +authors = ["Josh Stone "] +license = "Apache-2.0 OR MIT" +repository = "https://github.com/cuviper/autocfg" +documentation = "https://docs.rs/autocfg/" +description = "Automatic cfg for Rust compiler features" +readme = "README.md" +keywords = ["rustc", "build", "autoconf"] +categories = ["development-tools::build-utils"] +exclude = ["/.github/**"] +rust-version = "1.0" + +[[test]] +name = "no_std" + +[[test]] +name = "rustflags" + +[[test]] +name = "tests" + +[[test]] +name = "wrappers" +harness = false diff --git a/third_party/crates/autocfg-1.5.1/LICENSE-APACHE b/third_party/crates/autocfg-1.5.1/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/crates/autocfg-1.5.1/LICENSE-MIT b/third_party/crates/autocfg-1.5.1/LICENSE-MIT new file mode 100644 index 0000000..44fbc4d --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2018 Josh Stone + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third_party/crates/autocfg-1.5.1/README.md b/third_party/crates/autocfg-1.5.1/README.md new file mode 100644 index 0000000..05dbfc6 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/README.md @@ -0,0 +1,121 @@ +autocfg +======= + +[![autocfg crate](https://img.shields.io/crates/v/autocfg.svg)](https://crates.io/crates/autocfg) +[![autocfg documentation](https://docs.rs/autocfg/badge.svg)](https://docs.rs/autocfg) +![minimum rustc 1.0](https://img.shields.io/badge/rustc-1.0+-red.svg) +![build status](https://github.com/cuviper/autocfg/workflows/CI/badge.svg) + +A Rust library for build scripts to automatically configure code based on +compiler support. Code snippets are dynamically tested to see if the `rustc` +will accept them, rather than hard-coding specific version support. + + +## Usage + +Add this to your `Cargo.toml`: + +```toml +[build-dependencies] +autocfg = "1" +``` + +Then use it in your `build.rs` script to detect compiler features. For +example, to test for 128-bit integer support, it might look like: + +```rust +extern crate autocfg; + +fn main() { + let ac = autocfg::new(); + ac.emit_has_type("i128"); + + // (optional) We don't need to rerun for anything external. + autocfg::rerun_path("build.rs"); +} +``` + +If the type test succeeds, this will write a `cargo:rustc-cfg=has_i128` line +for Cargo, which translates to Rust arguments `--cfg has_i128`. Then in the +rest of your Rust code, you can add `#[cfg(has_i128)]` conditions on code that +should only be used when the compiler supports it. + + +## Release Notes + +- 1.5.1 (2026-05-22) + + - Refactor the `test_wrappers` test without any shell script. + +- 1.5.0 (2025-06-17) + + - Add `edition` and `set_edition` to control the Rust edition used in probes. + - Remove probe result files so they don't pollute the output directory. + +- 1.4.0 (2024-09-26) + + - Add `emit_possibility` for Rust 1.80's [checked cfgs], and call that + automatically for methods that conditionally `emit`, by @Techcable. + +[checked cfgs]: https://blog.rust-lang.org/2024/05/06/check-cfg.html + +- 1.3.0 (2024-05-03) + + - Add `probe_raw` for direct control of the code that will be test-compiled. + - Use wrappers when querying the `rustc` version information too. + +- 1.2.0 (2024-03-25) + + - Add `no_std` and `set_no_std` to control the use of `#![no_std]` in probes. + - Use `RUSTC_WRAPPER` and `RUSTC_WORKSPACE_WRAPPER` when they are set. + +- 1.1.0 (2022-02-07) + - Use `CARGO_ENCODED_RUSTFLAGS` when it is set. + +- 1.0.1 (2020-08-20) + - Apply `RUSTFLAGS` for more `--target` scenarios, by @adamreichold. + +- 1.0.0 (2020-01-08) + - 🎉 Release 1.0! 🎉 (no breaking changes) + - Add `probe_expression` and `emit_expression_cfg` to test arbitrary expressions. + - Add `probe_constant` and `emit_constant_cfg` to test arbitrary constant expressions. + +- 0.1.7 (2019-10-20) + - Apply `RUSTFLAGS` when probing `$TARGET != $HOST`, mainly for sysroot, by @roblabla. + +- 0.1.6 (2019-08-19) + - Add `probe`/`emit_sysroot_crate`, by @leo60228. + +- 0.1.5 (2019-07-16) + - Mask some warnings from newer rustc. + +- 0.1.4 (2019-05-22) + - Relax `std`/`no_std` probing to a warning instead of an error. + - Improve `rustc` bootstrap compatibility. + +- 0.1.3 (2019-05-21) + - Auto-detects if `#![no_std]` is needed for the `$TARGET`. + +- 0.1.2 (2019-01-16) + - Add `rerun_env(ENV)` to print `cargo:rerun-if-env-changed=ENV`. + - Add `rerun_path(PATH)` to print `cargo:rerun-if-changed=PATH`. + + +## Minimum Rust version policy + +This crate's minimum supported `rustc` version is `1.0.0`. Compatibility is +its entire reason for existence, so this crate will be extremely conservative +about raising this requirement. If this is ever deemed necessary, it will be +treated as a major breaking change for semver purposes. + + +## License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + https://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + https://opensource.org/licenses/MIT) + +at your option. diff --git a/third_party/crates/autocfg-1.5.1/examples/integers.rs b/third_party/crates/autocfg-1.5.1/examples/integers.rs new file mode 100644 index 0000000..23d4cba --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/examples/integers.rs @@ -0,0 +1,9 @@ +extern crate autocfg; + +fn main() { + // Normally, cargo will set `OUT_DIR` for build scripts. + let ac = autocfg::AutoCfg::with_dir("target").unwrap(); + for i in 3..8 { + ac.emit_has_type(&format!("i{}", 1 << i)); + } +} diff --git a/third_party/crates/autocfg-1.5.1/examples/nightly.rs b/third_party/crates/autocfg-1.5.1/examples/nightly.rs new file mode 100644 index 0000000..955e63d --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/examples/nightly.rs @@ -0,0 +1,18 @@ +extern crate autocfg; + +fn main() { + // Normally, cargo will set `OUT_DIR` for build scripts. + let ac = autocfg::AutoCfg::with_dir("target").unwrap(); + + // When this feature was stabilized, it also renamed the method to + // `chunk_by`, so it's important to *use* the feature in your probe. + let code = r#" + #![feature(slice_group_by)] + pub fn probe(slice: &[i32]) -> impl Iterator { + slice.group_by(|a, b| a == b) + } + "#; + if ac.probe_raw(code).is_ok() { + autocfg::emit("has_slice_group_by"); + } +} diff --git a/third_party/crates/autocfg-1.5.1/examples/paths.rs b/third_party/crates/autocfg-1.5.1/examples/paths.rs new file mode 100644 index 0000000..b7a6ca7 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/examples/paths.rs @@ -0,0 +1,22 @@ +extern crate autocfg; + +fn main() { + // Normally, cargo will set `OUT_DIR` for build scripts. + let ac = autocfg::AutoCfg::with_dir("target").unwrap(); + + // since ancient times... + ac.emit_has_path("std::vec::Vec"); + ac.emit_path_cfg("std::vec::Vec", "has_vec"); + + // rustc 1.10.0 + ac.emit_has_path("std::panic::PanicInfo"); + ac.emit_path_cfg("std::panic::PanicInfo", "has_panic_info"); + + // rustc 1.20.0 + ac.emit_has_path("std::mem::ManuallyDrop"); + ac.emit_path_cfg("std::mem::ManuallyDrop", "has_manually_drop"); + + // rustc 1.25.0 + ac.emit_has_path("std::ptr::NonNull"); + ac.emit_path_cfg("std::ptr::NonNull", "has_non_null"); +} diff --git a/third_party/crates/autocfg-1.5.1/examples/traits.rs b/third_party/crates/autocfg-1.5.1/examples/traits.rs new file mode 100644 index 0000000..c1ca003 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/examples/traits.rs @@ -0,0 +1,26 @@ +extern crate autocfg; + +fn main() { + // Normally, cargo will set `OUT_DIR` for build scripts. + let ac = autocfg::AutoCfg::with_dir("target").unwrap(); + + // since ancient times... + ac.emit_has_trait("std::ops::Add"); + ac.emit_trait_cfg("std::ops::Add", "has_ops"); + + // trait parameters have to be provided + ac.emit_has_trait("std::borrow::Borrow"); + ac.emit_trait_cfg("std::borrow::Borrow", "has_borrow"); + + // rustc 1.8.0 + ac.emit_has_trait("std::ops::AddAssign"); + ac.emit_trait_cfg("std::ops::AddAssign", "has_assign_ops"); + + // rustc 1.12.0 + ac.emit_has_trait("std::iter::Sum"); + ac.emit_trait_cfg("std::iter::Sum", "has_sum"); + + // rustc 1.28.0 + ac.emit_has_trait("std::alloc::GlobalAlloc"); + ac.emit_trait_cfg("std::alloc::GlobalAlloc", "has_global_alloc"); +} diff --git a/third_party/crates/autocfg-1.5.1/examples/versions.rs b/third_party/crates/autocfg-1.5.1/examples/versions.rs new file mode 100644 index 0000000..992919b --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/examples/versions.rs @@ -0,0 +1,9 @@ +extern crate autocfg; + +fn main() { + // Normally, cargo will set `OUT_DIR` for build scripts. + let ac = autocfg::AutoCfg::with_dir("target").unwrap(); + for i in 0..100 { + ac.emit_rustc_version(1, i); + } +} diff --git a/third_party/crates/autocfg-1.5.1/src/error.rs b/third_party/crates/autocfg-1.5.1/src/error.rs new file mode 100644 index 0000000..7839ee9 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/src/error.rs @@ -0,0 +1,81 @@ +use std::error; +use std::fmt; +use std::io; +use std::num; +use std::process; +use std::str; + +/// A common error type for the `autocfg` crate. +#[derive(Debug)] +pub struct Error { + kind: ErrorKind, +} + +impl error::Error for Error { + fn description(&self) -> &str { + "AutoCfg error" + } + + fn cause(&self) -> Option<&error::Error> { + match self.kind { + ErrorKind::Io(ref e) => Some(e), + ErrorKind::Num(ref e) => Some(e), + ErrorKind::Utf8(ref e) => Some(e), + ErrorKind::Process(_) | ErrorKind::Other(_) => None, + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + match self.kind { + ErrorKind::Io(ref e) => e.fmt(f), + ErrorKind::Num(ref e) => e.fmt(f), + ErrorKind::Utf8(ref e) => e.fmt(f), + ErrorKind::Process(ref status) => { + // Same message as the newer `ExitStatusError` + write!(f, "process exited unsuccessfully: {}", status) + } + ErrorKind::Other(s) => s.fmt(f), + } + } +} + +#[derive(Debug)] +enum ErrorKind { + Io(io::Error), + Num(num::ParseIntError), + Process(process::ExitStatus), + Utf8(str::Utf8Error), + Other(&'static str), +} + +pub fn from_exit(status: process::ExitStatus) -> Error { + Error { + kind: ErrorKind::Process(status), + } +} + +pub fn from_io(e: io::Error) -> Error { + Error { + kind: ErrorKind::Io(e), + } +} + +pub fn from_num(e: num::ParseIntError) -> Error { + Error { + kind: ErrorKind::Num(e), + } +} + +pub fn from_utf8(e: str::Utf8Error) -> Error { + Error { + kind: ErrorKind::Utf8(e), + } +} + +pub fn from_str(s: &'static str) -> Error { + Error { + kind: ErrorKind::Other(s), + } +} diff --git a/third_party/crates/autocfg-1.5.1/src/lib.rs b/third_party/crates/autocfg-1.5.1/src/lib.rs new file mode 100644 index 0000000..69e6ffc --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/src/lib.rs @@ -0,0 +1,636 @@ +//! A Rust library for build scripts to automatically configure code based on +//! compiler support. Code snippets are dynamically tested to see if the `rustc` +//! will accept them, rather than hard-coding specific version support. +//! +//! +//! ## Usage +//! +//! Add this to your `Cargo.toml`: +//! +//! ```toml +//! [build-dependencies] +//! autocfg = "1" +//! ``` +//! +//! Then use it in your `build.rs` script to detect compiler features. For +//! example, to test for 128-bit integer support, it might look like: +//! +//! ```rust +//! extern crate autocfg; +//! +//! fn main() { +//! # // Normally, cargo will set `OUT_DIR` for build scripts. +//! # let exe = std::env::current_exe().unwrap(); +//! # std::env::set_var("OUT_DIR", exe.parent().unwrap()); +//! let ac = autocfg::new(); +//! ac.emit_has_type("i128"); +//! +//! // (optional) We don't need to rerun for anything external. +//! autocfg::rerun_path("build.rs"); +//! } +//! ``` +//! +//! If the type test succeeds, this will write a `cargo:rustc-cfg=has_i128` line +//! for Cargo, which translates to Rust arguments `--cfg has_i128`. Then in the +//! rest of your Rust code, you can add `#[cfg(has_i128)]` conditions on code that +//! should only be used when the compiler supports it. +//! +//! ## Caution +//! +//! Many of the probing methods of `AutoCfg` document the particular template they +//! use, **subject to change**. The inputs are not validated to make sure they are +//! semantically correct for their expected use, so it's _possible_ to escape and +//! inject something unintended. However, such abuse is unsupported and will not +//! be considered when making changes to the templates. + +#![deny(missing_debug_implementations)] +#![deny(missing_docs)] +// allow future warnings that can't be fixed while keeping 1.0 compatibility +#![allow(unknown_lints)] +#![allow(bare_trait_objects)] +#![allow(ellipsis_inclusive_range_patterns)] + +/// Local macro to avoid `std::try!`, deprecated in Rust 1.39. +macro_rules! try { + ($result:expr) => { + match $result { + Ok(value) => value, + Err(error) => return Err(error), + } + }; +} + +use std::env; +use std::ffi::OsString; +use std::fmt::Arguments; +use std::fs; +use std::io::{stderr, Write}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +#[allow(deprecated)] +use std::sync::atomic::ATOMIC_USIZE_INIT; +use std::sync::atomic::{AtomicUsize, Ordering}; + +mod error; +pub use error::Error; + +mod rustc; +use rustc::Rustc; + +mod version; +use version::Version; + +#[cfg(test)] +mod tests; + +/// Helper to detect compiler features for `cfg` output in build scripts. +#[derive(Clone, Debug)] +pub struct AutoCfg { + out_dir: PathBuf, + rustc: Rustc, + rustc_version: Version, + target: Option, + no_std: bool, + edition: Option, + rustflags: Vec, + uuid: u64, +} + +/// Writes a config flag for rustc on standard out. +/// +/// This looks like: `cargo:rustc-cfg=CFG` +/// +/// Cargo will use this in arguments to rustc, like `--cfg CFG`. +/// +/// This does not automatically call [`emit_possibility`] +/// so the compiler my generate an [`unexpected_cfgs` warning][check-cfg-flags]. +/// However, all the builtin emit methods on [`AutoCfg`] call [`emit_possibility`] automatically. +/// +/// [check-cfg-flags]: https://blog.rust-lang.org/2024/05/06/check-cfg.html +pub fn emit(cfg: &str) { + println!("cargo:rustc-cfg={}", cfg); +} + +/// Writes a line telling Cargo to rerun the build script if `path` changes. +/// +/// This looks like: `cargo:rerun-if-changed=PATH` +/// +/// This requires at least cargo 0.7.0, corresponding to rustc 1.6.0. Earlier +/// versions of cargo will simply ignore the directive. +pub fn rerun_path(path: &str) { + println!("cargo:rerun-if-changed={}", path); +} + +/// Writes a line telling Cargo to rerun the build script if the environment +/// variable `var` changes. +/// +/// This looks like: `cargo:rerun-if-env-changed=VAR` +/// +/// This requires at least cargo 0.21.0, corresponding to rustc 1.20.0. Earlier +/// versions of cargo will simply ignore the directive. +pub fn rerun_env(var: &str) { + println!("cargo:rerun-if-env-changed={}", var); +} + +/// Indicates to rustc that a config flag should not generate an [`unexpected_cfgs` warning][check-cfg-flags] +/// +/// This looks like `cargo:rustc-check-cfg=cfg(VAR)` +/// +/// As of rust 1.80, the compiler does [automatic checking of cfgs at compile time][check-cfg-flags]. +/// All custom configuration flags must be known to rustc, or they will generate a warning. +/// This is done automatically when calling the builtin emit methods on [`AutoCfg`], +/// but not when calling [`autocfg::emit`](crate::emit) directly. +/// +/// Versions before rust 1.80 will simply ignore this directive. +/// +/// This function indicates to the compiler that the config flag never has a value. +/// If this is not desired, see [the blog post][check-cfg]. +/// +/// [check-cfg-flags]: https://blog.rust-lang.org/2024/05/06/check-cfg.html +pub fn emit_possibility(cfg: &str) { + println!("cargo:rustc-check-cfg=cfg({})", cfg); +} + +/// Creates a new `AutoCfg` instance. +/// +/// # Panics +/// +/// Panics if `AutoCfg::new()` returns an error. +pub fn new() -> AutoCfg { + AutoCfg::new().unwrap() +} + +impl AutoCfg { + /// Creates a new `AutoCfg` instance. + /// + /// # Common errors + /// + /// - `rustc` can't be executed, from `RUSTC` or in the `PATH`. + /// - The version output from `rustc` can't be parsed. + /// - `OUT_DIR` is not set in the environment, or is not a writable directory. + /// + pub fn new() -> Result { + match env::var_os("OUT_DIR") { + Some(d) => Self::with_dir(d), + None => Err(error::from_str("no OUT_DIR specified!")), + } + } + + /// Creates a new `AutoCfg` instance with the specified output directory. + /// + /// # Common errors + /// + /// - `rustc` can't be executed, from `RUSTC` or in the `PATH`. + /// - The version output from `rustc` can't be parsed. + /// - `dir` is not a writable directory. + /// + pub fn with_dir>(dir: T) -> Result { + let rustc = Rustc::new(); + let rustc_version = try!(rustc.version()); + + let target = env::var_os("TARGET"); + + // Sanity check the output directory + let dir = dir.into(); + let meta = try!(fs::metadata(&dir).map_err(error::from_io)); + if !meta.is_dir() || meta.permissions().readonly() { + return Err(error::from_str("output path is not a writable directory")); + } + + let mut ac = AutoCfg { + rustflags: rustflags(&target, &dir), + out_dir: dir, + rustc: rustc, + rustc_version: rustc_version, + target: target, + no_std: false, + edition: None, + uuid: new_uuid(), + }; + + // Sanity check with and without `std`. + if ac.probe_raw("").is_err() { + if ac.probe_raw("#![no_std]").is_ok() { + ac.no_std = true; + } else { + // Neither worked, so assume nothing... + let warning = b"warning: autocfg could not probe for `std`\n"; + stderr().write_all(warning).ok(); + } + } + Ok(ac) + } + + /// Returns whether `AutoCfg` is using `#![no_std]` in its probes. + /// + /// This is automatically detected during construction -- if an empty probe + /// fails while one with `#![no_std]` succeeds, then the attribute will be + /// used for all further probes. This is usually only necessary when the + /// `TARGET` lacks `std` altogether. If neither succeeds, `no_std` is not + /// set, but that `AutoCfg` will probably only work for version checks. + /// + /// This attribute changes the implicit [prelude] from `std` to `core`, + /// which may affect the paths you need to use in other probes. It also + /// restricts some types that otherwise get additional methods in `std`, + /// like floating-point trigonometry and slice sorting. + /// + /// See also [`set_no_std`](#method.set_no_std). + /// + /// [prelude]: https://doc.rust-lang.org/reference/names/preludes.html#the-no_std-attribute + pub fn no_std(&self) -> bool { + self.no_std + } + + /// Sets whether `AutoCfg` should use `#![no_std]` in its probes. + /// + /// See also [`no_std`](#method.no_std). + pub fn set_no_std(&mut self, no_std: bool) { + self.no_std = no_std; + } + + /// Returns the `--edition` string that is currently being passed to `rustc`, if any, + /// as configured by the [`set_edition`][Self::set_edition] method. + pub fn edition(&self) -> Option<&str> { + match self.edition { + Some(ref edition) => Some(&**edition), + None => None, + } + } + + /// Sets the `--edition` string that will be passed to `rustc`, + /// or `None` to leave the compiler at its default edition. + /// + /// See also [The Rust Edition Guide](https://doc.rust-lang.org/edition-guide/). + /// + /// **Warning:** Setting an unsupported edition will likely cause **all** subsequent probes to + /// fail! As of this writing, the known editions and their minimum Rust versions are: + /// + /// | Edition | Version | + /// | ------- | ---------- | + /// | 2015 | 1.27.0[^1] | + /// | 2018 | 1.31.0 | + /// | 2021 | 1.56.0 | + /// | 2024 | 1.85.0 | + /// + /// [^1]: Prior to 1.27.0, Rust was effectively 2015 Edition by default, but the concept hadn't + /// been established yet, so the explicit `--edition` flag wasn't supported either. + pub fn set_edition(&mut self, edition: Option) { + self.edition = edition; + } + + /// Tests whether the current `rustc` reports a version greater than + /// or equal to "`major`.`minor`". + pub fn probe_rustc_version(&self, major: usize, minor: usize) -> bool { + self.rustc_version >= Version::new(major, minor, 0) + } + + /// Sets a `cfg` value of the form `rustc_major_minor`, like `rustc_1_29`, + /// if the current `rustc` is at least that version. + pub fn emit_rustc_version(&self, major: usize, minor: usize) { + let cfg_flag = format!("rustc_{}_{}", major, minor); + emit_possibility(&cfg_flag); + if self.probe_rustc_version(major, minor) { + emit(&cfg_flag); + } + } + + /// Returns a new (hopefully unique) crate name for probes. + fn new_crate_name(&self) -> String { + #[allow(deprecated)] + static ID: AtomicUsize = ATOMIC_USIZE_INIT; + + let id = ID.fetch_add(1, Ordering::Relaxed); + format!("autocfg_{:016x}_{}", self.uuid, id) + } + + fn probe_fmt<'a>(&self, source: Arguments<'a>) -> Result<(), Error> { + let crate_name = self.new_crate_name(); + let mut command = self.rustc.command(); + command + .arg("--crate-name") + .arg(&crate_name) + .arg("--crate-type=lib") + .arg("--out-dir") + .arg(&self.out_dir) + .arg("--emit=llvm-ir"); + + if let Some(edition) = self.edition.as_ref() { + command.arg("--edition").arg(edition); + } + + if let Some(target) = self.target.as_ref() { + command.arg("--target").arg(target); + } + + command.args(&self.rustflags); + + command.arg("-").stdin(Stdio::piped()); + let mut child = try!(command.spawn().map_err(error::from_io)); + let mut stdin = child.stdin.take().expect("rustc stdin"); + + try!(stdin.write_fmt(source).map_err(error::from_io)); + drop(stdin); + + match child.wait() { + Ok(status) if status.success() => { + // Try to remove the output file so it doesn't look like a build product for + // systems like bazel -- but this is best-effort, so we can ignore failure. + // The probe itself is already considered successful at this point. + let mut file = self.out_dir.join(crate_name); + file.set_extension("ll"); + let _ = fs::remove_file(file); + + Ok(()) + } + Ok(status) => Err(error::from_exit(status)), + Err(error) => Err(error::from_io(error)), + } + } + + fn probe<'a>(&self, code: Arguments<'a>) -> bool { + let result = if self.no_std { + self.probe_fmt(format_args!("#![no_std]\n{}", code)) + } else { + self.probe_fmt(code) + }; + result.is_ok() + } + + /// Tests whether the given code can be compiled as a Rust library. + /// + /// This will only return `Ok` if the compiler ran and exited successfully, + /// per `ExitStatus::success()`. + /// The code is passed to the compiler exactly as-is, notably not even + /// adding the [`#![no_std]`][Self::no_std] attribute like other probes. + /// + /// Raw probes are useful for testing functionality that's not yet covered + /// by the rest of the `AutoCfg` API. For example, the following attribute + /// **must** be used at the crate level, so it wouldn't work within the code + /// templates used by other `probe_*` methods. + /// + /// ``` + /// # extern crate autocfg; + /// # // Normally, cargo will set `OUT_DIR` for build scripts. + /// # let exe = std::env::current_exe().unwrap(); + /// # std::env::set_var("OUT_DIR", exe.parent().unwrap()); + /// let ac = autocfg::new(); + /// assert!(ac.probe_raw("#![no_builtins]").is_ok()); + /// ``` + /// + /// Rust nightly features could be tested as well -- ideally including a + /// code sample to ensure the unstable feature still works as expected. + /// For example, `slice::group_by` was renamed to `chunk_by` when it was + /// stabilized, even though the feature name was unchanged, so testing the + /// `#![feature(..)]` alone wouldn't reveal that. For larger snippets, + /// [`include_str!`] may be useful to load them from separate files. + /// + /// ``` + /// # extern crate autocfg; + /// # // Normally, cargo will set `OUT_DIR` for build scripts. + /// # let exe = std::env::current_exe().unwrap(); + /// # std::env::set_var("OUT_DIR", exe.parent().unwrap()); + /// let ac = autocfg::new(); + /// let code = r#" + /// #![feature(slice_group_by)] + /// pub fn probe(slice: &[i32]) -> impl Iterator { + /// slice.group_by(|a, b| a == b) + /// } + /// "#; + /// if ac.probe_raw(code).is_ok() { + /// autocfg::emit("has_slice_group_by"); + /// } + /// ``` + pub fn probe_raw(&self, code: &str) -> Result<(), Error> { + self.probe_fmt(format_args!("{}", code)) + } + + /// Tests whether the given sysroot crate can be used. + /// + /// The test code is subject to change, but currently looks like: + /// + /// ```ignore + /// extern crate CRATE as probe; + /// ``` + pub fn probe_sysroot_crate(&self, name: &str) -> bool { + // Note: `as _` wasn't stabilized until Rust 1.33 + self.probe(format_args!("extern crate {} as probe;", name)) + } + + /// Emits a config value `has_CRATE` if `probe_sysroot_crate` returns true. + pub fn emit_sysroot_crate(&self, name: &str) { + let cfg_flag = format!("has_{}", mangle(name)); + emit_possibility(&cfg_flag); + if self.probe_sysroot_crate(name) { + emit(&cfg_flag); + } + } + + /// Tests whether the given path can be used. + /// + /// The test code is subject to change, but currently looks like: + /// + /// ```ignore + /// pub use PATH; + /// ``` + pub fn probe_path(&self, path: &str) -> bool { + self.probe(format_args!("pub use {};", path)) + } + + /// Emits a config value `has_PATH` if `probe_path` returns true. + /// + /// Any non-identifier characters in the `path` will be replaced with + /// `_` in the generated config value. + pub fn emit_has_path(&self, path: &str) { + self.emit_path_cfg(path, &format!("has_{}", mangle(path))); + } + + /// Emits the given `cfg` value if `probe_path` returns true. + pub fn emit_path_cfg(&self, path: &str, cfg: &str) { + emit_possibility(cfg); + if self.probe_path(path) { + emit(cfg); + } + } + + /// Tests whether the given trait can be used. + /// + /// The test code is subject to change, but currently looks like: + /// + /// ```ignore + /// pub trait Probe: TRAIT + Sized {} + /// ``` + pub fn probe_trait(&self, name: &str) -> bool { + self.probe(format_args!("pub trait Probe: {} + Sized {{}}", name)) + } + + /// Emits a config value `has_TRAIT` if `probe_trait` returns true. + /// + /// Any non-identifier characters in the trait `name` will be replaced with + /// `_` in the generated config value. + pub fn emit_has_trait(&self, name: &str) { + self.emit_trait_cfg(name, &format!("has_{}", mangle(name))); + } + + /// Emits the given `cfg` value if `probe_trait` returns true. + pub fn emit_trait_cfg(&self, name: &str, cfg: &str) { + emit_possibility(cfg); + if self.probe_trait(name) { + emit(cfg); + } + } + + /// Tests whether the given type can be used. + /// + /// The test code is subject to change, but currently looks like: + /// + /// ```ignore + /// pub type Probe = TYPE; + /// ``` + pub fn probe_type(&self, name: &str) -> bool { + self.probe(format_args!("pub type Probe = {};", name)) + } + + /// Emits a config value `has_TYPE` if `probe_type` returns true. + /// + /// Any non-identifier characters in the type `name` will be replaced with + /// `_` in the generated config value. + pub fn emit_has_type(&self, name: &str) { + self.emit_type_cfg(name, &format!("has_{}", mangle(name))); + } + + /// Emits the given `cfg` value if `probe_type` returns true. + pub fn emit_type_cfg(&self, name: &str, cfg: &str) { + emit_possibility(cfg); + if self.probe_type(name) { + emit(cfg); + } + } + + /// Tests whether the given expression can be used. + /// + /// The test code is subject to change, but currently looks like: + /// + /// ```ignore + /// pub fn probe() { let _ = EXPR; } + /// ``` + pub fn probe_expression(&self, expr: &str) -> bool { + self.probe(format_args!("pub fn probe() {{ let _ = {}; }}", expr)) + } + + /// Emits the given `cfg` value if `probe_expression` returns true. + pub fn emit_expression_cfg(&self, expr: &str, cfg: &str) { + emit_possibility(cfg); + if self.probe_expression(expr) { + emit(cfg); + } + } + + /// Tests whether the given constant expression can be used. + /// + /// The test code is subject to change, but currently looks like: + /// + /// ```ignore + /// pub const PROBE: () = ((), EXPR).0; + /// ``` + pub fn probe_constant(&self, expr: &str) -> bool { + self.probe(format_args!("pub const PROBE: () = ((), {}).0;", expr)) + } + + /// Emits the given `cfg` value if `probe_constant` returns true. + pub fn emit_constant_cfg(&self, expr: &str, cfg: &str) { + emit_possibility(cfg); + if self.probe_constant(expr) { + emit(cfg); + } + } +} + +fn mangle(s: &str) -> String { + s.chars() + .map(|c| match c { + 'A'...'Z' | 'a'...'z' | '0'...'9' => c, + _ => '_', + }) + .collect() +} + +fn dir_contains_target( + target: &Option, + dir: &Path, + cargo_target_dir: Option, +) -> bool { + target + .as_ref() + .and_then(|target| { + dir.to_str().and_then(|dir| { + let mut cargo_target_dir = cargo_target_dir + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("target")); + cargo_target_dir.push(target); + + cargo_target_dir + .to_str() + .map(|cargo_target_dir| dir.contains(cargo_target_dir)) + }) + }) + .unwrap_or(false) +} + +fn rustflags(target: &Option, dir: &Path) -> Vec { + // Starting with rust-lang/cargo#9601, shipped in Rust 1.55, Cargo always sets + // CARGO_ENCODED_RUSTFLAGS for any host/target build script invocation. This + // includes any source of flags, whether from the environment, toml config, or + // whatever may come in the future. The value is either an empty string, or a + // list of arguments separated by the ASCII unit separator (US), 0x1f. + if let Ok(a) = env::var("CARGO_ENCODED_RUSTFLAGS") { + return if a.is_empty() { + Vec::new() + } else { + a.split('\x1f').map(str::to_string).collect() + }; + } + + // Otherwise, we have to take a more heuristic approach, and we don't + // support values from toml config at all. + // + // Cargo only applies RUSTFLAGS for building TARGET artifact in + // cross-compilation environment. Sadly, we don't have a way to detect + // when we're building HOST artifact in a cross-compilation environment, + // so for now we only apply RUSTFLAGS when cross-compiling an artifact. + // + // See https://github.com/cuviper/autocfg/pull/10#issuecomment-527575030. + if *target != env::var_os("HOST") + || dir_contains_target(target, dir, env::var_os("CARGO_TARGET_DIR")) + { + if let Ok(rustflags) = env::var("RUSTFLAGS") { + // This is meant to match how cargo handles the RUSTFLAGS environment variable. + // See https://github.com/rust-lang/cargo/blob/69aea5b6f69add7c51cca939a79644080c0b0ba0/src/cargo/core/compiler/build_context/target_info.rs#L434-L441 + return rustflags + .split(' ') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(); + } + } + + Vec::new() +} + +/// Generates a numeric ID to use in probe crate names. +/// +/// This attempts to be random, within the constraints of Rust 1.0 and no dependencies. +fn new_uuid() -> u64 { + const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; + const FNV_PRIME: u64 = 0x100_0000_01b3; + + // This set should have an actual random hasher. + let set: std::collections::HashSet = (0..256).collect(); + + // Feed the `HashSet`-shuffled order into FNV-1a. + let mut hash: u64 = FNV_OFFSET_BASIS; + for x in set { + hash = (hash ^ x).wrapping_mul(FNV_PRIME); + } + hash +} diff --git a/third_party/crates/autocfg-1.5.1/src/rustc.rs b/third_party/crates/autocfg-1.5.1/src/rustc.rs new file mode 100644 index 0000000..e324ed1 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/src/rustc.rs @@ -0,0 +1,89 @@ +use std::env; +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Command; + +use super::error::Error; +use super::version::Version; + +#[derive(Clone, Debug)] +pub struct Rustc { + rustc: PathBuf, + rustc_wrapper: Option, + rustc_workspace_wrapper: Option, +} + +impl Rustc { + pub fn new() -> Self { + Rustc { + rustc: env::var_os("RUSTC") + .unwrap_or_else(|| "rustc".into()) + .into(), + rustc_wrapper: get_rustc_wrapper(false), + rustc_workspace_wrapper: get_rustc_wrapper(true), + } + } + + /// Build the command with possible wrappers. + pub fn command(&self) -> Command { + let mut rustc = self + .rustc_wrapper + .iter() + .chain(self.rustc_workspace_wrapper.iter()) + .chain(Some(&self.rustc)); + let mut command = Command::new(rustc.next().unwrap()); + for arg in rustc { + command.arg(arg); + } + command + } + + /// Try to get the `rustc` version. + pub fn version(&self) -> Result { + // Some wrappers like clippy-driver don't pass through version commands, + // so we try to fall back to combinations without each wrapper. + macro_rules! try_version { + ($command:expr) => { + if let Ok(value) = Version::from_command($command) { + return Ok(value); + } + }; + } + + let rustc = &self.rustc; + if let Some(ref rw) = self.rustc_wrapper { + if let Some(ref rww) = self.rustc_workspace_wrapper { + try_version!(Command::new(rw).args(&[rww, rustc])); + } + try_version!(Command::new(rw).arg(rustc)); + } + if let Some(ref rww) = self.rustc_workspace_wrapper { + try_version!(Command::new(rww).arg(rustc)); + } + Version::from_command(&mut Command::new(rustc)) + } +} + +fn get_rustc_wrapper(workspace: bool) -> Option { + // We didn't really know whether the workspace wrapper is applicable until Cargo started + // deliberately setting or unsetting it in rust-lang/cargo#9601. We'll use the encoded + // rustflags as a proxy for that change for now, but we could instead check version 1.55. + if workspace && env::var_os("CARGO_ENCODED_RUSTFLAGS").is_none() { + return None; + } + + let name = if workspace { + "RUSTC_WORKSPACE_WRAPPER" + } else { + "RUSTC_WRAPPER" + }; + + if let Some(wrapper) = env::var_os(name) { + // NB: `OsStr` didn't get `len` or `is_empty` until 1.9. + if wrapper != OsString::new() { + return Some(wrapper.into()); + } + } + + None +} diff --git a/third_party/crates/autocfg-1.5.1/src/tests.rs b/third_party/crates/autocfg-1.5.1/src/tests.rs new file mode 100644 index 0000000..19188c1 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/src/tests.rs @@ -0,0 +1,54 @@ +use std::path::Path; + +#[test] +fn version_cmp() { + use super::version::Version; + let v123 = Version::new(1, 2, 3); + + assert!(Version::new(1, 0, 0) < v123); + assert!(Version::new(1, 2, 2) < v123); + assert!(Version::new(1, 2, 3) == v123); + assert!(Version::new(1, 2, 4) > v123); + assert!(Version::new(1, 10, 0) > v123); + assert!(Version::new(2, 0, 0) > v123); +} + +#[test] +fn dir_does_not_contain_target() { + assert!(!super::dir_contains_target( + &Some("x86_64-unknown-linux-gnu".into()), + Path::new("/project/target/debug/build/project-ea75983148559682/out"), + None, + )); +} + +#[test] +fn dir_does_contain_target() { + assert!(super::dir_contains_target( + &Some("x86_64-unknown-linux-gnu".into()), + Path::new( + "/project/target/x86_64-unknown-linux-gnu/debug/build/project-0147aca016480b9d/out" + ), + None, + )); +} + +#[test] +fn dir_does_not_contain_target_with_custom_target_dir() { + assert!(!super::dir_contains_target( + &Some("x86_64-unknown-linux-gnu".into()), + Path::new("/project/custom/debug/build/project-ea75983148559682/out"), + Some("custom".into()), + )); +} + +#[test] +fn dir_does_contain_target_with_custom_target_dir() { + assert!(super::dir_contains_target( + &Some("x86_64-unknown-linux-gnu".into()), + Path::new( + "/project/custom/x86_64-unknown-linux-gnu/debug/build/project-0147aca016480b9d/out" + ), + Some("custom".into()), + )); +} diff --git a/third_party/crates/autocfg-1.5.1/src/version.rs b/third_party/crates/autocfg-1.5.1/src/version.rs new file mode 100644 index 0000000..9cf5b12 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/src/version.rs @@ -0,0 +1,65 @@ +use std::process::Command; +use std::str; + +use super::{error, Error}; + +/// A version structure for making relative comparisons. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Version { + major: usize, + minor: usize, + patch: usize, +} + +impl Version { + /// Creates a `Version` instance for a specific `major.minor.patch` version. + pub fn new(major: usize, minor: usize, patch: usize) -> Self { + Version { + major: major, + minor: minor, + patch: patch, + } + } + + pub fn from_command(command: &mut Command) -> Result { + // Get rustc's verbose version + let output = try!(command + .args(&["--version", "--verbose"]) + .output() + .map_err(error::from_io)); + if !output.status.success() { + return Err(error::from_str("could not execute rustc")); + } + let output = try!(str::from_utf8(&output.stdout).map_err(error::from_utf8)); + + // Find the release line in the verbose version output. + let release = match output.lines().find(|line| line.starts_with("release: ")) { + Some(line) => &line["release: ".len()..], + None => return Err(error::from_str("could not find rustc release")), + }; + + // Strip off any extra channel info, e.g. "-beta.N", "-nightly" + let version = match release.find('-') { + Some(i) => &release[..i], + None => release, + }; + + // Split the version into semver components. + let mut iter = version.splitn(3, '.'); + let major = try!(iter + .next() + .ok_or_else(|| error::from_str("missing major version"))); + let minor = try!(iter + .next() + .ok_or_else(|| error::from_str("missing minor version"))); + let patch = try!(iter + .next() + .ok_or_else(|| error::from_str("missing patch version"))); + + Ok(Version::new( + try!(major.parse().map_err(error::from_num)), + try!(minor.parse().map_err(error::from_num)), + try!(patch.parse().map_err(error::from_num)), + )) + } +} diff --git a/third_party/crates/autocfg-1.5.1/tests/no_std.rs b/third_party/crates/autocfg-1.5.1/tests/no_std.rs new file mode 100644 index 0000000..05ab445 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/tests/no_std.rs @@ -0,0 +1,28 @@ +extern crate autocfg; + +use std::env; + +mod support; + +/// Tests that we can control the use of `#![no_std]`. +#[test] +fn no_std() { + // Clear the CI `TARGET`, if any, so we're just dealing with the + // host target which always has `std` available. + env::remove_var("TARGET"); + + // Use the same path as this test binary. + let out = support::out_dir(); + + let mut ac = autocfg::AutoCfg::with_dir(out.as_ref()).unwrap(); + assert!(!ac.no_std()); + assert!(ac.probe_path("std::mem")); + + // `#![no_std]` was stabilized in Rust 1.6 + if ac.probe_rustc_version(1, 6) { + ac.set_no_std(true); + assert!(ac.no_std()); + assert!(!ac.probe_path("std::mem")); + assert!(ac.probe_path("core::mem")); + } +} diff --git a/third_party/crates/autocfg-1.5.1/tests/rustflags.rs b/third_party/crates/autocfg-1.5.1/tests/rustflags.rs new file mode 100644 index 0000000..8038a0c --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/tests/rustflags.rs @@ -0,0 +1,34 @@ +extern crate autocfg; + +use std::env; + +mod support; + +/// Tests that autocfg uses the RUSTFLAGS or CARGO_ENCODED_RUSTFLAGS +/// environment variables when running rustc. +#[test] +fn test_with_sysroot() { + let dir = support::exe_dir(); + let out = support::out_dir(); + + // If we have encoded rustflags, they take precedence, even if empty. + env::set_var("CARGO_ENCODED_RUSTFLAGS", ""); + env::set_var("RUSTFLAGS", &format!("-L {}", dir.display())); + let ac = autocfg::AutoCfg::with_dir(out.as_ref()).unwrap(); + assert!(ac.probe_sysroot_crate("std")); + assert!(!ac.probe_sysroot_crate("autocfg")); + + // Now try again with useful encoded args. + env::set_var( + "CARGO_ENCODED_RUSTFLAGS", + &format!("-L\x1f{}", dir.display()), + ); + let ac = autocfg::AutoCfg::with_dir(out.as_ref()).unwrap(); + assert!(ac.probe_sysroot_crate("autocfg")); + + // Try the old-style RUSTFLAGS, ensuring HOST != TARGET. + env::remove_var("CARGO_ENCODED_RUSTFLAGS"); + env::set_var("HOST", "lol"); + let ac = autocfg::AutoCfg::with_dir(out.as_ref()).unwrap(); + assert!(ac.probe_sysroot_crate("autocfg")); +} diff --git a/third_party/crates/autocfg-1.5.1/tests/support/mod.rs b/third_party/crates/autocfg-1.5.1/tests/support/mod.rs new file mode 100644 index 0000000..f50e717 --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/tests/support/mod.rs @@ -0,0 +1,21 @@ +use std::borrow::Cow; +use std::env; +use std::path::{Path, PathBuf}; + +/// The directory containing this test binary. +pub fn exe_dir() -> PathBuf { + let exe = env::current_exe().unwrap(); + exe.parent().unwrap().to_path_buf() +} + +/// The directory to use for test probes. +pub fn out_dir() -> Cow<'static, Path> { + if let Some(tmpdir) = option_env!("CARGO_TARGET_TMPDIR") { + Cow::Borrowed(tmpdir.as_ref()) + } else if let Some(tmpdir) = env::var_os("TESTS_TARGET_DIR") { + Cow::Owned(tmpdir.into()) + } else { + // Use the same path as this test binary. + Cow::Owned(exe_dir()) + } +} diff --git a/third_party/crates/autocfg-1.5.1/tests/tests.rs b/third_party/crates/autocfg-1.5.1/tests/tests.rs new file mode 100644 index 0000000..bd069ee --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/tests/tests.rs @@ -0,0 +1,189 @@ +extern crate autocfg; + +use autocfg::AutoCfg; + +mod support; + +fn core_std(ac: &AutoCfg, path: &str) -> String { + let krate = if ac.no_std() { "core" } else { "std" }; + format!("{}::{}", krate, path) +} + +fn assert_std(ac: &AutoCfg, probe_result: bool) { + assert_eq!(!ac.no_std(), probe_result); +} + +fn assert_min(ac: &AutoCfg, major: usize, minor: usize, probe_result: bool) { + assert_eq!(ac.probe_rustc_version(major, minor), probe_result); +} + +fn autocfg_for_test() -> AutoCfg { + AutoCfg::with_dir(support::out_dir().as_ref()).unwrap() +} + +#[test] +fn autocfg_version() { + let ac = autocfg_for_test(); + assert!(ac.probe_rustc_version(1, 0)); +} + +#[test] +fn probe_add() { + let ac = autocfg_for_test(); + let add = core_std(&ac, "ops::Add"); + let add_rhs = add.clone() + ""; + let add_rhs_output = add.clone() + ""; + let dyn_add_rhs_output = "dyn ".to_string() + &*add_rhs_output; + assert!(ac.probe_path(&add)); + assert!(ac.probe_trait(&add)); + assert!(ac.probe_trait(&add_rhs)); + assert!(ac.probe_trait(&add_rhs_output)); + assert_min(&ac, 1, 27, ac.probe_type(&dyn_add_rhs_output)); +} + +#[test] +fn probe_as_ref() { + let ac = autocfg_for_test(); + let as_ref = core_std(&ac, "convert::AsRef"); + let as_ref_str = as_ref.clone() + ""; + let dyn_as_ref_str = "dyn ".to_string() + &*as_ref_str; + assert!(ac.probe_path(&as_ref)); + assert!(ac.probe_trait(&as_ref_str)); + assert!(ac.probe_type(&as_ref_str)); + assert_min(&ac, 1, 27, ac.probe_type(&dyn_as_ref_str)); +} + +#[test] +fn probe_i128() { + let ac = autocfg_for_test(); + let i128_path = core_std(&ac, "i128"); + assert_min(&ac, 1, 26, ac.probe_path(&i128_path)); + assert_min(&ac, 1, 26, ac.probe_type("i128")); +} + +#[test] +fn probe_sum() { + let ac = autocfg_for_test(); + let sum = core_std(&ac, "iter::Sum"); + let sum_i32 = sum.clone() + ""; + let dyn_sum_i32 = "dyn ".to_string() + &*sum_i32; + assert_min(&ac, 1, 12, ac.probe_path(&sum)); + assert_min(&ac, 1, 12, ac.probe_trait(&sum)); + assert_min(&ac, 1, 12, ac.probe_trait(&sum_i32)); + assert_min(&ac, 1, 12, ac.probe_type(&sum_i32)); + assert_min(&ac, 1, 27, ac.probe_type(&dyn_sum_i32)); +} + +#[test] +fn probe_std() { + let ac = autocfg_for_test(); + assert_std(&ac, ac.probe_sysroot_crate("std")); +} + +#[test] +fn probe_alloc() { + let ac = autocfg_for_test(); + assert_min(&ac, 1, 36, ac.probe_sysroot_crate("alloc")); +} + +#[test] +fn probe_bad_sysroot_crate() { + let ac = autocfg_for_test(); + assert!(!ac.probe_sysroot_crate("doesnt_exist")); +} + +#[test] +fn probe_no_std() { + let ac = autocfg_for_test(); + assert!(ac.probe_type("i32")); + assert!(ac.probe_type("[i32]")); + assert_std(&ac, ac.probe_type("Vec")); +} + +#[test] +fn probe_expression() { + let ac = autocfg_for_test(); + assert!(ac.probe_expression(r#""test".trim_left()"#)); + assert_min(&ac, 1, 30, ac.probe_expression(r#""test".trim_start()"#)); + assert_std(&ac, ac.probe_expression("[1, 2, 3].to_vec()")); +} + +#[test] +fn probe_constant() { + let ac = autocfg_for_test(); + assert!(ac.probe_constant("1 + 2 + 3")); + assert_min( + &ac, + 1, + 33, + ac.probe_constant("{ let x = 1 + 2 + 3; x * x }"), + ); + assert_min(&ac, 1, 39, ac.probe_constant(r#""test".len()"#)); +} + +#[test] +fn probe_raw() { + let ac = autocfg_for_test(); + let prefix = if ac.no_std() { "#![no_std]\n" } else { "" }; + let f = |s| format!("{}{}", prefix, s); + + // This attribute **must** be used at the crate level. + assert!(ac.probe_raw(&f("#![no_builtins]")).is_ok()); + + assert!(ac.probe_raw(&f("#![deny(dead_code)] fn x() {}")).is_err()); + assert!(ac.probe_raw(&f("#![allow(dead_code)] fn x() {}")).is_ok()); + assert!(ac + .probe_raw(&f("#![deny(dead_code)] pub fn x() {}")) + .is_ok()); +} + +#[test] +fn probe_cleanup() { + let dir = support::out_dir().join("autocfg_test_probe_cleanup"); + std::fs::create_dir(&dir).unwrap(); + + let ac = AutoCfg::with_dir(&dir).unwrap(); + assert!(ac.probe_type("i32")); + + // NB: this is not `remove_dir_all`, so it will only work if the directory + // is empty -- i.e. the probe should have removed any output files. + std::fs::remove_dir(&dir).unwrap(); +} + +#[test] +fn editions() { + let mut ac = autocfg_for_test(); + assert!(ac.edition().is_none()); + assert!(ac.probe_raw("").is_ok()); + + for (edition, minor) in vec![(2015, 27), (2018, 31), (2021, 56), (2024, 85)] { + let edition = edition.to_string(); + ac.set_edition(Some(edition.clone())); + assert_eq!(ac.edition(), Some(&*edition)); + assert_min(&ac, 1, minor, ac.probe_raw("").is_ok()); + } + + ac.set_edition(Some("invalid".into())); + assert_eq!(ac.edition(), Some("invalid")); + assert!(ac.probe_raw("").is_err()); + + ac.set_edition(None); + assert!(ac.edition().is_none()); + assert!(ac.probe_raw("").is_ok()); +} + +#[test] +fn edition_keyword_try() { + let mut ac = autocfg_for_test(); + + if ac.probe_rustc_version(1, 27) { + ac.set_edition(Some(2015.to_string())); + } + assert!(ac.probe_expression("{ let try = 0; try }")); + + if ac.probe_rustc_version(1, 31) { + ac.set_edition(Some(2018.to_string())); + assert!(!ac.probe_expression("{ let try = 0; try }")); + assert!(ac.probe_expression("{ let r#try = 0; r#try }")); + } +} diff --git a/third_party/crates/autocfg-1.5.1/tests/wrappers.rs b/third_party/crates/autocfg-1.5.1/tests/wrappers.rs new file mode 100644 index 0000000..69a64dd --- /dev/null +++ b/third_party/crates/autocfg-1.5.1/tests/wrappers.rs @@ -0,0 +1,83 @@ +extern crate autocfg; + +use std::env; +use std::io; + +mod support; + +fn main() { + let args: Vec<_> = env::args().skip(1).collect(); + if args.is_empty() { + // With no arguments, this must be a (harness-free) test run. + // (we're using system binaries as wrappers, so ignore other platforms) + if cfg!(unix) { + test_wrappers(); + } + } else { + // Otherwise we're acting like a no-op rustc wrapper, ignoring almost everything. + for arg in args { + match &*arg { + // Add our own version so we can check that the wrapper is used for that. + "--version" => println!("release: 12345.6789.0"), + + // Read all input so the writer doesn't get EPIPE when we exit. + "-" => { + let stdin = io::stdin(); + let _ = io::copy(&mut stdin.lock(), &mut io::sink()); + } + + _ => {} + } + } + } +} + +/// Tests that autocfg uses the RUSTC_WRAPPER and/or RUSTC_WORKSPACE_WRAPPER +/// environment variables when running rustc. +fn test_wrappers() { + fn set(name: &str, value: Option) { + match value { + Some(true) => env::set_var(name, "/usr/bin/env"), + Some(false) => env::set_var(name, "/bin/false"), + None => env::remove_var(name), + } + } + + let out = support::out_dir(); + + // This is used as a heuristic to detect rust-lang/cargo#9601. + env::set_var("CARGO_ENCODED_RUSTFLAGS", ""); + + // No wrapper, a good pass-through wrapper, and a bad wrapper. + let variants = [None, Some(true), Some(false)]; + + for &workspace in &variants { + for &rustc in &variants { + set("RUSTC_WRAPPER", rustc); + set("RUSTC_WORKSPACE_WRAPPER", workspace); + + let ac = autocfg::AutoCfg::with_dir(out.as_ref()).unwrap(); + if rustc == Some(false) || workspace == Some(false) { + // Everything should fail with bad wrappers. + assert!(!ac.probe_type("usize")); + } else { + // Try known good and bad types for the wrapped rustc. + assert!(ac.probe_type("usize")); + assert!(!ac.probe_type("mesize")); + } + // Either way, we should have found the inner rustc version. + assert!(ac.probe_rustc_version(1, 0)); + } + } + + // Finally, make sure that `RUSTC_WRAPPER` is applied outermost + // by using something (ourself) that doesn't pass through at all. + let wrap_ignored = env::current_exe().unwrap(); + env::set_var("RUSTC_WRAPPER", &wrap_ignored); + env::set_var("RUSTC_WORKSPACE_WRAPPER", "/bin/false"); + let ac = autocfg::AutoCfg::with_dir(out.as_ref()).unwrap(); + assert!(ac.probe_type("mesize")); // anything goes! + + // Make sure we also got the version from that wrapper. + assert!(ac.probe_rustc_version(12345, 6789)); +} diff --git a/third_party/crates/crates.bzl b/third_party/crates/crates.bzl index 69d9612..b6d2518 100644 --- a/third_party/crates/crates.bzl +++ b/third_party/crates/crates.bzl @@ -341,6 +341,7 @@ _NORMAL_DEPENDENCIES = { _COMMON_CONDITION: { "clap": Label("//third_party/crates/clap-4.6.7:clap"), "rnix": Label("//third_party/crates/rnix-0.14.0:rnix"), + "rowan": Label("//third_party/crates/rowan-0.16.1:rowan"), }, }, } @@ -378,6 +379,9 @@ _NORMAL_DEV_DEPENDENCIES = { "src/status": { }, "src/xatu": { + _COMMON_CONDITION: { + "googletest": Label("//third_party/crates/googletest-0.14.3:googletest"), + }, }, } @@ -393,6 +397,8 @@ _NORMAL_DEV_ALIASES = { "src/status": { }, "src/xatu": { + _COMMON_CONDITION: { + }, }, } @@ -453,6 +459,8 @@ _PROC_MACRO_DEV_ALIASES = { "src/status": { }, "src/xatu": { + _COMMON_CONDITION: { + }, }, } diff --git a/third_party/crates/googletest-0.14.3/.cargo-checksum.json b/third_party/crates/googletest-0.14.3/.cargo-checksum.json new file mode 100644 index 0000000..b22446f --- /dev/null +++ b/third_party/crates/googletest-0.14.3/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"5719bd21217b0318653f958a7d307c4886d996625066ffea05b812601f0038f2","Cargo.lock":"1b2abac31ec1a7e8c90c938198866ff366675ada6eaa953544701ee9a146e4f5","Cargo.toml":"920bb8a199339a433626c2868c34f159187ea01dd106e27123e580685f62a422","Cargo.toml.orig":"4e8666c56120ab1c50436268e4766db70038d038173cd95a49ab1f97fc85dcda","LICENSE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","README.md":"d0b43ef465e4283f2837ef1f1f5ea5707aec9f7c94efe5a0b7d44c455400a687","crate_docs.md":"7bd062a021f0980dd4b55298cf7374fc11ec5d92fe6911a636b3365e95b26fcb","src/assertions.rs":"1caa262e4e45cce3adaa5cc4805e99439df64ee83372df446676aeebbc4f99d9","src/description.rs":"2a65872f0d5e89ef6bd3eb1b9cafbd8e0908b29fd922f0244d1a7a706fc38b8d","src/fixtures.rs":"6c06200f1cbae4ad9b49f543985d8936bede7b6c7606e794ab7ae70b6b346e80","src/fmt.rs":"94461bd1d5260644045acba95050f577b2b1b106968d9e2f2876ea2a4b446081","src/internal/description_renderer.rs":"a79e316a2b13c1e7c33920c7fe3e36f0857564d0d004b820b456d4b9a60ad16c","src/internal/glob.rs":"c8dbf5bbaf11c2a5087ee8175177c572cdccc29dff74b61ec89154332274e67f","src/internal/mod.rs":"1a704f723b97a16ff66bd53a5085ee8844377f73082cbc0c8a73506f7616a3a9","src/internal/scoped_trace.rs":"013318c0d8aebaa0c1dd86b571159f0c8a3ce16b7125a1a62ddd5932a19ffc05","src/internal/test_data.rs":"f84155d0a936cc4ba9645d6b50bd80e1304dd39723e40e28717f8374d2b59731","src/internal/test_filter.rs":"551faba774f1425694a82532d59a3168adb52298ee637b2756c1c74b3eb6cf1c","src/internal/test_outcome.rs":"dd49864d6e3fe3fff7e931fa05cab458f52fdf6fec3e02dd7212648a7fcbeea1","src/internal/test_sharding.rs":"b3f86d5d6776d30d62a18f288c399b76f3f2e8aac58821a6ca458d062a9d29d1","src/lib.rs":"49a7abc13d1b922b4bc3b6997709b59be1ef1739a88bb9a34d3e25c918722bc2","src/matcher.rs":"b63b50e7ecc4fc852ad27e0dedb4add0f3274a8760720269a86c70e78df77f4d","src/matcher_support/auto_eq.rs":"8cf337d9382d08df9f32a1ff5912895b6be170d16f956a7b819ee143c4bdf913","src/matcher_support/count_elements.rs":"4ef9b2ec6b2e05efb5c71dd24c643c3eaebbd8124d5f254bec1cd6f90a9507ab","src/matcher_support/edit_distance.rs":"23a734285ba3b298b65a219a74cca46a11c6096cc9aa0302b4768c79baee3d5e","src/matcher_support/match_matrix.rs":"1b414a154a8700d98595e3c798dbbde776dbfdfdfd0982a0086b4591287894bb","src/matcher_support/mod.rs":"ea00e7ed89695bbe435ef7637c2e19cb0a0bf13add3facfbbeabcf6cf2b1bd30","src/matcher_support/summarize_diff.rs":"2a54e6a91d77e09bd6f92208ce63409682725a78ca64c1e78b060c158afc3dac","src/matcher_support/zipped_iterator.rs":"ac1a1a54f866f977255403a4f7e7613a4116af163326029c54b65ca1ac57b538","src/matchers/all_matcher.rs":"c209faf3376d1280e80ef144607eb9dfa98ab9e034f82054ddbc7895c187e564","src/matchers/any_matcher.rs":"651875713b60bf799a8b6c6e060d159cf76d532dc289c53922cb8bca77b25119","src/matchers/anything_matcher.rs":"0f68327982c59d37c197d5e052790143724874e2ff9afa33f0b88c545706bb2c","src/matchers/bool_matcher.rs":"0917c57c952fe4de4f097933599ea5e69df7b36cb90c3a01d149d4ff214407ea","src/matchers/char_count_matcher.rs":"33e06cbcba56b1c37b5a0cd343dd847a0f1b6b1d3eed8266b653b8e4746a4c0f","src/matchers/conjunction_matcher.rs":"4fdb8ca6438b0bebd917da77263c8b08a4b994cd1858669e1bd8c2dd30268c9f","src/matchers/container_eq_matcher.rs":"3e0ebaafe0ef664a2bae48a46afb3fdbfa56cef519a8b7343437f6df1d5355ca","src/matchers/contains_matcher.rs":"964344d7ffc6b3d72ab62ba6ae088d9d1fc424a8a32f35f4e9bcde2249260a43","src/matchers/contains_regex_matcher.rs":"c35011b6f928480fc8c2a7c454ac8ae236631bcfdb87022e50ee861146fd9fbd","src/matchers/derefs_to_matcher.rs":"83b8a850a552becf981f42f77cee92c5f13962f974d25300c51caf2097420705","src/matchers/disjunction_matcher.rs":"ead7103fe8aeb1d29cca0346b6ee31920ace03e02a85cd299012b41365fb3ad3","src/matchers/display_matcher.rs":"bc8c92a598a9be81c4d1e576b5dea2fe1a953c4d78152a83508971cf7ce3738c","src/matchers/each_matcher.rs":"61220d1d7160aba20a471c76a60d3e4687488bbbbfd299d1e9acfb9be61a8194","src/matchers/elements_are_matcher.rs":"53030dd963b3c6cdb6c2285c60d5f54cefaa6c2d2e23c1942e7175ca3ab002a2","src/matchers/empty_matcher.rs":"d117eb481acf7f691a9eb5397ab9a26ecda95bd5b1471a072f3a9a8606d291ab","src/matchers/eq_matcher.rs":"f35c423dda202f546465217d444080ece6b687277c0b36af1e7bfbeab39e63b2","src/matchers/err_matcher.rs":"7d3f0b73a7729126d21ff590229f1aeabdbe5cab7ae1e0fe82c9f81aa3cb6fb0","src/matchers/field_matcher.rs":"fceddd5152e836a47881517a44fc5b0fe7c3c0ee6c6274d4a30ec024f7b74fdf","src/matchers/ge_matcher.rs":"5328c3180dfae507784140d8697d9251f063e0d18325cb77fd099dd37ec9d7ab","src/matchers/gt_matcher.rs":"b65d8dd38d5350435b8ffb265a26719b1699403271a599be79f3639308f9aac7","src/matchers/has_entry_matcher.rs":"1e2e82a18d1d5ff2e7e5e41afbe7f98efe08305e26b31a6e99186cfc4a4b1230","src/matchers/is_encoded_string_matcher.rs":"11282de32140638dd086f206382e1e8119611ebae4e4f320fb8952829d5c4e30","src/matchers/is_finite_matcher.rs":"1b644ad6b6a4428b41bf504f67eebb7395cf89293db5bf7dfa7a3f1d9e718a6e","src/matchers/is_infinite_matcher.rs":"1d8e46b1682829c6e8be255fd0707f39c2ecddd50b64b04ccf80e6793d888d12","src/matchers/is_matcher.rs":"7d1314a0e8c1cacbc5f359a9a2528fe4ee9edd0a10fbe974aa3e8da9f589dc28","src/matchers/is_nan_matcher.rs":"c759904a8cc1ec1258ae17aa8aafe93204402bea082bf535e90061c718ce296c","src/matchers/le_matcher.rs":"febacd9c5a283b7342aca82d383ebd6f9226afbb0ccf49ba732c661b88816d0b","src/matchers/len_matcher.rs":"33a66a238cb3e750bff0d77f717f8e19e8f81c9b249a8e43dd127c2df5ec2481","src/matchers/lt_matcher.rs":"ceb86f0fe5a2db0648aca7e5aeda72e8945023c493f57fbb05528861f85ee29a","src/matchers/matches_pattern.rs":"fe94f2789b76b81a19675460ed8a972a3977b9a3853d8231efe8609e9854c3d5","src/matchers/matches_regex_matcher.rs":"8a18f5e2d4446e31d8eb1c70a69f00b370478fd1d61ddce0217bbaa82c5f39e6","src/matchers/mod.rs":"1dc57f589184d70e70062d6f3e53c9b85af58970ca71abf69d78ce0712e01c79","src/matchers/ne_matcher.rs":"6390dbe8fbd30eb1848810a4de7715e061a862f6e93be72886f16212ba97a956","src/matchers/near_matcher.rs":"793e9e6d0acdf6a2a54d1ddb8ce60a7bb0e9fbee701d176ce3850c3fbb95327e","src/matchers/none_matcher.rs":"e81bc8422c98937f9712d22f29d5efa70e1036cd5325ad2588315fa2e3dcd629","src/matchers/not_matcher.rs":"a7f11381bc16b0e3509c91a96b8de849aeb59225435744e526752fceb796b4df","src/matchers/ok_matcher.rs":"76dee4c0207d7806eaa4485807162150aa2652aa60f667f956ce852f9b6332d3","src/matchers/points_to_matcher.rs":"9cbbe183b8433c56cfe5bce271d37c704fe37a713868432d4273f5cfab113b41","src/matchers/pointwise_matcher.rs":"3b2ed83dcd216c82063697e03bad998e9c6f2d93f0c18c8edb847b1ceeea2ff4","src/matchers/predicate_matcher.rs":"083ab17b65474f246baabb88ce76108ba301ff61bdb787d4dbd8d57ca5ea04e8","src/matchers/property_matcher.rs":"a247a0a1132428d66e45679e7a6a65e0e119144335bfea65b1b11fbbb1a91b1a","src/matchers/result_of_matcher.rs":"c079cce38eb9dc5af70c4531d3c98f40b25f4b3633a85b252665636bf3a795e2","src/matchers/some_matcher.rs":"042eb1637d01f70bae2c7795efc5c26df90ea73db40326752895b5780ab9838f","src/matchers/str_matcher.rs":"09e895374d22e3fe4e102a58f8821a1aaac6d5586547a134423079fb99f19de2","src/matchers/subset_of_matcher.rs":"9df35fa70c26fb974e8746900e282d34cb1e69d87256546ce37cd8b073c8d469","src/matchers/superset_of_matcher.rs":"f6570c8ab8d215328a53df97143f9c8e3071b8ba8322322139597568df760047","src/matchers/tuple_matcher.rs":"d00eb6fecd1d5671418e3bbe62c67f422b6a945ee1276e16edc1402c8e4f2edb","src/matchers/unordered_elements_are_matcher.rs":"7b81f3f036ddc03dccb33626148cdc2e4a4ea44b5f66c9e5aa87a67f4e875407","tests/all_matcher_test.rs":"5da33fcc13c47c2c1476877595d1832d2156becb8e463db662f1a2d5ac25df63","tests/any_matcher_test.rs":"04e8bc3402a012b55532edacfb10f392d928ab7ebd1749d97327c4bc5e1c2be7","tests/assertions_test.rs":"243c8fb089d091c9f159324840a542866b1b1323c1859ec7e2d1fca82fe2247f","tests/colorized_diff_test.rs":"105e7f67e4ee45071ca45faf497cbb6dc083a0498e093a9a1cc48914c935f88f","tests/composition_test.rs":"d845f2a4dfc5f289ef29251f921ca870d8e3e5f53964e405a5d805f9b3715dac","tests/elements_are_matcher_test.rs":"fa30f9b4901c929b7945578ca9298459805a9ca35d5f9fe8e98d006f2889ab6e","tests/field_matcher_test.rs":"d93f6cf67766f5aedb103d45de4c903a70928eff0bf9cbc75f4a9b10505b9142","tests/fmt_test.rs":"049211dfb8898d74705dfe7c597e9b03797f022a29baeb6ff54bfbef5407e728","tests/matches_pattern_enum_test.rs":"3a94738f87a707ac3f06a446068c1c770a1ecda25bd6a535eb912122c4a7b91d","tests/matches_pattern_struct_and_enum_test.rs":"06d8b8687d4dcb1c723c1d0b419ad576224e86c4d2ff3665666ff029be221ecb","tests/matches_pattern_struct_test.rs":"f4f762d8d5a21af3da20e75ef76486d5c7acb228eff9e73e21f01ca9ca12bcad","tests/matches_pattern_tuple_struct_test.rs":"ab5012b66137289f1a09894624844eaf270da6e349634b71586c8851ced05aec","tests/no_color_test.rs":"09ea645852d0708b02d64636d5dd38ca3a011dddc4e9b83ac8069215cd23f8e4","tests/no_std_test.rs":"51803f127f529597f249172f73b24cdd7467035e9004c8b343b1ceaa668bee60","tests/pointwise_matcher_test.rs":"66cc117dd5d1d163f1f2d84325065f4f40dd1320ff9ac2db310e00a09522d908","tests/property_matcher_test.rs":"fbf3780bfd3640b26e532c50de1777ec84bc9b4296ce18b2a654b384566cff3b","tests/proptest_integration_test.rs":"872f0a6030945433cab8c72479f791d597f175dce907507103ae2698f443b6ca","tests/reference_type_test.rs":"75fcc3085aff3a2e753eb268625b27738b04de7ccead993279529cb0090b9456","tests/scoped_trace_panic_test.golden":"aa60a14182f9019ad04a58f5c68449b5ab12e2645f517896f8a13809982bbf1b","tests/scoped_trace_panic_test.rs":"98cf4a939c713f4a73299cca62a860f890ec2e348725fdff8acffca14e5dfe9e","tests/scoped_trace_panic_test.sh":"62608f9b07abcc8d8f0c39051eb028f1a51ba9520ce27ad3fda758e318158f2d","tests/scoped_trace_test.rs":"4bb1e7e552b438a80ed48646ee11521ce3c665756df247be2f9195ba0bfa51e3","tests/tuple_matcher_test.rs":"ce60ffca6e66dcdda96768863b8f72f6c231b080c37a1bd9838cfc9ad2a40f90","tests/unordered_elements_are_matcher_test.rs":"60e90a96a242092a71960c4038ab5c78e7c5ac73b788842bc94f23a14abcd7a2"},"package":"f6b5e2f2b556b7b90297a5a35c8267dd43a537923d2b329beefdba2b4ec19d94"} \ No newline at end of file diff --git a/third_party/crates/googletest-0.14.3/.cargo_vcs_info.json b/third_party/crates/googletest-0.14.3/.cargo_vcs_info.json new file mode 100644 index 0000000..75883ec --- /dev/null +++ b/third_party/crates/googletest-0.14.3/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "871f28b74e91e1d2a38e139ea03a9809d9d85f10" + }, + "path_in_vcs": "googletest" +} \ No newline at end of file diff --git a/third_party/crates/googletest-0.14.3/BUILD.bazel b/third_party/crates/googletest-0.14.3/BUILD.bazel new file mode 100644 index 0000000..a8f48f4 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/BUILD.bazel @@ -0,0 +1,118 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_library( + name = "googletest", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_root = "src/lib.rs", + edition = "2021", + proc_macro_deps = [ + "//third_party/crates/googletest_macro-0.14.3:googletest_macro", + "//third_party/crates/rustversion-1.0.23:rustversion", + ], + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=googletest", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.14.3", + deps = [ + "//third_party/crates/num-traits-0.2.19:num_traits", + "//third_party/crates/regex-1.13.1:regex", + ], +) diff --git a/third_party/crates/googletest-0.14.3/Cargo.lock b/third_party/crates/googletest-0.14.3/Cargo.lock new file mode 100644 index 0000000..b402788 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/Cargo.lock @@ -0,0 +1,527 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "env_logger" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets", +] + +[[package]] +name = "googletest" +version = "0.14.3" +dependencies = [ + "anyhow", + "googletest_macro", + "indoc", + "num-traits", + "proptest", + "quickcheck", + "regex", + "rustversion", +] + +[[package]] +name = "googletest_macro" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae6abc96141edd26bf5aeec0f119c129c44de3ced09e5073711a02cb74725d0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "libc" +version = "0.2.178" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "log" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand 0.9.1", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quickcheck" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +dependencies = [ + "env_logger", + "log", + "rand 0.8.5", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.3", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom 0.3.1", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/third_party/crates/googletest-0.14.3/Cargo.toml b/third_party/crates/googletest-0.14.3/Cargo.toml new file mode 100644 index 0000000..62b6dc7 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/Cargo.toml @@ -0,0 +1,160 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.85.0" +name = "googletest" +version = "0.14.3" +authors = [ + "Bradford Hovinen ", + "Bastien Jacot-Guillarmod ", + "Maciej Pietrzak ", + "Martin Geisler ", +] +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "A rich assertion and matcher library inspired by GoogleTest for C++" +readme = "README.md" +keywords = [ + "unit", + "matcher", + "testing", + "assertions", +] +categories = [ + "development-tools", + "development-tools::testing", +] +license = "Apache-2.0" +repository = "https://github.com/google/googletest-rust" + +[lib] +name = "googletest" +path = "src/lib.rs" + +[[test]] +name = "all_matcher_test" +path = "tests/all_matcher_test.rs" + +[[test]] +name = "any_matcher_test" +path = "tests/any_matcher_test.rs" + +[[test]] +name = "assertions_test" +path = "tests/assertions_test.rs" + +[[test]] +name = "colorized_diff_test" +path = "tests/colorized_diff_test.rs" + +[[test]] +name = "composition_test" +path = "tests/composition_test.rs" + +[[test]] +name = "elements_are_matcher_test" +path = "tests/elements_are_matcher_test.rs" + +[[test]] +name = "field_matcher_test" +path = "tests/field_matcher_test.rs" + +[[test]] +name = "fmt_test" +path = "tests/fmt_test.rs" + +[[test]] +name = "matches_pattern_enum_test" +path = "tests/matches_pattern_enum_test.rs" + +[[test]] +name = "matches_pattern_struct_and_enum_test" +path = "tests/matches_pattern_struct_and_enum_test.rs" + +[[test]] +name = "matches_pattern_struct_test" +path = "tests/matches_pattern_struct_test.rs" + +[[test]] +name = "matches_pattern_tuple_struct_test" +path = "tests/matches_pattern_tuple_struct_test.rs" + +[[test]] +name = "no_color_test" +path = "tests/no_color_test.rs" + +[[test]] +name = "no_std_test" +path = "tests/no_std_test.rs" + +[[test]] +name = "pointwise_matcher_test" +path = "tests/pointwise_matcher_test.rs" + +[[test]] +name = "property_matcher_test" +path = "tests/property_matcher_test.rs" + +[[test]] +name = "proptest_integration_test" +path = "tests/proptest_integration_test.rs" + +[[test]] +name = "reference_type_test" +path = "tests/reference_type_test.rs" + +[[test]] +name = "scoped_trace_panic_test" +path = "tests/scoped_trace_panic_test.rs" + +[[test]] +name = "scoped_trace_test" +path = "tests/scoped_trace_test.rs" + +[[test]] +name = "tuple_matcher_test" +path = "tests/tuple_matcher_test.rs" + +[[test]] +name = "unordered_elements_are_matcher_test" +path = "tests/unordered_elements_are_matcher_test.rs" + +[dependencies.anyhow] +version = "1" +optional = true + +[dependencies.googletest_macro] +version = "0.14.3" + +[dependencies.num-traits] +version = "0.2.17" + +[dependencies.proptest] +version = "1.9.0" +optional = true + +[dependencies.regex] +version = "1.12.2" + +[dependencies.rustversion] +version = "1.0.22" + +[dev-dependencies.indoc] +version = "2" + +[dev-dependencies.quickcheck] +version = "1.0.3" diff --git a/third_party/crates/googletest-0.14.3/Cargo.toml.orig b/third_party/crates/googletest-0.14.3/Cargo.toml.orig new file mode 100644 index 0000000..31507b1 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/Cargo.toml.orig @@ -0,0 +1,43 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "googletest" +version = "0.14.3" +keywords = ["unit", "matcher", "testing", "assertions"] +categories = ["development-tools", "development-tools::testing"] +description = "A rich assertion and matcher library inspired by GoogleTest for C++" +repository = "https://github.com/google/googletest-rust" +readme = "../README.md" +license = "Apache-2.0" +edition = "2021" +rust-version = "1.85.0" # Needed for async closures. +authors = [ + "Bradford Hovinen ", + "Bastien Jacot-Guillarmod ", + "Maciej Pietrzak ", + "Martin Geisler ", +] + +[dependencies] +googletest_macro = { path = "../googletest_macro", version = "0.14.3" } +anyhow = { version = "1", optional = true } +num-traits = "0.2.17" +proptest = { version = "1.9.0", optional = true } +regex = "1.12.2" +rustversion = "1.0.22" + +[dev-dependencies] +indoc = "2" +quickcheck = "1.0.3" diff --git a/third_party/crates/googletest-0.14.3/LICENSE b/third_party/crates/googletest-0.14.3/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/crates/googletest-0.14.3/README.md b/third_party/crates/googletest-0.14.3/README.md new file mode 100644 index 0000000..e832878 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/README.md @@ -0,0 +1,367 @@ +# GoogleTest Rust + +[![crates.io][crates-badge]][crates-url] +[![docs.rs][docs-badge]][docs-url] +[![Apache licensed][license-badge]][license-url] +[![Build Status][actions-badge]][actions-url] +[![OpenSSF Best Practices][openssf-badge]][openssf-url] + +[crates-badge]: https://img.shields.io/crates/v/googletest.svg +[crates-url]: https://crates.io/crates/googletest +[docs-badge]: https://img.shields.io/badge/docs.rs-googletest-66c2a5 +[docs-url]: https://docs.rs/googletest/*/googletest/ +[license-badge]: https://img.shields.io/badge/license-Apache-blue.svg +[license-url]: https://github.com/google/googletest-rust/blob/main/LICENSE +[actions-badge]: https://github.com/google/googletest-rust/workflows/CI/badge.svg +[actions-url]: https://github.com/google/googletest-rust/actions?query=workflow%3ACI+branch%3Amain +[openssf-badge]: https://www.bestpractices.dev/projects/10037/badge +[openssf-url]: https://www.bestpractices.dev/projects/10037 + +This library brings the rich assertion types of Google's C++ testing library +[GoogleTest](https://github.com/google/googletest) to Rust. It provides: + + * A framework for writing matchers which can be combined to make a wide range + of assertions on data, + * A rich set of matchers providing similar functionality to those included in + [GoogleTest](https://google.github.io/googletest/reference/matchers.html), + and + * A new set of assertion macros offering similar functionality to those of + [GoogleTest](https://google.github.io/googletest/primer.html#assertions). + +**The minimum supported Rust version is 1.85**. + +> :warning: The API is not fully stable and may still be changed until we +> publish version 1.0. +> +> Moreover, any items or modules starting with `__` (double underscores) must +> not be used directly. Those items or modules are only for internal uses and +> their API may change without a major version update. + +## Learning resources + +If you're just getting started with `googletest`, consider going through +the first chapter of +["Advanced testing for Rust applications"](https://github.com/mainmatter/rust-advanced-testing-workshop), +a self-guided Rust course: it provides a guided introduction to the library, +with exercises to help you get comfortable with `googletest` macros, +its matchers and its overall philosophy. + +## Assertions and matchers + +The core of GoogleTest is its *matchers*. Matchers indicate what aspect of an +actual value one is asserting: (in-)equality, containment, regular expression +matching, and so on. + +To make an assertion using a matcher, GoogleTest offers three macros: + + * [`assert_that!`] panics if the assertion fails, aborting the test. + * [`expect_that!`] logs an assertion failure, marking the test as having + failed, but allows the test to continue running (called a _non-fatal + assertion_). It requires the use of the [`gtest`] attribute macro + on the test itself. + * [`verify_that!`] has no side effects and evaluates to a [`Result<()>`] whose + `Err` variant describes the assertion failure, if there is one. In + combination with the + [`?` operator](https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator), + this can be used to abort the test on assertion failure without panicking. It + is also the building block for the other two macros above. + +For example: + +```rust +use googletest::prelude::*; + +#[test] +fn fails_and_panics() { + let value = 2; + assert_that!(value, eq(4)); +} + +#[gtest] +fn two_logged_failures() { + let value = 2; + expect_that!(value, eq(4)); // Test now failed, but continues executing. + expect_that!(value, eq(5)); // Second failure is also logged. +} + +#[test] +fn fails_immediately_without_panic() -> Result<()> { + let value = 2; + verify_that!(value, eq(4))?; // Test fails and aborts. + verify_that!(value, eq(2))?; // Never executes. + Ok(()) +} + +#[test] +fn simple_assertion() -> Result<()> { + let value = 2; + verify_that!(value, eq(4)) // One can also just return the last assertion. +} +``` + +This library includes a rich set of matchers, covering: + + * Equality, numeric inequality, and approximate equality; + * Strings and regular expressions; + * Containers and set-theoretic matching. + +Matchers are composable: + +```rust +use googletest::prelude::*; + +#[gtest] +fn contains_at_least_one_item_at_least_3() { + let value = vec![1, 2, 3]; + expect_that!(value, contains(ge(3))); +} +``` + +They can also be logically combined: + +```rust +use googletest::prelude::*; + +#[gtest] +fn strictly_between_9_and_11() { + let value = 10; + expect_that!(value, gt(9).and(not(ge(11)))); +} +``` + +## Pattern-matching + +One can use the macro [`matches_pattern!`] to create a composite matcher for a +struct or enum that matches fields with other matchers: + +```rust +use googletest::prelude::*; + +struct AStruct { + a_field: i32, + another_field: i32, + a_third_field: &'static str, +} + +#[test] +fn struct_has_expected_values() { + let value = AStruct { + a_field: 10, + another_field: 100, + a_third_field: "A correct value", + }; + expect_that!(value, matches_pattern!(AStruct { + a_field: eq(10), + another_field: gt(50), + a_third_field: contains_substring("correct"), + })); +} +``` + +## Writing matchers + +One can extend the library by writing additional matchers. To do so, create a +struct holding the matcher's data and have it implement the trait [`Matcher`]: + +```rust +struct MyEqMatcher { + expected: T, +} + +impl Matcher for MyEqMatcher { + fn matches(&self, actual: T) -> MatcherResult { + (self.expected == actual).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> String { + match matcher_result { + MatcherResult::Match => { + format!("is equal to {:?} the way I define it", self.expected) + } + MatcherResult::NoMatch => { + format!("isn't equal to {:?} the way I define it", self.expected) + } + } + } +} +``` + +It is recommended to expose a function which constructs the matcher: + +```rust +pub fn eq_my_way(expected: T) -> impl Matcher { + MyEqMatcher { expected } +} +``` + +The new matcher can then be used in the assertion macros: + +```rust +#[gtest] +fn should_be_equal_by_my_definition() { + expect_that!(10, eq_my_way(10)); +} +``` + +## Non-fatal assertions + +Using non-fatal assertions, a single test is able to log multiple assertion +failures. Any single assertion failure causes the test to be considered having +failed, but execution continues until the test completes or otherwise aborts. + +This is analogous to the `EXPECT_*` family of macros in GoogleTest. + +To make a non-fatal assertion, use the macro [`expect_that!`]. The test must +also be marked with [`gtest`] instead of the Rust-standard `#[test]`. + +```rust +use googletest::prelude::*; + +#[gtest] +fn three_non_fatal_assertions() { + let value = 2; + expect_that!(value, eq(2)); // Passes; test still considered passing. + expect_that!(value, eq(3)); // Fails; logs failure and marks the test failed. + expect_that!(value, eq(4)); // A second failure, also logged. +} +``` + +This can be used in the same tests as `verify_that!`, in which case the test +function must also return [`Result<()>`]: + +```rust +use googletest::prelude::*; + +#[gtest] +fn failing_non_fatal_assertion() -> Result<()> { + let value = 2; + expect_that!(value, eq(3)); // Just marks the test as having failed. + verify_that!(value, eq(2))?; // Passes, so does not abort the test. + Ok(()) // Because of the failing expect_that! call above, the + // test fails despite returning Ok(()) +} +``` + +```rust +use googletest::prelude::*; + +#[gtest] +fn failing_fatal_assertion_after_non_fatal_assertion() -> Result<()> { + let value = 2; + verify_that!(value, eq(3))?; // Fails and aborts the test. + expect_that!(value, eq(3)); // Never executes, since the test already aborted. + Ok(()) +} +``` + +### Interoperability + +You can use the `#[gtest]` macro together with many other libraries +such as [rstest](https://crates.io/crates/rstest). Just apply both attribute +macros to the test: + +```rust +#[gtest] +#[rstest] +#[case(1)] +#[case(2)] +#[case(3)] +fn rstest_works_with_google_test(#[case] value: u32) -> Result<()> { + verify_that!(value, gt(0)) +} +``` + +Make sure to put `#[gtest]` *before* `#[rstest]`. Otherwise the +annotated test will run twice, since both macros will attempt to register a test +with the Rust test harness. + +The macro also works together with +[async tests with Tokio](https://docs.rs/tokio/latest/tokio/attr.gtest.html) in +the same way: + +```rust +#[gtest] +#[tokio::test] +async fn should_work_with_tokio() -> Result<()> { + verify_that!(3, gt(0)) +} +``` + +There is one caveat when running async tests: test failure reporting through +`and_log_failure` will not work properly if the assertion occurs on a different +thread than runs the test. + +## Predicate assertions + +The macro [`verify_pred!`] provides predicate assertions analogous to +GoogleTest's `EXPECT_PRED` family of macros. Wrap an invocation of a predicate +in a `verify_pred!` invocation to turn that into a test assertion which passes +precisely when the predicate returns `true`: + +```rust +fn stuff_is_correct(x: i32, y: i32) -> bool { + x == y +} + +let x = 3; +let y = 4; +verify_pred!(stuff_is_correct(x, y))?; +``` + +The assertion failure message shows the arguments and the values to which they +evaluate: + +``` +stuff_is_correct(x, y) was false with + x = 3, + y = 4 +``` + +The `verify_pred!` invocation evaluates to a [`Result<()>`] just like +[`verify_that!`]. There is also a macro [`expect_pred!`] to make a non-fatal +predicaticate assertion. + +## Unconditionally generating a test failure + +The macro [`fail!`] unconditionally evaluates to a `Result` indicating a test +failure. It can be used analogously to [`verify_that!`] and [`verify_pred!`] to +cause a test to fail, with an optional formatted message: + +```rust +#[test] +fn always_fails() -> Result<()> { + fail!("This test must fail with {}", "today") +} +``` + +## Configuration + +This library is configurable through environment variables. Since the +configuration does not impact whether a test fails or not but how a failure is +displayed, we recommend setting those variables in the personal +`~/.cargo/config.toml` instead of in the project-scoped `Cargo.toml`. + +### Configuration variable list + +| Variable name | Description | +| ------------- | ------------------------------------------------------- | +| NO_COLOR | Disables colored output. See . | +| FORCE_COLOR | Forces colors even when the output is piped to a file. | + +## Contributing Changes + +Please read [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to contribute +to this project. + +[`and_log_failure()`]: https://docs.rs/googletest/*/googletest/trait.GoogleTestSupport.html#tymethod.and_log_failure +[`assert_that!`]: https://docs.rs/googletest/*/googletest/macro.assert_that.html +[`expect_pred!`]: https://docs.rs/googletest/*/googletest/macro.expect_pred.html +[`expect_that!`]: https://docs.rs/googletest/*/googletest/macro.expect_that.html +[`fail!`]: https://docs.rs/googletest/*/googletest/macro.fail.html +[`gtest`]: https://docs.rs/googletest/*/googletest/attr.gtest.html +[`matches_pattern!`]: https://docs.rs/googletest/*/googletest/macro.matches_pattern.html +[`verify_pred!`]: https://docs.rs/googletest/*/googletest/macro.verify_pred.html +[`verify_that!`]: https://docs.rs/googletest/*/googletest/macro.verify_that.html +[`Describe`]: https://docs.rs/googletest/*/googletest/matcher/trait.Describe.html +[`Matcher`]: https://docs.rs/googletest/*/googletest/matcher/trait.Matcher.html +[`Result<()>`]: https://docs.rs/googletest/*/googletest/type.Result.html diff --git a/third_party/crates/googletest-0.14.3/crate_docs.md b/third_party/crates/googletest-0.14.3/crate_docs.md new file mode 100644 index 0000000..09aa35e --- /dev/null +++ b/third_party/crates/googletest-0.14.3/crate_docs.md @@ -0,0 +1,544 @@ +A rich test assertion library for Rust. + +This library provides: + + * A framework for writing matchers which can be combined to make a wide + range of assertions on data, + * A rich set of matchers, and + * A new set of test assertion macros. +## Learning resources + +If you're just getting started with `googletest`, consider going through +the first chapter of +["Advanced testing for Rust applications"](https://github.com/mainmatter/rust-advanced-testing-workshop), +a self-guided Rust course: it provides a guided introduction to the library, +with exercises to help you get comfortable with `googletest` macros, +its matchers and its overall philosophy. + +## Assertions and matchers + +The core of GoogleTest is its *matchers*. Matchers indicate what aspect of an +actual value one is asserting: (in-)equality, containment, regular expression +matching, and so on. + +To make an assertion using a matcher, GoogleTest offers three macros: + + * [`assert_that!`] panics if the assertion fails, aborting the test. + * [`expect_that!`] logs an assertion failure, marking the test as having + failed, but allows the test to continue running (called a _non-fatal + assertion_). It requires the use of the [`gtest`] + attribute macro on the test itself. + * [`verify_that!`] has no side effects and evaluates to a [`Result`] whose + `Err` variant describes the assertion failure, if there is one. In + combination with the + [`?` operator](https://doc.rust-lang.org/reference/expressions/operator-expr.html#the-question-mark-operator), + this can be used to abort the test on assertion failure without panicking. It + is also the building block for the other two macros above. + +For example: + +``` +use googletest::prelude::*; + +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[test] +# */ +fn fails_and_panics() { + let value = 2; + assert_that!(value, eq(4)); +} + +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[gtest] +# */ +fn two_logged_failures() { + let value = 2; + expect_that!(value, eq(4)); // Test now failed, but continues executing. + expect_that!(value, eq(5)); // Second failure is also logged. +} + +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[test] +# */ +fn fails_immediately_without_panic() -> Result<()> { + let value = 2; + verify_that!(value, eq(4))?; // Test fails and aborts. + verify_that!(value, eq(2))?; // Never executes. + Ok(()) +} + +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[test] +# */ +fn simple_assertion() -> Result<()> { + let value = 2; + verify_that!(value, eq(4)) // One can also just return the last assertion. +} +``` + +Matchers are composable: + +``` +use googletest::prelude::*; + +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[gtest] +# */ +fn contains_at_least_one_item_at_least_3() { +# googletest::internal::test_outcome::TestOutcome::init_current_test_outcome(); + let value = vec![1, 2, 3]; + expect_that!(value, contains(ge(&3))); +# googletest::internal::test_outcome::TestOutcome::close_current_test_outcome::<&str>(Ok(())) +# .unwrap(); +} +# contains_at_least_one_item_at_least_3(); +``` + +They can also be logically combined, with methods from [`MatcherBase`]: + +``` +use googletest::prelude::*; + +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[gtest] +# */ +fn strictly_between_9_and_11() { +# googletest::internal::test_outcome::TestOutcome::init_current_test_outcome(); + let value = 10; + expect_that!(value, gt(9).and(not(ge(11)))); +# googletest::internal::test_outcome::TestOutcome::close_current_test_outcome::<&str>(Ok(())) +# .unwrap(); +} +# strictly_between_9_and_11(); +``` + +## Available matchers + +The following matchers are provided in GoogleTest Rust: + +| Matcher | What it matches | +|----------------------|--------------------------------------------------------------------------| +| [`all!`] | Anything matched by all given matchers. | +| [`any!`] | Anything matched by at least one of the given matchers. | +| [`anything`] | Any input. | +| [`approx_eq`] | A floating point number within a standard tolerance of the argument. | +| [`char_count`] | A string with a Unicode scalar count matching the argument. | +| [`container_eq`] | Same as [`eq`], but for containers (with a better mismatch description). | +| [`contains`] | A container containing an element matched by the given matcher. | +| [`contains_each!`] | A container containing distinct elements each of the arguments match. | +| [`contains_regex`] | A string containing a substring matching the given regular expression. | +| [`contains_substring`] | A string containing the given substring. | +| [`derefs_to`] | A [`Deref`] which `deref()`s to a value that the argument matches. | +| [`displays_as`] | A [`Display`] value whose formatted string is matched by the argument. | +| [`each`] | A container all of whose elements the given argument matches. | +| [`elements_are!`] | A container whose elements the arguments match, in order. | +| [`ends_with`] | A string ending with the given suffix. | +| [`eq`] | A value equal to the argument, in the sense of the [`PartialEq`] trait. | +| [`err`] | A [`Result`][std::result::Result] containing an `Err` variant the argument matches. | +| [`field!`] | A struct or enum with a given field whose value the argument matches. | +| [`ge`] | A [`PartialOrd`] value greater than or equal to the given value. | +| [`gt`] | A [`PartialOrd`] value strictly greater than the given value. | +| [`has_entry`] | A [`HashMap`] containing a given key whose value the argument matches. | +| [`is_contained_in!`] | A container each of whose elements is matched by some given matcher. | +| [`is_empty`] | An empty collection. | +| [`is_finite`] | A floating point number which is neither infinite nor NaN. | +| [`is_infinite`] | A floating point number which is positive or negative infinity. | +| [`is_nan`] | A floating point number which is NaN. | +| [`le`] | A [`PartialOrd`] value less than or equal to the given value. | +| [`len`] | A container whose number of elements the argument matches. | +| [`lt`] | A [`PartialOrd`] value strictly less than the given value. | +| [`matches_pattern!`] | A struct or enum whose fields are matched according to the arguments. | +| [`matches_regex`] | A string matched by the given regular expression. | +| [`near`] | A floating point number within a given tolerance of the argument. | +| [`none`] | An [`Option`] containing `None`. | +| [`not`] | Any value the argument does not match. | +| [`ok`] | A [`Result`][std::result::Result] containing an `Ok` variant the argument matches. | +| [`pat!`] | Alias for [`matches_pattern!`]. | +| [`points_to`] | A reference `&` which points to a value that the argument matches. | +| [`pointwise!`] | A container whose contents the arguments match in a pointwise fashion. | +| [`predicate`] | A value on which the given predicate returns true. | +| [`some`] | An [`Option`] containing `Some` whose value the argument matches. | +| [`starts_with`] | A string starting with the given prefix. | +| [`subset_of`] | A container all of whose elements are contained in the argument. | +| [`superset_of`] | A container containing all elements of the argument. | +| [`unordered_elements_are!`] | A container whose elements the arguments match, in any order. | + +[`all!`]: matchers::all +[`any!`]: matchers::any +[`anything`]: matchers::anything +[`approx_eq`]: matchers::approx_eq +[`char_count`]: matchers::char_count +[`container_eq`]: matchers::container_eq +[`contains`]: matchers::contains +[`contains_each!`]: matchers::contains_each +[`contains_regex`]: matchers::contains_regex +[`contains_substring`]: matchers::contains_substring +[`displays_as`]: matchers::displays_as +[`derefs_to`]: matchers::derefs_to +[`each`]: matchers::each +[`elements_are!`]: matchers::elements_are +[`ends_with`]: matchers::ends_with +[`eq`]: matchers::eq +[`err`]: matchers::err +[`field!`]: matchers::field +[`ge`]: matchers::ge +[`gt`]: matchers::gt +[`has_entry`]: matchers::has_entry +[`is_contained_in!`]: matchers::is_contained_in +[`is_empty`]: matchers::is_empty +[`is_finite`]: matchers::is_finite +[`is_infinite`]: matchers::is_infinite +[`is_nan`]: matchers::is_nan +[`le`]: matchers::le +[`len`]: matchers::len +[`lt`]: matchers::lt +[`matches_regex`]: matchers::matches_regex +[`matches_pattern!`]: matchers::matches_pattern +[`near`]: matchers::near +[`none`]: matchers::none +[`not`]: matchers::not +[`pat!`]: matchers::pat +[`ok`]: matchers::ok +[`points_to`]: matchers::points_to +[`pointwise!`]: matchers::pointwise +[`predicate`]: matchers::predicate +[`some`]: matchers::some +[`starts_with`]: matchers::starts_with +[`subset_of`]: matchers::subset_of +[`superset_of`]: matchers::superset_of +[`unordered_elements_are!`]: matchers::unordered_elements_are +[`Deref`]: std::ops::Deref +[`Display`]: std::fmt::Display +[`HashMap`]: std::collections::HashMap +[`Option`]: std::option::Option +[`PartialEq`]: std::cmp::PartialEq +[`PartialOrd`]: std::cmp::PartialOrd + +## Writing matchers + +One can extend the library by writing additional matchers. To do so, create +a struct holding the matcher's data and have it implement the traits +[`Matcher`] and [`MatcherBase`]: + +```no_run +use googletest::{description::Description, matcher::{Matcher, MatcherBase, MatcherResult}}; +use std::fmt::Debug; + +#[derive(MatcherBase)] +struct MyEqMatcher { + expected: T, +} + +impl Matcher for MyEqMatcher { + fn matches(&self, actual: T) -> MatcherResult { + if self.expected == actual { + MatcherResult::Match + } else { + MatcherResult::NoMatch + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("is equal to {:?} the way I define it", self.expected).into() + } + MatcherResult::NoMatch => { + format!("isn't equal to {:?} the way I define it", self.expected).into() + } + } + } +} +``` + + It is recommended to expose a function which constructs the matcher: + + ```no_run + # use googletest::{description::Description, matcher::{Matcher, MatcherBase, MatcherResult}}; + # use std::fmt::Debug; + # #[derive(MatcherBase)] + # struct MyEqMatcher { + # expected: T, + # } + # + # impl Matcher for MyEqMatcher { + # fn matches(&self, actual: T) -> MatcherResult { + # if self.expected == actual { + # MatcherResult::Match + # } else { + # MatcherResult::NoMatch + # } + # } + # + # fn describe(&self, matcher_result: MatcherResult) -> Description { + # match matcher_result { + # MatcherResult::Match => { + # format!("is equal to {:?} the way I define it", self.expected).into() + # } + # MatcherResult::NoMatch => { + # format!("isn't equal to {:?} the way I define it", self.expected).into() + # } + # } + # } + # } + # + pub fn eq_my_way(expected: T) -> impl Matcher { + MyEqMatcher { expected } + } + ``` + + The new matcher can then be used in the assertion macros: + +``` +# use googletest::prelude::*; +# use googletest::{description::Description, matcher::{Matcher, MatcherBase, MatcherResult}}; +# use std::fmt::Debug; +# #[derive(MatcherBase)] +# struct MyEqMatcher { +# expected: T, +# } +# +# impl Matcher for MyEqMatcher { +# fn matches(&self, actual: T) -> MatcherResult { +# if self.expected == actual { +# MatcherResult::Match +# } else { +# MatcherResult::NoMatch +# } +# } +# +# fn describe(&self, matcher_result: MatcherResult) -> Description { +# match matcher_result { +# MatcherResult::Match => { +# format!("is equal to {:?} the way I define it", self.expected).into() +# } +# MatcherResult::NoMatch => { +# format!("isn't equal to {:?} the way I define it", self.expected).into() +# } +# } +# } +# } +# +# pub fn eq_my_way(expected: T) -> impl Matcher { +# MyEqMatcher { expected } +# } +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[gtest] +# */ +fn should_be_equal_by_my_definition() { +# googletest::internal::test_outcome::TestOutcome::init_current_test_outcome(); + expect_that!(10, eq_my_way(10)); +# googletest::internal::test_outcome::TestOutcome::close_current_test_outcome::<&str>(Ok(())) +# .unwrap(); +} +# should_be_equal_by_my_definition(); +``` + +## Non-fatal assertions + +Using non-fatal assertions, a single test is able to log multiple assertion +failures. Any single assertion failure causes the test to be considered +having failed, but execution continues until the test completes or otherwise +aborts. + +To make a non-fatal assertion, use the macro [`expect_that!`]. The test must +also be marked with [`gtest`] instead of the Rust-standard `#[test]`. + +```no_run +use googletest::prelude::*; + +#[gtest] +fn three_non_fatal_assertions() { + let value = 2; + expect_that!(value, eq(2)); // Passes; test still considered passing. + expect_that!(value, eq(3)); // Fails; logs failure and marks the test failed. + expect_that!(value, eq(4)); // A second failure, also logged. +} +``` + +This can be used in the same tests as `verify_that!`, in which case the test +function must also return [`Result<()>`]: + +```no_run +use googletest::prelude::*; + +# /* Make sure this also compiles as a doctest. +#[gtest] +# */ +fn failing_non_fatal_assertion() -> Result<()> { + let value = 2; + expect_that!(value, eq(3)); // Just marks the test as having failed. + verify_that!(value, eq(2))?; // Passes, so does not abort the test. + Ok(()) // Because of the failing expect_that! call above, the + // test fails despite returning Ok(()) +} +``` + +```no_run +use googletest::prelude::*; + +#[gtest] +fn failing_fatal_assertion_after_non_fatal_assertion() -> Result<()> { + let value = 2; + expect_that!(value, eq(2)); // Passes; test still considered passing. + verify_that!(value, eq(3))?; // Fails and aborts the test. + expect_that!(value, eq(3)); // Never executes, since the test already aborted. + Ok(()) +} +``` + +## Predicate assertions + +The macro [`verify_pred!`] provides predicate assertions analogous to +GoogleTest's `EXPECT_PRED` family of macros. Wrap an invocation of a +predicate in a `verify_pred!` invocation to turn that into a test assertion +which passes precisely when the predicate returns `true`: + +``` +# use googletest::prelude::*; +fn stuff_is_correct(x: i32, y: i32) -> bool { + x == y +} + +# fn run_test() -> Result<()> { +let x = 3; +let y = 4; +verify_pred!(stuff_is_correct(x, y))?; +# Ok(()) +# } +# run_test().unwrap_err(); +``` + +The assertion failure message shows the arguments and the values to which +they evaluate: + +```text +stuff_is_correct(x, y) was false with + x = 3, + y = 4 +``` + +The `verify_pred!` invocation evaluates to a [`Result<()>`] just like +[`verify_that!`]. There is also a macro [`expect_pred!`] to make a non-fatal +predicaticate assertion. + +## Unconditionally generating a test failure + +The macro [`fail!`] unconditionally evaluates to a `Result` indicating a +test failure. It can be used analogously to [`verify_that!`] and +[`verify_pred!`] to cause a test to fail, with an optional formatted +message: + +``` +# use googletest::prelude::*; +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[test] +# */ +fn always_fails() -> Result<()> { + fail!("This test must fail with {}", "today") +} +# always_fails().unwrap_err(); +``` + +## Conversion from `Result::Err` and `Option::None` + +To simplify error management during a test arrangement, [`Result`] +provides a few conversion utilities. + +If your setup function returns `std::result::Result` where `E: std::error::Error`, +the `std::result::Result` can simply be handled with the `?` operator. If an `Err(e)` +is returned, the test will report a failure at the line where the `?` operator has been +applied (or the lowest caller without `#[track_caller]`). + +``` +# use googletest::prelude::*; +struct PngImage { h: i32, w: i32 /* ... */ } +impl PngImage { + fn new_from_file(file_name: &str) -> std::result::Result { + Err(std::io::Error::new(std::io::ErrorKind::Other, "oh no!")) + + } + fn rotate(&mut self) { std::mem::swap(&mut self.h, &mut self.w);} + fn dimensions(&self) -> (i32, i32) { (self.h, self.w)} +} + +fn test_png_image_dimensions() -> googletest::Result<()> { + // Arrange + let mut png = PngImage::new_from_file("example.png")?; + verify_eq!(png.dimensions(), (128, 64))?; + + // Act + png.rotate(); + + // Assert + expect_eq!(png.dimensions(), (64, 128)); + Ok(()) +} + +# test_png_image_dimensions().unwrap_err(); +``` + +If your setup function returns `Option` or `std::result::Result` where +`E: !std::error::Error`, then you can convert these types with `or_fail()` +from the `OrFail` extension trait. + +``` +# use googletest::prelude::*; +# struct PngImage; +# static PNG_BINARY: [u8;0] = []; + +impl PngImage { + fn new_from_binary(bin: &[u8]) -> std::result::Result { + Err("Parsing failed".into()) + } +} + +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[gtest] +# */ +fn test_png_image_binary() -> googletest::Result<()> { + // Arrange + let png_image = PngImage::new_from_binary(&PNG_BINARY).or_fail()?; + /* ... */ + # Ok(()) +} +# test_png_image_binary().unwrap_err(); + +impl PngImage { + fn new_from_cache(key: u64) -> Option { + None + } +} + +# /* The attribute macro would prevent the function from being compiled in a doctest. +#[gtest] +# */ +fn test_png_from_cache() -> googletest::Result<()> { + // Arrange + let png_image = PngImage::new_from_cache(123).or_fail()?; + /* ... */ + # Ok(()) +} +# test_png_from_cache().unwrap_err(); +``` + + +## Multi-threaded tests + +Due to internal use of thread-local storage, the `expect_*` family of macros +should only be invoked from the same thread as that running the test itself. +`verify_*` and `assert_*` macros on the other hand may be called from other +threads. + +## Integrations with other crates + +GoogleTest Rust includes integrations with the +[Proptest](https://crates.io/crates/proptest) crates to simplify turning +GoogleTest assertion failures into Proptest +[`TestCaseError`](https://docs.rs/proptest/latest/proptest/test_runner/enum.TestCaseError.html) +through the `?` operator. + +[`and_log_failure()`]: GoogleTestSupport::and_log_failure +[`or_fail()`]: OrFail::or_fail +[`Matcher`]: matcher::Matcher +[`MatcherBase`]: matcher::MatcherBase diff --git a/third_party/crates/googletest-0.14.3/src/assertions.rs b/third_party/crates/googletest-0.14.3/src/assertions.rs new file mode 100644 index 0000000..1487422 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/assertions.rs @@ -0,0 +1,1918 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macros are documented at the top level. +#![doc(hidden)] + +/// Checks whether the `Matcher` given by the second argument matches the first +/// argument. +/// +/// Evaluates to `Result::Ok(())` if the matcher matches and +/// `Result::Err(TestAssertionFailure)` if it does not. The caller must then +/// decide how to handle the `Err` variant. It has a few options: +/// +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Note that unlike [`expect_that!`], this macro may be invoked in tests +/// without the [`gtest`][crate::gtest] attribute and also from other threads +/// (provided that `and_log_failure` is not called). +/// +/// Example: +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(42, eq(42))?; // This will pass. +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # fn should_fail() -> Result<()> { +/// # googletest::internal::test_outcome::TestOutcome::init_current_test_outcome(); +/// verify_that!(42, eq(123)).and_log_failure(); +/// // This will log a test failure and allow execution to continue. +/// let _ = verify_that!(42, eq(123)); // This will do nothing. +/// verify_that!(42, eq(123))?; // This will fail, returning immediately. +/// verify_that!(42, eq(0))?; // This will not run. +/// # googletest::internal::test_outcome::TestOutcome::close_current_test_outcome::<&str>(Ok(())) +/// # .unwrap_err(); +/// # Ok(()) +/// # } +/// # verify_that!(should_fail(), err(displays_as(contains_substring("Expected: is equal to 123")))) +/// # .unwrap(); +/// ``` +/// +/// This macro has special support for matching against container. Namely: +/// * `verify_that!(actual, [m1, m2, ...])` is equivalent to +/// `verify_that!(actual, elements_are![m1, m2, ...])` +/// * `verify_that!(actual, {m1, m2, ...})` is equivalent to +/// `verify_that!(actual, unordered_elements_are![m1, m2, ...])` +/// +/// ## Matching against tuples +/// +/// One can match against a tuple by constructing a tuple of matchers as +/// follows: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!((123, 456), (eq(123), eq(456)))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!((123, 456), (eq(123), eq(0)))?; // Fails: second matcher does not match +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// This also works with composed matchers: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!((123, 456), not((eq(456), eq(123))))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// Matchers must correspond to the actual tuple in count and type. Otherwise +/// the test will fail to compile. +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// verify_that!((123, 456), (eq(123),))?; // Does not compile: wrong tuple size +/// verify_that!((123, "A string"), (eq(123), eq(456)))?; // Does not compile: wrong type +/// # Ok(()) +/// # } +/// ``` +/// +/// All fields must be covered by matchers. Use +/// [`anything`][crate::matchers::anything] for fields which are not relevant +/// for the test. +/// +/// ``` +/// # use googletest::prelude::*; +/// verify_that!((123, 456), (eq(123), anything())) +/// # .unwrap(); +/// ``` +/// +/// This supports tuples of up to 12 elements. Tuples longer than that do not +/// automatically inherit the `Debug` trait from their members, so are generally +/// not supported; see [Rust by Example](https://doc.rust-lang.org/rust-by-example/primitives/tuples.html#tuples). +#[macro_export] +macro_rules! verify_that { + // specialized to sequences: + ($actual:expr, [$($expecteds:expr),+ $(,)?]) => { + { + use $crate::assertions::internal::Subject as _; + $actual.check( + $crate::matchers::elements_are![$($expecteds),+], + stringify!($actual), + ) + } + }; + + // specialized to unordered sequences: + ($actual:expr, {$($expecteds:expr),+ $(,)?}) => { + { + use $crate::assertions::internal::Subject as _; + $actual.check( + $crate::matchers::unordered_elements_are![$($expecteds),+], + stringify!($actual), + ) + } + }; + + // general case: + ($actual:expr, $expected:expr $(,)?) => { + { + use $crate::assertions::internal::Subject as _; + $actual.check( + $expected, + stringify!($actual), + ) + } + }; +} +pub use verify_that; + +/// Asserts that the given predicate applied to the given arguments returns +/// true. +/// +/// Similarly to [`verify_that`], this evaluates to a `Result` whose `Ok` +/// variant indicates that the given predicate returned true and whose `Err` +/// variant indicates that it returned false. +/// +/// The failure message contains detailed information about the arguments. For +/// example: +/// +/// ``` +/// # use googletest::prelude::*; +/// fn equals_modulo(a: i32, b: i32, n: i32) -> bool { a % n == b % n } +/// +/// # /* The attribute macro would prevent the function from being compiled in a doctest. +/// #[test] +/// # */ +/// fn test() -> Result<()> { +/// let a = 1; +/// fn b(_x: i32) -> i32 { 7 } +/// verify_pred!(equals_modulo(a, b(b(2)), 2 + 3))?; +/// Ok(()) +/// } +/// # verify_that!( +/// # test(), +/// # err(displays_as(contains_substring("equals_modulo(a, b(b(2)), 2 + 3) was false with"))) +/// # ).unwrap(); +/// ``` +/// +/// This results in the following message: +/// +/// ```text +/// equals_modulo(a, b(b(2)), 2 + 3) was false with +/// a = 1, +/// b(b(2)) = 7, +/// 2 + 3 = 5, +/// ``` +/// +/// The predicate can also be a method on a struct, e.g.: +/// +/// ```ignore +/// struct AStruct {}; +/// +/// impl AStruct { +/// fn equals_modulo(...) {...} +/// } +/// +/// verify_pred!((AStruct {}).equals_modulo(a, b, n))?; +/// ``` +/// +/// The expression passed to this macro must return `bool`. In the most general +/// case, it prints out each of the `.`-separated parts of the expression and +/// the arguments of all top-level method calls as long as they implement +/// `Debug`. It evaluates every value (including the method receivers) exactly +/// once. Effectively, for `verify_pred!((a + 1).b.c(x + y, &mut z, 2))`, it +/// generates code analogous to the following, which allows printing accurate +/// intermediate values even if they are subsequently consumed (moved out) or +/// mutated in-place by the expression: +/// +/// ```ignore +/// let mut error_message = "(a + 1).b.c(x + y, 2) was false with".to_string(); +/// let mut x1 = (a + 1); +/// write!(error_message, "\n (a + 1) = {:?},", x1); +/// write!(error_message, "\n (a + 1).b = {:?},", x1.b); +/// let mut x2 = x + y; +/// write!(error_message, "\n x + y = {:?},", x2); +/// let mut x3 = &mut z; +/// write!(error_message, "\n & mut z = {:?},", x3); +/// let mut x4 = x1.b.c(x2, x3, 2); +/// if (x4) { +/// Ok(()) +/// } else { +/// Err(error_message) +/// } +/// ``` +/// +/// Wrapping the passed-in expression in parens or curly braces will prevent the +/// detailed printing of the expression. +/// +/// ```ignore +/// verify_pred!((a.foo()).bar())?; +/// ``` +/// +/// would not print `a`, but would print `(a.foo())` and `(a.foo()).bar()` on +/// error. +#[macro_export] +macro_rules! verify_pred { + ($expr:expr $(,)?) => { + $crate::assertions::internal::__googletest_macro_verify_pred!($expr) + }; +} +pub use verify_pred; + +/// Evaluates to a `Result` which contains an `Err` variant with the given test +/// failure message. +/// +/// This can be used to force the test to fail if its execution reaches a +/// particular point. For example: +/// +/// ```ignore +/// match some_value { +/// ExpectedVariant => {...} +/// UnwantedVaraint => { +/// fail!("This thing which should not happen happened anyway")?; +/// } +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +/// +/// ```ignore +/// match some_value { +/// ExpectedVariant => {...} +/// UnwantedVaraint => { +/// fail!("This thing which should not happen happened anyway: {}", some_value)?; +/// } +/// } +/// ``` +/// +/// One may also omit the message, in which case the test failure message will +/// be generic: +/// +/// ```ignore +/// match some_value { +/// ExpectedVariant => {...} +/// UnwantedVaraint => { +/// fail!()?; +/// } +/// } +/// ``` +/// +/// Unlike `panic!` but analogously to [`verify_that`] and [`verify_pred`], this +/// macro has no effect on the flow of control but instead returns a `Result` +/// which must be handled by the invoking function. This can be done with the +/// question mark operator (as above) or the method +/// [`and_log_failure`](crate::GoogleTestSupport::and_log_failure). +#[macro_export] +macro_rules! fail { + ($($message:expr),+ $(,)?) => {{ + $crate::assertions::internal::create_fail_result( + format!($($message),*), + ) + }}; + + () => { $crate::fail!("Test failed") }; +} +pub use fail; + +/// Generates a success. This **does not** make the overall test succeed. A test +/// is only considered successful if none of its assertions fail during its +/// execution. +/// +/// The succeed!() assertion is purely documentary. The only user visible output +/// is a stdout with information on where the success was generated from. +/// +/// ```ignore +/// fn test_to_be_implemented() { +/// succeed!(); +/// } +/// ``` +/// +/// One may include formatted arguments in the success message: +/// +/// ```ignore +/// fn test_to_be_implemented() { +/// succeed!("I am just a fake test: {}", "a fake test indeed"); +/// } +/// ``` +#[macro_export] +macro_rules! succeed { + ($($message:expr),+ $(,)?) => {{ + println!( + "{}\n at {}:{}:{}", + format!($($message),*), + file!(), line!(), column!() + ); + }}; + + () => { + $crate::succeed!("Success") + }; +} +pub use succeed; + +/// Generates a failure marking the test as failed but continue execution. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail_but_not_abort() { +/// add_failure!(); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +/// +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail_but_not_abort() { +/// add_failure!("I am just a fake test: {}", "a fake test indeed"); +/// } +/// ``` +#[macro_export] +macro_rules! add_failure { + ($($message:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure( + $crate::assertions::internal::create_fail_result( + format!($($message),*), + )); + }}; + + () => { + add_failure!("Failed") + }; +} +pub use add_failure; + +/// Generates a failure at specified location marking the test as failed but +/// continue execution. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail_but_not_abort() { +/// add_failure_at!("src/my_file.rs", 32, 12); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +/// +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail_but_not_abort() { +/// add_failure_at!( +/// "src/my_file.rs", +/// 32, +/// 12, +/// "I am just a fake test: {}", "a fake test indeed", +/// ); +/// } +/// ``` +#[macro_export] +macro_rules! add_failure_at { + ($file:expr, $line:expr, $column:expr, $($message:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure( + $crate::assertions::internal::create_fail_result( + format!($($message),*), + ).map_err(|e| e.with_fake_location($file, $line, $column))); + }}; + + ($file:expr, $line:expr, $column:expr $(,)?) => { + add_failure_at!($file, $line, $column, "Failed") + }; +} +pub use add_failure_at; + +/// Verify if the condition evaluates to true and returns `Result`. +/// +/// Evaluates to `Result::Ok(())` if the condition is true and +/// `Result::Err(TestAssertionFailure)` if it evaluates to false. The caller +/// must then decide how to handle the `Err` variant. It has a few options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the failure and continue by calling the method `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_true!(2 + 2 == 5) +/// } +/// ``` +#[macro_export] +macro_rules! verify_true { + ($condition:expr) => {{ + use $crate::assertions::internal::Subject as _; + ($condition).check($crate::matchers::eq(true), stringify!($condition)) + }}; +} +pub use verify_true; + +/// Marks test as failed and continue execution if the expression evaluates to +/// false. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_true!(2 + 2 == 5); +/// println!("This will print"); +/// } +/// ``` +/// +/// One may optionally add arguments which will be formatted and appended to a +/// failure message. For example: +/// +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let extra_information = "Some additional information"; +/// expect_true!(false, "Test failed. Extra information: {extra_information}."); +/// } +/// ``` +/// +/// The output is as follows: +/// +/// ```text +/// Value of: false +/// Expected: is equal to true +/// Actual: false, +/// which isn't equal to true +/// Test failed. Extra information: Some additional information. +/// ``` +#[macro_export] +macro_rules! expect_true { + ($condition:expr) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_true!($condition)) + }}; + ($condition:expr, $($format_args:expr),* $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_true!($condition), + || format!($($format_args),*)); + }}; +} +pub use expect_true; + +/// Marks the test as failed and panics if the given value is `false`. +/// +/// This is a *fatal* assertion: the test panics +/// in the event of assertion failure. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The assertion must +/// occur in the same thread as that running the test itself. +/// +/// Example: +/// +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// assert_true!(false); // panic! +/// } +/// ``` +/// +/// One may optionally add arguments which will be formatted and appended to a +/// failure message. For example: +/// +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let extra_information = "Some additional information"; +/// assert_true!(false, "Test failed. Extra information: {extra_information}."); +/// } +/// ``` +/// +/// The output is as follows: +/// +/// ```text +/// Value of: false +/// Expected: is equal to true +/// Actual: false, +/// which isn't equal to true +/// Test failed. Extra information: Some additional information. +/// ``` +#[macro_export] +macro_rules! assert_true { + ($condition:expr) => { + if let Err(e) = $crate::verify_true!($condition) { + panic!("\n{}", e); + } + }; + ($condition:expr, $($format_args:expr),* $(,)?) => { + if let Err(e) = $crate::GoogleTestSupport::with_failure_message( + $crate::verify_true!($condition), + || format!($($format_args),*)) + { + panic!("\n{}", e); + } + }; +} +pub use assert_true; + +/// Verify if the condition evaluates to false and returns `Result`. +/// +/// Evaluates to `Result::Ok(())` if the condition is false and +/// `Result::Err(TestAssertionFailure)` if it evaluates to true. The caller +/// must then decide how to handle the `Err` variant. It has a few options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the failure and continue by calling the method `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_false!(2 + 2 == 4) +/// } +/// ``` +#[macro_export] +macro_rules! verify_false { + ($condition:expr) => {{ + use $crate::assertions::internal::Subject as _; + ($condition).check($crate::matchers::eq(false), stringify!($condition)) + }}; +} +pub use verify_false; + +/// Marks test as failed and continue execution if the expression evaluates to +/// true. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_false!(2 + 2 == 4); +/// println!("This will print"); +/// } +/// ``` +/// +/// One may optionally add arguments which will be formatted and appended to a +/// failure message. For example: +/// +/// ``` ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let extra_information = "Some additional information"; +/// expect_false!(true, "Test failed. Extra information: {extra_information}."); +/// } +/// ``` +/// +/// The output is as follows: +/// +/// ```text +/// Value of: true +/// Expected: is equal to false +/// Actual: true, +/// which isn't equal to false +/// Test failed. Extra information: Some additional information. +/// ``` +#[macro_export] +macro_rules! expect_false { + ($condition:expr) => {{ + $crate::GoogleTestSupport::and_log_failure(($crate::verify_false!($condition))) + }}; + ($condition:expr, $($format_args:expr),* $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_false!($condition), + || format!($($format_args),*)) + }}; +} +pub use expect_false; + +/// Marks the test as failed and panics if the given value is `true`. +/// +/// This is a *fatal* assertion: the test panics +/// in the event of assertion failure. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The assertion must +/// occur in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// assert_false!(true); // panic! +/// } +/// ``` +/// +/// One may optionally add arguments which will be formatted and appended to a +/// failure message. For example: +/// +/// ``` ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let extra_information = "Some additional information"; +/// assert_false!(true, "Test failed. Extra information: {extra_information}."); +/// } +/// ``` +/// +/// The output is as follows: +/// +/// ```text +/// Value of: true +/// Expected: is equal to false +/// Actual: true, +/// which isn't equal to false +/// Test failed. Extra information: Some additional information. +/// ``` +#[macro_export] +macro_rules! assert_false { + ($condition:expr) => { + if let Err(e) = $crate::verify_false!($condition) { + panic!("\n{}", e); + } + }; + ($condition:expr, $($format_args:expr),* $(,)?) => { + if let Err(e) = $crate::GoogleTestSupport::with_failure_message( + $crate::verify_false!($condition), + || format!($($format_args),*)) + { + panic!("\n{}", e); + } + }; +} +pub use assert_false; + +/// Checks whether the second argument is equal to the first argument. +/// +/// Evaluates to `Result::Ok(())` if they are equal and +/// `Result::Err(TestAssertionFailure)` if they are not. The caller must then +/// decide how to handle the `Err` variant. It has a few options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_eq!(2, 1) +/// } +/// ``` +/// +/// This macro has special support for matching against container. Namely: +/// * `verify_eq!(actual, [e1, e2, ...])` is equivalent to +/// `verify_that!(actual, elements_are![eq(e1), eq(e2), ...])` +/// * `verify_eq!(actual, {e1, e2, ...})` is equivalent to +/// `verify_that!(actual, unordered_elements_are![eq(e1), eq(e2), ...])` +#[macro_export] +macro_rules! verify_eq { + // Specialization for ordered sequences of tuples: + ($actual:expr, [ $( ( $($tuple_elt:expr),* ) ),+ $(,)? ] $(,)?) => { + $crate::verify_that!(&$actual, [ + $( + // tuple matching + ( + $( + $crate::matchers::eq(&$tuple_elt) + ),* + ) + ),* + ]) + }; + + // Specialization for unordered sequences of tuples: + ($actual:expr, { $( ( $($tuple_elt:expr),* ) ),+ $(,)?} $(,)?) => { + $crate::verify_that!(&$actual, { + $( + // tuple matching + ( + $( + $crate::matchers::eq(&$tuple_elt) + ),* + ) + ),* + }) + }; + + // Ordered sequences: + ($actual:expr, [$($expected:expr),+ $(,)?] $(,)?) => { + $crate::verify_that!(&$actual, [$($crate::matchers::eq(&$expected)),*]) + }; + + // Unordered sequences: + ($actual:expr, {$($expected:expr),+ $(,)?} $(,)?) => { + $crate::verify_that!(&$actual, {$($crate::matchers::eq(&$expected)),*}) + }; + + // General case: + ($actual:expr, $expected:expr $(,)?) => { + $crate::verify_that!(&$actual, $crate::matchers::eq(&$expected)) + }; +} +pub use verify_eq; + +/// Marks test as failed and continues execution if the second argument is not +/// equal to first argument. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_eq!(2, 1); +/// println!("This will print!"); +/// } +/// ``` +/// +/// This macro has special support for matching against container. Namely: +/// * `expect_eq!(actual, [e1, e2, ...])` for checking actual contains "e1, e2, +/// ..." in order. +/// * `expect_eq!(actual, {e1, e2, ...})` for checking actual contains "e1, e2, +/// ..." in any order. +/// +/// One may include formatted arguments in the failure message: +///```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let argument = "argument" +/// expect_eq!(2, 1, "custom failure message: {argument}"); +/// println!("This will print!"); +/// } +/// ``` +#[macro_export] +macro_rules! expect_eq { + ($actual:expr, [$($expected:expr),+ $(,)?] $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_eq!($actual, [$($expected),*])); + }}; + ($actual:expr, [$($expected:expr),+ $(,)?], $($format_args:expr),* $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_eq!($actual, [$($expected),*]), + || format!($($format_args),*)); + }}; + ($actual:expr, {$($expected:expr),+ $(,)?} $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_eq!($actual, {$($expected),*})); + }}; + ($actual:expr, {$($expected:expr),+ $(,)?}, $($format_args:expr),* $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_eq!($actual, {$($expected),*}), + || format!($($format_args),*)); + }}; + ($actual:expr, $expected:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_eq!($actual, $expected)); + }}; + ($actual:expr, $expected:expr, $($format_args:expr),* $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_eq!($actual, $expected), + || format!($($format_args),*)); + }}; +} +pub use expect_eq; + +/// Checks whether the second argument is not equal to the first argument. +/// +/// Evaluates to `Result::Ok(())` if they are not equal and +/// `Result::Err(TestAssertionFailure)` if they are equal. The caller must then +/// decide how to handle the `Err` variant. It has a few options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_ne!(1, 1) +/// } +/// ``` +#[macro_export] +macro_rules! verify_ne { + ($actual:expr, $expected:expr $(,)?) => { + $crate::verify_that!(&$actual, $crate::matchers::not($crate::matchers::eq(&$expected))) + }; +} +pub use verify_ne; + +/// Marks test as failed and continues execution if the second argument is +/// equal to first argument. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_ne!(1, 1); +/// println!("This will print!"); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +///```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let argument = "argument" +/// expect_ne!(1, 1, "custom failure message: {argument}"); +/// println!("This will print!"); +/// } +/// ``` +#[macro_export] +macro_rules! expect_ne { + ($actual:expr, $expected:expr, $($format_args:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_ne!($actual, $expected), + || format!($($format_args),*)); + }}; + ($actual:expr, $expected:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_ne!($actual, $expected)); + }}; +} +pub use expect_ne; + +/// Checks whether the first argument is less than second argument. +/// +/// Evaluates to `Result::Ok(())` if the first argument is less than the second +/// and `Result::Err(TestAssertionFailure)` if it is greater or equal. The +/// caller must then decide how to handle the `Err` variant. It has a few +/// options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_lt!(2, 1) +/// } +#[macro_export] +macro_rules! verify_lt { + ($actual:expr, $expected:expr $(,)?) => { + $crate::verify_that!($actual, $crate::matchers::lt($expected)) + }; +} +pub use verify_lt; + +/// Marks test as failed and continues execution if the first argument is +/// greater or equal to second argument. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_lt!(2, 1); +/// println!("This will print!"); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +///```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let argument = "argument" +/// expect_lt!(1, 1, "custom failure message: {argument}"); +/// println!("This will print!"); +/// } +/// ``` +#[macro_export] +macro_rules! expect_lt { + ($actual:expr, $expected:expr, $($format_args:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_lt!($actual, $expected), + || format!($($format_args),*)); + }}; + ($actual:expr, $expected:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_lt!($actual, $expected)); + }}; +} +pub use expect_lt; + +/// Checks whether the first argument is less than or equal to the second +/// argument. +/// +/// Evaluates to `Result::Ok(())` if the first argument is less than or equal to +/// the second and `Result::Err(TestAssertionFailure)` if it is greater. The +/// caller must then decide how to handle the `Err` variant. It has a few +/// options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_le!(2, 1) +/// } +#[macro_export] +macro_rules! verify_le { + ($actual:expr, $expected:expr $(,)?) => { + $crate::verify_that!($actual, $crate::matchers::le($expected)) + }; +} +pub use verify_le; + +/// Marks test as failed and continues execution if the first argument is +/// greater than the second argument. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_le!(2, 1); +/// println!("This will print!"); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +///```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let argument = "argument" +/// expect_le!(2, 1, "custom failure message: {argument}"); +/// println!("This will print!"); +/// } +/// ``` +#[macro_export] +macro_rules! expect_le { + ($actual:expr, $expected:expr, $($format_args:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_le!($actual, $expected), + || format!($($format_args),*)); + }}; + ($actual:expr, $expected:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_le!($actual, $expected)); + }}; +} +pub use expect_le; + +/// Checks whether the first argument is greater than the second argument. +/// +/// Evaluates to `Result::Ok(())` if the first argument is greater than +/// the second and `Result::Err(TestAssertionFailure)` if it is not. The +/// caller must then decide how to handle the `Err` variant. It has a few +/// options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_gt!(1, 2) +/// } +#[macro_export] +macro_rules! verify_gt { + ($actual:expr, $expected:expr $(,)?) => { + $crate::verify_that!($actual, $crate::matchers::gt($expected)) + }; +} +pub use verify_gt; + +/// Marks test as failed and continues execution if the first argument is +/// not greater than the second argument. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_gt!(1, 2); +/// println!("This will print!"); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +///```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let argument = "argument" +/// expect_gt!(1, 2, "custom failure message: {argument}"); +/// println!("This will print!"); +/// } +/// ``` +#[macro_export] +macro_rules! expect_gt { + ($actual:expr, $expected:expr, $($format_args:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_gt!($actual, $expected), + || format!($($format_args),*)); + }}; + ($actual:expr, $expected:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_gt!($actual, $expected)); + }}; +} +pub use expect_gt; + +/// Checks whether the first argument is greater than or equal to the second +/// argument. +/// +/// Evaluates to `Result::Ok(())` if the first argument is greater than or equal +/// to the second and `Result::Err(TestAssertionFailure)` if it is not. The +/// caller must then decide how to handle the `Err` variant. It has a few +/// options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_ge!(1, 2) +/// } +/// ``` +#[macro_export] +macro_rules! verify_ge { + ($actual:expr, $expected:expr $(,)?) => { + $crate::verify_that!($actual, $crate::matchers::ge($expected)) + }; +} +pub use verify_ge; + +/// Marks test as failed and continues execution if the first argument is +/// not greater than or equal to the second argument. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_ge!(1, 2); +/// println!("This will print!"); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +///```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let argument = "argument" +/// expect_ge!(1, 2, "custom failure message: {argument}"); +/// println!("This will print!"); +/// } +/// ``` +#[macro_export] +macro_rules! expect_ge { + ($actual:expr, $expected:expr, $($format_args:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_ge!($actual, $expected), + || format!($($format_args),*)); + }}; + ($actual:expr, $expected:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_ge!($actual, $expected)); + }}; +} +pub use expect_ge; + +/// Checks whether the float given by first argument is approximately +/// equal to second argument. +/// +/// This automatically computes a tolerance from the magnitude of `expected` and +/// matches any actual value within this tolerance of the expected value. The +/// tolerance is chosen to account for the inaccuracies in most ordinary +/// floating point calculations. To see details of how the tolerance is +/// calculated look at the implementation of +/// [`googletest::approx_eq`][crate::matchers::approx_eq]. +/// +/// Evaluates to `Result::Ok(())` if the first argument is approximately equal +/// to the second and `Result::Err(TestAssertionFailure)` if it is not. The +/// caller must then decide how to handle the `Err` variant. It has a few +/// options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_float_eq!(1.0, 2.0) +/// } +/// ``` +#[macro_export] +macro_rules! verify_float_eq { + ($actual:expr, $expected:expr $(,)?) => { + $crate::verify_that!($actual, $crate::matchers::approx_eq($expected)) + }; +} +pub use verify_float_eq; + +/// Marks test as failed and continues execution if the float given by the first +/// argument is not approximately equal to the float given by the second +/// argument. +/// +/// This automatically computes a tolerance from the magnitude of `expected` and +/// matches any actual value within this tolerance of the expected value. The +/// tolerance is chosen to account for the inaccuracies in most ordinary +/// floating point calculations. To see details of how the tolerance is +/// calculated look at the implementation of +/// [`googletest::approx_eq`][crate::matchers::approx_eq]. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_float_eq!(1.0, 2.0); +/// println!("This will print!"); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +///```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let argument = "argument" +/// expect_float_eq!(1.0, 2.0, "custom failure message: {argument}"); +/// println!("This will print!"); +/// } +/// ``` +#[macro_export] +macro_rules! expect_float_eq { + ($actual:expr, $expected:expr, $($format_args:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_float_eq!($actual, $expected), + || format!($($format_args),*)); + }}; + ($actual:expr, $expected:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_float_eq!($actual, $expected)); + }}; +} +pub use expect_float_eq; + +/// Checks whether the float given by first argument is equal to second argument +/// with error tolerance of max_abs_error. +/// +/// Evaluates to `Result::Ok(())` if the first argument is approximately equal +/// to the second and `Result::Err(TestAssertionFailure)` if it is not. The +/// caller must then decide how to handle the `Err` variant. It has a few +/// options: +/// * Abort the current function with the `?` operator. This requires that the +/// function return a suitable `Result`. +/// * Log the test failure and continue by calling the method +/// `and_log_failure`. +/// +/// Of course, one can also use all other standard methods on `Result`. +/// +/// **Invoking this macro by itself does not cause a test failure to be recorded +/// or output.** The resulting `Result` must be handled as described above to +/// cause the test to be recorded as a failure. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[test] +/// fn should_fail() -> Result<()> { +/// verify_near!(1.12345, 1.12346, 1e-6) +/// } +/// ``` +#[macro_export] +macro_rules! verify_near { + ($actual:expr, $expected:expr, $max_abs_error:expr $(,)?) => { + $crate::verify_that!($actual, $crate::matchers::near($expected, $max_abs_error)) + }; +} +pub use verify_near; + +/// Marks the test as failed and continues execution if the float given by first +/// argument is not equal to second argument with error tolerance of +/// max_abs_error. +/// +/// This is a **not-fatal** failure. The test continues execution even after the +/// macro execution. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The failure must be generated +/// in the same thread as that running the test itself. +/// +/// Example: +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// expect_near!(1.12345, 1.12346, 1e-6); +/// println!("This will print!"); +/// } +/// ``` +/// +/// One may include formatted arguments in the failure message: +///```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let argument = "argument" +/// expect_near!(1.12345, 1.12346, 1e-6, "custom failure message: {argument}"); +/// println!("This will print!"); +/// } +/// ``` +#[macro_export] +macro_rules! expect_near { + ($actual:expr, $expected:expr, $max_abs_error:expr, $($format_args:expr),+ $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_near!($actual, $expected, $max_abs_error), + || format!($($format_args),*)); + }}; + ($actual:expr, $expected:expr, $max_abs_error:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_near!($actual, $expected, $max_abs_error)); + }}; +} +pub use expect_near; + +/// Matches the given value against the given matcher, panicking if it does not +/// match. +/// +/// ```should_panic +/// # use googletest::prelude::*; +/// # fn should_fail() { +/// let value = 2; +/// assert_that!(value, eq(3)); // Fails and panics. +/// # } +/// # should_fail(); +/// ``` +/// +/// This is analogous to assertions in most Rust test libraries, where a failed +/// assertion causes a panic. +/// +/// One may optionally add arguments which will be formatted and appended to a +/// failure message. For example: +/// +/// ```should_panic +/// # use googletest::prelude::*; +/// # fn should_fail() { +/// let value = 2; +/// let extra_information = "Some additional information"; +/// assert_that!(value, eq(3), "Test failed. Extra information: {extra_information}."); +/// # } +/// # should_fail(); +/// ``` +/// +/// This is output as follows: +/// +/// ```text +/// Value of: value +/// Expected: is equal to 3 +/// Actual: 2, +/// which isn't equal to 3 +/// at ... +/// Test failed. Extra information: Some additional information. +/// ``` +/// +/// Note that unlike [`expect_that!`], this macro may be invoked in tests +/// without the [`gtest`][crate::gtest] attribute and also from other threads. +/// +/// **Note for users of [GoogleTest for C++](http://google.github.io/googletest/):** +/// This differs from the `ASSERT_THAT` macro in that it panics rather +/// than triggering an early return from the invoking function. To get behaviour +/// equivalent to `ASSERT_THAT`, use [`verify_that!`] with the `?` operator. +#[macro_export] +macro_rules! assert_that { + // specialized to sequence: + ($actual:expr, [ $($expected:expr),* ] $(,)?) => { + match $crate::verify_that!($actual, [ $($expected),* ]) { + Ok(_) => {} + Err(e) => { + // The extra newline before the assertion failure message makes the failure a + // bit easier to read when there's some generic boilerplate from the panic. + panic!("\n{}", e); + } + } + }; + + // specialized to unordered sequence + ($actual:expr, { $($expected:expr),* } $(,)?) => { + match $crate::verify_that!($actual, { $($expected),* }) { + Ok(_) => {} + Err(e) => { + // The extra newline before the assertion failure message makes the failure a + // bit easier to read when there's some generic boilerplate from the panic. + panic!("\n{}", e); + } + } + }; + + // w/ format args, specialized to sequence: + ($actual:expr, [ $($expected:expr),* ], $($format_args:expr),* $(,)?) => { + match $crate::GoogleTestSupport::with_failure_message( + $crate::verify_that!($actual, [ $($expected),* ]), + || format!($($format_args),*)) + { + Ok(_) => {} + Err(e) => { + // The extra newline before the assertion failure message makes the failure a + // bit easier to read when there's some generic boilerplate from the panic. + panic!("\n{}", e); + } + } + }; + + // w/ format args, specialized to unordered sequence: + ($actual:expr, { $($expected:expr),* }, $($format_args:expr),* $(,)?) => { + match $crate::GoogleTestSupport::with_failure_message( + $crate::verify_that!($actual, { $($expected),* }), + || format!($($format_args),*)) + { + Ok(_) => {} + Err(e) => { + // The extra newline before the assertion failure message makes the failure a + // bit easier to read when there's some generic boilerplate from the panic. + panic!("\n{}", e); + } + } + }; + + // general case: + ($actual:expr, $expected:expr $(,)?) => { + match $crate::verify_that!($actual, $expected) { + Ok(_) => {} + Err(e) => { + // The extra newline before the assertion failure message makes the failure a + // bit easier to read when there's some generic boilerplate from the panic. + panic!("\n{}", e); + } + } + }; + + // w/ format args, general case: + ($actual:expr, $expected:expr, $($format_args:expr),* $(,)?) => { + match $crate::GoogleTestSupport::with_failure_message( + $crate::verify_that!($actual, $expected), + || format!($($format_args),*)) + { + Ok(_) => {} + Err(e) => { + // The extra newline before the assertion failure message makes the failure a + // bit easier to read when there's some generic boilerplate from the panic. + panic!("\n{}", e); + } + } + }; +} +pub use assert_that; + +/// Asserts that the given predicate applied to the given arguments returns +/// true, panicking if it does not. +/// +/// One may optionally add arguments which will be formatted and appended to a +/// failure message. For example: +/// +/// ```should_panic +/// # use googletest::prelude::*; +/// # fn should_fail() { +/// let extra_information = "Some additional information"; +/// assert_pred!(1 == 2, "Test failed. Extra information: {extra_information}."); +/// # } +/// # should_fail(); +/// ``` +/// +/// The output is as follows: +/// +/// ```text +/// 1 == 2 was false with +/// Test failed. Extra information: Some additional information. +/// ``` +/// +/// **Note for users of [GoogleTest for C++](http://google.github.io/googletest/):** +/// This differs from the `ASSERT_PRED*` family of macros in that it panics +/// rather than triggering an early return from the invoking function. To get +/// behaviour equivalent to `ASSERT_PRED*`, use [`verify_pred!`] with the `?` +/// operator. +#[macro_export] +macro_rules! assert_pred { + ($content:expr $(,)?) => { + match $crate::verify_pred!($content) { + Ok(_) => {} + Err(e) => { + // The extra newline before the assertion failure message makes the failure a + // bit easier to read when there's some generic boilerplate from the panic. + panic!("\n{}", e); + } + } + }; + + // w/ format args + ($content:expr $(,)?, $($format_args:expr),* $(,)?) => { + match $crate::GoogleTestSupport::with_failure_message( + $crate::verify_pred!($content), + || format!($($format_args),*) + ) { + Ok(_) => {} + Err(e) => { + // The extra newline before the assertion failure message makes the failure a + // bit easier to read when there's some generic boilerplate from the panic. + panic!("\n{}", e); + } + } + }; +} +pub use assert_pred; + +/// Matches the given value against the given matcher, marking the test as +/// failed but continuing execution if it does not match. +/// +/// This is a *non-fatal* assertion: the test continues +/// execution in the event of assertion failure. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The assertion must +/// occur in the same thread as that running the test itself. +/// +/// Invoking this macro is equivalent to using +/// [`and_log_failure`](crate::GoogleTestSupport::and_log_failure) as follows: +/// +/// ```ignore +/// verify_that!(actual, expected).and_log_failure() +/// ``` +/// +/// One may optionally add arguments which will be formatted and appended to a +/// failure message. For example: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_fail() -> std::result::Result<(), googletest::internal::test_outcome::TestFailure> { +/// # googletest::internal::test_outcome::TestOutcome::init_current_test_outcome(); +/// let value = 2; +/// let extra_information = "Some additional information"; +/// expect_that!(value, eq(3), "Test failed. Extra information: {extra_information}."); +/// # googletest::internal::test_outcome::TestOutcome::close_current_test_outcome::<&str>(Ok(())) +/// # } +/// # should_fail().unwrap_err(); +/// ``` +/// +/// This is output as follows: +/// +/// ```text +/// Value of: value +/// Expected: is equal to 3 +/// Actual: 2, +/// which isn't equal to 3 +/// at ... +/// Test failed. Extra information: Some additional information. +/// ``` +#[macro_export] +macro_rules! expect_that { + // specialized to sequence: + ($actual:expr, [$($expected:expr),*] $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_that!($actual, [$($expected),*])); + }}; + + // specialized to unordered sequence: + ($actual:expr, {$($expected:expr),*} $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_that!($actual, {$($expected),*})); + }}; + + // w/ format args, specialized to sequence: + ($actual:expr, [$($expected:expr),*], $($format_args:expr),* $(,)?) => { + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_that!($actual, [$($expected),*]), + || format!($($format_args),*)) + }; + + // w/ format args, specialized to unordered sequence: + ($actual:expr, {$($expected:expr),*}, $($format_args:expr),* $(,)?) => { + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_that!($actual, {$($expected),*}), + || format!($($format_args),*)) + }; + + // general case: + ($actual:expr, $expected:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_that!($actual, $expected)); + }}; + + // w/ format args, general case: + ($actual:expr, $expected:expr, $($format_args:expr),* $(,)?) => { + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_that!($actual, $expected), + || format!($($format_args),*)) + }; +} +pub use expect_that; + +/// Asserts that the given predicate applied to the given arguments returns +/// true, failing the test but continuing execution if not. +/// +/// This is a *non-fatal* predicate assertion: the test +/// continues execution in the event of assertion failure. +/// +/// This can only be invoked inside tests with the +/// [`gtest`][crate::gtest] attribute. The assertion must +/// occur in the same thread as that running the test itself. +/// +/// Invoking this macro is equivalent to using +/// [`and_log_failure`](crate::GoogleTestSupport::and_log_failure) as follows: +/// +/// ```ignore +/// verify_pred!(predicate(...)).and_log_failure() +/// ``` +/// +/// One may optionally add arguments which will be formatted and appended to a +/// failure message. For example: +/// +/// ```ignore +/// use googletest::prelude::*; +/// +/// #[gtest] +/// fn should_fail() { +/// let extra_information = "Some additional information"; +/// expect_pred!(1 == 2, "Test failed. Extra information: {extra_information}."); +/// } +/// ``` +/// +/// The output is as follows: +/// +/// ```text +/// 1 == 2 was false with +/// Test failed. Extra information: Some additional information. +/// ``` +#[macro_export] +macro_rules! expect_pred { + ($content:expr $(,)?) => {{ + $crate::GoogleTestSupport::and_log_failure($crate::verify_pred!($content)); + }}; + // w/ format args + ($content:expr $(,)?, $($format_args:expr),* $(,)?) => { + $crate::GoogleTestSupport::and_log_failure_with_message( + $crate::verify_pred!($content), + || format!($($format_args),*)) + }; +} +pub use expect_pred; + +/// Functions for use only by the procedural macros in this module. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use crate::{ + internal::test_outcome::TestAssertionFailure, + matcher::{create_assertion_failure, Matcher, MatcherResult}, + }; + use std::fmt::Debug; + + pub use ::googletest_macro::__googletest_macro_verify_pred; + + /// Extension trait to perform autoref through method lookup in the + /// assertion macros. With this trait, the subject can be either a value + /// or a reference. For example, this trait makes the following code + /// compile and work: + /// ``` + /// # use googletest::prelude::*; + /// # fn would_not_compile_without_autoref() -> Result<()> { + /// let not_copyable = vec![1,2,3]; + /// verify_that!(not_copyable, is_empty())?; + /// # Ok(()) + /// # } + /// ``` + /// See [Method Lookup](https://rustc-dev-guide.rust-lang.org/method-lookup.html) + pub trait Subject: Copy + Debug { + /// Checks whether the matcher `expected` matches the `Subject `self`, + /// adding a test failure report if it does not match. + /// + /// Returns `Ok(())` if the value matches and `Err(_)` if it does not + /// match. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[must_use = "The assertion result must be evaluated to affect the test result."] + #[track_caller] + fn check( + self, + expected: impl Matcher, + actual_expr: &'static str, + ) -> Result<(), TestAssertionFailure> { + match expected.matches(self) { + MatcherResult::Match => Ok(()), + MatcherResult::NoMatch => { + Err(create_assertion_failure(&expected, self, actual_expr)) + } + } + } + } + + impl Subject for T {} + + /// Creates a failure at specified location. + /// + /// **For internal use only. API stability is not guaranteed!** + #[must_use = "The assertion result must be evaluated to affect the test result."] + #[track_caller] + pub fn create_fail_result(message: String) -> crate::Result<()> { + Err(crate::internal::test_outcome::TestAssertionFailure::create(message)) + } +} + +#[cfg(test)] +mod tests { + use crate::{ + self as googletest, + assertions::{verify_eq, verify_that}, + matchers::{anything, err}, + test, Result as TestResult, + }; + + #[test] + fn verify_of_hash_maps_with_str_string_matching() -> TestResult<()> { + let hash_map: std::collections::HashMap = + std::collections::HashMap::from([("a".into(), "A".into()), ("b".into(), "B".into())]); + verify_eq!(hash_map, {("a", "A"), ("b", "B")}) + } + + #[test] + fn verify_of_hash_maps_with_ad_hoc_struct() -> TestResult<()> { + #[derive(PartialEq, Debug)] + struct Greek(String); + + let hash_map: std::collections::HashMap = std::collections::HashMap::from([ + ("a".into(), Greek("Alpha".into())), + ("b".into(), Greek("Beta".into())), + ]); + verify_eq!(hash_map, { + ("b", Greek("Beta".into())), + ("a", Greek("Alpha".into())), + }) + } + + #[test] + fn verify_of_hash_maps_with_i32s() -> TestResult<()> { + let hash_map: std::collections::HashMap = + std::collections::HashMap::from([(1, 1), (2, 4), (-1, 1), (-3, 9)]); + verify_eq!(hash_map, { + (-3, 9), + (-1, 1), + (1, 1), + (2, 4), + }) + } + + #[test] + fn verify_eq_of_unordered_pairs() -> TestResult<()> { + verify_eq!(vec![(1, 2), (2, 3)], {(1, 2), (2, 3)})?; + verify_eq!(vec![(1, 2), (2, 3)], {(2, 3), (1, 2)}) + } + + #[test] + fn verify_eq_of_unordered_structs() -> TestResult<()> { + #[derive(PartialEq, Debug)] + struct P(i32, i32); + + verify_eq!(vec![P(1, 1), P(1, 2), P(3, 7)], + {P(1, 1), P(1, 2), P(3, 7)})?; + verify_eq!(vec![P(1, 1), P(1, 2), P(3, 7)], + {P(3,7), P(1, 1), P(1, 2)}) + } + + #[test] + fn verify_eq_of_ordered_pairs() -> TestResult<()> { + verify_eq!(vec![(1, 2), (2, 3)], [(1, 2), (2, 3)]) + } + + #[test] + fn verify_eq_of_ordered_structs() -> TestResult<()> { + #[derive(PartialEq, Debug)] + struct P(i32, i32); + + verify_eq!(vec![P(1, 1), P(1, 2), P(3, 7)], [P(1, 1), P(1, 2), P(3, 7)]) + } + + #[test] + fn verify_eq_of_ordered_pairs_order_matters() -> TestResult<()> { + let result = googletest::verify_eq!(vec![(1, 2), (2, 3)], [(2, 3), (1, 2)]); + verify_that!(result, err(anything())) + } + + #[test] + fn verify_eq_of_ordered_structs_order_matters() -> TestResult<()> { + #[derive(PartialEq, Debug)] + struct P(i32, i32); + + let result = verify_eq!(vec![P(1, 1), P(1, 2), P(3, 7)], [P(3, 7), P(1, 1), P(1, 2)]); + verify_that!(result, err(anything())) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/description.rs b/third_party/crates/googletest-0.14.3/src/description.rs new file mode 100644 index 0000000..4e5272d --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/description.rs @@ -0,0 +1,406 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + borrow::Cow, + fmt::{Display, Formatter, Result}, +}; + +use crate::internal::description_renderer::{List, INDENTATION_SIZE}; + +/// A structured description, either of a (composed) matcher or of an +/// assertion failure. +/// +/// One can compose blocks of text into a `Description`. Each one appears on a +/// new line. For example: +/// +/// ``` +/// # use googletest::prelude::*; +/// # use googletest::description::Description; +/// let description = Description::new() +/// .text("A block") +/// .text("Another block"); +/// verify_that!(description, displays_as(eq("A block\nAnother block"))) +/// # .unwrap(); +/// ``` +/// +/// One can embed nested descriptions into a `Description`. The resulting +/// nested description is then rendered with an additional level of +/// indentation. For example: +/// +/// ``` +/// # use googletest::prelude::*; +/// # use googletest::description::Description; +/// let inner_description = Description::new() +/// .text("A block") +/// .text("Another block"); +/// let outer_description = Description::new() +/// .text("Header") +/// .nested(inner_description); +/// verify_that!(outer_description, displays_as(eq("\ +/// Header +/// A block +/// Another block"))) +/// # .unwrap(); +/// ``` +/// +/// One can also enumerate or bullet list the elements of a `Description`: +/// +/// ``` +/// # use googletest::prelude::*; +/// # use googletest::description::Description; +/// let description = Description::new() +/// .text("First item") +/// .text("Second item") +/// .bullet_list(); +/// verify_that!(description, displays_as(eq("\ +/// * First item +/// * Second item"))) +/// # .unwrap(); +/// ``` +/// +/// One can construct a `Description` from a [`String`] or a string slice, an +/// iterator thereof, or from an iterator over other `Description`s: +/// +/// ``` +/// # use googletest::description::Description; +/// let single_element_description: Description = +/// "A single block description".into(); +/// let two_element_description: Description = +/// ["First item", "Second item"].into_iter().collect(); +/// let two_element_description_from_strings: Description = +/// ["First item".to_string(), "Second item".to_string()].into_iter().collect(); +/// ``` +/// +/// No newline is added after the last element during rendering. This makes it +/// easier to support single-line matcher descriptions and match explanations. +#[derive(Debug, Default)] +pub struct Description { + elements: List, + initial_indentation: usize, + is_conjunction: bool, + is_disjunction: bool, +} + +impl Description { + /// Returns a new empty [`Description`]. + pub fn new() -> Self { + Default::default() + } + + /// Appends a block of text to this instance. + /// + /// The block is indented uniformly when this instance is rendered. + pub fn text(mut self, text: impl Into>) -> Self { + self.elements.push_literal(text.into()); + self + } + + /// Appends a nested [`Description`] to this instance. + /// + /// The nested [`Description`] `inner` is indented uniformly at the next + /// level of indentation when this instance is rendered. + pub fn nested(mut self, inner: Description) -> Self { + self.elements.push_nested(inner.elements); + self + } + + /// Appends all [`Description`] in the given sequence `inner` to this + /// instance. + /// + /// Each element is treated as a nested [`Description`] in the sense of + /// [`Self::nested`]. + pub fn collect(self, inner: impl IntoIterator) -> Self { + inner.into_iter().fold(self, |outer, inner| outer.nested(inner)) + } + + /// Indents the lines in elements of this description. + /// + /// This operation will be performed lazily when [`self`] is displayed. + /// + /// This will indent every line inside each element. + /// + /// For example: + /// + /// ``` + /// # use googletest::prelude::*; + /// # use googletest::description::Description; + /// let description = std::iter::once("A B C\nD E F".to_string()).collect::(); + /// verify_that!(description.indent(), displays_as(eq(" A B C\n D E F"))) + /// # .unwrap(); + /// ``` + pub fn indent(self) -> Self { + Self { initial_indentation: INDENTATION_SIZE, ..self } + } + + /// Instructs this instance to render its elements as a bullet list. + /// + /// Each element (from either [`Description::text`] or + /// [`Description::nested`]) is rendered as a bullet point. If an element + /// contains multiple lines, the following lines are aligned with the first + /// one in the block. + /// + /// For instance: + /// + /// ``` + /// # use googletest::prelude::*; + /// # use googletest::description::Description; + /// let description = Description::new() + /// .text("First line\nsecond line") + /// .bullet_list(); + /// verify_that!(description, displays_as(eq("\ + /// * First line + /// second line"))) + /// # .unwrap(); + /// ``` + pub fn bullet_list(self) -> Self { + Self { elements: self.elements.bullet_list(), ..self } + } + + /// Instructs this instance to render its elements as an enumerated list. + /// + /// Each element (from either [`Description::text`] or + /// [`Description::nested`]) is rendered with its zero-based index. If an + /// element contains multiple lines, the following lines are aligned with + /// the first one in the block. + /// + /// For instance: + /// + /// ``` + /// # use googletest::prelude::*; + /// # use googletest::description::Description; + /// let description = Description::new() + /// .text("First line\nsecond line") + /// .enumerate(); + /// verify_that!(description, displays_as(eq("\ + /// 0. First line + /// second line"))) + /// # .unwrap(); + /// ``` + pub fn enumerate(self) -> Self { + Self { elements: self.elements.enumerate(), ..self } + } + + /// Returns the length of elements. + pub fn len(&self) -> usize { + self.elements.len() + } + + /// Returns whether the set of elements is empty. + pub fn is_empty(&self) -> bool { + self.elements.is_empty() + } + + pub(crate) fn push_in_last_nested(mut self, inner: Description) -> Self { + self.elements.push_at_end(inner.elements); + self + } + + pub(crate) fn conjunction_description(self) -> Self { + Self { is_conjunction: true, ..self } + } + + pub(crate) fn is_conjunction_description(&self) -> bool { + self.is_conjunction + } + + pub(crate) fn disjunction_description(self) -> Self { + Self { is_disjunction: true, ..self } + } + + pub(crate) fn is_disjunction_description(&self) -> bool { + self.is_disjunction + } +} + +impl Display for Description { + fn fmt(&self, f: &mut Formatter) -> Result { + self.elements.render(f, self.initial_indentation) + } +} + +impl>> FromIterator for Description { + fn from_iter(iter: T) -> Self + where + T: IntoIterator, + { + Self { elements: iter.into_iter().map(ElementT::into).collect(), ..Default::default() } + } +} + +impl FromIterator for Description { + fn from_iter(iter: T) -> Self + where + T: IntoIterator, + { + Self { elements: iter.into_iter().map(|s| s.elements).collect(), ..Default::default() } + } +} + +impl>> From for Description { + fn from(value: T) -> Self { + let mut elements = List::default(); + elements.push_literal(value.into()); + Self { elements, ..Default::default() } + } +} + +#[cfg(test)] +mod tests { + use super::Description; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn renders_single_fragment() -> Result<()> { + let description: Description = "A B C".into(); + verify_that!(description, displays_as(eq("A B C"))) + } + + #[test] + fn renders_two_fragments() -> Result<()> { + let description = + ["A B C".to_string(), "D E F".to_string()].into_iter().collect::(); + verify_that!(description, displays_as(eq("A B C\nD E F"))) + } + + #[test] + fn nested_description_is_indented() -> Result<()> { + let description = Description::new() + .text("Header") + .nested(["A B C".to_string()].into_iter().collect::()); + verify_that!(description, displays_as(eq("Header\n A B C"))) + } + + #[test] + fn nested_description_indents_two_elements() -> Result<()> { + let description = Description::new().text("Header").nested( + ["A B C".to_string(), "D E F".to_string()].into_iter().collect::(), + ); + verify_that!(description, displays_as(eq("Header\n A B C\n D E F"))) + } + + #[test] + fn nested_description_indents_one_element_on_two_lines() -> Result<()> { + let description = Description::new().text("Header").nested("A B C\nD E F".into()); + verify_that!(description, displays_as(eq("Header\n A B C\n D E F"))) + } + + #[test] + fn single_fragment_renders_with_bullet_when_bullet_list_enabled() -> Result<()> { + let description = Description::new().text("A B C").bullet_list(); + verify_that!(description, displays_as(eq("* A B C"))) + } + + #[test] + fn single_nested_fragment_renders_with_bullet_when_bullet_list_enabled() -> Result<()> { + let description = Description::new().nested("A B C".into()).bullet_list(); + verify_that!(description, displays_as(eq("* A B C"))) + } + + #[test] + fn two_fragments_render_with_bullet_when_bullet_list_enabled() -> Result<()> { + let description = Description::new().text("A B C").text("D E F").bullet_list(); + verify_that!(description, displays_as(eq("* A B C\n* D E F"))) + } + + #[test] + fn two_nested_fragments_render_with_bullet_when_bullet_list_enabled() -> Result<()> { + let description = + Description::new().nested("A B C".into()).nested("D E F".into()).bullet_list(); + verify_that!(description, displays_as(eq("* A B C\n* D E F"))) + } + + #[test] + fn single_fragment_with_more_than_one_line_renders_with_one_bullet() -> Result<()> { + let description = Description::new().text("A B C\nD E F").bullet_list(); + verify_that!(description, displays_as(eq("* A B C\n D E F"))) + } + + #[test] + fn single_fragment_renders_with_enumeration_when_enumerate_enabled() -> Result<()> { + let description = Description::new().text("A B C").enumerate(); + verify_that!(description, displays_as(eq("0. A B C"))) + } + + #[test] + fn two_fragments_render_with_enumeration_when_enumerate_enabled() -> Result<()> { + let description = Description::new().text("A B C").text("D E F").enumerate(); + verify_that!(description, displays_as(eq("0. A B C\n1. D E F"))) + } + + #[test] + fn single_fragment_with_two_lines_renders_with_one_enumeration_label() -> Result<()> { + let description = Description::new().text("A B C\nD E F").enumerate(); + verify_that!(description, displays_as(eq("0. A B C\n D E F"))) + } + + #[test] + fn multi_digit_enumeration_renders_with_correct_offset() -> Result<()> { + let description = ["A B C\nD E F"; 11] + .into_iter() + .map(str::to_string) + .collect::() + .enumerate(); + verify_that!( + description, + displays_as(eq(indoc!( + " + 0. A B C + D E F + 1. A B C + D E F + 2. A B C + D E F + 3. A B C + D E F + 4. A B C + D E F + 5. A B C + D E F + 6. A B C + D E F + 7. A B C + D E F + 8. A B C + D E F + 9. A B C + D E F + 10. A B C + D E F" + ))) + ) + } + + #[test] + fn new_is_empty() -> Result<()> { + verify_that!(Description::new(), predicate(Description::is_empty)) + } + + #[test] + fn text_is_not_empty() -> Result<()> { + verify_that!(Description::new().text("something"), not(predicate(Description::is_empty))) + } + + #[test] + fn new_zero_length() -> Result<()> { + verify_that!(Description::new().len(), eq(0)) + } + + #[test] + fn text_one_length() -> Result<()> { + verify_that!(Description::new().text("something").len(), eq(1)) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/fixtures.rs b/third_party/crates/googletest-0.14.3/src/fixtures.rs new file mode 100644 index 0000000..23e1b48 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/fixtures.rs @@ -0,0 +1,317 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + any::{Any, TypeId}, + collections::HashMap, + ops::{Deref, DerefMut}, + sync::{Mutex, OnceLock}, +}; + +/// Interface for structure to be set up and torn down as part of a test. +/// Types implementing `Fixture` can be passed as a reference argument to a +/// test function. +/// +/// ```ignore +/// struct MyFixture { ... } +/// +/// impl Fixture for MyFixture { ... } +/// +/// #[gtest] +/// fn test_with_fixture(my_fixture: &MyFixture) {...} +/// ``` +pub trait Fixture: Sized { + /// Factory method of the `Fixture`. + /// + /// This method is called by the test harness before the test case + /// that uses this fixture. If this method returns an `Err(...)`, + /// then the test case is not evaluated, automatically fails, and + /// only the fixtures previously set up are torn down. + fn set_up() -> crate::Result; + + /// Clean up method for the fixture. + /// + /// This method is called by the test harness after the test case + /// that uses this fixture. If the `Fixture` has been set up, the + /// test harness will call this method, even if the test case failed + /// or panicked. + fn tear_down(self) -> crate::Result<()>; +} + +/// Interface for structure to be set up before the test case. +/// Types implementing `ConsumableFixture` can be passed by value to +/// a test function. +/// +/// ```ignore +/// struct MyFixture { ... } +/// +/// impl ConsumableFixture for MyFixture { ... } +/// +/// #[gtest] +/// fn test_with_fixture(my_fixture: MyFixture) {...} +/// ``` +pub trait ConsumableFixture: Sized { + /// Factory method of the `ConsumableFixture`. + /// + /// This method is called by the test harness before the test case + /// that uses this fixture. If this method returns an `Err(...)`, + /// then the test case is not evaluated, automatically fails, and + /// only the fixtures previously set up are torn down. + fn set_up() -> crate::Result; +} + +/// Generic adapter to implement `ConsumableFixture` on any type implementing +/// `Default`. +pub struct FixtureOf(T); + +impl ConsumableFixture for FixtureOf { + fn set_up() -> crate::Result { + Ok(Self(T::default())) + } +} + +impl Deref for FixtureOf { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for FixtureOf { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// Interface for structure to be set up only once before all tests. +/// Types implementing `StaticFixture` can be passed as a double referenced +/// argument to a test function. +/// +/// ```ignore +/// struct MyFixture{ ... } +/// +/// impl StaticFixture for MyFixture { ... } +/// +/// #[gtest] +/// fn test_with_fixture(my_fixture: &&MyFixture){...} +/// ``` +pub trait StaticFixture: Sized + Sync + Send { + /// Factory method of the `StaticFixture`. + /// + /// This method is called by the test harness before the first test case + /// using this fixture. If this method returns an `Err(...)`, then every + /// test case using this fixture is not evaluated and automatically fails. + fn set_up_once() -> crate::Result; +} + +impl Fixture for &'static F { + fn set_up() -> crate::Result { + static ONCE_FIXTURE_REPO: OnceLock< + Mutex>, + > = OnceLock::new(); + let mut map = ONCE_FIXTURE_REPO.get_or_init(|| Mutex::new(HashMap::new())).lock()?; + let any = + map.entry(TypeId::of::()).or_insert_with(|| Box::leak(Box::new(F::set_up_once()))); + match any.downcast_ref::>() { + Some(Ok(fixture)) => Ok(fixture), + Some(Err(e)) => Err(e.clone()), + None => panic!("Downcast failed. This is a bug in GoogleTest Rust"), + } + } + + // Note that this is `&F` being torn down, not `F`. + fn tear_down(self) -> crate::Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + use std::sync::Once; + + use super::FixtureOf; + use super::StaticFixture; + use crate as googletest; + use crate::prelude::*; + use crate::test; + use crate::Result; + + #[test] + fn fixture_no_fixture() -> Result<()> { + Ok(()) + } + + struct AlwaysSucceed; + + impl Fixture for AlwaysSucceed { + fn set_up() -> crate::Result { + Ok(Self) + } + + fn tear_down(self) -> crate::Result<()> { + Ok(()) + } + } + + #[test] + fn fixture_one_fixture(_: &AlwaysSucceed) -> Result<()> { + Ok(()) + } + + #[test] + fn fixture_three_fixtures( + _: &AlwaysSucceed, + _: &AlwaysSucceed, + _: &AlwaysSucceed, + ) -> Result<()> { + Ok(()) + } + + struct NotAFixture { + a_field: i32, + } + + impl Default for NotAFixture { + fn default() -> Self { + Self { a_field: 33 } + } + } + + #[test] + fn fixture_of_non_fixture(not_a_fixture: FixtureOf) -> Result<()> { + verify_that!(not_a_fixture.a_field, eq(33)) + } + + #[test] + fn fixture_of_non_fixture_mut(mut not_a_fixture: FixtureOf) -> Result<()> { + not_a_fixture.a_field += 10; + verify_that!(not_a_fixture.a_field, eq(43)) + } + struct PanickyFixture; + + impl Fixture for PanickyFixture { + fn set_up() -> crate::Result { + Ok(Self) + } + + fn tear_down(self) -> crate::Result<()> { + panic!("Whoooops"); + } + } + + #[test] + #[should_panic(expected = "Whoooops")] + fn fixture_teardown_called_even_if_test_fail(_: &PanickyFixture) -> Result<()> { + Err(googletest::TestAssertionFailure::create("It must fail!".into())) + } + + struct AbortIfNotTornDownFixture { + has_been_torn_down: bool, + } + + impl Fixture for AbortIfNotTornDownFixture { + fn set_up() -> crate::Result { + Ok(Self { has_been_torn_down: false }) + } + fn tear_down(mut self) -> crate::Result<()> { + self.has_been_torn_down = true; + Ok(()) + } + } + + impl Drop for AbortIfNotTornDownFixture { + fn drop(&mut self) { + if !self.has_been_torn_down { + eprintln!("AbortIfNotTornDownFixture was not torn down"); + std::process::abort(); + } + } + } + + #[test] + #[should_panic(expected = "simple_fail")] + fn fixture_torn_down_after_test_panics(f: &AbortIfNotTornDownFixture) { + expect_that!(f.has_been_torn_down, eq(false)); + panic!("simple_fail"); + } + + struct OnlyOnce; + + impl StaticFixture for OnlyOnce { + fn set_up_once() -> crate::Result { + static ONCE: Once = Once::new(); + assert!(!ONCE.is_completed()); + ONCE.call_once(|| {}); + Ok(Self) + } + } + + #[test] + fn static_fixture_works(_: &&OnlyOnce) {} + + #[test] + fn static_fixture_same_static_fixture_twice(once: &&OnlyOnce, twice: &&OnlyOnce) { + // checks it points to the same memory address. + let once: *const OnlyOnce = *once; + let twice: *const OnlyOnce = *twice; + expect_eq!(once, twice); + } + + struct AnotherStaticFixture; + + impl StaticFixture for AnotherStaticFixture { + fn set_up_once() -> crate::Result { + Ok(Self) + } + } + + #[test] + fn static_fixture_two_different_static_fixtures(_: &&OnlyOnce, _: &&AnotherStaticFixture) {} + + struct FailingSetUp; + + impl Fixture for FailingSetUp { + fn set_up() -> crate::Result { + Err(googletest::TestAssertionFailure::create("sad fixture".into())) + } + + fn tear_down(self) -> crate::Result<()> { + unreachable!(); + } + } + + #[test] + #[should_panic(expected = "See failure output above")] + fn failing_fixture_causes_test_failure(_: &FailingSetUp) { + unreachable!() + } + + struct FailingTearDown; + + impl Fixture for FailingTearDown { + fn set_up() -> crate::Result { + Ok(Self) + } + + fn tear_down(self) -> crate::Result<()> { + Err(googletest::TestAssertionFailure::create("It must fail!".into())) + } + } + + #[test] + #[should_panic(expected = "See failure output above")] + fn failing_teardown_causes_test_failure(_: &FailingTearDown) {} +} diff --git a/third_party/crates/googletest-0.14.3/src/fmt.rs b/third_party/crates/googletest-0.14.3/src/fmt.rs new file mode 100644 index 0000000..e78228b --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/fmt.rs @@ -0,0 +1,67 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Functions for use only by the procedural macros in this module. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use std::fmt::{Debug, Write}; + + /// Wrapper to allow for inherent-method specialization based on whether a + /// type implements `Debug`. + pub struct FormatWrapper<'a, T: ?Sized>(pub &'a T); + + /// Default implementation to render values that implement `Debug`. + /// + /// Used for autoref specialization to conditionally + /// render only values that implement `Debug`. See also + /// [`FormatNonDebugFallback`]. + impl FormatWrapper<'_, T> { + #[track_caller] + pub fn __googletest_write_expr_value(&self, output: &mut dyn Write, expr_label: &str) { + write!(output, "\n {} = {:?},", expr_label, self.0) + .expect("Formatting to String should never fail"); + } + } + + /// Fallback implementation for rendering values for non-`Debug` types.. + /// + /// Used for inherent-method specialization to conditionally render only + /// values that implement `Debug`. See also the specialized inherent impl on + /// [`FormatWrapper`] above. + pub trait FormatNonDebugFallback { + fn __googletest_write_expr_value(&self, output: &mut dyn Write, expr_label: &str); + } + + impl FormatNonDebugFallback for FormatWrapper<'_, T> { + #[track_caller] + fn __googletest_write_expr_value(&self, output: &mut dyn Write, expr_label: &str) { + write!(output, "\n {expr_label} does not implement Debug,") + .expect("Formatting to String should never fail"); + } + } + + #[macro_export] + macro_rules! __googletest__write_expr_value( + ($output:expr, $expr_str:expr, $value:expr $(,)?) => { + { + use $crate::fmt::internal::FormatNonDebugFallback as _; + $crate::fmt::internal::FormatWrapper(&$value) + .__googletest_write_expr_value(&mut $output, $expr_str) + } + } + ); + pub use __googletest__write_expr_value; +} diff --git a/third_party/crates/googletest-0.14.3/src/internal/description_renderer.rs b/third_party/crates/googletest-0.14.3/src/internal/description_renderer.rs new file mode 100644 index 0000000..4be7001 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/internal/description_renderer.rs @@ -0,0 +1,626 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + borrow::Cow, + fmt::{Result, Write}, +}; + +/// Number of space used to indent lines when no alignement is required. +pub(crate) const INDENTATION_SIZE: usize = 2; + +/// A list of [`Block`] possibly rendered with a [`Decoration`]. +/// +/// This is the top-level renderable component, corresponding to the description +/// or match explanation of a single matcher. +/// +/// The constituent [`Block`] of a `List` can be decorated with either bullets +/// (`* `) or enumeration (`0. `, `1. `, ...). This is controlled via the +/// methods [`List::bullet_list`] and [`List::enumerate`]. By default, there is +/// no decoration. +/// +/// A `List` can be constructed as follows: +/// +/// * [`Default::default()`] constructs an empty `List`. +/// * [`Iterator::collect()`] on an [`Iterator`] of [`Block`]. +/// * [`Iterator::collect()`] on an [`Iterator`] of `String`, which produces a +/// [`Block::Literal`] for each `String`. +/// * [`Iterator::collect()`] on an [`Iterator`] of `List`, which produces a +/// [`Block::Nested`] for each `List`. +#[derive(Debug, Default)] +pub(crate) struct List(Vec, Decoration); + +impl List { + /// Render this instance using the formatter `f`. + /// + /// Indent each line of output by `indentation` spaces. + pub(crate) fn render(&self, f: &mut dyn Write, indentation: usize) -> Result { + self.render_with_prefix(f, indentation, "".into()) + } + + /// Append a new [`Block`] containing `literal`. + /// + /// The input `literal` is split into lines so that each line will be + /// indented correctly. + pub(crate) fn push_literal(&mut self, literal: Cow<'static, str>) { + self.0.push(literal.into()); + } + + /// Append a new [`Block`] containing `inner` as a nested [`List`]. + pub(crate) fn push_nested(&mut self, inner: List) { + self.0.push(Block::Nested(inner)); + } + + /// Render each [`Block`] of this instance preceded with a bullet "* ". + pub(crate) fn bullet_list(self) -> Self { + Self(self.0, Decoration::Bullet) + } + + /// Render each [`Block`] of this instance preceded with its 0-based index. + pub(crate) fn enumerate(self) -> Self { + Self(self.0, Decoration::Enumerate) + } + + /// Return the number of [`Block`] in this instance. + pub(crate) fn len(&self) -> usize { + self.0.len() + } + + /// Return `true` if there are no [`Block`] in this instance, `false` + /// otherwise. + pub(crate) fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Append a new [`List`] in the last element which must be a + /// [`Block::Nested`]. Panic if `self` is empty or the last element is + /// not [`Block::Nested`]. + pub(crate) fn push_at_end(&mut self, list: List) { + if let Some(Block::Nested(self_list)) = self.0.last_mut() { + self_list.push_nested(list); + } else { + panic!("pushing elements at the end of {self:#?} which last element is not Nested") + } + } + + fn render_with_prefix( + &self, + f: &mut dyn Write, + indentation: usize, + prefix: Cow<'static, str>, + ) -> Result { + if self.0.is_empty() { + return Ok(()); + } + + let enumeration_padding = self.enumeration_padding(); + + self.0[0].render( + f, + indentation, + self.full_prefix(0, enumeration_padding, &prefix).into(), + )?; + for (index, block) in self.0[1..].iter().enumerate() { + writeln!(f)?; + block.render( + f, + indentation + prefix.len(), + self.prefix(index + 1, enumeration_padding), + )?; + } + Ok(()) + } + + fn full_prefix(&self, index: usize, enumeration_padding: usize, prior_prefix: &str) -> String { + format!("{prior_prefix}{}", self.prefix(index, enumeration_padding)) + } + + fn prefix(&self, index: usize, enumeration_padding: usize) -> Cow<'static, str> { + match self.1 { + Decoration::None => "".into(), + Decoration::Bullet => "* ".into(), + Decoration::Enumerate => format!("{index:>enumeration_padding$}. ").into(), + } + } + + fn enumeration_padding(&self) -> usize { + match self.1 { + Decoration::None => 0, + Decoration::Bullet => 0, + Decoration::Enumerate => { + if self.0.len() > 1 { + ((self.0.len() - 1) as f64).log10().floor() as usize + 1 + } else { + // Avoid negative logarithm when there is only 0 or 1 element. + 1 + } + } + } + } +} + +impl FromIterator for List { + fn from_iter(iter: T) -> Self + where + T: IntoIterator, + { + Self(iter.into_iter().collect(), Decoration::None) + } +} + +impl>> FromIterator for List { + fn from_iter(iter: T) -> Self + where + T: IntoIterator, + { + Self(iter.into_iter().map(|b| b.into().into()).collect(), Decoration::None) + } +} + +impl FromIterator for List { + fn from_iter(iter: T) -> Self + where + T: IntoIterator, + { + Self(iter.into_iter().map(Block::nested).collect(), Decoration::None) + } +} + +/// A sequence of [`Fragment`] or a nested [`List`]. +/// +/// This may be rendered with a prefix specified by the [`Decoration`] of the +/// containing [`List`]. In this case, all lines are indented to align with the +/// first character of the first line of the block. +#[derive(Debug)] +enum Block { + /// A block of text. + /// + /// Each constituent [`Fragment`] contains one line of text. The lines are + /// indented uniformly to the current indentation of this block when + /// rendered. + Literal(Vec), + + /// A nested [`List`]. + /// + /// The [`List`] is rendered recursively at the next level of indentation. + Nested(List), +} + +impl Block { + fn nested(inner: List) -> Self { + Self::Nested(inner) + } + + fn render(&self, f: &mut dyn Write, indentation: usize, prefix: Cow<'static, str>) -> Result { + match self { + Self::Literal(fragments) => { + if fragments.is_empty() { + return Ok(()); + } + + write!(f, "{:indentation$}{prefix}", "")?; + fragments[0].render(f)?; + let block_indentation = indentation + prefix.as_ref().len(); + for fragment in &fragments[1..] { + writeln!(f)?; + write!(f, "{:block_indentation$}", "")?; + fragment.render(f)?; + } + Ok(()) + } + Self::Nested(inner) => inner.render_with_prefix( + f, + indentation + INDENTATION_SIZE.saturating_sub(prefix.len()), + prefix, + ), + } + } +} + +impl From for Block { + fn from(value: String) -> Self { + Block::Literal(value.lines().map(|v| Fragment(v.to_string().into())).collect()) + } +} + +impl From<&'static str> for Block { + fn from(value: &'static str) -> Self { + Block::Literal(value.lines().map(|v| Fragment(v.into())).collect()) + } +} + +impl From> for Block { + fn from(value: Cow<'static, str>) -> Self { + match value { + Cow::Borrowed(value) => value.into(), + Cow::Owned(value) => value.into(), + } + } +} + +/// A string representing one line of a description or match explanation. +#[derive(Debug)] +struct Fragment(Cow<'static, str>); + +impl Fragment { + fn render(&self, f: &mut dyn Write) -> Result { + write!(f, "{}", self.0) + } +} + +/// The decoration which appears on [`Block`] of a [`List`] when rendered. +#[derive(Debug, Default)] +enum Decoration { + /// No decoration on each [`Block`]. The default. + #[default] + None, + + /// Each [`Block`] is preceded by a bullet (`* `). + Bullet, + + /// Each [`Block`] is preceded by its index in the [`List`] (`0. `, `1. `, + /// ...). + Enumerate, +} + +#[cfg(test)] +mod tests { + use super::{Block, Fragment, List}; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn renders_fragment() -> Result<()> { + let fragment = Fragment("A fragment".into()); + let mut result = String::new(); + + fragment.render(&mut result)?; + + verify_that!(result, eq("A fragment")) + } + + #[test] + fn renders_empty_block() -> Result<()> { + let block = Block::Literal(vec![]); + let mut result = String::new(); + + block.render(&mut result, 0, "".into())?; + + verify_that!(result, eq("")) + } + + #[test] + fn renders_block_with_one_fragment() -> Result<()> { + let block: Block = "A fragment".into(); + let mut result = String::new(); + + block.render(&mut result, 0, "".into())?; + + verify_that!(result, eq("A fragment")) + } + + #[test] + fn renders_block_with_two_fragments() -> Result<()> { + let block: Block = "A fragment\nAnother fragment".into(); + let mut result = String::new(); + + block.render(&mut result, 0, "".into())?; + + verify_that!(result, eq("A fragment\nAnother fragment")) + } + + #[test] + fn renders_indented_block() -> Result<()> { + let block: Block = "A fragment\nAnother fragment".into(); + let mut result = String::new(); + + block.render(&mut result, 2, "".into())?; + + verify_that!(result, eq(" A fragment\n Another fragment")) + } + + #[test] + fn renders_block_with_prefix() -> Result<()> { + let block: Block = "A fragment\nAnother fragment".into(); + let mut result = String::new(); + + block.render(&mut result, 0, "* ".into())?; + + verify_that!(result, eq("* A fragment\n Another fragment")) + } + + #[test] + fn renders_indented_block_with_prefix() -> Result<()> { + let block: Block = "A fragment\nAnother fragment".into(); + let mut result = String::new(); + + block.render(&mut result, 2, "* ".into())?; + + verify_that!(result, eq(" * A fragment\n Another fragment")) + } + + #[test] + fn renders_empty_list() -> Result<()> { + let list = list(vec![]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("")) + } + + #[test] + fn renders_plain_list_with_one_block() -> Result<()> { + let list = list(vec!["A fragment".into()]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("A fragment")) + } + + #[test] + fn renders_plain_list_with_two_blocks() -> Result<()> { + let list = list(vec!["A fragment".into(), "A fragment in a second block".into()]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("A fragment\nA fragment in a second block")) + } + + #[test] + fn renders_plain_list_with_one_block_with_two_fragments() -> Result<()> { + let list = list(vec!["A fragment\nA second fragment".into()]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("A fragment\nA second fragment")) + } + + #[test] + fn renders_nested_plain_list_with_one_block() -> Result<()> { + let list = list(vec![Block::nested(list(vec!["A fragment".into()]))]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq(" A fragment")) + } + + #[test] + fn renders_nested_plain_list_with_two_blocks() -> Result<()> { + let list = list(vec![Block::nested(list(vec![ + "A fragment".into(), + "A fragment in a second block".into(), + ]))]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq(" A fragment\n A fragment in a second block")) + } + + #[test] + fn renders_nested_plain_list_with_one_block_with_two_fragments() -> Result<()> { + let list = list(vec![Block::nested(list(vec!["A fragment\nA second fragment".into()]))]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq(" A fragment\n A second fragment")) + } + + #[test] + fn renders_bulleted_list_with_one_block() -> Result<()> { + let list = list(vec!["A fragment".into()]).bullet_list(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("* A fragment")) + } + + #[test] + fn renders_bulleted_list_with_two_blocks() -> Result<()> { + let list = + list(vec!["A fragment".into(), "A fragment in a second block".into()]).bullet_list(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("* A fragment\n* A fragment in a second block")) + } + + #[test] + fn renders_bulleted_list_with_one_block_with_two_fragments() -> Result<()> { + let list = list(vec!["A fragment\nA second fragment".into()]).bullet_list(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("* A fragment\n A second fragment")) + } + + #[test] + fn renders_nested_bulleted_list_with_one_block() -> Result<()> { + let list = list(vec![Block::nested(list(vec!["A fragment".into()]).bullet_list())]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq(" * A fragment")) + } + + #[test] + fn renders_nested_bulleted_list_with_two_blocks() -> Result<()> { + let list = list(vec![Block::nested( + list(vec!["A fragment".into(), "A fragment in a second block".into()]).bullet_list(), + )]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq(" * A fragment\n * A fragment in a second block")) + } + + #[test] + fn renders_nested_bulleted_list_with_one_block_with_two_fragments() -> Result<()> { + let list = list(vec![Block::nested( + list(vec!["A fragment\nA second fragment".into()]).bullet_list(), + )]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq(" * A fragment\n A second fragment")) + } + + #[test] + fn renders_enumerated_list_with_one_block() -> Result<()> { + let list = list(vec!["A fragment".into()]).enumerate(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("0. A fragment")) + } + + #[test] + fn renders_enumerated_list_with_two_blocks() -> Result<()> { + let list = + list(vec!["A fragment".into(), "A fragment in a second block".into()]).enumerate(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("0. A fragment\n1. A fragment in a second block")) + } + + #[test] + fn renders_enumerated_list_with_one_block_with_two_fragments() -> Result<()> { + let list = list(vec!["A fragment\nA second fragment".into()]).enumerate(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("0. A fragment\n A second fragment")) + } + + #[test] + fn aligns_renders_enumerated_list_with_more_than_ten_blocks() -> Result<()> { + let list = + (0..11).map(|i| Block::from(format!("Fragment {i}"))).collect::().enumerate(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!( + result, + eq(indoc! {" + 0. Fragment 0 + 1. Fragment 1 + 2. Fragment 2 + 3. Fragment 3 + 4. Fragment 4 + 5. Fragment 5 + 6. Fragment 6 + 7. Fragment 7 + 8. Fragment 8 + 9. Fragment 9 + 10. Fragment 10"}) + ) + } + + #[test] + fn renders_fragment_plus_nested_plain_list_with_one_block() -> Result<()> { + let list = + list(vec!["A fragment".into(), Block::nested(list(vec!["Another fragment".into()]))]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("A fragment\n Another fragment")) + } + + #[test] + fn renders_double_nested_plain_list_with_one_block() -> Result<()> { + let list = + list(vec![Block::nested(list(vec![Block::nested(list(vec!["A fragment".into()]))]))]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq(" A fragment")) + } + + #[test] + fn renders_headers_plus_double_nested_plain_list() -> Result<()> { + let list = list(vec![ + "First header".into(), + Block::nested(list(vec![ + "Second header".into(), + Block::nested(list(vec!["A fragment".into()])), + ])), + ]); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("First header\n Second header\n A fragment")) + } + + #[test] + fn renders_double_nested_bulleted_list() -> Result<()> { + let list = + list(vec![Block::nested(list(vec!["A fragment".into()]).bullet_list())]).bullet_list(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("* * A fragment")) + } + + #[test] + fn renders_nested_enumeration_with_two_blocks_inside_bulleted_list() -> Result<()> { + let list = + list(vec![Block::nested(list(vec!["Block 1".into(), "Block 2".into()]).enumerate())]) + .bullet_list(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("* 0. Block 1\n 1. Block 2")) + } + + #[test] + fn renders_nested_enumeration_with_block_with_two_fragments_inside_bulleted_list() -> Result<()> + { + let list = list(vec![Block::nested( + list(vec!["A fragment\nAnother fragment".into()]).enumerate(), + )]) + .bullet_list(); + let mut result = String::new(); + + list.render(&mut result, 0)?; + + verify_that!(result, eq("* 0. A fragment\n Another fragment")) + } + + fn list(blocks: Vec) -> List { + List(blocks, super::Decoration::None) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/internal/glob.rs b/third_party/crates/googletest-0.14.3/src/internal/glob.rs new file mode 100644 index 0000000..cd549fe --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/internal/glob.rs @@ -0,0 +1,312 @@ +//! Pattern matches using glob wildcards `*` and `?` with the +//! linear-time algorithm . + +use std::iter::Peekable; +use std::str::Chars; + +pub struct Pattern(String); + +impl Pattern { + /// Creates a new pattern matcher. Each pattern consists of + /// regular characters, single-character wildcards `'?'`, and + /// multi-character wildcards `'*'`. + pub fn new(pattern: String) -> Self { + Self(pattern) + } + + /// Returns true if and only if the wildcard pattern matches the + /// string. + pub fn matches(&self, string: &str) -> bool { + let processor = Processor { + pattern: self.0.chars().peekable(), + string: string.chars().peekable(), + restart: None, + }; + + processor.process() + } +} + +type PeekableChars<'a> = Peekable>; + +/// Represents the state we need to restart search from a star wildcard (`*`). +struct Restart<'a> { + pattern_next: PeekableChars<'a>, + string_next: PeekableChars<'a>, +} + +/// The runtime state for glob matching. +struct Processor<'a> { + pattern: PeekableChars<'a>, + string: PeekableChars<'a>, + restart: Option>, +} + +/// Represents what to do after any step through the processor. +enum StepOutcome { + Proceed, + TryRestart, +} + +impl Processor<'_> { + /// Runs the pattern matching until we find an unrecoverable + /// mismatch, or the input is consumed. + fn process(mut self) -> bool { + while self.is_unfinished() { + let outcome = self.step(); + if let StepOutcome::TryRestart = outcome { + let restarted = self.try_restart(); + if !restarted { + return false; + } + } + } + + true + } + + fn is_unfinished(&mut self) -> bool { + self.pattern.peek().is_some() || self.string.peek().is_some() + } + + /// Takes a single step forward, and returns whether to proceed or try to + /// restart. + fn step(&mut self) -> StepOutcome { + match self.pattern.peek() { + Some('?') => self.step_question_wildcard(), + Some('*') => self.step_star_wildcard(), + Some(pattern_ch) => { + let pattern_ch = *pattern_ch; + self.step_ordinary_character(pattern_ch) + } + None => StepOutcome::TryRestart, + } + } + + /// Match any single character. + fn step_question_wildcard(&mut self) -> StepOutcome { + if self.string.peek().is_some() { + _ = self.pattern.next(); + _ = self.string.next(); + StepOutcome::Proceed + } else { + StepOutcome::TryRestart + } + } + + /// Match zero or more characters. Start by skipping over the + /// wildcard and matching zero characters from string. If that + /// fails, restart and match one more character than the last + /// attempt. + fn step_star_wildcard(&mut self) -> StepOutcome { + self.restart = if self.string.peek().is_none() { + // Subtle: if the string is already exhausted, we mark + // that we can't restart. + None + } else { + let pattern_next = self.pattern.clone(); + let mut string_next = self.string.clone(); + string_next.next(); + Some(Restart { pattern_next, string_next }) + }; + + _ = self.pattern.next(); + StepOutcome::Proceed + } + + /// Match an ordinary (non-wildcard) character. + fn step_ordinary_character(&mut self, pattern_ch: char) -> StepOutcome { + if self.string.peek() == Some(&pattern_ch) { + _ = self.pattern.next(); + _ = self.string.next(); + StepOutcome::Proceed + } else { + StepOutcome::TryRestart + } + } + + /// Try to restart from failing to match a character. If true, the + /// matching can restart. + fn try_restart(&mut self) -> bool { + if let Some(Restart { pattern_next, string_next }) = &self.restart { + self.pattern = pattern_next.clone(); + self.string = string_next.clone(); + true + } else { + false + } + } +} + +/// Returns true if `s` contains glob wildcards. +pub fn is_glob_pattern(s: &str) -> bool { + s.contains(['?', '*']) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::*; + + #[test] + fn simple_character_match() -> Result<()> { + verify_that!(Pattern::new("a".into()).matches("a"), is_true()) + } + + #[test] + fn simple_character_mismatch() -> Result<()> { + verify_that!(Pattern::new("b".into()).matches("a"), is_false()) + } + + #[test] + fn simple_word_mismatch() -> Result<()> { + verify_that!(Pattern::new("judgement".into()).matches("judgment"), is_false()) + } + + #[test] + fn question_match() -> Result<()> { + verify_that!(Pattern::new("?".into()).matches("a"), is_true()) + } + + #[test] + fn simple_word_question_match() -> Result<()> { + let pattern = Pattern::new("judg?ment".into()); + verify_that!(pattern.matches("judgment"), is_false())?; + verify_that!(pattern.matches("judgement"), is_true())?; + verify_that!(pattern.matches("judge ment"), is_false())?; + Ok(()) + } + + #[test] + fn question_mismatch() -> Result<()> { + let pattern = Pattern::new("?".into()); + verify_that!(pattern.matches(""), is_false())?; + verify_that!(pattern.matches("aa"), is_false())?; + Ok(()) + } + + #[test] + fn glob_on_empty() -> Result<()> { + verify_that!(Pattern::new("*".into()).matches(""), is_true())?; + verify_that!(Pattern::new("**".into()).matches(""), is_true())?; + Ok(()) + } + + #[test] + fn glob_prefix() -> Result<()> { + let pattern = Pattern::new("*a".into()); + verify_that!(pattern.matches(""), is_false())?; + verify_that!(pattern.matches("a"), is_true())?; + verify_that!(pattern.matches("ba"), is_true())?; + verify_that!(pattern.matches("bba"), is_true())?; + verify_that!(pattern.matches("bbab"), is_false())?; + Ok(()) + } + + #[test] + fn glob_within() -> Result<()> { + let pattern = Pattern::new("b*a".into()); + verify_that!(pattern.matches(""), is_false())?; + verify_that!(pattern.matches("b"), is_false())?; + verify_that!(pattern.matches("bb"), is_false())?; + verify_that!(pattern.matches("ba"), is_true())?; + verify_that!(pattern.matches("bbbba"), is_true())?; + verify_that!(pattern.matches("baa"), is_true())?; + Ok(()) + } + + #[test] + fn glob_suffix() -> Result<()> { + let pattern = Pattern::new("ba*".into()); + verify_that!(pattern.matches(""), is_false())?; + verify_that!(pattern.matches("b"), is_false())?; + verify_that!(pattern.matches("bb"), is_false())?; + verify_that!(pattern.matches("ba"), is_true())?; + verify_that!(pattern.matches("baa"), is_true())?; + verify_that!(pattern.matches("bab"), is_true())?; + verify_that!(pattern.matches("bba"), is_false())?; + verify_that!(pattern.matches("bbbba"), is_false())?; + Ok(()) + } + + #[test] + fn redundant_stars() -> Result<()> { + let pattern = Pattern::new("**a".into()); + verify_that!(pattern.matches(""), is_false())?; + verify_that!(pattern.matches("a"), is_true())?; + verify_that!(pattern.matches("ba"), is_true())?; + verify_that!(pattern.matches("bba"), is_true())?; + verify_that!(pattern.matches("bbab"), is_false())?; + verify_that!(pattern.matches("bbaba"), is_true())?; + Ok(()) + } + + #[test] + fn star_question_star_case() -> Result<()> { + let pattern = Pattern::new("*?*".into()); + verify_that!(pattern.matches(""), is_false())?; + verify_that!(pattern.matches("a"), is_true())?; + verify_that!(pattern.matches("aa"), is_true())?; + verify_that!(pattern.matches("aaa"), is_true())?; + Ok(()) + } + + #[test] + fn another_case_finding_two_separated_a() -> Result<()> { + let pattern = Pattern::new("*a?a*".into()); + verify_that!(pattern.matches(""), is_false())?; + verify_that!(pattern.matches("a"), is_false())?; + verify_that!(pattern.matches("aa"), is_false())?; + verify_that!(pattern.matches("aaa"), is_true())?; + verify_that!(pattern.matches("aba"), is_true())?; + verify_that!(pattern.matches("baba"), is_true())?; + verify_that!(pattern.matches("abab"), is_true())?; + verify_that!(pattern.matches("babab"), is_true())?; + Ok(()) + } + + #[test] + fn banana() -> Result<()> { + let pattern = Pattern::new("b?n???".into()); + verify_that!(pattern.matches("banana"), is_true())?; + verify_that!(pattern.matches("binary"), is_true())?; + verify_that!(pattern.matches("bundle"), is_true())?; + verify_that!(pattern.matches("bindir"), is_true())?; + + verify_that!(pattern.matches("bananas"), is_false())?; + verify_that!(pattern.matches("bucket"), is_false())?; + verify_that!(pattern.matches("budget"), is_false())?; + verify_that!(pattern.matches("bazzar"), is_false())?; + verify_that!(pattern.matches("burger"), is_false())?; + Ok(()) + } + + #[test] + fn glob_word() -> Result<()> { + let word = Pattern::new("*word*".into()); + verify_that!(word.matches("bird"), is_false())?; + verify_that!(word.matches("This is a wordy sentence"), is_true())?; + verify_that!(word.matches("word soup"), is_true())?; + verify_that!(word.matches("bird is the word"), is_true())?; + verify_that!(word.matches("word"), is_true())?; + Ok(()) + } + + #[test] + fn degenerate_glob() -> Result<()> { + verify_that!( + Pattern::new("************************************************.*".into()) + .matches("this is a test.com"), + is_true() + ) + } + + #[test] + fn degenerate_glob_2_mismatch() -> Result<()> { + // The first example from https://research.swtch.com/glob, with N=1000. + const N: usize = 1000; + let long_pattern = ["a*"; N].into_iter().collect::() + "b"; + let long_string = ["a"; N].into_iter().collect::(); + verify_that!(Pattern::new(long_pattern).matches(&long_string), is_false()) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/internal/mod.rs b/third_party/crates/googletest-0.14.3/src/internal/mod.rs new file mode 100644 index 0000000..b3198fe --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/internal/mod.rs @@ -0,0 +1,23 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc(hidden)] + +pub(crate) mod description_renderer; +pub mod glob; +pub mod scoped_trace; +pub mod test_data; +pub mod test_filter; +pub mod test_outcome; +pub mod test_sharding; diff --git a/third_party/crates/googletest-0.14.3/src/internal/scoped_trace.rs b/third_party/crates/googletest-0.14.3/src/internal/scoped_trace.rs new file mode 100644 index 0000000..80e1e28 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/internal/scoped_trace.rs @@ -0,0 +1,126 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cell::RefCell; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT_TRACE_ID: AtomicUsize = AtomicUsize::new(0); + +/// Information about a scoped trace. +#[derive(Clone, Debug)] +pub struct TraceInfo { + pub id: usize, + pub file: &'static str, + pub line: u32, + pub message: String, +} + +thread_local! { + static TRACE_STACK: RefCell> = const { RefCell::new(Vec::new()) }; +} + +/// RAII guard to manage the push and pop of trace information. +/// +/// This struct is `!Send` and `!Sync` to prevent it from being held across +/// `.await` points in async tests, which would cause incorrect trace tracking +/// if the task moves between threads. +#[doc(hidden)] +pub struct ScopedTraceGuard { + id: usize, + _phantom: std::marker::PhantomData<*mut ()>, +} + +impl ScopedTraceGuard { + #[doc(hidden)] + #[track_caller] + pub fn new(message: String) -> Self { + let caller = std::panic::Location::caller(); + let id = NEXT_TRACE_ID.fetch_add(1, Ordering::Relaxed); + TRACE_STACK.with(|stack| { + // Use try_borrow_mut to avoid double panic if called during unwinding. + if let Ok(mut s) = stack.try_borrow_mut() { + s.push(TraceInfo { id, file: caller.file(), line: caller.line(), message }); + } + }); + Self { id, _phantom: std::marker::PhantomData } + } +} + +impl Drop for ScopedTraceGuard { + fn drop(&mut self) { + TRACE_STACK.with(|stack| { + // Use try_borrow_mut to avoid double panic if called during unwinding. + if let Ok(mut s) = stack.try_borrow_mut() { + if let Some(pos) = s.iter().rposition(|t| t.id == self.id) { + s.remove(pos); + } + } + }); + } +} + +/// Retrieves a clone of the current thread's trace stack. +pub fn get_scoped_traces() -> Vec { + TRACE_STACK.with(|stack| stack.try_borrow().map(|s| s.clone()).unwrap_or_default()) +} + +// Test-only state and helpers, hidden from production API. +#[cfg(test)] +pub(crate) mod test_helpers { + use super::*; + use std::cell::Cell; + + thread_local! { + pub static CAPTURED_TRACES_IN_HOOK: RefCell> = const { RefCell::new(Vec::new()) }; + pub static USE_CAPTURE_HOOK: Cell = const { Cell::new(false) }; + } + + pub fn enable_capture_in_hook(enable: bool) { + USE_CAPTURE_HOOK.with(|v| v.set(enable)); + } + + pub fn get_captured_traces_in_hook() -> Vec { + CAPTURED_TRACES_IN_HOOK.with(|v| v.borrow().clone()) + } + + pub fn clear_captured_traces_in_hook() { + CAPTURED_TRACES_IN_HOOK.with(|v| v.borrow_mut().clear()); + } +} + +#[cfg(test)] +mod tests { + use super::test_helpers::*; + use super::*; + + #[test] + fn test_scoped_trace_fatal() { + enable_capture_in_hook(true); + clear_captured_traces_in_hook(); + + // Ensure hook is installed + crate::internal::test_outcome::TestOutcome::init_current_test_outcome(); + + let _ = std::panic::catch_unwind(|| { + let _guard = ScopedTraceGuard::new("Second trace".to_string()); + panic!("Intentional panic"); + }); + + let captured = get_captured_traces_in_hook(); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].message, "Second trace"); + + enable_capture_in_hook(false); + } +} diff --git a/third_party/crates/googletest-0.14.3/src/internal/test_data.rs b/third_party/crates/googletest-0.14.3/src/internal/test_data.rs new file mode 100644 index 0000000..bc9c817 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/internal/test_data.rs @@ -0,0 +1,29 @@ +//! This is for testing only to provide fully-qualified struct paths to macros. + +#[doc(hidden)] +#[derive(Debug)] +pub struct TestStruct { + #[doc(hidden)] + pub value: u32, +} + +impl TestStruct { + #[doc(hidden)] + pub fn get_value(&self) -> u32 { + self.value + } +} + +#[doc(hidden)] +#[derive(Debug)] +pub struct GenericTestStruct { + #[doc(hidden)] + pub value: T, +} + +impl GenericTestStruct { + #[doc(hidden)] + pub fn get_value(&self, a: U) -> U { + a + } +} diff --git a/third_party/crates/googletest-0.14.3/src/internal/test_filter.rs b/third_party/crates/googletest-0.14.3/src/internal/test_filter.rs new file mode 100644 index 0000000..559bb40 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/internal/test_filter.rs @@ -0,0 +1,261 @@ +//! Implements the googletest test filtering protocol. +//! +//! The Google test filtering protocol consists of the following +//! environment variable: +//! +//! * TESTBRIDGE_TEST_ONLY: string passed from --test_filter +//! +//! The format of a filter is a ‘,‘-separated list of wildcard +//! patterns (called the positive patterns) optionally followed by a +//! ‘-’ and another ‘,‘-separated pattern list (called the negative +//! patterns). A test matches the filter if and only if it matches any +//! of the positive patterns but does not match any of the negative +//! patterns. (Note that this is a deliberate devation from GTest +//! C++, which uses colons to separate the patterns, as colons will +//! unfortunately clash with Rust's "::" namespacing operator.) +//! +//! As an example: "*mount*-*doom*" will accept any string that contains the +//! substring "mount", as long as it also doesn't contain "doom" +use crate::internal::glob::{is_glob_pattern, Pattern}; +use std::sync::OnceLock; + +static TEST_FILTER: OnceLock> = OnceLock::new(); + +pub fn test_should_run(test_name: &str) -> bool { + let test_filter = TEST_FILTER.get_or_init(|| { + if let Ok(testbridge_test_only) = std::env::var("TESTBRIDGE_TEST_ONLY") { + Box::new(get_test_filter(&testbridge_test_only)) + } else { + Box::new(AcceptAll) + } + }); + + test_filter.filter(test_name) +} + +trait TestFilter { + /// Returns true if the test should run. + fn filter(&self, test_name: &str) -> bool; +} + +struct AcceptAll; +impl TestFilter for AcceptAll { + fn filter(&self, _test_name: &str) -> bool { + true + } +} + +struct Equals(String); +impl TestFilter for Equals { + fn filter(&self, test_name: &str) -> bool { + test_name == self.0 + } +} + +struct Matches(Pattern); +impl TestFilter for Matches { + fn filter(&self, test_name: &str) -> bool { + self.0.matches(test_name) + } +} + +struct Collection { + // The positive portion: + positive_equals: Box<[Equals]>, + positive_matches: Box<[Matches]>, + + // The negative portion: + negative_equals: Box<[Equals]>, + negative_matches: Box<[Matches]>, +} + +impl TestFilter for Collection { + fn filter(&self, test_name: &str) -> bool { + (self.positive_equals.iter().any(|f| f.filter(test_name)) + || self.positive_matches.iter().any(|f| f.filter(test_name))) + && (!self.negative_equals.iter().any(|f| f.filter(test_name))) + && (!self.negative_matches.iter().any(|f| f.filter(test_name))) + } +} + +fn get_test_filter(testbridge_test_only: &str) -> Collection { + let positive_negative: Vec<&str> = testbridge_test_only.splitn(2, '-').collect(); + + let (positive_with_globs, positive_literals): (Vec<_>, Vec<_>) = { + let positive = positive_negative[0]; + if positive.is_empty() { + // Forces the empty positive filter to accept everything: + (vec!["*"], vec![]) + } else { + positive.split(',').partition(|s| is_glob_pattern(s)) + } + }; + + let (negative_with_globs, negative_literals): (Vec<_>, Vec<_>) = match positive_negative.get(1) + { + Some(negative) if !negative.is_empty() => { + negative.split(',').partition(|s| is_glob_pattern(s)) + } + _ => (vec![], vec![]), + }; + + Collection { + positive_equals: positive_literals.into_iter().map(|s| Equals(s.to_string())).collect(), + positive_matches: positive_with_globs + .into_iter() + .map(|s| Matches(Pattern::new(s.to_string()))) + .collect(), + negative_equals: negative_literals.into_iter().map(|s| Equals(s.to_string())).collect(), + negative_matches: negative_with_globs + .into_iter() + .map(|s| Matches(Pattern::new(s.to_string()))) + .collect(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::*; + + #[test] + fn accept_all_accepts_all() -> Result<()> { + let filter = AcceptAll; + + verify_that!(filter.filter(""), is_true())?; + verify_that!(filter.filter("abcdefg"), is_true())?; + Ok(()) + } + + #[test] + fn empty_filter_accepts_all() -> Result<()> { + let filter = get_test_filter(""); + + verify_that!(filter.filter(""), is_true())?; + verify_that!(filter.filter("abcdefg"), is_true())?; + Ok(()) + } + + #[test] + fn empty_negation_filter_accepts_all() -> Result<()> { + let filter = get_test_filter("-"); + + verify_that!(filter.filter(""), is_true())?; + verify_that!(filter.filter("abcdefg"), is_true())?; + Ok(()) + } + + #[test] + fn simple_literal_filter() -> Result<()> { + let filter = get_test_filter("*magic*"); + + verify_that!(filter.filter("this_is_magic"), is_true())?; + verify_that!(filter.filter(""), is_false())?; + verify_that!(filter.filter("magic"), is_true())?; + verify_that!(filter.filter("science"), is_false())?; + Ok(()) + } + + #[test] + fn star_globs() -> Result<()> { + let filter = get_test_filter("a*b"); + + verify_that!(filter.filter(""), is_false())?; + verify_that!(filter.filter("a"), is_false())?; + verify_that!(filter.filter("ab"), is_true())?; + verify_that!(filter.filter("a b"), is_true())?; + verify_that!(filter.filter("b"), is_false())?; + verify_that!(filter.filter("b a"), is_false())?; + Ok(()) + } + + #[test] + fn question_globs() -> Result<()> { + let filter = get_test_filter("a?b"); + + verify_that!(filter.filter(""), is_false())?; + verify_that!(filter.filter("a"), is_false())?; + verify_that!(filter.filter("ab"), is_false())?; + verify_that!(filter.filter("aXb"), is_true())?; + verify_that!(filter.filter("a b"), is_true())?; + verify_that!(filter.filter("b"), is_false())?; + verify_that!(filter.filter("b a"), is_false())?; + Ok(()) + } + + #[test] + fn collection() -> Result<()> { + let filter = get_test_filter("a,b"); + + verify_that!(filter.filter(""), is_false())?; + verify_that!(filter.filter("a"), is_true())?; + verify_that!(filter.filter("ab"), is_false())?; + verify_that!(filter.filter("a b"), is_false())?; + verify_that!(filter.filter("b"), is_true())?; + verify_that!(filter.filter("b a"), is_false())?; + verify_that!(filter.filter("c"), is_false())?; + Ok(()) + } + + #[test] + fn collection_with_globs() -> Result<()> { + let filter = get_test_filter("*test1*,*test2*"); + + verify_that!(filter.filter(""), is_false())?; + verify_that!(filter.filter("this is test1"), is_true())?; + verify_that!(filter.filter("and test2 is it"), is_true())?; + verify_that!(filter.filter("but test3 is not"), is_false())?; + Ok(()) + } + + #[test] + fn collection_with_globs_negation() -> Result<()> { + let filter = get_test_filter("*test*-*testbad"); + + verify_that!(filter.filter(""), is_false())?; + verify_that!(filter.filter("module"), is_false())?; + verify_that!(filter.filter("module::my_test1"), is_true())?; + verify_that!(filter.filter("module::my_test2"), is_true())?; + verify_that!(filter.filter("module::my_testbad"), is_false())?; + Ok(()) + } + + #[test] + fn mount_doom() -> Result<()> { + let filter = get_test_filter("*mount*-*doom*"); + + verify_that!(filter.filter(""), is_false())?; + verify_that!(filter.filter("mount rushmore"), is_true())?; + verify_that!(filter.filter("doom mount"), is_false())?; + verify_that!(filter.filter("dismount"), is_true())?; + verify_that!(filter.filter("mountains of moria"), is_true())?; + verify_that!(filter.filter("frodo and sam went to mount doom"), is_false())?; + Ok(()) + } + + #[test] + fn collection_with_only_negation() -> Result<()> { + let filter = get_test_filter("-testbad1,testbad2"); + + verify_that!(filter.filter(""), is_true())?; + verify_that!(filter.filter("test"), is_true())?; + verify_that!(filter.filter("testbad1"), is_false())?; + verify_that!(filter.filter("testbad2"), is_false())?; + verify_that!(filter.filter("testbad3"), is_true())?; + Ok(()) + } + + #[test] + fn magic_words_a_and_e() -> Result<()> { + let filter = get_test_filter("a*,e*-abracadabra,elbereth,avada kedavra"); + + verify_that!(filter.filter("alakazam"), is_true())?; + verify_that!(filter.filter("abracadabra"), is_false())?; + verify_that!(filter.filter("avada kedavra"), is_false())?; + verify_that!(filter.filter("enchantment"), is_true())?; + verify_that!(filter.filter("elbereth"), is_false())?; + verify_that!(filter.filter("expecto patronum"), is_true())?; + verify_that!(filter.filter("fuego"), is_false())?; + Ok(()) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/internal/test_outcome.rs b/third_party/crates/googletest-0.14.3/src/internal/test_outcome.rs new file mode 100644 index 0000000..ef7ff43 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/internal/test_outcome.rs @@ -0,0 +1,325 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::cell::{RefCell, RefMut}; +use std::fmt::{Debug, Display, Error, Formatter}; +use std::sync::OnceLock; +use std::thread_local; + +/// The outcome hitherto of running a test. +/// +/// This is kept as a running record as the test progresses. One can access it +/// with `TestOutcome::with_current_test_outcome`. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub enum TestOutcome { + /// The test ran or is currently running and no assertions have failed. + Success, + /// The test ran or is currently running and at least one assertion has + /// failed. + Failure, +} + +thread_local! { + static CURRENT_TEST_OUTCOME: RefCell> = const { RefCell::new(None) }; +} + +impl TestOutcome { + /// Resets the current test's [`TestOutcome`]. + /// + /// This is intended only for use by the attribute macro `#[gtest]`. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + pub fn init_current_test_outcome() { + static INSTALL_HOOK: OnceLock<()> = OnceLock::new(); + INSTALL_HOOK.get_or_init(|| { + let prev_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + let traces = crate::internal::scoped_trace::get_scoped_traces(); + if !traces.is_empty() { + #[cfg(test)] + { + use crate::internal::scoped_trace::test_helpers::*; + let use_capture = USE_CAPTURE_HOOK.with(|v| v.get()); + if use_capture { + let _ = CAPTURED_TRACES_IN_HOOK + .with(|v| v.try_borrow_mut().map(|mut b| *b = traces.clone())); + prev_hook(info); + return; + } + } + + eprintln!("Google Test trace:"); + for trace in traces.iter().rev() { + eprintln!(" {}:{}: {}", trace.file, trace.line, trace.message); + } + } + prev_hook(info); + })); + }); + + Self::with_current_test_outcome(|mut current_test_outcome| { + *current_test_outcome = Some(TestOutcome::Success); + }) + } + + /// Evaluates the current test's [`TestOutcome`], producing a suitable + /// `Result`. + /// + /// The parameter `result` is the value returned by the test function + /// itself. This returns `Result::Err` with a `Display`-formatted string of + /// the error if `result` is `Result::Err`. + /// + /// Otherwise, this returns `Result::Err` precisely when a test failure has + /// been recorded with + /// [`and_log_failure`](crate::GoogleTestSupport::and_log_failure). + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + pub fn close_current_test_outcome( + inner_result: Result<(), E>, + ) -> Result<(), TestFailure> { + TestOutcome::with_current_test_outcome(|mut outcome| { + let outer_result = match &*outcome { + Some(TestOutcome::Success) => match inner_result { + Ok(()) => Ok(()), + Err(_) => Err(TestFailure), + }, + Some(TestOutcome::Failure) => Err(TestFailure), + None => { + panic!("No test context found. This indicates a bug in GoogleTest.") + } + }; + if let Err(fatal_assertion_failure) = inner_result { + println!("{fatal_assertion_failure}"); + } + *outcome = None; + outer_result + }) + } + + /// Returns a `Result` corresponding to the outcome of the currently running + /// test. + #[track_caller] + pub(crate) fn get_current_test_outcome() -> Result<(), TestAssertionFailure> { + TestOutcome::with_current_test_outcome(|mut outcome| { + let outcome = outcome + .as_mut() + .expect("No test context found. This indicates a bug in GoogleTest."); + match outcome { + TestOutcome::Success => Ok(()), + TestOutcome::Failure => Err(TestAssertionFailure::create("Test failed".into())), + } + }) + } + + /// Records that the currently running test has failed. + fn fail_current_test() { + TestOutcome::with_current_test_outcome(|mut outcome| { + let outcome = outcome + .as_mut() + .expect("No test context found. This indicates a bug in GoogleTest."); + *outcome = TestOutcome::Failure; + }) + } + + /// Runs `action` with the [`TestOutcome`] for the currently running test. + /// + /// This is primarily intended for use by assertion macros like + /// `expect_that!`. + fn with_current_test_outcome(action: impl FnOnce(RefMut>) -> T) -> T { + CURRENT_TEST_OUTCOME.with(|current_test_outcome| action(current_test_outcome.borrow_mut())) + } + + /// Ensure that there is a test context present and panic if there is not. + pub(crate) fn ensure_test_context_present() { + if FAILURE_REPORTER_HOOK.get().is_some() { + // Bypassed because an external runner hook is registered. + return; + } + TestOutcome::with_current_test_outcome(|outcome| { + outcome.as_ref().expect( + " +No test context found. + * Did you annotate the test with gtest? + * Is the assertion running in the original test thread? +", + ); + }) + } +} + +/// A marking struct indicating that a test has failed. +/// +/// This exists to implement the [Error][std::error::Error] trait. It displays +/// to a message indicating that the actual test assertion failure messages are +/// in the text above. +pub struct TestFailure; + +impl std::error::Error for TestFailure {} + +impl std::fmt::Debug for TestFailure { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { + writeln!(f, "See failure output above")?; + Ok(()) + } +} + +impl std::fmt::Display for TestFailure { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { + writeln!(f, "See failure output above")?; + Ok(()) + } +} + +/// A report that a single test assertion failed. +/// +/// **For internal use only. API stablility is not guaranteed!** +/// +/// See pub type Result in googletest/src/lib.rs for the public alias. +#[doc(hidden)] +#[derive(Clone)] +pub struct TestAssertionFailure { + /// A human-readable formatted string describing the error. + pub description: String, + pub custom_message: Option, + pub traces: Vec, + location: Location, +} + +/// A code location. +/// +/// `std::panic::Location` does not provide a constructor, hence we cannot +/// construct a fake value. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +#[derive(Clone)] +enum Location { + Real(&'static std::panic::Location<'static>), + Fake { file: &'static str, line: u32, column: u32 }, +} + +impl Display for Location { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Location::Real(l) => write!(f, "{l}"), + Location::Fake { file, line, column } => write!(f, "{file}:{line}:{column}"), + } + } +} + +/// A hook to capture non fatal test failures. This is a global static, and it +/// is initialized exactly once. +#[doc(hidden)] +static FAILURE_REPORTER_HOOK: OnceLock = OnceLock::new(); + +/// Initializes the failure reporter hook. +/// +/// This is a global static, and it is initialized at most once and returns true +/// if the initialization was successful. +/// If the hook is already set, this will do nothing and return false. +#[doc(hidden)] +pub fn set_failure_reporter_hook_if_not_set(capture_fn: fn(&TestAssertionFailure)) -> bool { + FAILURE_REPORTER_HOOK.set(capture_fn).is_ok() +} + +impl TestAssertionFailure { + /// Creates a new instance with the given `description`. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[track_caller] + pub fn create(description: String) -> Self { + Self { + description, + custom_message: None, + traces: crate::internal::scoped_trace::get_scoped_traces(), + location: Location::Real(std::panic::Location::caller()), + } + } + + /// Set `location`` to a fake value. + /// + /// **For internal use only. API stablility is not guaranteed!** + pub fn with_fake_location(mut self, file: &'static str, line: u32, column: u32) -> Self { + self.location = Location::Fake { file, line, column }; + self + } + + pub(crate) fn log(&self) { + println!("{self}"); + if let Some(capture_fn) = FAILURE_REPORTER_HOOK.get() { + capture_fn(self); + return; + } + TestOutcome::fail_current_test(); + } + + /// Returns the file name of the location. + pub fn file(&self) -> &'static str { + match self.location { + Location::Real(l) => l.file(), + Location::Fake { file, .. } => file, + } + } + + /// Returns the line number of the location. + pub fn line(&self) -> u32 { + match self.location { + Location::Real(l) => l.line(), + Location::Fake { line, .. } => line, + } + } +} + +impl Display for TestAssertionFailure { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { + writeln!(f, "{}", self.description)?; + if let Some(custom_message) = &self.custom_message { + writeln!(f, "{custom_message}")?; + } + if !self.traces.is_empty() { + writeln!(f, "Google Test trace:")?; + for trace in self.traces.iter().rev() { + writeln!(f, " {}:{}: {}", trace.file, trace.line, trace.message)?; + } + } + writeln!(f, " at {}", self.location) + } +} + +// The standard Rust test harness outputs the TestAssertionFailure with the +// Debug trait. We want the output to be formatted, so we use a custom Debug +// implementation which defers to Display. +impl Debug for TestAssertionFailure { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { + Display::fmt(self, f) + } +} + +impl From for TestAssertionFailure { + #[track_caller] + fn from(value: T) -> Self { + TestAssertionFailure::create(format!("{value}")) + } +} + +#[cfg(feature = "proptest")] +impl From for proptest::test_runner::TestCaseError { + fn from(value: TestAssertionFailure) -> Self { + proptest::test_runner::TestCaseError::Fail(format!("{value}").into()) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/internal/test_sharding.rs b/third_party/crates/googletest-0.14.3/src/internal/test_sharding.rs new file mode 100644 index 0000000..e3704a8 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/internal/test_sharding.rs @@ -0,0 +1,107 @@ +/// This module implements the googletest test sharding protocol. The Google +/// sharding protocol consists of the following environment variables: +/// +/// * GTEST_TOTAL_SHARDS: total number of shards. +/// * GTEST_SHARD_INDEX: number of this shard +/// * GTEST_SHARD_STATUS_FILE: touch this file to indicate support for sharding. +/// +/// See also +use std::cell::OnceCell; +use std::env::{var, var_os}; +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File}; +use std::num::NonZeroU64; +use std::path::{Path, PathBuf}; + +/// Environment variable specifying the total number of test shards. +const TEST_TOTAL_SHARDS: &[&str] = &["GTEST_TOTAL_SHARDS", "TEST_TOTAL_SHARDS"]; + +/// Environment variable specifyign the index of this test shard. +const TEST_SHARD_INDEX: &[&str] = &["GTEST_SHARD_INDEX", "TEST_SHARD_INDEX"]; + +/// Environment variable specifying the name of the file we create (or cause a +/// timestamp change on) to indicate that we support the sharding protocol. +const TEST_SHARD_STATUS_FILE: &[&str] = &["GTEST_SHARD_STATUS_FILE", "TEST_SHARD_STATUS_FILE"]; + +thread_local! { + static SHARDING: OnceCell = const { OnceCell::new() }; +} + +struct Sharding { + this_shard: u64, + total_shards: NonZeroU64, +} + +impl Default for Sharding { + fn default() -> Self { + Self { this_shard: 0, total_shards: NonZeroU64::MIN } + } +} + +pub fn test_should_run(test_case_hash: u64) -> bool { + SHARDING.with(|sharding_cell| { + sharding_cell.get_or_init(Sharding::from_environment).test_should_run(test_case_hash) + }) +} + +fn get_var(keys: &[&str]) -> Option { + for key in keys { + if let Ok(value) = var(OsStr::new(key)) { + return Some(value); + } + } + + None +} + +fn get_var_os(keys: &[&str]) -> Option { + for key in keys { + if let Some(value) = var_os(OsStr::new(key)) { + return Some(value); + } + } + + None +} + +impl Sharding { + fn test_should_run(&self, test_case_hash: u64) -> bool { + (test_case_hash % self.total_shards.get()) == self.this_shard + } + + fn from_environment() -> Sharding { + let this_shard: Option = + { get_var(TEST_SHARD_INDEX).and_then(|value| value.parse().ok()) }; + let total_shards: Option = { + get_var(TEST_TOTAL_SHARDS) + .and_then(|value| value.parse().ok()) + .and_then(NonZeroU64::new) + }; + + match (this_shard, total_shards) { + (Some(this_shard), Some(total_shards)) if this_shard < total_shards.get() => { + if let Some(name) = get_var_os(TEST_SHARD_STATUS_FILE) { + let pathbuf = PathBuf::from(name); + if let Err(e) = create_status_file(&pathbuf) { + eprintln!( + "failed to create $GTEST_SHARD_STATUS_FILE file {}: {}", + pathbuf.display(), + e + ); + } + } + + Sharding { this_shard, total_shards } + } + _ => Sharding::default(), + } + } +} + +fn create_status_file(path: &Path) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + File::create(path).map(|_| ()) +} diff --git a/third_party/crates/googletest-0.14.3/src/lib.rs b/third_party/crates/googletest-0.14.3/src/lib.rs new file mode 100644 index 0000000..20c75a7 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/lib.rs @@ -0,0 +1,381 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc = include_str!("../crate_docs.md")] + +extern crate googletest_macro; + +#[cfg(test)] +extern crate quickcheck; + +#[macro_use] +pub mod assertions; +pub mod description; +pub mod fixtures; +#[macro_use] +pub mod fmt; +pub mod internal; +pub mod matcher; +pub mod matcher_support; +pub mod matchers; + +pub use googletest_macro::__abbreviated_stringify; + +/// Re-exports of the symbols in this crate which are most likely to be used. +/// +/// This includes: +/// * All assertion macros, +/// * Traits and type definitions normally used by tests, and +/// * All built-in matchers. +/// +/// Typically, one imports everything in the prelude in one's test module: +/// +/// ``` +/// mod tests { +/// use googletest::prelude::*; +/// } +/// ``` +pub mod prelude { + pub use super::fixtures::{ConsumableFixture, Fixture, FixtureOf, StaticFixture}; + pub use super::gtest; + pub use super::matcher::{Matcher, MatcherBase}; + pub use super::matchers::*; + pub use super::scoped_trace; + pub use super::verify_current_test_outcome; + pub use super::GoogleTestSupport; + pub use super::OrFail; + pub use super::Result; + // Assert macros + pub use super::{ + add_failure, add_failure_at, assert_false, assert_pred, assert_that, assert_true, + expect_eq, expect_false, expect_float_eq, expect_ge, expect_gt, expect_le, expect_lt, + expect_ne, expect_near, expect_pred, expect_that, expect_true, fail, succeed, verify_eq, + verify_false, verify_float_eq, verify_ge, verify_gt, verify_le, verify_lt, verify_ne, + verify_near, verify_pred, verify_that, verify_true, + }; +} + +/// Causes a trace to be included in every test failure message generated +/// by code in the current scope. +#[macro_export] +macro_rules! scoped_trace { + ($($arg:tt)*) => { + #[allow(clippy::shadow_same, clippy::shadow_unrelated)] + let _gtest_trace_guard = $crate::internal::scoped_trace::ScopedTraceGuard::new( + format!($($arg)*), + ); + }; +} + +pub use googletest_macro::gtest; +pub use googletest_macro::test; + +use internal::test_outcome::{TestAssertionFailure, TestOutcome}; + +/// A `Result` whose `Err` variant indicates a test failure. +/// +/// The assertions [`verify_that!`][crate::verify_that], +/// [`verify_pred!`][crate::verify_pred], and [`fail!`][crate::fail] evaluate +/// to `Result<()>`. A test function may return `Result<()>` in combination with +/// those macros to abort immediately on assertion failure. +/// +/// This can be used with subroutines which may cause the test to fatally fail +/// and which return some value needed by the caller. For example: +/// +/// ```ignore +/// fn load_file_content_as_string() -> Result { +/// let file_stream = load_file().err_to_test_failure()?; +/// Ok(file_stream.to_string()) +/// } +/// ``` +/// +/// The `Err` variant contains a [`TestAssertionFailure`] which carries the data +/// of the (fatal) assertion failure which generated this result. Non-fatal +/// assertion failures, which log the failure and report the test as having +/// failed but allow it to continue running, are not encoded in this type. +pub type Result = std::result::Result; + +/// Returns a [`Result`] corresponding to the outcome of the currently running +/// test. +/// +/// This returns `Result::Err` precisely if the current test has recorded at +/// least one test assertion failure via [`expect_that!`][crate::expect_that], +/// [`expect_pred!`][crate::expect_pred], or +/// [`GoogleTestSupport::and_log_failure`]. It can be used in concert with the +/// `?` operator to continue execution of the test conditionally on there not +/// having been any failure yet. +/// +/// This requires the use of the [`#[gtest]`][crate::gtest] attribute macro. +/// +/// ``` +/// # use googletest::prelude::*; +/// # /* Make sure this also compiles as a doctest. +/// #[gtest] +/// # */ +/// # fn foo() -> u32 { 1 } +/// # fn bar() -> u32 { 2 } +/// fn should_fail_and_not_execute_last_assertion() -> Result<()> { +/// # googletest::internal::test_outcome::TestOutcome::init_current_test_outcome(); +/// expect_that!(foo(), eq(2)); // May fail, but will not abort the test. +/// expect_that!(bar(), gt(1)); // May fail, but will not abort the test. +/// verify_current_test_outcome()?; // Aborts the test if one of the previous assertions failed. +/// verify_that!(foo(), gt(0)) // Does not execute if the line above aborts. +/// } +/// # verify_that!(should_fail_and_not_execute_last_assertion(), err(displays_as(contains_substring("Test failed")))).unwrap(); +/// ``` +#[track_caller] +pub fn verify_current_test_outcome() -> Result<()> { + TestOutcome::get_current_test_outcome() +} + +/// Adds to `Result` support for GoogleTest Rust functionality. +pub trait GoogleTestSupport { + /// If `self` is a `Result::Err`, writes to `stdout` a failure report + /// and marks the test failed. Otherwise, does nothing. + /// + /// This can be used for non-fatal test assertions, for example: + /// + /// ``` + /// # use googletest::prelude::*; + /// # use googletest::internal::test_outcome::TestOutcome; + /// # TestOutcome::init_current_test_outcome(); + /// let actual = 42; + /// verify_that!(actual, eq(42)).and_log_failure(); + /// // Test still passing; nothing happens + /// verify_that!(actual, eq(10)).and_log_failure(); + /// // Test now fails and failure output to stdout + /// verify_that!(actual, eq(100)).and_log_failure(); + /// // Test still fails and new failure also output to stdout + /// # TestOutcome::close_current_test_outcome::<&str>(Ok(())).unwrap_err(); + /// ``` + fn and_log_failure(self); + + /// If `self` is a `Result::Err`, writes to `stdout` with a failure report + /// and the message returned by `provider`. + /// + /// This is equivalent to combining [`GoogleTestSupport::and_log_failure`] + /// with a call to [`GoogleTestSupport::with_failure_message`]. + /// + /// Example: + /// + /// ``` + /// # use googletest::GoogleTestSupport; + /// # use googletest::assertions::verify_eq; + /// # use googletest::internal::test_outcome::TestOutcome; + /// # TestOutcome::init_current_test_outcome(); + /// let actual = 0; + /// verify_eq!(actual, 42) + /// .and_log_failure_with_message(|| format!("Actual {} was wrong!", actual)); + /// # TestOutcome::close_current_test_outcome::<&str>(Ok(())).unwrap_err(); + /// ``` + fn and_log_failure_with_message(self, provider: impl FnOnce() -> String) + where + Self: Sized, + { + self.with_failure_message(provider).and_log_failure(); + } + + /// Adds `message` to the logged failure message if `self` is a + /// `Result::Err`. Otherwise, does nothing. + /// + /// If this method is called more than once, only `message` from the last + /// invocation is output. + /// + /// For example: + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_fail() -> Result<()> { + /// let actual = 0; + /// verify_that!(actual, eq(42)).failure_message("Actual was wrong!")?; + /// # Ok(()) + /// # } + /// # verify_that!(should_fail(), err(displays_as(contains_substring("Actual was wrong")))) + /// # .unwrap(); + /// ``` + /// + /// results in the following failure message: + /// + /// ```text + /// Expected: actual equal to 42 + /// but was: 0 + /// Actual was wrong! + /// ``` + /// + /// One can pass a `String` too: + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_fail() -> Result<()> { + /// let actual = 0; + /// verify_that!(actual, eq(42)) + /// .failure_message(format!("Actual {} was wrong!", actual))?; + /// # Ok(()) + /// # } + /// # verify_that!(should_fail(), err(displays_as(contains_substring("Actual 0 was wrong")))) + /// # .unwrap(); + /// ``` + /// + /// However, consider using [`GoogleTestSupport::with_failure_message`] + /// instead in that case to avoid unnecessary memory allocation when the + /// message is not needed. + fn failure_message(self, message: impl Into) -> Self; + + /// Adds the output of the closure `provider` to the logged failure message + /// if `self` is a `Result::Err`. Otherwise, does nothing. + /// + /// This is analogous to [`GoogleTestSupport::failure_message`] but + /// only executes the closure `provider` if it actually produces the + /// message, thus saving possible memory allocation. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_fail() -> Result<()> { + /// let actual = 0; + /// verify_that!(actual, eq(42)) + /// .with_failure_message(|| format!("Actual {} was wrong!", actual))?; + /// # Ok(()) + /// # } + /// # verify_that!(should_fail(), err(displays_as(contains_substring("Actual 0 was wrong")))) + /// # .unwrap(); + /// ``` + fn with_failure_message(self, provider: impl FnOnce() -> String) -> Self; +} + +impl GoogleTestSupport for std::result::Result { + fn and_log_failure(self) { + TestOutcome::ensure_test_context_present(); + if let Err(failure) = self { + failure.log(); + } + } + + fn failure_message(mut self, message: impl Into) -> Self { + if let Err(ref mut failure) = self { + failure.custom_message = Some(message.into()); + } + self + } + + fn with_failure_message(mut self, provider: impl FnOnce() -> String) -> Self { + if let Err(ref mut failure) = self { + failure.custom_message = Some(provider()); + } + self + } +} + +/// Provides an extension method for converting an arbitrary type into +/// `googletest`'s [`Result`] type. +/// +/// A type can implement this trait to provide an easy way to return immediately +/// from a test in conjunction with the `?` operator. This is useful for +/// [`Option`] and [`Result`][std::result::Result] types whose `Result::Err` +/// variant does not implement [`std::error::Error`]. +/// +/// If `Result::Err` implements [`std::error::Error`] you can just use the `?` +/// operator directly. +/// +/// ```ignore +/// #[test] +/// fn should_work() -> googletest::Result<()> { +/// let value = something_which_can_fail().or_fail()?; +/// let value = something_which_can_fail_with_option().or_fail()?; +/// ... +/// } +/// +/// fn something_which_can_fail() -> std::result::Result { ... } +/// fn something_which_can_fail_with_option() -> Option { ... } +/// ``` +pub trait OrFail { + /// The success type of the test result. + type Output; + + /// Converts a value into a [`Result`] containing + /// either the [`Self::Output`] type or a [`TestAssertionFailure`]. + /// + /// The most frequently used implementations convert + /// `Result` into `Result` and + /// `Option` into `Result`. + fn or_fail(self) -> Result; +} + +impl OrFail for std::result::Result { + type Output = T; + + #[track_caller] + fn or_fail(self) -> std::result::Result { + match self { + Ok(t) => Ok(t), + Err(e) => Err(TestAssertionFailure::create(format!("{e:?}"))), + } + } +} + +impl OrFail for Option { + type Output = T; + + #[track_caller] + fn or_fail(self) -> std::result::Result { + match self { + Some(t) => Ok(t), + None => Err(TestAssertionFailure::create(format!( + "called `Option::or_fail()` on a `Option::<{}>::None` value", + std::any::type_name::() + ))), + } + } +} + +#[doc(hidden)] +pub mod __internal_macro_support { + use crate::fixtures::Fixture; + + pub struct FixtureTearDownOnDrop { + fixture: Option, + } + + impl FixtureTearDownOnDrop { + pub fn new(fixture: T) -> Self { + Self { fixture: Some(fixture) } + } + pub fn tear_down(mut self) -> crate::Result<()> { + if let Some(fixture) = self.fixture.take() { + fixture.tear_down() + } else { + Ok(()) + } + } + } + + impl AsRef for FixtureTearDownOnDrop { + fn as_ref(&self) -> &T { + self.fixture.as_ref().unwrap() + } + } + + impl AsMut for FixtureTearDownOnDrop { + fn as_mut(&mut self) -> &mut T { + self.fixture.as_mut().unwrap() + } + } + + impl Drop for FixtureTearDownOnDrop { + fn drop(&mut self) { + if let Some(fixture) = self.fixture.take() { + fixture.tear_down().unwrap(); + } + } + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matcher.rs b/third_party/crates/googletest-0.14.3/src/matcher.rs new file mode 100644 index 0000000..454af7a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matcher.rs @@ -0,0 +1,333 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The components required to implement matchers. + +use crate::description::Description; +use crate::internal::test_outcome::TestAssertionFailure; +use crate::matchers::__internal_unstable_do_not_depend_on_these::ConjunctionMatcher; +use crate::matchers::__internal_unstable_do_not_depend_on_these::DisjunctionMatcher; +pub use googletest_macro::MatcherBase; +use std::fmt::Debug; + +/// An interface for checking an arbitrary condition on a datum. +/// +/// This trait is automatically implemented for a reference of any type +/// implementing `Matcher`. This simplifies reusing a matcher in different +/// assertions. +/// +/// It is also implemented for tuple of `Matcher`. If `MatcherT: Matcher` and +/// `MatcherU: Matcher`, then `(MatcherT, MatcherU): Matcher<(T, U)>`, and so +/// on, up to 12 elements. Tuples longer than that do not automatically inherit +/// the `Debug` trait from their members, so are generally not well-supported; +/// see [Rust by Example](https://doc.rust-lang.org/rust-by-example/primitives/tuples.html#tuples). +// `ActualT` requires `Copy` so that `actual` could be passed to `matches` and +// if it fails passed to `explain_match`. We can relax this constraint later by +// requiring only `Clone`. +pub trait Matcher: MatcherBase { + /// Returns whether the condition matches the datum `actual`. + /// + /// The trait implementation defines what it means to "match". Often the + /// matching condition is based on data stored in the matcher. For example, + /// `eq` matches when its stored expected value is equal (in the sense of + /// the `==` operator) to the value `actual`. + fn matches(&self, actual: ActualT) -> MatcherResult; + + /// Returns a description of `self` according to the supplied matching + /// sense: a positive description if `sense` is `Match`, and a negative + /// description if it's `NoMatch`. + /// + /// The function should print a verb phrase that describes the property a + /// value matching, respectively not matching, this matcher should have. + /// The subject of the verb phrase is the value being matched. + /// + /// The output appears next to `Expected` in an assertion failure message. + /// For example: + /// + /// ```text + /// Value of: ... + /// Expected: is equal to 7 + /// ^^^^^^^^^^^^^ + /// Actual: ... + /// ``` + /// + /// When the matcher contains one or more inner matchers, the implementation + /// should invoke [`Self::describe`] on the inner matchers to complete the + /// description. It should place the inner description at a point where a + /// verb phrase would fit. For example, the matcher + /// [`some`][crate::matchers::some] implements `describe` as follows: + /// + /// ```ignore + /// fn describe(&self, sense: MatcherResult) -> Description { + /// match sense { + /// MatcherResult::Match => { + /// Description::new() + /// .text("has a value which") + /// .nested(self.inner.describe(MatcherResult::Match)) + /// // Inner matcher: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + /// } + /// MatcherResult::NoMatch => { + /// Description::new() + /// .text("is None or has a value which") + /// .nested(self.inner.describe(MatcherResult::NoMatch)) + /// } + /// } + /// } + /// ``` + /// + /// The output expectation differs from that of + /// [`explain_match`][Self::explain_match] in that it is a verb phrase + /// (beginning with a verb like "is") rather than a relative clause + /// (beginning with "which" or "whose"). This difference is because the + /// output of `explain_match` is always used adjectivally to describe the + /// actual value, while `describe` is used in contexts where a relative + /// clause would not make sense. + fn describe(&self, sense: MatcherResult) -> Description; + + /// Prepares a [`String`] describing how the expected value + /// encoded in this instance matches or does not match the given value + /// `actual`. + /// + /// This should be in the form of a relative clause, i.e. something starting + /// with a relative pronoun such as "which" or "whose". It will appear next + /// to the actual value in an assertion failure. For example: + /// + /// ```text + /// Value of: ... + /// Expected: ... + /// Actual: ["Something"], which does not contain "Something else" + /// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + /// ``` + /// + /// The default implementation relies on [`describe`][Self::describe]. Thus + /// it does not make any use of the actual value itself, but rather only + /// whether the value is matched. + /// + /// Override the default implementation to provide additional context on why + /// a particular value matched or did not match. For example, the + /// [`container_eq`][crate::matchers::container_eq] matcher displays + /// information on which elements of the actual value were not present in + /// the expected value and vice versa. + /// + /// This implementation should be overridden in any matcher which contains + /// one or more inner matchers. The implementation should invoke + /// `explain_match` on the inner matchers, so that the generated match + /// explanation also reflects their implementation. Without this, the match + /// explanation of the inner matchers will not be able to make use of the + /// actual value at all. + /// + /// For example, the `explain_match` implementation of the matcher + /// [`points_to`][crate::matchers::points_to] defers immediately to the + /// inner matcher and appears as follows: + /// + /// ```ignore + /// fn explain_match(&self, actual: &ActualT) -> Description { + /// self.expected.explain_match(*actual) + /// } + /// ``` + /// + /// The matcher can also provide some additional context before deferring to + /// an inner matcher. In that case it should invoke `explain_match` on the + /// inner matcher at a point where a relative clause would fit. For example: + /// + /// ```ignore + /// fn explain_match(&self, actual: ActualT) -> Description { + /// Description::new() + /// .text("which points to a value") + /// .nested(self.expected.explain_match(*actual)) + /// } + /// ``` + fn explain_match(&self, actual: ActualT) -> Description { + format!("which {}", self.describe(self.matches(actual))).into() + } +} + +/// Base trait for matchers. Any type implementing `Matcher` must implement +/// `MatcherBase`, but that should be done through the `#[derive(MatcherBase)]` +/// macro. +// The `and` and `or` functions cannot be part of the `Matcher` trait since it +// can be implemented multiple times for a given matcher type. Consider that +// `and` and `or` are part of the `Matcher` trait and `MyMatcher` implements +// both `Matcher` and `Matcher`. Then `MyMatcher{...}.and(...)` can be +// either: +// * `Matcher::::and(MyMatcher{...}, ...)` or +// * `Matcher::::and(MyMatcher{...}, ...)`. +// Moving the `and` and `or` functions in a non-generic trait removes this +// confusion by making `and` and `or` unique for a given type. +pub trait MatcherBase { + /// Constructs a matcher that matches both `self` and `right`. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_pass() -> Result<()> { + /// verify_that!("A string", starts_with("A").and(ends_with("string")))?; // Passes + /// # Ok(()) + /// # } + /// # fn should_fail_1() -> Result<()> { + /// verify_that!("A string", starts_with("Another").and(ends_with("string")))?; // Fails + /// # Ok(()) + /// # } + /// # fn should_fail_2() -> Result<()> { + /// verify_that!("A string", starts_with("A").and(ends_with("non-string")))?; // Fails + /// # Ok(()) + /// # } + /// # should_pass().unwrap(); + /// # should_fail_1().unwrap_err(); + /// # should_fail_2().unwrap_err(); + /// ``` + // TODO(b/264518763): Replace the return type with impl Matcher and reduce + // visibility of ConjunctionMatcher once impl in return position in trait + // methods is stable. + fn and(self, right: Right) -> ConjunctionMatcher + where + Self: Sized, + { + ConjunctionMatcher::new(self, right) + } + + /// Constructs a matcher that matches when at least one of `self` or `right` + /// matches the input. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_pass() -> Result<()> { + /// verify_that!(10, eq(2).or(ge(5)))?; // Passes + /// verify_that!(10, eq(2).or(eq(5)).or(ge(9)))?; // Passes + /// # Ok(()) + /// # } + /// # fn should_fail() -> Result<()> { + /// verify_that!(10, eq(2).or(ge(15)))?; // Fails + /// # Ok(()) + /// # } + /// # should_pass().unwrap(); + /// # should_fail().unwrap_err(); + /// ``` + // TODO(b/264518763): Replace the return type with impl Matcher and reduce + // visibility of DisjunctionMatcher once impl in return position in trait + // methods is stable. + fn or(self, right: Right) -> DisjunctionMatcher + where + Self: Sized, + { + DisjunctionMatcher::new(self, right) + } +} + +/// Any actual value whose debug length is greater than this value will be +/// pretty-printed. Otherwise, it will have normal debug output formatting. +const PRETTY_PRINT_LENGTH_THRESHOLD: usize = 60; + +/// Constructs a [`TestAssertionFailure`] reporting that the given `matcher` +/// does not match the value `actual`. +/// +/// The parameter `actual_expr` contains the expression which was evaluated to +/// obtain `actual`. +#[track_caller] +pub(crate) fn create_assertion_failure( + matcher: &impl Matcher, + actual: T, + actual_expr: &'static str, +) -> TestAssertionFailure { + let actual_formatted = format!("{actual:?}"); + let actual_formatted = if actual_formatted.len() > PRETTY_PRINT_LENGTH_THRESHOLD { + format!("{actual:#?}") + } else { + actual_formatted + }; + TestAssertionFailure::create(format!( + "\ +Value of: {actual_expr} +Expected: {} +Actual: {actual_formatted}, +{}", + matcher.describe(MatcherResult::Match), + matcher.explain_match(actual).indent(), + )) +} + +/// The result of applying a [`Matcher`] on an actual value. +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum MatcherResult { + /// The actual value matches according to the [`Matcher`] definition. + Match, + /// The actual value does not match according to the [`Matcher`] definition. + NoMatch, +} + +impl From for MatcherResult { + fn from(b: bool) -> Self { + if b { + MatcherResult::Match + } else { + MatcherResult::NoMatch + } + } +} + +impl From for bool { + fn from(matcher_result: MatcherResult) -> Self { + matcher_result.is_match() + } +} + +impl MatcherResult { + /// Returns `true` if `self` is [`MatcherResult::Match`], otherwise + /// `false`. + pub fn is_match(self) -> bool { + matches!(self, MatcherResult::Match) + } + + /// Returns `true` if `self` is [`MatcherResult::NoMatch`], otherwise + /// `false`. + pub fn is_no_match(self) -> bool { + matches!(self, MatcherResult::NoMatch) + } +} + +impl MatcherBase for &M {} + +impl> Matcher for &M { + fn matches(&self, actual: T) -> MatcherResult { + (*self).matches(actual) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + (*self).describe(matcher_result) + } + + fn explain_match(&self, actual: T) -> Description { + (*self).explain_match(actual) + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + + #[test] + fn ref_matchers_can_be_reused() -> Result<()> { + let matcher = eq(1); + + verify_that!(1, &matcher)?; + verify_that!(1, &matcher) + } + + #[test] + fn ref_matchers_as_inner_matcher() -> Result<()> { + let matcher = gt(1); + + verify_that!([2, 3, 4, 5], [&matcher, &matcher, &matcher, &matcher]) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matcher_support/auto_eq.rs b/third_party/crates/googletest-0.14.3/src/matcher_support/auto_eq.rs new file mode 100644 index 0000000..59a7bb2 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matcher_support/auto_eq.rs @@ -0,0 +1,107 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc(hidden)] + +/// Macro that wraps the expression with `eq(...)` if the expression is +/// not a matcher. +/// +/// This is useful to let users pass expected value to macro matchers like +/// `field!` and `property!`. +///` +/// **For internal use only. API stablility is not guaranteed!** +/// If you are interested in using it in your matcher, please file an issue to +/// stabilize this. +#[macro_export] +macro_rules! __auto_eq { + ($e:expr) => {{ + #[allow(unused_imports)] + use $crate::matcher_support::__internal_unstable_do_not_depend_on_these::ExpectedKind as _; + match $e { + expected => { + $crate::matcher_support::__internal_unstable_do_not_depend_on_these::Wrapper( + &expected, + ) + .kind() + .matcher(expected) + } + } + }}; +} + +// This reimplements the pattern presented in +// https://github.com/dtolnay/case-studies/issues/14 +pub mod internal { + use crate::{ + matcher::MatcherBase, + matchers::{eq, EqMatcher}, + }; + + pub struct Wrapper(pub T); + + impl Wrapper<&'_ T> { + #[inline] + pub fn kind(&self) -> MatcherTag { + MatcherTag + } + } + + pub trait ExpectedKind { + #[inline] + fn kind(&self) -> ExpectedTag { + ExpectedTag + } + } + + impl ExpectedKind for Wrapper {} + + pub struct MatcherTag; + + impl MatcherTag { + #[inline] + pub fn matcher(self, matcher: M) -> M { + matcher + } + } + pub struct ExpectedTag; + + impl ExpectedTag { + #[inline] + pub fn matcher(self, expected: T) -> EqMatcher { + eq(expected) + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + + #[test] + fn auto_ref_matcher() -> Result<()> { + verify_that!(123, __auto_eq!(ge(9))) + } + + #[test] + fn auto_ref_expected() -> Result<()> { + verify_that!(123, __auto_eq!(123)) + } + + #[test] + fn auto_ref_on_ref_matcher() -> Result<()> { + let matcher = eq(123); + verify_that!(123, __auto_eq!(&matcher)) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matcher_support/count_elements.rs b/third_party/crates/googletest-0.14.3/src/matcher_support/count_elements.rs new file mode 100644 index 0000000..6d5b2bf --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matcher_support/count_elements.rs @@ -0,0 +1,79 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Counts the number of elements in `value`. +/// +/// This uses [`Iterator::size_hint`] when that function returns an +/// unambiguous answer, i.e., the upper bound exists and the lower and upper +/// bounds agree. Otherwise it iterates through `value` and counts the +/// elements. +pub(crate) fn count_elements(value: ContainerT) -> usize { + let iterator = value.into_iter(); + if let (lower, Some(higher)) = iterator.size_hint() { + if lower == higher { + return lower; + } + } + iterator.count() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::*; + use crate::Result; + + #[test] + fn count_elements_vec() -> Result<()> { + verify_that!(count_elements(vec![1, 2, 3]), eq(3)) + } + + #[test] + fn count_elements_with_imprecise_hint() -> Result<()> { + struct FakeIterator; + + impl Iterator for FakeIterator { + type Item = (); + + fn next(&mut self) -> Option { + None + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(123)) + } + } + + verify_that!(count_elements(FakeIterator), eq(0)) + } + + #[test] + fn count_elements_with_no_hint() -> Result<()> { + struct FakeIterator; + + impl Iterator for FakeIterator { + type Item = (); + + fn next(&mut self) -> Option { + None + } + + fn size_hint(&self) -> (usize, Option) { + (0, None) + } + } + + verify_that!(count_elements(FakeIterator), eq(0)) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matcher_support/edit_distance.rs b/third_party/crates/googletest-0.14.3/src/matcher_support/edit_distance.rs new file mode 100644 index 0000000..2ace1a8 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matcher_support/edit_distance.rs @@ -0,0 +1,649 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Debug; + +/// Maximum number of edits which can exist before [`edit_list`] falls back to a +/// complete rewrite to produce the edit list. +/// +/// Increasing this limit increases the accuracy of [`edit_list`] while +/// quadratically increasing its worst-case runtime. +const MAX_DISTANCE: i32 = 50; + +/// The difference between two inputs as produced by [`edit_list`]. +#[derive(Debug)] +pub(crate) enum Difference { + /// No differences were detected at all. + Equal, + + /// At most [`MAX_DISTANCE`] edits are required to convert one input to the + /// other. + /// + /// Contains the list of [`Edit`] to perform the transformation. + Editable(Vec>), + + /// More than [`MAX_DISTANCE`] edits are required to convert one input to + /// the other. + /// + /// The inputs are therefore considered unrelated and no edit list is + /// provided. + Unrelated, +} + +/// An edit operation on two sequences of `T`. +#[derive(Debug, Clone)] +pub(crate) enum Edit { + /// An extra `T` was added to the actual sequence. + ExtraActual(T), + + /// An extra `T` was added to the expected sequence. + ExtraExpected(T), + + /// An element was added to each sequence. + Both(T), + + /// Additional (unlisted) elements are present in the actual sequence. + /// + /// This is only output in the mode [`Mode::Prefix`]. Its presence precludes + /// reconstructing the actual sequence from the expected sequence. + AdditionalActual, +} + +/// Controls the termination condition of [`edit_list`]. +#[derive(Clone, Copy)] +pub(crate) enum Mode { + /// Indicates that the two arguments are intended to be equal. + /// + /// The entire edit list to transform between `actual` and `expected` is + /// returned. + Exact, + + /// Indicates that `expected` is inteded to be a prefix of `actual`. + /// + /// Any additional parts of `actual` after the prefix `expected` are omitted + /// from the output. + Prefix, + + /// Similar to [`Mode::Prefix`], except it is also assumed that `actual` has + /// some number of initial lines which should not be in the output. + /// + /// Any initial [`Edit::ExtraActual`] entries are replaced with + /// [`Edit::AdditionalActual`] in the edit list. If the first entry which is + /// not an [`Edit::ExtraActual`] is [`Edit::ExtraExpected`], then the last + /// [`Edit::ExtraActual`] is actual in the output. + Contains, +} + +/// Computes the edit list of `actual` and `expected`. +/// +/// If `actual` and `expected` are equal, then this returns +/// [`Difference::Equal`]. If they are different but have an +/// [edit distance](https://en.wikipedia.org/wiki/Edit_distance) +/// of at most [`MAX_DISTANCE`], this returns [`Difference::Editable`] with the +/// sequence of [`Edit`] which can be applied to `actual` to obtain `expected`. +/// Otherwise this returns [`Difference::Unrelated`]. +/// +/// This uses [Myers Algorithm](https://neil.fraser.name/writing/diff/myers.pdf) +/// with a maximum edit distance of [`MAX_DISTANCE`]. Thus the worst-case +/// runtime is linear in both the input length and [`MAX_DISTANCE`]. +pub(crate) fn edit_list( + actual: impl IntoIterator, + expected: impl IntoIterator, + mode: Mode, +) -> Difference { + let actual: Vec<_> = actual.into_iter().collect(); + let expected: Vec<_> = expected.into_iter().collect(); + + let mut paths_last: Vec> = Vec::new(); + + for distance in 0..=MAX_DISTANCE { + let mut paths_current = Vec::new(); + for k in (-distance..=distance).step_by(2) { + // The following will be None when k is at the edges of the range, + // since no paths have been created for k outside the range in the + // previous iteration. + let path_k_minus_1 = paths_last.get(index_of_k(k - 1, -distance + 1)); + let path_k_plus_1 = paths_last.get(index_of_k(k + 1, -distance + 1)); + + let (mut path, edit) = match (path_k_minus_1, path_k_plus_1) { + // This the first (outer) iteration. + (None, None) => (Path::default(), None), + + // k = -distance. There is no previous parent path yet. + (None, Some(path_k_plus_1)) => ( + path_k_plus_1.clone(), + expected.get(path_k_plus_1.expected_endpoint).copied().map(Edit::ExtraExpected), + ), + + // k = distance. There is no next parent path yet. + (Some(path_k_minus_1), None) => ( + path_k_minus_1.extend_actual_endpoint(), + actual.get(path_k_minus_1.actual_endpoint).copied().map(Edit::ExtraActual), + ), + + // k is strictly between -distance and distance. Both parent paths were set in the + // last iteration. + (Some(path_k_minus_1), Some(path_k_plus_1)) => { + // This decides whether the algorithm prefers to add an edit + // from the actual or from the expected when the rows differ. We + // alternate so that the elements of differing blocks + // interleave rather than all elements of each respective + // side being output in a single block. + if (distance % 2 == 0 + && path_k_plus_1.actual_endpoint > path_k_minus_1.actual_endpoint) + || (distance % 2 == 1 + && path_k_plus_1.expected_endpoint > path_k_minus_1.expected_endpoint) + { + ( + path_k_plus_1.clone(), + expected + .get(path_k_plus_1.expected_endpoint) + .copied() + .map(Edit::ExtraExpected), + ) + } else { + ( + path_k_minus_1.extend_actual_endpoint(), + actual + .get(path_k_minus_1.actual_endpoint) + .copied() + .map(Edit::ExtraActual), + ) + } + } + }; + path.edits.extend(edit); + + // Advance through any common elements starting at the current path. + let (mut actual_endpoint, mut expected_endpoint) = + (path.actual_endpoint, (path.actual_endpoint as i32 - k) as usize); + while actual_endpoint < actual.len() + && expected_endpoint < expected.len() + && actual[actual_endpoint] == expected[expected_endpoint] + { + path.edits.push(Edit::Both(actual[actual_endpoint])); + (actual_endpoint, expected_endpoint) = (actual_endpoint + 1, expected_endpoint + 1); + } + + // If we have exhausted both inputs, we are done. + if actual_endpoint == actual.len() && expected_endpoint == expected.len() { + return if path.edits.iter().any(|v| !matches!(v, Edit::Both(_))) { + if matches!(mode, Mode::Contains) { + compress_prefix_and_suffix(&mut path.edits); + } + Difference::Editable(path.edits) + } else { + Difference::Equal + }; + } + + path.actual_endpoint = actual_endpoint; + path.expected_endpoint = expected_endpoint; + paths_current.push(path); + } + + if matches!(mode, Mode::Prefix) { + if let Some(path) = paths_current + .iter_mut() + .filter(|p| p.expected_endpoint == expected.len()) + .max_by(|p1, p2| p1.edits.len().cmp(&p2.edits.len())) + { + // We've reached the end of the expected side but there could still be a + // corresponding line on the actual which we haven't picked up into the edit + // list. We'll just add it manually to the edit list. There's no + // real harm doing so -- worst case is that there's an + // additional line when there didn't have to be. + if let Some(Edit::ExtraExpected(_)) = path.edits.last() { + if path.actual_endpoint < actual.len() { + // The edits from the actual should come before the corresponding one from + // the expected, so we insert rather than push. + path.edits.insert( + path.edits.len() - 1, + Edit::ExtraActual(actual[path.actual_endpoint]), + ); + } + } + path.edits.push(Edit::AdditionalActual); + return if path.edits.iter().any(|v| !matches!(v, Edit::Both(_))) { + Difference::Editable(std::mem::take(&mut path.edits)) + } else { + Difference::Equal + }; + } + } + + paths_last = paths_current; + } + + Difference::Unrelated +} + +fn index_of_k(k: i32, k_min: i32) -> usize { + ((k - k_min) / 2) as usize +} + +fn compress_prefix_and_suffix(edits: &mut Vec>) { + if let Some(mut first_non_extra_actual_edit) = + edits.iter().position(|e| !matches!(e, Edit::ExtraActual(_))) + { + if first_non_extra_actual_edit > 1 + && matches!(edits[first_non_extra_actual_edit], Edit::ExtraExpected(_)) + { + first_non_extra_actual_edit -= 1; + } + edits.splice(..first_non_extra_actual_edit, [Edit::AdditionalActual]); + } + + if let Some(mut last_non_extra_actual_edit) = + edits.iter().rposition(|e| !matches!(e, Edit::ExtraActual(_))) + { + if last_non_extra_actual_edit < edits.len() - 1 + && matches!(edits[last_non_extra_actual_edit], Edit::ExtraExpected(_)) + { + last_non_extra_actual_edit += 1; + } + edits.splice(last_non_extra_actual_edit + 1.., [Edit::AdditionalActual]); + } +} + +#[derive(Clone)] +struct Path { + actual_endpoint: usize, + expected_endpoint: usize, + edits: Vec>, +} + +impl Default for Path { + fn default() -> Self { + Self { actual_endpoint: 0, expected_endpoint: 0, edits: vec![] } + } +} + +impl Path { + fn extend_actual_endpoint(&self) -> Self { + Self { + actual_endpoint: self.actual_endpoint + 1, + expected_endpoint: self.expected_endpoint, + edits: self.edits.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::*; + use crate::Result; + use quickcheck::{quickcheck, Arbitrary, TestResult}; + + #[test] + fn returns_equal_when_strings_are_equal() -> Result<()> { + let result = edit_list(["A string"], ["A string"], Mode::Exact); + verify_that!(result, matches_pattern!(&Difference::Equal)) + } + + #[test] + fn returns_sequence_of_two_common_parts() -> Result<()> { + let result = edit_list( + ["A string (1)", "A string (2)"], + ["A string (1)", "A string (2)"], + Mode::Exact, + ); + verify_that!(result, matches_pattern!(&Difference::Equal)) + } + + #[test] + fn returns_extra_actual_when_only_actual_has_content() -> Result<()> { + let result = edit_list(["A string"], [], Mode::Exact); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![matches_pattern!( + &Edit::ExtraActual(eq("A string")) + )])) + ) + } + + #[test] + fn returns_extra_expected_when_only_expected_has_content() -> Result<()> { + let result = edit_list([], ["A string"], Mode::Exact); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![matches_pattern!( + &Edit::ExtraExpected(eq("A string")) + )])) + ) + } + + #[test] + fn returns_extra_actual_followed_by_extra_expected_with_two_unequal_strings() -> Result<()> { + let result = edit_list(["A string"], ["Another string"], Mode::Exact); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::ExtraActual(eq("A string"))), + matches_pattern!(&Edit::ExtraExpected(eq("Another string"))), + ])) + ) + } + + #[test] + fn interleaves_extra_actual_and_extra_expected_when_multiple_lines_differ() -> Result<()> { + let result = + edit_list(["A string", "A string"], ["Another string", "Another string"], Mode::Exact); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::ExtraActual(eq("A string"))), + matches_pattern!(&Edit::ExtraExpected(eq("Another string"))), + matches_pattern!(&Edit::ExtraActual(eq("A string"))), + matches_pattern!(&Edit::ExtraExpected(eq("Another string"))), + ])) + ) + } + + #[test] + fn returns_common_part_plus_difference_when_there_is_common_prefix() -> Result<()> { + let result = edit_list( + ["Common part", "Actual only"], + ["Common part", "Expected only"], + Mode::Exact, + ); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::Both(eq("Common part"))), + matches_pattern!(&Edit::ExtraActual(eq("Actual only"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only"))), + ])) + ) + } + + #[test] + fn returns_common_part_plus_extra_actual_when_actual_has_extra_suffix() -> Result<()> { + let result = edit_list(["Common part", "Actual only"], ["Common part"], Mode::Exact); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::Both(eq("Common part"))), + matches_pattern!(&Edit::ExtraActual(eq("Actual only"))), + ])) + ) + } + + #[test] + fn returns_common_part_plus_extra_expected_when_expected_has_extra_suffix() -> Result<()> { + let result = edit_list(["Common part"], ["Common part", "Expected only"], Mode::Exact); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::Both(eq("Common part"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only"))), + ])) + ) + } + + #[test] + fn returns_difference_plus_common_part_when_there_is_common_suffix() -> Result<()> { + let result = edit_list( + ["Actual only", "Common part"], + ["Expected only", "Common part"], + Mode::Exact, + ); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::ExtraActual(eq("Actual only"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only"))), + matches_pattern!(&Edit::Both(eq("Common part"))), + ])) + ) + } + + #[test] + fn returns_difference_plus_common_part_plus_difference_when_there_is_common_infix() -> Result<()> + { + let result = edit_list( + ["Actual only (1)", "Common part", "Actual only (2)"], + ["Expected only (1)", "Common part", "Expected only (2)"], + Mode::Exact, + ); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::ExtraActual(eq("Actual only (1)"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only (1)"))), + matches_pattern!(&Edit::Both(eq("Common part"))), + matches_pattern!(&Edit::ExtraActual(eq("Actual only (2)"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only (2)"))), + ])) + ) + } + + #[test] + fn returns_common_part_plus_difference_plus_common_part_when_there_is_common_prefix_and_suffix( + ) -> Result<()> { + let result = edit_list( + ["Common part (1)", "Actual only", "Common part (2)"], + ["Common part (1)", "Expected only", "Common part (2)"], + Mode::Exact, + ); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::Both(eq("Common part (1)"))), + matches_pattern!(&Edit::ExtraActual(eq("Actual only"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only"))), + matches_pattern!(&Edit::Both(eq("Common part (2)"))), + ])) + ) + } + + #[test] + fn returns_common_part_plus_extra_actual_plus_common_part_when_there_is_common_prefix_and_suffix( + ) -> Result<()> { + let result = edit_list( + ["Common part (1)", "Actual only", "Common part (2)"], + ["Common part (1)", "Common part (2)"], + Mode::Exact, + ); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::Both(eq("Common part (1)"))), + matches_pattern!(&Edit::ExtraActual(eq("Actual only"))), + matches_pattern!(&Edit::Both(eq("Common part (2)"))), + ])) + ) + } + + #[test] + fn returns_common_part_plus_extra_expected_plus_common_part_when_there_is_common_prefix_and_suffix( + ) -> Result<()> { + let result = edit_list( + ["Common part (1)", "Common part (2)"], + ["Common part (1)", "Expected only", "Common part (2)"], + Mode::Exact, + ); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::Both(eq("Common part (1)"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only"))), + matches_pattern!(&Edit::Both(eq("Common part (2)"))), + ])) + ) + } + + #[test] + fn skips_extra_parts_on_actual_at_end_in_prefix_mode() -> Result<()> { + let result = edit_list( + ["Common part", "Actual only"], + ["Expected only", "Common part"], + Mode::Prefix, + ); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref not(contains(matches_pattern!( + &Edit::ExtraActual(eq("Actual only")) + ))))) + ) + } + + #[test] + fn does_not_skip_extra_parts_on_actual_in_prefix_mode_at_end_when_they_are_in_common( + ) -> Result<()> { + let result = edit_list( + ["Actual only", "Common part"], + ["Expected only", "Common part"], + Mode::Prefix, + ); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::ExtraActual(eq("Actual only"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only"))), + matches_pattern!(&Edit::Both(eq("Common part"))), + ])) + ) + } + + #[test] + fn does_not_skip_corresponding_line_on_actual_when_actual_and_expected_differ_in_prefix_mode( + ) -> Result<()> { + let result = edit_list(["Actual only"], ["Expected only"], Mode::Prefix); + verify_that!( + result, + matches_pattern!(&Difference::Editable(ref elements_are![ + matches_pattern!(&Edit::ExtraActual(eq("Actual only"))), + matches_pattern!(&Edit::ExtraExpected(eq("Expected only"))), + matches_pattern!(&Edit::AdditionalActual), + ])) + ) + } + + #[test] + fn returns_unrelated_when_maximum_distance_exceeded() -> Result<()> { + let result = edit_list(0..=50, 60..110, Mode::Exact); + verify_that!(result, matches_pattern!(&Difference::Unrelated)) + } + + quickcheck! { + fn edit_list_edits_actual_to_expected( + actual: Vec, + expected: Vec + ) -> TestResult { + match edit_list(actual.clone(), expected.clone(), Mode::Exact) { + Difference::Equal => TestResult::from_bool(actual == expected), + Difference::Editable(edit_list) => { + TestResult::from_bool(apply_edits_to_actual(&edit_list, &actual) == expected) + } + Difference::Unrelated => { + if actual == expected { + TestResult::failed() + } else { + TestResult::discard() + } + } + } + } + } + + quickcheck! { + fn edit_list_edits_expected_to_actual( + actual: Vec, + expected: Vec + ) -> TestResult { + match edit_list(actual.clone(), expected.clone(), Mode::Exact) { + Difference::Equal => TestResult::from_bool(actual == expected), + Difference::Editable(edit_list) => { + TestResult::from_bool(apply_edits_to_expected(&edit_list, &expected) == actual) + } + Difference::Unrelated => { + if actual == expected { + TestResult::failed() + } else { + TestResult::discard() + } + } + } + } + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Alphabet { + A, + B, + C, + } + + impl Arbitrary for Alphabet { + fn arbitrary(g: &mut quickcheck::Gen) -> Self { + g.choose(&[Alphabet::A, Alphabet::B, Alphabet::C]).copied().unwrap() + } + } + + fn apply_edits_to_actual( + edit_list: &[Edit], + actual: &[T], + ) -> Vec { + let mut result = Vec::new(); + let mut actual_iter = actual.iter(); + for edit in edit_list { + match edit { + Edit::ExtraActual(value) => { + assert_that!(actual_iter.next(), some(eq(value))); + } + Edit::ExtraExpected(value) => { + result.push(*value); + } + Edit::Both(value) => { + assert_that!(actual_iter.next(), some(eq(value))); + result.push(*value); + } + Edit::AdditionalActual => { + fail!("Unexpected Edit::AdditionalActual").unwrap(); + } + } + } + assert_that!(actual_iter.next(), none()); + result + } + + fn apply_edits_to_expected( + edit_list: &[Edit], + expected: &[T], + ) -> Vec { + let mut result = Vec::new(); + let mut expected_iter = expected.iter(); + for edit in edit_list { + match edit { + Edit::ExtraActual(value) => { + result.push(*value); + } + Edit::ExtraExpected(value) => { + assert_that!(expected_iter.next(), some(eq(value))); + } + Edit::Both(value) => { + assert_that!(expected_iter.next(), some(eq(value))); + result.push(*value); + } + Edit::AdditionalActual => { + fail!("Unexpected Edit::AdditionalActual").unwrap(); + } + } + } + assert_that!(expected_iter.next(), none()); + result + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matcher_support/match_matrix.rs b/third_party/crates/googletest-0.14.3/src/matcher_support/match_matrix.rs new file mode 100644 index 0000000..98dc52a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matcher_support/match_matrix.rs @@ -0,0 +1,518 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[doc(hidden)] +pub mod internal { + use crate::description::Description; + use crate::matcher::{Matcher, MatcherResult}; + use crate::matcher_support::count_elements::count_elements; + use std::collections::HashSet; + use std::fmt::{Debug, Display}; + + /// The requirements of the mapping between matchers and actual values by + /// which [`UnorderedElementsAre`] is deemed to match its input. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + #[derive(Clone, Copy)] + pub enum Requirements { + /// There must be a 1:1 correspondence between the actual values and the + /// matchers. + PerfectMatch, + + /// The mapping from matched actual values to their corresponding + /// matchers must be surjective. + Superset, + + /// The mapping from matchers to matched actual values must be + /// surjective. + Subset, + } + + impl Requirements { + pub fn explain_size_mismatch( + &self, + actual: ContainerT, + expected_size: usize, + ) -> Option { + let actual_size = count_elements(actual); + match self { + Requirements::PerfectMatch if actual_size != expected_size => { + Some(format!("which has size {actual_size} (expected {expected_size})").into()) + } + + Requirements::Superset if actual_size < expected_size => Some( + format!("which has size {actual_size} (expected at least {expected_size})") + .into(), + ), + + Requirements::Subset if actual_size > expected_size => Some( + format!("which has size {actual_size} (expected at most {expected_size})") + .into(), + ), + + _ => None, + } + } + } + + impl Display for Requirements { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Requirements::PerfectMatch => { + write!(f, "perfect") + } + Requirements::Superset => { + write!(f, "superset") + } + Requirements::Subset => { + write!(f, "subset") + } + } + } + } + + /// The bipartite matching graph between actual and expected elements. + pub(crate) struct MatchMatrix { + graph: Vec>, // graph[actual_idx][expected_idx] + expected_len: usize, + } + + impl MatchMatrix { + pub(crate) fn generate<'a, T: Debug + Copy + 'a>( + actual: impl IntoIterator, + expected: &[Box + 'a>], + ) -> Self { + let expected_len = expected.len(); + let graph = actual + .into_iter() + .map(|actual| { + expected + .iter() + .map(|expected_matcher| expected_matcher.matches(actual)) + .collect() + }) + .collect(); + MatchMatrix { graph, expected_len } + } + + pub(crate) fn is_match_for(&self, requirements: Requirements) -> bool { + match requirements { + Requirements::PerfectMatch => { + !self.find_unmatchable_elements().has_unmatchable_elements() + && self.find_best_match().is_full_match() + } + Requirements::Superset => { + !self.find_unmatched_expected().has_unmatchable_elements() + && self.find_best_match().is_superset_match() + } + Requirements::Subset => { + !self.find_unmatched_actual().has_unmatchable_elements() + && self.find_best_match().is_subset_match() + } + } + } + + pub(crate) fn explain_unmatchable( + &self, + requirements: Requirements, + ) -> Option { + let unmatchable_elements = match requirements { + Requirements::PerfectMatch => self.find_unmatchable_elements(), + Requirements::Superset => self.find_unmatched_expected(), + Requirements::Subset => self.find_unmatched_actual(), + }; + unmatchable_elements.get_explanation() + } + + // Verifies that each actual matches at least one expected and that + // each expected matches at least one actual. + // This is a necessary condition but not sufficient. But it is faster + // than `find_best_match()`. + fn find_unmatchable_elements(&self) -> UnmatchableElements { + let unmatchable_actual = + self.graph.iter().map(|row| row.iter().all(|&e| e.is_no_match())).collect(); + let mut unmatchable_expected = vec![false; self.expected_len]; + for (col_idx, expected) in unmatchable_expected.iter_mut().enumerate() { + *expected = self.graph.iter().map(|row| row[col_idx]).all(|e| e.is_no_match()); + } + UnmatchableElements { unmatchable_actual, unmatchable_expected } + } + + fn find_unmatched_expected(&self) -> UnmatchableElements { + let mut unmatchable_expected = vec![false; self.expected_len]; + for (col_idx, expected) in unmatchable_expected.iter_mut().enumerate() { + *expected = self.graph.iter().map(|row| row[col_idx]).all(|e| e.is_no_match()); + } + UnmatchableElements { + unmatchable_actual: vec![false; self.expected_len], + unmatchable_expected, + } + } + + fn find_unmatched_actual(&self) -> UnmatchableElements { + let unmatchable_actual = + self.graph.iter().map(|row| row.iter().all(|e| e.is_no_match())).collect(); + UnmatchableElements { + unmatchable_actual, + unmatchable_expected: vec![false; self.expected_len], + } + } + + // Verifies that a full match exists. + // + // Uses the well-known Ford-Fulkerson max flow method to find a maximum + // bipartite matching. Flow is considered to be from actual to expected. + // There is an implicit source node that is connected to all of the actual + // nodes, and an implicit sink node that is connected to all of the + // expected nodes. All edges have unit capacity. + // + // Neither the flow graph nor the residual flow graph are represented + // explicitly. Instead, they are implied by the information in `self.graph` and + // the local `actual_match : vec![Option; self.expected_len]` whose + // elements are initialized to `None`. This represents the initial state + // of the algorithm, where the flow graph is empty, and the residual + // flow graph has the following edges: + // - An edge from source to each actual element node + // - An edge from each expected element node to sink + // - An edge from each actual element node to each expected element node, if + // the actual element matches the expected element, i.e. + // `matches!(self.graph[actual_id][expected_id], Matches)` + // + // When the `try_augment(...)` method adds a flow, it sets `actual_match[l] = + // Some(r)` for some nodes l and r. This induces the following changes: + // - The edges (source, l), (l, r), and (r, sink) are added to the flow graph. + // - The same three edges are removed from the residual flow graph. + // - The reverse edges (l, source), (r, l), and (sink, r) are added to the + // residual flow graph, which is a directional graph representing unused + // flow capacity. + // + // When the method augments a flow (changing `actual_match[l]` from `Some(r1)` + // to `Some(r2)`), this can be thought of as "undoing" the above steps + // with respect to r1 and "redoing" them with respect to r2. + // + // It bears repeating that the flow graph and residual flow graph are + // never represented explicitly, but can be derived by looking at the + // information in 'self.graph' and in `actual_match`. + // + // As an optimization, there is a second local `expected_match: + // vec![Option; self.expected_len]` which does not provide any + // new information. Instead, it enables more efficient queries about + // edges entering or leaving the expected elements nodes of the flow or + // residual flow graphs. The following invariants are maintained: + // + // actual_match[a] == None or expected_match[actual_match[a].unwrap()] == + // Some(a) + // expected_match[r] == None or actual_match[expected_match[e].unwrap()] == + // Some(e) + // + // | [ source ] | + // | ||| | + // | ||| | + // | ||\-> actual_match[0]=Some(1) -\ expected_match[0]=None ---\ | + // | || | | | + // | |\--> actual_match[1]=None \-> expected_match[1]=Some(0) --\| | + // | | || | + // | \---> actual_match[2]=Some(2) --> expected_match[2]=Some(2) -\|| | + // | ||| | + // | elements matchers vvv | + // | [ sink ] | + // + // See Also: + // [1] Cormen, et al (2001). "Section 26.2: The Ford-Fulkerson method". + // "Introduction to Algorithms (Second ed.)", pp. 651-664. + // [2] "Ford-Fulkerson algorithm", Wikipedia, + // 'http://en.wikipedia.org/wiki/Ford%E2%80%93Fulkerson_algorithm' + pub(crate) fn find_best_match(&self) -> BestMatch { + let mut actual_match = vec![None; self.graph.len()]; + let mut expected_match: Vec> = vec![None; self.expected_len]; + // Searches the residual flow graph for a path from each actual node to + // the sink in the residual flow graph, and if one is found, add this path + // to the graph. + // It's okay to search through the actual nodes once. The + // edge from the implicit source node to each previously-visited actual + // node will have flow if that actual node has any path to the sink + // whatsoever. Subsequent augmentations can only add flow to the + // network, and cannot take away that previous flow unit from the source. + // Since the source-to-actual edge can only carry one flow unit (or, + // each actual element can be matched to only one expected element), there is no + // need to visit the actual nodes more than once looking for + // augmented paths. The flow is known to be possible or impossible + // by looking at the node once. + for actual_idx in 0..self.graph.len() { + assert!(actual_match[actual_idx].is_none()); + let mut seen = vec![false; self.expected_len]; + self.try_augment(actual_idx, &mut seen, &mut actual_match, &mut expected_match); + } + BestMatch::new(actual_match, self.expected_len) + } + + // Perform a depth-first search from actual node `actual_idx` to the sink by + // searching for an unassigned expected node. If a path is found, flow + // is added to the network by linking the actual and expected vector elements + // corresponding each segment of the path. Returns true if a path to + // sink was found, which means that a unit of flow was added to the + // network. The 'seen' array elements correspond to expected nodes and are + // marked to eliminate cycles from the search. + // + // Actual nodes will only be explored at most once because they + // are accessible from at most one expected node in the residual flow + // graph. + // + // Note that `actual_match[actual_idx]` is the only element of `actual_match` + // that `try_augment(...)` will potentially transition from `None` to + // `Some(...)`. Any other `actual_match` element holding `None` before + // `try_augment(...)` will be holding it when `try_augment(...)` + // returns. + // + fn try_augment( + &self, + actual_idx: usize, + seen: &mut Vec, + actual_match: &mut [Option], + expected_match: &mut Vec>, + ) -> bool { + for expected_idx in 0..self.expected_len { + if seen[expected_idx] { + continue; + } + if self.graph[actual_idx][expected_idx].is_no_match() { + continue; + } + // There is an edge between `actual_idx` and `expected_idx`. + seen[expected_idx] = true; + // Next a search is performed to determine whether + // this edge is a dead end or leads to the sink. + // + // `expected_match[expected_idx].is_none()` means that there is residual flow + // from expected node at index expected_idx to the sink, so we + // can use that to finish this flow path and return success. + // + // Otherwise, we look for a residual flow starting from + // `expected_match[expected_idx].unwrap()` by calling + // ourselves recursively to see if this ultimately leads to + // sink. + if expected_match[expected_idx].is_none() + || self.try_augment( + expected_match[expected_idx].unwrap(), + seen, + actual_match, + expected_match, + ) + { + // We found a residual flow from source to sink. We thus need to add the new + // edge to the current flow. + // Note: this also remove the potential flow that existed by overwriting the + // value in the `expected_match` and `actual_match`. + expected_match[expected_idx] = Some(actual_idx); + actual_match[actual_idx] = Some(expected_idx); + return true; + } + } + false + } + } + + /// The list of elements that do not match any element in the corresponding + /// set. + /// TODO - Use BitVec. + pub(crate) struct UnmatchableElements { + unmatchable_actual: Vec, + unmatchable_expected: Vec, + } + + impl UnmatchableElements { + fn has_unmatchable_elements(&self) -> bool { + self.unmatchable_actual.iter().any(|b| *b) + || self.unmatchable_expected.iter().any(|b| *b) + } + + fn get_explanation(&self) -> Option { + let unmatchable_actual = self.unmatchable_actual(); + let actual_idx = unmatchable_actual + .iter() + .map(|idx| format!("#{idx}")) + .collect::>() + .join(", "); + let unmatchable_expected = self.unmatchable_expected(); + let expected_idx = unmatchable_expected + .iter() + .map(|idx| format!("#{idx}")) + .collect::>() + .join(", "); + match (unmatchable_actual.len(), unmatchable_expected.len()) { + (0, 0) => None, + (1, 0) => { + Some(format!("whose element {actual_idx} does not match any expected elements").into()) + } + (_, 0) => { + Some(format!("whose elements {actual_idx} do not match any expected elements",).into()) + } + (0, 1) => Some(format!( + "which has no element matching the expected element {expected_idx}" + ).into()), + (0, _) => Some(format!( + "which has no elements matching the expected elements {expected_idx}" + ).into()), + (1, 1) => Some(format!( + "whose element {actual_idx} does not match any expected elements and no elements match the expected element {expected_idx}" + ).into()), + (_, 1) => Some(format!( + "whose elements {actual_idx} do not match any expected elements and no elements match the expected element {expected_idx}" + ).into()), + (1, _) => Some(format!( + "whose element {actual_idx} does not match any expected elements and no elements match the expected elements {expected_idx}" + ).into()), + (_, _) => Some(format!( + "whose elements {actual_idx} do not match any expected elements and no elements match the expected elements {expected_idx}" + ).into()), + } + } + + fn unmatchable_actual(&self) -> Vec { + self.unmatchable_actual + .iter() + .enumerate() + .filter_map(|(idx, b)| if *b { Some(idx) } else { None }) + .collect() + } + + fn unmatchable_expected(&self) -> Vec { + self.unmatchable_expected + .iter() + .enumerate() + .filter_map(|(idx, b)| if *b { Some(idx) } else { None }) + .collect() + } + } + + /// The representation of a match between actual and expected. + /// The value at idx represents to which expected the actual at idx is + /// matched with. For example, `BestMatch::new([Some(0), None, Some(1)], + /// ..)` means: + /// * The 0th element in actual matches the 0th element in expected. + /// * The 1st element in actual does not match. + /// * The 2nd element in actual matches the 1st element in expected. + pub(crate) struct BestMatch { + actual_match: Vec>, + expected_len: usize, + } + + impl BestMatch { + fn new(actual_match: Vec>, expected_len: usize) -> BestMatch { + BestMatch { actual_match, expected_len } + } + + pub(crate) fn is_full_match(&self) -> bool { + self.actual_match.iter().all(|o| o.is_some()) + } + + pub(crate) fn is_subset_match(&self) -> bool { + self.is_full_match() + } + + pub(crate) fn is_superset_match(&self) -> bool { + self.get_unmatched_expected().is_empty() + } + + pub(crate) fn get_matches(&self) -> impl Iterator + '_ { + self.actual_match.iter().enumerate().filter_map(|(actual_idx, maybe_expected_idx)| { + maybe_expected_idx.map(|expected_idx| (actual_idx, expected_idx)) + }) + } + + pub(crate) fn get_unmatched_actual(&self) -> impl Iterator + '_ { + self.actual_match + .iter() + .enumerate() + .filter(|&(_, o)| o.is_none()) + .map(|(actual_idx, _)| actual_idx) + } + + pub(crate) fn get_unmatched_expected(&self) -> Vec { + let matched_expected: HashSet<_> = self.actual_match.iter().flatten().collect(); + (0..self.expected_len) + .filter(|expected_idx| !matched_expected.contains(expected_idx)) + .collect() + } + + pub(crate) fn get_explanation<'a, T: Debug + Copy>( + &self, + actual: impl IntoIterator, + expected: &[Box + 'a>], + requirements: Requirements, + ) -> Option { + let actual: Vec<_> = actual.into_iter().collect(); + if self.is_full_match() { + return None; + } + let mut error_message = + format!("which does not have a {requirements} match with the expected elements."); + + error_message.push_str("\n The best match found was: "); + + let matches = self.get_matches().map(|(actual_idx, expected_idx)|{ + format!( + "Actual element {:?} at index {actual_idx} matched expected element `{}` at index {expected_idx}.", + actual[actual_idx], + expected[expected_idx].describe(MatcherResult::Match), + )}); + + let unmatched_actual = self.get_unmatched_actual().map(|actual_idx| { + format!( + "Actual element {:#?} at index {actual_idx} did not match any remaining expected element.", + actual[actual_idx] + ) + }); + + let unmatched_expected = self.get_unmatched_expected().into_iter().map(|expected_idx|{format!( + "Expected element `{}` at index {expected_idx} did not match any remaining actual element.", + expected[expected_idx].describe(MatcherResult::Match) + )}); + + let best_match = matches + .chain(unmatched_actual) + .chain(unmatched_expected) + .collect::() + .indent(); + Some(format!( + "which does not have a {requirements} match with the expected elements. The best match found was:\n{best_match}" + ).into()) + } + } + + #[cfg(test)] + mod tests { + use super::*; + use crate::prelude::*; + + #[test] + fn match_matrix_generate() { + let actual = vec![1, 2, 3, 4]; + let expected: Vec>> = + vec![Box::new(eq(1)), Box::new(eq(2)), Box::new(eq(3))]; + let matrix = MatchMatrix::generate(actual, &expected); + + assert_eq!( + matrix.graph, + vec![ + vec![true.into(), false.into(), false.into()], + vec![false.into(), true.into(), false.into()], + vec![false.into(), false.into(), true.into()], + vec![false.into(), false.into(), false.into()], + ] + ); + } + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matcher_support/mod.rs b/third_party/crates/googletest-0.14.3/src/matcher_support/mod.rs new file mode 100644 index 0000000..1d99b7a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matcher_support/mod.rs @@ -0,0 +1,31 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Utilities to facilitate writing matchers. +//! +//! Tests normally do not need to import anything from this module. Some of +//! these facilities could be useful to downstream users writing custom +//! matchers. + +mod auto_eq; +pub(crate) mod count_elements; +pub(crate) mod edit_distance; +pub(crate) mod match_matrix; +pub(crate) mod summarize_diff; +pub(crate) mod zipped_iterator; + +pub mod __internal_unstable_do_not_depend_on_these { + pub use super::auto_eq::internal::{ExpectedKind, Wrapper}; + pub use crate::__auto_eq as auto_eq; +} diff --git a/third_party/crates/googletest-0.14.3/src/matcher_support/summarize_diff.rs b/third_party/crates/googletest-0.14.3/src/matcher_support/summarize_diff.rs new file mode 100644 index 0000000..09b64bc --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matcher_support/summarize_diff.rs @@ -0,0 +1,591 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc(hidden)] + +use crate::matcher_support::edit_distance; +#[rustversion::since(1.70)] +use std::io::IsTerminal; +use std::{borrow::Cow, cell::Cell, fmt::Display}; + +/// Returns a string describing how the expected and actual lines differ. +/// +/// This is included in a match explanation for [`EqMatcher`] and +/// [`crate::matchers::str_matcher::StrMatcher`]. +/// +/// If the actual value has less than two lines, or the two differ by more than +/// the maximum edit distance, then this returns the empty string. If the two +/// are equal, it returns a simple statement that they are equal. Otherwise, +/// this constructs a unified diff view of the actual and expected values. +pub(crate) fn create_diff( + actual_debug: &str, + expected_debug: &str, + diff_mode: edit_distance::Mode, +) -> Cow<'static, str> { + if actual_debug.lines().count() < 2 { + // If the actual debug is only one line, then there is no point in doing a + // line-by-line diff. + return "".into(); + } + + match edit_distance::edit_list(actual_debug.lines(), expected_debug.lines(), diff_mode) { + edit_distance::Difference::Equal => { + // str.lines() is oblivious to the last newline in a + // string, so we need to check this to make sure we don't spuriously + // claim that 'hello' and 'hello\n' are identical debug strings. + // + // Although we would have liked to resolve by replacing + // str::lines() with str::split('\n'), the potentially + // empty last element interferes with good diff output for + // "contains" checks. + let actual_newline_terminated = actual_debug.ends_with('\n'); + let expected_newline_terminated = expected_debug.ends_with('\n'); + if actual_newline_terminated && !expected_newline_terminated { + "Actual includes a terminating newline that is absent from expected.".into() + } else if !actual_newline_terminated && expected_newline_terminated { + "Actual omits a terminating newline that is present in expected.".into() + } else { + "No difference found between debug strings.".into() + } + } + edit_distance::Difference::Editable(edit_list) => { + format!("{}{}", summary_header(), edit_list.into_iter().collect::(),) + .into() + } + edit_distance::Difference::Unrelated => "".into(), + } +} + +/// Returns a string describing how the expected and actual differ after +/// reversing the lines in each. +/// +/// This is similar to [`create_diff`] except that it first reverses the lines +/// in both the expected and actual values, then reverses the constructed edit +/// list. When `diff_mode` is [`edit_distance::Mode::Prefix`], this becomes a +/// diff of the suffix for use by [`ends_with`][crate::matchers::ends_with]. +pub(crate) fn create_diff_reversed( + actual_debug: &str, + expected_debug: &str, + diff_mode: edit_distance::Mode, +) -> Cow<'static, str> { + if actual_debug.lines().count() < 2 { + // If the actual debug is only one line, then there is no point in doing a + // line-by-line diff. + return "".into(); + } + let mut actual_lines_reversed = actual_debug.lines().collect::>(); + let mut expected_lines_reversed = expected_debug.lines().collect::>(); + actual_lines_reversed.reverse(); + expected_lines_reversed.reverse(); + match edit_distance::edit_list(actual_lines_reversed, expected_lines_reversed, diff_mode) { + edit_distance::Difference::Equal => "No difference found between debug strings.".into(), + edit_distance::Difference::Editable(mut edit_list) => { + edit_list.reverse(); + format!("{}{}", summary_header(), edit_list.into_iter().collect::(),) + .into() + } + edit_distance::Difference::Unrelated => "".into(), + } +} + +// Produces the header, with or without coloring depending on +// USE_COLOR +fn summary_header() -> Cow<'static, str> { + if USE_COLOR.with(Cell::get) { + format!( + "Difference(-{ACTUAL_ONLY_STYLE}actual{RESET_ALL} / +{EXPECTED_ONLY_STYLE}expected{RESET_ALL}):" + ).into() + } else { + "Difference(-actual / +expected):".into() + } +} + +// Aggregator collecting the lines to be printed in the difference summary. +// +// This is buffered in order to allow a future line to potentially impact how +// the current line would be printed. +#[derive(Default)] +struct BufferedSummary<'a> { + summary: SummaryBuilder, + buffer: Buffer<'a>, +} + +impl<'a> BufferedSummary<'a> { + // Appends a new line which is common to both actual and expected. + fn feed_common_lines(&mut self, common_line: &'a str) { + if let Buffer::CommonLines(ref mut common_lines) = self.buffer { + common_lines.push(common_line); + } else { + self.flush_buffer(); + self.buffer = Buffer::CommonLines(vec![common_line]); + } + } + + // Appends a new line which is found only in the actual string. + fn feed_extra_actual(&mut self, extra_actual: &'a str) { + if let Buffer::ExtraExpectedLineChunk(extra_expected) = self.buffer { + self.print_inline_diffs(extra_actual, extra_expected); + self.buffer = Buffer::Empty; + } else { + self.flush_buffer(); + self.buffer = Buffer::ExtraActualLineChunk(extra_actual); + } + } + + // Appends a new line which is found only in the expected string. + fn feed_extra_expected(&mut self, extra_expected: &'a str) { + if let Buffer::ExtraActualLineChunk(extra_actual) = self.buffer { + self.print_inline_diffs(extra_actual, extra_expected); + self.buffer = Buffer::Empty; + } else { + self.flush_buffer(); + self.buffer = Buffer::ExtraExpectedLineChunk(extra_expected); + } + } + + // Appends a comment for the additional line at the start or the end of the + // actual string which should be omitted. + fn feed_additional_actual(&mut self) { + self.flush_buffer(); + self.summary.new_line(); + self.summary.push_str_as_comment("<---- remaining lines omitted ---->"); + } + + fn flush_buffer(&mut self) { + self.buffer.flush(&mut self.summary); + } + + fn print_inline_diffs(&mut self, actual_line: &str, expected_line: &str) { + let line_edits = edit_distance::edit_list( + actual_line.chars(), + expected_line.chars(), + edit_distance::Mode::Exact, + ); + + if let edit_distance::Difference::Editable(edit_list) = line_edits { + let mut actual_summary = SummaryBuilder::default(); + actual_summary.new_line_for_actual(); + let mut expected_summary = SummaryBuilder::default(); + expected_summary.new_line_for_expected(); + for edit in &edit_list { + match edit { + edit_distance::Edit::ExtraActual(c) => actual_summary.push_actual_only(*c), + edit_distance::Edit::ExtraExpected(c) => { + expected_summary.push_expected_only(*c) + } + edit_distance::Edit::Both(c) => { + actual_summary.push_actual_with_match(*c); + expected_summary.push_expected_with_match(*c); + } + edit_distance::Edit::AdditionalActual => { + // Calling edit_distance::edit_list(_, _, Mode::Exact) should never return + // this enum + panic!("This should not happen. This is a bug in gtest_rust") + } + } + } + actual_summary.reset_ansi(); + expected_summary.reset_ansi(); + self.summary.push_str(&actual_summary.summary); + self.summary.push_str(&expected_summary.summary); + } else { + self.summary.new_line_for_actual(); + self.summary.push_str_actual_only(actual_line); + self.summary.new_line_for_expected(); + self.summary.push_str_expected_only(expected_line); + } + } +} + +impl<'a> FromIterator> for BufferedSummary<'a> { + fn from_iter>>(iter: T) -> Self { + let mut buffered_summary = BufferedSummary::default(); + for edit in iter { + match edit { + edit_distance::Edit::Both(same) => { + buffered_summary.feed_common_lines(same); + } + edit_distance::Edit::ExtraActual(actual) => { + buffered_summary.feed_extra_actual(actual); + } + edit_distance::Edit::ExtraExpected(expected) => { + buffered_summary.feed_extra_expected(expected); + } + edit_distance::Edit::AdditionalActual => { + buffered_summary.feed_additional_actual(); + } + }; + } + buffered_summary.flush_buffer(); + buffered_summary.summary.reset_ansi(); + + buffered_summary + } +} + +impl Display for BufferedSummary<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !matches!(self.buffer, Buffer::Empty) { + panic!("Buffer is not empty. This is a bug in gtest_rust.") + } + if !self.summary.last_ansi_style.is_empty() { + panic!("ANSI style has not been reset. This is a bug in gtest_rust.") + } + self.summary.summary.fmt(f) + } +} + +enum Buffer<'a> { + Empty, + CommonLines(Vec<&'a str>), + ExtraActualLineChunk(&'a str), + ExtraExpectedLineChunk(&'a str), +} + +impl<'a> Buffer<'a> { + fn flush(&mut self, summary: &mut SummaryBuilder) { + match self { + Buffer::Empty => {} + Buffer::CommonLines(common_lines) => { + Self::flush_common_lines(std::mem::take(common_lines), summary); + } + Buffer::ExtraActualLineChunk(extra_actual) => { + summary.new_line_for_actual(); + summary.push_str_actual_only(extra_actual); + } + Buffer::ExtraExpectedLineChunk(extra_expected) => { + summary.new_line_for_expected(); + summary.push_str_expected_only(extra_expected); + } + }; + *self = Buffer::Empty; + } + + fn flush_common_lines(common_lines: Vec<&'a str>, summary: &mut SummaryBuilder) { + // The number of the lines kept before and after the compressed lines. + const COMMON_LINES_CONTEXT_SIZE: usize = 2; + + if common_lines.len() <= 2 * COMMON_LINES_CONTEXT_SIZE + 1 { + for line in common_lines { + summary.new_line(); + summary.push_str(line); + } + return; + } + + let start_context = &common_lines[0..COMMON_LINES_CONTEXT_SIZE]; + + for line in start_context { + summary.new_line(); + summary.push_str(line); + } + + summary.new_line(); + summary.push_str_as_comment(&format!( + "<---- {} common lines omitted ---->", + common_lines.len() - 2 * COMMON_LINES_CONTEXT_SIZE, + )); + + let end_context = + &common_lines[common_lines.len() - COMMON_LINES_CONTEXT_SIZE..common_lines.len()]; + + for line in end_context { + summary.new_line(); + summary.push_str(line); + } + } +} + +impl Default for Buffer<'_> { + fn default() -> Self { + Self::Empty + } +} + +thread_local! { + pub(crate) static USE_COLOR: Cell = Cell::new(stdout_supports_color()); +} + +#[rustversion::since(1.70)] +fn stdout_supports_color() -> bool { + #[allow(clippy::incompatible_msrv)] + match (is_env_var_set("NO_COLOR"), is_env_var_set("FORCE_COLOR")) { + (true, _) => false, + (false, true) => true, + (false, false) => std::io::stdout().is_terminal(), + } +} + +#[rustversion::not(since(1.70))] +fn stdout_supports_color() -> bool { + is_env_var_set("FORCE_COLOR") && !is_env_var_set("NO_COLOR") +} + +fn is_env_var_set(var: &'static str) -> bool { + std::env::var(var).map(|s| !s.is_empty()).unwrap_or(false) +} + +// Font in italic +const COMMENT_STYLE: &str = "\x1B[3m"; +// Font in green and bold +const EXPECTED_ONLY_STYLE: &str = "\x1B[1;32m"; +// Font in red and bold +const ACTUAL_ONLY_STYLE: &str = "\x1B[1;31m"; +// Font in green onlyh +const EXPECTED_WITH_MATCH_STYLE: &str = "\x1B[32m"; +// Font in red only +const ACTUAL_WITH_MATCH_STYLE: &str = "\x1B[31m"; +// Reset all ANSI formatting +const RESET_ALL: &str = "\x1B[0m"; + +#[derive(Default)] +struct SummaryBuilder { + summary: String, + last_ansi_style: &'static str, +} + +impl SummaryBuilder { + fn push_str(&mut self, element: &str) { + self.reset_ansi(); + self.summary.push_str(element); + } + + fn push_str_as_comment(&mut self, element: &str) { + self.set_ansi(COMMENT_STYLE); + self.summary.push_str(element); + } + + fn push_str_actual_only(&mut self, element: &str) { + self.set_ansi(ACTUAL_ONLY_STYLE); + self.summary.push_str(element); + } + + fn push_str_expected_only(&mut self, element: &str) { + self.set_ansi(EXPECTED_ONLY_STYLE); + self.summary.push_str(element); + } + + fn push_actual_only(&mut self, element: char) { + self.set_ansi(ACTUAL_ONLY_STYLE); + self.summary.push(element); + } + + fn push_expected_only(&mut self, element: char) { + self.set_ansi(EXPECTED_ONLY_STYLE); + self.summary.push(element); + } + + fn push_actual_with_match(&mut self, element: char) { + self.set_ansi(ACTUAL_WITH_MATCH_STYLE); + self.summary.push(element); + } + + fn push_expected_with_match(&mut self, element: char) { + self.set_ansi(EXPECTED_WITH_MATCH_STYLE); + self.summary.push(element); + } + + fn new_line(&mut self) { + self.reset_ansi(); + self.summary.push_str("\n "); + } + + fn new_line_for_actual(&mut self) { + self.reset_ansi(); + self.summary.push_str("\n-"); + } + + fn new_line_for_expected(&mut self) { + self.reset_ansi(); + self.summary.push_str("\n+"); + } + + fn reset_ansi(&mut self) { + if !self.last_ansi_style.is_empty() && USE_COLOR.with(Cell::get) { + self.summary.push_str(RESET_ALL); + self.last_ansi_style = ""; + } + } + + fn set_ansi(&mut self, ansi_style: &'static str) { + if !USE_COLOR.with(Cell::get) || self.last_ansi_style == ansi_style { + return; + } + if !self.last_ansi_style.is_empty() { + self.summary.push_str(RESET_ALL); + } + self.summary.push_str(ansi_style); + self.last_ansi_style = ansi_style; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{matcher_support::edit_distance::Mode, prelude::*, Result}; + use indoc::indoc; + use std::fmt::Write; + + // Make a long text with each element of the iterator on one line. + // `collection` must contains at least one element. + fn build_text(mut collection: impl Iterator) -> String { + let mut text = String::new(); + write!(&mut text, "{}", collection.next().expect("Provided collection without elements")) + .unwrap(); + for item in collection { + write!(&mut text, "\n{item}").unwrap(); + } + text + } + + #[test] + fn create_diff_smaller_than_one_line() -> Result<()> { + verify_that!(create_diff("One", "Two", Mode::Exact), eq("")) + } + + #[test] + fn create_diff_exact_same() -> Result<()> { + let expected = indoc! {" + One + Two + "}; + let actual = indoc! {" + One + Two + "}; + verify_that!( + create_diff(expected, actual, Mode::Exact), + eq("No difference found between debug strings.") + ) + } + + #[test] + fn create_diff_multiline_diff() -> Result<()> { + let expected = indoc! {" + prefix + Actual#1 + Actual#2 + Actual#3 + suffix"}; + let actual = indoc! {" + prefix + Expected@one + Expected@two + suffix"}; + // TODO: It would be better to have all the Actual together followed by all the + // Expected together. + verify_that!( + create_diff(expected, actual, Mode::Exact), + eq(indoc!( + " + Difference(-actual / +expected): + prefix + -Actual#1 + +Expected@one + -Actual#2 + +Expected@two + -Actual#3 + suffix" + )) + ) + } + + #[test] + fn create_diff_exact_unrelated() -> Result<()> { + verify_that!(create_diff(&build_text(1..500), &build_text(501..1000), Mode::Exact), eq("")) + } + + #[test] + fn create_diff_exact_small_difference() -> Result<()> { + verify_that!( + create_diff(&build_text(1..50), &build_text(1..51), Mode::Exact), + eq(indoc! { + " + Difference(-actual / +expected): + 1 + 2 + <---- 45 common lines omitted ----> + 48 + 49 + +50" + }) + ) + } + + #[test] + fn create_diff_exact_small_difference_with_color() -> Result<()> { + USE_COLOR.with(|cell| cell.set(true)); + + verify_that!( + create_diff(&build_text(1..50), &build_text(1..51), Mode::Exact), + eq(indoc! { + " + Difference(-\x1B[1;31mactual\x1B[0m / +\x1B[1;32mexpected\x1B[0m): + 1 + 2 + \x1B[3m<---- 45 common lines omitted ---->\x1B[0m + 48 + 49 + +\x1B[1;32m50\x1B[0m" + }) + ) + } + + #[test] + fn create_diff_exact_difference_with_inline_color() -> Result<()> { + USE_COLOR.with(|cell| cell.set(true)); + + let actual = indoc!( + "There is a home in Nouvelle Orleans + They say, it is the rising sons + And it has been the ruin of many a po'boy" + ); + + let expected = indoc!( + "There is a house way down in New Orleans + They call the rising sun + And it has been the ruin of many a poor boy" + ); + + verify_that!( + create_diff(actual, expected, Mode::Exact), + eq(indoc! { + " + Difference(-\x1B[1;31mactual\x1B[0m / +\x1B[1;32mexpected\x1B[0m): + -\x1B[31mThere is a ho\x1B[0m\x1B[1;31mm\x1B[0m\x1B[31me in N\x1B[0m\x1B[1;31mouv\x1B[0m\x1B[31me\x1B[0m\x1B[1;31mlle\x1B[0m\x1B[31m Orleans\x1B[0m + +\x1B[32mThere is a ho\x1B[0m\x1B[1;32mus\x1B[0m\x1B[32me \x1B[0m\x1B[1;32mway down \x1B[0m\x1B[32min Ne\x1B[0m\x1B[1;32mw\x1B[0m\x1B[32m Orleans\x1B[0m + -\x1B[31mThey \x1B[0m\x1B[1;31ms\x1B[0m\x1B[31ma\x1B[0m\x1B[1;31my,\x1B[0m\x1B[31m \x1B[0m\x1B[1;31mi\x1B[0m\x1B[31mt\x1B[0m\x1B[1;31m is t\x1B[0m\x1B[31mhe rising s\x1B[0m\x1B[1;31mo\x1B[0m\x1B[31mn\x1B[0m\x1B[1;31ms\x1B[0m + +\x1B[32mThey \x1B[0m\x1B[1;32mc\x1B[0m\x1B[32ma\x1B[0m\x1B[1;32mll\x1B[0m\x1B[32m the rising s\x1B[0m\x1B[1;32mu\x1B[0m\x1B[32mn\x1B[0m + -\x1B[31mAnd it has been the ruin of many a po\x1B[0m\x1B[1;31m'\x1B[0m\x1B[31mboy\x1B[0m + +\x1B[32mAnd it has been the ruin of many a po\x1B[0m\x1B[1;32mor \x1B[0m\x1B[32mboy\x1B[0m" + }) + ) + } + + #[test] + fn create_diff_line_termination_diff() -> Result<()> { + verify_that!( + create_diff("1\n2\n3", "1\n2\n3\n", Mode::Exact), + eq("Actual omits a terminating newline that is present in expected.") + )?; + verify_that!( + create_diff("1\n2\n3\n", "1\n2\n3", Mode::Exact), + eq("Actual includes a terminating newline that is absent from expected.") + )?; + verify_that!( + create_diff("1\n2\n3\n", "1\n2\n3\n", Mode::Exact), + eq("No difference found between debug strings.") + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matcher_support/zipped_iterator.rs b/third_party/crates/googletest-0.14.3/src/matcher_support/zipped_iterator.rs new file mode 100644 index 0000000..d1cbaf3 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matcher_support/zipped_iterator.rs @@ -0,0 +1,85 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Zips up two iterators into a single iterator of pairs. +/// +/// This is identical to [`Iterator::zip`] except that this version allows the +/// caller to determine whether the two iterators had mismatching sizes using +/// the method [`ZippedIterator::has_size_mismatch`]. +/// +/// [`Iterator::zip`]: https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.zip +pub(crate) fn zip(left: I1, right: I2) -> ZippedIterator { + ZippedIterator { left, right, has_size_mismatch: false, consumed_elements: 0 } +} + +/// An iterator over pairs of the elements of two constituent iterators, which +/// keeps track of whether the two iterators have the same size. +/// +/// This is identical to [`Zip`] except that it allows the caller to determine +/// whether the two iterators had mismatching sizes using the method +/// [`ZippedIterator::has_size_mismatch`]. +/// +/// [`Zip`]: https://doc.rust-lang.org/std/iter/struct.Zip.html +pub(crate) struct ZippedIterator { + left: I1, + right: I2, + has_size_mismatch: bool, + consumed_elements: usize, +} + +impl ZippedIterator { + /// Returns whether a mismatch in the two sizes of the two iterators was + /// detected during iteration. + /// + /// This returns `true` if and only if, at some previous call to + /// [`Iterator::next`] on this instance, one of the constituent iterators + /// had a next element and the other did not. + /// + /// [`Iterator::next`]: https://doc.rust-lang.org/std/iter/trait.Iterator.html#tymethod.next + pub(crate) fn has_size_mismatch(&self) -> bool { + self.has_size_mismatch + } + + /// Returns the number of elements in the left iterator. + /// + /// This iterates through the remainder of the left iterator if necessary in + /// order to get the true number of elements. It therefore consumes `self`. + pub(crate) fn left_size(mut self) -> usize { + self.consumed_elements + self.left.by_ref().count() + } +} + +impl Iterator for ZippedIterator { + type Item = (I1::Item, I2::Item); + + fn next(&mut self) -> Option<(I1::Item, I2::Item)> { + match (self.left.next(), self.right.next()) { + (Some(v1), Some(v2)) => { + self.consumed_elements += 1; + Some((v1, v2)) + } + (Some(_), None) => { + // Consumed elements counts only elements from self.left + self.consumed_elements += 1; + self.has_size_mismatch = true; + None + } + (None, Some(_)) => { + self.has_size_mismatch = true; + None + } + (None, None) => None, + } + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/all_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/all_matcher.rs new file mode 100644 index 0000000..111f38c --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/all_matcher.rs @@ -0,0 +1,148 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented in the matcher module. +#![doc(hidden)] + +/// Matches a value which all of the given matchers match. +/// +/// Each argument is a [`Matcher`][crate::matcher::Matcher] which matches +/// against the actual value. +/// +/// For example: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!("A string", all!(starts_with("A"), ends_with("string")))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!("A string", all!(starts_with("A"), ends_with("not a string")))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// Using this macro is equivalent to using the +/// [`and`][crate::matcher::MatcherBase::and] method: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(10, gt(9).and(lt(11)))?; // Also passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// Assertion failure messages are not guaranteed to be identical, however. +/// +/// If an inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// +/// verify_that!(123, all![123, lt(1000), gt(100)]) +/// # .unwrap(); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! __all { + ($(,)?) => {{ + $crate::matchers::anything() + }} ; + ($matcher:expr $(,)?) => {{ + use $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq; + auto_eq!($matcher) + }}; + ($head:expr, $head2:expr $(,)?) => {{ + use $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq; + $crate::matchers::__internal_unstable_do_not_depend_on_these::ConjunctionMatcher::new(auto_eq!($head), auto_eq!($head2)) + }}; + ($head:expr, $head2:expr, $($tail:expr),+ $(,)?) => {{ + use $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq; + $crate::__all![ + $crate::matchers::__internal_unstable_do_not_depend_on_these::ConjunctionMatcher::new(auto_eq!($head), auto_eq!($head2)), + $($tail),+ + ] + }} +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn description_shows_more_than_one_matcher() -> Result<()> { + let first_matcher = starts_with("A"); + let second_matcher = ends_with("string"); + let matcher = all!(first_matcher, second_matcher); + + verify_that!( + Matcher::<&String>::describe(&matcher, MatcherResult::Match), + displays_as(eq(indoc!( + " + has all the following properties: + * starts with prefix \"A\" + * ends with suffix \"string\"" + ))) + ) + } + + #[test] + fn description_shows_one_matcher_directly() -> Result<()> { + let first_matcher = starts_with("A"); + let matcher = all!(first_matcher); + + verify_that!( + Matcher::<&String>::describe(&matcher, MatcherResult::Match), + displays_as(eq("starts with prefix \"A\"")) + ) + } + + #[test] + fn mismatch_description_shows_which_matcher_failed_if_more_than_one_constituent() -> Result<()> + { + let first_matcher = starts_with("Another"); + let second_matcher = ends_with("string"); + let matcher = all!(first_matcher, second_matcher); + + verify_that!( + matcher.explain_match("A string"), + displays_as(eq("which does not start with \"Another\"")) + ) + } + + #[test] + fn mismatch_description_is_simple_when_only_one_consistuent() -> Result<()> { + let first_matcher = starts_with("Another"); + let matcher = all!(first_matcher); + + verify_that!( + matcher.explain_match("A string"), + displays_as(eq("which does not start with \"Another\"")) + ) + } + + #[test] + fn all_with_auto_eq() -> Result<()> { + verify_that!(42, all![eq(42), 42, lt(100)]) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/any_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/any_matcher.rs new file mode 100644 index 0000000..5db399f --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/any_matcher.rs @@ -0,0 +1,155 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented in the matchers module. +#![doc(hidden)] + +/// Matches a value which at least one of the given matchers match. +/// +/// Each argument is a [`Matcher`][crate::matcher::Matcher] which matches +/// against the actual value. +/// +/// For example: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!("A string", any!(starts_with("A"), ends_with("string")))?; // Passes +/// verify_that!("A string", any!(starts_with("A"), starts_with("string")))?; // Passes +/// verify_that!("A string", any!(ends_with("A"), ends_with("string")))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!("A string", any!(starts_with("An"), ends_with("not a string")))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// Using this macro is equivalent to using the +/// [`or`][crate::matcher::MatcherBase::or] method: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(10, gt(9).or(lt(8)))?; // Also passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// Assertion failure messages are not guaranteed to be identical, however. +/// +/// If an inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// +/// verify_that!(123, any![lt(1), 123, gt(1000)]) +/// # .unwrap(); +/// ``` +#[macro_export] +#[doc(hidden)] +macro_rules! __any { + ($(,)?) => {{ + $crate::matchers::not($crate::matchers::anything()) + }} ; + ($matcher:expr $(,)?) => {{ + use $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq; + auto_eq!($matcher) + }}; + ($head:expr, $head2:expr $(,)?) => {{ + use $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq; + $crate::matchers::__internal_unstable_do_not_depend_on_these::DisjunctionMatcher::new(auto_eq!($head), auto_eq!($head2)) + }}; + ($head:expr, $head2:expr, $($tail:expr),+ $(,)?) => {{ + use $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq; + $crate::__any![ + $crate::matchers::__internal_unstable_do_not_depend_on_these::DisjunctionMatcher::new(auto_eq!($head), auto_eq!($head2)), + $($tail),+ + ] + }} +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn description_shows_more_than_one_matcher() -> Result<()> { + let first_matcher = starts_with("A"); + let second_matcher = ends_with("string"); + let matcher = any!(first_matcher, second_matcher); + + verify_that!( + Matcher::<&String>::describe(&matcher, MatcherResult::Match), + displays_as(eq(indoc!( + " + has at least one of the following properties: + * starts with prefix \"A\" + * ends with suffix \"string\"" + ))) + ) + } + + #[test] + fn description_shows_one_matcher_directly() -> Result<()> { + let first_matcher = starts_with("A"); + let matcher = any!(first_matcher); + + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("starts with prefix \"A\"")) + ) + } + + #[test] + fn mismatch_description_shows_which_matcher_failed_if_more_than_one_constituent() -> Result<()> + { + let first_matcher = starts_with("Another"); + let second_matcher = ends_with("string"); + let matcher = any!(first_matcher, second_matcher); + + verify_that!( + matcher.explain_match("A string"), + displays_as(eq("which does not start with \"Another\"")) + ) + } + + #[test] + fn mismatch_description_is_simple_when_only_one_constituent() -> Result<()> { + let first_matcher = starts_with("Another"); + let matcher = any!(first_matcher); + + verify_that!( + matcher.explain_match("A string"), + displays_as(eq("which does not start with \"Another\"")) + ) + } + + #[test] + fn empty_any_matcher_never_matches() -> Result<()> { + verify_that!(123, not(any![])) + } + + #[test] + fn any_with_auto_eq() -> Result<()> { + verify_that!(42, any![1, 2, 42, gt(123)]) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/anything_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/anything_matcher.rs new file mode 100644 index 0000000..d37bb68 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/anything_matcher.rs @@ -0,0 +1,80 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches anything. This matcher always succeeds. +/// +/// This is useful to check if `actual` matches the specific structure (like +/// `Some(...)`) but without caring about the internal value. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let option = Some("Some value"); +/// verify_that!(option, some(anything()))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +pub fn anything() -> Anything { + Anything +} + +#[derive(MatcherBase)] +pub struct Anything; + +impl Matcher for Anything { + fn matches(&self, _: T) -> MatcherResult { + MatcherResult::Match + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => "is anything".into(), + MatcherResult::NoMatch => "never matches".into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + + #[test] + fn anything_matches_i32() -> Result<()> { + let value = 32; + verify_that!(value, anything())?; + Ok(()) + } + + #[test] + fn anything_matches_str() -> Result<()> { + let value = "32"; + verify_that!(value, anything())?; + Ok(()) + } + + #[test] + fn anything_matches_option() -> Result<()> { + let value = Some(32); + verify_that!(value, some(anything()))?; + Ok(()) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/bool_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/bool_matcher.rs new file mode 100644 index 0000000..113d677 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/bool_matcher.rs @@ -0,0 +1,100 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; + +/// Matches boolean value `true`. +pub fn is_true() -> BoolMatcher { + BoolMatcher { expected: true } +} + +/// Matches boolean value `false`. +pub fn is_false() -> BoolMatcher { + BoolMatcher { expected: false } +} + +/// Matches a bool value or bool reference. +#[derive(MatcherBase)] +pub struct BoolMatcher { + expected: bool, +} + +impl BoolMatcher { + fn matches(&self, actual: bool) -> MatcherResult { + (actual == self.expected).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match (matcher_result, self.expected) { + (MatcherResult::Match, true) | (MatcherResult::NoMatch, false) => "is true".into(), + (MatcherResult::Match, false) | (MatcherResult::NoMatch, true) => "is false".into(), + } + } +} + +impl Matcher for BoolMatcher { + fn matches(&self, actual: bool) -> MatcherResult { + self.matches(actual) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + self.describe(matcher_result) + } +} + +impl<'a> Matcher<&'a bool> for BoolMatcher { + fn matches(&self, actual: &'a bool) -> MatcherResult { + self.matches(*actual) + } + fn describe(&self, matcher_result: MatcherResult) -> Description { + self.describe(matcher_result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::*; + use crate::Result; + + #[test] + fn match_value() -> Result<()> { + verify_that!(true, is_true())?; + verify_that!(true, not(is_false()))?; + verify_that!(false, is_false())?; + verify_that!(false, not(is_true())) + } + + #[test] + fn match_ref() -> Result<()> { + let t = true; + let f = false; + + verify_that!(&t, is_true())?; + verify_that!(&t, not(is_false()))?; + verify_that!(&f, is_false())?; + verify_that!(&f, not(is_true())) + } + + #[test] + fn describe() { + assert_eq!(is_true().describe(MatcherResult::Match).to_string(), "is true"); + assert_eq!(is_true().describe(MatcherResult::NoMatch).to_string(), "is false"); + assert_eq!(is_false().describe(MatcherResult::Match).to_string(), "is false"); + assert_eq!(is_false().describe(MatcherResult::NoMatch).to_string(), "is true"); + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/char_count_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/char_count_matcher.rs new file mode 100644 index 0000000..0492edb --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/char_count_matcher.rs @@ -0,0 +1,164 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a string whose number of Unicode scalars matches `expected`. +/// +/// In other words, the argument must match the output of +/// [`actual_string.chars().count()`][std::str::Chars]. +/// +/// This can have surprising effects when what appears to be a single character +/// is composed of multiple Unicode scalars. See [Rust documentation on +/// character +/// representation](https://doc.rust-lang.org/std/primitive.char.html#representation) +/// for more information. +/// +/// This matches against owned strings and string slices. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let string_slice = "A string"; +/// verify_that!(string_slice, char_count(eq(8)))?; +/// let non_ascii_string_slice = "Ä ſtřiɲğ"; +/// verify_that!(non_ascii_string_slice, char_count(eq(8)))?; +/// let owned_string = String::from("A string"); +/// verify_that!(owned_string, char_count(eq(8)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// The parameter `expected` can be any integer numeric matcher. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let string_slice = "A string"; +/// verify_that!(string_slice, char_count(gt(4)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +pub fn char_count>(expected: E) -> CharLenMatcher { + CharLenMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct CharLenMatcher { + expected: E, +} + +impl, E: Matcher> Matcher for CharLenMatcher { + fn matches(&self, actual: T) -> MatcherResult { + self.expected.matches(actual.as_ref().chars().count()) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!( + "has character count, which {}", + self.expected.describe(MatcherResult::Match) + ) + .into(), + MatcherResult::NoMatch => format!( + "has character count, which {}", + self.expected.describe(MatcherResult::NoMatch) + ) + .into(), + } + } + + fn explain_match(&self, actual: T) -> Description { + let actual_size = actual.as_ref().chars().count(); + format!( + "which has character count {}, {}", + actual_size, + self.expected.explain_match(actual_size) + ) + .into() + } +} + +#[cfg(test)] +mod tests { + use crate::description::Description; + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::fmt::Debug; + + #[test] + fn char_count_matches_string_slice() -> Result<()> { + let value = "abcd"; + verify_that!(value, char_count(eq(4))) + } + + #[test] + fn char_count_matches_owned_string() -> Result<()> { + let value = String::from("abcd"); + verify_that!(value, char_count(eq(4))) + } + + #[test] + fn char_count_counts_non_ascii_characters_correctly() -> Result<()> { + let value = "äöüß"; + verify_that!(value, char_count(eq(4))) + } + + #[test] + fn char_count_explains_match() -> Result<()> { + #[derive(MatcherBase)] + struct TestMatcher; + + impl Matcher for TestMatcher { + fn matches(&self, _: T) -> MatcherResult { + false.into() + } + + fn describe(&self, _: MatcherResult) -> Description { + "called described".into() + } + + fn explain_match(&self, _: T) -> Description { + "called explain_match".into() + } + } + verify_that!( + char_count(TestMatcher).explain_match("A string"), + displays_as(eq("which has character count 8, called explain_match")) + ) + } + + #[test] + fn char_count_has_correct_failure_message() -> Result<()> { + let result = verify_that!("äöüß", char_count(eq(3))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: "äöüß" + Expected: has character count, which is equal to 3 + Actual: "äöüß", + which has character count 4, which isn't equal to 3"# + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/conjunction_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/conjunction_matcher.rs new file mode 100644 index 0000000..5b2dac5 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/conjunction_matcher.rs @@ -0,0 +1,218 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module. +#![doc(hidden)] + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matcher created by [`Matcher::and`] and [`all!`]. +/// +/// Both [`Matcher::and`] and [`all!`] nest on m1. In other words, +/// both `x.and(y).and(z)` and `all![x, y, z]` produce: +/// ```ignore +/// ConjunctionMatcher { +/// m1: ConjunctionMatcher { +/// m1: x, +/// m2: y +/// }, +/// m2: z +/// } +/// ``` +/// +/// This behavior must be respected +/// to ensure that [`Matcher::explain_match`] and [`Matcher::describe`] produce +/// useful descriptions. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +#[derive(MatcherBase)] +pub struct ConjunctionMatcher { + m1: M1, + m2: M2, +} + +impl ConjunctionMatcher { + pub fn new(m1: M1, m2: M2) -> Self { + Self { m1, m2 } + } +} + +impl, M2: Matcher> Matcher for ConjunctionMatcher { + fn matches(&self, actual: T) -> MatcherResult { + match (self.m1.matches(actual), self.m2.matches(actual)) { + (MatcherResult::Match, MatcherResult::Match) => MatcherResult::Match, + _ => MatcherResult::NoMatch, + } + } + + fn explain_match(&self, actual: T) -> Description { + match (self.m1.matches(actual), self.m2.matches(actual)) { + (MatcherResult::NoMatch, MatcherResult::Match) => self.m1.explain_match(actual), + (MatcherResult::Match, MatcherResult::NoMatch) => self.m2.explain_match(actual), + (_, _) => { + let m1_description = self.m1.explain_match(actual); + if m1_description.is_conjunction_description() { + m1_description.nested(self.m2.explain_match(actual)) + } else { + Description::new() + .bullet_list() + .collect([m1_description, self.m2.explain_match(actual)]) + .conjunction_description() + } + } + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + let m1_description = self.m1.describe(matcher_result); + if m1_description.is_conjunction_description() { + m1_description.push_in_last_nested(self.m2.describe(matcher_result)) + } else { + let header = if matcher_result.into() { + "has all the following properties:" + } else { + "has at least one of the following properties:" + }; + Description::new() + .text(header) + .nested( + Description::new() + .bullet_list() + .collect([m1_description, self.m2.describe(matcher_result)]), + ) + .conjunction_description() + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn and_true_true_matches() -> Result<()> { + verify_that!(1, anything().and(anything())) + } + + #[test] + fn and_true_false_does_not_match() -> Result<()> { + let result = verify_that!(1, anything().and(not(anything()))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 1 + Expected: has all the following properties: + * is anything + * never matches + Actual: 1, + which is anything + " + )))) + ) + } + + #[test] + fn and_false_true_does_not_match() -> Result<()> { + let result = verify_that!(1, not(anything()).and(anything())); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 1 + Expected: has all the following properties: + * never matches + * is anything + Actual: 1, + which is anything + " + )))) + ) + } + + #[test] + fn and_false_false_does_not_match() -> Result<()> { + let result = verify_that!(1, not(anything()).and(not(anything()))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 1 + Expected: has all the following properties: + * never matches + * never matches + Actual: 1, + * which is anything + * which is anything + " + )))) + ) + } + + #[test] + fn and_long_chain_of_matchers() -> Result<()> { + let result = verify_that!( + 1, + anything().and(not(anything())).and(anything()).and(not(anything())).and(anything()) + ); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 1 + Expected: has all the following properties: + * is anything + * never matches + * is anything + * never matches + * is anything + Actual: 1, + * which is anything + * which is anything + " + )))) + ) + } + + #[test] + fn chained_and_matches() -> Result<()> { + #[derive(Debug, Clone, Copy)] + struct Struct { + a: i32, + b: i32, + c: i32, + } + verify_that!( + Struct { a: 1, b: 2, c: 3 }, + field!(Struct.a, eq(1)).and(field!(Struct.b, eq(2))).and(field!(Struct.c, eq(3))) + ) + } + + #[test] + fn works_with_str_slices() -> Result<()> { + verify_that!("A string", starts_with("A").and(ends_with("string"))) + } + + #[test] + fn works_with_owned_strings() -> Result<()> { + verify_that!("A string".to_string(), starts_with("A").and(ends_with("string"))) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/container_eq_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/container_eq_matcher.rs new file mode 100644 index 0000000..4829f95 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/container_eq_matcher.rs @@ -0,0 +1,517 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use crate::matcher_support::match_matrix::internal::{MatchMatrix, Requirements}; +use crate::matchers::eq_matcher::eq; +use std::fmt::Debug; + +/// Matches a container equal (in the sense of `==`) to `expected`. +/// +/// This is similar to [`crate::matchers::eq`] except that an assertion failure +/// message generated from this matcher will include the missing and unexpected +/// items in the actual value, e.g.: +/// +/// ```text +/// Expected container to equal [1, 2, 3] +/// but was: [1, 2, 4] +/// Missing: [3] +/// Unexpected: [4] +/// ``` +/// +/// The actual value must be a container such as a `&Vec`, an array, or a +/// dereferenced slice. More precisely, the actual value must +/// implement [`IntoIterator`] whose `Item` type implements +/// [`PartialEq`], where `ExpectedT` is the element type of the +/// expected value. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let vec = vec![1, 2, 3]; +/// verify_that!(vec, container_eq([1, 2, 3]))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// *Performance note*: In the event of a mismatch leading to an assertion +/// failure, the construction of the lists of missing and unexpected values +/// uses a naive algorithm requiring time proportional to the product of the +/// sizes of the expected and actual values. This should therefore only be used +/// when the containers are small enough that this is not a problem. +pub fn container_eq( + expected: ExpectedContainerT, +) -> ContainerEqMatcher +where + ExpectedContainerT: Debug, +{ + ContainerEqMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct ContainerEqMatcher { + expected: ExpectedContainerT, +} + +impl + Matcher for ContainerEqMatcher +where + ActualElementT: for<'a> PartialEq<&'a ExpectedElementT> + Debug + Copy, + ActualContainerT: for<'a> PartialEq<&'a ExpectedContainerT> + Debug + Copy, + ExpectedElementT: Debug, + ExpectedContainerT: Debug, + ActualContainerT: IntoIterator, + for<'a> &'a ExpectedContainerT: IntoIterator, +{ + fn matches(&self, actual: ActualContainerT) -> MatcherResult { + (actual == &self.expected).into() + } + + fn explain_match(&self, actual: ActualContainerT) -> Description { + build_explanation(self.get_missing_items(actual), self.get_unexpected_items(actual)).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!("is equal to {:?}", self.expected).into(), + MatcherResult::NoMatch => format!("isn't equal to {:?}", self.expected).into(), + } + } +} + +impl ContainerEqMatcher +where + for<'a> &'a ExpectedContainerT: IntoIterator, +{ + fn get_missing_items( + &self, + actual: ActualContainerT, + ) -> Vec<&'_ ExpectedElementT> + where + ActualElementT: for<'a> PartialEq<&'a ExpectedElementT> + Copy, + ActualContainerT: for<'a> PartialEq<&'a ExpectedContainerT> + Copy, + ActualContainerT: IntoIterator, + { + self.expected.into_iter().filter(|i| !actual.into_iter().any(|j| j == *i)).collect() + } + + fn get_unexpected_items( + &self, + actual: ActualContainerT, + ) -> Vec + where + ActualElementT: for<'a> PartialEq<&'a ExpectedElementT> + Copy, + ActualContainerT: for<'a> PartialEq<&'a ExpectedContainerT> + Copy, + ActualContainerT: IntoIterator, + { + actual.into_iter().filter(|i| !self.expected.into_iter().any(|j| i == &j)).collect() + } +} + +fn build_explanation(missing: Vec, unexpected: Vec) -> String { + match (missing.len(), unexpected.len()) { + // TODO(b/261175849) add more data here (out of order elements, duplicated elements, etc...) + (0, 0) => "which contains all the elements".to_string(), + (0, 1) => format!("which contains the unexpected element {:?}", unexpected[0]), + (0, _) => format!("which contains the unexpected elements {unexpected:?}",), + (1, 0) => format!("which is missing the element {:?}", missing[0]), + (1, 1) => { + format!( + "which is missing the element {:?} and contains the unexpected element {:?}", + missing[0], unexpected[0] + ) + } + (1, _) => { + format!( + "which is missing the element {:?} and contains the unexpected elements {unexpected:?}", + missing[0] + ) + } + (_, 0) => format!("which is missing the elements {missing:?}"), + (_, 1) => { + format!( + "which is missing the elements {missing:?} and contains the unexpected element {:?}", + unexpected[0] + ) + } + (_, _) => { + format!( + "which is missing the elements {missing:?} and contains the unexpected elements {unexpected:?}", + ) + } + } +} + +impl ContainerEqMatcher { + /// Match container equality, but ignoring element order. + pub fn ignore_order(self) -> IgnoringOrder { + IgnoringOrder { expected: self.expected } + } +} + +#[derive(MatcherBase)] +pub struct IgnoringOrder { + expected: ExpectedContainerT, +} + +/// Implements a matcher that ignores the relative order of the elements. +impl + Matcher for IgnoringOrder +where + ActualElementT: Debug + Copy + for<'a> PartialEq<&'a ExpectedElementT>, + ActualContainerT: Debug + Copy + IntoIterator, + ExpectedElementT: Debug, + for<'a> &'a ExpectedContainerT: IntoIterator, +{ + fn matches(&self, actual: ActualContainerT) -> MatcherResult { + let expected: Vec>> = self + .expected + .into_iter() + .map(|x| Box::new(eq(x)) as Box>) + .collect(); + let match_matrix = MatchMatrix::generate(actual, &expected); + // TODO: investigate why Requirements::PerfectMatch and + // match_matrix.is_full_match is not doing what we expect + // here. + (match_matrix.is_match_for(Requirements::Subset) + && match_matrix.is_match_for(Requirements::Superset)) + .into() + } + + fn explain_match(&self, actual: ActualContainerT) -> Description { + // We need to materialize the collections in order to have reliable iteration + // order when generating our reports. + let expected_items: Vec<&ExpectedElementT> = self.expected.into_iter().collect(); + let actual_items: Vec = actual.into_iter().collect(); + + let expected_matchers: Vec<_> = expected_items + .iter() + .map(|&x| Box::new(eq(x)) as Box>) + .collect(); + let match_matrix = MatchMatrix::generate(actual_items.iter().copied(), &expected_matchers); + + let best_match = match_matrix.find_best_match(); + + // Since we are doing equality checks, we can generate a slightly less verbose + // message than BestMatch::get_explanation. + let matches = best_match.get_matches().map(|(actual_idx, expected_idx)|{ + Description::new().text( + format!( + "Actual element {:?} at index {actual_idx} is equal to expected element at index {expected_idx}.", + actual_items[actual_idx], + ))}); + + let unmatched_actual = best_match.get_unmatched_actual().map(|actual_idx| { + Description::new().text( + format!( + "Actual element {:?} at index {actual_idx} did not match any remaining expected element.", + actual_items[actual_idx], + )) + }); + + let unmatched_expected = + best_match.get_unmatched_expected().into_iter().map(|expected_idx| { + Description::new().text(format!( + "Expected element {:?} at index {expected_idx} did not match any remaining actual element.", + expected_items[expected_idx] + )) + }); + + Description::new() + .text("which does not have a perfect match. The best match found was:") + .collect(matches.chain(unmatched_actual).chain(unmatched_expected)) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + Description::new() + .text(format!( + "{} all elements matching in any order:", + if matcher_result.into() { "contains" } else { "doesn't contain" }, + )) + .nested( + self.expected + .into_iter() + .map(|element| format!("{element:?}")) + .collect::() + .bullet_list(), + ) + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::collections::HashSet; + + #[test] + fn container_eq_returns_match_when_containers_match() -> Result<()> { + verify_that!(vec![1, 2, 3], container_eq(vec![1, 2, 3])) + } + + #[test] + fn container_eq_matches_array_with_slice() -> Result<()> { + let value = &[1, 2, 3]; + verify_that!(value, container_eq([1, 2, 3])) + } + + #[test] + fn container_eq_matches_hash_set() -> Result<()> { + let value: HashSet = [1, 2, 3].into(); + verify_that!(value, container_eq([1, 2, 3].into())) + } + + #[test] + fn container_eq_full_error_message() -> Result<()> { + let result = verify_that!(vec![1, 3, 2], container_eq(vec![1, 2, 3])); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 3, 2] + Expected: is equal to [1, 2, 3] + Actual: [1, 3, 2], + which contains all the elements + " + )))) + ) + } + + #[test] + fn container_eq_returns_mismatch_when_elements_out_of_order() -> Result<()> { + verify_that!( + container_eq(vec![1, 2, 3]).explain_match(&vec![1, 3, 2]), + displays_as(eq("which contains all the elements")) + ) + } + + #[test] + fn container_eq_mismatch_shows_missing_elements_in_container() -> Result<()> { + verify_that!( + container_eq(vec![1, 2, 3]).explain_match(&vec![1, 2]), + displays_as(eq("which is missing the element 3")) + ) + } + + #[test] + fn container_eq_mismatch_shows_surplus_elements_in_container() -> Result<()> { + verify_that!( + container_eq(vec![1, 2]).explain_match(&vec![1, 2, 3]), + displays_as(eq("which contains the unexpected element 3")) + ) + } + + #[test] + fn container_eq_mismatch_shows_missing_and_surplus_elements_in_container() -> Result<()> { + verify_that!( + container_eq(vec![1, 2, 3]).explain_match(&vec![1, 2, 4]), + displays_as(eq("which is missing the element 3 and contains the unexpected element 4")) + ) + } + + #[test] + fn container_eq_mismatch_does_not_show_duplicated_element() -> Result<()> { + verify_that!( + container_eq(vec![1, 2, 3]).explain_match(&vec![1, 2, 3, 3]), + displays_as(eq("which contains all the elements")) + ) + } + + #[test] + fn container_eq_matches_owned_vec_with_array() -> Result<()> { + let vector = vec![123, 234]; + verify_that!(vector, container_eq([123, 234])) + } + + #[test] + fn container_eq_matches_owned_vec_of_owned_strings_with_slice_of_string_references( + ) -> Result<()> { + let vector = vec!["A string".to_string(), "Another string".to_string()]; + verify_that!(vector, container_eq(["A string", "Another string"])) + } + + #[test] + fn container_eq_matches_owned_vec_of_owned_strings_with_shorter_slice_of_string_references( + ) -> Result<()> { + let actual = vec!["A string".to_string(), "Another string".to_string()]; + let matcher = container_eq(["A string"]); + + let result = matcher.matches(&actual); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn container_eq_mismatch_with_slice_shows_missing_elements_in_container() -> Result<()> { + verify_that!( + container_eq([1, 2, 3]).explain_match(&vec![1, 2]), + displays_as(eq("which is missing the element 3")) + ) + } + + #[test] + fn container_eq_mismatch_with_str_slice_shows_missing_elements_in_container() -> Result<()> { + verify_that!( + container_eq(["A", "B", "C"]).explain_match(&vec!["A".to_string(), "B".to_string()]), + displays_as(eq("which is missing the element \"C\"")) + ) + } + + #[test] + fn container_eq_mismatch_with_str_slice_shows_surplus_elements_in_container() -> Result<()> { + verify_that!( + container_eq(["A", "B"]).explain_match(&vec![ + "A".to_string(), + "B".to_string(), + "C".to_string() + ]), + displays_as(eq("which contains the unexpected element \"C\"")) + ) + } + + #[test] + fn ignoring_order_match() -> Result<()> { + verify_that!(vec!["a", "b"], container_eq(["b", "a"]).ignore_order()) + } + + #[test] + fn ignoring_order_mismatch() -> Result<()> { + verify_that!(vec!["a", "b"], not(container_eq(["1", "2"]).ignore_order())) + } + + #[test] + fn ignoring_order_mismatch_explain() -> Result<()> { + let expected_err = verify_that!(vec!["a", "b"], container_eq(["1", "2"]).ignore_order()); + verify_that!( + expected_err, + err(displays_as(contains_substring(indoc!( + r#" + Value of: vec!["a", "b"] + Expected: contains all elements matching in any order: + * "1" + * "2" + Actual: ["a", "b"], + which does not have a perfect match. The best match found was: + Actual element "a" at index 0 did not match any remaining expected element. + Actual element "b" at index 1 did not match any remaining expected element. + Expected element "1" at index 0 did not match any remaining actual element. + Expected element "2" at index 1 did not match any remaining actual element. + "# + )))) + ) + } + + #[test] + fn ignoring_order_unaccounted_extra_expected() -> Result<()> { + verify_that!(vec!["a", "b"], not(container_eq(["a", "b", "a"]).ignore_order())) + } + + #[test] + fn ignoring_order_unaccounted_extra_expected_explain() -> Result<()> { + let expected_err = + verify_that!(vec!["a", "b"], container_eq(["a", "b", "a"]).ignore_order()); + verify_that!( + expected_err, + err(displays_as(contains_substring(indoc!( + r#" + Value of: vec!["a", "b"] + Expected: contains all elements matching in any order: + * "a" + * "b" + * "a" + Actual: ["a", "b"], + which does not have a perfect match. The best match found was: + Actual element "a" at index 0 is equal to expected element at index 0. + Actual element "b" at index 1 is equal to expected element at index 1. + Expected element "a" at index 2 did not match any remaining actual element. + "# + )))) + ) + } + + #[test] + fn ignoring_order_unaccounted_extra_actual() -> Result<()> { + verify_that!(vec!["a", "b", "a"], not(container_eq(["b", "a"]).ignore_order())) + } + + #[test] + fn ignoring_order_unaccounted_extra_actual_explain() -> Result<()> { + let expected_err = + verify_that!(vec!["a", "b", "a"], container_eq(["b", "a"]).ignore_order()); + + verify_that!( + expected_err, + err(displays_as(contains_substring(indoc!( + r#" + Value of: vec!["a", "b", "a"] + Expected: contains all elements matching in any order: + * "b" + * "a" + Actual: ["a", "b", "a"], + which does not have a perfect match. The best match found was: + Actual element "a" at index 0 is equal to expected element at index 1. + Actual element "b" at index 1 is equal to expected element at index 0. + Actual element "a" at index 2 did not match any remaining expected element. + "# + )))) + ) + } + + #[test] + fn ignoring_order_on_sets() -> Result<()> { + let mut actual = std::collections::HashSet::new(); + actual.insert("b"); + actual.insert("a"); + actual.insert("c"); + verify_that!(actual, container_eq(["c", "b", "a"]).ignore_order()) + } + + #[test] + fn ignoring_order_on_sets_explain() -> Result<()> { + let mut actual = std::collections::HashSet::new(); + actual.insert("b"); + actual.insert("a"); + actual.insert("c"); + let expected_err = verify_that!(actual, container_eq(["c", "a"]).ignore_order()); + verify_that!( + expected_err, + err(displays_as(contains_regex(indoc!( + r#" + Value of: actual + Expected: contains all elements matching in any order: + \* "c" + \* "a" + Actual: \{"\w", "\w", "\w"\}, + which does not have a perfect match. The best match found was: + Actual element "\w" at index \d is equal to expected element at index \d\. + Actual element "\w" at index \d is equal to expected element at index \d\. + Actual element "\w" at index \d did not match any remaining expected element\. + "# + )))) + ) + } + + #[test] + fn ignoring_order_on_number_sets() -> Result<()> { + let mut actual = std::collections::HashSet::new(); + actual.insert(1); + actual.insert(2); + actual.insert(3); + verify_that!(actual, container_eq([3, 2, 1]).ignore_order()) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/contains_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/contains_matcher.rs new file mode 100644 index 0000000..b4ebb1a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/contains_matcher.rs @@ -0,0 +1,279 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches an [`IntoIterator`] type whose elements contain a value matched by +/// `inner`. +/// +/// By default, this matches a container with any number of elements matched +/// by `inner`. Use the method [`ContainsMatcher::times`] to constrain the +/// matched containers to a specific number of matching elements. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(["Some value"], contains(eq("Some value")))?; // Passes +/// verify_that!(vec!["Some value"], contains(eq(&"Some value")))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!([] as [&String; 0], contains(eq("Some value")))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!(["Some value"], contains(eq("Some other value")))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// ``` +pub fn contains(inner: InnerMatcherT) -> ContainsMatcher { + ContainsMatcher { inner, count: None } +} + +/// A matcher which matches a container containing one or more elements a given +/// inner [`Matcher`] matches. +#[derive(MatcherBase)] +pub struct ContainsMatcher { + inner: InnerMatcherT, + count: Option>>, +} + +impl ContainsMatcher { + /// Configures this instance to match containers which contain a number of + /// matching items matched by `count`. + /// + /// For example, to assert that exactly three matching items must be + /// present, use: + /// + /// ```ignore + /// contains(...).times(eq(3)) + /// ``` + /// + /// One can also use `times(eq(0))` to test for the *absence* of an item + /// matching the expected value. + pub fn times(mut self, count: impl Matcher + 'static) -> Self { + self.count = Some(Box::new(count)); + self + } +} + +// TODO(hovinen): Revisit the trait bounds to see whether this can be made more +// flexible. Namely, the following doesn't compile currently: +// +// let matcher = contains(eq(&42)); +// let val = 42; +// let _ = matcher.matches(&vec![&val]); +// +// because val is dropped before matcher but the trait bound requires that +// the argument to matches outlive the matcher. It works fine if one defines +// val before matcher. +impl, ContainerT: Debug + Copy> Matcher + for ContainsMatcher +where + ContainerT: IntoIterator, +{ + fn matches(&self, actual: ContainerT) -> MatcherResult { + if let Some(count) = &self.count { + count.matches(self.count_matches(actual)) + } else { + for v in actual.into_iter() { + if self.inner.matches(v).into() { + return MatcherResult::Match; + } + } + MatcherResult::NoMatch + } + } + + fn explain_match(&self, actual: ContainerT) -> Description { + let count = self.count_matches(actual); + match (count, &self.count) { + (_, Some(_)) => format!("which contains {count} matching elements").into(), + (0, None) => "which does not contain a matching element".into(), + (_, None) => "which contains a matching element".into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match (matcher_result, &self.count) { + (MatcherResult::Match, Some(count)) => format!( + "contains n elements which {}\n where n {}", + self.inner.describe(MatcherResult::Match), + count.describe(MatcherResult::Match) + ) + .into(), + (MatcherResult::NoMatch, Some(count)) => format!( + "doesn't contain n elements which {}\n where n {}", + self.inner.describe(MatcherResult::Match), + count.describe(MatcherResult::Match) + ) + .into(), + (MatcherResult::Match, None) => format!( + "contains at least one element which {}", + self.inner.describe(MatcherResult::Match) + ) + .into(), + (MatcherResult::NoMatch, None) => { + format!("contains no element which {}", self.inner.describe(MatcherResult::Match)) + .into() + } + } + } +} + +impl ContainsMatcher { + fn count_matches(&self, actual: ContainerT) -> usize + where + ContainerT: IntoIterator, + InnerMatcherT: Matcher, + { + let mut count = 0; + for v in actual.into_iter() { + if self.inner.matches(v).into() { + count += 1; + } + } + count + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + + #[test] + fn contains_matches_singleton_slice_with_value() -> Result<()> { + let matcher = contains(eq(&1)); + + let result = matcher.matches(&vec![1]); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn contains_matches_singleton_vec_with_value() -> Result<()> { + let matcher = contains(eq(&1)); + + let result = matcher.matches(&vec![1]); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn contains_matches_two_element_slice_with_value() -> Result<()> { + let matcher = contains(eq(&1)); + + let result = matcher.matches(&[0, 1]); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn contains_does_not_match_singleton_slice_with_wrong_value() -> Result<()> { + let matcher = contains(eq(&1)); + + let result = matcher.matches(&[0]); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn contains_does_not_match_empty_slice() -> Result<()> { + let matcher = contains(eq(&1)); + + let result = matcher.matches(&[1; 0]); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn contains_matches_slice_with_repeated_value() -> Result<()> { + let matcher = contains(eq(&1)).times(eq(2)); + + let result = matcher.matches(&[1, 1]); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn contains_does_not_match_slice_with_too_few_of_value() -> Result<()> { + let matcher = contains(eq(&1)).times(eq(2)); + + let result = matcher.matches(&[0, 1]); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn contains_does_not_match_slice_with_too_many_of_value() -> Result<()> { + let matcher = contains(eq(&1)).times(eq(1)); + + let result = matcher.matches(&[1, 1]); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn contains_formats_without_multiplicity_by_default() -> Result<()> { + let matcher = contains(eq(&1)); + + verify_that!( + Matcher::<&Vec>::describe(&matcher, MatcherResult::Match), + displays_as(eq("contains at least one element which is equal to 1")) + ) + } + + #[test] + fn contains_formats_with_multiplicity_when_specified() -> Result<()> { + let matcher = contains(eq(&1)).times(eq(2)); + + verify_that!( + Matcher::<&Vec>::describe(&matcher, MatcherResult::Match), + displays_as(eq("contains n elements which is equal to 1\n where n is equal to 2")) + ) + } + + #[test] + fn contains_mismatch_shows_number_of_times_element_was_found() -> Result<()> { + verify_that!( + contains(eq(&3)).times(eq(1)).explain_match(&vec![1, 2, 3, 3]), + displays_as(eq("which contains 2 matching elements")) + ) + } + + #[test] + fn contains_mismatch_shows_when_matches() -> Result<()> { + verify_that!( + contains(eq(&3)).explain_match(&vec![1, 2, 3, 3]), + displays_as(eq("which contains a matching element")) + ) + } + + #[test] + fn contains_mismatch_shows_when_no_matches() -> Result<()> { + verify_that!( + contains(eq(&3)).explain_match(&vec![1, 2]), + displays_as(eq("which does not contain a matching element")) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/contains_regex_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/contains_regex_matcher.rs new file mode 100644 index 0000000..473d94a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/contains_regex_matcher.rs @@ -0,0 +1,137 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use regex::Regex; +use std::fmt::Debug; +use std::ops::Deref; + +/// Matches a string containing a substring which matches the given regular +/// expression. +/// +/// Both the actual value and the expected regular expression may be either a +/// `String` or a string reference. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass_1() -> Result<()> { +/// verify_that!("Some value", contains_regex("S.*e"))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!("Another value", contains_regex("Some"))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_pass_2() -> Result<()> { +/// verify_that!("Some value".to_string(), contains_regex("v.*e"))?; // Passes +/// verify_that!("Some value", contains_regex("v.*e".to_string()))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail().unwrap_err(); +/// # should_pass_2().unwrap(); +/// ``` +/// +/// Panics if the given `pattern` is not a syntactically valid regular +/// expression. +#[track_caller] +pub fn contains_regex>(pattern: PatternT) -> ContainsRegexMatcher { + ContainsRegexMatcher { regex: Regex::new(pattern.deref()).unwrap() } +} + +/// A matcher matching a string-like type containing a substring matching a +/// given regular expression. +/// +/// Intended only to be used from the function [`contains_regex`] only. +/// Should not be referenced by code outside this library. +#[derive(MatcherBase)] +pub struct ContainsRegexMatcher { + regex: Regex, +} + +impl + Debug + Copy> Matcher for ContainsRegexMatcher { + fn matches(&self, actual: ActualT) -> MatcherResult { + self.regex.is_match(actual.as_ref()).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("contains the regular expression {:#?}", self.regex.as_str()).into() + } + MatcherResult::NoMatch => { + format!("doesn't contain the regular expression {:#?}", self.regex.as_str()).into() + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + + #[test] + fn contains_regex_matches_string_reference_with_pattern() -> Result<()> { + let matcher = contains_regex("S.*val"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn contains_regex_does_not_match_string_without_pattern() -> Result<()> { + let matcher = contains_regex("Another"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn contains_regex_matches_owned_string_with_pattern() -> Result<()> { + let matcher = contains_regex("value"); + + let result = matcher.matches(&"Some value".to_string()); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn contains_regex_matches_string_reference_with_owned_string() -> Result<()> { + let matcher = contains_regex("value"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn verify_that_works_with_owned_string() -> Result<()> { + verify_that!("Some value".to_string(), contains_regex("value")) + } + + #[test] + fn contains_regex_displays_quoted_debug_of_pattern() -> Result<()> { + let matcher = contains_regex("\n"); + + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("contains the regular expression \"\\n\"")) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/derefs_to_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/derefs_to_matcher.rs new file mode 100644 index 0000000..103cdb2 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/derefs_to_matcher.rs @@ -0,0 +1,100 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::{fmt::Debug, ops::Deref}; + +/// Dereferences the `actual` value and verifies that the returned reference +/// matches the `inner` matcher. +/// +/// ``` +/// # use googletest::{matchers::{derefs_to, eq}, verify_that}; +/// verify_that!(Box::new(123), derefs_to(eq(&123))) +/// # .unwrap() +/// ``` +pub fn derefs_to(inner: Inner) -> DerefsTo { + DerefsTo { inner } +} + +/// A matcher which derefs a value and verifies that the result matches the +/// `inner` matcher. +/// +/// See [`derefs_to`]. +#[derive(MatcherBase)] +pub struct DerefsTo { + pub(crate) inner: InnerT, +} + +impl<'a, ActualT, ExpectedT, Inner> Matcher<&'a ActualT> for DerefsTo +where + ActualT: Deref + Debug, + ExpectedT: Copy + Debug + 'a, + Inner: Matcher<&'a ExpectedT>, +{ + fn matches(&self, actual: &'a ActualT) -> MatcherResult { + self.inner.matches(actual.deref()) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + self.inner.describe(matcher_result) + } + + fn explain_match(&self, actual: &'a ActualT) -> Description { + self.inner.explain_match(actual.deref()) + } +} + +#[cfg(test)] +mod tests { + use std::rc::Rc; + + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn deref_to_matches_box_of_int_with_int() -> Result<()> { + let actual = Box::new(123); + verify_that!(actual, derefs_to(eq(&123))) + } + + #[test] + fn deref_to_matches_rc_of_int_with_int() -> Result<()> { + verify_that!(Rc::new(123), derefs_to(eq(&123))) + } + + #[test] + fn deref_to_combines_with_points_to_for_copy() -> Result<()> { + verify_that!(Rc::new(123), derefs_to(points_to(eq(123)))) + } + + #[test] + fn match_explanation_references_actual_value() -> Result<()> { + let actual = Box::new(1); + let result = verify_that!(actual, derefs_to(eq(&0))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Actual: 1, + which isn't equal to 0 + " + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/disjunction_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/disjunction_matcher.rs new file mode 100644 index 0000000..ccf8743 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/disjunction_matcher.rs @@ -0,0 +1,152 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module. +#![doc(hidden)] + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matcher created by [`Matcher::or`] and [`any!`]. +/// +/// Both [`Matcher::or`] and [`any!`] nest on m1. In other words, +/// both `x.or(y).or(z)` and `any![x, y, z]` produce: +/// ```ignore +/// DisjunctionMatcher { +/// m1: DisjunctionMatcher { +/// m1: x, m2: y +/// }, +/// m2: z +/// } +/// ``` +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +#[derive(MatcherBase)] +pub struct DisjunctionMatcher { + m1: M1, + m2: M2, +} + +impl DisjunctionMatcher { + pub fn new(m1: M1, m2: M2) -> Self { + Self { m1, m2 } + } +} + +impl, M2: Matcher> Matcher for DisjunctionMatcher { + fn matches(&self, actual: T) -> MatcherResult { + match (self.m1.matches(actual), self.m2.matches(actual)) { + (MatcherResult::NoMatch, MatcherResult::NoMatch) => MatcherResult::NoMatch, + _ => MatcherResult::Match, + } + } + + fn explain_match(&self, actual: T) -> Description { + match (self.m1.matches(actual), self.m2.matches(actual)) { + (MatcherResult::NoMatch, MatcherResult::Match) => self.m1.explain_match(actual), + (MatcherResult::Match, MatcherResult::NoMatch) => self.m2.explain_match(actual), + (_, _) => { + let m1_description = self.m1.explain_match(actual); + if m1_description.is_disjunction_description() { + m1_description.nested(self.m2.explain_match(actual)) + } else { + Description::new() + .bullet_list() + .collect([m1_description, self.m2.explain_match(actual)]) + .disjunction_description() + } + } + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + let m1_description = self.m1.describe(matcher_result); + if m1_description.is_disjunction_description() { + m1_description.push_in_last_nested(self.m2.describe(matcher_result)) + } else { + let header = if matcher_result.into() { + "has at least one of the following properties:" + } else { + "has all of the following properties:" + }; + Description::new() + .text(header) + .nested( + Description::new() + .bullet_list() + .collect([m1_description, self.m2.describe(matcher_result)]), + ) + .disjunction_description() + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn or_true_true_matches() -> Result<()> { + verify_that!(1, anything().or(anything())) + } + + #[test] + fn or_true_false_matches() -> Result<()> { + verify_that!(1, anything().or(not(anything()))) + } + + #[test] + fn or_false_true_matches() -> Result<()> { + verify_that!(1, not(anything()).or(anything())) + } + + #[test] + fn or_false_false_does_not_match() -> Result<()> { + let result = verify_that!(1, not(anything()).or(not(anything()))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 1 + Expected: has at least one of the following properties: + * never matches + * never matches + Actual: 1, + * which is anything + * which is anything + " + )))) + ) + } + + #[test] + fn chained_or_matches() -> Result<()> { + verify_that!(10, eq(1).or(eq(5)).or(ge(9))) + } + + #[test] + fn works_with_str_slices() -> Result<()> { + verify_that!("A string", ends_with("A").or(ends_with("string"))) + } + + #[test] + fn works_with_owned_strings() -> Result<()> { + verify_that!("A string".to_string(), ends_with("A").or(ends_with("string"))) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/display_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/display_matcher.rs new file mode 100644 index 0000000..42b6bf0 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/display_matcher.rs @@ -0,0 +1,124 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use std::fmt::{Debug, Display}; + +/// Matches the string representation of types that implement `Display`. +/// +/// ```ignore +/// let result: impl Display = ...; +/// verify_that!(result, displays_as(eq(format!("{}", result))))?; +/// ``` +pub fn displays_as Matcher<&'a str>>( + inner: InnerMatcher, +) -> DisplayMatcher { + DisplayMatcher { inner } +} + +#[derive(MatcherBase)] +pub struct DisplayMatcher { + inner: InnerMatcher, +} + +impl Matcher<&'a str>> Matcher + for DisplayMatcher +{ + fn matches(&self, actual: T) -> MatcherResult { + self.inner.matches(&format!("{actual}")) + } + + fn explain_match(&self, actual: T) -> Description { + format!( + "which displays as {:?} {}", + actual.to_string(), + self.inner.explain_match(&format!("{actual}")) + ) + .into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("displays as a string which {}", self.inner.describe(MatcherResult::Match)) + .into() + } + MatcherResult::NoMatch => format!( + "doesn't display as a string which {}", + self.inner.describe(MatcherResult::Match) + ) + .into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::fmt::{Debug, Display, Error, Formatter}; + + #[test] + fn display_matches_i32() -> Result<()> { + let value = 32; + verify_that!(value, displays_as(eq("32")))?; + Ok(()) + } + + #[test] + fn display_matches_str() -> Result<()> { + let value = "32"; + verify_that!(value, displays_as(eq("32")))?; + Ok(()) + } + + #[test] + fn display_matches_struct() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + struct Struct { + a: i32, + b: i64, + } + impl Display for Struct { + fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), Error> { + write!(f, "{self:?}") + } + } + verify_that!(Struct { a: 123, b: 321 }, displays_as(eq("Struct { a: 123, b: 321 }")))?; + Ok(()) + } + + #[test] + fn display_displays_error_message_with_explanation_from_inner_matcher() -> Result<()> { + let result = verify_that!("123\n234", displays_as(eq("123\n345"))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Actual: \"123\\n234\", + which displays as \"123\\n234\" which isn't equal to \"123\\n345\" + + Difference(-actual / +expected): + 123 + -234 + +345 + " + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/each_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/each_matcher.rs new file mode 100644 index 0000000..8f17a0a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/each_matcher.rs @@ -0,0 +1,231 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use std::fmt::Debug; + +/// Matches a container all of whose elements are matched by the matcher +/// `inner`. +/// +/// `T` must implement [`IntoIterator`]. This +/// includes `&Vec`, arrays, and slices. +/// +/// ``` +/// # use googletest::prelude::*; +/// # use std::collections::HashSet; +/// # fn should_pass_1() -> Result<()> { +/// let value = vec![1, 2, 3]; +/// verify_that!(value, each(gt(&0)))?; // Passes +/// let array_value = [1, 2, 3]; +/// verify_that!(array_value, each(gt(0)))?; // Passes +/// let slice_value = &[1, 2, 3]; +/// verify_that!(slice_value, each(gt(&0)))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// # let value = vec![1, 2, 3]; +/// verify_that!(value, each(lt(&2)))?; // Fails: 2 and 3 are not less than 2 +/// # Ok(()) +/// # } +/// +/// # fn should_pass_2() -> Result<()> { +/// let value: HashSet = [1, 2, 3].into(); +/// verify_that!(value, each(gt(&0)))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail().unwrap_err(); +/// # should_pass_2().unwrap(); +/// ``` +pub fn each(inner: MatcherT) -> EachMatcher { + EachMatcher { inner } +} + +#[derive(MatcherBase)] +pub struct EachMatcher { + inner: MatcherT, +} + +impl Matcher + for EachMatcher +where + ActualT: IntoIterator, + MatcherT: Matcher, +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + for element in actual { + if self.inner.matches(element).is_no_match() { + return MatcherResult::NoMatch; + } + } + MatcherResult::Match + } + + fn explain_match(&self, actual: ActualT) -> Description { + let mut non_matching_elements = Vec::new(); + for (index, element) in actual.into_iter().enumerate() { + if self.inner.matches(element).is_no_match() { + non_matching_elements.push((index, element, self.inner.explain_match(element))); + } + } + if non_matching_elements.is_empty() { + return format!("whose each element {}", self.inner.describe(MatcherResult::Match)) + .into(); + } + if non_matching_elements.len() == 1 { + let (idx, element, explanation) = non_matching_elements.remove(0); + return format!("whose element #{idx} is {element:?}, {explanation}").into(); + } + + let failed_indexes = non_matching_elements + .iter() + .map(|&(idx, _, _)| format!("#{idx}")) + .collect::>() + .join(", "); + let element_explanations = non_matching_elements + .iter() + .map(|&(_, element, ref explanation)| format!("{element:?}, {explanation}")) + .collect::() + .indent(); + format!("whose elements {failed_indexes} don't match\n{element_explanations}").into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("only contains elements that {}", self.inner.describe(MatcherResult::Match)) + .into() + } + MatcherResult::NoMatch => { + format!("contains no element that {}", self.inner.describe(MatcherResult::Match)) + .into() + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::collections::HashSet; + + #[test] + fn each_matches_empty_vec() -> Result<()> { + let value: Vec = vec![]; + verify_that!(value, each(gt(&0))) + } + + #[test] + fn each_matches_vec_with_one_element() -> Result<()> { + let value = vec![1]; + verify_that!(value, each(gt(&0))) + } + + #[test] + fn each_matches_vec_with_two_elements() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, each(gt(&0))) + } + + #[test] + fn each_matches_slice_with_one_element() -> Result<()> { + let value = &[1]; + verify_that!(*value, each(gt(0))) + } + + #[test] + fn each_matches_hash_set_with_one_element() -> Result<()> { + let value: HashSet = [1].into(); + verify_that!(value, each(gt(&0))) + } + + #[test] + fn each_does_not_match_when_first_element_does_not_match() -> Result<()> { + let value = vec![0]; + verify_that!(value, not(each(gt(&1)))) + } + + #[test] + fn each_does_not_match_when_second_element_does_not_match() -> Result<()> { + let value = vec![2, 0]; + verify_that!(value, not(each(gt(&1)))) + } + + #[test] + fn each_shows_correct_message_when_first_item_does_not_match() -> Result<()> { + let result = verify_that!(vec![0, 2, 3], each(gt(&0))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![0, 2, 3] + Expected: only contains elements that is greater than 0 + Actual: [0, 2, 3], + whose element #0 is 0, which is less than or equal to 0" + )))) + ) + } + + #[test] + fn each_shows_correct_message_when_second_item_does_not_match() -> Result<()> { + let result = verify_that!(vec![1, 0, 3], each(gt(&0))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 0, 3] + Expected: only contains elements that is greater than 0 + Actual: [1, 0, 3], + whose element #1 is 0, which is less than or equal to 0" + )))) + ) + } + + #[test] + fn each_shows_correct_message_when_first_two_items_do_not_match() -> Result<()> { + let result = verify_that!(vec![0, 1, 3], each(gt(&1))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![0, 1, 3] + Expected: only contains elements that is greater than 1 + Actual: [0, 1, 3], + whose elements #0, #1 don't match + 0, which is less than or equal to 1 + 1, which is less than or equal to 1" + )))) + ) + } + #[test] + fn each_shows_inner_explanation() -> Result<()> { + let result = verify_that!(vec![vec![1, 2], vec![1]], each(each(eq(&1)))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Expected: only contains elements that only contains elements that is equal to 1 + Actual: [[1, 2], [1]], + whose element #0 is [1, 2], whose element #1 is 2, which isn't equal to 1" + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/elements_are_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/elements_are_matcher.rs new file mode 100644 index 0000000..075f649 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/elements_are_matcher.rs @@ -0,0 +1,185 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented in the matchers module. +#![doc(hidden)] + +/// Matches a container's elements to each matcher in order. +/// +/// This macro produces a matcher against a container. It takes as arguments a +/// sequence of matchers each of which should respectively match the +/// corresponding element of the actual value. +/// +/// ``` +/// # use googletest::prelude::*; +/// verify_that!(vec![1, 2, 3], elements_are![eq(&1), anything(), gt(&0).and(lt(&123))]) +/// # .unwrap(); +/// ``` +/// +/// The actual value must be a container such as a `&Vec`, an array, or a slice. +/// More precisely, the actual value must implement [`IntoIterator`]. +/// +/// ``` +/// # use googletest::prelude::*; +/// let vector = vec![1, 2, 3]; +/// let slice = vector.as_slice(); +/// verify_that!(slice, elements_are![eq(&1), anything(), gt(&0).and(lt(&123))]) +/// # .unwrap(); +/// ``` +/// +/// This can also be omitted in [`verify_that!`] macros and replaced with square +/// brackets. +/// +/// ``` +/// # use googletest::prelude::*; +/// verify_that!(vec![1, 2], [eq(&1), eq(&2)]) +/// # .unwrap(); +/// ``` +/// +/// Note: This behavior is only possible in [`verify_that!`] macros. In any +/// other cases, it is still necessary to use the +/// [`elements_are!`][crate::matchers::elements_are] macro. +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// verify_that!(vec![vec![1,2], vec![3]], [[eq(&1), eq(&2)], [eq(&3)]]) +/// # .unwrap(); +/// ``` +/// +/// Use this instead: +/// ``` +/// # use googletest::prelude::*; +/// verify_that!(vec![vec![1,2], vec![3]], [elements_are![eq(&1), eq(&2)], elements_are![eq(&3)]]) +/// # .unwrap(); +/// ``` +/// +/// If an inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// +/// verify_that!(vec![1,2,3], elements_are![&1, lt(&1000), gt(&1)]) +/// # .unwrap(); +/// ``` +/// +/// Do not use this with unordered containers, since that will lead to flaky +/// tests. Use +/// [`unordered_elements_are!`][crate::matchers::unordered_elements_are] +/// instead. +/// +/// [`IntoIterator`]: std::iter::IntoIterator +/// [`Iterator`]: std::iter::Iterator +/// [`Iterator::collect`]: std::iter::Iterator::collect +/// [`Vec`]: std::vec::Vec +#[macro_export] +#[doc(hidden)] +macro_rules! __elements_are { + ($($matcher:expr),* $(,)?) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::ElementsAre::new( + vec![$(Box::new( + $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq!( + $matcher + ) + )),*]) + }} +} + +/// Module for use only by the procedural macros in this module. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use crate::description::Description; + use crate::matcher::{Matcher, MatcherBase, MatcherResult}; + use crate::matcher_support::zipped_iterator::zip; + use std::fmt::Debug; + + /// This struct is meant to be used only by the macro `elements_are!`. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + #[derive(MatcherBase)] + pub struct ElementsAre<'a, T: Debug + Copy> { + elements: Vec + 'a>>, + } + + impl<'a, T: Debug + Copy> ElementsAre<'a, T> { + /// Factory only intended for use in the macro `elements_are!`. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + pub fn new(elements: Vec + 'a>>) -> Self { + Self { elements } + } + } + + impl Matcher for ElementsAre<'_, T> + where + ContainerT: IntoIterator, + { + fn matches(&self, actual: ContainerT) -> MatcherResult { + let mut zipped_iterator = zip(actual.into_iter(), self.elements.iter()); + for (a, e) in zipped_iterator.by_ref() { + if e.matches(a).is_no_match() { + return MatcherResult::NoMatch; + } + } + if !zipped_iterator.has_size_mismatch() { + MatcherResult::Match + } else { + MatcherResult::NoMatch + } + } + + fn explain_match(&self, actual: ContainerT) -> Description { + let actual_iterator = actual.into_iter(); + let mut zipped_iterator = zip(actual_iterator, self.elements.iter()); + let mut mismatches = Vec::new(); + for (idx, (a, e)) in zipped_iterator.by_ref().enumerate() { + if e.matches(a).is_no_match() { + mismatches.push(format!("element #{idx} is {a:?}, {}", e.explain_match(a))); + } + } + if mismatches.is_empty() { + if !zipped_iterator.has_size_mismatch() { + "whose elements all match".into() + } else { + format!("whose size is {}", zipped_iterator.left_size()).into() + } + } else if mismatches.len() == 1 { + let mismatches = mismatches.into_iter().collect::(); + format!("where {mismatches}").into() + } else { + let mismatches = mismatches.into_iter().collect::(); + format!("where:\n{}", mismatches.bullet_list().indent()).into() + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + format!( + "{} elements:\n{}", + if matcher_result.into() { "has" } else { "doesn't have" }, + &self + .elements + .iter() + .map(|matcher| matcher.describe(MatcherResult::Match)) + .collect::() + .enumerate() + .indent() + ) + .into() + } + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/empty_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/empty_matcher.rs new file mode 100644 index 0000000..3673f6a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/empty_matcher.rs @@ -0,0 +1,90 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches an empty container. +/// +/// `T` can be any container that implements `IntoIterator`. For instance, `T` +/// can be the reference of a common container like `&Vec` and +/// [`&HashSet`][std::collections::HashSet]. +/// +/// ``` +/// # use googletest::prelude::*; +/// # use std::collections::HashSet; +/// # fn should_pass() -> Result<()> { +/// let value: Vec = vec![]; +/// verify_that!(value, is_empty())?; +/// let value: HashSet = HashSet::new(); +/// verify_that!(value, is_empty())?; +/// let value: &[u32] = &[]; +/// verify_that!(value, is_empty())?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +pub fn is_empty() -> EmptyMatcher { + EmptyMatcher +} + +#[derive(MatcherBase)] +pub struct EmptyMatcher; + +impl Matcher for EmptyMatcher +where + T: IntoIterator, +{ + fn matches(&self, actual: T) -> MatcherResult { + actual.into_iter().next().is_none().into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + if matcher_result.into() { "is empty" } else { "isn't empty" }.into() + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use std::collections::HashSet; + + #[test] + fn empty_matcher_match_empty_vec() -> Result<()> { + let value: Vec = vec![]; + verify_that!(value, is_empty()) + } + + #[test] + fn empty_matcher_does_not_match_empty_vec() -> Result<()> { + let value = vec![1, 2, 3]; + verify_that!(value, not(is_empty())) + } + + #[test] + fn empty_matcher_matches_empty_slice() -> Result<()> { + let value: &[i32] = &[]; + verify_that!(value, is_empty()) + } + + #[test] + fn empty_matcher_matches_empty_hash_set() -> Result<()> { + let value: HashSet = HashSet::new(); + verify_that!(value, is_empty()) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/eq_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/eq_matcher.rs new file mode 100644 index 0000000..33b3f90 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/eq_matcher.rs @@ -0,0 +1,422 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use crate::matcher_support::edit_distance; +use crate::matcher_support::summarize_diff::create_diff; + +use std::fmt::Debug; + +/// Matches a value equal (in the sense of `==`) to `expected`. +/// +/// The type of `expected` must implement the [`PartialEq`] trait so that the +/// expected and actual values can be compared. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(123, eq(123))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(123, eq(234))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// `expected` to `actual` must be comparable with one another via the +/// [`PartialEq`] trait. In most cases, this means that they must be of the same +/// type. However, there are a few cases where different but closely related +/// types are comparable, for example `String` with `&str`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(String::from("Some value"), eq("Some value"))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// In most cases however, one must convert one of the arguments explicitly. +/// This can be surprising when comparing integer types or references. +/// +/// ```compile_fail +/// verify_that!(123u32, eq(123u64))?; // Does not compile +/// verify_that!(123u32 as u64, eq(123u64))?; // Passes +/// ``` +/// +/// ```ignore +/// let actual: &T = ...; +/// let expected: T = T{...}; +/// verify_that(actual, eq(expected))?; // Does not compile +/// verify_that(actual, eq(&expected))?; // Compiles +/// ``` +/// +/// When matching with string types (`&str` and `String`), one can set more +/// options on how equality is checked through the +/// [`StrMatcherConfigurator`][crate::matchers::str_matcher::StrMatcherConfigurator] +/// extension trait, which is implemented for this matcher. +pub fn eq(expected: T) -> EqMatcher { + EqMatcher { expected } +} + +/// A matcher which matches a value equal to `expected`. +/// +/// See [`eq`]. +#[derive(MatcherBase)] +pub struct EqMatcher { + pub(crate) expected: T, +} + +impl> Matcher for EqMatcher { + fn matches(&self, actual: A) -> MatcherResult { + (actual == self.expected).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!("is equal to {:?}", self.expected).into(), + MatcherResult::NoMatch => format!("isn't equal to {:?}", self.expected).into(), + } + } + + fn explain_match(&self, actual: A) -> Description { + let expected_debug = format!("{:#?}", self.expected); + let actual_debug = format!("{actual:#?}"); + let description = Matcher::::describe(self, self.matches(actual)); + + let diff = if is_multiline_string_debug(&actual_debug) + && is_multiline_string_debug(&expected_debug) + { + create_diff( + // The two calls below return None if and only if the strings expected_debug + // respectively actual_debug are not enclosed in ". The calls to + // is_multiline_string_debug above ensure that they are. So the calls cannot + // actually return None and unwrap() should not panic. + &to_display_output(&actual_debug).unwrap(), + &to_display_output(&expected_debug).unwrap(), + edit_distance::Mode::Exact, + ) + } else { + create_diff(&actual_debug, &expected_debug, edit_distance::Mode::Exact) + }; + + if diff.is_empty() { + format!("which {description}").into() + } else { + format!("which {description}\n\n{diff}").into() + } + } +} + +fn is_multiline_string_debug(string: &str) -> bool { + string.starts_with('"') + && string.ends_with('"') + && !string.contains('\n') + && string.contains("\\n") +} + +fn to_display_output(string: &str) -> Option { + Some(string.strip_prefix('"')?.strip_suffix('"')?.split("\\n").collect::>().join("\n")) +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn eq_matches_string_reference_with_string_reference() -> Result<()> { + verify_that!("A string", eq("A string")) + } + + #[test] + fn eq_matches_owned_string_with_string_reference() -> Result<()> { + let value = "A string".to_string(); + verify_that!(value, eq("A string")) + } + + #[test] + fn eq_matches_owned_string_reference_with_string_reference() -> Result<()> { + let value = "A string".to_string(); + verify_that!(&value, eq("A string")) + } + + #[test] + fn eq_matches_i32_with_i32() -> Result<()> { + verify_that!(123, eq(123)) + } + + #[test] + fn eq_struct_debug_diff() -> Result<()> { + #[derive(Debug, PartialEq)] + struct Strukt { + int: i32, + string: String, + } + + let result = verify_that!( + Strukt { int: 123, string: "something".into() }, + eq(&Strukt { int: 321, string: "someone".into() }) + ); + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + " + Actual: Strukt { int: 123, string: \"something\" }, + which isn't equal to Strukt { int: 321, string: \"someone\" } + + Difference(-actual / +expected): + Strukt { + - int: 123, + + int: 321, + - string: \"something\", + + string: \"someone\", + } + "}))) + ) + } + + #[test] + fn eq_vec_debug_diff() -> Result<()> { + let result = verify_that!(vec![1, 2, 3], eq(&vec![1, 3, 4])); + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + " + Value of: vec![1, 2, 3] + Expected: is equal to [1, 3, 4] + Actual: [1, 2, 3], + which isn't equal to [1, 3, 4] + + Difference(-actual / +expected): + [ + 1, + - 2, + 3, + + 4, + ] + "}))) + ) + } + + #[test] + fn eq_vec_debug_diff_length_mismatch() -> Result<()> { + let result = verify_that!(vec![1, 2, 3, 4, 5], eq(&vec![1, 3, 5])); + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + " + Value of: vec![1, 2, 3, 4, 5] + Expected: is equal to [1, 3, 5] + Actual: [1, 2, 3, 4, 5], + which isn't equal to [1, 3, 5] + + Difference(-actual / +expected): + [ + 1, + - 2, + 3, + - 4, + 5, + ] + "}))) + ) + } + + #[test] + fn eq_debug_diff_common_lines_omitted() -> Result<()> { + let result = verify_that!((1..50).collect::>(), eq(&(3..52).collect::>())); + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + " + ], + which isn't equal to [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51] + + Difference(-actual / +expected): + [ + - 1, + - 2, + 3, + 4, + <---- 43 common lines omitted ----> + 48, + 49, + + 50, + + 51, + ]"}))) + ) + } + + #[test] + fn eq_debug_diff_5_common_lines_not_omitted() -> Result<()> { + let result = verify_that!((1..8).collect::>(), eq(&(3..10).collect::>())); + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + " + Actual: [1, 2, 3, 4, 5, 6, 7], + which isn't equal to [3, 4, 5, 6, 7, 8, 9] + + Difference(-actual / +expected): + [ + - 1, + - 2, + 3, + 4, + 5, + 6, + 7, + + 8, + + 9, + ]"}))) + ) + } + + #[test] + fn eq_debug_diff_start_common_lines_omitted() -> Result<()> { + let result = verify_that!((1..50).collect::>(), eq(&(1..52).collect::>())); + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + " + ], + which isn't equal to [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51] + + Difference(-actual / +expected): + [ + 1, + <---- 46 common lines omitted ----> + 48, + 49, + + 50, + + 51, + ]"}))) + ) + } + + #[test] + fn eq_debug_diff_end_common_lines_omitted() -> Result<()> { + let result = verify_that!((1..52).collect::>(), eq(&(3..52).collect::>())); + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + " + ], + which isn't equal to [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51] + + Difference(-actual / +expected): + [ + - 1, + - 2, + 3, + 4, + <---- 46 common lines omitted ----> + 51, + ]"}))) + ) + } + + #[test] + fn eq_multi_line_string_debug_diff() -> Result<()> { + let result = verify_that!("One\nTwo\nThree", eq("One\nSix\nThree")); + // TODO: b/257454450 - Make this more useful, by potentially unescaping the + // line return. + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + r#" + Value of: "One\nTwo\nThree" + Expected: is equal to "One\nSix\nThree" + Actual: "One\nTwo\nThree", + which isn't equal to "One\nSix\nThree" + "#}))) + ) + } + + #[test] + fn match_explanation_contains_diff_of_strings_if_more_than_one_line() -> Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + " + ), + eq(indoc!( + " + First line + Second lines + Third line + " + )) + ); + + verify_that!( + result, + err(displays_as(contains_substring( + "\ + First line + -Second line + +Second lines + Third line" + ))) + ) + } + + #[test] + fn match_explanation_does_not_show_diff_if_actual_value_is_single_line() -> Result<()> { + let result = verify_that!( + "First line", + eq(indoc!( + " + First line + Second line + Third line + " + )) + ); + + verify_that!( + result, + err(displays_as(not(contains_substring("Difference(-actual / +expected):")))) + ) + } + + #[test] + fn match_explanation_does_not_show_diff_if_expected_value_is_single_line() -> Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + " + ), + eq("First line") + ); + + verify_that!( + result, + err(displays_as(not(contains_substring("Difference(-actual / +expected):")))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/err_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/err_matcher.rs new file mode 100644 index 0000000..1875f93 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/err_matcher.rs @@ -0,0 +1,282 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a `Result` containing `Err` with a value matched by `inner`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> googletest::Result<()> { +/// verify_that!(Err::<(), _>("Some error"), err(eq("Some error")))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> googletest::Result<()> { +/// verify_that!(Ok::<_, &str>("A value"), err(eq("A value")))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> googletest::Result<()> { +/// verify_that!(Err::<(), _>("Some error"), err(eq("Some other error")))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// ``` +pub fn err(inner: Inner) -> ErrMatcher { + ErrMatcher { inner } +} + +#[derive(MatcherBase)] +pub struct ErrMatcher { + inner: InnerMatcherT, +} + +impl> Matcher> + for ErrMatcher +{ + fn matches(&self, actual: std::result::Result) -> MatcherResult { + actual.err().map(|v| self.inner.matches(v)).unwrap_or(MatcherResult::NoMatch) + } + + fn explain_match(&self, actual: std::result::Result) -> Description { + match actual { + Err(e) => { + Description::new().text("which is an error").nested(self.inner.explain_match(e)) + } + Ok(_) => "which is a success".into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("is an error which {}", self.inner.describe(MatcherResult::Match)).into() + } + MatcherResult::NoMatch => format!( + "is a success or is an error containing a value which {}", + self.inner.describe(MatcherResult::NoMatch) + ) + .into(), + } + } +} + +impl<'a, T: Debug, E: Debug, InnerMatcherT: Matcher<&'a E>> Matcher<&'a std::result::Result> + for ErrMatcher +{ + fn matches(&self, actual: &'a std::result::Result) -> MatcherResult { + actual.as_ref().err().map(|v| self.inner.matches(v)).unwrap_or(MatcherResult::NoMatch) + } + + fn explain_match(&self, actual: &'a std::result::Result) -> Description { + match actual { + Err(e) => { + Description::new().text("which is an error").nested(self.inner.explain_match(e)) + } + Ok(_) => "which is a success".into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("is an error which {}", self.inner.describe(MatcherResult::Match)).into() + } + MatcherResult::NoMatch => format!( + "is a success or is an error containing a value which {}", + self.inner.describe(MatcherResult::NoMatch) + ) + .into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn err_matches_result_with_err_value() -> Result<()> { + let matcher = err(eq(1)); + let value: std::result::Result = Err(1); + + let result = matcher.matches(value); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn err_does_not_match_result_with_wrong_err_value() -> Result<()> { + let matcher = err(eq(1)); + let value: std::result::Result = Err(0); + + let result = matcher.matches(value); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn err_does_not_match_result_with_ok() -> Result<()> { + let matcher = err(eq(1)); + let value: std::result::Result = Ok(1); + + let result = matcher.matches(value); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn err_matches_result_with_err_value_by_ref() -> Result<()> { + let value: std::result::Result = Err("123".to_string()); + verify_that!(value, err(eq("123"))) + } + + #[test] + fn err_does_not_match_result_with_wrong_err_value_by_ref() -> Result<()> { + let value: std::result::Result = Err("321".to_string()); + verify_that!(value, not(err(eq("123")))) + } + + #[test] + fn err_does_not_match_result_with_ok_by_ref() -> Result<()> { + let value: std::result::Result = Ok("123".to_string()); + verify_that!(value, not(err(eq("123")))) + } + + #[test] + fn err_full_error_message() -> Result<()> { + let result = verify_that!(Err::(1), err(eq(2))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: Err::(1) + Expected: is an error which is equal to 2 + Actual: Err(1), + which is an error + which isn't equal to 2 + " + )))) + ) + } + + #[test] + fn err_describe_match() -> Result<()> { + let matcher = err(eq(1)); + verify_that!( + Matcher::>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is an error which is equal to 1")) + ) + } + + #[test] + fn err_describe_no_match() -> Result<()> { + let matcher = err(eq(1)); + verify_that!( + Matcher::>::describe(&matcher, MatcherResult::NoMatch), + displays_as(eq( + "is a success or is an error containing a value which isn't equal to 1" + )) + ) + } + + #[test] + fn err_describe_match_by_ref() -> Result<()> { + let matcher = err(eq("123")); + verify_that!( + Matcher::<&std::result::Result>::describe( + &matcher, + MatcherResult::Match + ), + displays_as(eq("is an error which is equal to \"123\"")) + ) + } + + #[test] + fn err_describe_no_match_by_ref() -> Result<()> { + let matcher = err(eq("123")); + verify_that!( + Matcher::<&std::result::Result>::describe( + &matcher, + MatcherResult::NoMatch + ), + displays_as(eq( + "is a success or is an error containing a value which isn't equal to \"123\"" + )) + ) + } + + #[test] + fn err_explain_match_success() -> Result<()> { + let actual: std::result::Result = Err(1); + let matcher = err(eq(1)); + verify_that!( + matcher.explain_match(actual), + displays_as(eq("which is an error\n which is equal to 1")) + ) + } + + #[test] + fn err_explain_match_fail() -> Result<()> { + let actual: std::result::Result = Err(2); + let matcher = err(eq(1)); + verify_that!( + matcher.explain_match(actual), + displays_as(eq("which is an error\n which isn't equal to 1")) + ) + } + + #[test] + fn err_explain_match_ok() -> Result<()> { + let actual: std::result::Result = Ok(1); + let matcher = err(eq(1)); + verify_that!(matcher.explain_match(actual), displays_as(eq("which is a success"))) + } + + #[test] + fn err_explain_match_success_by_ref() -> Result<()> { + let actual: std::result::Result = Err("123".to_string()); + let matcher = err(eq("123")); + verify_that!( + matcher.explain_match(&actual), + displays_as(eq("which is an error\n which is equal to \"123\"")) + ) + } + + #[test] + fn err_explain_match_fail_by_ref() -> Result<()> { + let actual: std::result::Result = Err("321".to_string()); + let matcher = err(eq("123")); + verify_that!( + matcher.explain_match(&actual), + displays_as(eq("which is an error\n which isn't equal to \"123\"")) + ) + } + + #[test] + fn err_explain_match_ok_by_ref() -> Result<()> { + let actual: std::result::Result = Ok("123".to_string()); + let matcher = err(eq("123")); + verify_that!(matcher.explain_match(&actual), displays_as(eq("which is a success"))) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/field_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/field_matcher.rs new file mode 100644 index 0000000..98eba6d --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/field_matcher.rs @@ -0,0 +1,360 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented in the matchers module. +#![doc(hidden)] + +/// Matches a structure or enum with a given field which is matched by a given +/// matcher. +/// +/// This takes two arguments: +/// +/// * a specification of the field against which to match, and +/// * an inner [`Matcher`][crate::matcher::Matcher] to apply to that field. +/// +/// For example: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct IntField { +/// int: i32 +/// } +/// # fn should_pass() -> Result<()> { +/// verify_that!(IntField{int: 32}, field!(&IntField.int, eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// Tuple structs are also supported via the index syntax: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct IntField(i32); +/// # fn should_pass() -> Result<()> { +/// verify_that!(IntField(32), field!(&IntField.0, eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// Enums are also supported, in which case only the specified variant is +/// matched: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// enum MyEnum { +/// A(i32), +/// B, +/// } +/// # fn should_pass() -> Result<()> { +/// verify_that!(MyEnum::A(32), field!(&MyEnum::A.0, eq(32)))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(MyEnum::B, field!(&MyEnum::A.0, eq(32)))?; // Fails: wrong enum variant +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// The structure or enum may also be referenced from a separate module: +/// +/// ``` +/// # use googletest::prelude::*; +/// mod a_module { +/// #[derive(Debug)] +/// pub struct AStruct(pub i32); +/// } +/// # fn should_pass() -> Result<()> { +/// verify_that!(a_module::AStruct(32), field!(&a_module::AStruct.0, eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// If the inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct IntField { +/// int: i32 +/// } +/// # fn should_pass() -> Result<()> { +/// verify_that!(IntField{int: 32}, field!(&IntField.int, 32))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// Nested structures are *not supported*, however: +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct InnerStruct(i32); +/// #[derive(Debug)] +/// struct OuterStruct { +/// inner: InnerStruct, +/// } +/// # fn should_not_compile() -> Result<()> { +/// verify_that!(value, field!(OuterStruct.inner.0, eq(32)))?; // Does not compile +/// # Ok(()) +/// # } +/// ``` +/// +/// # Specification of the field pattern +/// +/// The specification of the field follow the syntax: `(ref)? (&)? +/// $TYPE.$FIELD`. +/// The `&` allows to specify whether this matcher matches against an actual of +/// type `$TYPE` (`$TYPE`` must implement `Copy`) or a `&$TYPE`. +/// +/// For instance: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// pub struct AStruct{a_field: i32}; +/// # fn should_pass() -> Result<()> { +/// verify_that!(AStruct{a_field: 32}, field!(&AStruct.a_field, eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug, Clone, Copy)] +/// pub struct AStruct{a_field: i32}; +/// # fn should_pass() -> Result<()> { +/// verify_that!(AStruct{a_field: 32}, field!(AStruct.a_field, eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// The `ref` allows to bind the field value by reference, which is required if +/// the field type does not implement `Copy`. +/// +/// For instance: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// pub struct AStruct{a_field: i32}; +/// # fn should_pass() -> Result<()> { +/// verify_that!(AStruct{a_field: 32}, field!(&AStruct.a_field, eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// If `field!` is qualified by both `&` and `ref`, they can both be omitted. +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// pub struct AStruct{a_field: String}; +/// # fn should_pass() -> Result<()> { +/// verify_that!(AStruct{a_field: "32".into()}, field!(&AStruct.a_field, ref eq("32")))?; +/// verify_that!(AStruct{a_field: "32".into()}, field!(AStruct.a_field, eq("32")))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// See also the macro [`property`][crate::matchers::property] for an analogous +/// mechanism to extract a datum by invoking a method. +#[macro_export] +#[doc(hidden)] +macro_rules! __field { + ($($t:tt)*) => { $crate::field_internal!($($t)*) } +} + +// Internal-only macro created so that the macro definition does not appear in +// generated documentation. +// We cannot use `path` or `ty` to capture the type as we are terminating the +// type with a . (dot). +#[doc(hidden)] +#[macro_export] +macro_rules! field_internal { + // `ref` variant. + ( + // The 3 cases of `$(& $($_amp:literal)?)?` -> `$(& $($_amp)*)*` + // 1. outer group captures nothing => expansion produces nothing + // 2. outer group captures just `&` => expansion produces `&` + // 3. outer group captures `& ` => disallowed by `@assert_empty` subrule invocation + // + // `$_amp:literal` works only because the following `$t:ident` or `::` can't be captured by + // it. + $(& $($_amp:literal)?)? // Optional `&`'s presence is implicitly captured by `_amp`. + $(:: $($_cs:literal)?)? // Optional `::`'s presence is implicitly captured by `_cs`. + $($t:ident)::+ $(::<$($t_ty_args:ty),* $(,)?>)? .$field:tt, + ref $m:expr) => {{ + $crate::field_internal!(@assert_empty $($($_amp)*)* $($($_cs)*)*); + $crate::field_internal!(@internal + struct_type: [&_] + field_prefix: [$(& $($_amp)*)* $(:: $($_cs)*)* $($t)::* $(::<$($t_ty_args),*>)*] + [$field] [ref] [$m]) + }}; + + // Non-`ref` variant. + ( + // See comment on previous variant above. + $(& $($_amp:literal)?)? // Optional `&`'s presence is implicitly captured by `_amp`. + $(:: $($_cs:literal)?)? // Optional `::`'s presence is implicitly captured by `_cs`. + $($t:ident)::+ $(::<$($t_ty_args:ty),* $(,)?>)? .$field:tt, $m:expr) => {{ + $crate::field_internal!(@assert_empty $($($_amp)*)* $($($_cs)*)*); + $crate::field_internal!(@internal + struct_type: [$(& $($_amp)*)* &_] + field_prefix: [$(& $($_amp)*)* $(:: $($_cs)*)* $($t)::* $(::<$($t_ty_args),*>)*] + [$field] [] [$m]) + }}; + + (@assert_empty) => {}; + (@assert_empty $($l:literal)+) => { + compile_error!("property! argument must start with an optional `&` followed by a path") + }; + + (@internal struct_type: [$struct_ty:ty] + field_prefix: [$($field_prefix:tt)*] + [$field:tt] [$($ref:tt)?] [$m:expr]) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::field_matcher( + |o: $struct_ty| { + match o { + $($field_prefix)* {$field: $($ref)* value, .. } => Some(value), + // The pattern below is unreachable if the type is a struct (as opposed to an + // enum). Since the macro can't know which it is, we always include it and just + // tell the compiler not to complain. + #[allow(unreachable_patterns)] + _ => None, + } + }, + &stringify!($field), + $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq!($m)) + }} +} + +/// Functions for use only by the declarative macros in this module. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, + }; + use std::fmt::Debug; + + /// Creates a matcher to verify a specific field of the actual struct using + /// the provided inner matcher. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + pub fn field_matcher( + field_accessor: fn(&OuterT) -> Option<&InnerT>, + field_path: &'static str, + inner: InnerMatcher, + ) -> FieldMatcher { + FieldMatcher { field_accessor, field_path, inner } + } + + #[derive(MatcherBase)] + pub struct FieldMatcher { + field_accessor: fn(&OuterT) -> Option<&InnerT>, + field_path: &'static str, + inner: InnerMatcher, + } + + impl<'a, OuterT: Debug + 'a, InnerT: Debug + 'a, InnerMatcher: Matcher<&'a InnerT>> + Matcher<&'a OuterT> for FieldMatcher + { + fn matches(&self, actual: &'a OuterT) -> MatcherResult { + if let Some(value) = (self.field_accessor)(actual) { + self.inner.matches(value) + } else { + MatcherResult::NoMatch + } + } + + fn explain_match(&self, actual: &'a OuterT) -> Description { + if let Some(actual) = (self.field_accessor)(actual) { + format!( + "which has field `{}`, {}", + self.field_path, + self.inner.explain_match(actual) + ) + .into() + } else { + let formatted_actual_value = format!("{actual:?}"); + let without_fields = formatted_actual_value.split('(').next().unwrap_or(""); + let without_fields = without_fields.split('{').next().unwrap_or("").trim_end(); + format!("which has the wrong enum variant `{without_fields}`").into() + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + format!( + "has field `{}`, which {}", + self.field_path, + self.inner.describe(matcher_result) + ) + .into() + } + } + + impl> Matcher + for FieldMatcher + { + fn matches(&self, actual: OuterT) -> MatcherResult { + if let Some(value) = (self.field_accessor)(&actual) { + self.inner.matches(*value) + } else { + MatcherResult::NoMatch + } + } + + fn explain_match(&self, actual: OuterT) -> Description { + if let Some(actual) = (self.field_accessor)(&actual) { + format!( + "which has field `{}`, {}", + self.field_path, + self.inner.explain_match(*actual) + ) + .into() + } else { + let formatted_actual_value = format!("{actual:?}"); + let without_fields = formatted_actual_value.split('(').next().unwrap_or(""); + let without_fields = without_fields.split('{').next().unwrap_or("").trim_end(); + format!("which has the wrong enum variant `{without_fields}`").into() + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + format!( + "has field `{}`, which {}", + self.field_path, + self.inner.describe(matcher_result) + ) + .into() + } + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/ge_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/ge_matcher.rs new file mode 100644 index 0000000..d0d8d34 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/ge_matcher.rs @@ -0,0 +1,219 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a value greater than or equal to (in the sense of `>=`) `expected`. +/// +/// The types of `ActualT` of `actual` and `ExpectedT` of `expected` must be +/// comparable via the `PartialOrd` trait. Namely, `ActualT` must implement +/// `PartialOrd`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(1, ge(0))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(0, ge(1))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// In most cases the params neeed to be the same type or they need to be cast +/// explicitly. This can be surprising when comparing integer types or +/// references: +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// verify_that!(123u32, ge(0u64))?; // Does not compile +/// verify_that!(123u32 as u64, ge(0u64))?; // Passes +/// # Ok(()) +/// # } +/// ``` +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// let actual: &u32 = &2; +/// let expected: u32 = 0; +/// verify_that!(actual, ge(expected))?; // Does not compile +/// # Ok(()) +/// # } +/// ``` +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let actual: &u32 = &2; +/// let expected: u32 = 0; +/// verify_that!(actual, ge(&expected))?; // Compiles and passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// You can find the standard library `PartialOrd` implementation in +/// +pub fn ge(expected: ExpectedT) -> GeMatcher { + GeMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct GeMatcher { + expected: ExpectedT, +} + +impl + Copy, ExpectedT: Debug> Matcher + for GeMatcher +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + (actual >= self.expected).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("is greater than or equal to {:?}", self.expected).into() + } + MatcherResult::NoMatch => format!("is less than {:?}", self.expected).into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::ffi::OsString; + + #[test] + fn ge_matches_i32_with_i32() -> Result<()> { + let actual: i32 = 0; + let expected: i32 = 0; + verify_that!(actual, ge(expected)) + } + + #[test] + fn ge_does_not_match_smaller_i32() -> Result<()> { + let matcher = ge(10); + let result = matcher.matches(9); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn ge_matches_bigger_str() -> Result<()> { + verify_that!("B", ge("A")) + } + + #[test] + fn ge_does_not_match_lesser_str() -> Result<()> { + let matcher = ge("z"); + let result = matcher.matches("a"); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn ge_mismatch_contains_actual_and_expected() -> Result<()> { + let result = verify_that!(591, ge(927)); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 591 + Expected: is greater than or equal to 927 + Actual: 591, + which is less than 927 + " + )))) + ) + } + + // Test `ge` matcher where actual is `&OsString` and expected is `&str`. + // Note that stdlib is a little bit inconsistent: `PartialOrd` exists for + // `OsString` and `str`, but only in one direction: it's only possible to + // compare `OsString` with `str` if `OsString` is on the left side of the + // ">=" operator (`impl PartialOrd for OsString`). + // + // The comparison in the other direction is not defined. + // + // This means that the test case bellow effectively ensures that + // `verify_that(actual, ge(expected))` works if `actual >= expected` works + // (regardless whether the `expected >= actual` works`). + #[test] + fn ge_matches_owned_osstring_reference_with_string_reference() -> Result<()> { + let expected = "A"; + let actual: OsString = "B".to_string().into(); + verify_that!(&actual, ge(expected)) + } + + #[test] + fn ge_matches_ipv6addr_with_ipaddr() -> Result<()> { + use std::net::IpAddr; + use std::net::Ipv6Addr; + let actual: Ipv6Addr = "2001:4860:4860::8844".parse().unwrap(); + let expected: IpAddr = "127.0.0.1".parse().unwrap(); + verify_that!(actual, ge(expected)) + } + + #[test] + fn ge_matches_with_custom_partial_ord() -> Result<()> { + /// A custom "number" that is lower than all other numbers. The only + /// things we define about this "special" number is `PartialOrd` and + /// `PartialEq` against `u32`. + #[derive(Debug)] + struct VeryLowNumber {} + + impl std::cmp::PartialEq for VeryLowNumber { + fn eq(&self, _other: &u32) -> bool { + false + } + } + + // PartialOrd (required for >) requires PartialEq. + impl std::cmp::PartialOrd for VeryLowNumber { + fn partial_cmp(&self, _other: &u32) -> Option { + Some(std::cmp::Ordering::Less) + } + } + + impl std::cmp::PartialEq for u32 { + fn eq(&self, _other: &VeryLowNumber) -> bool { + false + } + } + + impl std::cmp::PartialOrd for u32 { + fn partial_cmp(&self, _other: &VeryLowNumber) -> Option { + Some(std::cmp::Ordering::Greater) + } + } + + let actual: u32 = 42; + let expected = VeryLowNumber {}; + + verify_that!(actual, ge(expected)) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/gt_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/gt_matcher.rs new file mode 100644 index 0000000..cf5381e --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/gt_matcher.rs @@ -0,0 +1,258 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a value greater (in the sense of `>`) than `expected`. +/// +/// The types of `ActualT` of `actual` and `ExpectedT` of `expected` must be +/// comparable via the `PartialOrd` trait. Namely, `ActualT` must implement +/// `PartialOrd`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(38, gt(1))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(234, gt(234))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// In most cases the params neeed to be the same type or they need to be cast +/// explicitly. This can be surprising when comparing integer types or +/// references: +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// verify_that!(123u32, gt(0u64))?; // Does not compile +/// verify_that!(123u32 as u64, gt(0u64))?; // Passes +/// # Ok(()) +/// # } +/// ``` +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// let actual: &u32 = &2; +/// let expected: u32 = 1; +/// verify_that!(actual, gt(expected))?; // Does not compile +/// # Ok(()) +/// # } +/// ``` +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let actual: &u32 = &2; +/// let expected: u32 = 1; +/// verify_that!(actual, gt(&expected))?; // Compiles and passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// You can find the standard library `PartialOrd` implementation in +/// +pub fn gt(expected: ExpectedT) -> GtMatcher { + GtMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct GtMatcher { + expected: ExpectedT, +} + +impl + Copy, ExpectedT: Debug> Matcher + for GtMatcher +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + (actual > self.expected).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!("is greater than {:?}", self.expected).into(), + MatcherResult::NoMatch => { + format!("is less than or equal to {:?}", self.expected).into() + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::ffi::OsString; + + #[test] + fn gt_matches_i32_with_i32() -> Result<()> { + let actual: i32 = 321; + let expected: i32 = 123; + verify_that!(actual, gt(expected)) + } + + #[test] + fn gt_does_not_match_equal_i32() -> Result<()> { + let matcher = gt(10); + let result = matcher.matches(10); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn gt_does_not_match_lower_i32() -> Result<()> { + let matcher = gt(-50); + let result = matcher.matches(-51); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn gt_matches_greater_str() -> Result<()> { + verify_that!("B", gt("A")) + } + + #[test] + fn gt_does_not_match_lesser_str() -> Result<()> { + let matcher = gt("B"); + let result = matcher.matches("A"); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn gt_mismatch_contains_actual_and_expected() -> Result<()> { + let result = verify_that!(481, gt(632)); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 481 + Expected: is greater than 632 + Actual: 481, + " + )))) + ) + } + + #[test] + fn gt_mismatch_combined_with_each() -> Result<()> { + let result = verify_that!(vec![19, 23, 11], each(gt(&15))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![19, 23, 11] + Expected: only contains elements that is greater than 15 + Actual: [19, 23, 11], + whose element #2 is 11, which is less than or equal to 15 + " + )))) + ) + } + + #[test] + fn gt_describe_matches() -> Result<()> { + verify_that!( + Matcher::::describe(>(232), MatcherResult::Match), + displays_as(eq("is greater than 232")) + ) + } + + #[test] + fn gt_describe_does_not_match() -> Result<()> { + verify_that!( + Matcher::::describe(>(232), MatcherResult::NoMatch), + displays_as(eq("is less than or equal to 232")) + ) + } + + // Test `gt` matcher where actual is `&OsString` and expected is `&str`. + // Note that stdlib is a little bit inconsistent: `PartialOrd` exists for + // `OsString` and `str`, but only in one direction: it's only possible to + // compare `OsString` with `str` if `OsString` is on the left side of the + // ">" operator (`impl PartialOrd for OsString`). + // + // The comparison in the other direction is not defined. + // + // This means that the test case bellow effectively ensures that + // `verify_that(actual, gt(expected))` works if `actual > expected` works + // (regardless whether the `expected > actual` works`). + #[test] + fn gt_matches_owned_osstring_reference_with_string_reference() -> Result<()> { + let expected = "A"; + let actual: OsString = "B".to_string().into(); + verify_that!(&actual, gt(expected)) + } + + #[test] + fn gt_matches_ipv6addr_with_ipaddr() -> Result<()> { + use std::net::IpAddr; + use std::net::Ipv6Addr; + let actual: Ipv6Addr = "2001:4860:4860::8888".parse().unwrap(); + let expected: IpAddr = "127.0.0.1".parse().unwrap(); + verify_that!(actual, gt(expected)) + } + + #[test] + fn gt_matches_with_custom_partial_ord() -> Result<()> { + /// A custom "number" that is smaller than all other numbers. The only + /// things we define about this "special" number is `PartialOrd` and + /// `PartialEq` against `u32`. + #[derive(Debug)] + struct VeryLowNumber {} + + impl std::cmp::PartialEq for VeryLowNumber { + fn eq(&self, _other: &u32) -> bool { + false + } + } + + // PartialOrd (required for >) requires PartialEq. + impl std::cmp::PartialOrd for VeryLowNumber { + fn partial_cmp(&self, _other: &u32) -> Option { + Some(std::cmp::Ordering::Less) + } + } + + impl std::cmp::PartialEq for u32 { + fn eq(&self, _other: &VeryLowNumber) -> bool { + false + } + } + + impl std::cmp::PartialOrd for u32 { + fn partial_cmp(&self, _other: &VeryLowNumber) -> Option { + Some(std::cmp::Ordering::Greater) + } + } + + let actual: u32 = 42; + let expected = VeryLowNumber {}; + + verify_that!(actual, gt(expected)) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/has_entry_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/has_entry_matcher.rs new file mode 100644 index 0000000..7c94420 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/has_entry_matcher.rs @@ -0,0 +1,178 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use std::collections::HashMap; +use std::fmt::Debug; +use std::hash::Hash; + +/// Matches a `&HashMap` containing the given `key` whose value is matched by +/// the matcher `inner`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # use std::collections::HashMap; +/// # fn should_pass() -> Result<()> { +/// let value = HashMap::from([(0, 1), (1, -1)]); +/// verify_that!(value, has_entry(0, eq(&1)))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// # let value = HashMap::from([(0, 1), (1, -1)]); +/// verify_that!(value, has_entry(1, gt(&0)))?; // Fails: value not matched +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// # let value = HashMap::from([(0, 1), (1, -1)]); +/// verify_that!(value, has_entry(2, eq(&0)))?; // Fails: key not present +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// ``` +/// +/// Note: One could obtain the same effect by using `contains` and a +/// `Matcher<(&Key, &Value)>`: +/// +/// ``` +/// # use googletest::prelude::*; +/// # use std::collections::HashMap; +/// # fn should_pass() -> Result<()> { +/// let value = HashMap::from([(0, 1), (1, -1)]); +/// verify_that!(value, contains(eq((&0, &1))))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +pub fn has_entry(key: KeyT, inner: MatcherT) -> HasEntryMatcher { + HasEntryMatcher { key, inner } +} + +#[derive(MatcherBase)] +pub struct HasEntryMatcher { + key: KeyT, + inner: MatcherT, +} + +impl<'a, KeyT: Debug + Eq + Hash, ValueT: Debug, MatcherT: Matcher<&'a ValueT>> + Matcher<&'a HashMap> for HasEntryMatcher +{ + fn matches(&self, actual: &'a HashMap) -> MatcherResult { + if let Some(value) = actual.get(&self.key) { + self.inner.matches(value) + } else { + MatcherResult::NoMatch + } + } + + fn explain_match(&self, actual: &'a HashMap) -> Description { + if let Some(value) = actual.get(&self.key) { + format!( + "which contains key {:?}, but is mapped to value {:#?}, {}", + self.key, + value, + self.inner.explain_match(value) + ) + .into() + } else { + format!("which doesn't contain key {:?}", self.key).into() + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!( + "contains key {:?}, which value {}", + self.key, + self.inner.describe(MatcherResult::Match) + ) + .into(), + MatcherResult::NoMatch => format!( + "doesn't contain key {:?} or contains key {:?}, which value {}", + self.key, + self.key, + self.inner.describe(MatcherResult::NoMatch) + ) + .into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::collections::HashMap; + + #[test] + fn has_entry_does_not_match_empty_hash_map() -> Result<()> { + let value: HashMap = HashMap::new(); + verify_that!(&value, not(has_entry(0, eq(&0)))) + } + + #[test] + fn has_entry_matches_hash_map_with_value() -> Result<()> { + let value: HashMap = HashMap::from([(0, 0)]); + verify_that!(&value, has_entry(0, eq(&0))) + } + + #[test] + fn has_entry_does_not_match_hash_map_with_wrong_value() -> Result<()> { + let value: HashMap = HashMap::from([(0, 1)]); + verify_that!(&value, not(has_entry(0, eq(&0)))) + } + + #[test] + fn has_entry_does_not_match_hash_map_with_wrong_key() -> Result<()> { + let value: HashMap = HashMap::from([(1, 0)]); + verify_that!(&value, not(has_entry(0, eq(&0)))) + } + + #[test] + fn has_entry_shows_correct_message_when_key_is_not_present() -> Result<()> { + let result = verify_that!(HashMap::from([(0, 0)]), has_entry(1, eq(&0))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: HashMap::from([(0, 0)]) + Expected: contains key 1, which value is equal to 0 + Actual: {0: 0}, + which doesn't contain key 1 + " + )))) + ) + } + + #[test] + fn has_entry_shows_correct_message_when_key_has_non_matching_value() -> Result<()> { + let result = verify_that!(HashMap::from([(0, 0)]), has_entry(0, eq(&1))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: HashMap::from([(0, 0)]) + Expected: contains key 0, which value is equal to 1 + Actual: {0: 0}, + which contains key 0, but is mapped to value 0, which isn't equal to 1 + " + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/is_encoded_string_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/is_encoded_string_matcher.rs new file mode 100644 index 0000000..db7461c --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/is_encoded_string_matcher.rs @@ -0,0 +1,176 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a byte sequence which is a UTF-8 encoded string matched by `inner`. +/// +/// The matcher reports no match if either the string is not UTF-8 encoded or if +/// `inner` does not match on the decoded string. +/// +/// The input may be a slice `&[u8]` or a `Vec` of bytes. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let bytes: &[u8] = "A string".as_bytes(); +/// verify_that!(bytes, is_utf8_string(eq("A string")))?; // Passes +/// let bytes: Vec = "A string".as_bytes().to_vec(); +/// verify_that!(bytes, is_utf8_string(eq("A string")))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// # let bytes: &[u8] = "A string".as_bytes(); +/// verify_that!(bytes, is_utf8_string(eq("Another string")))?; // Fails (inner matcher does not match) +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// let bytes: Vec = vec![255, 64, 128, 32]; +/// verify_that!(bytes, is_utf8_string(anything()))?; // Fails (not UTF-8 encoded) +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// ``` +pub fn is_utf8_string(inner: InnerMatcherT) -> IsEncodedStringMatcher +where + InnerMatcherT: for<'a> Matcher<&'a str>, +{ + IsEncodedStringMatcher { inner } +} + +#[derive(MatcherBase)] +pub struct IsEncodedStringMatcher { + inner: InnerMatcherT, +} + +impl + Debug + Copy, InnerMatcherT> Matcher + for IsEncodedStringMatcher +where + InnerMatcherT: for<'a> Matcher<&'a str>, +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + std::str::from_utf8(actual.as_ref()) + .map(|s| self.inner.matches(s)) + .unwrap_or(MatcherResult::NoMatch) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!( + "is a UTF-8 encoded string which {}", + self.inner.describe(MatcherResult::Match) + ) + .into(), + MatcherResult::NoMatch => format!( + "is not a UTF-8 encoded string which {}", + self.inner.describe(MatcherResult::Match) + ) + .into(), + } + } + + fn explain_match(&self, actual: ActualT) -> Description { + match std::str::from_utf8(actual.as_ref()) { + Ok(s) => { + format!("which is a UTF-8 encoded string {}", self.inner.explain_match(s)).into() + } + Err(e) => format!("which is not a UTF-8 encoded string: {e}").into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + + #[test] + fn matches_string_as_byte_slice() -> Result<()> { + verify_that!("A string".as_bytes(), is_utf8_string(eq("A string"))) + } + + #[test] + fn matches_string_as_byte_vec() -> Result<()> { + verify_that!("A string".as_bytes().to_vec(), is_utf8_string(eq("A string"))) + } + + #[test] + fn matches_string_with_utf_8_encoded_sequences() -> Result<()> { + verify_that!("äöüÄÖÜ".as_bytes().to_vec(), is_utf8_string(eq("äöüÄÖÜ"))) + } + + #[test] + fn does_not_match_non_equal_string() -> Result<()> { + verify_that!("äöüÄÖÜ".as_bytes().to_vec(), not(is_utf8_string(eq("A string")))) + } + + #[test] + fn does_not_match_non_utf_8_encoded_byte_sequence() -> Result<()> { + verify_that!(&[192, 64, 255, 32], not(is_utf8_string(eq("A string")))) + } + + #[test] + fn has_correct_description_in_matched_case() -> Result<()> { + let matcher = is_utf8_string(eq("A string")); + + verify_that!( + Matcher::<&[u8]>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is a UTF-8 encoded string which is equal to \"A string\"")) + ) + } + + #[test] + fn has_correct_description_in_not_matched_case() -> Result<()> { + let matcher = is_utf8_string(eq("A string")); + + verify_that!( + Matcher::<&[u8]>::describe(&matcher, MatcherResult::NoMatch), + displays_as(eq("is not a UTF-8 encoded string which is equal to \"A string\"")) + ) + } + + #[test] + fn has_correct_explanation_in_matched_case() -> Result<()> { + let explanation = is_utf8_string(eq("A string")).explain_match("A string".as_bytes()); + + verify_that!( + explanation, + displays_as(eq("which is a UTF-8 encoded string which is equal to \"A string\"")) + ) + } + + #[test] + fn has_correct_explanation_when_byte_array_is_not_utf8_encoded() -> Result<()> { + let explanation = is_utf8_string(eq("A string")).explain_match([192, 128, 0, 64]); + + verify_that!(explanation, displays_as(starts_with("which is not a UTF-8 encoded string: "))) + } + + #[test] + fn has_correct_explanation_when_inner_matcher_does_not_match() -> Result<()> { + let explanation = is_utf8_string(eq("A string")).explain_match("Another string".as_bytes()); + + verify_that!( + explanation, + displays_as(eq("which is a UTF-8 encoded string which isn't equal to \"A string\"")) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/is_finite_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/is_finite_matcher.rs new file mode 100644 index 0000000..3571294 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/is_finite_matcher.rs @@ -0,0 +1,84 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use num_traits::float::Float; +use std::fmt::Debug; + +/// Matches a floating point value which is Finite. +pub fn is_finite() -> IsFiniteMatcher { + IsFiniteMatcher +} + +#[derive(MatcherBase)] +pub struct IsFiniteMatcher; + +impl Matcher for IsFiniteMatcher { + fn matches(&self, actual: T) -> MatcherResult { + actual.is_finite().into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + if matcher_result.into() { "is Finite" } else { "isn't Finite" }.into() + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + + #[test] + fn matches_f32_number() -> Result<()> { + verify_that!(0.0f32, is_finite()) + } + + #[test] + fn does_not_match_f32_pos_infinity() -> Result<()> { + verify_that!(f32::INFINITY, not(is_finite())) + } + + #[test] + fn does_not_match_f32_neg_infinity() -> Result<()> { + verify_that!(f32::NEG_INFINITY, not(is_finite())) + } + + #[test] + fn does_not_match_f32_nan() -> Result<()> { + verify_that!(f32::NAN, not(is_finite())) + } + + #[test] + fn matches_f64_number() -> Result<()> { + verify_that!(0.0f64, is_finite()) + } + + #[test] + fn does_not_match_f64_pos_infinity() -> Result<()> { + verify_that!(f64::INFINITY, not(is_finite())) + } + + #[test] + fn does_not_match_f64_neg_infinity() -> Result<()> { + verify_that!(f64::NEG_INFINITY, not(is_finite())) + } + + #[test] + fn does_not_match_f64_nan() -> Result<()> { + verify_that!(f64::NAN, not(is_finite())) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/is_infinite_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/is_infinite_matcher.rs new file mode 100644 index 0000000..f97aeff --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/is_infinite_matcher.rs @@ -0,0 +1,84 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use num_traits::float::Float; +use std::fmt::Debug; + +/// Matches a floating point value which is Infinite. +pub fn is_infinite() -> IsInfiniteMatcher { + IsInfiniteMatcher +} + +#[derive(MatcherBase)] +pub struct IsInfiniteMatcher; + +impl Matcher for IsInfiniteMatcher { + fn matches(&self, actual: T) -> MatcherResult { + actual.is_infinite().into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + if matcher_result.into() { "is Infinite" } else { "isn't Infinite" }.into() + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + + #[test] + fn matches_f32_pos_infinity() -> Result<()> { + verify_that!(f32::INFINITY, is_infinite()) + } + + #[test] + fn matches_f32_neg_infinity() -> Result<()> { + verify_that!(f32::NEG_INFINITY, is_infinite()) + } + + #[test] + fn does_not_match_f32_number() -> Result<()> { + verify_that!(0.0f32, not(is_infinite())) + } + + #[test] + fn does_not_match_f32_nan() -> Result<()> { + verify_that!(f32::NAN, not(is_finite())) + } + + #[test] + fn matches_f64_pos_infinity() -> Result<()> { + verify_that!(f64::INFINITY, is_infinite()) + } + + #[test] + fn matches_f64_neg_infinity() -> Result<()> { + verify_that!(f64::NEG_INFINITY, is_infinite()) + } + + #[test] + fn does_not_match_f64_number() -> Result<()> { + verify_that!(0.0f64, not(is_infinite())) + } + + #[test] + fn does_not_match_f64_nan() -> Result<()> { + verify_that!(f64::NAN, not(is_finite())) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/is_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/is_matcher.rs new file mode 100644 index 0000000..0429986 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/is_matcher.rs @@ -0,0 +1,65 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc(hidden)] + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches precisely values matched by `inner`. +/// +/// The returned matcher produces a description prefixed by the string +/// `description`. This is useful in contexts where the test assertion failure +/// output must include the additional description. +pub fn is(description: &str, inner: InnerMatcherT) -> IsMatcher<'_, InnerMatcherT> { + IsMatcher { description, inner } +} + +#[derive(MatcherBase)] +pub struct IsMatcher<'a, InnerMatcherT> { + description: &'a str, + inner: InnerMatcherT, +} + +impl> Matcher + for IsMatcher<'_, InnerMatcherT> +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + self.inner.matches(actual) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!( + "is {} which {}", + self.description, + self.inner.describe(MatcherResult::Match) + ) + .into(), + MatcherResult::NoMatch => format!( + "is not {} which {}", + self.description, + self.inner.describe(MatcherResult::Match) + ) + .into(), + } + } + + fn explain_match(&self, actual: ActualT) -> Description { + self.inner.explain_match(actual) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/is_nan_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/is_nan_matcher.rs new file mode 100644 index 0000000..9a35081 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/is_nan_matcher.rs @@ -0,0 +1,64 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use num_traits::float::Float; +use std::fmt::Debug; + +/// Matches a floating point value which is NaN. +pub fn is_nan() -> IsNanMatcher { + IsNanMatcher +} + +#[derive(MatcherBase)] +pub struct IsNanMatcher; + +impl Matcher for IsNanMatcher { + fn matches(&self, actual: T) -> MatcherResult { + actual.is_nan().into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + if matcher_result.into() { "is NaN" } else { "isn't NaN" }.into() + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + + #[test] + fn matches_f32_nan() -> Result<()> { + verify_that!(f32::NAN, is_nan()) + } + + #[test] + fn does_not_match_f32_number() -> Result<()> { + verify_that!(0.0f32, not(is_nan())) + } + + #[test] + fn matches_f64_nan() -> Result<()> { + verify_that!(f64::NAN, is_nan()) + } + + #[test] + fn does_not_match_f64_number() -> Result<()> { + verify_that!(0.0f64, not(is_nan())) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/le_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/le_matcher.rs new file mode 100644 index 0000000..a355e63 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/le_matcher.rs @@ -0,0 +1,218 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a value less than or equal to (in the sense of `<=`) `expected`. +/// +/// The types of `ActualT` of `actual` and `ExpectedT` of `expected` must be +/// comparable via the `PartialOrd` trait. Namely, `ActualT` must implement +/// `PartialOrd`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(0, le(0))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(1, le(0))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// In most cases the params neeed to be the same type or they need to be cast +/// explicitly. This can be surprising when comparing integer types or +/// references: +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// verify_that!(1u32, le(2u64))?; // Does not compile +/// verify_that!(1u32 as u64, le(2u64))?; // Passes +/// # Ok(()) +/// # } +/// ``` +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// let actual: &u32 = &1; +/// let expected: u32 = 2; +/// verify_that!(actual, le(expected))?; // Does not compile +/// # Ok(()) +/// # } +/// ``` +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let actual: &u32 = &1; +/// let expected: u32 = 2; +/// verify_that!(actual, le(&expected))?; // Compiles and passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// You can find the standard library `PartialOrd` implementation in +/// +pub fn le(expected: ExpectedT) -> LeMatcher { + LeMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct LeMatcher { + expected: ExpectedT, +} + +impl + Copy, ExpectedT: Debug> Matcher + for LeMatcher +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + (actual <= self.expected).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!("is less than or equal to {:?}", self.expected).into(), + MatcherResult::NoMatch => format!("is greater than {:?}", self.expected).into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::ffi::OsString; + + #[test] + fn le_matches_i32_with_i32() -> Result<()> { + let actual: i32 = 0; + let expected: i32 = 0; + verify_that!(actual, le(expected)) + } + + #[test] + fn le_does_not_match_bigger_i32() -> Result<()> { + let matcher = le(0); + let result = matcher.matches(1); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn le_matches_smaller_str() -> Result<()> { + verify_that!("A", le("B")) + } + + #[test] + fn le_does_not_match_bigger_str() -> Result<()> { + let matcher = le("a"); + let result = matcher.matches("z"); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn le_mismatch_contains_actual_and_expected() -> Result<()> { + let result = verify_that!(489, le(294)); + let formatted_message = format!("{}", result.unwrap_err()); + + verify_that!( + formatted_message.as_str(), + contains_substring(indoc!( + " + Value of: 489 + Expected: is less than or equal to 294 + Actual: 489, + which is greater than 294 + " + )) + ) + } + + // Test `le` matcher where actual is `&OsString` and expected is `&str`. + // Note that stdlib is a little bit inconsistent: `PartialOrd` exists for + // `OsString` and `str`, but only in one direction: it's only possible to + // compare `OsString` with `str` if `OsString` is on the left side of the + // "<=" operator (`impl PartialOrd for OsString`). + // + // The comparison in the other direction is not defined. + // + // This means that the test case bellow effectively ensures that + // `verify_that(actual, le(expected))` works if `actual <= expected` works + // (regardless whether the `expected <= actual` works`). + #[test] + fn le_matches_owned_osstring_reference_with_string_reference() -> Result<()> { + let expected = "B"; + let actual: OsString = "A".into(); + verify_that!(&actual, le(expected)) + } + + #[test] + fn le_matches_ipv6addr_with_ipaddr() -> Result<()> { + use std::net::IpAddr; + use std::net::Ipv6Addr; + let actual: IpAddr = "127.0.0.1".parse().unwrap(); + let expected: Ipv6Addr = "2001:4860:4860::8844".parse().unwrap(); + verify_that!(actual, le(expected)) + } + + #[test] + fn le_matches_with_custom_partial_ord() -> Result<()> { + /// A custom "number" that is lower than all other numbers. The only + /// things we define about this "special" number is `PartialOrd` and + /// `PartialEq` against `u32`. + #[derive(Debug, Clone, Copy)] + struct VeryLowNumber {} + + impl std::cmp::PartialEq for VeryLowNumber { + fn eq(&self, _other: &u32) -> bool { + false + } + } + + // PartialOrd (required for >) requires PartialEq. + impl std::cmp::PartialOrd for VeryLowNumber { + fn partial_cmp(&self, _other: &u32) -> Option { + Some(std::cmp::Ordering::Less) + } + } + + impl std::cmp::PartialEq for u32 { + fn eq(&self, _other: &VeryLowNumber) -> bool { + false + } + } + + impl std::cmp::PartialOrd for u32 { + fn partial_cmp(&self, _other: &VeryLowNumber) -> Option { + Some(std::cmp::Ordering::Greater) + } + } + + let actual = VeryLowNumber {}; + let expected: u32 = 42; + + verify_that!(actual, le(expected)) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/len_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/len_matcher.rs new file mode 100644 index 0000000..709df87 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/len_matcher.rs @@ -0,0 +1,209 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use crate::matcher_support::count_elements::count_elements; +use std::fmt::Debug; + +/// Matches a container whose number of elements matches `expected`. +/// +/// This matches against a container over which one can iterate. This includes +/// the standard Rust containers, arrays, and slices. More +/// precisely, the actual type must implement [`IntoIterator`]. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let array = [1,2,3]; +/// verify_that!(array, len(eq(3)))?; +/// let vec = vec![1,2,3]; +/// verify_that!(vec, len(eq(3)))?; +/// let slice = vec.as_slice(); +/// verify_that!(*slice, len(eq(3)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// The parameter `expected` can be any integer numeric matcher. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let vec = vec![1,2,3]; +/// verify_that!(vec, len(gt(1)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +pub fn len(expected: E) -> LenMatcher { + LenMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct LenMatcher { + expected: E, +} + +impl> Matcher for LenMatcher +where + T: IntoIterator, +{ + fn matches(&self, actual: T) -> MatcherResult { + self.expected.matches(count_elements(actual)) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("has length, which {}", self.expected.describe(MatcherResult::Match)).into() + } + MatcherResult::NoMatch => { + format!("has length, which {}", self.expected.describe(MatcherResult::NoMatch)) + .into() + } + } + } + + fn explain_match(&self, actual: T) -> Description { + let actual_size = count_elements(actual); + format!("which has length {}, {}", actual_size, self.expected.explain_match(actual_size)) + .into() + } +} + +#[cfg(test)] +mod tests { + use crate::description::Description; + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::collections::{ + BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet, LinkedList, VecDeque, + }; + use std::fmt::Debug; + + #[test] + fn len_matcher_match_vec() -> Result<()> { + let value = vec![1, 2, 3]; + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_array_reference() -> Result<()> { + let value = &[1, 2, 3]; + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_slice_of_array() -> Result<()> { + let value = &[1, 2, 3]; + verify_that!(value[0..1], len(eq(1))) + } + + #[test] + fn len_matcher_match_slice_of_vec() -> Result<()> { + let value = vec![1, 2, 3]; + let slice = value.as_slice(); + verify_that!(slice, len(eq(3))) + } + + #[test] + fn len_matcher_match_sized_slice() -> Result<()> { + let value = [1, 2, 3]; + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_btreemap() -> Result<()> { + let value = BTreeMap::from([(1, 2), (2, 3), (3, 4)]); + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_btreeset() -> Result<()> { + let value = BTreeSet::from([1, 2, 3]); + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_binaryheap() -> Result<()> { + let value = BinaryHeap::from([1, 2, 3]); + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_hashmap() -> Result<()> { + let value = HashMap::from([(1, 2), (2, 3), (3, 4)]); + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_hashset() -> Result<()> { + let value = HashSet::from([1, 2, 3]); + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_linkedlist() -> Result<()> { + let value = LinkedList::from([1, 2, 3]); + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_match_vecdeque() -> Result<()> { + let value = VecDeque::from([1, 2, 3]); + verify_that!(value, len(eq(3))) + } + + #[test] + fn len_matcher_explain_match() -> Result<()> { + #[derive(MatcherBase)] + struct TestMatcher; + impl Matcher for TestMatcher { + fn matches(&self, _: T) -> MatcherResult { + false.into() + } + + fn describe(&self, _: MatcherResult) -> Description { + "called described".into() + } + + fn explain_match(&self, _: T) -> Description { + "called explain_match".into() + } + } + verify_that!( + len(TestMatcher).explain_match([1, 2, 3]), + displays_as(eq("which has length 3, called explain_match")) + ) + } + + #[test] + fn len_matcher_error_message() -> Result<()> { + let result = verify_that!(vec![1, 2, 3, 4], len(eq(3))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 2, 3, 4] + Expected: has length, which is equal to 3 + Actual: [1, 2, 3, 4], + which has length 4, which isn't equal to 3" + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/lt_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/lt_matcher.rs new file mode 100644 index 0000000..dc5e36d --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/lt_matcher.rs @@ -0,0 +1,227 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a value less (in the sense of `<`) than `expected`. +/// +/// The types of `ActualT` of `actual` and `ExpectedT` of `expected` must be +/// comparable via the `PartialOrd` trait. Namely, `ActualT` must implement +/// `PartialOrd`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(1, lt(2))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(2, lt(2))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// In most cases the params neeed to be the same type or they need to be cast +/// explicitly. This can be surprising when comparing integer types or +/// references: +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// verify_that!(123u32, lt(0u64))?; // Does not compile +/// verify_that!(123u32 as u64, lt(100000000u64))?; // Passes +/// # Ok(()) +/// # } +/// ``` +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// # fn should_not_compile() -> Result<()> { +/// let actual: &u32 = &2; +/// let expected: u32 = 70; +/// verify_that!(actual, lt(expected))?; // Does not compile +/// # Ok(()) +/// # } +/// ``` +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let actual: &u32 = &2; +/// let expected: u32 = 70; +/// verify_that!(actual, lt(&expected))?; // Compiles and passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// You can find the standard library `PartialOrd` implementation in +/// +pub fn lt(expected: ExpectedT) -> LtMatcher { + LtMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct LtMatcher { + expected: ExpectedT, +} + +impl + Copy, ExpectedT: Debug> Matcher + for LtMatcher +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + (actual < self.expected).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!("is less than {:?}", self.expected).into(), + MatcherResult::NoMatch => { + format!("is greater than or equal to {:?}", self.expected).into() + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::ffi::OsString; + + #[test] + fn lt_matches_i32_with_i32() -> Result<()> { + let actual: i32 = 10000; + let expected: i32 = 20000; + verify_that!(actual, lt(expected)) + } + + #[test] + fn lt_does_not_match_equal_i32() -> Result<()> { + let matcher = lt(10); + let result = matcher.matches(10); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn lt_does_not_match_lower_i32() -> Result<()> { + let matcher = lt(-50); + let result = matcher.matches(50); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn lt_matches_lesser_str() -> Result<()> { + verify_that!("A", lt("B")) + } + + #[test] + fn lt_does_not_match_bigger_str() -> Result<()> { + let matcher = lt("ab"); + let result = matcher.matches("az"); + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn lt_mismatch_contains_actual_and_expected() -> Result<()> { + let result = verify_that!(481, lt(45)); + let formatted_message = format!("{}", result.unwrap_err()); + + verify_that!( + formatted_message.as_str(), + contains_substring(indoc!( + " + Value of: 481 + Expected: is less than 45 + Actual: 481, + which is greater than or equal to 45 + " + )) + ) + } + + // Test `lt` matcher where actual is `&OsString` and expected is `&str`. + // Note that stdlib is a little bit inconsistent: `PartialOrd` exists for + // `OsString` and `str`, but only in one direction: it's only possible to + // compare `OsString` with `str` if `OsString` is on the left side of the + // "<" operator (`impl PartialOrd for OsString`). + // + // The comparison in the other direction is not defined. + // + // This means that the test case bellow effectively ensures that + // `verify_that(actual, lt(expected))` works if `actual < expected` works + // (regardless whether the `expected < actual` works`). + #[test] + fn lt_matches_owned_osstring_reference_with_string_reference() -> Result<()> { + let expected = "C"; + let actual: OsString = "B".to_string().into(); + verify_that!(&actual, lt(expected)) + } + + #[test] + fn lt_matches_ipv6addr_with_ipaddr() -> Result<()> { + use std::net::IpAddr; + use std::net::Ipv6Addr; + let actual: IpAddr = "127.0.0.1".parse().unwrap(); + let expected: Ipv6Addr = "2001:4860:4860::8844".parse().unwrap(); + verify_that!(actual, lt(expected)) + } + + #[test] + fn lt_matches_with_custom_partial_ord() -> Result<()> { + /// A custom "number" that is smaller than all other numbers. The only + /// things we define about this "special" number is `PartialOrd` and + /// `PartialEq` against `u32`. + #[derive(Debug, Clone, Copy)] + struct VeryLowNumber {} + + impl std::cmp::PartialEq for VeryLowNumber { + fn eq(&self, _other: &u32) -> bool { + false + } + } + + // PartialOrd (required for >) requires PartialEq. + impl std::cmp::PartialOrd for VeryLowNumber { + fn partial_cmp(&self, _other: &u32) -> Option { + Some(std::cmp::Ordering::Less) + } + } + + impl std::cmp::PartialEq for u32 { + fn eq(&self, _other: &VeryLowNumber) -> bool { + false + } + } + + impl std::cmp::PartialOrd for u32 { + fn partial_cmp(&self, _other: &VeryLowNumber) -> Option { + Some(std::cmp::Ordering::Greater) + } + } + + let actual = VeryLowNumber {}; + let expected: u32 = 42; + + verify_that!(actual, lt(expected)) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/matches_pattern.rs b/third_party/crates/googletest-0.14.3/src/matchers/matches_pattern.rs new file mode 100644 index 0000000..1ae2cdd --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/matches_pattern.rs @@ -0,0 +1,523 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented in the matchers module. +#![doc(hidden)] + +/// Matches a value according to a pattern of matchers. +/// +/// This takes as an argument a specification similar to a struct or enum +/// initialiser, where each value is a [`Matcher`][crate::matcher::Matcher] +/// which is applied to the corresponding field. +/// +/// This can be used to match arbitrary combinations of fields on structures +/// using arbitrary matchers: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct MyStruct { +/// a_field: String, +/// another_field: String, +/// } +/// +/// let my_struct = MyStruct { +/// a_field: "Something to believe in".into(), +/// another_field: "Something else".into() +/// }; +/// verify_that!(my_struct, matches_pattern!(MyStruct { +/// a_field: starts_with("Something"), +/// another_field: ends_with("else"), +/// })) +/// # .unwrap(); +/// ``` +/// +/// If any fields are provided in the pattern, then all fields must be +/// specified, or the pattern must end with `..`, just like regular match +/// patterns. Omitted fields have no effect on the output of the matcher. +/// The `..` is unnecessary when no fields are provided and only method +/// values are checked. +/// +/// ``` +/// # use googletest::prelude::*; +/// # #[derive(Debug)] +/// # struct MyStruct { +/// # a_field: String, +/// # another_field: String, +/// # } +/// # +/// # let my_struct = MyStruct { +/// # a_field: "Something to believe in".into(), +/// # another_field: "Something else".into() +/// # }; +/// verify_that!(my_struct, matches_pattern!(MyStruct { +/// a_field: starts_with("Something"), +/// .. // another_field is missing, so it may be anything. +/// })) +/// # .unwrap(); +/// ``` +/// +/// One can use it recursively to match nested structures: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct MyStruct { +/// a_nested_struct: MyInnerStruct, +/// } +/// +/// #[derive(Debug)] +/// struct MyInnerStruct { +/// a_field: String, +/// } +/// +/// let my_struct = MyStruct { +/// a_nested_struct: MyInnerStruct { a_field: "Something to believe in".into() }, +/// }; +/// verify_that!(my_struct, matches_pattern!(MyStruct { +/// a_nested_struct: matches_pattern!(MyInnerStruct { +/// a_field: starts_with("Something"), +/// }), +/// })) +/// # .unwrap(); +/// ``` +/// +/// One can use the alias [`pat`][crate::matchers::pat] to make this less +/// verbose: +/// +/// ``` +/// # use googletest::prelude::*; +/// # #[derive(Debug)] +/// # struct MyStruct { +/// # a_nested_struct: MyInnerStruct, +/// # } +/// # +/// # #[derive(Debug)] +/// # struct MyInnerStruct { +/// # a_field: String, +/// # } +/// # +/// # let my_struct = MyStruct { +/// # a_nested_struct: MyInnerStruct { a_field: "Something to believe in".into() }, +/// # }; +/// verify_that!(my_struct, matches_pattern!(MyStruct { +/// a_nested_struct: pat!(MyInnerStruct { +/// a_field: starts_with("Something"), +/// }), +/// })) +/// # .unwrap(); +/// ``` +/// +/// In addition to fields, one can match on the outputs of methods +/// ("properties"): +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct MyStruct { +/// a_field: String, +/// } +/// +/// impl MyStruct { +/// fn get_a_field(&self) -> String { self.a_field.clone() } +/// } +/// +/// let my_struct = MyStruct { a_field: "Something to believe in".into() }; +/// verify_that!(my_struct, matches_pattern!(MyStruct { +/// get_a_field(): starts_with("Something"), +/// })) +/// # .unwrap(); +/// ``` +/// +/// If an inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct MyStruct { +/// a_field: String, +/// another_field: String, +/// } +/// +/// let my_struct = MyStruct { +/// a_field: "this".into(), +/// another_field: "that".into() +/// }; +/// verify_that!(my_struct, matches_pattern!(MyStruct { +/// a_field: "this", +/// another_field: "that", +/// })) +/// # .unwrap(); +/// ``` +/// +/// **Important**: The method should be pure function with a deterministic +/// output and no side effects. In particular, in the event of an assertion +/// failure, it will be invoked a second time, with the assertion failure output +/// reflecting the *second* invocation. +/// +/// These may also include extra litteral parameters you pass in: +/// +/// ``` +/// # use googletest::prelude::*; +/// # #[derive(Debug)] +/// # struct MyStruct { +/// # a_field: String, +/// # } +/// # +/// impl MyStruct { +/// fn append_to_a_field(&self, suffix: &str) -> String { self.a_field.clone() + suffix } +/// } +/// +/// # let my_struct = MyStruct { a_field: "Something to believe in".into() }; +/// verify_that!(my_struct, matches_pattern!(&MyStruct { +/// append_to_a_field("a suffix"): ref ends_with("a suffix"), +/// })) +/// # .unwrap(); +/// ``` +/// +/// You can precede both field and property matchers with a `ref` to match the +/// result by reference: +/// +/// ``` +/// # use googletest::prelude::*; +/// # #[derive(Debug)] +/// # struct MyStruct { +/// # a_field: String, +/// # } +/// # +/// impl MyStruct { +/// fn get_a_field_ref(&self) -> String { self.a_field.clone() } +/// } +/// +/// # let my_struct = MyStruct { a_field: "Something to believe in".into() }; +/// verify_that!(my_struct, matches_pattern!(&MyStruct { +/// get_a_field_ref(): ref starts_with("Something"), +/// })) +/// # .unwrap(); +/// ``` +/// +/// Note that if the `actual` is of type `&ActualT` and the pattern type is +/// `ActualT`, this is automatically performed. This behavior is similar to the +/// reference binding mode in pattern matching. +/// +/// ``` +/// # use googletest::prelude::*; +/// # #[derive(Debug)] +/// # struct MyStruct { +/// # a_field: String, +/// # } +/// # +/// impl MyStruct { +/// fn get_a_field_ref(&self) -> String { self.a_field.clone() } +/// } +/// +/// # let my_struct = MyStruct { a_field: "Something to believe in".into() }; +/// verify_that!(my_struct, matches_pattern!(MyStruct { +/// get_a_field_ref(): starts_with("Something"), +/// })) +/// # .unwrap(); +/// ``` +/// +/// One can also match tuple structs with up to 10 fields. In this case, all +/// fields must have matchers: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// struct MyTupleStruct(String, String); +/// +/// let my_struct = MyTupleStruct("Something".into(), "Some other thing".into()); +/// verify_that!( +/// my_struct, +/// matches_pattern!(&MyTupleStruct(ref eq("Something"), ref eq("Some other thing"))) +/// ) +/// # .unwrap(); +/// ``` +/// +/// The macro also allows matching on specific enum values and supports wildcard +/// patterns like `MyEnum::Case(_)`. +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// enum MyEnum { +/// A(u32), +/// B, +/// } +/// +/// # fn should_pass() -> Result<()> { +/// verify_that!(MyEnum::A(123), matches_pattern!(&MyEnum::A(eq(123))))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_pass_with_wildcard() -> Result<()> { +/// verify_that!(MyEnum::A(123), matches_pattern!(MyEnum::A(_)))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(MyEnum::B, matches_pattern!(&MyEnum::A(eq(123))))?; // Fails - wrong enum variant +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_pass_with_wildcard().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// This macro does not support plain (non-struct) tuples. But it should not be +/// necessary as tuple of matchers are matchers of tuple. In other words, if +/// `MatcherU: Matcher` and `MatcherT: Matcher`, then `(MatcherU, +/// MatcherT): Matcher<(U, T)>`. +/// +/// Trailing commas are allowed (but not required) in both ordinary and tuple +/// structs. +/// +/// Note that the default format (rustfmt) can format macros if the macro +/// argument is parseable Rust code. This is mostly true for this macro with two +/// exceptions: +/// * property matching +/// * `ref` keyword with named fields +/// +/// An option for formatting large is to avoid these exceptions (by removing the +/// parenthesis of properties and the `ref` keywords), run `rustfmt` and add +/// them back. +#[macro_export] +#[doc(hidden)] +macro_rules! __matches_pattern { + ($($t:tt)*) => { $crate::matches_pattern_internal!($($t)*) } +} + +// Internal-only macro created so that the macro definition does not appear in +// generated documentation. +#[doc(hidden)] +#[macro_export] +macro_rules! matches_pattern_internal { + ($($tt:tt)*) => { + { + use $crate::{self as googletest}; + #[allow(unused)] + use $crate::matchers::{all, field, property}; + $crate::matchers::__internal_unstable_do_not_depend_on_these::__googletest_macro_matches_pattern!($($tt)*) + } + }; +} + +/// An alias for [`matches_pattern`][crate::matchers::matches_pattern!]. +#[macro_export] +#[doc(hidden)] +macro_rules! __pat { + ($($t:tt)*) => { $crate::matches_pattern_internal!($($t)*) } +} + +#[doc(hidden)] +pub mod internal { + use crate::matcher::{Matcher, MatcherBase}; + use std::fmt::Debug; + + pub use ::googletest_macro::__googletest_macro_matches_pattern; + + // Specialized implementation of the `predicate` matcher to support ref binding + // mode for `matches_pattern`. + pub fn pattern_only( + matcher_function: fn(&T) -> bool, + match_description: &'static str, + no_match_description: &'static str, + ) -> PatternOnlyMatcher { + PatternOnlyMatcher { matcher_function, match_description, no_match_description } + } + + #[derive(MatcherBase)] + #[doc(hidden)] + pub struct PatternOnlyMatcher { + matcher_function: fn(&T) -> bool, + match_description: &'static str, + no_match_description: &'static str, + } + + impl<'a, T: Debug> Matcher<&'a T> for PatternOnlyMatcher { + fn matches(&self, actual: &'a T) -> crate::matcher::MatcherResult { + (self.matcher_function)(actual).into() + } + + fn describe( + &self, + matcher_result: crate::matcher::MatcherResult, + ) -> crate::description::Description { + match matcher_result { + crate::matcher::MatcherResult::Match => self.match_description.into(), + crate::matcher::MatcherResult::NoMatch => self.no_match_description.into(), + } + } + } + + impl Matcher for PatternOnlyMatcher { + fn matches(&self, actual: T) -> crate::matcher::MatcherResult { + (self.matcher_function)(&actual).into() + } + + fn describe( + &self, + matcher_result: crate::matcher::MatcherResult, + ) -> crate::description::Description { + match matcher_result { + crate::matcher::MatcherResult::Match => self.match_description.into(), + crate::matcher::MatcherResult::NoMatch => self.no_match_description.into(), + } + } + } + + /// A matcher that ensures that the passed-in function compiles with the + /// benefit of inference from the value being tested. The passed-in + /// matcher is what is actually invoked for matching. + /// + /// It forwards all description responsibilities to the passed-in matcher. + pub fn compile_assert_and_match( + must_compile_function: fn(&T), + matcher: M, + ) -> CompileAssertAndMatch { + CompileAssertAndMatch { must_compile_function, matcher } + } + + #[derive(MatcherBase)] + #[doc(hidden)] + pub struct CompileAssertAndMatch { + #[allow(dead_code)] + must_compile_function: fn(&T), + matcher: M, + } + + impl<'a, T: Debug, M> Matcher<&'a T> for CompileAssertAndMatch + where + M: Matcher<&'a T>, + { + fn matches(&self, actual: &'a T) -> crate::matcher::MatcherResult { + self.matcher.matches(actual) + } + + fn describe( + &self, + matcher_result: crate::matcher::MatcherResult, + ) -> crate::description::Description { + self.matcher.describe(matcher_result) + } + + fn explain_match(&self, actual: &'a T) -> crate::description::Description { + self.matcher.explain_match(actual) + } + } + + impl Matcher for CompileAssertAndMatch + where + M: Matcher, + { + fn matches(&self, actual: T) -> crate::matcher::MatcherResult { + self.matcher.matches(actual) + } + + fn describe( + &self, + matcher_result: crate::matcher::MatcherResult, + ) -> crate::description::Description { + self.matcher.describe(matcher_result) + } + + fn explain_match(&self, actual: T) -> crate::description::Description { + self.matcher.explain_match(actual) + } + } +} + +mod compile_fail_tests { + /// ```compile_fail + /// use ::googletest::prelude::*; + /// #[derive(Debug)] + /// struct Foo { a: u32 } + /// impl Foo { + /// fn b() {} + /// } + /// let actual = Foo { a: 1 }; + /// verify_that!(actual, matches_pattern!(Foo { a: eq(&1), b(): _ })); + /// ``` + fn _underscore_unsupported_for_methods() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// #[derive(Debug)] + /// struct Foo { a: u32, b: u32 } + /// let actual = Foo { a: 1, b: 2 }; + /// verify_that!(actual, matches_pattern!(Foo { a: eq(&1), .., })); + /// ``` + fn _dot_dot_supported_only_at_end_of_struct_pattern() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// #[derive(Debug)] + /// struct Foo(u32, u32); + /// let actual = Foo(1, 2); + /// verify_that!(actual, matches_pattern!(Foo(eq(&1), .., ))); + /// ``` + fn _dot_dot_supported_only_at_end_of_tuple_struct_pattern() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// #[derive(Debug)] + /// struct Foo { a: u32, b: u32 } + /// let actual = Foo { a: 1, b: 2 }; + /// verify_that!(actual, matches_pattern!(Foo { a: eq(&1) })); + /// ``` + fn _unexhaustive_struct_field_check_requires_dot_dot() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// #[derive(Debug)] + /// enum Foo { + /// Bar { a: u32, b: u32 } + /// } + /// let actual = Foo::Bar { a: 1, b: 2 }; + /// verify_that!(actual, matches_pattern!(Foo::Bar { a: eq(&1) })); + /// ``` + fn _unexhaustive_enum_struct_field_check_requires_dot_dot() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// #[derive(Debug)] + /// struct Foo(u32, u32, u32); + /// let actual = Foo(1, 2, 3); + /// verify_that!(actual, matches_pattern!(Foo(eq(&1), eq(&2) ))); + /// ``` + fn _unexhaustive_tuple_struct_field_check_requires_dot_dot() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// verify_that!(1, matches_pattern!(UndefinedSymbol { })); + /// ``` + fn _should_fail_to_compile_unknown_struct_with_no_fields() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// verify_that!(1, matches_pattern!(UndefinedSymbol { a: 1 })); + /// ``` + fn _should_fail_to_compile_unknown_struct_with_field() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// verify_that!(1, matches_pattern!(UndefinedSymbol { .. })); + /// ``` + fn _should_fail_to_compile_unknown_struct_with_dot_dot() {} + + /// ```compile_fail + /// use ::googletest::prelude::*; + /// verify_that!(1, matches_pattern!(UndefinedSymbol( .. ))); + /// ``` + fn _should_fail_to_compile_unknown_tuple_struct_with_dot_dot() {} +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/matches_regex_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/matches_regex_matcher.rs new file mode 100644 index 0000000..10f190a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/matches_regex_matcher.rs @@ -0,0 +1,203 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use regex::Regex; +use std::fmt::Debug; +use std::ops::Deref; + +/// Matches a string the entirety of which which matches the given regular +/// expression. +/// +/// This is similar to [`contains_regex`][crate::matchers::contains_regex], +/// except that the match must cover the whole string and not a substring. +/// +/// Both the actual value and the expected regular expression may be either a +/// `String` or a string reference. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass_1() -> Result<()> { +/// verify_that!("Some value", matches_regex("S.*e"))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!("Another value", matches_regex("Some"))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!("Some value", matches_regex("Some"))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_pass_2() -> Result<()> { +/// verify_that!("Some value".to_string(), matches_regex(".*v.*e"))?; // Passes +/// verify_that!("Some value", matches_regex(".*v.*e".to_string()))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_pass_2().unwrap(); +/// ``` +/// +/// Panics if the given `pattern` is not a syntactically valid regular +/// expression. +// N.B. This returns the concrete type rather than an impl Matcher so that it +// can act simultaneously as a Matcher and a Matcher. Otherwise the +// compiler treats it as a Matcher only and the code +// verify_that!("Some value".to_string(), matches_regex(".*value"))?; +// doesn't compile. +pub fn matches_regex>( + pattern: PatternT, +) -> MatchesRegexMatcher { + let adjusted_pattern = format!("^{}$", pattern.deref()); + let regex = Regex::new(adjusted_pattern.as_str()).unwrap(); + MatchesRegexMatcher { regex, pattern, _adjusted_pattern: adjusted_pattern } +} + +/// A matcher matching a string-like type matching a given regular expression. +/// +/// Intended only to be used from the function [`matches_regex`] only. +/// Should not be referenced by code outside this library. +#[derive(MatcherBase)] +pub struct MatchesRegexMatcher> { + regex: Regex, + pattern: PatternT, + _adjusted_pattern: String, +} + +impl Matcher for MatchesRegexMatcher +where + PatternT: Deref, + ActualT: AsRef + Debug + Copy, +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + self.regex.is_match(actual.as_ref()).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("matches the regular expression {:#?}", self.pattern.deref()).into() + } + MatcherResult::NoMatch => { + format!("doesn't match the regular expression {:#?}", self.pattern.deref()).into() + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + + #[test] + fn matches_regex_matches_string_reference_with_pattern() -> Result<()> { + let matcher = matches_regex("S.*e"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn matches_regex_does_not_match_string_without_pattern() -> Result<()> { + let matcher = matches_regex("Another"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn matches_regex_does_not_match_string_only_beginning_of_which_matches() -> Result<()> { + let matcher = matches_regex("Some"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn matches_regex_does_not_match_string_only_end_of_which_matches() -> Result<()> { + let matcher = matches_regex("value"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn matches_regex_matches_owned_string_with_pattern() -> Result<()> { + let matcher = matches_regex(".*value"); + + let result = matcher.matches(&"Some value".to_string()); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn matches_regex_matches_string_when_regex_has_beginning_of_string_marker() -> Result<()> { + let matcher = matches_regex("^Some value"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn matches_regex_matches_string_when_regex_has_end_of_string_marker() -> Result<()> { + let matcher = matches_regex("Some value$"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn matches_regex_matches_string_when_regex_has_both_end_markers() -> Result<()> { + let matcher = matches_regex("^Some value$"); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn matches_regex_matches_string_reference_with_owned_string() -> Result<()> { + let matcher = matches_regex(".*value".to_string()); + + let result = matcher.matches("Some value"); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn verify_that_works_with_owned_string() -> Result<()> { + verify_that!("Some value".to_string(), matches_regex(".*value")) + } + + #[test] + fn matches_regex_displays_quoted_debug_of_pattern() -> Result<()> { + let matcher = matches_regex("\n"); + + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("matches the regular expression \"\\n\"")) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/mod.rs b/third_party/crates/googletest-0.14.3/src/matchers/mod.rs new file mode 100644 index 0000000..975dd2b --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/mod.rs @@ -0,0 +1,130 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! All built-in matchers of this crate are in submodules of this module. + +mod all_matcher; +mod any_matcher; +mod anything_matcher; +mod bool_matcher; +mod char_count_matcher; +mod conjunction_matcher; +mod container_eq_matcher; +mod contains_matcher; +mod contains_regex_matcher; +mod derefs_to_matcher; +mod disjunction_matcher; +mod display_matcher; +mod each_matcher; +mod elements_are_matcher; +mod empty_matcher; +mod eq_matcher; +mod err_matcher; +mod field_matcher; +mod ge_matcher; +mod gt_matcher; +mod has_entry_matcher; +mod is_encoded_string_matcher; +mod is_finite_matcher; +mod is_infinite_matcher; +mod is_matcher; +mod is_nan_matcher; +mod le_matcher; +mod len_matcher; +mod lt_matcher; +mod matches_pattern; +mod matches_regex_matcher; +mod ne_matcher; +mod near_matcher; +mod none_matcher; +mod not_matcher; +mod ok_matcher; +mod points_to_matcher; +mod pointwise_matcher; +mod predicate_matcher; +mod property_matcher; +mod result_of_matcher; +mod some_matcher; +mod str_matcher; +mod subset_of_matcher; +mod superset_of_matcher; +mod tuple_matcher; +mod unordered_elements_are_matcher; + +pub use anything_matcher::anything; +pub use bool_matcher::{is_false, is_true}; +pub use char_count_matcher::char_count; +pub use container_eq_matcher::container_eq; +pub use contains_matcher::{contains, ContainsMatcher}; +pub use contains_regex_matcher::contains_regex; +pub use derefs_to_matcher::derefs_to; +pub use display_matcher::displays_as; +pub use each_matcher::each; +pub use empty_matcher::is_empty; +pub use eq_matcher::{eq, EqMatcher}; +pub use err_matcher::err; +pub use ge_matcher::ge; +pub use gt_matcher::gt; +pub use has_entry_matcher::has_entry; +pub use is_encoded_string_matcher::is_utf8_string; +pub use is_finite_matcher::is_finite; +pub use is_infinite_matcher::is_infinite; +pub use is_nan_matcher::is_nan; +pub use le_matcher::le; +pub use len_matcher::len; +pub use lt_matcher::lt; +pub use matches_regex_matcher::matches_regex; +pub use ne_matcher::ne; +pub use near_matcher::{approx_eq, near, NearMatcher}; +pub use none_matcher::none; +pub use not_matcher::not; +pub use ok_matcher::ok; +pub use points_to_matcher::points_to; +pub use predicate_matcher::{predicate, PredicateMatcher}; +pub use some_matcher::some; +pub use str_matcher::{ + contains_substring, ends_with, starts_with, StrMatcher, StrMatcherConfigurator, +}; +pub use subset_of_matcher::subset_of; +pub use superset_of_matcher::superset_of; + +// Reexport and unmangle the macros. +#[doc(inline)] +pub use crate::{ + __all as all, __any as any, __contains_each as contains_each, __elements_are as elements_are, + __field as field, __is_contained_in as is_contained_in, __matches_pattern as matches_pattern, + __pat as pat, __pointwise as pointwise, __property as property, __result_of as result_of, + __result_of_ref as result_of_ref, __unordered_elements_are as unordered_elements_are, +}; + +// Types and functions used by macros matchers. +// Do not use directly. +// We may perform incompatible changes without major release. These elements +// should only be used through their respective macros. +#[doc(hidden)] +pub mod __internal_unstable_do_not_depend_on_these { + pub use super::conjunction_matcher::ConjunctionMatcher; + pub use super::disjunction_matcher::DisjunctionMatcher; + pub use super::elements_are_matcher::internal::ElementsAre; + pub use super::field_matcher::internal::field_matcher; + pub use super::is_matcher::is; + pub use super::matches_pattern::internal::{ + __googletest_macro_matches_pattern, compile_assert_and_match, pattern_only, + }; + pub use super::pointwise_matcher::internal::PointwiseMatcher; + pub use super::property_matcher::internal::{property_matcher, property_ref_matcher}; + pub use super::result_of_matcher::internal::{result_of, result_of_ref}; + pub use super::unordered_elements_are_matcher::internal::UnorderedElementsAreMatcher; + pub use crate::matcher_support::match_matrix::internal::Requirements; +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/ne_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/ne_matcher.rs new file mode 100644 index 0000000..66b2474 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/ne_matcher.rs @@ -0,0 +1,108 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a value is not equal (in the sense of `!=`) to `expected`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(0, ne(1))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(0, ne(0))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +pub fn ne(expected: T) -> NEMatcher { + NEMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct NEMatcher { + expected: T, +} + +impl> Matcher for NEMatcher { + fn matches(&self, actual: A) -> MatcherResult { + (actual != self.expected).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!("is not equal to {:?}", self.expected).into(), + MatcherResult::NoMatch => format!("is equal to {:?}", self.expected).into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn ne_matches_i32_with_i32() -> Result<()> { + verify_that!(123, ne(234)) + } + + #[test] + fn ne_matches_string_reference_with_string_reference() -> Result<()> { + verify_that!("A string", ne("B string")) + } + + #[test] + fn ne_matches_owned_string_with_string_reference() -> Result<()> { + let value = "A string".to_string(); + verify_that!(value, ne("B string")) + } + + #[test] + fn ne_matches_owned_string_reference_with_string_reference() -> Result<()> { + let value = "A string".to_string(); + verify_that!(&value, ne("B string")) + } + + #[test] + fn ne_struct_debug_diff() -> Result<()> { + #[derive(Debug, PartialEq)] + struct Strukt { + int: i32, + string: String, + } + + let result = verify_that!( + Strukt { int: 123, string: "something".into() }, + ne(&Strukt { int: 123, string: "something".into() }) + ); + verify_that!( + result, + err(displays_as(contains_substring(indoc! { + " + Expected: is not equal to Strukt { int: 123, string: \"something\" } + Actual: Strukt { int: 123, string: \"something\" }, + which is equal to Strukt { int: 123, string: \"something\" } + "}))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/near_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/near_matcher.rs new file mode 100644 index 0000000..1713e75 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/near_matcher.rs @@ -0,0 +1,363 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use num_traits::{Float, FloatConst}; +use std::{borrow::Borrow, fmt::Debug}; + +/// Matches a value equal within `max_abs_error` of `expected`. +/// +/// The type `T` of the actual, `expected`, and `max_abs_error` values must +/// implement [`Float`]. +/// +/// The values `expected` and `max_abs_error` may not be NaN. The value +/// `max_abs_error` must be non-negative. The matcher panics on construction +/// otherwise. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass_1() -> Result<()> { +/// verify_that!(1.0, near(1.0, 0.1))?; // Passes +/// verify_that!(1.01, near(1.0, 0.1))?; // Passes +/// verify_that!(1.25, near(1.0, 0.25))?; // Passes +/// verify_that!(0.75, near(1.0, 0.25))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!(1.101, near(1.0, 0.1))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!(0.899, near(1.0, 0.1))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_pass_2() -> Result<()> { +/// verify_that!(100.25, near(100.0, 0.25))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_pass_2().unwrap(); +/// ``` +/// +/// The default behaviour for special values is consistent with the IEEE +/// floating point standard. Thus infinity is infinitely far away from any +/// floating point value: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_fail_1() -> Result<()> { +/// verify_that!(f64::INFINITY, near(0.0, f64::MAX))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!(0.0, near(f64::INFINITY, f64::MAX))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_3() -> Result<()> { +/// verify_that!(f64::INFINITY, near(f64::INFINITY, f64::MAX))?; // Fails +/// # Ok(()) +/// # } +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_fail_3().unwrap_err(); +/// ``` +/// +/// Similarly, by default, `NaN` is infinitely far away from any value: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_fail_1() -> Result<()> { +/// verify_that!(f64::NAN, near(0.0, f64::MAX))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!(0.0, near(f64::NAN, f64::MAX))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_3() -> Result<()> { +/// verify_that!(f64::NAN, near(f64::NAN, f64::MAX))?; // Fails +/// # Ok(()) +/// # } +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_fail_3().unwrap_err(); +/// ``` +/// +/// To treat two `NaN` values as equal, use the method +/// [`NearMatcher::nans_are_equal`]. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(f64::NAN, near(f64::NAN, f64::MAX).nans_are_equal())?; // Passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +#[track_caller] +pub fn near(expected: T, max_abs_error: T) -> NearMatcher { + if max_abs_error.is_nan() { + panic!("max_abs_error must not be NaN"); + } + if max_abs_error < T::zero() { + panic!("max_abs_error must be non-negative"); + } + NearMatcher { expected, max_abs_error, nans_are_equal: false } +} + +/// Matches a value approximately equal to `expected`. +/// +/// This automatically computes a tolerance from the magnitude of `expected` and +/// matches any actual value within this tolerance of the expected value. The +/// tolerance is chosen to account for the inaccuracies in most ordinary +/// floating point calculations. +/// +/// Otherwise this works analogously to [`near`]; see its documentation for +/// further notes. +pub fn approx_eq(expected: T) -> NearMatcher { + // The FloatConst trait doesn't offer 2 as a constant but does offer 1. + let five_bits_of_mantissa = (T::one() + T::one()).powi(5); + let abs_tolerance = five_bits_of_mantissa * T::epsilon(); + let max_abs_error = T::max(expected.abs() * abs_tolerance, abs_tolerance); + NearMatcher { expected, max_abs_error, nans_are_equal: false } +} + +/// A matcher which matches floating-point numbers approximately equal to its +/// expected value. +#[derive(MatcherBase)] +pub struct NearMatcher { + expected: T, + max_abs_error: T, + nans_are_equal: bool, +} + +impl NearMatcher { + /// Configures this instance to treat two NaNs as equal. + /// + /// This behaviour differs from the IEEE standad for floating point which + /// treats two NaNs as infinitely far apart. + pub fn nans_are_equal(mut self) -> Self { + self.nans_are_equal = true; + self + } + + /// Configures this instance to treat two NaNs as not equal. + /// + /// This behaviour complies with the IEEE standad for floating point. It is + /// the default behaviour for this matcher, so invoking this method is + /// usually redunant. + pub fn nans_are_not_equal(mut self) -> Self { + self.nans_are_equal = false; + self + } +} + +impl + Debug + Copy, F: Debug + Float> Matcher for NearMatcher { + fn matches(&self, actual: T) -> MatcherResult { + if self.nans_are_equal && self.expected.is_nan() && actual.borrow().is_nan() { + return MatcherResult::Match; + } + + let delta = *actual.borrow() - self.expected; + if delta >= -self.max_abs_error && delta <= self.max_abs_error { + MatcherResult::Match + } else { + MatcherResult::NoMatch + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("is within {:?} of {:?}", self.max_abs_error, self.expected).into() + } + MatcherResult::NoMatch => { + format!("isn't within {:?} of {:?}", self.max_abs_error, self.expected).into() + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + + #[test] + fn matches_value_inside_range() -> Result<()> { + let matcher = near(1.0f64, 0.1f64); + + let result = matcher.matches(1.0f64); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn matches_value_at_low_end_of_range() -> Result<()> { + let matcher = near(1.0f64, 0.1f64); + + let result = matcher.matches(0.9f64); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn matches_value_at_high_end_of_range() -> Result<()> { + let matcher = near(1.0f64, 0.25f64); + + let result = matcher.matches(1.25f64); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn does_not_match_value_below_low_end_of_range() -> Result<()> { + let matcher = near(1.0f64, 0.1f64); + + let result = matcher.matches(0.899999f64); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn does_not_match_value_above_high_end_of_range() -> Result<()> { + let matcher = near(1.0f64, 0.1f64); + + let result = matcher.matches(1.100001f64); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn nan_is_not_near_a_number() -> Result<()> { + let matcher = near(0.0f64, f64::MAX); + + let result = matcher.matches(f64::NAN); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn nan_is_not_near_nan_by_default() -> Result<()> { + verify_that!(f64::NAN, not(near(f64::NAN, f64::MAX))) + } + + #[test] + fn nan_is_not_near_nan_when_explicitly_configured() -> Result<()> { + verify_that!(f64::NAN, not(near(f64::NAN, f64::MAX).nans_are_not_equal())) + } + + #[test] + fn nan_is_near_nan_if_nans_are_equal() -> Result<()> { + verify_that!(f64::NAN, near(f64::NAN, f64::MAX).nans_are_equal()) + } + + #[test] + fn nan_is_not_near_number_when_nans_are_equal() -> Result<()> { + verify_that!(f64::NAN, not(near(0.0, f64::MAX).nans_are_equal())) + } + + #[test] + fn number_is_not_near_nan_when_nans_are_equal() -> Result<()> { + verify_that!(0.0, not(near(f64::NAN, f64::MAX).nans_are_equal())) + } + + #[test] + fn inf_is_not_near_inf() -> Result<()> { + let matcher = near(f64::INFINITY, f64::MAX); + + let result = matcher.matches(f64::INFINITY); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn inf_is_not_near_a_number() -> Result<()> { + let matcher = near(f64::INFINITY, f64::MAX); + + let result = matcher.matches(f64::MIN); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn any_two_numbers_are_within_inf_of_each_other() -> Result<()> { + let matcher = near(f64::MIN, f64::INFINITY); + + let result = matcher.matches(f64::MAX); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[::core::prelude::v1::test] + #[should_panic] + fn panics_if_max_abs_error_is_nan() { + near(0.0, f64::NAN); + } + + #[::core::prelude::v1::test] + #[should_panic] + fn panics_if_tolerance_is_negative() { + near(0.0, -1.0); + } + + #[test] + fn approx_eq_matches_equal_number() -> Result<()> { + verify_that!(1.0f64, approx_eq(1.0f64)) + } + + #[test] + fn approx_eq_matches_really_close_f64_number() -> Result<()> { + verify_that!(1.0f64, approx_eq(1.0 + 16.0 * f64::EPSILON)) + } + + #[test] + fn approx_eq_matches_really_close_f64_number_to_large_number() -> Result<()> { + verify_that!(1000f64, approx_eq(1000.0 + 16000.0 * f64::EPSILON)) + } + + #[test] + fn approx_eq_matches_really_close_f64_number_to_zero() -> Result<()> { + verify_that!(16.0 * f64::EPSILON, approx_eq(0.0)) + } + + #[test] + fn approx_eq_matches_really_close_f32_number() -> Result<()> { + verify_that!(1.0f32, approx_eq(1.0 + 16.0 * f32::EPSILON)) + } + + #[test] + fn approx_eq_does_not_match_distant_number() -> Result<()> { + verify_that!(0.0f64, not(approx_eq(1.0f64))) + } + + #[test] + fn approx_eq_supports_ref() -> Result<()> { + verify_that!(&0.0f64, approx_eq(0.0f64)) + } + + #[test] + fn approx_eq_supports_container_matchers() -> Result<()> { + verify_that!( + vec![1., 2., 3.], + unordered_elements_are![approx_eq(2.), approx_eq(3.), approx_eq(1.)] + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/none_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/none_matcher.rs new file mode 100644 index 0000000..501bb4b --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/none_matcher.rs @@ -0,0 +1,129 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use std::fmt::Debug; + +/// Matches an `Option` containing `None`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(None::<()>, none())?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(Some("Some value"), none())?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +pub fn none() -> NoneMatcher { + NoneMatcher +} + +#[derive(MatcherBase)] +pub struct NoneMatcher; + +impl Matcher> for NoneMatcher { + fn matches(&self, actual: Option) -> MatcherResult { + actual.is_none().into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => "is none".into(), + MatcherResult::NoMatch => "is some(_)".into(), + } + } +} + +impl<'a, T: Debug> Matcher<&'a Option> for NoneMatcher { + fn matches(&self, actual: &'a Option) -> MatcherResult { + actual.is_none().into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => "is none".into(), + MatcherResult::NoMatch => "is some(_)".into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + + #[test] + fn none_matches_option_with_none() -> Result<()> { + let matcher = none(); + + let result = matcher.matches(None::); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn none_does_not_match_option_with_value() -> Result<()> { + let matcher = none(); + + let result = matcher.matches(Some(0)); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn none_matches_option_by_ref() -> Result<()> { + verify_that!(None::, none()) + } + #[test] + fn none_does_not_match_option_with_value_by_ref() -> Result<()> { + verify_that!(Some("123".to_string()), not(none())) + } + + #[test] + fn none_describe_match_option_by_ref() -> Result<()> { + verify_that!( + Matcher::<&Option>::describe(&none(), MatcherResult::Match), + displays_as(eq("is none")) + ) + } + #[test] + fn none_describe_no_match_option_by_ref() -> Result<()> { + verify_that!( + Matcher::<&Option>::describe(&none(), MatcherResult::NoMatch), + displays_as(eq("is some(_)")) + ) + } + + #[test] + fn none_describe_match_option() -> Result<()> { + verify_that!( + Matcher::>::describe(&none(), MatcherResult::Match), + displays_as(eq("is none")) + ) + } + #[test] + fn none_describe_no_match_option() -> Result<()> { + verify_that!( + Matcher::>::describe(&none(), MatcherResult::NoMatch), + displays_as(eq("is some(_)")) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/not_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/not_matcher.rs new file mode 100644 index 0000000..efe22d6 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/not_matcher.rs @@ -0,0 +1,105 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches the actual value exactly when the inner matcher does _not_ match. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(0, not(eq(1)))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!(0, not(eq(0)))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +pub fn not(inner: InnerMatcherT) -> NotMatcher { + NotMatcher { inner } +} + +#[derive(MatcherBase)] +pub struct NotMatcher { + inner: InnerMatcherT, +} + +impl> Matcher for NotMatcher { + fn matches(&self, actual: T) -> MatcherResult { + match self.inner.matches(actual) { + MatcherResult::Match => MatcherResult::NoMatch, + MatcherResult::NoMatch => MatcherResult::Match, + } + } + + fn explain_match(&self, actual: T) -> Description { + self.inner.explain_match(actual) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + self.inner.describe(if matcher_result.into() { + MatcherResult::NoMatch + } else { + MatcherResult::Match + }) + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn matches_when_inner_matcher_does_not_match() -> Result<()> { + let matcher = not(eq(1)); + + let result = matcher.matches(0); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn does_not_match_when_inner_matcher_matches() -> Result<()> { + let matcher = not(eq(1)); + + let result = matcher.matches(1); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn match_explanation_references_actual_value() -> Result<()> { + let result = verify_that!(&[1], not(container_eq([1]))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Actual: [1], + which contains all the elements + " + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/ok_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/ok_matcher.rs new file mode 100644 index 0000000..790b87c --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/ok_matcher.rs @@ -0,0 +1,291 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a `Result` containing `Ok` with a value matched by `inner`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> googletest::Result<()> { +/// verify_that!(Ok::<_, ()>("Some value"), ok(eq("Some value")))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> googletest::Result<()> { +/// verify_that!(Err::<&str, _>("An error"), ok(eq("An error")))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> googletest::Result<()> { +/// verify_that!(Ok::<_, ()>("Some value"), ok(eq("Some other value")))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// ``` +pub fn ok(inner: InnerMatcherT) -> OkMatcher { + OkMatcher { inner } +} + +#[derive(MatcherBase)] +pub struct OkMatcher { + inner: InnerMatcherT, +} + +impl> Matcher> + for OkMatcher +{ + fn matches(&self, actual: std::result::Result) -> MatcherResult { + actual.map(|v| self.inner.matches(v)).unwrap_or(MatcherResult::NoMatch) + } + + fn explain_match(&self, actual: std::result::Result) -> Description { + match actual { + Ok(o) => { + Description::new().text("which is a success").nested(self.inner.explain_match(o)) + } + Err(_) => "which is an error".into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!( + "is a success containing a value, which {}", + self.inner.describe(MatcherResult::Match) + ) + .into(), + MatcherResult::NoMatch => format!( + "is an error or a success containing a value, which {}", + self.inner.describe(MatcherResult::NoMatch) + ) + .into(), + } + } +} + +impl<'a, T: Debug, E: Debug, InnerMatcherT: Matcher<&'a T>> Matcher<&'a std::result::Result> + for OkMatcher +{ + fn matches(&self, actual: &'a std::result::Result) -> MatcherResult { + actual.as_ref().map(|v| self.inner.matches(v)).unwrap_or(MatcherResult::NoMatch) + } + + fn explain_match(&self, actual: &'a std::result::Result) -> Description { + match actual { + Ok(o) => { + Description::new().text("which is a success").nested(self.inner.explain_match(o)) + } + Err(_) => "which is an error".into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!( + "is a success containing a value, which {}", + self.inner.describe(MatcherResult::Match) + ) + .into(), + MatcherResult::NoMatch => format!( + "is an error or a success containing a value, which {}", + self.inner.describe(MatcherResult::NoMatch) + ) + .into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn ok_matches_result_with_value() -> Result<()> { + let matcher = ok(eq(1)); + let value: std::result::Result = Ok(1); + + let result = matcher.matches(value); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn ok_does_not_match_result_with_wrong_value() -> Result<()> { + let matcher = ok(eq(1)); + let value: std::result::Result = Ok(0); + + let result = matcher.matches(value); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn ok_does_not_match_result_with_err() -> Result<()> { + let matcher = ok(eq(1)); + let value: std::result::Result = Err(1); + + let result = matcher.matches(value); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn ok_matches_result_with_value_by_ref() -> Result<()> { + let result: std::result::Result = Ok("123".into()); + verify_that!(result, ok(eq("123"))) + } + + #[test] + fn ok_does_not_match_result_with_wrong_value_by_ref() -> Result<()> { + let result: std::result::Result = Ok("321".into()); + verify_that!(result, not(ok(eq("123")))) + } + + #[test] + fn ok_does_not_match_result_with_err_by_ref() -> Result<()> { + let result: std::result::Result = Err("123".into()); + verify_that!(result, not(ok(eq("123")))) + } + + #[test] + fn ok_full_error_message() -> Result<()> { + let result = verify_that!(Ok::(1), ok(eq(2))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: Ok::(1) + Expected: is a success containing a value, which is equal to 2 + Actual: Ok(1), + which is a success + which isn't equal to 2 + " + )))) + ) + } + + #[test] + fn ok_describe_match() -> Result<()> { + let matcher = ok(eq(1)); + verify_that!( + Matcher::>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is a success containing a value, which is equal to 1")) + ) + } + + #[test] + fn ok_describe_no_match() -> Result<()> { + let matcher = ok(eq(1)); + verify_that!( + Matcher::>::describe(&matcher, MatcherResult::NoMatch), + displays_as(eq("is an error or a success containing a value, which isn't equal to 1")) + ) + } + + #[test] + fn ok_describe_match_by_ref() -> Result<()> { + let matcher = ok(eq(&1)); + verify_that!( + Matcher::<&std::result::Result>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is a success containing a value, which is equal to 1")) + ) + } + + #[test] + fn ok_describe_no_match_by_ref() -> Result<()> { + let matcher = ok(eq(&1)); + verify_that!( + Matcher::<&std::result::Result>::describe( + &matcher, + MatcherResult::NoMatch + ), + displays_as(eq("is an error or a success containing a value, which isn't equal to 1")) + ) + } + + #[test] + fn ok_explain_match_ok_success() -> Result<()> { + let actual = Ok(1); + let matcher = ok(eq(1)); + + verify_that!( + Matcher::>::explain_match(&matcher, actual), + displays_as(eq("which is a success\n which is equal to 1")) + ) + } + + #[test] + fn ok_explain_match_ok_fail() -> Result<()> { + let actual = Ok(1); + let matcher = ok(eq(2)); + + verify_that!( + Matcher::>::explain_match(&matcher, actual), + displays_as(eq("which is a success\n which isn't equal to 2")) + ) + } + + #[test] + fn ok_explain_match_ok_err() -> Result<()> { + let actual = Err(1); + let matcher = ok(eq(2)); + + verify_that!( + Matcher::>::explain_match(&matcher, actual), + displays_as(eq("which is an error")) + ) + } + + #[test] + fn ok_explain_match_ok_success_by_ref() -> Result<()> { + let actual = Ok("123".to_string()); + let matcher = ok(eq("123")); + + verify_that!( + Matcher::<&std::result::Result>::explain_match(&matcher, &actual), + displays_as(eq("which is a success\n which is equal to \"123\"")) + ) + } + + #[test] + fn ok_explain_match_ok_fail_by_ref() -> Result<()> { + let actual = Ok("321".to_string()); + let matcher = ok(eq("123")); + + verify_that!( + Matcher::<&std::result::Result>::explain_match(&matcher, &actual), + displays_as(eq("which is a success\n which isn't equal to \"123\"")) + ) + } + + #[test] + fn ok_explain_match_ok_err_by_ref() -> Result<()> { + let actual = Err("123".to_string()); + let matcher = ok(eq("123")); + + verify_that!( + Matcher::<&std::result::Result>::explain_match(&matcher, &actual), + displays_as(eq("which is an error")) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/points_to_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/points_to_matcher.rs new file mode 100644 index 0000000..39e70ad --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/points_to_matcher.rs @@ -0,0 +1,88 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::description::Description; +use crate::matcher::{Matcher, MatcherBase, MatcherResult}; +use std::fmt::Debug; + +/// Matches a reference pointing to a value matched by the [`Matcher`] +/// `expected`. +/// +/// This is useful for combining matchers, especially when working with +/// iterators. +/// +/// For example: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(&123, points_to(eq(123)))?; +/// verify_that!(vec![1,2,3], each(points_to(gt(0))))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +pub fn points_to(expected: MatcherT) -> PointsToMatcher { + PointsToMatcher { expected } +} + +#[derive(MatcherBase)] +pub struct PointsToMatcher { + expected: MatcherT, +} + +impl<'a, ExpectedT, MatcherT> Matcher<&'a ExpectedT> for PointsToMatcher +where + ExpectedT: Debug + Copy, + MatcherT: Matcher, +{ + fn matches(&self, actual: &'a ExpectedT) -> MatcherResult { + self.expected.matches(*actual) + } + + fn explain_match(&self, actual: &'a ExpectedT) -> Description { + self.expected.explain_match(*actual) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + self.expected.describe(matcher_result) + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn points_to_matches_ref() -> Result<()> { + verify_that!(&123, points_to(eq(123))) + } + + #[test] + fn match_explanation_references_actual_value() -> Result<()> { + let result = verify_that!(&1, points_to(eq(0))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Actual: 1, + which isn't equal to 0 + " + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/pointwise_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/pointwise_matcher.rs new file mode 100644 index 0000000..a3a0d74 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/pointwise_matcher.rs @@ -0,0 +1,232 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented in the matcher module. +#![doc(hidden)] + +/// Generates a matcher which matches a container each of whose elements match +/// the given matcher name applied respectively to each element of the given +/// container. +/// +/// For example, the following matches a container of integers each of which +/// does not exceed the given upper bounds: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let value = vec![1, 2, 3]; +/// verify_that!(value, pointwise!(le, [&1, &2, &3]))?; // Passes +/// verify_that!(value, pointwise!(|e| points_to(le(e)), [1, 2, 3]))?; // Passes +/// verify_that!(value, pointwise!(|e| points_to(le(e)), vec![1, 3, 3]))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// # let value = vec![1, 2, 3]; +/// verify_that!(value, pointwise!(|e| points_to(le(e)), [1, 1, 3]))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail().unwrap_err(); +/// ``` +/// +/// One can also use a closure which returns a matcher: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let value = vec![1.00001, 2.000001, 3.00001]; +/// verify_that!(value, pointwise!(|v| points_to(near(v, 0.001)), [1.0, 2.0, 3.0]))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// One can pass up to three containers to supply arguments to the function +/// creating the matcher: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let value = vec![1.00001, 2.000001, 3.00001]; +/// verify_that!(value, pointwise!(|v, t| points_to(near(v, t)), [1.0, 2.0, 3.0], [0.001, 0.0001, 0.01]))?; +/// verify_that!( +/// value, +/// pointwise!( +/// |v, t, u| points_to(near(v, t * u)), +/// [1.0, 2.0, 3.0], +/// [0.001, 0.0001, 0.01], +/// [0.5, 0.5, 1.0] +/// ) +/// )?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// When using `pointwise!` with multiple containers, the caller must ensure +/// that all of the containers have the same size. This matcher does not check +/// whether the sizes match. +/// +/// The actual value must be a container such as a `&Vec`, an array, or a +/// slice. More precisely, the actual value must implement [`IntoIterator`]. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let value = vec![1, 2, 3]; +/// verify_that!(value, pointwise!(|i| points_to(le(i)), [1, 3, 3]))?; // Passes +/// verify_that!([1, 2, 3], pointwise!(le, [1, 3, 3]))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// The second argument can be any value implementing `IntoIterator`, such as a +/// `Vec` or an array. The container does not have to have the same type as the +/// actual value, but the value type must be the same. +/// +/// **Note for users of the [`Pointwise`] matcher in C++ GoogleTest:** +/// +/// This macro differs from `Pointwise` in that the first parameter is not a +/// matcher which matches a pair but rather the name of a function of one +/// argument whose output is a matcher. This means that one can use standard +/// matchers like `eq`, `le`, and so on with `pointwise!` but certain C++ tests +/// using `Pointwise` will require some extra work to port. +/// +/// [`IntoIterator`]: std::iter::IntoIterator +/// [`Iterator`]: std::iter::Iterator +/// [`Iterator::collect`]: std::iter::Iterator::collect +/// [`Pointwise`]: https://google.github.io/googletest/reference/matchers.html#container-matchers +/// [`Vec`]: std::vec::Vec +#[macro_export] +#[doc(hidden)] +macro_rules! __pointwise { + ($matcher:expr, $container:expr) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::PointwiseMatcher::new( + $container.into_iter().map($matcher).collect(), + ) + }}; + + ($matcher:expr, $left_container:expr, $right_container:expr) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::PointwiseMatcher::new( + $left_container + .into_iter() + .zip($right_container.into_iter()) + .map(|(l, r)| $matcher(l, r)) + .collect(), + ) + }}; + + ($matcher:expr, $left_container:expr, $middle_container:expr, $right_container:expr) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::PointwiseMatcher::new( + $left_container + .into_iter() + .zip($right_container.into_iter().zip($middle_container.into_iter())) + .map(|(l, (m, r))| $matcher(l, m, r)) + .collect(), + ) + }}; +} + +/// Module for use only by the procedural macros in this module. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use crate::description::Description; + use crate::matcher::{Matcher, MatcherBase, MatcherResult}; + use crate::matcher_support::zipped_iterator::zip; + use std::fmt::Debug; + + /// This struct is meant to be used only through the `pointwise` macro. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + #[derive(MatcherBase)] + pub struct PointwiseMatcher { + matchers: Vec, + } + + impl PointwiseMatcher { + pub fn new(matchers: Vec) -> Self { + Self { matchers } + } + } + + impl, ContainerT: Copy + Debug> Matcher + for PointwiseMatcher + where + ContainerT: IntoIterator, + { + fn matches(&self, actual: ContainerT) -> MatcherResult { + let mut zipped_iterator = zip(actual.into_iter(), self.matchers.iter()); + for (element, matcher) in zipped_iterator.by_ref() { + if matcher.matches(element).is_no_match() { + return MatcherResult::NoMatch; + } + } + if zipped_iterator.has_size_mismatch() { + MatcherResult::NoMatch + } else { + MatcherResult::Match + } + } + + fn explain_match(&self, actual: ContainerT) -> Description { + // TODO(b/260819741) This code duplicates elements_are_matcher.rs. Consider + // extract as a separate library. (or implement pointwise! with + // elements_are) + let actual_iterator = actual.into_iter(); + let mut zipped_iterator = zip(actual_iterator, self.matchers.iter()); + let mut mismatches = Vec::new(); + for (idx, (a, e)) in zipped_iterator.by_ref().enumerate() { + if e.matches(a).is_no_match() { + mismatches.push(format!("element #{idx} is {a:?}, {}", e.explain_match(a))); + } + } + if mismatches.is_empty() { + if !zipped_iterator.has_size_mismatch() { + "which matches all elements".into() + } else { + format!( + "which has size {} (expected {})", + zipped_iterator.left_size(), + self.matchers.len() + ) + .into() + } + } else if mismatches.len() == 1 { + format!("where {}", mismatches[0]).into() + } else { + let mismatches = mismatches.into_iter().collect::(); + format!("where:\n{}", mismatches.bullet_list().indent()).into() + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + format!( + "{} elements satisfying respectively:\n{}", + if matcher_result.into() { "has" } else { "doesn't have" }, + self.matchers + .iter() + .map(|m| m.describe(MatcherResult::Match)) + .collect::() + .enumerate() + .indent() + ) + .into() + } + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/predicate_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/predicate_matcher.rs new file mode 100644 index 0000000..13799e6 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/predicate_matcher.rs @@ -0,0 +1,215 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Creates a matcher based on the predicate provided. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(3, predicate(|x: i32| x % 2 == 1))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// The predicate should take the subject type and return a +/// boolean. +/// +/// Note: even if the Rust compiler should be able to infer the type of +/// the closure argument, it is likely that it won't. +/// See for update on this issue. +/// This is easily fixed by explicitly declaring the type of the argument +pub fn predicate

(predicate: P) -> PredicateMatcher { + PredicateMatcher { + predicate, + positive_description: NoDescription, + negative_description: NoDescription, + } +} + +impl

PredicateMatcher { + /// Configures this instance to provide a more meaningful description. + /// + /// For example, to make sure the error message is more useful + /// + /// ``` + /// # use googletest::matchers::{predicate, PredicateMatcher}; + /// # let _ = + /// predicate(|x: i32| x % 2 == 1) + /// .with_description("is odd", "is even") + /// # ; + /// ``` + /// + /// This is optional as it only provides value when the test fails. + /// + /// Description can be passed by `&str`, `String` or `Fn() -> Into`. + pub fn with_description( + self, + positive_description: D1, + negative_description: D2, + ) -> PredicateMatcher { + PredicateMatcher { predicate: self.predicate, positive_description, negative_description } + } +} + +/// A matcher which applies `predicate` on the value. +/// +/// See [`predicate`]. +#[derive(MatcherBase)] +pub struct PredicateMatcher { + predicate: P, + positive_description: D1, + negative_description: D2, +} + +/// A trait to allow [`PredicateMatcher::with_description`] to accept multiple +/// types. +/// +/// See [`PredicateMatcher::with_description`] +pub trait PredicateDescription { + fn to_description(&self) -> Description; +} + +impl PredicateDescription for &str { + fn to_description(&self) -> Description { + self.to_string().into() + } +} + +impl PredicateDescription for String { + fn to_description(&self) -> Description { + self.to_string().into() + } +} + +impl PredicateDescription for T +where + T: Fn() -> S, + S: Into, +{ + fn to_description(&self) -> Description { + self().into().into() + } +} + +// Sentinel type to tag a MatcherBuilder as without a description. +#[doc(hidden)] +pub struct NoDescription; + +impl Matcher for PredicateMatcher +where + P: Fn(T) -> bool, +{ + fn matches(&self, actual: T) -> MatcherResult { + (self.predicate)(actual).into() + } + + fn describe(&self, result: MatcherResult) -> Description { + match result { + MatcherResult::Match => "matches".into(), + MatcherResult::NoMatch => "does not match".into(), + } + } +} + +impl Matcher + for PredicateMatcher +where + P: Fn(T) -> bool, +{ + fn matches(&self, actual: T) -> MatcherResult { + (self.predicate)(actual).into() + } + + fn describe(&self, result: MatcherResult) -> Description { + match result { + MatcherResult::Match => self.positive_description.to_description(), + MatcherResult::NoMatch => self.negative_description.to_description(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + + // Simple matcher with a description + fn is_odd() -> impl Matcher { + predicate(|x| x % 2 == 1).with_description("is odd", "is even") + } + + #[test] + fn predicate_matcher_odd() -> Result<()> { + verify_that!(1, is_odd()) + } + + #[test] + fn predicate_matcher_odd_explain_match_matches() -> Result<()> { + verify_that!(is_odd().explain_match(1), displays_as(eq("which is odd"))) + } + + #[test] + fn predicate_matcher_odd_explain_match_does_not_match() -> Result<()> { + verify_that!(is_odd().explain_match(2), displays_as(eq("which is even"))) + } + + // Simple Matcher without description + fn is_even() -> impl Matcher { + predicate(|x| x % 2 == 0) + } + + #[test] + fn predicate_matcher_even() -> Result<()> { + verify_that!(2, is_even()) + } + + #[test] + fn predicate_matcher_even_explain_match_matches() -> Result<()> { + verify_that!(is_even().explain_match(2), displays_as(eq("which matches"))) + } + + #[test] + fn predicate_matcher_even_explain_match_does_not_match() -> Result<()> { + verify_that!(is_even().explain_match(1), displays_as(eq("which does not match"))) + } + + #[test] + fn predicate_matcher_generator_lambda() -> Result<()> { + let is_divisible_by = |quotient| { + predicate(move |x: i32| x % quotient == 0).with_description( + move || format!("is divisible by {quotient}"), + move || format!("is not divisible by {quotient}"), + ) + }; + verify_that!(49, is_divisible_by(7)) + } + + #[test] + fn predicate_matcher_inline() -> Result<()> { + verify_that!(2048, predicate(|x: i32| x.count_ones() == 1)) + } + + #[test] + fn predicate_matcher_function_pointer() -> Result<()> { + use std::time::Duration; + verify_that!(&Duration::new(0, 0), predicate(Duration::is_zero)) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/property_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/property_matcher.rs new file mode 100644 index 0000000..c77b726 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/property_matcher.rs @@ -0,0 +1,384 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented in the matcher module. +#![doc(hidden)] + +/// Matches an object which, upon calling the given method on it with the given +/// arguments, produces a value matched by the given inner matcher. +/// +/// This is particularly useful as a nested matcher when the desired +/// property cannot be accessed through a field and must instead be +/// extracted through a method call. For example: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// pub struct MyStruct { +/// a_field: u32, +/// } +/// +/// impl MyStruct { +/// pub fn get_a_field(&self) -> u32 { self.a_field } +/// } +/// +/// let value = vec![MyStruct { a_field: 100 }]; +/// verify_that!(value, contains(property!(&MyStruct.get_a_field(), eq(100)))) +/// # .unwrap(); +/// ``` +/// +/// +/// If the inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// pub struct MyStruct { +/// a_field: u32, +/// } +/// +/// impl MyStruct { +/// pub fn get_a_field(&self) -> u32 { self.a_field } +/// } +/// +/// let value = vec![MyStruct { a_field: 100 }]; +/// verify_that!(value, contains(property!(&MyStruct.get_a_field(), 100))) +/// # .unwrap(); +/// ``` +/// +/// **Important**: The method should be pure function with a deterministic +/// output and no side effects. In particular, in the event of an assertion +/// failure, it will be invoked a second time, with the assertion failure output +/// reflecting the *second* invocation. +/// +/// The method may also take additional litteral arguments: +/// +/// ``` +/// # use googletest::prelude::*; +/// # #[derive(Debug)] +/// # pub struct MyStruct { +/// # a_field: u32, +/// # } +/// impl MyStruct { +/// pub fn add_to_a_field(&self, a: u32) -> u32 { self.a_field + a } +/// } +/// +/// # let value = vec![MyStruct { a_field: 100 }]; +/// verify_that!(value, contains(property!(&MyStruct.add_to_a_field(50), eq(150)))) +/// # .unwrap(); +/// ``` +/// +/// The arguments must be litteral as `property!` is not able to capture them. +/// +/// # Specification of the property pattern +/// +/// The specification of the field follow the syntax: `(ref)? (&)? +/// $TYPE.$PROPERTY\($ARGUMENT\)`. +/// +/// The `&` allows to specify whether this matcher matches against an actual of +/// type `$TYPE` (`$TYPE` must implement `Copy`) or a `&$TYPE`. +/// +/// For instance: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// pub struct AStruct; +/// +/// impl AStruct { +/// fn a_property(&self) -> i32 {32} +/// } +/// # fn should_pass() -> Result<()> { +/// verify_that!(AStruct, property!(&AStruct.a_property(), eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug, Clone, Copy)] +/// pub struct AStruct; +/// +/// impl AStruct { +/// fn a_property(self) -> i32 {32} +/// } +/// # fn should_pass() -> Result<()> { +/// verify_that!(AStruct, property!(AStruct.a_property(), eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// The `ref` allows to bind the property returned value by reference, which is +/// required if the field type does not implement `Copy`. +/// +/// For instance: +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// pub struct AStruct; +/// +/// impl AStruct { +/// fn a_property(&self) -> i32 {32} +/// } +/// # fn should_pass() -> Result<()> { +/// verify_that!(AStruct, property!(&AStruct.a_property(), eq(32)))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// If `property!` is qualified by both `&` and `ref`, they can both be omitted. +/// +/// ``` +/// # use googletest::prelude::*; +/// #[derive(Debug)] +/// pub struct AStruct; +/// +/// impl AStruct { +/// fn a_property(&self) -> String {"32".into()} +/// } +/// # fn should_pass() -> Result<()> { +/// verify_that!(AStruct, property!(&AStruct.a_property(), ref eq("32")))?; +/// verify_that!(AStruct, property!(AStruct.a_property(), eq("32")))?; +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// This macro is analogous to [`field`][crate::matchers::field], except that it +/// extracts the datum to be matched from the given object by invoking a method +/// rather than accessing a field. +/// +/// The list of arguments may optionally have a trailing comma. +#[macro_export] +#[doc(hidden)] +macro_rules! __property { + ($($t:tt)*) => { $crate::property_internal!($($t)*) } +} + +// Internal-only macro created so that the macro definition does not appear in +// generated documentation. +#[doc(hidden)] +#[macro_export] +macro_rules! property_internal { + // `ref` variant. + ( + // The 3 cases of `$(& $($_amp:literal)?)?` -> `$(& $($_amp)*)*` + // 1. outer group captures nothing => expansion produces nothing + // 2. outer group captures just `&` => expansion produces `&` + // 3. outer group captures `& ` => disallowed by `@assert_empty` subrule invocation + // + // `$_amp:literal` works only because the following `$t:ident` or `::` can't be captured by + // it. + $(& $($_amp:literal)?)? // Optional `&`'s presence is implicitly captured by `_amp`. + $(:: $($_cs:literal)?)? // Optional `::`'s presence is implicitly captured by `_cs`. + $($t:ident)::+ $(::<$($t_ty_args:ty),* $(,)?>)? + .$method:tt $(::<$($m_ty_args:ty),* $(,)?>)? ($($argument:expr),* $(,)?), + ref $m:expr + ) => {{ + $crate::property_internal!(@assert_empty $($($_amp)*)* $($($_cs)*)*); + $crate::property_internal!( + @self_arg + struct_type: [$(& $($_amp)*)* $(:: $($_cs)*)* $($t)::+ $( <$($t_ty_args),*>)*] + method_prefix: [ $(:: $($_cs)*)* $($t)::+ $(::<$($t_ty_args),*>)*] + [$method $(::<$($m_ty_args),*>)*] [$($argument),*] [$m]) + }}; + + // Non-`ref` variant. + ( + // See comment on previous variant above. + $(& $($_amp:literal)?)? // Optional `&`'s presence is implicitly captured by `_amp`. + $(:: $($_cs:literal)?)? // Optional `::`'s presence is implicitly captured by `_cs`. + $($t:ident)::+ $(::<$($t_ty_args:ty),* $(,)?>)? + .$method:tt $(::<$($m_ty_args:ty),* $(,)?>)? ($($argument:expr),* $(,)?), + $m:expr) => {{ + $crate::property_internal!(@assert_empty $($($_amp)*)* $($($_cs)*)*); + $crate::property_internal!( + @self_dot + struct_type: [$(& $($_amp)*)* $(:: $($_cs)*)* $($t)::+ $(<$($t_ty_args),*>)*] + [$method $(::<$($m_ty_args),*>)*] [$($argument),*] [$m]) + }}; + + (@assert_empty) => {}; + (@assert_empty $($l:literal)+) => { + compile_error!("property! argument must start with an optional `&` followed by a path") + }; + + (@self_arg struct_type: [$struct_ty:ty] + method_prefix: [$($method_prefix:tt)+] + [$($method:tt)*] [$($argument:expr),*] [$m:expr]) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::property_ref_matcher( + |o: $struct_ty| $($method_prefix)*::$($method)* (o, $($argument),*), + &stringify!($($method)* ($($argument),*)), + $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq!($m)) + }}; + + (@self_dot struct_type: [$struct_ty:ty] + [$($method:tt)*] [$($argument:expr),*] [$m:expr]) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::property_matcher( + |o: &$struct_ty| o.$($method)* ($($argument),*), + &stringify!($($method)* ($($argument),*)), + $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq!($m)) + }}; +} + +/// Items for use only by the declarative macros in this module. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, + }; + use std::fmt::Debug; + + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + pub fn property_matcher( + extractor: fn(&OuterT) -> InnerT, + property_desc: &'static str, + inner: MatcherT, + ) -> PropertyMatcher { + PropertyMatcher { extractor, property_desc, inner } + } + + #[derive(MatcherBase)] + pub struct PropertyMatcher { + extractor: fn(&OuterT) -> InnerT, + property_desc: &'static str, + inner: MatcherT, + } + + impl Matcher for PropertyMatcher + where + InnerT: Debug + Copy, + OuterT: Debug + Copy, + MatcherT: Matcher, + { + fn matches(&self, actual: OuterT) -> MatcherResult { + self.inner.matches((self.extractor)(&actual)) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + format!( + "has property `{}`, which {}", + self.property_desc, + self.inner.describe(matcher_result) + ) + .into() + } + + fn explain_match(&self, actual: OuterT) -> Description { + let actual_inner = (self.extractor)(&actual); + format!( + "whose property `{}` is `{:#?}`, {}", + self.property_desc, + actual_inner, + self.inner.explain_match(actual_inner) + ) + .into() + } + } + + impl<'a, InnerT, OuterT, MatcherT> Matcher<&'a OuterT> for PropertyMatcher + where + InnerT: Debug, + OuterT: Debug, + MatcherT: for<'b> Matcher<&'b InnerT>, + { + fn matches(&self, actual: &'a OuterT) -> MatcherResult { + self.inner.matches(&(self.extractor)(actual)) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + format!( + "has property `{}`, which {}", + self.property_desc, + self.inner.describe(matcher_result) + ) + .into() + } + + fn explain_match(&self, actual: &'a OuterT) -> Description { + let actual_inner = (self.extractor)(actual); + format!( + "whose property `{}` is `{:#?}`, {}", + self.property_desc, + actual_inner, + self.inner.explain_match(&actual_inner) + ) + .into() + } + } + + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + pub fn property_ref_matcher( + extractor: ExtractorT, + property_desc: &'static str, + inner: MatcherT, + ) -> PropertyRefMatcher + where + OuterT: Debug, + InnerT: Debug, + MatcherT: for<'a> Matcher<&'a InnerT>, + ExtractorT: Fn(OuterT) -> InnerT, + { + PropertyRefMatcher { extractor, property_desc, inner } + } + + #[derive(MatcherBase)] + pub struct PropertyRefMatcher { + extractor: ExtractorT, + property_desc: &'static str, + inner: MatcherT, + } + + impl< + InnerT: Debug, + OuterT: Debug + Copy, + MatcherT: for<'a> Matcher<&'a InnerT>, + ExtractorT: Fn(OuterT) -> InnerT, + > Matcher for PropertyRefMatcher + { + fn matches(&self, actual: OuterT) -> MatcherResult { + self.inner.matches(&(self.extractor)(actual)) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + format!( + "has property `{}`, which {}", + self.property_desc, + self.inner.describe(matcher_result) + ) + .into() + } + + fn explain_match(&self, actual: OuterT) -> Description { + let actual_inner = (self.extractor)(actual); + format!( + "whose property `{}` is `{:#?}`, {}", + self.property_desc, + actual_inner, + self.inner.explain_match(&actual_inner) + ) + .into() + } + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/result_of_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/result_of_matcher.rs new file mode 100644 index 0000000..b4f8a64 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/result_of_matcher.rs @@ -0,0 +1,441 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Matches a value where the result of `callable` applied to the value matches +/// the inner matcher. +/// +/// The `callable` will be called twice, so make sure it is pure. +/// ``` +/// use googletest::prelude::*; +/// fn should_pass() -> googletest::Result<()> { +/// verify_that!(100, result_of!(|value| value + 1, eq(101)))?; // Passes +/// Ok(()) +/// } +/// +/// fn should_fail() -> googletest::Result<()> { +/// verify_that!(100, result_of!(|value| value * 2, eq(100)))?; // Fails +/// Ok(()) +/// } +/// should_pass().unwrap(); +/// should_fail().unwrap_err(); +/// ``` +#[macro_export] +macro_rules! __result_of { + ($function: expr, $matcher: expr) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::result_of( + $function, + $matcher, + stringify!($function), + ) + }}; +} + +/// Matches a value where the reference to the result of `callable` applied to +/// the value matches the inner matcher. +/// +/// The `callable` will be called twice, so make sure it is pure. +/// ``` +/// use googletest::prelude::*; +/// fn should_pass_1() -> googletest::Result<()> { +/// verify_that!("hello", result_of_ref!(|s: &str| s.to_uppercase(), eq("HELLO")))?; // Passes +/// Ok(()) +/// } +/// +/// fn should_pass_2() -> googletest::Result<()> { +/// verify_that!(100, result_of_ref!(|value| value + 1, eq(&101)))?; // Passes +/// Ok(()) +/// } +/// +/// fn should_fail() -> googletest::Result<()> { +/// verify_that!("world", result_of_ref!(|s: &str| s.to_uppercase(), eq("HELLO")))?; // Passes +/// Ok(()) +/// } +/// should_pass_1().unwrap(); +/// should_pass_2().unwrap(); +/// should_fail().unwrap_err(); +/// ``` +#[macro_export] +macro_rules! __result_of_ref { + ($function: expr, $matcher: expr) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these::result_of_ref( + $function, + $matcher, + stringify!($function), + ) + }}; +} + +/// Items for use only by the declarative macros in this module. +/// +/// **For internal use only. API stability is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use crate::description::Description; + use crate::matcher::{Matcher, MatcherBase, MatcherResult}; + use std::fmt::Debug; + + pub fn result_of( + callable: Callable, + inner_matcher: InnerMatcher, + callable_description: &'static str, + ) -> ResultOfMatcher { + ResultOfMatcher { callable, inner_matcher, callable_description } + } + + #[derive(MatcherBase)] + pub struct ResultOfMatcher { + callable: Callable, + inner_matcher: InnerMatcher, + callable_description: &'static str, + } + + impl T, InnerMatcherT: Matcher> + Matcher for ResultOfMatcher + { + fn matches(&self, actual: I) -> MatcherResult { + self.inner_matcher.matches((self.callable)(actual)) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + Description::new() + .text(format!("by applying {},", self.callable_description)) + .nested(self.inner_matcher.describe(matcher_result)) + } + + fn explain_match(&self, actual: I) -> Description { + let actual_result = (self.callable)(actual); + Description::new() + .text(format!("which, results into {actual_result:?}",)) + .nested(self.describe(self.matches(actual))) + } + } + + pub fn result_of_ref( + callable: Callable, + inner_matcher: InnerMatcher, + callable_description: &'static str, + ) -> ResultOfRefMatcher { + ResultOfRefMatcher { callable, inner_matcher, callable_description } + } + #[derive(MatcherBase)] + pub struct ResultOfRefMatcher { + callable: Callable, + inner_matcher: InnerMatcher, + callable_description: &'static str, + } + + impl< + I: Copy + Debug, + T: Debug, + Callable: Fn(I) -> T, + InnerMatcherT: for<'a> Matcher<&'a T>, + > Matcher for ResultOfRefMatcher + { + fn matches(&self, actual: I) -> MatcherResult { + self.inner_matcher.matches(&(self.callable)(actual)) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + Description::new() + .text(format!("by applying {},", self.callable_description)) + .nested(self.inner_matcher.describe(matcher_result)) + } + + fn explain_match(&self, actual: I) -> Description { + let actual_result = (self.callable)(actual); + Description::new() + .text(format!("which, results into {actual_result:?}",)) + .nested(self.describe(self.matches(actual))) + } + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn result_of_match_with_value() -> Result<()> { + verify_that!(1, result_of!(|value| value + 1, eq(2))) + } + + #[test] + fn result_of_match_with_value_function() -> Result<()> { + fn inc_by_one(value: i32) -> i32 { + value + 1 + } + verify_that!(1, result_of!(inc_by_one, eq(2))) + } + + #[test] + fn result_of_match_with_different_value() -> Result<()> { + let result = verify_that!(0, result_of!(|value| value - 1, eq(2))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 0 + Expected: by applying |value| value - 1, + is equal to 2 + Actual: 0, + which, results into -1 + by applying |value| value - 1, + isn't equal to 2 + " + )))) + ) + } + + #[test] + fn result_of_match_with_different_value_block_closure() -> Result<()> { + let result = verify_that!(0, result_of!(|value| { value - 1 }, eq(2))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 0 + Expected: by applying |value| { value - 1 }, + is equal to 2 + Actual: 0, + which, results into -1 + by applying |value| { value - 1 }, + isn't equal to 2 + " + )))) + ) + } + + #[test] + fn result_of_match_with_different_value_multiline_closure() -> Result<()> { + let result = verify_that!( + 0, + result_of!( + |value| { + let dec = value - 1; + let inc = dec + 1; + inc - 2 + }, + eq(2) + ) + ); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 0 + Expected: by applying |value| { let dec = value - 1; let inc = dec + 1; inc - 2 }, + is equal to 2 + Actual: 0, + which, results into -2 + by applying |value| { let dec = value - 1; let inc = dec + 1; inc - 2 }, + isn't equal to 2 + " + )))) + ) + } + + #[test] + fn result_of_match_with_different_value_function() -> Result<()> { + fn dec_by_one(value: i32) -> i32 { + value - 1 + } + let result = verify_that!(0, result_of!(dec_by_one, eq(2))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 0 + Expected: by applying dec_by_one, + is equal to 2 + Actual: 0, + which, results into -1 + by applying dec_by_one, + isn't equal to 2 + " + )))) + ) + } + + #[test] + fn result_of_ref_match_with_string_reference() -> Result<()> { + verify_that!("hello", result_of_ref!(|s: &str| s.to_uppercase(), eq("HELLO"))) + } + + #[test] + fn result_of_ref_match_with_string_reference_function() -> Result<()> { + fn to_upper_case>(s: S) -> String { + s.as_ref().to_uppercase() + } + verify_that!("hello", result_of_ref!(to_upper_case, eq("HELLO"))) + } + + #[test] + fn result_of_ref_match_with_copy_types() -> Result<()> { + verify_that!(100, result_of_ref!(|value| value + 1, eq(&101))) + } + + #[test] + fn result_of_ref_match_with_different_value() -> Result<()> { + let result = verify_that!("world", result_of_ref!(|s: &str| s.to_uppercase(), eq("HELLO"))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: "world" + Expected: by applying |s: &str| s.to_uppercase(), + is equal to "HELLO" + Actual: "world", + which, results into "WORLD" + by applying |s: &str| s.to_uppercase(), + isn't equal to "HELLO""# + )))) + ) + } + + #[test] + fn result_of_ref_match_with_different_value_block_closure() -> Result<()> { + let result = + verify_that!("world", result_of_ref!(|s: &str| { s.to_uppercase() }, eq("HELLO"))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: "world" + Expected: by applying |s: &str| { s.to_uppercase() }, + is equal to "HELLO" + Actual: "world", + which, results into "WORLD" + by applying |s: &str| { s.to_uppercase() }, + isn't equal to "HELLO" + "# + )))) + ) + } + + #[test] + fn result_of_ref_match_with_different_value_function() -> Result<()> { + fn to_upper_case>(s: S) -> String { + s.as_ref().to_uppercase() + } + let result = verify_that!("world", result_of_ref!(to_upper_case, eq("HELLO"))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: "world" + Expected: by applying to_upper_case, + is equal to "HELLO" + Actual: "world", + which, results into "WORLD" + by applying to_upper_case, + isn't equal to "HELLO" + "# + )))) + ) + } + + #[test] + fn result_of_ref_match_different_with_closure_variable() -> Result<()> { + let to_upper_case = |s: &str| s.to_uppercase(); + let result = verify_that!("world", result_of_ref!(to_upper_case, eq("HELLO"))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: "world" + Expected: by applying to_upper_case, + is equal to "HELLO" + Actual: "world", + which, results into "WORLD" + by applying to_upper_case, + isn't equal to "HELLO" + "# + )))) + ) + } + + #[test] + fn result_of_ref_match_different_with_method_literal() -> Result<()> { + let result = verify_that!("world", result_of_ref!(str::to_uppercase, eq("HELLO"))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: "world" + Expected: by applying str::to_uppercase, + is equal to "HELLO" + Actual: "world", + which, results into "WORLD" + by applying str::to_uppercase, + isn't equal to "HELLO" + "# + )))) + ) + } + + #[test] + fn result_of_ref_match_different_with_function_return_closure() -> Result<()> { + fn upper_case() -> impl Fn(&str) -> String { + |s: &str| s.to_uppercase() + } + let result = verify_that!("world", result_of_ref!(upper_case(), eq("HELLO"))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: "world" + Expected: by applying upper_case(), + is equal to "HELLO" + Actual: "world", + which, results into "WORLD" + by applying upper_case(), + isn't equal to "HELLO" + "# + )))) + ) + } + + #[test] + fn test_describe_simple() -> Result<()> { + let matcher = result_of!(|x| x + 1, eq(2)); + let description = matcher.describe(matcher.matches(0)); + verify_that!( + description, + displays_as(eq(indoc!( + r#" + by applying |x| x + 1, + isn't equal to 2"# + ))) + ) + } + + #[test] + fn test_describe_complicated() -> Result<()> { + let matcher = result_of_ref!( + |s: &str| s.chars().collect::>(), + each(predicate(char::is_ascii_alphabetic)) + ); + let description = matcher.describe(matcher.matches("A quick brown fox")); + verify_that!( + description, + displays_as(eq(indoc!( + r#" + by applying |s: &str| s.chars().collect::>(), + contains no element that matches"# + ))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/some_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/some_matcher.rs new file mode 100644 index 0000000..ce21527 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/some_matcher.rs @@ -0,0 +1,250 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches an `Option` containing a value matched by `inner`. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(Some("Some value"), some(eq("Some value")))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!(None::<&str>, some(eq("Some value")))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!(Some("Some value"), some(eq("Some other value")))?; // Fails +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// ``` +pub fn some(inner: Inner) -> SomeMatcher { + SomeMatcher { inner } +} + +#[derive(MatcherBase)] +pub struct SomeMatcher { + inner: InnerMatcherT, +} + +impl> Matcher> for SomeMatcher { + fn matches(&self, actual: Option) -> MatcherResult { + actual.map(|v| self.inner.matches(v)).unwrap_or(MatcherResult::NoMatch) + } + + fn explain_match(&self, actual: Option) -> Description { + match (self.matches(actual), actual) { + (_, Some(t)) => { + Description::new().text("which has a value").nested(self.inner.explain_match(t)) + } + (_, None) => "which is None".into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("has a value which {}", self.inner.describe(MatcherResult::Match)).into() + } + MatcherResult::NoMatch => format!( + "is None or has a value which {}", + self.inner.describe(MatcherResult::NoMatch) + ) + .into(), + } + } +} + +impl<'a, T: Debug, InnerMatcherT: Matcher<&'a T>> Matcher<&'a Option> + for SomeMatcher +{ + fn matches(&self, actual: &'a Option) -> MatcherResult { + actual.as_ref().map(|v| self.inner.matches(v)).unwrap_or(MatcherResult::NoMatch) + } + + fn explain_match(&self, actual: &'a Option) -> Description { + match (self.matches(actual), actual) { + (_, Some(t)) => { + Description::new().text("which has a value").nested(self.inner.explain_match(t)) + } + (_, None) => "which is None".into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + format!("has a value which {}", self.inner.describe(MatcherResult::Match)).into() + } + MatcherResult::NoMatch => format!( + "is None or has a value which {}", + self.inner.describe(MatcherResult::NoMatch) + ) + .into(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::matcher::MatcherResult; + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + + #[test] + fn some_matches_option_with_value() -> Result<()> { + let matcher = some(eq(1)); + + let result = matcher.matches(Some(1)); + + verify_that!(result, eq(MatcherResult::Match)) + } + + #[test] + fn some_does_not_match_option_with_wrong_value() -> Result<()> { + let matcher = some(eq(1)); + + let result = matcher.matches(Some(0)); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn some_does_not_match_option_with_none() -> Result<()> { + let matcher = some(eq(1)); + + let result = matcher.matches(None::); + + verify_that!(result, eq(MatcherResult::NoMatch)) + } + + #[test] + fn some_matches_option_with_by_ref_value() -> Result<()> { + verify_that!(Some("123".to_string()), some(eq("123"))) + } + + #[test] + fn some_does_not_match_option_with_wrong_by_ref_value() -> Result<()> { + verify_that!(Some("321".to_string()), not(some(eq("123")))) + } + + #[test] + fn some_does_not_match_option_with_by_ref_none() -> Result<()> { + verify_that!(None::, not(some(eq("123")))) + } + + #[test] + fn some_full_error_message() -> Result<()> { + let result = verify_that!(Some(2), some(eq(1))); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: Some(2) + Expected: has a value which is equal to 1 + Actual: Some(2), + which has a value + which isn't equal to 1 + " + )))) + ) + } + + #[test] + fn some_describe_matches() -> Result<()> { + verify_that!( + Matcher::>::describe(&some(eq(1)), MatcherResult::Match), + displays_as(eq("has a value which is equal to 1")) + ) + } + + #[test] + fn some_describe_does_not_match() -> Result<()> { + verify_that!( + Matcher::>::describe(&some(eq(1)), MatcherResult::NoMatch), + displays_as(eq("is None or has a value which isn't equal to 1")) + ) + } + + #[test] + fn some_describe_matches_of_by_ref() -> Result<()> { + verify_that!( + Matcher::>::describe(&some(eq("123")), MatcherResult::Match), + displays_as(eq("has a value which is equal to \"123\"")) + ) + } + + #[test] + fn some_describe_does_not_match_of_by_ref() -> Result<()> { + verify_that!( + Matcher::>::describe(&some(eq("123")), MatcherResult::NoMatch), + displays_as(eq("is None or has a value which isn't equal to \"123\"")) + ) + } + + #[test] + fn some_explain_match_with_none() -> Result<()> { + verify_that!(some(eq(1)).explain_match(None::), displays_as(eq("which is None"))) + } + + #[test] + fn some_explain_match_with_some_success() -> Result<()> { + verify_that!( + some(eq(1)).explain_match(Some(1)), + displays_as(eq("which has a value\n which is equal to 1")) + ) + } + + #[test] + fn some_explain_match_with_some_fail() -> Result<()> { + verify_that!( + some(eq(1)).explain_match(Some(2)), + displays_as(eq("which has a value\n which isn't equal to 1")) + ) + } + + #[test] + fn some_explain_match_with_none_by_ref() -> Result<()> { + verify_that!( + some(eq("123")).explain_match(&None::), + displays_as(eq("which is None")) + ) + } + + #[test] + fn some_explain_match_with_some_success_by_ref() -> Result<()> { + verify_that!( + some(eq("123")).explain_match(&Some("123".to_string())), + displays_as(eq("which has a value\n which is equal to \"123\"")) + ) + } + + #[test] + fn some_explain_match_with_some_fail_by_ref() -> Result<()> { + verify_that!( + some(eq("123")).explain_match(&Some("321".to_string())), + displays_as(eq("which has a value\n which isn't equal to \"123\"")) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/str_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/str_matcher.rs new file mode 100644 index 0000000..eea7e72 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/str_matcher.rs @@ -0,0 +1,1322 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, + matcher_support::{ + edit_distance, + summarize_diff::{create_diff, create_diff_reversed}, + }, + matchers::eq_matcher::EqMatcher, +}; +use std::borrow::Cow; +use std::fmt::Debug; +use std::ops::Deref; + +/// Matches a string containing a given substring. +/// +/// Both the actual value and the expected substring may be either a `String` or +/// a string reference. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass_1() -> Result<()> { +/// verify_that!("Some value", contains_substring("Some"))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// verify_that!("Another value", contains_substring("Some"))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_pass_2() -> Result<()> { +/// verify_that!("Some value".to_string(), contains_substring("value"))?; // Passes +/// verify_that!("Some value", contains_substring("value".to_string()))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail().unwrap_err(); +/// # should_pass_2().unwrap(); +/// ``` +/// +/// See the [`StrMatcherConfigurator`] extension trait for more options on how +/// the string is matched. +/// +/// > Note on memory use: In most cases, this matcher does not allocate memory +/// > when matching strings. However, it must allocate copies of both the actual +/// > and expected values when matching strings while +/// > [`ignoring_ascii_case`][StrMatcherConfigurator::ignoring_ascii_case] or +/// > [`ignoring_unicode_case`][StrMatcherConfigurator::ignoring_unicode_case] +/// > are set. +pub fn contains_substring(expected: T) -> StrMatcher { + StrMatcher { + configuration: Configuration { mode: MatchMode::Contains, ..Default::default() }, + expected, + } +} + +/// Matches a string which starts with the given prefix. +/// +/// Both the actual value and the expected prefix may be either a `String` or +/// a string reference. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass_1() -> Result<()> { +/// verify_that!("Some value", starts_with("Some"))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!("Another value", starts_with("Some"))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!("Some value", starts_with("value"))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_pass_2() -> Result<()> { +/// verify_that!("Some value".to_string(), starts_with("Some"))?; // Passes +/// verify_that!("Some value", starts_with("Some".to_string()))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_pass_2().unwrap(); +/// ``` +/// +/// See the [`StrMatcherConfigurator`] extension trait for more options on how +/// the string is matched. +pub fn starts_with(expected: T) -> StrMatcher { + StrMatcher { + configuration: Configuration { mode: MatchMode::StartsWith, ..Default::default() }, + expected, + } +} + +/// Matches a string which ends with the given suffix. +/// +/// Both the actual value and the expected suffix may be either a `String` or +/// a string reference. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass_1() -> Result<()> { +/// verify_that!("Some value", ends_with("value"))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!("Some value", ends_with("other value"))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!("Some value", ends_with("Some"))?; // Fails +/// # Ok(()) +/// # } +/// # fn should_pass_2() -> Result<()> { +/// verify_that!("Some value".to_string(), ends_with("value"))?; // Passes +/// verify_that!("Some value", ends_with("value".to_string()))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_pass_2().unwrap(); +/// ``` +/// +/// See the [`StrMatcherConfigurator`] extension trait for more options on how +/// the string is matched. +pub fn ends_with(expected: T) -> StrMatcher { + StrMatcher { + configuration: Configuration { mode: MatchMode::EndsWith, ..Default::default() }, + expected, + } +} + +/// Extension trait to configure [`StrMatcher`]. +/// +/// Matchers which match against string values and, through configuration, +/// specialise to [`StrMatcher`] implement this trait. That includes +/// [`EqMatcher`] and [`StrMatcher`]. +pub trait StrMatcherConfigurator { + /// Configures the matcher to ignore any leading whitespace in either the + /// actual or the expected value. + /// + /// Whitespace is defined as in [`str::trim_start`]. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_pass() -> Result<()> { + /// verify_that!("A string", eq(" A string").ignoring_leading_whitespace())?; // Passes + /// verify_that!(" A string", eq("A string").ignoring_leading_whitespace())?; // Passes + /// # Ok(()) + /// # } + /// # should_pass().unwrap(); + /// ``` + /// + /// When all other configuration options are left as the defaults, this is + /// equivalent to invoking [`str::trim_start`] on both the expected and + /// actual value. + fn ignoring_leading_whitespace(self) -> StrMatcher; + + /// Configures the matcher to ignore any trailing whitespace in either the + /// actual or the expected value. + /// + /// Whitespace is defined as in [`str::trim_end`]. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_pass() -> Result<()> { + /// verify_that!("A string", eq("A string ").ignoring_trailing_whitespace())?; // Passes + /// verify_that!("A string ", eq("A string").ignoring_trailing_whitespace())?; // Passes + /// # Ok(()) + /// # } + /// # should_pass().unwrap(); + /// ``` + /// + /// When all other configuration options are left as the defaults, this is + /// equivalent to invoking [`str::trim_end`] on both the expected and + /// actual value. + fn ignoring_trailing_whitespace(self) -> StrMatcher; + + /// Configures the matcher to ignore both leading and trailing whitespace in + /// either the actual or the expected value. + /// + /// Whitespace is defined as in [`str::trim`]. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_pass() -> Result<()> { + /// verify_that!("A string", eq(" A string ").ignoring_outer_whitespace())?; // Passes + /// verify_that!(" A string ", eq("A string").ignoring_outer_whitespace())?; // Passes + /// # Ok(()) + /// # } + /// # should_pass().unwrap(); + /// ``` + /// + /// This is equivalent to invoking both + /// [`ignoring_leading_whitespace`][StrMatcherConfigurator::ignoring_leading_whitespace] and + /// [`ignoring_trailing_whitespace`][StrMatcherConfigurator::ignoring_trailing_whitespace]. + /// + /// When all other configuration options are left as the defaults, this is + /// equivalent to invoking [`str::trim`] on both the expected and actual + /// value. + fn ignoring_outer_whitespace(self) -> StrMatcher; + + /// Configures the matcher to ignore ASCII case when comparing values. + /// + /// This uses the same rules for case as [`str::eq_ignore_ascii_case`]. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_pass() -> Result<()> { + /// verify_that!("Some value", eq("SOME VALUE").ignoring_ascii_case())?; // Passes + /// # Ok(()) + /// # } + /// # fn should_fail() -> Result<()> { + /// verify_that!("Another value", eq("Some value").ignoring_ascii_case())?; // Fails + /// # Ok(()) + /// # } + /// # should_pass().unwrap(); + /// # should_fail().unwrap_err(); + /// ``` + /// + /// This is **not guaranteed** to match strings with differing upper/lower + /// case characters outside of the codepoints 0-127 covered by ASCII. + fn ignoring_ascii_case(self) -> StrMatcher; + + /// Configures the matcher to ignore Unicode case when comparing values. + /// + /// This uses the same rules for case as [`str::to_lowercase`]. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_pass() -> Result<()> { + /// verify_that!("ὈΔΥΣΣΕΎΣ", eq("ὀδυσσεύς").ignoring_unicode_case())?; // Passes + /// # Ok(()) + /// # } + /// # fn should_fail() -> Result<()> { + /// verify_that!("secret", eq("비밀").ignoring_unicode_case())?; // Fails + /// # Ok(()) + /// # } + /// # should_pass().unwrap(); + /// # should_fail().unwrap_err(); + /// ``` + fn ignoring_unicode_case(self) -> StrMatcher; + + /// Configures the matcher to match only strings which otherwise satisfy the + /// conditions a number times matched by the matcher `times`. + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_pass() -> Result<()> { + /// verify_that!("Some value\nSome value", contains_substring("value").times(eq(2)))?; // Passes + /// # Ok(()) + /// # } + /// # fn should_fail() -> Result<()> { + /// verify_that!("Some value", contains_substring("value").times(eq(2)))?; // Fails + /// # Ok(()) + /// # } + /// # should_pass().unwrap(); + /// # should_fail().unwrap_err(); + /// ``` + /// + /// The matched substrings must be disjoint from one another to be counted. + /// For example: + /// + /// ``` + /// # use googletest::prelude::*; + /// # fn should_fail() -> Result<()> { + /// // Fails: substrings distinct but not disjoint! + /// verify_that!("ababab", contains_substring("abab").times(eq(2)))?; + /// # Ok(()) + /// # } + /// # should_fail().unwrap_err(); + /// ``` + /// + /// This is only meaningful when the matcher was constructed with + /// [`contains_substring`]. This method will panic when it is used with any + /// other matcher construction. + fn times(self, times: impl Matcher + 'static) -> StrMatcher; +} + +/// A matcher which matches equality or containment of a string-like value in a +/// configurable way. +/// +/// The following matcher methods instantiate this: +/// +/// * [`eq`][crate::matchers::eq_matcher::eq], +/// * [`contains_substring`], +/// * [`starts_with`], +/// * [`ends_with`]. +#[derive(MatcherBase)] +pub struct StrMatcher { + expected: ExpectedT, + configuration: Configuration, +} + +impl Matcher for StrMatcher +where + ExpectedT: Deref + Debug, + ActualT: AsRef + Debug + Copy, +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + self.configuration.do_strings_match(self.expected.deref(), actual.as_ref()).into() + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + self.configuration.describe(matcher_result, self.expected.deref()) + } + + fn explain_match(&self, actual: ActualT) -> Description { + self.configuration.explain_match(self.expected.deref(), actual.as_ref()) + } +} + +impl>> StrMatcherConfigurator + for MatcherT +{ + fn ignoring_leading_whitespace(self) -> StrMatcher { + let existing = self.into(); + StrMatcher { + configuration: existing.configuration.ignoring_leading_whitespace(), + ..existing + } + } + + fn ignoring_trailing_whitespace(self) -> StrMatcher { + let existing = self.into(); + StrMatcher { + configuration: existing.configuration.ignoring_trailing_whitespace(), + ..existing + } + } + + fn ignoring_outer_whitespace(self) -> StrMatcher { + let existing = self.into(); + StrMatcher { configuration: existing.configuration.ignoring_outer_whitespace(), ..existing } + } + + fn ignoring_ascii_case(self) -> StrMatcher { + let existing = self.into(); + StrMatcher { configuration: existing.configuration.ignoring_ascii_case(), ..existing } + } + + fn ignoring_unicode_case(self) -> StrMatcher { + let existing = self.into(); + StrMatcher { configuration: existing.configuration.ignoring_unicode_case(), ..existing } + } + + fn times(self, times: impl Matcher + 'static) -> StrMatcher { + let existing = self.into(); + if !matches!(existing.configuration.mode, MatchMode::Contains) { + panic!("The times() configurator is only meaningful with contains_substring()."); + } + StrMatcher { configuration: existing.configuration.times(times), ..existing } + } +} + +impl> From> for StrMatcher { + fn from(value: EqMatcher) -> Self { + Self::with_default_config(value.expected) + } +} + +impl StrMatcher { + /// Returns a [`StrMatcher`] with a default configuration to match against + /// the given expected value. + /// + /// This default configuration is sensitive to whitespace and case. + fn with_default_config(expected: T) -> Self { + Self { expected, configuration: Default::default() } + } +} + +// Holds all the information on how the expected and actual strings are to be +// compared. Its associated functions perform the actual matching operations +// on string references. The struct and comparison methods therefore need not be +// parameterised, saving compilation time and binary size on monomorphisation. +// +// The default value represents exact equality of the strings. +struct Configuration { + mode: MatchMode, + ignore_leading_whitespace: bool, + ignore_trailing_whitespace: bool, + case_policy: CasePolicy, + times: Option>>, +} + +#[derive(Clone)] +enum MatchMode { + Equals, + Contains, + StartsWith, + EndsWith, +} + +impl MatchMode { + fn to_diff_mode(&self) -> edit_distance::Mode { + match self { + MatchMode::StartsWith | MatchMode::EndsWith => edit_distance::Mode::Prefix, + MatchMode::Contains => edit_distance::Mode::Contains, + MatchMode::Equals => edit_distance::Mode::Exact, + } + } +} + +#[derive(Clone)] +enum CasePolicy { + Respect, + IgnoreAscii, + IgnoreUnicode, +} + +impl Configuration { + // The entry point for all string matching. StrMatcher::matches redirects + // immediately to this function. + fn do_strings_match(&self, expected: &str, actual: &str) -> bool { + let (expected, actual) = + match (self.ignore_leading_whitespace, self.ignore_trailing_whitespace) { + (true, true) => (expected.trim(), actual.trim()), + (true, false) => (expected.trim_start(), actual.trim_start()), + (false, true) => (expected.trim_end(), actual.trim_end()), + (false, false) => (expected, actual), + }; + match self.mode { + MatchMode::Equals => match self.case_policy { + CasePolicy::Respect => expected == actual, + CasePolicy::IgnoreAscii => expected.eq_ignore_ascii_case(actual), + CasePolicy::IgnoreUnicode => expected.to_lowercase() == actual.to_lowercase(), + }, + MatchMode::Contains => match self.case_policy { + CasePolicy::Respect => self.does_containment_match(actual, expected), + CasePolicy::IgnoreAscii => self.does_containment_match( + actual.to_ascii_lowercase().as_str(), + expected.to_ascii_lowercase().as_str(), + ), + CasePolicy::IgnoreUnicode => self.does_containment_match( + actual.to_lowercase().as_str(), + expected.to_lowercase().as_str(), + ), + }, + MatchMode::StartsWith => match self.case_policy { + CasePolicy::Respect => actual.starts_with(expected), + CasePolicy::IgnoreAscii => { + actual.len() >= expected.len() + && actual[..expected.len()].eq_ignore_ascii_case(expected) + } + CasePolicy::IgnoreUnicode => { + actual.to_lowercase().starts_with(&expected.to_lowercase()) + } + }, + MatchMode::EndsWith => match self.case_policy { + CasePolicy::Respect => actual.ends_with(expected), + CasePolicy::IgnoreAscii => { + actual.len() >= expected.len() + && actual[actual.len() - expected.len()..].eq_ignore_ascii_case(expected) + } + CasePolicy::IgnoreUnicode => { + actual.to_lowercase().ends_with(&expected.to_lowercase()) + } + }, + } + } + + // Returns whether actual contains expected a number of times matched by the + // matcher self.times. Does not take other configuration into account. + fn does_containment_match(&self, actual: &str, expected: &str) -> bool { + if let Some(times) = self.times.as_ref() { + // Split returns an iterator over the "boundaries" left and right of the + // substring to be matched, of which there is one more than the number of + // substrings. + matches!(times.matches(actual.split(expected).count() - 1), MatcherResult::Match) + } else { + actual.contains(expected) + } + } + + // StrMatcher::::describe redirects immediately to this function. + fn describe(&self, matcher_result: MatcherResult, expected: &str) -> Description { + let mut addenda: Vec> = Vec::with_capacity(3); + match (self.ignore_leading_whitespace, self.ignore_trailing_whitespace) { + (true, true) => addenda.push("ignoring leading and trailing whitespace".into()), + (true, false) => addenda.push("ignoring leading whitespace".into()), + (false, true) => addenda.push("ignoring trailing whitespace".into()), + (false, false) => {} + } + match self.case_policy { + CasePolicy::Respect => {} + CasePolicy::IgnoreAscii => addenda.push("ignoring ASCII case".into()), + CasePolicy::IgnoreUnicode => addenda.push("ignoring Unicode case".into()), + } + if let Some(times) = self.times.as_ref() { + addenda.push(format!("count {}", times.describe(matcher_result)).into()); + } + let extra = + if !addenda.is_empty() { format!(" ({})", addenda.join(", ")) } else { "".into() }; + let match_mode_description = match self.mode { + MatchMode::Equals => match matcher_result { + MatcherResult::Match => "is equal to", + MatcherResult::NoMatch => "isn't equal to", + }, + MatchMode::Contains => match matcher_result { + MatcherResult::Match => "contains a substring", + MatcherResult::NoMatch => "does not contain a substring", + }, + MatchMode::StartsWith => match matcher_result { + MatcherResult::Match => "starts with prefix", + MatcherResult::NoMatch => "does not start with", + }, + MatchMode::EndsWith => match matcher_result { + MatcherResult::Match => "ends with suffix", + MatcherResult::NoMatch => "does not end with", + }, + }; + format!("{match_mode_description} {expected:?}{extra}").into() + } + + fn explain_match(&self, expected: &str, actual: &str) -> Description { + let default_explanation = format!( + "which {}", + self.describe(self.do_strings_match(expected, actual).into(), expected) + ) + .into(); + if !expected.contains('\n') || !actual.contains('\n') { + return default_explanation; + } + + if self.ignore_leading_whitespace { + // TODO - b/283448414 : Support StrMatcher with ignore_leading_whitespace. + return default_explanation; + } + + if self.ignore_trailing_whitespace { + // TODO - b/283448414 : Support StrMatcher with ignore_trailing_whitespace. + return default_explanation; + } + + if self.times.is_some() { + // TODO - b/283448414 : Support StrMatcher with times. + return default_explanation; + } + if matches!(self.case_policy, CasePolicy::IgnoreAscii) { + // TODO - b/283448414 : Support StrMatcher with ignore ascii case policy. + return default_explanation; + } + if matches!(self.case_policy, CasePolicy::IgnoreUnicode) { + // TODO - b/283448414 : Support StrMatcher with ignore unicode case policy. + return default_explanation; + } + if self.do_strings_match(expected, actual) { + // TODO - b/283448414 : Consider supporting debug difference if the + // strings match. This can be useful when a small contains is found + // in a long string. + return default_explanation; + } + + let diff = match self.mode { + MatchMode::Equals | MatchMode::StartsWith | MatchMode::Contains => { + // TODO(b/287632452): Also consider improving the output in MatchMode::Contains + // when the substring begins or ends in the middle of a line of the actual + // value. + create_diff(actual, expected, self.mode.to_diff_mode()) + } + MatchMode::EndsWith => create_diff_reversed(actual, expected, self.mode.to_diff_mode()), + }; + + if diff.is_empty() { + format!("{default_explanation}").into() + } else { + format!("{default_explanation}\n\n{diff}").into() + } + } + + fn ignoring_leading_whitespace(self) -> Self { + Self { ignore_leading_whitespace: true, ..self } + } + + fn ignoring_trailing_whitespace(self) -> Self { + Self { ignore_trailing_whitespace: true, ..self } + } + + fn ignoring_outer_whitespace(self) -> Self { + Self { ignore_leading_whitespace: true, ignore_trailing_whitespace: true, ..self } + } + + fn ignoring_ascii_case(self) -> Self { + Self { case_policy: CasePolicy::IgnoreAscii, ..self } + } + + fn ignoring_unicode_case(self) -> Self { + Self { case_policy: CasePolicy::IgnoreUnicode, ..self } + } + + fn times(self, times: impl Matcher + 'static) -> Self { + Self { times: Some(Box::new(times)), ..self } + } +} + +impl Default for Configuration { + fn default() -> Self { + Self { + mode: MatchMode::Equals, + ignore_leading_whitespace: false, + ignore_trailing_whitespace: false, + case_policy: CasePolicy::Respect, + times: None, + } + } +} + +#[cfg(test)] +mod tests { + use crate as googletest; + use crate::matcher::MatcherResult; + use crate::prelude::*; + use indoc::indoc; + + #[test] + fn matches_string_reference_with_equal_string_reference() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + verify_that!("A string", matcher) + } + + #[test] + fn does_not_match_string_reference_with_non_equal_string_reference() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("Another string"); + verify_that!("A string", not(matcher)) + } + + #[test] + fn matches_owned_string_with_string_reference() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + let value = "A string".to_string(); + verify_that!(value, matcher) + } + + #[test] + fn matches_owned_string_reference_with_string_reference() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + let value = "A string".to_string(); + verify_that!(&value, matcher) + } + + #[test] + fn ignores_leading_whitespace_in_expected_when_requested() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config(" \n\tA string"); + verify_that!("A string", matcher.ignoring_leading_whitespace()) + } + + #[test] + fn ignores_leading_whitespace_in_actual_when_requested() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + verify_that!(" \n\tA string", matcher.ignoring_leading_whitespace()) + } + + #[test] + fn does_not_match_unequal_remaining_string_when_ignoring_leading_whitespace( + ) -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config(" \n\tAnother string"); + verify_that!("A string", not(matcher.ignoring_leading_whitespace())) + } + + #[test] + fn remains_sensitive_to_trailing_whitespace_when_ignoring_leading_whitespace( + ) -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string \n\t"); + verify_that!("A string", not(matcher.ignoring_leading_whitespace())) + } + + #[test] + fn ignores_trailing_whitespace_in_expected_when_requested() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string \n\t"); + verify_that!("A string", matcher.ignoring_trailing_whitespace()) + } + + #[test] + fn ignores_trailing_whitespace_in_actual_when_requested() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + verify_that!("A string \n\t", matcher.ignoring_trailing_whitespace()) + } + + #[test] + fn does_not_match_unequal_remaining_string_when_ignoring_trailing_whitespace( + ) -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("Another string \n\t"); + verify_that!("A string", not(matcher.ignoring_trailing_whitespace())) + } + + #[test] + fn remains_sensitive_to_leading_whitespace_when_ignoring_trailing_whitespace( + ) -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config(" \n\tA string"); + verify_that!("A string", not(matcher.ignoring_trailing_whitespace())) + } + + #[test] + fn ignores_leading_and_trailing_whitespace_in_expected_when_requested() -> googletest::Result<()> + { + let matcher = StrMatcher::with_default_config(" \n\tA string \n\t"); + verify_that!("A string", matcher.ignoring_outer_whitespace()) + } + + #[test] + fn ignores_leading_and_trailing_whitespace_in_actual_when_requested() -> googletest::Result<()> + { + let matcher = StrMatcher::with_default_config("A string"); + verify_that!(" \n\tA string \n\t", matcher.ignoring_outer_whitespace()) + } + + #[test] + fn respects_ascii_case_by_default() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + verify_that!("A STRING", not(matcher)) + } + + #[test] + fn ignores_ascii_case_when_requested() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + verify_that!("A STRING", matcher.ignoring_ascii_case()) + } + + #[test] + fn ignores_unicode_case_when_requested() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("ὈΔΥΣΣΕΎΣ"); + verify_that!("ὀδυσσεύς", matcher.ignoring_unicode_case()) + } + + #[test] + fn allows_ignoring_leading_whitespace_from_eq() -> googletest::Result<()> { + verify_that!("A string", eq(" \n\tA string").ignoring_leading_whitespace()) + } + + #[test] + fn allows_ignoring_trailing_whitespace_from_eq() -> googletest::Result<()> { + verify_that!("A string", eq("A string \n\t").ignoring_trailing_whitespace()) + } + + #[test] + fn allows_ignoring_outer_whitespace_from_eq() -> googletest::Result<()> { + verify_that!("A string", eq(" \n\tA string \n\t").ignoring_outer_whitespace()) + } + + #[test] + fn allows_ignoring_ascii_case_from_eq() -> googletest::Result<()> { + verify_that!("A string", eq("A STRING").ignoring_ascii_case()) + } + + #[test] + fn allows_ignoring_unicode_case_from_eq() -> googletest::Result<()> { + verify_that!("ὈΔΥΣΣΕΎΣ", eq("ὀδυσσεύς").ignoring_unicode_case()) + } + + #[test] + fn unicode_case_sensitive_from_eq() -> googletest::Result<()> { + verify_that!("ὈΔΥΣΣΕΎΣ", not(eq("ὀδυσσεύς"))) + } + + #[test] + fn matches_string_containing_expected_value_in_contains_mode() -> googletest::Result<()> { + verify_that!("Some string", contains_substring("str")) + } + + #[test] + fn matches_string_containing_expected_value_in_contains_mode_while_ignoring_ascii_case( + ) -> googletest::Result<()> { + verify_that!("Some string", contains_substring("STR").ignoring_ascii_case()) + } + + #[test] + fn matches_string_containing_expected_value_in_contains_mode_while_ignoring_unicode_case( + ) -> googletest::Result<()> { + verify_that!("Some σpsilon", contains_substring("Σps").ignoring_unicode_case()) + } + + #[test] + fn contains_substring_matches_correct_number_of_substrings() -> googletest::Result<()> { + verify_that!("Some string", contains_substring("str").times(eq(1))) + } + + #[test] + fn contains_substring_does_not_match_incorrect_number_of_substrings() -> googletest::Result<()> + { + verify_that!("Some string\nSome string", not(contains_substring("string").times(eq(1)))) + } + + #[test] + fn contains_substring_does_not_match_when_substrings_overlap() -> googletest::Result<()> { + verify_that!("ababab", not(contains_substring("abab").times(eq(2)))) + } + + #[test] + fn starts_with_matches_string_reference_with_prefix() -> googletest::Result<()> { + verify_that!("Some value", starts_with("Some")) + } + + #[test] + fn starts_with_matches_string_reference_with_prefix_ignoring_ascii_case( + ) -> googletest::Result<()> { + verify_that!("Some value", starts_with("SOME").ignoring_ascii_case()) + } + + #[test] + fn starts_with_does_not_match_wrong_prefix_ignoring_ascii_case() -> googletest::Result<()> { + verify_that!("Some value", not(starts_with("OTHER").ignoring_ascii_case())) + } + + #[test] + fn starts_with_does_not_match_short_string_ignoring_ascii_case() -> googletest::Result<()> { + verify_that!("Some", not(starts_with("OTHER").ignoring_ascii_case())) + } + + #[test] + fn starts_with_matches_string_reference_with_prefix_ignoring_unicode_case( + ) -> googletest::Result<()> { + verify_that!("비밀 santa", starts_with("비밀").ignoring_unicode_case()) + } + + #[test] + fn starts_with_does_not_match_wrong_prefix_ignoring_unicode_case() -> googletest::Result<()> { + verify_that!("secret santa", not(starts_with("비밀").ignoring_unicode_case())) + } + + #[test] + fn starts_with_does_not_match_short_string_ignoring_unicode_case() -> googletest::Result<()> { + verify_that!("비밀", not(starts_with("秘密").ignoring_unicode_case())) + } + + #[test] + fn starts_with_does_not_match_string_without_prefix() -> googletest::Result<()> { + verify_that!("Some value", not(starts_with("Another"))) + } + + #[test] + fn starts_with_does_not_match_string_with_substring_not_at_beginning() -> googletest::Result<()> + { + verify_that!("Some value", not(starts_with("value"))) + } + + #[test] + fn ends_with_matches_string_reference_with_suffix() -> googletest::Result<()> { + verify_that!("Some value", ends_with("value")) + } + + #[test] + fn ends_with_matches_string_reference_with_suffix_ignoring_ascii_case() -> googletest::Result<()> + { + verify_that!("Some value", ends_with("VALUE").ignoring_ascii_case()) + } + + #[test] + fn ends_with_does_not_match_wrong_suffix_ignoring_ascii_case() -> googletest::Result<()> { + verify_that!("Some value", not(ends_with("OTHER").ignoring_ascii_case())) + } + + #[test] + fn ends_with_does_not_match_too_short_string_ignoring_ascii_case() -> googletest::Result<()> { + verify_that!("Some", not(ends_with("OTHER").ignoring_ascii_case())) + } + + #[test] + fn ends_with_does_not_match_string_without_suffix() -> googletest::Result<()> { + verify_that!("Some value", not(ends_with("other value"))) + } + + #[test] + fn ends_with_does_not_match_string_with_substring_not_at_end() -> googletest::Result<()> { + verify_that!("Some value", not(ends_with("Some"))) + } + + #[test] + fn ends_with_matches_string_reference_with_suffix_ignoring_unicode_case( + ) -> googletest::Result<()> { + verify_that!("santa 비밀", ends_with("비밀").ignoring_unicode_case()) + } + + #[test] + fn ends_with_does_not_match_wrong_suffix_ignoring_unicode_case() -> googletest::Result<()> { + verify_that!("secret santa", not(ends_with("비밀").ignoring_unicode_case())) + } + + #[test] + fn ends_with_does_not_match_short_string_ignoring_unicode_case() -> googletest::Result<()> { + verify_that!("비밀", not(ends_with("秘密").ignoring_unicode_case())) + } + + #[test] + fn describes_itself_for_matching_result() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is equal to \"A string\"")) + ) + } + + #[test] + fn describes_itself_for_non_matching_result() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string"); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::NoMatch), + displays_as(eq("isn't equal to \"A string\"")) + ) + } + + #[test] + fn describes_itself_for_matching_result_ignoring_leading_whitespace() -> googletest::Result<()> + { + let matcher = StrMatcher::with_default_config("A string").ignoring_leading_whitespace(); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is equal to \"A string\" (ignoring leading whitespace)")) + ) + } + + #[test] + fn describes_itself_for_non_matching_result_ignoring_leading_whitespace( + ) -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string").ignoring_leading_whitespace(); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::NoMatch), + displays_as(eq("isn't equal to \"A string\" (ignoring leading whitespace)")) + ) + } + + #[test] + fn describes_itself_for_matching_result_ignoring_trailing_whitespace() -> googletest::Result<()> + { + let matcher = StrMatcher::with_default_config("A string").ignoring_trailing_whitespace(); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is equal to \"A string\" (ignoring trailing whitespace)")) + ) + } + + #[test] + fn describes_itself_for_matching_result_ignoring_leading_and_trailing_whitespace( + ) -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string").ignoring_outer_whitespace(); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is equal to \"A string\" (ignoring leading and trailing whitespace)")) + ) + } + + #[test] + fn describes_itself_for_matching_result_ignoring_ascii_case() -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string").ignoring_ascii_case(); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("is equal to \"A string\" (ignoring ASCII case)")) + ) + } + + #[test] + fn describes_itself_for_matching_result_ignoring_ascii_case_and_leading_whitespace( + ) -> googletest::Result<()> { + let matcher = StrMatcher::with_default_config("A string") + .ignoring_leading_whitespace() + .ignoring_ascii_case(); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq( + "is equal to \"A string\" (ignoring leading whitespace, ignoring ASCII case)" + )) + ) + } + + #[test] + fn describes_itself_for_matching_result_in_contains_mode() -> googletest::Result<()> { + let matcher = contains_substring("A string"); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("contains a substring \"A string\"")) + ) + } + + #[test] + fn describes_itself_for_non_matching_result_in_contains_mode() -> googletest::Result<()> { + let matcher = contains_substring("A string"); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::NoMatch), + displays_as(eq("does not contain a substring \"A string\"")) + ) + } + + #[test] + fn describes_itself_with_count_number() -> googletest::Result<()> { + let matcher = contains_substring("A string").times(gt(2)); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("contains a substring \"A string\" (count is greater than 2)")) + ) + } + + #[test] + fn describes_itself_for_matching_result_in_starts_with_mode() -> googletest::Result<()> { + let matcher = starts_with("A string"); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("starts with prefix \"A string\"")) + ) + } + + #[test] + fn describes_itself_for_non_matching_result_in_starts_with_mode() -> googletest::Result<()> { + let matcher = starts_with("A string"); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::NoMatch), + displays_as(eq("does not start with \"A string\"")) + ) + } + + #[test] + fn describes_itself_for_matching_result_in_ends_with_mode() -> googletest::Result<()> { + let matcher = ends_with("A string"); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::Match), + displays_as(eq("ends with suffix \"A string\"")) + ) + } + + #[test] + fn describes_itself_for_non_matching_result_in_ends_with_mode() -> googletest::Result<()> { + let matcher = ends_with("A string"); + verify_that!( + Matcher::<&str>::describe(&matcher, MatcherResult::NoMatch), + displays_as(eq("does not end with \"A string\"")) + ) + } + + #[test] + fn match_explanation_contains_diff_of_strings_if_more_than_one_line() -> googletest::Result<()> + { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + " + ), + starts_with(indoc!( + " + First line + Second lines + Third line + " + )) + ); + + verify_that!( + result, + err(displays_as(contains_substring( + "\ + First line + -Second line + +Second lines + Third line" + ))) + ) + } + + #[test] + fn match_explanation_for_starts_with_ignores_trailing_lines_in_actual_string( + ) -> googletest::Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + Fourth line + " + ), + starts_with(indoc!( + " + First line + Second lines + Third line + " + )) + ); + + verify_that!( + result, + err(displays_as(contains_substring( + " + First line + -Second line + +Second lines + Third line + <---- remaining lines omitted ---->" + ))) + ) + } + + #[test] + fn match_explanation_for_starts_with_includes_both_versions_of_differing_last_line( + ) -> googletest::Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + " + ), + starts_with(indoc!( + " + First line + Second lines + " + )) + ); + + verify_that!( + result, + err(displays_as(contains_substring( + "\ + First line + -Second line + +Second lines + <---- remaining lines omitted ---->" + ))) + ) + } + + #[test] + fn match_explanation_for_ends_with_ignores_leading_lines_in_actual_string( + ) -> googletest::Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + Fourth line + " + ), + ends_with(indoc!( + " + Second line + Third lines + Fourth line + " + )) + ); + + verify_that!( + result, + err(displays_as(contains_substring( + " + Difference(-actual / +expected): + <---- remaining lines omitted ----> + Second line + -Third line + +Third lines + Fourth line" + ))) + ) + } + + #[test] + fn match_explanation_for_contains_substring_ignores_outer_lines_in_actual_string( + ) -> googletest::Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + Fourth line + Fifth line + " + ), + contains_substring(indoc!( + " + Second line + Third lines + Fourth line + " + )) + ); + + verify_that!( + result, + err(displays_as(contains_substring( + " + Difference(-actual / +expected): + <---- remaining lines omitted ----> + Second line + -Third line + +Third lines + Fourth line + <---- remaining lines omitted ---->" + ))) + ) + } + + #[test] + fn match_explanation_for_contains_substring_shows_diff_when_first_and_last_line_are_incomplete( + ) -> googletest::Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + Fourth line + Fifth line + " + ), + contains_substring(indoc!( + " + line + Third line + Foorth line + Fifth" + )) + ); + + verify_that!( + result, + err(displays_as(contains_substring( + " + Difference(-actual / +expected): + <---- remaining lines omitted ----> + -Second line + +line + Third line + -Fourth line + +Foorth line + -Fifth line + +Fifth + <---- remaining lines omitted ---->" + ))) + ) + } + + #[test] + fn match_explanation_for_eq_does_not_ignore_trailing_lines_in_actual_string( + ) -> googletest::Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + Fourth line + " + ), + eq(indoc!( + " + First line + Second lines + Third line + " + )) + ); + + verify_that!( + result, + err(displays_as(contains_substring( + "\ + First line + -Second line + +Second lines + Third line + -Fourth line" + ))) + ) + } + + #[test] + fn match_explanation_does_not_show_diff_if_actual_value_is_single_line( + ) -> googletest::Result<()> { + let result = verify_that!( + "First line", + starts_with(indoc!( + " + Second line + Third line + " + )) + ); + + verify_that!( + result, + err(displays_as(not(contains_substring("Difference(-actual / +expected):")))) + ) + } + + #[test] + fn match_explanation_does_not_show_diff_if_expected_value_is_single_line( + ) -> googletest::Result<()> { + let result = verify_that!( + indoc!( + " + First line + Second line + Third line + " + ), + starts_with("Second line") + ); + + verify_that!( + result, + err(displays_as(not(contains_substring("Difference(-actual / +expected):")))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/subset_of_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/subset_of_matcher.rs new file mode 100644 index 0000000..9e52d73 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/subset_of_matcher.rs @@ -0,0 +1,253 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a container all of whose items are in the given container +/// `superset`. +/// +/// The element type `ElementT` must implement `PartialEq` to allow element +/// comparison. +/// +/// `ActualT` and `ExpectedT` can each be any container which +/// implements `IntoIterator`. For instance, `T` can be a common container like +/// `&Vec` or arrays. They need not be the same container type. +/// +/// ``` +/// # use googletest::prelude::*; +/// # use std::collections::HashSet; +/// # fn should_pass_1() -> Result<()> { +/// let value = vec![1, 2, 3]; +/// verify_that!(value, subset_of([&1, &2, &3, &4]))?; // Passes +/// let array_value = [1, 2, 3]; +/// verify_that!(array_value, subset_of([1, 2, 3, 4]))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// # let value = vec![1, 2, 3]; +/// verify_that!(value, subset_of([&1, &2]))?; // Fails: 3 is not in the superset +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail().unwrap_err(); +/// +/// # fn should_pass_2() -> Result<()> { +/// let value: HashSet = [1, 2, 3].into(); +/// verify_that!(value, subset_of([&1, &2, &3]))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_2().unwrap(); +/// ``` +/// +/// Item multiplicity in both the actual and expected containers is ignored: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let value: Vec = vec![0, 0, 1]; +/// verify_that!(value, subset_of([&0, &1]))?; // Passes +/// verify_that!(value, subset_of([&0, &1, &1]))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// A note on performance: This matcher uses a naive algorithm with a worst-case +/// runtime proportional to the *product* of the sizes of the actual and +/// expected containers as well as the time to check equality of each pair of +/// items. It should not be used on especially large containers. +pub fn subset_of(superset: ExpectedT) -> SubsetOfMatcher { + SubsetOfMatcher { superset } +} + +#[derive(MatcherBase)] +pub struct SubsetOfMatcher { + superset: ExpectedT, +} + +impl Matcher + for SubsetOfMatcher +where + ActualT: IntoIterator, + for<'a> &'a ExpectedT: IntoIterator, +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + for actual_item in actual { + if self.expected_is_missing(&actual_item) { + return MatcherResult::NoMatch; + } + } + MatcherResult::Match + } + + fn explain_match(&self, actual: ActualT) -> Description { + let unexpected_elements = actual + .into_iter() + .enumerate() + .filter(|item| self.expected_is_missing(&item.1)) + .map(|(idx, actual_item)| format!("{actual_item:#?} at #{idx}")) + .collect::>(); + + match unexpected_elements.len() { + 0 => "which no element is unexpected".into(), + 1 => format!("whose element {} is unexpected", &unexpected_elements[0]).into(), + _ => format!("whose elements {} are unexpected", unexpected_elements.join(", ")).into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!("is a subset of {:#?}", self.superset).into(), + MatcherResult::NoMatch => format!("isn't a subset of {:#?}", self.superset).into(), + } + } +} + +impl SubsetOfMatcher +where + for<'a> &'a ExpectedT: IntoIterator, +{ + fn expected_is_missing(&self, needle: &ElementT) -> bool { + !self.superset.into_iter().any(|item| item == needle) + } +} + +#[cfg(test)] +mod tests { + use crate::prelude::*; + use crate::Result; + use indoc::indoc; + use std::collections::HashSet; + + #[test] + fn subset_of_matches_empty_vec() -> Result<()> { + let value: Vec = vec![]; + verify_that!(value, subset_of([])) + } + + #[test] + fn subset_of_matches_vec_with_one_element() -> Result<()> { + let value = vec![1]; + verify_that!(value, subset_of([&1])) + } + + #[test] + fn subset_of_matches_vec_with_two_elements() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, subset_of([&1, &2])) + } + + #[test] + fn subset_of_matches_vec_when_expected_has_excess_element() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, subset_of([&1, &2, &3])) + } + + #[test] + fn subset_of_matches_vec_when_expected_has_excess_element_first() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, subset_of([&3, &1, &2])) + } + + #[test] + fn subset_of_matches_slice_with_one_element() -> Result<()> { + let value = &[1]; + verify_that!(value, subset_of([&1])) + } + + #[test] + fn subset_of_matches_hash_set_with_one_element() -> Result<()> { + let value: HashSet = [1].into(); + verify_that!(value, subset_of([&1])) + } + + #[test] + fn subset_of_does_not_match_when_first_element_does_not_match() -> Result<()> { + let value = vec![0]; + verify_that!(value, not(subset_of([&1]))) + } + + #[test] + fn subset_of_does_not_match_when_second_element_does_not_match() -> Result<()> { + let value = vec![2, 0]; + verify_that!(value, not(subset_of([&2]))) + } + + #[test] + fn subset_of_shows_correct_message_when_first_item_does_not_match() -> Result<()> { + let result = verify_that!(vec![0, 2, 3], subset_of([&1, &2, &3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![0, 2, 3] + Expected: is a subset of [ + 1, + 2, + 3, + ] + Actual: [0, 2, 3], + whose element 0 at #0 is unexpected + " + )))) + ) + } + + #[test] + fn subset_of_shows_correct_message_when_second_item_does_not_match() -> Result<()> { + let result = verify_that!(vec![1, 0, 3], subset_of([&1, &2, &3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 0, 3] + Expected: is a subset of [ + 1, + 2, + 3, + ] + Actual: [1, 0, 3], + whose element 0 at #1 is unexpected + " + )))) + ) + } + + #[test] + fn subset_of_shows_correct_message_when_first_two_items_do_not_match() -> Result<()> { + let result = verify_that!(vec![0, 0, 3], subset_of([&1, &2, &3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![0, 0, 3] + Expected: is a subset of [ + 1, + 2, + 3, + ] + Actual: [0, 0, 3], + whose elements 0 at #0, 0 at #1 are unexpected + " + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/superset_of_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/superset_of_matcher.rs new file mode 100644 index 0000000..a176612 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/superset_of_matcher.rs @@ -0,0 +1,253 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, +}; +use std::fmt::Debug; + +/// Matches a container containing all of the items in the given container +/// `subset`. +/// +/// The element type `ElementT` must implement `PartialEq` to allow element +/// comparison. +/// +/// `ActualT` and `ExpectedT` can each be any container which implements +/// `IntoIterator`. For instance, `ActualT` and `ExpectedT` can be a +/// common container like `&Vec`, arrays or slices. They need not be the same +/// container type. +/// +/// ``` +/// # use googletest::prelude::*; +/// # use std::collections::HashSet; +/// # fn should_pass_1() -> Result<()> { +/// let value = vec![1, 2, 3]; +/// verify_that!(value, superset_of([&1, &2]))?; // Passes +/// let array_value = [1, 2, 3]; +/// verify_that!(array_value, superset_of([1, 2]))?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail() -> Result<()> { +/// # let value = vec![1, 2, 3]; +/// verify_that!(value, superset_of([&1, &2, &4]))?; // Fails: 4 is not in the subset +/// # Ok(()) +/// # } +/// # should_pass_1().unwrap(); +/// # should_fail().unwrap_err(); +/// +/// # fn should_pass_2() -> Result<()> { +/// let value: HashSet = [1, 2, 3].into(); +/// verify_that!(value, superset_of([&1, &2, &3]))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass_2().unwrap(); +/// ``` +/// +/// Item multiplicity in both the actual and expected containers is ignored: +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// let value: Vec = vec![0, 0, 1]; +/// verify_that!(value, superset_of([&0, &1]))?; // Passes +/// verify_that!(value, superset_of([&0, &1, &1]))?; // Passes +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// ``` +/// +/// A note on performance: This matcher uses a naive algorithm with a worst-case +/// runtime proportional to the *product* of the sizes of the actual and +/// expected containers as well as the time to check equality of each pair of +/// items. It should not be used on especially large containers. +pub fn superset_of(subset: ExpectedT) -> SupersetOfMatcher { + SupersetOfMatcher { subset } +} + +#[derive(MatcherBase)] +pub struct SupersetOfMatcher { + subset: ExpectedT, +} + +impl Matcher + for SupersetOfMatcher +where + ActualT: IntoIterator, + for<'a> &'a ExpectedT: IntoIterator, +{ + fn matches(&self, actual: ActualT) -> MatcherResult { + for expected_item in &self.subset { + if actual_is_missing(actual, expected_item) { + return MatcherResult::NoMatch; + } + } + MatcherResult::Match + } + + fn explain_match(&self, actual: ActualT) -> Description { + let missing_items: Vec<_> = self + .subset + .into_iter() + .filter(|expected_item| actual_is_missing(actual, *expected_item)) + .map(|expected_item| format!("{expected_item:#?}")) + .collect(); + match missing_items.len() { + 0 => "whose no element is missing".into(), + 1 => format!("whose element {} is missing", &missing_items[0]).into(), + _ => format!("whose elements {} are missing", missing_items.join(", ")).into(), + } + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => format!("is a superset of {:#?}", self.subset).into(), + MatcherResult::NoMatch => format!("isn't a superset of {:#?}", self.subset).into(), + } + } +} + +fn actual_is_missing(actual: ActualT, needle: &ElementT) -> bool +where + ActualT: IntoIterator, +{ + !actual.into_iter().any(|item| &item == needle) +} +#[cfg(test)] +mod tests { + use crate as googletest; + use crate::prelude::*; + use indoc::indoc; + use std::collections::HashSet; + + #[test] + fn superset_of_matches_empty_vec() -> googletest::Result<()> { + let value: Vec = vec![]; + verify_that!(value, superset_of([])) + } + + #[test] + fn superset_of_matches_vec_with_one_element() -> googletest::Result<()> { + let value = vec![1]; + verify_that!(value, superset_of([&1])) + } + + #[test] + fn superset_of_matches_vec_with_two_items() -> googletest::Result<()> { + let value = vec![1, 2]; + verify_that!(value, superset_of([&1, &2])) + } + + #[test] + fn superset_of_matches_vec_when_actual_has_excess_element() -> googletest::Result<()> { + let value = vec![1, 2, 3]; + verify_that!(value, superset_of([&1, &2])) + } + + #[test] + fn superset_of_matches_vec_when_actual_has_excess_element_first() -> googletest::Result<()> { + let value = vec![3, 1, 2]; + verify_that!(value, superset_of([&1, &2])) + } + + #[test] + fn superset_of_matches_slice_with_one_element() -> googletest::Result<()> { + let value = &[1]; + verify_that!(value, superset_of([&1])) + } + + #[test] + fn superset_of_matches_hash_set_with_one_element() -> googletest::Result<()> { + let value: HashSet = [1].into(); + verify_that!(value, superset_of([&1])) + } + + #[test] + fn superset_of_does_not_match_when_first_element_does_not_match() -> googletest::Result<()> { + let value = vec![0]; + verify_that!(value, not(superset_of([&1]))) + } + + #[test] + fn superset_of_does_not_match_when_second_element_does_not_match() -> googletest::Result<()> { + let value = vec![2]; + verify_that!(value, not(superset_of([&2, &0]))) + } + + #[test] + fn superset_of_shows_correct_message_when_first_item_does_not_match() -> googletest::Result<()> + { + let result = verify_that!(vec![0, 2, 3], superset_of([&1, &2, &3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![0, 2, 3] + Expected: is a superset of [ + 1, + 2, + 3, + ] + Actual: [0, 2, 3], + whose element 1 is missing + " + )))) + ) + } + + #[test] + fn superset_of_shows_correct_message_when_second_item_does_not_match() -> googletest::Result<()> + { + let result = verify_that!(vec![1, 0, 3], superset_of([&1, &2, &3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 0, 3] + Expected: is a superset of [ + 1, + 2, + 3, + ] + Actual: [1, 0, 3], + whose element 2 is missing + " + )))) + ) + } + + #[test] + fn superset_of_shows_correct_message_when_first_two_items_do_not_match( + ) -> googletest::Result<()> { + let result = verify_that!(vec![0, 0, 3], superset_of([&1, &2, &3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![0, 0, 3] + Expected: is a superset of [ + 1, + 2, + 3, + ] + Actual: [0, 0, 3], + whose elements 1, 2 are missing + " + )))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/tuple_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/tuple_matcher.rs new file mode 100644 index 0000000..4822d52 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/tuple_matcher.rs @@ -0,0 +1,255 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented at the top level. +#![doc(hidden)] + +/// Functions for use only by the declarative macros in this module. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use crate::{ + description::Description, + matcher::{Matcher, MatcherBase, MatcherResult}, + }; + use std::fmt::Debug; + + impl MatcherBase for () {} + + // This implementation is provided for completeness, but is completely trivial. + // The only actual value which can be supplied is (), which must match. + impl Matcher<()> for () { + fn matches(&self, _: ()) -> MatcherResult { + MatcherResult::Match + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => "is the empty tuple".into(), + MatcherResult::NoMatch => "is not the empty tuple".into(), + } + } + } + + impl Matcher<&()> for () { + fn matches(&self, _: &()) -> MatcherResult { + MatcherResult::Match + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + >::describe(self, matcher_result) + } + } + + /// Generates a tuple matcher for tuples of a specific length. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + macro_rules! tuple_matcher_n { + ($([$field_number:tt, $matcher_type:ident, $field_type:ident]),*) => { + impl<$($matcher_type: MatcherBase),*> MatcherBase for ($($matcher_type,)*){} + + impl<$($field_type: Debug + Copy, $matcher_type: Matcher<$field_type>),*> + Matcher<($($field_type,)*)> for ($($matcher_type,)*) + { + fn matches(&self, actual: ($($field_type,)*)) -> MatcherResult { + $(match self.$field_number.matches(actual.$field_number) { + MatcherResult::Match => {}, + MatcherResult::NoMatch => { + return MatcherResult::NoMatch; + } + })* + MatcherResult::Match + } + + fn explain_match(&self, actual: ($($field_type,)*)) -> Description { + let mut explanation = Description::new().text("which").nested( + self.describe(self.matches(actual))); + $(match self.$field_number.matches(actual.$field_number) { + MatcherResult::Match => {}, + MatcherResult::NoMatch => { + explanation = explanation + .text(format!(concat!("Element #", $field_number, " is {:?},"), + actual.$field_number)) + .nested(self.$field_number.explain_match(actual.$field_number)); + } + })* + explanation + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + let mut description = Description::new().text( + "is a tuple whose values respectively match:"); + $(description = description.nested( + self.$field_number.describe(matcher_result));)* + description + } + MatcherResult::NoMatch => { + let mut description = Description::new().text( + "is a tuple whose values do not respectively match:"); + $(description = description.nested( + self.$field_number.describe(MatcherResult::Match));)* + description + } + } + } + } + impl<'a, $($field_type: Debug, $matcher_type: Matcher<&'a $field_type>),*> + Matcher<&'a ($($field_type,)*)> for ($($matcher_type,)*) + { + fn matches(&self, actual: &'a ($($field_type,)*)) -> MatcherResult { + $(match self.$field_number.matches(&actual.$field_number) { + MatcherResult::Match => {}, + MatcherResult::NoMatch => { + return MatcherResult::NoMatch; + } + })* + MatcherResult::Match + } + + fn explain_match(&self, actual: &'a ($($field_type,)*)) -> Description { + let mut explanation = Description::new() + .text("which") + .nested( + Matcher::<&'a ($($field_type,)*)>::describe( + self, self.matches(actual))); + $(match self.$field_number.matches(&actual.$field_number) { + MatcherResult::Match => {}, + MatcherResult::NoMatch => { + explanation = explanation + .text(format!( + concat!( + "Element #", + $field_number, + " is {:?},"), + actual.$field_number)) + .nested(self.$field_number.explain_match(&actual.$field_number)); + } + })* + explanation + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + match matcher_result { + MatcherResult::Match => { + let mut description = Description::new().text( + "is a tuple whose values respectively match:"); + $(description = description.nested( + self.$field_number.describe(matcher_result));)* + description + } + MatcherResult::NoMatch => { + let mut description = Description::new().text( + "is a tuple whose values do not respectively match:"); + $(description = description.nested( + self.$field_number.describe(MatcherResult::Match));)* + description + } + } + } + } + }; + } + + tuple_matcher_n!([0, I0, T0]); + + tuple_matcher_n!([0, I0, T0], [1, I1, T1]); + + tuple_matcher_n!([0, I0, T0], [1, I1, T1], [2, I2, T2]); + + tuple_matcher_n!([0, I0, T0], [1, I1, T1], [2, I2, T2], [3, I3, T3]); + + tuple_matcher_n!([0, I0, T0], [1, I1, T1], [2, I2, T2], [3, I3, T3], [4, I4, T4]); + + tuple_matcher_n!([0, I0, T0], [1, I1, T1], [2, I2, T2], [3, I3, T3], [4, I4, T4], [5, I5, T5]); + + tuple_matcher_n!( + [0, I0, T0], + [1, I1, T1], + [2, I2, T2], + [3, I3, T3], + [4, I4, T4], + [5, I5, T5], + [6, I6, T6] + ); + + tuple_matcher_n!( + [0, I0, T0], + [1, I1, T1], + [2, I2, T2], + [3, I3, T3], + [4, I4, T4], + [5, I5, T5], + [6, I6, T6], + [7, I7, T7] + ); + + tuple_matcher_n!( + [0, I0, T0], + [1, I1, T1], + [2, I2, T2], + [3, I3, T3], + [4, I4, T4], + [5, I5, T5], + [6, I6, T6], + [7, I7, T7], + [8, I8, T8] + ); + + tuple_matcher_n!( + [0, I0, T0], + [1, I1, T1], + [2, I2, T2], + [3, I3, T3], + [4, I4, T4], + [5, I5, T5], + [6, I6, T6], + [7, I7, T7], + [8, I8, T8], + [9, I9, T9] + ); + + tuple_matcher_n!( + [0, I0, T0], + [1, I1, T1], + [2, I2, T2], + [3, I3, T3], + [4, I4, T4], + [5, I5, T5], + [6, I6, T6], + [7, I7, T7], + [8, I8, T8], + [9, I9, T9], + [10, I10, T10] + ); + + tuple_matcher_n!( + [0, I0, T0], + [1, I1, T1], + [2, I2, T2], + [3, I3, T3], + [4, I4, T4], + [5, I5, T5], + [6, I6, T6], + [7, I7, T7], + [8, I8, T8], + [9, I9, T9], + [10, I10, T10], + [11, I11, T11] + ); +} diff --git a/third_party/crates/googletest-0.14.3/src/matchers/unordered_elements_are_matcher.rs b/third_party/crates/googletest-0.14.3/src/matchers/unordered_elements_are_matcher.rs new file mode 100644 index 0000000..b7d6b91 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/src/matchers/unordered_elements_are_matcher.rs @@ -0,0 +1,609 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// There are no visible documentation elements in this module; the declarative +// macro is documented in the matchers module. +#![doc(hidden)] + +/// Matches a container whose elements in any order have a 1:1 correspondence +/// with the provided element matchers. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(vec![3, 2, 1], unordered_elements_are![eq(&1), ge(&2), anything()])?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!(vec![1], unordered_elements_are![eq(&1), ge(&2)])?; // Fails: container has wrong size +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!(vec![3, 2, 1], unordered_elements_are![eq(&1), ge(&4), eq(&2)])?; // Fails: second matcher not matched +/// # Ok(()) +/// # } +/// # fn should_fail_3() -> Result<()> { +/// verify_that!(vec![3, 2, 1], unordered_elements_are![ge(&3), ge(&3), ge(&3)])?; // Fails: no 1:1 correspondence +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_fail_3().unwrap_err(); +/// ``` +/// +/// The actual value must be a container such as a `&Vec`, an array, or a +/// slice. More precisely, the actual value must implement [`IntoIterator`]. +/// +/// This can also be omitted in [`verify_that!`] macros and replaced with curly +/// brackets. +/// +/// ``` +/// # use googletest::prelude::*; +/// verify_that!(vec![1, 2], {eq(&2), eq(&1)}) +/// # .unwrap(); +/// ``` +/// +/// Note: This behavior is only possible in [`verify_that!`] macros. In any +/// other cases, it is still necessary to use the +/// [`unordered_elements_are!`][crate::matchers::unordered_elements_are] macro. +/// +/// ```compile_fail +/// # use googletest::prelude::*; +/// verify_that!(vec![vec![1,2], vec![3]], {{eq(2), eq(1)}, {eq(3)}}) +/// # .unwrap(); +/// ``` +/// +/// Use this instead: +/// ``` +/// # use googletest::prelude::*; +/// verify_that!(vec![vec![1,2], vec![3]], +/// {unordered_elements_are![eq(&2), eq(&1)], unordered_elements_are![eq(&3)]}) +/// # .unwrap(); +/// ``` +/// +/// If an inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// +/// verify_that!(vec![1,2,3], unordered_elements_are![lt(&2), gt(&1), &3]) +/// # .unwrap(); +/// ``` +/// +/// The matcher proceeds in three stages: +/// +/// 1. It first checks whether the actual value is of the right size to possibly +/// be matched by each of the given matchers. If not, then it immediately +/// fails explaining that the size is incorrect. +/// +/// 2. It then checks whether each matcher matches at least one corresponding +/// element in the actual container and each element in the actual container +/// is matched by at least one matcher. If not, it fails with a message +/// indicating which matcher respectively container elements had no +/// counterparts. +/// +/// 3. Finally, it checks whether the mapping of matchers to corresponding +/// actual elements is a 1-1 correspondence and fails if that is not the +/// case. The failure message then shows the best matching it could find, +/// including which matchers did not have corresponding unique elements in +/// the container and which container elements had no corresponding matchers. +/// +/// [`IntoIterator`]: std::iter::IntoIterator +/// [`Iterator`]: std::iter::Iterator +/// [`Iterator::collect`]: std::iter::Iterator::collect +/// [`Vec`]: std::vec::Vec +#[macro_export] +#[doc(hidden)] +macro_rules! __unordered_elements_are { + ($(,)?) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these:: + UnorderedElementsAreMatcher::new( + (), + $crate::matchers::__internal_unstable_do_not_depend_on_these:: + Requirements::PerfectMatch) + }}; + + ($($matcher:expr),* $(,)?) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these:: + UnorderedElementsAreMatcher::new( + ($( + $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq!( + $matcher + ) + ,)*), + $crate::matchers::__internal_unstable_do_not_depend_on_these:: + Requirements::PerfectMatch) + }}; +} + +/// Matches a container containing elements matched by the given matchers. +/// +/// To match, each given matcher must have a corresponding element in the +/// container which it matches. There must be a mapping uniquely matching each +/// matcher to a container element. The container can, however, contain +/// additional elements that don't correspond to any matcher. +/// +/// Put another way, `contains_each!` matches if there is a subset of the actual +/// container which +/// [`unordered_elements_are`][crate::matchers::unordered_elements_are] would +/// match. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(vec![3, 2, 1], contains_each![eq(&2), ge(&3)])?; // Passes +/// verify_that!(vec![3, 2, 1], contains_each![ge(&2), ge(&2)])?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!(vec![1], contains_each![eq(&1), ge(&2)])?; // Fails: container too small +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!(vec![3, 2, 1], contains_each![eq(&1), ge(&4)])?; // Fails: second matcher unmatched +/// # Ok(()) +/// # } +/// # fn should_fail_3() -> Result<()> { +/// verify_that!(vec![3, 2, 1], contains_each![ge(&3), ge(&3), ge(&3)])?; // Fails: no matching +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_fail_3().unwrap_err(); +/// ``` +/// +/// The actual value must be a container such as a `&Vec`, an array, or a +/// slice. More precisely, the actual value must implement [`IntoIterator`]. +/// +/// If an inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// +/// verify_that!(vec![1,2,3], contains_each![lt(&2), &3]) +/// # .unwrap(); +/// ``` +/// +/// The matcher proceeds in three stages: +/// +/// 1. It first checks whether the actual value is large enough to possibly be +/// matched by each of the given matchers. If not, then it immediately fails +/// explaining that the size is too small. +/// +/// 2. It then checks whether each matcher matches at least one corresponding +/// element in the actual container and fails if that is not the case. The +/// failure message indicates which matcher had no corresponding element. +/// +/// 3. Finally, it checks whether the mapping of matchers to corresponding +/// actual elements is 1-1 and fails if that is not the case. The failure +/// message then shows the best matching it could find, including which +/// matchers did not have corresponding unique elements in the container. +/// +/// [`IntoIterator`]: std::iter::IntoIterator +/// [`Iterator`]: std::iter::Iterator +/// [`Iterator::collect`]: std::iter::Iterator::collect +/// [`Vec`]: std::vec::Vec +#[macro_export] +#[doc(hidden)] +macro_rules! __contains_each { + ($(,)?) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these:: + UnorderedElementsAreMatcher::new( + (), + $crate::matchers::__internal_unstable_do_not_depend_on_these::Requirements::Superset) + }}; + + ($($matcher:expr),* $(,)?) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these:: + UnorderedElementsAreMatcher::new( + ($( + $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq!( + $matcher + ) + ,)*), + $crate::matchers::__internal_unstable_do_not_depend_on_these::Requirements::Superset) + }} +} + +/// Matches a container all of whose elements are matched by the given matchers. +/// +/// To match, each element in the container must have a corresponding matcher +/// which matches it. There must be a 1-1 mapping from container elements to +/// matchers, so that no matcher has more than one corresponding element. +/// +/// There may, however, be matchers not corresponding to any elements in the +/// container. +/// +/// Put another way, `is_contained_in!` matches if there is a subset of the +/// matchers which would match with +/// [`unordered_elements_are`][crate::matchers::unordered_elements_are]. +/// +/// ``` +/// # use googletest::prelude::*; +/// # fn should_pass() -> Result<()> { +/// verify_that!(vec![2, 1], is_contained_in![eq(&1), ge(&2)])?; // Passes +/// verify_that!(vec![2, 1], is_contained_in![ge(&1), ge(&1)])?; // Passes +/// # Ok(()) +/// # } +/// # fn should_fail_1() -> Result<()> { +/// verify_that!(vec![1, 2, 3], is_contained_in![eq(&1), ge(&2)])?; // Fails: container too large +/// # Ok(()) +/// # } +/// # fn should_fail_2() -> Result<()> { +/// verify_that!(vec![2, 1], is_contained_in![eq(&1), ge(&4)])?; // Fails: second matcher unmatched +/// # Ok(()) +/// # } +/// # fn should_fail_3() -> Result<()> { +/// verify_that!(vec![3, 1], is_contained_in![ge(&3), ge(&3), ge(&3)])?; // Fails: no matching +/// # Ok(()) +/// # } +/// # should_pass().unwrap(); +/// # should_fail_1().unwrap_err(); +/// # should_fail_2().unwrap_err(); +/// # should_fail_3().unwrap_err(); +/// ``` +/// +/// The actual value must be a container such as a `&Vec`, an array, or a slice. +/// More precisely, the actual value must implement [`IntoIterator`]. +/// +/// If an inner matcher is `eq(...)`, it can be omitted: +/// +/// ``` +/// # use googletest::prelude::*; +/// +/// verify_that!(vec![1,2,3], is_contained_in![lt(&2), &3, &4, gt(&0)]) +/// # .unwrap(); +/// ``` +/// +/// The matcher proceeds in three stages: +/// +/// 1. It first checks whether the actual value is too large to possibly be +/// matched by each of the given matchers. If so, it immediately fails +/// explaining that the size is too large. +/// +/// 2. It then checks whether each actual container element is matched by at +/// least one matcher and fails if that is not the case. The failure message +/// indicates which element had no corresponding matcher. +/// +/// 3. Finally, it checks whether the mapping of elements to corresponding +/// matchers is 1-1 and fails if that is not the case. The failure message +/// then shows the best matching it could find, including which container +/// elements did not have corresponding matchers. +/// +/// [`IntoIterator`]: std::iter::IntoIterator +/// [`Iterator`]: std::iter::Iterator +/// [`Iterator::collect`]: std::iter::Iterator::collect +/// [`Vec`]: std::vec::Vec +#[macro_export] +#[doc(hidden)] +macro_rules! __is_contained_in { + ($(,)?) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these:: + UnorderedElementsAreMatcher::new( + (), $crate::matchers::__internal_unstable_do_not_depend_on_these::Requirements::Subset) + }}; + + ($($matcher:expr),* $(,)?) => {{ + $crate::matchers::__internal_unstable_do_not_depend_on_these:: + UnorderedElementsAreMatcher::new( + ($( + $crate::matcher_support::__internal_unstable_do_not_depend_on_these::auto_eq!( + $matcher + ) + ,)*), + $crate::matchers::__internal_unstable_do_not_depend_on_these::Requirements::Subset) + }} +} + +/// Module for use only by the macros in this module. +/// +/// **For internal use only. API stablility is not guaranteed!** +#[doc(hidden)] +pub mod internal { + use crate::description::Description; + use crate::matcher::{Matcher, MatcherBase, MatcherResult}; + use crate::matcher_support::match_matrix::internal::{MatchMatrix, Requirements}; + use std::fmt::Debug; + + /// A trait for a tuple of matchers. + pub trait MatcherTuple { + /// Turn the tuple into a vector of matchers. + fn to_matchers(&self) -> Vec + '_>>; + } + + // Base case, no matchers. + impl MatcherTuple for () { + fn to_matchers(&self) -> Vec + '_>> { + vec![] + } + } + + // A macro for generating a MatcherTuple implementation for a given number + // of matchers. + macro_rules! matcher_tuple_n { + // Given list of (field_number, matcher) pairs, generate a MatcherTuple + // implementation. + ($([$field_number:tt, $matcher_type:ident]),*) => { + impl),*> MatcherTuple + for ($($matcher_type,)*) { + // Our implementation of to_matchers just converts our tuple into a vector of + // matchers by expanding self.0, self.1, etc. + fn to_matchers(&self) -> Vec + '_>> { + vec![$(Box::new(&self.$field_number)),*] + } + } + }; + } + + matcher_tuple_n!([0, M0]); + matcher_tuple_n!([0, M0], [1, M1]); + matcher_tuple_n!([0, M0], [1, M1], [2, M2]); + matcher_tuple_n!([0, M0], [1, M1], [2, M2], [3, M3]); + matcher_tuple_n!([0, M0], [1, M1], [2, M2], [3, M3], [4, M4]); + matcher_tuple_n!([0, M0], [1, M1], [2, M2], [3, M3], [4, M4], [5, M5]); + matcher_tuple_n!([0, M0], [1, M1], [2, M2], [3, M3], [4, M4], [5, M5], [6, M6]); + matcher_tuple_n!([0, M0], [1, M1], [2, M2], [3, M3], [4, M4], [5, M5], [6, M6], [7, M7]); + matcher_tuple_n!( + [0, M0], + [1, M1], + [2, M2], + [3, M3], + [4, M4], + [5, M5], + [6, M6], + [7, M7], + [8, M8] + ); + matcher_tuple_n!( + [0, M0], + [1, M1], + [2, M2], + [3, M3], + [4, M4], + [5, M5], + [6, M6], + [7, M7], + [8, M8], + [9, M9] + ); + matcher_tuple_n!( + [0, M0], + [1, M1], + [2, M2], + [3, M3], + [4, M4], + [5, M5], + [6, M6], + [7, M7], + [8, M8], + [9, M9], + [10, M10] + ); + matcher_tuple_n!( + [0, M0], + [1, M1], + [2, M2], + [3, M3], + [4, M4], + [5, M5], + [6, M6], + [7, M7], + [8, M8], + [9, M9], + [10, M10], + [11, M11] + ); + + /// This struct is meant to be used only through the + /// `unordered_elements_are![...]` macro. + /// + /// **For internal use only. API stablility is not guaranteed!** + #[doc(hidden)] + #[derive(MatcherBase)] + pub struct UnorderedElementsAreMatcher { + matchers: M, + requirements: Requirements, + } + + impl UnorderedElementsAreMatcher { + pub fn new(matchers: M, requirements: Requirements) -> Self { + Self { matchers, requirements } + } + } + + // This matcher performs the checks in three different steps in both `matches` + // and `explain_match`. This is useful for performance but also to produce + // an actionable error message. + // 1. `UnorderedElementsAreMatcher` verifies that both collections have the same + // size + // 2. `UnorderedElementsAreMatcher` verifies that each actual element matches at + // least one expected element and vice versa. + // 3. `UnorderedElementsAreMatcher` verifies that a perfect matching exists + // using Ford-Fulkerson. + impl Matcher + for UnorderedElementsAreMatcher + where + ContainerT: IntoIterator, + M: MatcherTuple, + { + fn matches(&self, actual: ContainerT) -> MatcherResult { + let matchers = self.matchers.to_matchers(); + let match_matrix = MatchMatrix::generate(actual, &matchers); + match_matrix.is_match_for(self.requirements).into() + } + + fn explain_match(&self, actual: ContainerT) -> Description { + let matchers = self.matchers.to_matchers(); + if let Some(size_mismatch_explanation) = + self.requirements.explain_size_mismatch(actual, matchers.len()) + { + return size_mismatch_explanation; + } + + let match_matrix = MatchMatrix::generate(actual, &matchers); + if let Some(unmatchable_explanation) = + match_matrix.explain_unmatchable(self.requirements) + { + return unmatchable_explanation; + } + + let best_match = match_matrix.find_best_match(); + best_match + .get_explanation(actual, &matchers, self.requirements) + .unwrap_or("whose elements all match".into()) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + let matchers = self.matchers.to_matchers(); + format!( + "{} elements matching in any order:\n{}", + if matcher_result.into() { "contains" } else { "doesn't contain" }, + matchers + .iter() + .map(|matcher| matcher.describe(MatcherResult::Match)) + .collect::() + .enumerate() + .indent() + ) + .into() + } + } +} + +#[cfg(test)] +mod tests { + use crate as googletest; + use crate::description::Description; + use crate::matcher::MatcherResult; + use crate::prelude::*; + use indoc::indoc; + use std::collections::{HashMap, HashSet}; + + #[test] + fn has_correct_description_for_map() -> googletest::Result<()> { + // UnorderedElementsAreMatcher maintains references to the matchers, so the + // constituent matchers must live longer. Inside a verify_that! macro, the + // compiler takes care of that, but when the matcher is created separately, + // we must create the constitute matchers separately so that they + // aren't dropped too early. + let matchers = ((eq(&2), eq(&"Two")), (eq(&1), eq(&"One")), (eq(&3), eq(&"Three"))); + let matcher = unordered_elements_are![ + (matchers.0 .0, matchers.0 .1), + (matchers.1 .0, matchers.1 .1), + (matchers.2 .0, matchers.2 .1) + ]; + verify_that!( + Matcher::<&HashMap>::describe(&matcher, MatcherResult::Match), + displays_as(eq(indoc!( + " + contains elements matching in any order: + 0. is a tuple whose values respectively match: + is equal to 2 + is equal to \"Two\" + 1. is a tuple whose values respectively match: + is equal to 1 + is equal to \"One\" + 2. is a tuple whose values respectively match: + is equal to 3 + is equal to \"Three\"" + ))) + ) + } + + #[test] + fn unordered_elements_are_description_no_full_match_with_map() -> googletest::Result<()> { + // UnorderedElementsAreMatcher maintains references to the matchers, so the + // constituent matchers must live longer. Inside a verify_that! macro, the + // compiler takes care of that, but when the matcher is created separately, + // we must create the constitute matchers separately so that they + // aren't dropped too early. + let value: HashMap = HashMap::from_iter([(0, 1), (1, 1), (2, 2)]); + let matchers = ((anything(), eq(&1)), (anything(), eq(&2)), (anything(), eq(&2))); + let matcher = unordered_elements_are![ + (matchers.0 .0, matchers.0 .1), + (matchers.1 .0, matchers.1 .1), + (matchers.2 .0, matchers.2 .1), + ]; + verify_that!( + matcher.explain_match(&value), + all![ + displays_as(contains_regex( + "Actual element \\(2, 2\\) at index [0-2] matched expected element `is a tuple whose values respectively match:\n is anything\n is equal to 2` at index [0-2]." + )), + displays_as(contains_regex( + "Actual element \\(\n [0-1],\n [0-1],\n \\) at index [0-2] did not match any remaining expected element." + )), + displays_as(contains_substring( + "Expected element `is a tuple whose values respectively match:\n is anything\n is equal to 2` at index 2 did not match any remaining actual element." + )) + ] + ) + } + + // This test verifies that unordered_elements_are can be used as the inner + // matcher of another matcher, where the inner matcher is given a reference + // to a local variable. If unordered_elements_are stores its matchers in a + // Vec with the same type and lifetimes then this fails to compile. + #[test] + fn works_with_inner_matcher_that_modifies_the_input() -> googletest::Result<()> { + // A custom matcher that applies an operation to the input hashmap (filtering + // out any keys present in `to_filter`) and then applies the inner + // matcher to the modified input. + fn matches_filtered<'a, M>( + to_filter: &'a HashSet, + inner: M, + ) -> impl Matcher<&'a HashSet> + where + M: for<'b> Matcher<&'b HashSet>, + { + #[derive(MatcherBase)] + struct FilteredMatcher<'a, M> { + to_filter: &'a HashSet, + inner: M, + } + + impl<'a, M> Matcher<&'a HashSet> for FilteredMatcher<'a, M> + where + M: for<'b> Matcher<&'b HashSet>, + { + fn matches(&self, actual: &'a HashSet) -> MatcherResult { + let filtered: HashSet = + actual.iter().filter(|k| !self.to_filter.contains(k)).cloned().collect(); + self.inner.matches(&filtered) + } + + fn describe(&self, matcher_result: MatcherResult) -> Description { + self.inner.describe(matcher_result) + } + + fn explain_match(&self, actual: &'a HashSet) -> Description { + let filtered: HashSet = + actual.iter().filter(|k| !self.to_filter.contains(k)).cloned().collect(); + self.inner.explain_match(&filtered) + } + } + + FilteredMatcher { to_filter, inner } + } + + let before: HashSet = HashSet::from([1]); + let after: HashSet = HashSet::from([1, 2]); + + verify_that!(&after, matches_filtered(&before, unordered_elements_are![eq(&2)]))?; + + Ok(()) + } +} diff --git a/third_party/crates/googletest-0.14.3/tests/all_matcher_test.rs b/third_party/crates/googletest-0.14.3/tests/all_matcher_test.rs new file mode 100644 index 0000000..34b8515 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/all_matcher_test.rs @@ -0,0 +1,155 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; +use indoc::indoc; + +#[test] +fn matches_any_value_when_list_is_empty() -> Result<()> { + verify_that!((), all!()) +} + +#[test] +fn matches_value_with_single_matching_component() -> Result<()> { + verify_that!(123, all!(eq(123))) +} + +#[test] +fn does_not_match_value_with_single_non_matching_component() -> Result<()> { + verify_that!(123, not(all!(eq(456)))) +} + +#[test] +fn matches_value_with_two_matching_components() -> Result<()> { + verify_that!("A string", all!(starts_with("A"), ends_with("string"))) +} + +#[test] +fn does_not_match_value_with_one_non_matching_component_among_two_components() -> Result<()> { + verify_that!(123, not(all!(eq(123), eq(456)))) +} + +#[test] +fn supports_trailing_comma() -> Result<()> { + verify_that!( + "An important string", + all!(starts_with("An"), contains_substring("important"), ends_with("string"),) + ) +} + +#[test] +fn admits_matchers_without_static_lifetime() -> Result<()> { + #[derive(Debug, PartialEq)] + struct AStruct(i32); + let expected_value = AStruct(123); + verify_that!(AStruct(123), all![eq(&expected_value)]) +} + +#[test] +fn mismatch_description_two_failed_matchers() -> Result<()> { + verify_that!( + all!(starts_with("One"), starts_with("Two")).explain_match("Three"), + displays_as(eq("* which does not start with \"One\"\n* which does not start with \"Two\"")) + ) +} + +#[test] +fn mismatch_description_empty_matcher() -> Result<()> { + verify_that!(all!().explain_match("Three"), displays_as(eq("which is anything"))) +} + +#[test] +fn all_multiple_failed_assertions() -> Result<()> { + let result = verify_that!(4, all![eq(1), eq(2), eq(3)]); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 4 + Expected: has all the following properties: + * is equal to 1 + * is equal to 2 + * is equal to 3 + Actual: 4, + * which isn't equal to 1 + * which isn't equal to 2 + * which isn't equal to 3" + )))) + ) +} + +#[test] +fn formats_error_message_correctly_when_all_is_inside_some() -> Result<()> { + let value = Some(4); + let result = verify_that!(value, some(all![eq(1), eq(2), eq(3)])); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: value + Expected: has a value which has all the following properties: + * is equal to 1 + * is equal to 2 + * is equal to 3 + Actual: Some(4), + which has a value + * which isn't equal to 1 + * which isn't equal to 2 + * which isn't equal to 3" + )))) + ) +} + +#[test] +fn formats_error_message_correctly_when_all_is_inside_ok() -> Result<()> { + let value: std::result::Result = Ok(4); + let result = verify_that!(value, ok(all![eq(&1), eq(&2), eq(&3)])); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: value + Expected: is a success containing a value, which has all the following properties: + * is equal to 1 + * is equal to 2 + * is equal to 3 + Actual: Ok(4), + which is a success + * which isn't equal to 1 + * which isn't equal to 2 + * which isn't equal to 3" + )))) + ) +} + +#[test] +fn formats_error_message_correctly_when_all_is_inside_err() -> Result<()> { + let value: std::result::Result<(), &'static str> = Err("An error"); + let result = verify_that!(value, err(all![starts_with("Not"), ends_with("problem")])); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: value + Expected: is an error which has all the following properties: + * starts with prefix "Not" + * ends with suffix "problem" + Actual: Err("An error"), + which is an error + * which does not start with "Not" + * which does not end with "problem""# + )))) + ) +} diff --git a/third_party/crates/googletest-0.14.3/tests/any_matcher_test.rs b/third_party/crates/googletest-0.14.3/tests/any_matcher_test.rs new file mode 100644 index 0000000..4ff6a87 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/any_matcher_test.rs @@ -0,0 +1,145 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; +use indoc::indoc; + +#[test] +fn matches_value_with_single_matching_component() -> Result<()> { + verify_that!(123, any!(eq(123))) +} + +#[test] +fn does_not_match_value_with_single_non_matching_component() -> Result<()> { + verify_that!(123, not(any!(eq(456)))) +} + +#[test] +fn matches_value_with_first_of_two_matching_components() -> Result<()> { + verify_that!("A string", any!(starts_with("A"), starts_with("string"))) +} + +#[test] +fn matches_value_with_second_of_two_matching_components() -> Result<()> { + verify_that!("A string", any!(starts_with("string"), starts_with("A"))) +} + +#[test] +fn supports_trailing_comma() -> Result<()> { + verify_that!( + "An important string", + any!(starts_with("An"), contains_substring("important"), ends_with("string"),) + ) +} + +#[test] +fn admits_matchers_without_static_lifetime() -> Result<()> { + #[derive(Debug, PartialEq)] + struct AStruct(i32); + let expected_value = AStruct(123); + verify_that!(AStruct(123), any![eq(&expected_value)]) +} + +#[test] +fn mismatch_description_two_failed_matchers() -> Result<()> { + verify_that!( + any!(starts_with("One"), starts_with("Two")).explain_match("Three"), + displays_as(eq("* which does not start with \"One\"\n* which does not start with \"Two\"")) + ) +} + +#[test] +fn all_multiple_failed_assertions() -> Result<()> { + let result = verify_that!(4, any![eq(1), eq(2), eq(3)]); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: 4 + Expected: has at least one of the following properties: + * is equal to 1 + * is equal to 2 + * is equal to 3 + Actual: 4, + * which isn't equal to 1 + * which isn't equal to 2 + * which isn't equal to 3" + )))) + ) +} + +#[test] +fn formats_error_message_correctly_when_any_is_inside_some() -> Result<()> { + let value = Some(4); + let result = verify_that!(value, some(any![eq(1), eq(2), eq(3)])); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: value + Expected: has a value which has at least one of the following properties: + * is equal to 1 + * is equal to 2 + * is equal to 3 + Actual: Some(4), + which has a value + * which isn't equal to 1 + * which isn't equal to 2 + * which isn't equal to 3" + )))) + ) +} + +#[test] +fn formats_error_message_correctly_when_any_is_inside_ok() -> Result<()> { + let value: std::result::Result = Ok(4); + let result = verify_that!(value, ok(any![eq(&1), eq(&2), eq(&3)])); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: value + Expected: is a success containing a value, which has at least one of the following properties: + * is equal to 1 + * is equal to 2 + * is equal to 3 + Actual: Ok(4), + which is a success + * which isn't equal to 1 + * which isn't equal to 2 + * which isn't equal to 3" + )))) + ) +} + +#[test] +fn formats_error_message_correctly_when_any_is_inside_err() -> Result<()> { + let value: std::result::Result<(), &'static str> = Err("An error"); + let result = verify_that!(value, err(any![starts_with("Not"), ends_with("problem")])); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + r#" + Value of: value + Expected: is an error which has at least one of the following properties: + * starts with prefix "Not" + * ends with suffix "problem" + Actual: Err("An error"), + which is an error + * which does not start with "Not" + * which does not end with "problem""# + )))) + ) +} diff --git a/third_party/crates/googletest-0.14.3/tests/assertions_test.rs b/third_party/crates/googletest-0.14.3/tests/assertions_test.rs new file mode 100644 index 0000000..de872a1 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/assertions_test.rs @@ -0,0 +1,428 @@ +mod verify_pred { + use googletest::prelude::*; + use googletest::Result; + use indoc::indoc; + + #[test] + fn supports_function_call_with_non_debug_types() -> Result<()> { + // Non-Debug - cannot be printed. + struct Apple; + fn f(_a: &Apple, _b: u32, _c: u32) -> bool { + false + } + fn g(_a: u32) -> u32 { + 5 + } + + let a = &Apple; + let res = verify_pred!(f(a, g(g(3)), 1 + 2)); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + f(a, g(g(3)), 1 + 2) was false with + a does not implement Debug, + g(g(3)) = 5, + 1 + 2 = 3, + at" + }))) + ) + } + + #[test] + fn supports_trailing_comma() -> Result<()> { + verify_that!(verify_pred!(false,), err(anything())) + } + + #[test] + fn supports_non_function() -> Result<()> { + verify_pred!(true)?; + verify_that!(verify_pred!(false), err(anything())) + } + + #[test] + fn does_not_print_literals() -> Result<()> { + trait Foo { + fn f(&self, _a: u32, _b: i32, _c: u32, _d: &str) -> bool { + false + } + } + impl Foo for i32 {} + + let res = verify_pred!(0.f(1, 2_i32.abs(), 1 + 2, "hello")); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {r#" + 0.f(1, 2_i32.abs(), 1 + 2, "hello") was false with + 2_i32.abs() = 2, + 1 + 2 = 3, + at"# + }))) + ) + } + + #[test] + fn supports_chained_field_access_and_method_calls_with_non_debug_types() -> Result<()> { + // Non-Debug + struct Apple { + b: Banana, + } + #[derive(Debug)] + struct Banana; + impl Banana { + fn c(&self, _c: &Cherry, _d: u32) -> bool { + false + } + } + // Non-Debug - cannot be printed. + struct Cherry; + + let a = Apple { b: Banana }; + let c = &Cherry; + let d = 3; + let res = verify_pred!(a.b.c(c, d)); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + a.b.c(c, d) was false with + a does not implement Debug, + a.b = Banana, + c does not implement Debug, + d = 3, + at" + }))) + ) + } + + #[test] + fn evaluates_functions_and_arguments_exactly_once() -> Result<()> { + let mut a = 0; + let mut foo = |_b: u32| { + a += 1; + false + }; + let mut b = 0; + let mut bar = || { + b += 10; + b + }; + + let res = verify_pred!(foo(bar())); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + foo(bar()) was false with + bar() = 10, + at" + }))) + )?; + + verify_that!((a, b), eq((1, 10))) + } + + #[test] + fn evaluates_methods_and_arguments_exactly_once() -> Result<()> { + struct Apple(u32); + impl Apple { + fn c(&mut self, _b: bool) -> bool { + self.0 += 1; + false + } + } + let mut a = Apple(0); + let mut b = Apple(10); + + let res = verify_pred!(a.c(b.c(false))); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + a.c(b.c(false)) was false with + a does not implement Debug, + b.c(false) = false, + at" + }))) + )?; + + verify_that!((a.0, b.0), eq((1, 11))) + } + + #[test] + fn supports_chained_method_calls() -> Result<()> { + #[derive(Debug)] + struct Apple; + impl Apple { + fn b(&self, _b: u32) -> Banana { + Banana + } + } + // Non-Debug: not printed on error. + struct Banana; + impl Banana { + fn c(&self, _c0: u32, _c1: Cherry) -> bool { + false + } + } + // Non-Debug: not printed on error. + #[derive(Copy, Clone)] + struct Cherry; + + let a = Apple; + let v = 10; + let res = verify_pred!(a.b(v).c(11, Cherry)); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + a.b(v).c(11, Cherry) was false with + a = Apple, + v = 10, + a.b(v) does not implement Debug, + Cherry does not implement Debug, + at" + }))) + ) + } + + #[test] + fn prints_consumed_values() -> Result<()> { + // Non-Debug + struct Apple; + impl Apple { + fn b(self) -> Banana { + Banana + } + } + #[derive(Debug)] + struct Banana; + impl Banana { + fn c(self) -> bool { + false + } + } + + let a = Apple; + let res = verify_pred!(a.b().c()); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + a.b().c() was false with + a does not implement Debug, + a.b() = Banana, + at" + }))) + ) + } + + #[test] + fn works_with_realistic_example_with_consumed_intermediate_values() -> Result<()> { + let res = + verify_pred!(vec![1, 2].into_iter().map(|x| x * 2).collect::>().is_empty()); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + vec! [1, 2].into_iter().map(| x | x * 2).collect :: < Vec < _ > > + ().is_empty() was false with + vec! [1, 2] = [1, 2], + vec! [1, 2].into_iter() = IntoIter([1, 2]), + | x | x * 2 does not implement Debug, + vec! [1, 2].into_iter().map(| x | x * 2) = Map { iter: IntoIter([1, 2]) }, + vec! [1, 2].into_iter().map(| x | x * 2).collect :: < Vec < _ > > () = [2, 4], + at" + }))) + ) + } + + #[test] + fn values_should_be_accessible_after_test() -> Result<()> { + // Not `Copy` and should not be consumed by the generated test code. + #[derive(Debug)] + struct Apple; + impl Apple { + fn b(&self, _c: &mut u32) -> bool { + false + } + } + + let mut c = 0; + let a = Apple; + let res = verify_pred!(a.b(&mut c)); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + a.b(& mut c) was false with + a = Apple, + & mut c = 0, + at" + }))) + )?; + + // `a` and `&mut c` should still be accessible after the test despite not being + // `Copy`. + let _ = a.b(&mut c); + + Ok(()) + } + + #[test] + fn prints_values_for_mutating_expressions() -> Result<()> { + let mut a = 1; + let mut b = 2; + let mut c = 0; + trait Mutator { + fn mutate_and_false(&mut self, b: &mut u32) -> bool; + } + impl Mutator for u32 { + fn mutate_and_false(&mut self, b: &mut u32) -> bool { + *self += 10; + *b += 20; + false + } + } + + // Macro to to avoid the inconsistency in how `;` and `&mut` are printed between + // Rust versions when printing out the stringified version of the block. + macro_rules! block_a { + () => {{ + c += 10; + &mut a + }}; + } + macro_rules! block_b { + () => {{ + c += 100; + &mut b + }}; + } + let res = verify_pred! { block_a!().mutate_and_false(block_b!()) }; + + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + block_a! ().mutate_and_false(block_b! ()) was false with + block_a! () = 1, + block_b! () = 2, + at" + }))) + )?; + + verify_that!((a, b, c), eq((11, 22, 110))) + } + + #[test] + fn values_can_be_insulated_with_parens() -> Result<()> { + // Not `Copy` and has a consuming method. + struct Apple; + impl Apple { + fn b(self) -> Banana { + Banana + } + } + #[derive(Debug)] + struct Banana; + impl Banana { + fn c(&self) -> bool { + false + } + } + + let a = Apple; + let res = verify_pred!({ a.b() }.c()); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + { a.b() }.c() was false with + { a.b() } = Banana, + at" + }))) + ) + } + + #[test] + fn binary_operator() -> Result<()> { + // Add chaining and function calls. + fn f(x: u32) -> u32 { + x + 1 + } + #[derive(Debug)] + struct Apple; + impl Apple { + fn b(&self, y: u32) -> u32 { + y + 10 + } + } + let a = Apple; + let x = 1; + let y = 2; + let res = verify_pred!(f(x) - 1 == a.b(y + 1) + f(y)); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + f(x) - 1 == a.b(y + 1) + f(y) was false with + x = 1, + f(x) = 2, + f(x) - 1 = 1, + a = Apple, + y + 1 = 3, + a.b(y + 1) = 13, + y = 2, + f(y) = 3, + a.b(y + 1) + f(y) = 16, + at" + }))) + ) + } + + #[rustversion::before(1.77)] + #[test] + fn unary_operator() -> Result<()> { + #[derive(Debug)] + struct Apple; + impl Apple { + fn b(&self, _b: u32, _c: i32) -> i32 { + 0 + } + } + + let a = Apple; + let b = 1; + let res = verify_pred!(!a.b(b, -1) == !-2); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + ! a.b(b, - 1) ==! - 2 was false with + a = Apple, + b = 1, + a.b(b, - 1) = 0, + ! a.b(b, - 1) = -1, + ! - 2 = 1, + at" + }))) + ) + } + + #[rustversion::since(1.77)] + #[test] + fn unary_operator() -> Result<()> { + #[derive(Debug)] + struct Apple; + impl Apple { + fn b(&self, _b: u32, _c: i32) -> i32 { + 0 + } + } + + let a = Apple; + let b = 1; + let res = verify_pred!(!a.b(b, -1) == !-2); + verify_that!( + res, + err(displays_as(contains_substring(indoc! {" + ! a.b(b, - 1) == ! - 2 was false with + a = Apple, + b = 1, + a.b(b, - 1) = 0, + ! a.b(b, - 1) = -1, + ! - 2 = 1, + at" + }))) + ) + } +} diff --git a/third_party/crates/googletest-0.14.3/tests/colorized_diff_test.rs b/third_party/crates/googletest-0.14.3/tests/colorized_diff_test.rs new file mode 100644 index 0000000..98d575b --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/colorized_diff_test.rs @@ -0,0 +1,50 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use std::fmt::{Display, Write}; + +// Make a long text with each element of the iterator on one line. +// `collection` must contains at least one element. +fn build_text(mut collection: impl Iterator) -> String { + let mut text = String::new(); + write!(&mut text, "{}", collection.next().expect("Provided collection without elements")) + .unwrap(); + for item in collection { + write!(&mut text, "\n{item}").unwrap(); + } + text +} + +#[test] +fn colors_appear_when_no_color_is_no_set_and_force_color_is_set() -> googletest::Result<()> { + std::env::remove_var("NO_COLOR"); + std::env::set_var("FORCE_COLOR", "1"); + + let result = verify_that!(build_text(1..50), eq(&build_text(1..51))); + + verify_that!( + result, + err(displays_as(contains_substring( + " + Difference(-\x1B[1;31mactual\x1B[0m / +\x1B[1;32mexpected\x1B[0m): + 1 + 2 + \x1B[3m<---- 45 common lines omitted ---->\x1B[0m + 48 + 49 + +\x1B[1;32m50\x1B[0m" + ))) + ) +} diff --git a/third_party/crates/googletest-0.14.3/tests/composition_test.rs b/third_party/crates/googletest-0.14.3/tests/composition_test.rs new file mode 100644 index 0000000..e1afdb2 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/composition_test.rs @@ -0,0 +1,187 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; + +#[test] +fn all_matcher_works_as_inner_matcher() -> Result<()> { + let value = vec![1]; + verify_that!(value, contains_each![all!(gt(&0), lt(&2))]) +} + +#[test] +fn matches_pattern_works_as_inner_matcher() -> Result<()> { + #[derive(Debug)] + struct AStruct(i32); + verify_that!(vec![AStruct(123)], contains_each![matches_pattern!(&AStruct(eq(123)))]) +} + +#[test] +fn matches_pattern_works_with_property_as_inner_matcher() -> Result<()> { + #[derive(Debug)] + struct AStruct(i32); + impl AStruct { + fn get_value(&self) -> i32 { + self.0 + } + } + verify_that!( + vec![AStruct(123)], + contains_each![matches_pattern!(&AStruct { + get_value(): eq(123) + })] + ) +} + +#[test] +fn contains_each_works_as_inner_matcher() -> Result<()> { + #[derive(Debug)] + struct AStruct(Vec); + verify_that!(AStruct(vec![123]), matches_pattern!(&AStruct(ref contains_each![eq(&123)]))) +} + +#[test] +fn pointwise_works_as_inner_matcher() -> Result<()> { + #[derive(Debug)] + struct AStruct(Vec); + verify_that!(AStruct(vec![123]), matches_pattern!(&AStruct(ref pointwise!(eq, [&123])))) +} + +#[test] +fn elements_are_works_as_inner_matcher() -> Result<()> { + #[derive(Debug)] + struct AStruct(Vec); + verify_that!(AStruct(vec![123]), matches_pattern!(&AStruct(ref elements_are![eq(&123)]))) +} + +#[test] +fn tuple_works_as_inner_matcher() -> Result<()> { + verify_that!(vec![(123,)], elements_are![(eq(&123),)]) +} + +#[test] +fn matches_struct_with_method_returning_option_of_non_copy_value() -> Result<()> { + #[derive(Debug)] + struct AnInnerStruct; + + #[derive(Debug)] + struct AStruct; + + impl AStruct { + fn get_value(&self) -> Option { + Some(AnInnerStruct) + } + } + + verify_that!( + AStruct, + matches_pattern!(&AStruct { + get_value(): ref some(matches_pattern!(&AnInnerStruct)) + }) + ) +} + +#[test] +fn matches_struct_with_method_returning_option_of_non_copy_enum() -> Result<()> { + #[derive(Debug)] + enum AnInnerStruct { + ThisCase, + #[allow(unused)] + ThatCase, + } + #[derive(Debug)] + struct AStruct; + impl AStruct { + fn get_value(&self) -> Option { + Some(AnInnerStruct::ThisCase) + } + } + + verify_that!( + AStruct, + matches_pattern!(&AStruct { + get_value(): ref some(matches_pattern!(&AnInnerStruct::ThisCase)) + }) + ) +} + +#[test] +fn matches_struct_with_method_returning_option_ref_binding_mode() -> Result<()> { + #[derive(Debug)] + struct AnInnerStruct; + #[derive(Debug)] + struct AStruct; + impl AStruct { + fn get_value(&self) -> Option { + Some(AnInnerStruct) + } + } + + verify_that!( + AStruct, + matches_pattern!(AStruct { + get_value(): some(matches_pattern!(AnInnerStruct)) + }) + ) +} + +#[test] +fn matches_struct_with_method_returning_option_enum_ref_binding_mode() -> Result<()> { + #[derive(Debug)] + enum AnInnerStruct { + ThisCase, + #[allow(unused)] + ThatCase, + } + #[derive(Debug)] + struct AStruct; + impl AStruct { + fn get_value(&self) -> Option { + Some(AnInnerStruct::ThisCase) + } + } + + verify_that!( + AStruct, + matches_pattern!(AStruct { + get_value(): some(matches_pattern!(AnInnerStruct::ThisCase)) + }) + ) +} + +#[test] +fn matches_struct_with_property_against_predicate() -> Result<()> { + #[derive(Debug)] + enum AnInnerStruct { + ThisCase, + #[allow(unused)] + ThatCase, + } + + #[derive(Debug)] + struct AStruct; + impl AStruct { + fn get_value(&self) -> AnInnerStruct { + AnInnerStruct::ThisCase + } + } + + verify_that!( + AStruct, + matches_pattern!(AStruct { + get_value(): predicate(|_: &_| true) + }) + ) +} diff --git a/third_party/crates/googletest-0.14.3/tests/elements_are_matcher_test.rs b/third_party/crates/googletest-0.14.3/tests/elements_are_matcher_test.rs new file mode 100644 index 0000000..760bc28 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/elements_are_matcher_test.rs @@ -0,0 +1,147 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; +use indoc::indoc; + +#[test] +fn elements_are_matches_vector() -> Result<()> { + let value = vec![1, 2, 3]; + verify_that!(value, elements_are![eq(&1), eq(&2), eq(&3)]) +} + +#[test] +fn elements_are_matches_slice() -> Result<()> { + let value = vec![1, 2, 3]; + let slice = value.as_slice(); + verify_that!(slice, elements_are![eq(&1), eq(&2), eq(&3)]) +} + +#[test] +fn elements_are_matches_array() -> Result<()> { + verify_that!([1, 2, 3], elements_are![eq(1), eq(2), eq(3)]) +} + +#[test] +fn elements_are_supports_trailing_comma() -> Result<()> { + let value = vec![1, 2, 3]; + verify_that!(value, elements_are![eq(&1), eq(&2), eq(&3),]) +} + +#[test] +fn elements_are_returns_no_match_when_expected_and_actual_sizes_differ() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, not(elements_are![eq(&1), eq(&2), eq(&3)])) +} + +#[test] +fn elements_are_admits_matchers_without_static_lifetime() -> Result<()> { + #[derive(Debug, PartialEq)] + struct AStruct(i32); + let expected_value = AStruct(123); + verify_that!(vec![AStruct(123)], elements_are![eq(&expected_value)]) +} + +#[test] +fn elements_are_matches_iterator_returning_by_value() -> Result<()> { + #[derive(Debug, Copy, Clone)] + struct Countdown(i32); + impl Iterator for Countdown { + type Item = i32; + + fn next(&mut self) -> Option { + match self.0 { + 0 => None, + x => { + self.0 -= 1; + Some(x) + } + } + } + } + verify_that!(Countdown(3), elements_are![eq(3), eq(2), eq(1)]) +} + +#[test] +fn elements_are_produces_correct_failure_message() -> Result<()> { + let result = verify_that!(vec![1, 4, 3], elements_are![eq(&1), eq(&2), eq(&3)]); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 4, 3] + Expected: has elements: + 0. is equal to 1 + 1. is equal to 2 + 2. is equal to 3 + Actual: [1, 4, 3], + where element #1 is 4, which isn't equal to 2" + )))) + ) +} + +#[test] +fn elements_are_produces_correct_failure_message_nested() -> Result<()> { + let result = verify_that!( + vec![vec![0, 1], vec![1, 2]], + elements_are![elements_are![eq(&1), eq(&2)], elements_are![eq(&2), eq(&3)]] + ); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Expected: has elements: + 0. has elements: + 0. is equal to 1 + 1. is equal to 2 + 1. has elements: + 0. is equal to 2 + 1. is equal to 3 + Actual: [[0, 1], [1, 2]], + where: + * element #0 is [0, 1], where: + * element #0 is 0, which isn't equal to 1 + * element #1 is 1, which isn't equal to 2 + * element #1 is [1, 2], where: + * element #0 is 1, which isn't equal to 2 + * element #1 is 2, which isn't equal to 3" + )))) + ) +} + +#[test] +fn elements_are_explain_match_wrong_size() -> Result<()> { + let matcher = elements_are![eq(&1)]; + verify_that!(matcher.explain_match(&vec![1, 2]), displays_as(eq("whose size is 2"))) +} + +fn create_matcher<'a>() -> impl Matcher<&'a Vec> { + elements_are![eq(&1)] +} + +#[test] +fn elements_are_works_when_matcher_is_created_in_subroutine() -> Result<()> { + verify_that!(vec![1], create_matcher()) +} + +#[test] +fn elements_are_implicitly_called() -> Result<()> { + verify_that!(vec![1, 2, 3], [eq(&1), eq(&2), eq(&3)]) +} + +#[test] +fn elements_are_with_auto_eq() -> Result<()> { + verify_that!(vec![1, 2, 3], [&1, &2, lt(&43)]) +} diff --git a/third_party/crates/googletest-0.14.3/tests/field_matcher_test.rs b/third_party/crates/googletest-0.14.3/tests/field_matcher_test.rs new file mode 100644 index 0000000..0a818e3 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/field_matcher_test.rs @@ -0,0 +1,315 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::matcher::MatcherResult; +use googletest::prelude::*; +use googletest::Result; + +#[derive(Debug)] +struct IntField { + int: i32, +} + +#[test] +fn field_matches_integer_field() -> Result<()> { + verify_that!(IntField { int: 32 }, field!(&IntField.int, eq(32))) +} + +#[derive(Debug)] +struct StringField { + strink: String, +} + +#[test] +fn field_matches_string_field() -> Result<()> { + verify_that!( + StringField { strink: "yes".to_string() }, + field!(&StringField.strink, ref eq("yes")) + ) +} + +#[test] +fn field_error_message_shows_field_name_and_inner_matcher() -> Result<()> { + let matcher = field!(&IntField.int, eq(31)); + + verify_that!( + matcher.describe(MatcherResult::Match), + displays_as(eq("has field `int`, which is equal to 31")) + ) +} + +mod sub { + #[derive(Debug)] + pub struct SubStruct { + pub field: i32, + } +} + +#[test] +fn struct_in_other_module_matches() -> Result<()> { + verify_that!(sub::SubStruct { field: 32 }, field!(&sub::SubStruct.field, eq(32))) +} + +#[derive(Debug)] +struct Tuple(i32, #[allow(unused)] String); + +#[test] +fn tuple_matches_with_index() -> Result<()> { + verify_that!(Tuple(32, "yes".to_string()), field!(&Tuple.0, eq(32))) +} + +#[test] +fn matches_enum_value() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + AValue(u32), + } + let value = AnEnum::AValue(123); + + verify_that!(value, field!(&AnEnum::AValue.0, eq(123))) +} + +#[test] +fn shows_correct_failure_message_for_wrong_struct_entry() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a: Vec, + } + let value = AStruct { a: vec![1] }; + + let arr: [u32; 0] = []; + let result = verify_that!(value, field!(&AStruct.a, ref container_eq(arr))); + + verify_that!( + result, + err(displays_as(contains_substring( + "which has field `a`, which contains the unexpected element 1" + ))) + ) +} + +#[test] +fn does_not_match_enum_value_with_wrong_enum_variant() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + #[allow(dead_code)] // This variant is intentionally unused. + AValue(u32), + AnotherValue, + } + let value = AnEnum::AnotherValue; + + verify_that!(&value, not(field!(&AnEnum::AValue.0, eq(123)))) +} + +#[test] +fn shows_correct_failure_message_for_wrong_enum_value() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + #[allow(dead_code)] // This variant is intentionally unused. + AValue { + a: u32, + }, + AnotherValue, + } + let value = AnEnum::AnotherValue; + + let result = verify_that!(value, field!(&AnEnum::AValue.a, eq(123))); + + verify_that!( + result, + err(displays_as(contains_substring("which has the wrong enum variant `AnotherValue`"))) + ) +} + +#[test] +fn shows_correct_failure_message_for_wrong_enum_value_with_tuple_field() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + #[allow(dead_code)] // This variant is intentionally unused. + AValue(u32), + #[allow(unused)] + AnotherValue(u32), + } + let value = AnEnum::AnotherValue(123); + + let result = verify_that!(value, field!(&AnEnum::AValue.0, eq(123))); + + verify_that!( + result, + err(displays_as(contains_substring("which has the wrong enum variant `AnotherValue`"))) + ) +} + +#[test] +fn shows_correct_failure_message_for_wrong_enum_value_with_named_field() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + #[allow(dead_code)] // This variant is intentionally unused. + AValue(u32), + AnotherValue { + #[allow(unused)] + a: u32, + }, + } + let value = AnEnum::AnotherValue { a: 123 }; + + let result = verify_that!(value, field!(&AnEnum::AValue.0, eq(123))); + + verify_that!( + result, + err(displays_as(contains_substring("which has the wrong enum variant `AnotherValue`"))) + ) +} + +#[test] +fn matches_struct_like_enum_value() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + AValue { a_field: u32 }, + } + let value = AnEnum::AValue { a_field: 123 }; + + verify_that!(value, field!(&AnEnum::AValue.a_field, eq(123))) +} + +#[test] +fn matches_struct_copy_to_copy() -> Result<()> { + #[derive(Debug, Clone, Copy)] + struct Strukt { + a_field: i32, + } + + verify_that!(Strukt { a_field: 32 }, field!(Strukt.a_field, eq(32))) +} + +#[test] +fn matches_struct_ref_to_copy() -> Result<()> { + #[derive(Debug)] + struct Strukt { + a_field: i32, + } + + verify_that!(Strukt { a_field: 32 }, field!(&Strukt.a_field, eq(32))) +} + +#[test] +fn matches_struct_ref_to_ref() -> Result<()> { + #[derive(Debug)] + struct Strukt { + a_field: String, + } + + verify_that!(Strukt { a_field: "32".into() }, field!(&Strukt.a_field, ref eq("32"))) +} + +#[test] +fn matches_struct_copy_to_ref() -> Result<()> { + // It is not possible to have a copy struct with non-copy field. Hence, this + // test case is not necessary. + Ok(()) +} + +#[test] +fn matches_struct_ref_to_ref_binding_mode() -> Result<()> { + #[derive(Debug)] + struct Strukt { + a_field: String, + } + + verify_that!(Strukt { a_field: "32".into() }, field!(Strukt.a_field, eq("32"))) +} + +#[test] +fn matches_struct_with_auto_eq() -> Result<()> { + #[derive(Debug)] + struct Strukt { + a_field: String, + } + + verify_that!(Strukt { a_field: "32".into() }, field!(Strukt.a_field, "32")) +} + +#[test] +fn matches_enum_with_auto_eq() -> Result<()> { + #[derive(Debug)] + enum Enum { + Str(String), + #[allow(unused)] + Int(i32), + } + + verify_that!(Enum::Str("32".into()), field!(Enum::Str.0, "32")) +} + +#[test] +fn matches_enum_with_auto_eq_with_wrapper() -> Result<()> { + #[derive(Debug)] + struct Wrapper { + wrapped: I, + } + + verify_that!( + Wrapper { wrapped: Wrapper { wrapped: 23 } }, + field!(Wrapper.wrapped, field!(Wrapper.wrapped, &23)) + ) +} + +#[test] +fn supports_fully_qualified_struct_path() -> Result<()> { + // Ensure that the macro expands to the fully-qualified struct path. + mod googletest {} + + let value = ::googletest::internal::test_data::TestStruct { value: 10 }; + verify_that!(value, field!(&::googletest::internal::test_data::TestStruct.value, ref eq(&10)))?; + verify_that!(value, field!(&::googletest::internal::test_data::TestStruct.value, eq(10)))?; + verify_that!(value, field!(::googletest::internal::test_data::TestStruct.value, eq(&10)))?; + Ok(()) +} + +#[test] +fn supports_generic_struct() -> Result<()> { + #[derive(Debug)] + struct Struct { + #[allow(dead_code)] // The property1 field is used only for adding the type parameter. + property1: S, + property2: T, + } + + let value = Struct { property1: 1, property2: 10 }; + verify_that!(&value, field!(&Struct::.property2, ref eq(&10)))?; + verify_that!(value, field!(&Struct::.property2, eq(10)))?; + verify_that!(value, field!(Struct::.property2, eq(&10)))?; + Ok(()) +} + +#[test] +fn supports_fully_qualified_generic_struct() -> Result<()> { + // Ensure that the macro expands to the fully-qualified struct path. + mod googletest {} + + let value = ::googletest::internal::test_data::GenericTestStruct { value: 10 }; + verify_that!( + &value, + field!(&::googletest::internal::test_data::GenericTestStruct::.value, ref eq(&10)) + )?; + verify_that!( + value, + field!(&::googletest::internal::test_data::GenericTestStruct::.value, eq(10)) + )?; + verify_that!( + value, + field!(::googletest::internal::test_data::GenericTestStruct::.value, eq(&10)) + )?; + Ok(()) +} diff --git a/third_party/crates/googletest-0.14.3/tests/fmt_test.rs b/third_party/crates/googletest-0.14.3/tests/fmt_test.rs new file mode 100644 index 0000000..9aefa8c --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/fmt_test.rs @@ -0,0 +1,58 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod write_expr_value { + use googletest::prelude::*; + + // Converts the formatting call to a `String` for testing. + macro_rules! write_expr_value { + ($expr_str:expr, $expr: expr $(,)?) => {{ + let mut s = String::new(); + ::googletest::fmt::internal::__googletest__write_expr_value!(s, $expr_str, $expr); + s + }}; + } + + #[test] + fn test_with_debug_value_references() -> googletest::Result<()> { + #[derive(Debug)] + struct Foo; + let mut val = Foo; + + verify_that!(write_expr_value!("val", val), eq("\n val = Foo,"))?; + verify_that!(write_expr_value!("val", &val), eq("\n val = Foo,"))?; + verify_that!(write_expr_value!("val", &&val), eq("\n val = Foo,"))?; + verify_that!(write_expr_value!("val", &mut val), eq("\n val = Foo,"))?; + verify_that!(write_expr_value!("val", &mut &mut val), eq("\n val = Foo,"))?; + + Ok(()) + } + + #[test] + fn test_with_non_debug_value_references() -> googletest::Result<()> { + struct Foo; + let mut val = Foo; + + verify_that!(write_expr_value!("val", val), eq("\n val does not implement Debug,"))?; + verify_that!(write_expr_value!("val", &val), eq("\n val does not implement Debug,"))?; + verify_that!(write_expr_value!("val", &&val), eq("\n val does not implement Debug,"))?; + verify_that!(write_expr_value!("val", &mut val), eq("\n val does not implement Debug,"))?; + verify_that!( + write_expr_value!("val", &mut &mut val), + eq("\n val does not implement Debug,") + )?; + + Ok(()) + } +} diff --git a/third_party/crates/googletest-0.14.3/tests/matches_pattern_enum_test.rs b/third_party/crates/googletest-0.14.3/tests/matches_pattern_enum_test.rs new file mode 100644 index 0000000..1bda28a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/matches_pattern_enum_test.rs @@ -0,0 +1,469 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; +use indoc::indoc; + +#[test] +fn matches_enum_without_field() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A, + } + let actual = AnEnum::A; + + verify_that!(actual, matches_pattern!(&AnEnum::A))?; + verify_that!(actual, matches_pattern!(&AnEnum::A,)) +} + +#[test] +fn matches_enum_without_field_ref_binding_mode() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A, + } + let actual = AnEnum::A; + + verify_that!(actual, matches_pattern!(AnEnum::A))?; + verify_that!(actual, matches_pattern!(AnEnum::A,)) +} + +#[test] +fn matches_enum_without_field_copy() -> Result<()> { + #[derive(Debug, Clone, Copy)] + enum AnEnum { + A, + } + let actual = AnEnum::A; + + verify_that!(actual, matches_pattern!(AnEnum::A))?; + verify_that!(actual, matches_pattern!(AnEnum::A,)) +} + +#[test] +fn generates_correct_failure_output_when_enum_variant_without_field_is_not_matched() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + #[allow(unused)] + A, + B, + } + let actual = AnEnum::B; + + let result = verify_that!(actual, matches_pattern!(&AnEnum::A)); + + const EXPECTED: &str = "is not & AnEnum :: A"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn generates_correct_failure_output_when_enum_variant_without_field_is_matched() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A, + } + let actual = AnEnum::A; + + let result = verify_that!(actual, not(matches_pattern!(&AnEnum::A))); + + const EXPECTED: &str = "is & AnEnum :: A"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn has_meaningful_assertion_failure_message_when_wrong_enum_variant_is_used() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + #[allow(unused)] + A(u32), + #[allow(unused)] + B(u32), + } + let actual = AnEnum::A(123); + + let result = verify_that!(actual, matches_pattern!(&AnEnum::B(eq(123)))); + + verify_that!( + result, + err(displays_as(contains_substring(indoc! {" + Actual: A(123), + which has the wrong enum variant `A` + " + }))) + ) +} + +#[test] +fn matches_enum_struct_non_exhaustive() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1 { a_field: u32, another_field: u32 }, + } + let actual: AnEnum = AnEnum::Variant1 { a_field: 123, another_field: 234 }; + + verify_that!(actual, matches_pattern!(&AnEnum::Variant1 { a_field: eq(123), .. })) +} + +#[test] +fn matches_enum_struct_with_all_non_exhaustive_fields() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1 { a: u32, b: u32 }, + Variant2 { c: u32, d: u32 }, + } + let actual: AnEnum = AnEnum::Variant1 { a: 123, b: 234 }; + + verify_that!(actual, matches_pattern!(&AnEnum::Variant1 { .. })) +} + +#[test] +fn has_failure_when_wrong_enum_struct_variant_is_matched_with_all_non_exhaustive_fields( +) -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1 { a: u32, b: u32 }, + Variant2 { c: u32, d: u32 }, + } + let actual: AnEnum = AnEnum::Variant1 { a: 123, b: 234 }; + + let result = verify_that!(actual, matches_pattern!(&AnEnum::Variant2 { .. })); + + const EXPECTED: &str = indoc!( + " + Expected: is & AnEnum :: Variant2 { .. } + Actual: Variant1 { a: 123, b: 234 }, + which is not & AnEnum :: Variant2 { .. } + " + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn matches_enum_struct_with_all_wildcard_fields() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1 { a: u32, b: u32 }, + Variant2 { c: u32, d: u32 }, + } + let actual: AnEnum = AnEnum::Variant1 { a: 123, b: 234 }; + + verify_that!(actual, matches_pattern!(&AnEnum::Variant1 { a: _, b: _ })) +} + +#[test] +fn has_failure_when_wrong_enum_struct_variant_is_matched_with_all_wildcard_fields() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1 { a: u32, b: u32 }, + Variant2 { c: u32, d: u32 }, + } + let actual: AnEnum = AnEnum::Variant1 { a: 123, b: 234 }; + + let result = verify_that!(actual, matches_pattern!(&AnEnum::Variant2 { c: _, d: _ })); + + const EXPECTED: &str = indoc!( + " + Expected: is & AnEnum :: Variant2 { c : _, d : _, } + Actual: Variant1 { a: 123, b: 234 }, + which is not & AnEnum :: Variant2 { c : _, d : _, } + " + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn matches_enum_struct_non_exhaustive_with_wildcard_fields() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1 { a: u32, b: u32 }, + Variant2 { c: u32, d: u32 }, + } + let actual: AnEnum = AnEnum::Variant1 { a: 123, b: 234 }; + + verify_that!(actual, matches_pattern!(&AnEnum::Variant1 { a: _, .. })) +} + +#[test] +fn has_failure_when_wrong_enum_struct_variant_is_matched_non_exhaustive_with_wildcard_fields( +) -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1 { a: u32, b: u32 }, + Variant2 { c: u32, d: u32 }, + } + let actual: AnEnum = AnEnum::Variant1 { a: 123, b: 234 }; + + let result = verify_that!(actual, matches_pattern!(&AnEnum::Variant2 { c: _, .. })); + + const EXPECTED: &str = indoc!( + " + Expected: is & AnEnum :: Variant2 { c : _, .. } + Actual: Variant1 { a: 123, b: 234 }, + which is not & AnEnum :: Variant2 { c : _, .. } + " + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn matches_enum_struct_exhaustive_with_multiple_variants() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1 { a_field: u32 }, + Variant2, + } + let actual: AnEnum = AnEnum::Variant1 { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&AnEnum::Variant1 { a_field: eq(123) })) +} + +#[test] +fn matches_match_pattern_literal() -> Result<()> { + let actual = false; + #[allow(clippy::redundant_pattern_matching)] + verify_that!(actual, matches_pattern!(false))?; + #[allow(clippy::redundant_pattern_matching)] + verify_that!(actual, matches_pattern!(false,))?; + let actual = 1; + verify_that!(actual, matches_pattern!(1))?; + verify_that!(actual, matches_pattern!(1,))?; + let actual = "test"; + verify_that!(actual, matches_pattern!(&"test"))?; + verify_that!(actual, matches_pattern!(&"test",)) +} + +#[test] +fn has_failure_when_wrong_enum_variant_is_matched_non_exhaustively() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1(i8), + Variant2, + } + let actual: AnEnum = AnEnum::Variant2; + + let result = verify_that!(actual, matches_pattern!(&AnEnum::Variant1(..))); + + const EXPECTED: &str = indoc!( + " + Expected: is & AnEnum :: Variant1(..) + Actual: Variant2, + which is not & AnEnum :: Variant1(..) + " + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn has_failure_when_wrong_enum_variant_is_matched_with_underscore() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1(i8), + Variant2, + } + let actual: AnEnum = AnEnum::Variant2; + + let result = verify_that!(actual, matches_pattern!(&AnEnum::Variant1(_))); + + const EXPECTED: &str = indoc!( + " + Expected: is & AnEnum :: Variant1(_) + Actual: Variant2, + which is not & AnEnum :: Variant1(_) + " + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn has_failure_when_wrong_enum_variant_is_matched_with_value() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1(i8), + Variant2, + } + let actual: AnEnum = AnEnum::Variant2; + + let result = verify_that!(actual, matches_pattern!(&AnEnum::Variant1(123))); + + const EXPECTED: &str = indoc!( + " + Expected: is & AnEnum :: Variant1 which has field `0`, which is equal to 123 + Actual: Variant2, + which has the wrong enum variant `Variant2` + " + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn matches_enum_struct_field_with_mutliple_variants() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1(i8), + Variant2, + } + let actual: AnEnum = AnEnum::Variant2; + + verify_that!(actual, matches_pattern!(&AnEnum::Variant2)) +} + +#[test] +fn matches_enum_struct_field_with_multiple_variants_non_exhaustive() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1(i8), + Variant2, + } + let actual: AnEnum = AnEnum::Variant1(123); + + verify_that!(actual, matches_pattern!(&AnEnum::Variant1(..))) +} + +#[test] +fn matches_enum_struct_field_with_multiple_variants_with_wildcard() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + enum AnEnum { + Variant1(i8), + Variant2, + } + let actual: AnEnum = AnEnum::Variant1(123); + + verify_that!(actual, matches_pattern!(&AnEnum::Variant1(_))) +} + +#[test] +fn matches_enum_with_field() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A(u32), + } + let actual = AnEnum::A(123); + + verify_that!(actual, matches_pattern!(&AnEnum::A(eq(123)))) +} + +#[test] +fn does_not_match_wrong_enum_value() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + #[allow(unused)] + A(u32), + B, + } + let actual = AnEnum::B; + + verify_that!(actual, not(matches_pattern!(&AnEnum::A(eq(123))))) +} + +#[test] +fn includes_enum_variant_in_description_with_field() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A(u32), + } + let actual = AnEnum::A(123); + + let result = verify_that!(actual, matches_pattern!(&AnEnum::A(eq(234)))); + + const EXPECTED: &str = "Expected: is & AnEnum :: A which has field `0`"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn includes_enum_variant_in_negative_description_with_field() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A(u32), + } + let actual = AnEnum::A(123); + + let result = verify_that!(actual, not(matches_pattern!(&AnEnum::A(eq(123))))); + + const EXPECTED: &str = "Expected: is not & AnEnum :: A which has field `0`, which is equal to"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn includes_enum_variant_in_description_with_two_fields() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A(u32, u32), + } + let actual = AnEnum::A(123, 234); + + let result = verify_that!(actual, matches_pattern!(&AnEnum::A(eq(234), eq(234)))); + + const EXPECTED: &str = "Expected: is & AnEnum :: A which has all the following properties"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn includes_enum_variant_in_description_with_three_fields() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A(u32, u32, u32), + } + let actual = AnEnum::A(123, 234, 345); + + let result = verify_that!(actual, matches_pattern!(&AnEnum::A(eq(234), eq(234), eq(345)))); + + const EXPECTED: &str = "Expected: is & AnEnum :: A which has all the following properties"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn includes_enum_variant_in_description_with_named_field() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A { field: u32 }, + } + let actual = AnEnum::A { field: 123 }; + + let result = verify_that!(actual, matches_pattern!(&AnEnum::A { field: eq(234) })); + + const EXPECTED: &str = "Expected: is & AnEnum :: A which has field `field`"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn includes_enum_variant_in_description_with_two_named_fields() -> Result<()> { + #[derive(Debug)] + enum AnEnum { + A { field: u32, another_field: u32 }, + } + let actual = AnEnum::A { field: 123, another_field: 234 }; + + let result = verify_that!( + actual, + matches_pattern!(&AnEnum::A { field: eq(234), another_field: eq(234) }) + ); + + const EXPECTED: &str = "Expected: is & AnEnum :: A which has all the following properties"; + verify_that!(&result, err(displays_as(contains_substring(EXPECTED)))) +} diff --git a/third_party/crates/googletest-0.14.3/tests/matches_pattern_struct_and_enum_test.rs b/third_party/crates/googletest-0.14.3/tests/matches_pattern_struct_and_enum_test.rs new file mode 100644 index 0000000..58478a1 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/matches_pattern_struct_and_enum_test.rs @@ -0,0 +1,234 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; + +#[test] +fn matches_struct_with_a_method_taking_enum_value_parameter_followed_by_field() -> Result<()> { + enum AnEnum { + AVariant, + } + + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field(&self, _value: AnEnum) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 1, another_field: 2 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { get_field(AnEnum::AVariant): eq(1), another_field: eq(2), .. }) + ) +} + +#[test] +fn matches_struct_with_a_method_taking_enum_value_param_ret_ref_followed_by_field() -> Result<()> { + enum AnEnum { + AVariant, + } + + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _value: AnEnum) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 1, another_field: 2 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + get_field_ref(AnEnum::AVariant): eq(&1), + another_field: eq(2), + .. + }) + ) +} + +#[test] +fn matches_struct_with_field_followed_by_method_taking_enum_value_param() -> Result<()> { + enum AnEnum { + AVariant, + } + + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field(&self, _value: AnEnum) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 1, another_field: 2 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(2), + get_field(AnEnum::AVariant): eq(1), + .. + }) + ) +} + +#[test] +fn matches_struct_with_field_followed_by_method_taking_enum_value_param_ret_ref() -> Result<()> { + enum AnEnum { + AVariant, + } + + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _value: AnEnum) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 1, another_field: 2 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(2), + get_field_ref(AnEnum::AVariant): eq(&1), + .. + }) + ) +} + +#[test] +fn matches_struct_with_field_followed_by_method_taking_enum_value_param_followed_by_field( +) -> Result<()> { + enum AnEnum { + AVariant, + } + + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn get_field(&self, _value: AnEnum) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 1, another_field: 2, a_third_field: 3 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(2), + get_field(AnEnum::AVariant): eq(1), + a_third_field: eq(3), + .. + }) + ) +} + +#[test] +fn matches_struct_with_field_followed_by_method_taking_enum_value_param_ret_ref_followed_by_field( +) -> Result<()> { + enum AnEnum { + AVariant, + } + + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _value: AnEnum) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 1, another_field: 2, a_third_field: 3 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(2), + get_field_ref(AnEnum::AVariant): eq(&1), + a_third_field: eq(3), + .. + }) + ) +} + +#[test] +fn matches_struct_with_a_method_returning_reference_taking_enum_value_parameter() -> Result<()> { + enum AnEnum { + AVariant, + } + + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _value: AnEnum) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 1 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_field_ref(AnEnum::AVariant): eq(&1) })) +} + +#[test] +fn matches_struct_with_a_method_taking_enum_value_parameter() -> Result<()> { + enum AnEnum { + AVariant, + } + + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_a_field(&self, _value: AnEnum) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 1 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_a_field(AnEnum::AVariant): eq(1) })) +} diff --git a/third_party/crates/googletest-0.14.3/tests/matches_pattern_struct_test.rs b/third_party/crates/googletest-0.14.3/tests/matches_pattern_struct_test.rs new file mode 100644 index 0000000..1a8668f --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/matches_pattern_struct_test.rs @@ -0,0 +1,1250 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; +use indoc::indoc; + +#[test] +fn matches_struct_containing_single_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + let actual = AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&AStruct { a_field: eq(123) })) +} + +#[test] +fn matches_struct_containing_two_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!(actual, matches_pattern!(&AStruct { a_field: eq(123), another_field: eq(234) })) +} + +#[test] +fn matches_struct_non_exhaustive() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!(actual, matches_pattern!(&AStruct { a_field: eq(123), .. })) +} + +#[test] +fn matches_braced_struct_with_no_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct {} + let actual = AStruct {}; + + verify_that!(actual, matches_pattern!(AStruct {})) +} + +#[test] +#[rustfmt::skip]// Otherwise fmt strips the trailing comma +fn supports_trailing_comma_with_one_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + let actual = AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&AStruct { + a_field: eq(123), // Block reformatting + })) +} + +#[test] +fn supports_trailing_comma_with_two_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + a_field: eq(123), + another_field: eq(234), // Block reformatting + }) + ) +} + +#[test] +fn supports_trailing_comma_with_three_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + let actual = AStruct { a_field: 123, another_field: 234, a_third_field: 345 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + a_field: eq(123), + another_field: eq(234), + a_third_field: eq(345), + }) + ) +} + +#[test] +fn matches_struct_containing_nested_struct_with_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_nested_struct: ANestedStruct, + } + #[derive(Debug)] + struct ANestedStruct { + a_field: u32, + } + let actual = AStruct { a_nested_struct: ANestedStruct { a_field: 123 } }; + + verify_that!( + actual, + matches_pattern!(&AStruct { a_nested_struct: ref pat!(&ANestedStruct { a_field: eq(123) }) }) + ) +} + +#[test] +fn matches_struct_containing_nested_struct_with_field_with_binding_mode() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_nested_struct: ANestedStruct, + } + #[derive(Debug)] + struct ANestedStruct { + a_field: u32, + } + let actual = AStruct { a_nested_struct: ANestedStruct { a_field: 123 } }; + + verify_that!( + actual, + matches_pattern!(AStruct { a_nested_struct: pat!(ANestedStruct { a_field: eq(&123) }) }) + ) +} + +#[test] +fn matches_struct_containing_non_copy_field_binding_mode() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_string: String, + } + let actual = AStruct { a_string: "123".into() }; + + verify_that!(actual, matches_pattern!(AStruct { a_string: eq("123") })) +} + +#[test] +fn matches_generic_struct_exhaustively() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + struct AStruct { + a: T, + b: u32, + } + let actual = AStruct { a: 1, b: 3 }; + + verify_that!(actual, matches_pattern!(&AStruct { a: _, b: _ }))?; + verify_that!(actual, matches_pattern!(AStruct { a: _, b: _ })) +} + +#[test] +fn matches_struct_with_interleaved_underscore() -> Result<()> { + #[derive(Debug)] + struct NonCopy; + #[allow(dead_code)] + #[derive(Debug)] + struct AStruct { + a: u32, + b: NonCopy, + c: u32, + } + let actual = AStruct { a: 1, b: NonCopy, c: 3 }; + + verify_that!(actual, matches_pattern!(&AStruct { a: eq(1), b: _, c: eq(3) }))?; + verify_that!(actual, matches_pattern!(AStruct { a: eq(&1), b: _, c: eq(&3) })) +} + +#[test] +fn has_correct_assertion_failure_message_for_single_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + let actual = AStruct { a_field: 123 }; + + let result = verify_that!(actual, matches_pattern!(&AStruct { a_field: eq(234) })); + + const EXPECTED: &str = indoc!( + " + Value of: actual + Expected: is & AStruct which has field `a_field`, which is equal to 234 + Actual: AStruct { a_field: 123 }, + which has field `a_field`, which isn't equal to 234 + " + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn has_correct_assertion_failure_message_for_two_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + let result = verify_that!( + actual, + matches_pattern!(&AStruct { a_field: eq(234), another_field: eq(123) }) + ); + + const EXPECTED: &str = indoc!( + " + Value of: actual + Expected: is & AStruct which has all the following properties: + * has field `a_field`, which is equal to 234 + * has field `another_field`, which is equal to 123 + Actual: AStruct { a_field: 123, another_field: 234 }, + * which has field `a_field`, which isn't equal to 234 + * which has field `another_field`, which isn't equal to 123" + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn has_correct_assertion_failure_message_for_field_and_property() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + impl AStruct { + fn get_field(&self) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + let result = verify_that!( + actual, + matches_pattern!(&AStruct { get_field(): eq(234), another_field: eq(123), .. }) + ); + + const EXPECTED: &str = indoc!( + " + Value of: actual + Expected: is & AStruct which has all the following properties: + * has property `get_field()`, which is equal to 234 + * has field `another_field`, which is equal to 123 + Actual: AStruct { a_field: 123, another_field: 234 }, + * whose property `get_field()` is `123`, which isn't equal to 234 + * which has field `another_field`, which isn't equal to 123" + ); + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn supports_qualified_struct_names() -> Result<()> { + mod a_module { + #[derive(Debug)] + pub(super) struct AStruct { + pub(super) a_field: u32, + } + } + let actual = a_module::AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&a_module::AStruct { a_field: eq(123) })) +} + +#[test] +fn matches_match_pattern_struct() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + struct AStruct { + a: u32, + } + let actual = AStruct { a: 123 }; + + verify_that!(actual, matches_pattern!(AStruct { .. })) +} + +#[test] +fn includes_struct_name_in_description_with_property() -> Result<()> { + #[derive(Debug)] + struct AStruct { + field: u32, + } + impl AStruct { + fn get_field(&self) -> u32 { + self.field + } + } + let actual = AStruct { field: 123 }; + + let result = verify_that!(actual, matches_pattern!(&AStruct { get_field(): eq(234) })); + + const EXPECTED: &str = "Expected: is & AStruct which has property `get_field()`"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn includes_struct_name_in_description_with_ref_property() -> Result<()> { + #[derive(Debug)] + struct AStruct { + field: u32, + } + impl AStruct { + fn get_field(&self) -> &u32 { + &self.field + } + } + let actual = AStruct { field: 123 }; + + let result = verify_that!(actual, matches_pattern!(&AStruct { get_field(): eq(&234) })); + + const EXPECTED: &str = "Expected: is & AStruct which has property `get_field()`"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn includes_struct_name_in_description_with_property_after_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + field: u32, + } + impl AStruct { + fn get_field(&self) -> u32 { + self.field + } + } + let actual = AStruct { field: 123 }; + + let result = + verify_that!(actual, matches_pattern!(&AStruct { field: eq(123), get_field(): eq(234) })); + + const EXPECTED: &str = "Expected: is & AStruct which has all the following properties"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn includes_struct_name_in_description_with_ref_property_after_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + field: u32, + } + impl AStruct { + fn get_field(&self) -> &u32 { + &self.field + } + } + let actual = AStruct { field: 123 }; + + let result = + verify_that!(actual, matches_pattern!(&AStruct { field: eq(123), get_field(): eq(&234) })); + + const EXPECTED: &str = "Expected: is & AStruct which has all the following properties"; + verify_that!(result, err(displays_as(contains_substring(EXPECTED)))) +} + +#[test] +fn matches_struct_with_a_method() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field(&self) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_field(): eq(123) })) +} + +#[test] +fn matches_struct_with_a_method_and_trailing_comma() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field(&self) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_field(): eq(123), })) +} + +#[test] +fn matches_struct_with_a_method_taking_parameter() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn add_to_field(&self, a: u32) -> u32 { + self.a_field + a + } + } + let actual = AStruct { a_field: 1 }; + + verify_that!(actual, matches_pattern!(&AStruct { add_to_field(2): eq(3) })) +} + +#[test] +fn matches_struct_with_a_method_taking_two_parameters() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_field + a * b + } + } + let actual = AStruct { a_field: 1 }; + + verify_that!(actual, matches_pattern!(&AStruct { add_product_to_field(2, 3): eq(7) })) +} + +#[test] +fn matches_struct_with_a_method_taking_two_parameters_with_trailing_comma() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_field + a * b + } + } + let actual = AStruct { a_field: 1 }; + + verify_that!(actual, matches_pattern!(&AStruct { add_product_to_field(2, 3,): eq(7) })) +} + +#[test] +fn matches_struct_with_a_method_returning_a_reference() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field_ref(&self) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_field_ref(): eq(&123) })) +} + +#[test] +fn matches_struct_with_a_method_returning_a_reference_with_trailing_comma() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field_ref(&self) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_field_ref(): eq(&123), })) +} + +#[test] +fn matches_struct_with_a_method_taking_two_parameters_ret_ref() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _a: u32, _b: u32) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 1 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_field_ref(2, 3): eq(&1) })) +} + +#[test] +fn matches_struct_with_a_method_taking_two_parameters_with_trailing_comma_ret_ref() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _a: u32, _b: u32) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 1 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_field_ref(2, 3,): eq(&1) })) +} + +#[test] +fn matches_struct_with_a_method_followed_by_a_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field(&self) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { get_field(): eq(123), another_field: eq(234), .. }) + ) +} + +#[test] +fn matches_struct_with_a_method_followed_by_a_field_with_trailing_comma() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_value(&self) -> u32 { + 123 + } + } + let actual = AStruct { a_field: 234 }; + + verify_that!(actual, matches_pattern!(&AStruct { get_value(): eq(123), a_field: eq(234), })) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_params_trailing_comma_ret_ref_followed_by_a_field( +) -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _a: u32, _b: u32) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234, a_third_field: 345 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(234), + get_field_ref(2, 3,): eq(&123), + a_third_field: eq(345), + .. + }) + ) +} + +#[test] +fn matches_struct_field_copy() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + let actual = AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(&AStruct { a_field: eq(123) })) +} + +#[test] +fn matches_struct_field_non_copy() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: String, + } + let actual = AStruct { a_field: "123".into() }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + a_field: ref eq("123"), + }) + ) +} + +#[test] +fn matches_copy_struct_field_copy() -> Result<()> { + #[derive(Debug, Clone, Copy)] + struct AStruct { + a_field: i32, + } + let actual = AStruct { a_field: 123 }; + + verify_that!(actual, matches_pattern!(AStruct { a_field: eq(123) })) +} + +#[test] +fn matches_struct_property_copy() -> Result<()> { + #[derive(Debug)] + struct AStruct; + + impl AStruct { + fn prop(&self) -> i32 { + 123 + } + } + let actual = AStruct; + + verify_that!(actual, matches_pattern!(&AStruct { prop(): eq(123) })) +} + +#[test] +fn matches_struct_property_non_copy() -> Result<()> { + #[derive(Debug)] + struct AStruct; + + impl AStruct { + fn prop(&self) -> String { + "123".into() + } + } + let actual = AStruct; + + verify_that!(actual, matches_pattern!(&AStruct { prop(): ref eq("123") })) +} + +#[test] +fn matches_copy_struct_property_copy() -> Result<()> { + #[derive(Debug, Clone, Copy)] + struct AStruct; + + impl AStruct { + fn prop(self) -> i32 { + 123 + } + } + let actual = AStruct; + + verify_that!(actual, matches_pattern!(AStruct { prop(): eq(123) })) +} + +#[test] +fn matches_copy_struct_property_non_copy() -> Result<()> { + #[derive(Debug, Clone, Copy)] + struct AStruct; + + impl AStruct { + fn prop(self) -> String { + "123".into() + } + } + let actual = AStruct; + + verify_that!(actual, matches_pattern!(AStruct { prop(): ref eq("123") })) +} + +#[test] +fn matches_struct_auto_eq() -> Result<()> { + #[derive(Debug, Clone)] + struct AStruct { + int: i32, + string: String, + option: Option, + } + + verify_that!( + AStruct { int: 123, string: "123".into(), option: Some(123) }, + matches_pattern!(&AStruct { int: 123, string: ref "123", option: Some(123) }) + ) +} + +#[test] +fn matches_struct_with_a_method_taking_two_parameters_and_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_field + a * b + } + } + let actual = AStruct { a_field: 1, another_field: 123 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { add_product_to_field(2, 3): eq(7), another_field: eq(123), .. }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_ret_ref_followed_by_a_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn get_field_ref(&self) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234, a_third_field: 345 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(234), + get_field_ref(): eq(&123), + a_third_field: eq(345), + .. + }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_ret_ref_followed_by_a_field_with_trailing_comma( +) -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn get_field_ref(&self) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234, a_third_field: 345 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(234), + get_field_ref(): eq(&123), + a_third_field: eq(345), + .. + }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_params_ret_ref_followed_by_a_field( +) -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _a: u32, _b: u32) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234, a_third_field: 345 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(234), + get_field_ref(2, 3): eq(&123), + a_third_field: eq(345), + .. + }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_params_and_trailing_comma_ret_ref( +) -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _a: u32, _b: u32) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { another_field: eq(234), get_field_ref(2, 3,): eq(&123), .. }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_followed_by_a_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn get_field(&self) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234, a_third_field: 345 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(234), + get_field(): eq(123), + a_third_field: eq(345), + .. + }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_followed_by_a_field_with_trailing_comma( +) -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field(&self) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + a_field: eq(123), + get_field(): eq(123), + another_field: eq(234), + }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_params_followed_by_a_field() -> Result<()> +{ + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_field + a * b + } + } + let actual = AStruct { a_field: 1, another_field: 234, a_third_field: 345 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(234), + add_product_to_field(2, 3): eq(7), + a_third_field: eq(345), + .. + }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_params_and_trailing_comma_followed_by_a_field( +) -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + a_third_field: u32, + } + + impl AStruct { + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_field + a * b + } + } + let actual = AStruct { a_field: 1, another_field: 234, a_third_field: 345 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + another_field: eq(234), + add_product_to_field(2, 3,): eq(7), + a_third_field: eq(345), + .. + }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_params_and_trailing_comma() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_field + a * b + } + } + let actual = AStruct { a_field: 1, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { another_field: eq(234), add_product_to_field(2, 3,): eq(7), .. }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_returning_reference() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field_ref(&self) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { another_field: eq(234), get_field_ref(): eq(&123), .. }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_returning_ref_and_trailing_comma() -> Result<()> +{ + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field_ref(&self) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { a_field: eq(123), get_field_ref(): eq(&123), }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_params_ret_ref() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _a: u32, _b: u32) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { another_field: eq(234), get_field_ref(2, 3): eq(&123), .. }) + ) +} + +#[test] +fn matches_struct_with_a_method_taking_two_parameters_with_trailing_comma_ret_ref_and_field( +) -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _a: u32, _b: u32) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 1, another_field: 123 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { get_field_ref(2, 3,): eq(&1), another_field: eq(123), .. }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field(&self) -> u32 { + self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { another_field: eq(234), get_field(): eq(123), .. }) + ) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_trailing_comma() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_value(&self) -> u32 { + 123 + } + } + let actual = AStruct { a_field: 234 }; + + verify_that!(actual, matches_pattern!(&AStruct { a_field: eq(234), get_value(): eq(123), })) +} + +#[test] +fn matches_struct_with_a_field_followed_by_a_method_with_params() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_field + a * b + } + } + let actual = AStruct { a_field: 1, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { another_field: eq(234), add_product_to_field(2, 3): eq(7), .. }) + ) +} + +#[test] +fn matches_struct_with_a_method_taking_two_parameters_with_trailing_comma_and_field() -> Result<()> +{ + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_field + a * b + } + } + let actual = AStruct { a_field: 1, another_field: 123 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { + add_product_to_field(2, 3,): eq(7), + another_field: eq(123), + .. + }) + ) +} + +#[test] +fn matches_struct_with_a_method_returning_reference_followed_by_a_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field_ref(&self) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123, another_field: 234 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { get_field_ref(): eq(&123), another_field: eq(234), .. }) + ) +} + +#[test] +fn matches_struct_with_a_method_returning_reference_followed_by_a_field_with_trailing_comma( +) -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + } + + impl AStruct { + fn get_field_ref(&self) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 123 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { get_field_ref(): eq(&123), a_field: eq(123), }) + ) +} + +#[test] +fn matches_struct_with_a_method_taking_two_parameters_ret_ref_and_field() -> Result<()> { + #[derive(Debug)] + struct AStruct { + a_field: u32, + another_field: u32, + } + + impl AStruct { + fn get_field_ref(&self, _a: u32, _b: u32) -> &u32 { + &self.a_field + } + } + let actual = AStruct { a_field: 1, another_field: 123 }; + + verify_that!( + actual, + matches_pattern!(&AStruct { get_field_ref(2, 3): eq(&1), another_field: eq(123), .. }) + ) +} + +#[test] +fn matches_with_fully_qualified_struct_path() -> Result<()> { + // Ensure that the macro expands to the fully-qualified struct path. + mod googletest {} + + let value = ::googletest::internal::test_data::TestStruct { value: 10 }; + verify_that!( + value, + matches_pattern!( + &::googletest::internal::test_data::TestStruct { + value: eq(10), + get_value(): eq(10) + } + ) + ) +} + +#[test] +fn matches_with_explicit_type_parameters() -> Result<()> { + #[derive(Debug)] + struct AStruct { + field: T, + } + + impl AStruct { + fn get_default(&self) -> U { + Default::default() + } + } + + let actual = AStruct:: { field: 1 }; + verify_that!( + actual, + matches_pattern!(&AStruct:: { field: eq(1), get_default::(): eq(0) }) + ) +} + +#[test] +fn matches_with_fully_qualified_generic_struct_path() -> Result<()> { + // Ensure that the macro expands to the fully-qualified struct path. + mod googletest {} + + let value = ::googletest::internal::test_data::GenericTestStruct { value: 10 }; + verify_that!( + value, + matches_pattern!( + &::googletest::internal::test_data::GenericTestStruct:: { + value: eq(10), + get_value::(20): eq(20) + } + ) + ) +} diff --git a/third_party/crates/googletest-0.14.3/tests/matches_pattern_tuple_struct_test.rs b/third_party/crates/googletest-0.14.3/tests/matches_pattern_tuple_struct_test.rs new file mode 100644 index 0000000..25fe2b0 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/matches_pattern_tuple_struct_test.rs @@ -0,0 +1,237 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; + +#[test] +fn matches_tuple_struct_containing_single_field() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32); + let actual = AStruct(123); + + verify_that!(actual, matches_pattern!(&AStruct(eq(123)))) +} + +#[test] +fn matches_tuple_struct_containing_two_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32); + let actual = AStruct(123, 234); + + verify_that!(actual, matches_pattern!(&AStruct(eq(123), eq(234)))) +} + +#[test] +fn matches_tuple_struct_containing_three_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32); + let actual = AStruct(123, 234, 345); + + verify_that!(actual, matches_pattern!(&AStruct(eq(123), eq(234), eq(345)))) +} + +#[test] +fn matches_tuple_struct_containing_four_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32, u32); + let actual = AStruct(123, 234, 345, 456); + + verify_that!(actual, matches_pattern!(&AStruct(eq(123), eq(234), eq(345), eq(456)))) +} + +#[test] +fn matches_tuple_struct_containing_five_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32, u32, u32); + let actual = AStruct(123, 234, 345, 456, 567); + + verify_that!(actual, matches_pattern!(&AStruct(eq(123), eq(234), eq(345), eq(456), eq(567)))) +} + +#[test] +fn matches_tuple_struct_containing_six_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32, u32, u32, u32); + let actual = AStruct(123, 234, 345, 456, 567, 678); + + verify_that!( + actual, + matches_pattern!(&AStruct(eq(123), eq(234), eq(345), eq(456), eq(567), eq(678))) + ) +} + +#[test] +fn matches_tuple_struct_containing_seven_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32, u32, u32, u32, u32); + let actual = AStruct(123, 234, 345, 456, 567, 678, 789); + + verify_that!( + actual, + matches_pattern!(&AStruct(eq(123), eq(234), eq(345), eq(456), eq(567), eq(678), eq(789))) + ) +} + +#[test] +fn matches_tuple_struct_containing_eight_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32, u32, u32, u32, u32, u32); + let actual = AStruct(123, 234, 345, 456, 567, 678, 789, 890); + + verify_that!( + actual, + matches_pattern!(&AStruct( + eq(123), + eq(234), + eq(345), + eq(456), + eq(567), + eq(678), + eq(789), + eq(890) + )) + ) +} + +#[test] +fn matches_tuple_struct_containing_nine_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32, u32, u32, u32, u32, u32, u32); + let actual = AStruct(123, 234, 345, 456, 567, 678, 789, 890, 901); + + verify_that!( + actual, + matches_pattern!(&AStruct( + eq(123), + eq(234), + eq(345), + eq(456), + eq(567), + eq(678), + eq(789), + eq(890), + eq(901) + )) + ) +} + +#[test] +fn matches_tuple_struct_containing_ten_fields() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32, u32, u32, u32, u32, u32, u32, u32); + let actual = AStruct(123, 234, 345, 456, 567, 678, 789, 890, 901, 12); + + verify_that!( + actual, + matches_pattern!(&AStruct( + eq(123), + eq(234), + eq(345), + eq(456), + eq(567), + eq(678), + eq(789), + eq(890), + eq(901), + eq(12) + )) + ) +} + +#[test] +fn matches_tuple_struct_containing_ten_fields_by_ref() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32, u32, u32, u32, u32, u32, u32, u32, u32); + let actual = AStruct(123, 234, 345, 456, 567, 678, 789, 890, 901, 12); + + verify_that!( + actual, + matches_pattern!(&AStruct( + ref eq(&123), + ref eq(&234), + ref eq(&345), + ref eq(&456), + ref eq(&567), + ref eq(&678), + ref eq(&789), + ref eq(&890), + ref eq(&901), + ref eq(&12) + )) + ) +} + +#[test] +fn matches_tuple_struct_with_trailing_comma() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32); + let actual = AStruct(123); + + verify_that!( + actual, + matches_pattern!(&AStruct( + eq(123), // Keep the trailing comma, block reformatting + )) + ) +} + +#[test] +fn matches_tuple_struct_with_two_fields_and_trailing_comma() -> Result<()> { + #[derive(Debug)] + struct AStruct(u32, u32); + let actual = AStruct(123, 234); + + verify_that!( + actual, + matches_pattern!(&AStruct( + eq(123), + eq(234), // Keep the trailing comma, block reformatting + )) + ) +} + +#[test] +fn matches_tuple_struct_with_interleaved_underscore() -> Result<()> { + #[derive(Debug)] + struct NonCopy; + #[derive(Debug)] + struct AStruct(u32, NonCopy, u32); + let actual = AStruct(1, NonCopy, 3); + + verify_that!(actual, matches_pattern!(&AStruct(eq(1), _, eq(3))))?; + verify_that!(actual, matches_pattern!(AStruct(eq(&1), _, eq(&3)))) +} + +#[test] +fn matches_tuple_struct_non_exhaustive() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + struct AStruct(i32, u32); + let actual = AStruct(1, 3); + + verify_that!(actual, matches_pattern!(&AStruct(_, ..)))?; + verify_that!(actual, matches_pattern!(AStruct(_, ..))) +} + +#[test] +fn matches_generic_tuple_struct_exhaustively() -> Result<()> { + #[allow(dead_code)] + #[derive(Debug)] + struct AStruct(T, u32); + let actual = AStruct(1, 3); + + verify_that!(actual, matches_pattern!(&AStruct(_, _)))?; + verify_that!(actual, matches_pattern!(AStruct(_, _))) +} diff --git a/third_party/crates/googletest-0.14.3/tests/no_color_test.rs b/third_party/crates/googletest-0.14.3/tests/no_color_test.rs new file mode 100644 index 0000000..081a335 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/no_color_test.rs @@ -0,0 +1,50 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use std::fmt::{Display, Write}; + +// Make a long text with each element of the iterator on one line. +// `collection` must contains at least one element. +fn build_text(mut collection: impl Iterator) -> String { + let mut text = String::new(); + write!(&mut text, "{}", collection.next().expect("Provided collection without elements")) + .unwrap(); + for item in collection { + write!(&mut text, "\n{item}").unwrap(); + } + text +} + +#[test] +fn colors_suppressed_when_both_no_color_and_force_color_are_set() -> googletest::Result<()> { + std::env::set_var("NO_COLOR", "1"); + std::env::set_var("FORCE_COLOR", "1"); + + let result = verify_that!(build_text(1..50), eq(&build_text(1..51))); + + verify_that!( + result, + err(displays_as(contains_substring( + " + Difference(-actual / +expected): + 1 + 2 + <---- 45 common lines omitted ----> + 48 + 49 + +50" + ))) + ) +} diff --git a/third_party/crates/googletest-0.14.3/tests/no_std_test.rs b/third_party/crates/googletest-0.14.3/tests/no_std_test.rs new file mode 100644 index 0000000..6afbc03 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/no_std_test.rs @@ -0,0 +1,37 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_std] + +/// A simple, no_std function. +fn no_std_identity(value: u32) -> u32 { + value +} + +#[cfg(test)] +mod tests { + use super::*; + use googletest::prelude::*; + + #[gtest] + fn no_std_verify() -> Result<()> { + verify_eq!(no_std_identity(42), 42)?; + Ok(()) + } + + #[gtest] + fn no_std_expect() { + expect_eq!(no_std_identity(214), 214) + } +} diff --git a/third_party/crates/googletest-0.14.3/tests/pointwise_matcher_test.rs b/third_party/crates/googletest-0.14.3/tests/pointwise_matcher_test.rs new file mode 100644 index 0000000..1a282e8 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/pointwise_matcher_test.rs @@ -0,0 +1,191 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; +use indoc::indoc; + +#[test] +fn pointwise_matches_single_element() -> Result<()> { + let value = vec![1]; + verify_that!(value, pointwise!(lt, vec![&2])) +} + +#[test] +fn pointwise_matches_two_elements() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, pointwise!(|x| points_to(lt(x)), vec![2, 3])) +} + +#[test] +fn pointwise_matches_iterator_returning_by_value() -> Result<()> { + #[derive(Clone, Copy, Debug)] + struct Countdown(i32); + impl Iterator for Countdown { + type Item = i32; + + fn next(&mut self) -> Option { + match self.0 { + 0 => None, + x => { + self.0 -= 1; + Some(x) + } + } + } + } + verify_that!(Countdown(3), pointwise!(eq, vec![3, 2, 1])) +} + +#[test] +fn pointwise_matches_two_elements_with_array() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, pointwise!(lt, [&2, &3])) +} + +#[test] +fn pointwise_matches_two_element_slice() -> Result<()> { + let value = vec![1, 2]; + let slice = value.as_slice(); + verify_that!(slice, pointwise!(lt, [&2, &3])) +} + +#[test] +fn pointwise_does_not_match_value_of_wrong_length() -> Result<()> { + let value = vec![1]; + verify_that!(value, not(pointwise!(lt, vec![&2, &3]))) +} + +#[test] +fn pointwise_does_not_match_value_not_matching_in_first_position() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, not(pointwise!(lt, vec![&1, &3]))) +} + +#[test] +fn pointwise_does_not_match_value_not_matching_in_second_position() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, not(pointwise!(lt, vec![&2, &2]))) +} + +#[test] +fn pointwise_allows_qualified_matcher_name() -> Result<()> { + mod submodule { + pub(super) use super::lt; + } + let value = vec![1]; + verify_that!(value, pointwise!(submodule::lt, vec![&2])) +} + +#[test] +fn pointwise_returns_mismatch_when_actual_value_has_wrong_length() -> Result<()> { + let result = verify_that!(vec![1, 2, 3], pointwise!(eq, vec![&1, &2])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 2, 3] + Expected: has elements satisfying respectively: + 0. is equal to 1 + 1. is equal to 2 + Actual: [1, 2, 3], + which has size 3 (expected 2) + " + )))) + ) +} + +#[test] +fn pointwise_returns_mismatch_when_actual_value_does_not_match_on_first_item() -> Result<()> { + let result = verify_that!(vec![1, 2, 3], pointwise!(eq, vec![&2, &2, &3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 2, 3] + Expected: has elements satisfying respectively: + 0. is equal to 2 + 1. is equal to 2 + 2. is equal to 3 + Actual: [1, 2, 3], + where element #0 is 1, which isn't equal to 2 + " + )))) + ) +} + +#[test] +fn pointwise_returns_mismatch_when_actual_value_does_not_match_on_second_item() -> Result<()> { + let result = verify_that!(vec![1, 2, 3], pointwise!(eq, &vec![1, 3, 3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 2, 3] + Expected: has elements satisfying respectively: + 0. is equal to 1 + 1. is equal to 3 + 2. is equal to 3 + Actual: [1, 2, 3], + where element #1 is 2, which isn't equal to 3 + " + )))) + ) +} + +#[test] +fn pointwise_returns_mismatch_when_actual_value_does_not_match_on_first_and_second_items( +) -> Result<()> { + let result = verify_that!(vec![1, 2, 3], pointwise!(eq, &vec![2, 3, 3])); + + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 2, 3] + Expected: has elements satisfying respectively: + 0. is equal to 2 + 1. is equal to 3 + 2. is equal to 3 + Actual: [1, 2, 3], + where: + * element #0 is 1, which isn't equal to 2 + * element #1 is 2, which isn't equal to 3" + )))) + ) +} + +#[test] +fn pointwise_matches_single_element_with_lambda_expression_with_extra_value() -> Result<()> { + let value = [1.00001f32]; + verify_that!(value, pointwise!(|v| near(v, 0.0001), vec![1.0])) +} + +#[test] +fn pointwise_matches_single_element_with_two_containers() -> Result<()> { + let value = [1.00001f32]; + verify_that!(value, pointwise!(near, vec![1.0], vec![0.0001])) +} + +#[test] +fn pointwise_matches_single_element_with_three_containers() -> Result<()> { + let value = [1.00001f32]; + verify_that!( + value, + pointwise!(|v, t, u| near(v, t + u), vec![1.0f32], vec![0.0001f32], vec![0.5f32]) + ) +} diff --git a/third_party/crates/googletest-0.14.3/tests/property_matcher_test.rs b/third_party/crates/googletest-0.14.3/tests/property_matcher_test.rs new file mode 100644 index 0000000..8c20a35 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/property_matcher_test.rs @@ -0,0 +1,396 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::matcher::MatcherResult; +use googletest::prelude::*; +use googletest::Result; + +#[derive(Debug)] +struct SomeStruct { + a_property: u32, +} + +impl SomeStruct { + fn get_property(&self) -> u32 { + self.a_property + } + + fn get_property_ref(&self) -> &u32 { + &self.a_property + } + + fn add_product_to_field(&self, a: u32, b: u32) -> u32 { + self.a_property + a * b + } + + fn get_property_ref_with_params(&self, _a: u32, _b: u32) -> &u32 { + &self.a_property + } + + fn generic_method(&self, _a: U, _b: V) -> u32 { + self.a_property + } + + fn generic_method_ref(&self, _a: U, _b: V) -> &u32 { + &self.a_property + } +} + +#[test] +fn matches_struct_with_matching_property() -> Result<()> { + let value = SomeStruct { a_property: 10 }; + verify_that!(value, property!(&SomeStruct.get_property(), eq(10))) +} + +#[test] +fn matches_struct_with_matching_property_with_parameters() -> Result<()> { + let value = SomeStruct { a_property: 10 }; + verify_that!(value, property!(&SomeStruct.add_product_to_field(2, 3), eq(16))) +} + +#[test] +fn matches_struct_with_matching_property_with_parameters_with_trailing_comma() -> Result<()> { + let value = SomeStruct { a_property: 10 }; + verify_that!(value, property!(&SomeStruct.add_product_to_field(2, 3,), eq(16))) +} + +#[test] +fn matches_struct_with_matching_property_ref() -> Result<()> { + let value = SomeStruct { a_property: 10 }; + verify_that!(value, property!(&SomeStruct.get_property_ref(), eq(&10))) +} + +#[test] +fn matches_struct_with_matching_string_reference_property() -> Result<()> { + #[derive(Debug)] + struct StructWithString { + property: String, + } + impl StructWithString { + fn get_property_ref(&self) -> &String { + &self.property + } + } + let value = StructWithString { property: "Something".into() }; + verify_that!(value, property!(&StructWithString.get_property_ref(), eq("Something"))) +} + +#[test] +fn matches_struct_with_matching_slice_property() -> Result<()> { + #[derive(Debug)] + struct StructWithVec { + property: Vec, + } + impl StructWithVec { + fn get_property_ref(&self) -> &[u32] { + &self.property + } + } + let value = StructWithVec { property: vec![1, 2, 3] }; + verify_that!(value, property!(&StructWithVec.get_property_ref(), eq([1, 2, 3]))) +} + +#[test] +fn matches_struct_with_matching_property_ref_with_parameters() -> Result<()> { + let value = SomeStruct { a_property: 10 }; + verify_that!(value, property!(&SomeStruct.get_property_ref_with_params(2, 3), eq(&10))) +} + +#[test] +fn matches_struct_with_matching_property_ref_with_parameters_and_trailing_comma() -> Result<()> { + let value = SomeStruct { a_property: 10 }; + verify_that!(value, property!(&SomeStruct.get_property_ref_with_params(2, 3,), eq(&10))) +} + +#[test] +fn does_not_match_struct_with_non_matching_property() -> Result<()> { + let value = SomeStruct { a_property: 2 }; + verify_that!(value, not(property!(&SomeStruct.get_property(), eq(1)))) +} + +#[test] +fn supports_fully_qualified_struct_path() -> Result<()> { + // Ensure that the macro expands to the fully-qualified struct path. + mod googletest {} + + let value = ::googletest::internal::test_data::TestStruct { value: 10 }; + verify_that!( + value, + property!(::googletest::internal::test_data::TestStruct.get_value(), eq(&10)) + )?; + verify_that!( + value, + property!(&::googletest::internal::test_data::TestStruct.get_value(), eq(10)) + )?; + verify_that!( + value, + property!(&::googletest::internal::test_data::TestStruct.get_value(), ref eq(&10)) + )?; + Ok(()) +} + +#[test] +fn supports_generic_struct_matches() -> Result<()> { + #[derive(Debug)] + struct GenericStruct { + #[allow(dead_code)] // The property1 field is used only for adding the type parameter. + property1: S, + property2: T, + } + + impl GenericStruct { + fn get_property(&self) -> T { + self.property2 + } + + fn get_property_ref(&self) -> &T { + &self.property2 + } + } + + let value = GenericStruct { property1: 1, property2: 10 }; + verify_that!(value, property!(GenericStruct::.get_property(), eq(&10)))?; + verify_that!(value, property!(&GenericStruct::.get_property(), eq(10)))?; + verify_that!(value, property!(&GenericStruct::.get_property(), ref eq(&10)))?; + verify_that!(value, property!(&GenericStruct::.get_property_ref(), eq(&10)))?; + verify_that!(value, property!(&GenericStruct::.get_property_ref(), ref eq(&&10)))?; + Ok(()) +} + +#[test] +fn supports_generic_method_matches() -> Result<()> { + let value = SomeStruct { a_property: 10 }; + verify_that!(value, property!(SomeStruct.generic_method::(1, 2), eq(&10)))?; + verify_that!(value, property!(&SomeStruct.generic_method::(1, 2), eq(10)))?; + verify_that!(value, property!(&SomeStruct.generic_method::(1, 2), ref eq(&10)))?; + verify_that!(value, property!(&SomeStruct.generic_method_ref::(1, 2), eq(&10)))?; + verify_that!(value, property!(&SomeStruct.generic_method_ref::(1, 2), ref eq(&&10)))?; + Ok(()) +} + +#[test] +fn supports_fully_qualified_generic_struct_path() -> Result<()> { + // Ensure that the macro expands to the fully-qualified struct path. + mod googletest {} + + let value = ::googletest::internal::test_data::GenericTestStruct { value: 1 }; + verify_that!( + value, + property!( + ::googletest::internal::test_data::GenericTestStruct::.get_value::(10), + eq(&10) + ) + )?; + verify_that!( + value, + property!( + &::googletest::internal::test_data::GenericTestStruct::.get_value::(10), + eq(10) + ) + )?; + verify_that!( + value, + property!( + &::googletest::internal::test_data::GenericTestStruct::.get_value::(10), + ref eq(&10)) + )?; + Ok(()) +} + +#[test] +fn describes_itself_in_matching_case() -> Result<()> { + verify_that!( + property!(&SomeStruct.get_property(), eq(1)).describe(MatcherResult::Match), + displays_as(eq("has property `get_property()`, which is equal to 1")) + ) +} + +#[test] +fn describes_itself_in_not_matching_case() -> Result<()> { + verify_that!( + property!(&SomeStruct.get_property(), eq(1)).describe(MatcherResult::NoMatch), + displays_as(eq("has property `get_property()`, which isn't equal to 1")) + ) +} + +#[test] +fn explains_mismatch_referencing_explanation_of_inner_matcher() -> Result<()> { + #[derive(Debug)] + struct SomeStruct; + + impl SomeStruct { + fn get_a_collection(&self) -> Vec { + vec![] + } + } + let value = SomeStruct; + let result = + verify_that!(value, property!(&SomeStruct.get_a_collection(), ref container_eq([1]))); + + verify_that!( + result, + err(displays_as(contains_substring( + "whose property `get_a_collection()` is `[]`, which is missing the element 1" + ))) + ) +} + +#[test] +fn explains_mismatch_referencing_explanation_of_inner_matcher_binding_mode() -> Result<()> { + #[derive(Debug)] + struct SomeStruct; + impl SomeStruct { + fn get_a_collection(&self) -> Vec { + vec![] + } + } + let result = + verify_that!(SomeStruct, property!(SomeStruct.get_a_collection(), container_eq([1]))); + + verify_that!( + result, + err(displays_as(contains_substring( + "whose property `get_a_collection()` is `[]`, which is missing the element 1" + ))) + ) +} + +#[test] +fn describes_itself_in_matching_case_for_ref() -> Result<()> { + verify_that!( + property!(&SomeStruct.get_property_ref(), eq(&1)).describe(MatcherResult::Match), + displays_as(eq("has property `get_property_ref()`, which is equal to 1")) + ) +} + +#[test] +fn describes_itself_in_not_matching_case_for_ref() -> Result<()> { + verify_that!( + property!(&SomeStruct.get_property_ref(), eq(&1)).describe(MatcherResult::NoMatch), + displays_as(eq("has property `get_property_ref()`, which isn't equal to 1")) + ) +} + +#[test] +fn explains_mismatch_referencing_explanation_of_inner_matcher_for_ref() -> Result<()> { + static EMPTY_COLLECTION: Vec = vec![]; + #[derive(Debug)] + struct SomeStruct; + impl SomeStruct { + fn get_a_collection_ref(&self) -> &[u32] { + &EMPTY_COLLECTION + } + } + let value = SomeStruct; + let result = + verify_that!(value, property!(&SomeStruct.get_a_collection_ref(), container_eq([1]))); + + verify_that!( + result, + err(displays_as(contains_substring( + "whose property `get_a_collection_ref()` is `[]`, which is missing the element 1" + ))) + ) +} + +#[test] +fn matches_copy_to_copy() -> Result<()> { + #[derive(Debug, Clone, Copy)] + struct Struct; + impl Struct { + fn property(self) -> i32 { + 32 + } + } + + verify_that!(Struct, property!(Struct.property(), eq(32))) +} + +#[test] +fn matches_copy_to_ref() -> Result<()> { + #[derive(Debug, Clone, Copy)] + struct Struct; + impl Struct { + fn property(self) -> String { + "something".into() + } + } + + verify_that!(Struct, property!(Struct.property(), ref eq("something"))) +} + +#[test] +fn matches_copy_but_by_ref() -> Result<()> { + #[derive(Debug, Clone, Copy)] + struct Struct; + impl Struct { + fn property(&self) -> String { + "something".into() + } + } + + verify_that!(&Struct, property!(&Struct.property(), ref eq("something"))) +} + +#[test] +fn matches_ref_to_ref() -> Result<()> { + #[derive(Debug)] + struct Struct; + impl Struct { + fn property(&self) -> String { + "something".into() + } + } + + verify_that!(Struct, property!(&Struct.property(), ref eq("something"))) +} + +#[test] +fn matches_ref_to_copy() -> Result<()> { + #[derive(Debug)] + struct Struct; + impl Struct { + fn property(&self) -> i32 { + 32 + } + } + + verify_that!(Struct, property!(&Struct.property(), eq(32))) +} + +#[test] +fn matches_ref_to_ref_with_binding_mode() -> Result<()> { + #[derive(Debug)] + struct Struct; + impl Struct { + fn property(&self) -> String { + "something".into() + } + } + + verify_that!(Struct, property!(Struct.property(), eq("something"))) +} + +#[test] +fn matches_property_auto_eq() -> Result<()> { + #[derive(Debug)] + struct Struct; + impl Struct { + fn property(&self) -> String { + "something".into() + } + } + + verify_that!(Struct, property!(Struct.property(), "something")) +} diff --git a/third_party/crates/googletest-0.14.3/tests/proptest_integration_test.rs b/third_party/crates/googletest-0.14.3/tests/proptest_integration_test.rs new file mode 100644 index 0000000..8e38b8a --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/proptest_integration_test.rs @@ -0,0 +1,30 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(feature = "proptest")] + +use googletest::prelude::*; +use proptest::test_runner::{Config, TestRunner}; + +#[test] +fn numbers_are_greater_than_zero() -> Result<()> { + let mut runner = TestRunner::new(Config::default()); + runner.run(&(1..100i32), |v| Ok(verify_that!(v, gt(0))?)).or_fail() +} + +#[test] +fn strings_are_nonempty() -> Result<()> { + let mut runner = TestRunner::new(Config::default()); + runner.run(&"[a-zA-Z0-9]+", |v| Ok(verify_that!(v, not(eq("")))?)).or_fail() +} diff --git a/third_party/crates/googletest-0.14.3/tests/reference_type_test.rs b/third_party/crates/googletest-0.14.3/tests/reference_type_test.rs new file mode 100644 index 0000000..3ba27b3 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/reference_type_test.rs @@ -0,0 +1,51 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[derive(Debug)] +struct ArenaHolder<'a, T: ?Sized> { + value: &'a T, +} + +impl<'a, T: PartialEq + ?Sized> PartialEq for ArenaHolder<'a, T> { + fn eq(&self, other: &T) -> bool { + self.value == other + } +} + +struct Strukt { + a_field: String, +} + +impl<'a> ArenaHolder<'a, Strukt> { + fn get_a_field(&self) -> ArenaHolder<'_, str> { + ArenaHolder { value: &self.value.a_field } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use googletest::prelude::*; + + #[gtest] + fn test() { + let arena = [Strukt { a_field: "hello".into() }]; + + let holder = ArenaHolder { value: &arena[0] }; + + let inner_holder = holder.get_a_field(); + + expect_that!(inner_holder, eq("hello")); + } +} diff --git a/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.golden b/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.golden new file mode 100644 index 0000000..d8faeff --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.golden @@ -0,0 +1,3 @@ +Google Test trace: + scoped_trace_panic_test.rs:24: Inner trace + scoped_trace_panic_test.rs:22: Outer trace diff --git a/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.rs b/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.rs new file mode 100644 index 0000000..d6e5009 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.rs @@ -0,0 +1,27 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::internal::test_outcome::TestOutcome; +use googletest::prelude::*; + +fn main() { + // Initialize the panic hook to capture and print scoped traces on panic. + TestOutcome::init_current_test_outcome(); + + scoped_trace!("Outer trace"); + { + scoped_trace!("Inner trace"); + panic!("Intentional panic"); + } +} diff --git a/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.sh b/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.sh new file mode 100644 index 0000000..714def6 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/scoped_trace_panic_test.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# fail on error +set -e + +# Read binary path from argument +BINARY_PATH="$1" + +if [ -z "$BINARY_PATH" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Run the binary, capture output, and process with sed +# We ignore the failure of the binary as per the original genrule (|| true) +( "${BINARY_PATH}" 2>&1 || true ) | \ +sed -n -e '/^Google Test trace:/p' -e '/^ .*trace/p' | \ +sed 's#^.*scoped_trace_panic_test.rs# scoped_trace_panic_test.rs#g' diff --git a/third_party/crates/googletest-0.14.3/tests/scoped_trace_test.rs b/third_party/crates/googletest-0.14.3/tests/scoped_trace_test.rs new file mode 100644 index 0000000..f28e95d --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/scoped_trace_test.rs @@ -0,0 +1,46 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(test)] +mod tests { + use googletest::internal::scoped_trace::get_scoped_traces; + use googletest::prelude::*; + + #[gtest] + fn test_scoped_trace_non_fatal() -> googletest::Result<()> { + let result = { + scoped_trace!("First trace"); + verify_eq!(1, 2) + }; + + verify_that!(result, err(displays_as(contains_substring("First trace"))))?; + + verify_that!(result, err(displays_as(contains_substring("Google Test trace:")))) + } + + #[gtest] + fn test_scoped_trace_nesting() { + scoped_trace!("Outer trace"); + { + scoped_trace!("Inner trace"); + let traces = get_scoped_traces(); + assert_eq!(traces.len(), 2); + assert_eq!(traces[0].message, "Outer trace"); + assert_eq!(traces[1].message, "Inner trace"); + } + let traces = get_scoped_traces(); + assert_eq!(traces.len(), 1); + assert_eq!(traces[0].message, "Outer trace"); + } +} diff --git a/third_party/crates/googletest-0.14.3/tests/tuple_matcher_test.rs b/third_party/crates/googletest-0.14.3/tests/tuple_matcher_test.rs new file mode 100644 index 0000000..c5ea0f5 --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/tuple_matcher_test.rs @@ -0,0 +1,265 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::matcher::MatcherResult; +use googletest::prelude::*; +use googletest::Result; +use indoc::indoc; + +#[test] +fn empty_matcher_matches_empty_tuple() -> Result<()> { + verify_that!((), ()) +} + +#[test] +fn empty_matcher_matches_empty_tuple_reference() -> Result<()> { + let a_ok: std::result::Result<(), String> = Ok(()); // Non copy + verify_that!(a_ok, ok(())) +} + +#[test] +fn singleton_matcher_matches_matching_singleton_tuple() -> Result<()> { + verify_that!((123,), (eq(123),)) +} + +#[test] +fn singleton_matcher_does_not_match_non_matching_singleton_tuple() -> Result<()> { + verify_that!((123,), not((eq(456),))) +} + +#[test] +fn pair_matcher_matches_matching_pair_tuple() -> Result<()> { + verify_that!((123, 456), (eq(123), eq(456))) +} + +#[test] +fn pair_matcher_matches_matching_pair_tuple_with_different_types() -> Result<()> { + verify_that!((123, "A string"), (eq(123), eq("A string"))) +} + +#[test] +fn pair_matcher_with_trailing_comma_matches_matching_pair_tuple() -> Result<()> { + verify_that!((123, 456), (eq(123), eq(456),)) +} + +#[test] +fn tuple_matcher_matches_matching_3_tuple() -> Result<()> { + verify_that!((1, 2, 3), (eq(1), eq(2), eq(3))) +} + +#[test] +fn tuple_matcher_matches_matching_4_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4), (eq(1), eq(2), eq(3), eq(4))) +} + +#[test] +fn tuple_matcher_matches_matching_5_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4, 5), (eq(1), eq(2), eq(3), eq(4), eq(5))) +} + +#[test] +fn tuple_matcher_matches_matching_6_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4, 5, 6), (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6))) +} + +#[test] +fn tuple_matcher_matches_matching_7_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4, 5, 6, 7), (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7))) +} + +#[test] +fn tuple_matcher_matches_matching_8_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4, 5, 6, 7, 8), (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8))) +} + +#[test] +fn tuple_matcher_matches_matching_9_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8, 9), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8), eq(9)) + ) +} + +#[test] +fn tuple_matcher_matches_matching_10_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8), eq(9), eq(10)) + ) +} + +#[test] +fn tuple_matcher_matches_matching_11_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8), eq(9), eq(10), eq(11)) + ) +} + +#[test] +fn tuple_matcher_matches_matching_12_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8), eq(9), eq(10), eq(11), eq(12)) + ) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_3_tuple() -> Result<()> { + verify_that!((1, 2, 3), (eq(1), eq(2), eq(3),)) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_4_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4), (eq(1), eq(2), eq(3), eq(4),)) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_5_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4, 5), (eq(1), eq(2), eq(3), eq(4), eq(5),)) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_6_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4, 5, 6), (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6),)) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_7_tuple() -> Result<()> { + verify_that!((1, 2, 3, 4, 5, 6, 7), (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7),)) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_8_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8),) + ) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_9_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8, 9), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8), eq(9),) + ) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_10_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8), eq(9), eq(10),) + ) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_11_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8), eq(9), eq(10), eq(11),) + ) +} + +#[test] +fn tuple_matcher_with_trailing_comma_matches_matching_12_tuple() -> Result<()> { + verify_that!( + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12), + (eq(1), eq(2), eq(3), eq(4), eq(5), eq(6), eq(7), eq(8), eq(9), eq(10), eq(11), eq(12),) + ) +} + +#[test] +fn tuple_matcher_1_has_correct_description_for_match() -> Result<()> { + verify_that!( + Matcher::<(i32,)>::describe(&(eq(1),), MatcherResult::Match), + displays_as(eq(indoc!( + " + is a tuple whose values respectively match: + is equal to 1" + ))) + ) +} + +#[test] +fn tuple_matcher_1_has_correct_description_for_mismatch() -> Result<()> { + verify_that!( + Matcher::<(i32,)>::describe(&(eq(1),), MatcherResult::NoMatch), + displays_as(eq(indoc!( + " + is a tuple whose values do not respectively match: + is equal to 1" + ))) + ) +} + +#[test] +fn tuple_matcher_2_has_correct_description_for_match() -> Result<()> { + verify_that!( + Matcher::<(i32, i32)>::describe(&(eq(1), eq(2)), MatcherResult::Match), + displays_as(eq(indoc!( + " + is a tuple whose values respectively match: + is equal to 1 + is equal to 2" + ))) + ) +} + +#[test] +fn tuple_matcher_2_has_correct_description_for_mismatch() -> Result<()> { + verify_that!( + Matcher::<(i32, i32)>::describe(&(eq(1), eq(2)), MatcherResult::NoMatch), + displays_as(eq(indoc!( + " + is a tuple whose values do not respectively match: + is equal to 1 + is equal to 2" + ))) + ) +} + +#[test] +fn describe_match_shows_which_tuple_element_did_not_match() -> Result<()> { + verify_that!( + (eq(1), eq(2)).explain_match((1, 3)), + displays_as(eq(indoc!( + " + which + is a tuple whose values do not respectively match: + is equal to 1 + is equal to 2 + Element #1 is 3, + which isn't equal to 2" + ))) + ) +} + +#[test] +fn describe_match_shows_which_two_tuple_elements_did_not_match() -> Result<()> { + verify_that!( + (eq(1), eq(2)).explain_match((2, 3)), + displays_as(eq(indoc!( + " + which + is a tuple whose values do not respectively match: + is equal to 1 + is equal to 2 + Element #0 is 2, + which isn't equal to 1 + Element #1 is 3, + which isn't equal to 2" + ))) + ) +} diff --git a/third_party/crates/googletest-0.14.3/tests/unordered_elements_are_matcher_test.rs b/third_party/crates/googletest-0.14.3/tests/unordered_elements_are_matcher_test.rs new file mode 100644 index 0000000..7b939da --- /dev/null +++ b/third_party/crates/googletest-0.14.3/tests/unordered_elements_are_matcher_test.rs @@ -0,0 +1,466 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use googletest::prelude::*; +use googletest::Result; +use indoc::indoc; +use std::collections::HashMap; + +#[test] +fn unordered_elements_are_matches_empty_vector() -> Result<()> { + let value: Vec = vec![]; + verify_that!(value, unordered_elements_are![]) +} + +#[test] +fn unordered_elements_are_matches_empty_vector_with_trailing_comma() -> Result<()> { + let value: Vec = vec![]; + verify_that!(value, unordered_elements_are![,]) +} + +#[test] +fn unordered_elements_are_matches_vector() -> Result<()> { + let value = vec![1, 2, 3]; + verify_that!(value, unordered_elements_are![eq(&1), eq(&2), eq(&3)]) +} + +#[test] +fn unordered_elements_are_matches_iterator_returning_by_value() -> Result<()> { + #[derive(Debug, Copy, Clone)] + struct Countdown(i32); + impl Iterator for Countdown { + type Item = i32; + + fn next(&mut self) -> Option { + match self.0 { + 0 => None, + x => { + self.0 -= 1; + Some(x) + } + } + } + } + verify_that!(Countdown(3), unordered_elements_are![eq(1), eq(2), eq(3)]) +} + +#[test] +fn unordered_elements_are_omitted() -> Result<()> { + let value = vec![1, 2, 3]; + verify_that!(value, {eq(&3), eq(&2), eq(&1)}) +} + +#[test] +fn unordered_elements_are_matches_slice() -> Result<()> { + let value = vec![1, 2, 3]; + let slice = value.as_slice(); + verify_that!(slice, unordered_elements_are![eq(&1), eq(&2), eq(&3)]) +} + +#[test] +fn unordered_elements_are_matches_hash_map() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two"), (3, "Three")]); + verify_that!( + value, + unordered_elements_are![(eq(&2), eq(&"Two")), (eq(&1), eq(&"One")), (eq(&3), eq(&"Three"))] + ) +} + +#[test] +fn unordered_elements_are_matches_hash_map_with_trailing_comma() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two"), (3, "Three")]); + verify_that!( + value, + unordered_elements_are![(eq(&2), eq(&"Two")), (eq(&1), eq(&"One")), (eq(&3), eq(&"Three")),] + ) +} + +#[test] +fn unordered_elements_are_does_not_match_hash_map_with_wrong_key() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two"), (4, "Three")]); + verify_that!( + value, + not(unordered_elements_are![ + (eq(&2), eq(&"Two")), + (eq(&1), eq(&"One")), + (eq(&3), eq(&"Three")) + ]) + ) +} + +#[test] +fn unordered_elements_are_does_not_match_hash_map_with_wrong_value() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two"), (3, "Four")]); + verify_that!( + value, + not(unordered_elements_are![ + (eq(&2), eq(&"Two")), + (eq(&1), eq(&"One")), + (eq(&3), eq(&"Three")) + ]) + ) +} + +#[test] +fn unordered_elements_are_does_not_match_hash_map_missing_element() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two")]); + verify_that!( + value, + not(unordered_elements_are![ + (eq(&2), eq(&"Two")), + (eq(&1), eq(&"One")), + (eq(&3), eq(&"Three")) + ]) + ) +} + +#[test] +fn unordered_elements_are_does_not_match_hash_map_with_extra_element() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two"), (3, "Three")]); + verify_that!(value, not(unordered_elements_are![(eq(&2), eq(&"Two")), (eq(&1), eq(&"One"))])) +} + +#[test] +fn unordered_elements_are_does_not_match_hash_map_with_mismatched_key_and_value() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Three"), (3, "Two")]); + verify_that!( + value, + not(unordered_elements_are![ + (eq(&2), eq(&"Two")), + (eq(&1), eq(&"One")), + (eq(&3), eq(&"Three")) + ]) + ) +} + +#[test] +fn unordered_elements_are_matches_vector_with_trailing_comma() -> Result<()> { + let value = vec![1, 2, 3]; + verify_that!(value, unordered_elements_are![eq(&1), eq(&2), eq(&3),]) +} + +#[test] +fn unordered_elements_are_matches_size() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, not(unordered_elements_are![eq(&1), eq(&2), eq(&3)])) +} + +#[test] +fn unordered_elements_are_admits_matchers_without_static_lifetime() -> Result<()> { + #[derive(Debug, PartialEq)] + struct AStruct(i32); + let expected_value = AStruct(123); + verify_that!(vec![AStruct(123)], unordered_elements_are![eq(&expected_value)]) +} + +#[test] +fn unordered_elements_are_with_map_admits_matchers_without_static_lifetime() -> Result<()> { + #[derive(Debug, PartialEq)] + struct AStruct(i32); + let expected_value = AStruct(123); + verify_that!( + HashMap::from([(1, AStruct(123))]), + unordered_elements_are![(eq(&1), eq(&expected_value))] + ) +} + +#[test] +fn unordered_elements_are_description_mismatch() -> Result<()> { + let result = verify_that!(vec![1, 4, 3], unordered_elements_are![eq(&1), eq(&2), eq(&3)]); + verify_that!( + result, + err(displays_as(contains_substring(indoc!( + " + Value of: vec![1, 4, 3] + Expected: contains elements matching in any order: + 0. is equal to 1 + 1. is equal to 2 + 2. is equal to 3 + Actual: [1, 4, 3], + whose element #1 does not match any expected elements and no elements match the expected element #1" + )))) + ) +} + +#[test] +fn unordered_elements_are_matches_unordered() -> Result<()> { + let value = vec![1, 2]; + verify_that!(value, unordered_elements_are![eq(&2), eq(&1)]) +} + +#[test] +fn unordered_elements_are_matches_unordered_with_repetition() -> Result<()> { + let value = vec![1, 2, 1, 2, 1]; + verify_that!(value, unordered_elements_are![eq(&1), eq(&1), eq(&1), eq(&2), eq(&2)]) +} + +#[test] +fn unordered_elements_are_explains_mismatch_due_to_wrong_size() -> Result<()> { + let matcher = unordered_elements_are![eq(&2), eq(&3), eq(&4)]; + verify_that!( + matcher.explain_match(&vec![2, 3]), + displays_as(eq("which has size 2 (expected 3)")) + ) +} + +#[test] +fn unordered_elements_are_description_no_full_match() -> Result<()> { + let matcher = unordered_elements_are![eq(&1), eq(&2), eq(&2)]; + verify_that!( + matcher.explain_match(&vec![1, 1, 2]), + displays_as(eq(indoc!( + " + which does not have a perfect match with the expected elements. The best match found was: + Actual element 1 at index 0 matched expected element `is equal to 1` at index 0. + Actual element 2 at index 2 matched expected element `is equal to 2` at index 1. + Actual element 1 at index 1 did not match any remaining expected element. + Expected element `is equal to 2` at index 2 did not match any remaining actual element." + ))) + ) +} + +#[test] +fn unordered_elements_are_unmatchable_expected_description_mismatch() -> Result<()> { + let matcher = unordered_elements_are![eq(&1), eq(&2), eq(&3)]; + verify_that!( + matcher.explain_match(&vec![1, 1, 3]), + displays_as(eq("which has no element matching the expected element #1")) + ) +} + +#[test] +fn unordered_elements_are_unmatchable_actual_description_mismatch() -> Result<()> { + let matcher = unordered_elements_are![eq(&1), eq(&1), eq(&3)]; + verify_that!( + matcher.explain_match(&vec![1, 2, 3]), + displays_as(eq("whose element #1 does not match any expected elements")) + ) +} + +fn create_matcher<'a>() -> impl Matcher<&'a Vec> { + unordered_elements_are![eq(&1)] +} + +#[test] +fn unordered_elements_are_works_when_matcher_is_created_in_subroutine() -> Result<()> { + verify_that!(vec![1], create_matcher()) +} + +fn create_matcher_for_map<'a>() -> impl Matcher<&'a HashMap> { + unordered_elements_are![(eq(&1), eq(&1))] +} + +#[test] +fn unordered_elements_are_works_when_matcher_for_maps_is_created_in_subroutine() -> Result<()> { + verify_that!(HashMap::from([(1, 1)]), create_matcher_for_map()) +} + +#[test] +fn contains_each_matches_when_one_to_one_correspondence_present() -> Result<()> { + verify_that!(vec![2, 3, 4], contains_each!(eq(&2), eq(&3), eq(&4))) +} + +#[test] +fn contains_each_supports_trailing_comma() -> Result<()> { + verify_that!(vec![2, 3, 4], contains_each!(eq(&2), eq(&3), eq(&4),)) +} + +#[test] +fn contains_each_matches_hash_map() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two"), (3, "Three")]); + verify_that!(value, contains_each![(eq(&2), eq(&"Two")), (eq(&1), eq(&"One"))]) +} + +#[test] +fn contains_each_matches_hash_map_with_trailing_comma() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two"), (3, "Three")]); + verify_that!(value, contains_each![(eq(&2), eq(&"Two")), (eq(&1), eq(&"One")),]) +} + +#[test] +fn contains_each_matches_when_no_matchers_present() -> Result<()> { + verify_that!(vec![2, 3, 4], contains_each!()) +} + +#[test] +fn contains_each_matches_when_no_matchers_present_and_trailing_comma() -> Result<()> { + verify_that!(vec![2, 3, 4], contains_each!(,)) +} + +#[test] +fn contains_each_matches_when_list_is_empty_and_no_matchers_present() -> Result<()> { + verify_that!(Vec::::new(), contains_each!()) +} + +#[test] +fn contains_each_matches_when_excess_elements_present() -> Result<()> { + verify_that!(vec![1, 2, 3, 4], contains_each!(eq(&2), eq(&3), eq(&4))) +} + +#[test] +fn contains_each_does_not_match_when_matchers_are_unmatched() -> Result<()> { + verify_that!(vec![1, 2, 3], not(contains_each!(eq(&2), eq(&3), eq(&4)))) +} + +#[test] +fn contains_each_explains_mismatch_due_to_wrong_size() -> Result<()> { + let matcher = contains_each![eq(&2), eq(&3), eq(&4)]; + verify_that!( + matcher.explain_match(&vec![2, 3]), + displays_as(eq("which has size 2 (expected at least 3)")) + ) +} + +#[test] +fn contains_each_explains_missing_element_in_mismatch() -> Result<()> { + let matcher = contains_each![eq(&2), eq(&3), eq(&4)]; + verify_that!( + matcher.explain_match(&vec![1, 2, 3]), + displays_as(eq("which has no element matching the expected element #2")) + ) +} + +#[test] +fn contains_each_explains_missing_elements_in_mismatch() -> Result<()> { + let matcher = contains_each![eq(&2), eq(&3), eq(&4), eq(&5)]; + verify_that!( + matcher.explain_match(&vec![0, 1, 2, 3]), + displays_as(eq("which has no elements matching the expected elements #2, #3")) + ) +} + +#[test] +fn contains_each_explains_mismatch_due_to_no_graph_matching_found() -> Result<()> { + let matcher = contains_each![ge(&2), ge(&2)]; + verify_that!( + matcher .explain_match(&vec![1, 2]), + displays_as(eq(indoc!( + " + which does not have a superset match with the expected elements. The best match found was: + Actual element 2 at index 1 matched expected element `is greater than or equal to 2` at index 0. + Actual element 1 at index 0 did not match any remaining expected element. + Expected element `is greater than or equal to 2` at index 1 did not match any remaining actual element.")) + )) +} + +#[test] +fn is_contained_in_matches_with_empty_vector() -> Result<()> { + let value: Vec = vec![]; + verify_that!(value, is_contained_in!()) +} + +#[test] +fn is_contained_in_matches_with_empty_vector_and_trailing_comma() -> Result<()> { + let value: Vec = vec![]; + verify_that!(value, is_contained_in!(,)) +} + +#[test] +fn is_contained_in_matches_when_one_to_one_correspondence_present() -> Result<()> { + verify_that!(vec![2, 3, 4], is_contained_in!(eq(&2), eq(&3), eq(&4))) +} + +#[test] +fn is_contained_supports_trailing_comma() -> Result<()> { + verify_that!(vec![2, 3, 4], is_contained_in!(eq(&2), eq(&3), eq(&4),)) +} + +#[test] +fn is_contained_in_matches_hash_map() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two")]); + verify_that!( + value, + is_contained_in![(eq(&2), eq(&"Two")), (eq(&1), eq(&"One")), (eq(&3), eq(&"Three"))] + ) +} + +#[test] +fn is_contained_in_matches_hash_map_with_trailing_comma() -> Result<()> { + let value: HashMap = HashMap::from([(1, "One"), (2, "Two")]); + verify_that!( + value, + is_contained_in![(eq(&2), eq(&"Two")), (eq(&1), eq(&"One")), (eq(&3), eq(&"Three")),] + ) +} + +#[test] +fn is_contained_in_matches_when_container_is_empty() -> Result<()> { + verify_that!(vec![1; 0], is_contained_in!(eq(&2), eq(&3), eq(&4))) +} + +#[test] +fn is_contained_in_matches_when_excess_matchers_present() -> Result<()> { + verify_that!(vec![3, 4], is_contained_in!(eq(&2), eq(&3), eq(&4))) +} + +#[test] +fn is_contained_in_does_not_match_when_elements_are_unmatched() -> Result<()> { + verify_that!(vec![1, 2, 3], not(is_contained_in!(eq(&2), eq(&3), eq(&4)))) +} + +#[test] +fn is_contained_in_explains_mismatch_due_to_wrong_size() -> Result<()> { + let matcher = is_contained_in![eq(&2), eq(&3)]; + verify_that!( + matcher.explain_match(&vec![2, 3, 4]), + displays_as(eq("which has size 3 (expected at most 2)")) + ) +} + +#[test] +fn is_contained_in_explains_missing_element_in_mismatch() -> Result<()> { + let matcher = is_contained_in![eq(&2), eq(&3), eq(&4)]; + verify_that!( + matcher.explain_match(&vec![1, 2, 3]), + displays_as(eq("whose element #0 does not match any expected elements")) + ) +} + +#[test] +fn is_contained_in_explains_missing_elements_in_mismatch() -> Result<()> { + let matcher = is_contained_in![eq(&2), eq(&3), eq(&4), eq(&5)]; + verify_that!( + matcher.explain_match(&vec![0, 1, 2, 3]), + displays_as(eq("whose elements #0, #1 do not match any expected elements")) + ) +} + +#[test] +fn is_contained_in_explains_mismatch_due_to_no_graph_matching_found() -> Result<()> { + let matcher = is_contained_in![ge(&1), ge(&3)]; + verify_that!( + matcher.explain_match(&vec![1, 2]), + displays_as(eq(indoc!( + " + which does not have a subset match with the expected elements. The best match found was: + Actual element 1 at index 0 matched expected element `is greater than or equal to 1` at index 0. + Actual element 2 at index 1 did not match any remaining expected element. + Expected element `is greater than or equal to 3` at index 1 did not match any remaining actual element.")) + )) +} + +#[test] +fn unordered_elements_are_with_auto_eq() -> Result<()> { + verify_that!(vec![3, 4, 2], unordered_elements_are![&2, &3, &4]) +} + +#[test] +fn contains_each_with_auto_eq() -> Result<()> { + verify_that!(vec![3, 4, 2], contains_each![&2, &4]) +} + +#[test] +fn is_contained_in_with_auto_eq() -> Result<()> { + verify_that!(vec![3, 4, 2], is_contained_in![&1, &2, &3, &4]) +} diff --git a/third_party/crates/googletest_macro-0.14.3/.cargo-checksum.json b/third_party/crates/googletest_macro-0.14.3/.cargo-checksum.json new file mode 100644 index 0000000..d1fa858 --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"bd02a169c37bfb4d4180fde42695267a0ae90eae8f047d82ec43802aece8b342","Cargo.lock":"61eaa8a3d699f5246996d53825aea5cbd241011dd9c1adea60064f9dbbbf26e5","Cargo.toml":"9ec02c08ca5bf9ae77d9509b5c99d7bff70406bb773017bc7e7d45900e3e25bd","Cargo.toml.orig":"0e528fbf7a0659c362227b2be56bbe85a5b12e22e6efbdc81a03795df9fa602f","LICENSE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","README.md":"416912f481d1b8c7580ebae4ed036e6c72adb5acdaef5e9e152f5d9ea4c5e479","src/lib.rs":"436715fbff5b895e88b288b8d40c521bb74b68f7202845a1f71483660b7624e6","src/matches_pattern.rs":"af3d59cf8235bc1da31f07fabcd3d769fb238767e77617a906f027df44333108","src/verify_pred.rs":"ad94e1afaffda7fcb4adaeb67422c33d2800fcb54e16a32edffdc33d95eb1b48"},"package":"2ae6abc96141edd26bf5aeec0f119c129c44de3ced09e5073711a02cb74725d0"} \ No newline at end of file diff --git a/third_party/crates/googletest_macro-0.14.3/.cargo_vcs_info.json b/third_party/crates/googletest_macro-0.14.3/.cargo_vcs_info.json new file mode 100644 index 0000000..442444c --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "871f28b74e91e1d2a38e139ea03a9809d9d85f10" + }, + "path_in_vcs": "googletest_macro" +} \ No newline at end of file diff --git a/third_party/crates/googletest_macro-0.14.3/BUILD.bazel b/third_party/crates/googletest_macro-0.14.3/BUILD.bazel new file mode 100644 index 0000000..5a2d461 --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/BUILD.bazel @@ -0,0 +1,115 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars") +load("@rules_rust//rust:defs.bzl", "rust_proc_macro") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_proc_macro( + name = "googletest_macro", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_root = "src/lib.rs", + edition = "2021", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=googletest_macro", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.14.3", + deps = [ + "//third_party/crates/proc-macro2-1.0.107:proc_macro2", + "//third_party/crates/quote-1.0.47:quote", + "//third_party/crates/syn-2.0.119:syn", + ], +) diff --git a/third_party/crates/googletest_macro-0.14.3/Cargo.lock b/third_party/crates/googletest_macro-0.14.3/Cargo.lock new file mode 100644 index 0000000..7e4ebbe --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/Cargo.lock @@ -0,0 +1,47 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "googletest_macro" +version = "0.14.3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" diff --git a/third_party/crates/googletest_macro-0.14.3/Cargo.toml b/third_party/crates/googletest_macro-0.14.3/Cargo.toml new file mode 100644 index 0000000..ffafff8 --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/Cargo.toml @@ -0,0 +1,54 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "googletest_macro" +version = "0.14.3" +authors = ["Bradford Hovinen "] +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Procedural macros for GoogleTest Rust" +readme = "README.md" +keywords = [ + "unit", + "matcher", + "testing", + "assertions", +] +categories = [ + "development-tools", + "development-tools::testing", +] +license = "Apache-2.0" +repository = "https://github.com/google/googletest-rust" + +[lib] +name = "googletest_macro" +path = "src/lib.rs" +proc-macro = true + +[dependencies.proc-macro2] +version = "1.0.106" + +[dependencies.quote] +version = "1.0.44" + +[dependencies.syn] +version = "2.0.114" +features = [ + "full", + "extra-traits", +] diff --git a/third_party/crates/googletest_macro-0.14.3/Cargo.toml.orig b/third_party/crates/googletest_macro-0.14.3/Cargo.toml.orig new file mode 100644 index 0000000..357aaa4 --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/Cargo.toml.orig @@ -0,0 +1,35 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "googletest_macro" +version = "0.14.3" +keywords = ["unit", "matcher", "testing", "assertions"] +categories = ["development-tools", "development-tools::testing"] +description = "Procedural macros for GoogleTest Rust" +repository = "https://github.com/google/googletest-rust" +license = "Apache-2.0" +edition = "2021" +authors = [ + "Bradford Hovinen ", +] + +[dependencies] +quote = "1.0.44" +syn = {version = "2.0.114", features = ["full", "extra-traits"]} +proc-macro2 = "1.0.106" + +[lib] +name = "googletest_macro" +proc-macro = true diff --git a/third_party/crates/googletest_macro-0.14.3/LICENSE b/third_party/crates/googletest_macro-0.14.3/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/crates/googletest_macro-0.14.3/README.md b/third_party/crates/googletest_macro-0.14.3/README.md new file mode 100644 index 0000000..9b66ba9 --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/README.md @@ -0,0 +1,5 @@ +## Procedural macros for GoogleTest Rust + +This crate is only useful as a dependency of +[GoogleTest Rust](https://crates.io/crates/googletest). It cannot be used on its +own. diff --git a/third_party/crates/googletest_macro-0.14.3/src/lib.rs b/third_party/crates/googletest_macro-0.14.3/src/lib.rs new file mode 100644 index 0000000..8795e53 --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/src/lib.rs @@ -0,0 +1,415 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use quote::quote; +use syn::{ + parse_macro_input, parse_quote, punctuated::Punctuated, spanned::Spanned, Attribute, + DeriveInput, Expr, ExprLit, FnArg, ItemFn, Lit, MetaNameValue, PatType, ReturnType, Signature, + Type, +}; + +/// Marks a test to be run by the Google Rust test runner. +/// +/// Annotate tests the same way ordinary Rust tests are annotated: +/// +/// ```ignore +/// #[gtest] +/// fn should_work() { +/// ... +/// } +/// ``` +/// +/// The test function is not required to have a return type. If it does have a +/// return type, that type must be [`googletest::Result`]. One may do this if +/// one wishes to use both fatal and non-fatal assertions in the same test. For +/// example: +/// +/// ```ignore +/// #[gtest] +/// fn should_work() -> googletest::Result<()> { +/// let value = 2; +/// expect_that!(value, gt(0)); +/// verify_that!(value, eq(2)) +/// } +/// ``` +/// +/// This macro can be used with `#[should_panic]` to indicate that the test is +/// expected to panic. For example: +/// +/// ```ignore +/// #[gtest] +/// #[should_panic] +/// fn passes_due_to_should_panic() { +/// let value = 2; +/// expect_that!(value, gt(0)); +/// panic!("This panics"); +/// } +/// ``` +/// +/// Using `#[should_panic]` modifies the behaviour of `#[gtest]` so +/// that the test panics (and passes) if any non-fatal assertion occurs. +/// For example, the following test passes: +/// +/// ```ignore +/// #[gtest] +/// #[should_panic] +/// fn passes_due_to_should_panic_and_failing_assertion() { +/// let value = 2; +/// expect_that!(value, eq(0)); +/// } +/// ``` +/// +/// [`googletest::Result`]: type.Result.html +#[proc_macro_attribute] +pub fn gtest( + _args: proc_macro::TokenStream, + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + let ItemFn { mut attrs, sig, block, .. } = parse_macro_input!(input as ItemFn); + + let sig_ident = &sig.ident; + let test_case_hash: u64 = { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut h = DefaultHasher::new(); + + // Only consider attrs and name for stability. Changing the function body should + // not affect the test case distribution. + attrs.hash(&mut h); + sig_ident.hash(&mut h); + h.finish() + }; + + let (skipped_test_result, outer_return_type, trailer) = attrs + .iter() + .find(|attr| attr.path().is_ident("should_panic")) + .map(|attr| { + let error_message = extract_should_panic_expected(attr).unwrap_or("".to_string()); + ( + quote! { + { + panic!("{}", #error_message); + } + }, + quote! { () }, + quote! { .unwrap(); } + )}) + .unwrap_or_else(|| + ( + quote! {Ok(())}, + quote! { ::core::result::Result<(), googletest::internal::test_outcome::TestFailure> }, + quote! {}, + )); + + let is_rstest_enabled = is_rstest_enabled(&attrs); + let outer_sig = { + let mut outer_sig = sig.clone(); + outer_sig.output = ReturnType::Default; + if !is_rstest_enabled { + outer_sig.inputs = Punctuated::new(); + } + outer_sig + }; + + let output_type = match sig.output { + ReturnType::Default => None, + ReturnType::Type(_, ref ty) => Some(quote! {#ty}), + }; + let is_async = sig.asyncness.is_some(); + let maybe_async = is_async.then(|| quote! { async }); + let maybe_await = is_async.then(|| quote! { .await }); + let invocation = if is_rstest_enabled { + // Rstest may refer in block to its fixtures. Hence, we only wrap it in a + // closure to capture them. + let result_type = output_type.clone().unwrap_or_else(|| quote! {()}); + let mut invocation = quote! { + (#maybe_async move || -> #result_type { + #block + })() #maybe_await + }; + if output_type.is_none() { + invocation = quote! { { let () = #invocation; googletest::Result::<()>::Ok(()) } }; + } + invocation + } else { + let closure_body = match closure_body(&sig) { + Ok(body) => body, + Err(e) => return e.into_compile_error().into(), + }; + + // In the sync case, the ? operator returns from the surrounding function. So + // we redeclare the original test function internally. + let result_type = output_type.unwrap_or_else(|| quote! {googletest::Result<()>}); + quote! { + (#maybe_async move || -> #result_type { + #sig { #block } + #closure_body + })() #maybe_await + } + }; + if !attrs.iter().any(is_test_attribute) && !is_rstest_enabled { + let test_attr: Attribute = parse_quote! { + #[::core::prelude::v1::test] + }; + attrs.push(test_attr); + }; + let function = quote! { + #(#attrs)* + #outer_sig -> #outer_return_type { + if !googletest::internal::test_filter::test_should_run(concat!(module_path!(), "::", stringify!(#sig_ident))) { + #skipped_test_result + } else if googletest::internal::test_sharding::test_should_run(#test_case_hash) { + use googletest::internal::test_outcome::TestOutcome; + TestOutcome::init_current_test_outcome(); + TestOutcome::close_current_test_outcome(#invocation) + } else { + #skipped_test_result + } + #trailer + } + }; + function.into() +} + +/// Extract the optional "expected" string literal from a `should_panic` +/// attribute. +fn extract_should_panic_expected(attr: &Attribute) -> Option { + let Ok(name_value) = attr.parse_args::() else { + return None; + }; + match name_value.value { + Expr::Lit(ExprLit { lit: Lit::Str(expected), .. }) + if name_value.path.is_ident("expected") => + { + Some(expected.value()) + } + _ => None, + } +} + +/// Alias for [`googletest::gtest`]. +/// +/// Generally, prefer using `#[gtest]` to mark googletest-based tests. +/// +/// Use `#[test]` instead of `#[gtest]` to satisfy compatibility +/// requirements. For example, the rstest crate can be composed with other test +/// attributes but it requires the attribute to be named `test`. +/// +/// ```ignore +/// #[rstest] +/// #[googletest::test] +/// fn rstest_with_googletest() -> Result<()> { +/// verify_that!(1, eq(1)) +/// } +/// ``` +/// +/// [`googletest::gtest`]: attr.gtest.html +#[proc_macro_attribute] +pub fn test( + args: proc_macro::TokenStream, + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + gtest(args, input) +} + +fn is_test_attribute(attr: &Attribute) -> bool { + match attr.path().segments.last() { + Some(last_segment) => last_segment.ident == "test", + None => false, + } +} + +fn is_rstest_enabled(attributes: &[Attribute]) -> bool { + attributes.iter().any(|attr| matches!(attr.path().segments.last(), Some(last_segment) if last_segment.ident == "rstest")) +} + +enum FixtureKind { + Consumable, + MutableRef, + SharedRef, +} + +struct Fixture { + identifier: syn::Ident, + ty: Box, + kind: FixtureKind, +} + +impl Fixture { + fn new(index: usize, ty: Box) -> syn::Result { + let identifier = syn::Ident::new(&format!("__googletest__fixture__{index}"), ty.span()); + match &*ty { + Type::Reference(reference) => Ok(Self { + identifier, + ty: reference.elem.clone(), + kind: if reference.mutability.is_some() { + FixtureKind::MutableRef + } else { + FixtureKind::SharedRef + }, + }), + Type::Path(..) => Ok(Self { identifier, ty, kind: FixtureKind::Consumable }), + _ => Err(syn::Error::new( + ty.span(), + "Unexpected fixture type. Only references (&T or &mut T) and paths (T) are supported.", + )), + } + } + + fn wrap_call(&self, inner_call: proc_macro2::TokenStream) -> proc_macro2::TokenStream { + let Self { identifier, ty, kind } = self; + let ref_method = match kind { + FixtureKind::Consumable => { + return quote! { + #[allow(non_snake_case)] + let #identifier = <#ty as googletest::fixtures::ConsumableFixture>::set_up()?; + { #inner_call } + }; + } + FixtureKind::MutableRef => quote! { .as_mut() }, + FixtureKind::SharedRef => quote! { .as_ref() }, + }; + quote! { { + #[allow(non_snake_case, unused_mut)] + let mut #identifier = + googletest::__internal_macro_support::FixtureTearDownOnDrop::new( + <#ty as googletest::fixtures::Fixture>::set_up()?); + let result = { + #[allow(non_snake_case)] + let #identifier = #identifier #ref_method; + #inner_call + }; + #identifier.tear_down()?; + result + } } + } +} + +fn closure_body(signature: &Signature) -> syn::Result { + let input_types = signature + .inputs + .iter() + .enumerate() + .map(|(index, typed)| { + let FnArg::Typed(PatType { ty, .. }) = typed else { + return Err(syn::Error::new( + typed.span(), + "`self` receiver is not accepted as test argument", + )); + }; + Fixture::new(index, ty.clone()) + }) + .collect::>>()?; + + let mut block = { + let parameters = input_types.iter().map(|fixture| &fixture.identifier); + + let test_name = &signature.ident; + let mut invocation = quote! { + #test_name(#(#parameters, )*) + }; + if signature.asyncness.is_some() { + invocation = quote! { #invocation.await }; + } + if let ReturnType::Default = signature.output { + invocation = quote! { { let () = #invocation; googletest::Result::Ok(()) } }; + } + invocation + }; + + for fixture in input_types.iter().rev() { + block = fixture.wrap_call(block); + } + + Ok(block) +} + +#[proc_macro_derive(MatcherBase)] +pub fn derive(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let ast = parse_macro_input!(input as DeriveInput); + let DeriveInput { ident, generics, .. } = ast; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + quote! { + impl #impl_generics MatcherBase for #ident #ty_generics #where_clause {} + } + .into() +} + +mod matches_pattern; + +/// This is an implementation detail of `googletest::matches_pattern!`. +/// +/// It's not intended to be used directly. +#[doc(hidden)] +#[proc_macro] +pub fn __googletest_macro_matches_pattern( + input: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + matches_pattern::matches_pattern_impl(input) +} + +mod verify_pred; + +/// This is an implementation detail of `googletest::verify_pred!`. +/// +/// It's not intended to be used directly. +#[doc(hidden)] +#[proc_macro] +pub fn __googletest_macro_verify_pred(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + verify_pred::verify_pred_impl(input) +} + +/// Stringifies its argument (like `stringify!()` from the standard library) but +/// limits the output to a provided maximum length. +/// +/// The input is a tuple of `target` and `max_length` seprated by a comma. +/// The `max_length` is the maximum number of characters to include in the +/// abbreviated string. For example: +/// +/// ```ignore +/// #[rstest] +/// #[gtest] +/// fn test_abbreviated_string() -> Result<()> { +/// verifiy_eq!(__abbreviated_stringify!(|x| x + 1, 6), "|x|...")?; +/// Ok(()) +/// } +/// ``` +#[doc(hidden)] +#[proc_macro] +pub fn __abbreviated_stringify(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let input = input.to_string(); + let abbreviated = abbreviated_string(&input).unwrap(); + quote! { + #abbreviated + } + .into() +} + +fn abbreviated_string(target: &str) -> Result, &'static str> { + use std::borrow::Cow; + match target.rsplit_once(',') { + None => Err("Expect a `max_length` argument, but got none"), + Some((expr, limit)) => match limit.trim().parse::() { + Ok(limit) if expr.len() > limit => { + if limit >= 4 { + Ok(Cow::Owned(format!("{}...", &expr[..limit - 3]))) + } else { + Err("The `max_length` argument is too small. It must be at least 4.") + } + } + Ok(_) => Ok(Cow::Borrowed(expr)), + Err(_) => Err("The `max_length` argument is not a number."), + }, + } +} diff --git a/third_party/crates/googletest_macro-0.14.3/src/matches_pattern.rs b/third_party/crates/googletest_macro-0.14.3/src/matches_pattern.rs new file mode 100644 index 0000000..9c6151c --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/src/matches_pattern.rs @@ -0,0 +1,415 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use proc_macro2::{Delimiter, Group, Ident, Span, TokenStream, TokenTree}; +use quote::{quote, ToTokens}; +use syn::{ + parse::{Parse, ParseStream, Parser as _}, + parse_macro_input, + token::DotDot, + Expr, ExprCall, Pat, Token, +}; + +/// This is an implementation detail of `googletest::matches_pattern!`. It +/// assumes that a few symbols from `googletest::matchers` have been imported +/// and that `$crate` has been aliased to `googletest` (which might otherwise +/// have been imported as a different alias), both of which are done +/// by `googletest::matches_pattern!` before calling this proc macro. +pub(crate) fn matches_pattern_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let result = parse_macro_input!(input as ParsedMatchPattern).into_matcher_expr(); + quote! { #result }.into() +} + +/// Represents one of the root-level pseudo-patterns supported by +/// `match_pattern!`. +/// +/// Examples with `struct_name` set and `group` being `None`: +/// * `true` +/// * `Enum::Variant` +/// * `&Enum::Variant` +/// +/// Examples with `group` being non-`None`. +/// * `Struct { a: eq(1), b: ends_with("foo") }` +/// * `&Struct(ref eq(&1), ends_with("foo"))` +#[derive(Debug)] +struct ParsedMatchPattern { + struct_name: TokenStream, + group: Option, +} + +impl Parse for ParsedMatchPattern { + fn parse(input: ParseStream) -> syn::Result { + let mut struct_name: Vec = vec![]; + let mut group: Option = None; + + // Use a TT-muncher (as opposed to parsing `Expr` or `Pat` or such) since: + // - The `struct_name` can be a struct/enum type or literal value (`true`, `1`, + // `&"test"`). + // - The braced part supports syntax that is not valid rust like `&Foo { a: ref + // eq(1) }`. + input.step(|cursor| { + let mut rest = *cursor; + + while let Some((tt, next)) = rest.token_tree() { + // If we reached a group (`{...}` or `(...)`), that should be the end of the + // pattern. Other groups (`[...]`) are not supported in struct names, but that's + // checked later. + if let TokenTree::Group(g) = tt { + group = Some(g); + return Ok(((), next)); + } + + // Everything before the group is the struct/enum name, possibly prefixed with + // `&`. + struct_name.push(tt); + rest = next; + } + + // If no group found, remove any trailing comma that might have been + // incorporated into the struct/enum name. + if matches!(struct_name.last(), Some(TokenTree::Punct(p)) if p.as_char() == ',') { + struct_name.pop(); + } + + Ok(((), rest)) + })?; + + input.parse::>()?; + let struct_name = struct_name.into_iter().collect(); + Ok(ParsedMatchPattern { struct_name, group }) + } +} + +impl ParsedMatchPattern { + fn into_matcher_expr(self) -> TokenStream { + let Self { struct_name, group } = self; + // `matcher_pattern` supports both its custom (not necessarily valid rust) + // syntax and also native match pattern (like `Struct { .. }` and + // `Struct(_)`). So we need to speculatively attempt the first and then + // fall back to the latter if the first fails. If both fail, we return + // the error from the first attempt. + let mut first_err = None; + + // `matcher_pattern` special syntax. + if let Some(ref g) = group { + let res = match g.delimiter() { + Delimiter::Parenthesis => parse_tuple_pattern_args(struct_name.clone(), g.stream()), + Delimiter::Brace => parse_braced_pattern_args(struct_name.clone(), g.stream()), + Delimiter::Bracket => compile_err(g.span(), "[...] syntax is not meaningful"), + Delimiter::None => compile_err(g.span(), "undelimited group not supported"), + }; + match res { + Ok(res) => return res, + Err(e) => first_err = Some(e), + } + } + + // Standard `match` pattern (prioritize `first_err` if both fail). + into_match_pattern_expr(quote! { #struct_name #group }) + .map_err(|e| first_err.unwrap_or(e)) + .unwrap_or_else(syn::Error::into_compile_error) + } +} + +/// Returns a pattern match expression as long as `stream` is a valid pattern. +/// Otherwise, returns failure. +fn into_match_pattern_expr(stream: TokenStream) -> syn::Result { + // Only produce if stream successfully parses as a pattern. Otherwise, return + // failure so that we can instead return the error due to failing to parse + // the `matcher_pattern` custom syntax. + Pat::parse_multi.parse2(stream.clone())?; + Ok(quote! { + googletest::matchers::__internal_unstable_do_not_depend_on_these::pattern_only( + |v| matches!(v, #stream), + concat!("is ", stringify!(#stream)), + concat!("is not ", stringify!(#stream))) + }) +} + +//////////////////////////////////////////////////////////////////////////////// +// Parse tuple struct patterns + +/// Each part in a tuple matcher pattern that's between the commas. When `None`, +/// it represents `_` which matches anything. +struct MaybeTupleFieldPattern(Option); + +struct TupleFieldPattern { + ref_token: Option, + matcher: Expr, +} + +impl Parse for MaybeTupleFieldPattern { + fn parse(input: ParseStream) -> syn::Result { + let pattern = match input.parse::>()? { + Some(_) => None, + None => Some(TupleFieldPattern { ref_token: input.parse()?, matcher: input.parse()? }), + }; + Ok(MaybeTupleFieldPattern(pattern)) + } +} + +/// Parses a tuple struct's fields into a match expression. +fn parse_tuple_pattern_args( + struct_name: TokenStream, + group_content: TokenStream, +) -> syn::Result { + let (patterns, dot_dot) = + parse_list_terminated_pattern::.parse2(group_content)?; + let field_count = patterns.len(); + let field_patterns: Vec<_> = patterns + .into_iter() + .enumerate() + .filter_map(|(index, maybe_pattern)| maybe_pattern.0.map(|pattern| (index, pattern))) + .map(|(index, TupleFieldPattern { ref_token, matcher })| { + let index = syn::Index::from(index); + quote! { googletest::matchers::field!(#struct_name.#index, #ref_token #matcher) } + }) + .collect(); + + if field_patterns.is_empty() { + // It is possible that the logic above didn't generate any field matchers + // (e.g., for patterns like `Some(_)`). + // In this case we verify that the enum has the correct case, but don't + // verify the payload. + #[allow(clippy::manual_repeat_n)] + // `repeat_n` is not available on the Rust MSRV that we support in OSS + let ignored_fields = std::iter::repeat(quote! { _ }) + .take(field_count) + .chain(dot_dot.map(ToTokens::into_token_stream)); + let full_pattern = quote! { #struct_name ( #(#ignored_fields),* ) }; + + Ok(quote! { + googletest::matchers::__internal_unstable_do_not_depend_on_these::pattern_only( + |actual| { matches!(actual, #full_pattern) }, + concat!("is ", stringify!(#full_pattern)), + concat!("is not ", stringify!(#full_pattern)) + ) + }) + } else { + // We have created at least one field matcher. Each field matcher will verify + // not only its part of the payload, but also that the enum has the + // correct case. + let matcher = quote! { + googletest::matchers::__internal_unstable_do_not_depend_on_these::is( + stringify!(#struct_name), + all!( #(#field_patterns),* ) + ) + }; + + // Do a match to ensure: + // - Fields are exhaustively listed unless the pattern ended with `..`. + // - `UNDEFINED_SYMBOL(..)` fails to compile. + #[allow(clippy::manual_repeat_n)] + // `repeat_n` is not available on the Rust MSRV that we support in OSS + let empty_fields = std::iter::repeat(quote! { _ }) + .take(field_count) + .chain(dot_dot.map(ToTokens::into_token_stream)); + Ok(quote! { + googletest::matchers::__internal_unstable_do_not_depend_on_these::compile_assert_and_match( + |actual| { + // Exhaustively check that all field names are specified. + match actual { + #struct_name ( #(#empty_fields),* ) => (), + // The pattern below is unreachable if the type is a struct (as opposed to + // an enum). Since the macro can't know which it is, we always include it + // and just tell the compiler not to complain. + #[allow(unreachable_patterns)] + _ => {}, + } + }, + #matcher) + }) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Parse braced structs patterns + +enum FieldOrMethod { + Field(Ident), + Method(ExprCall), +} + +impl Parse for FieldOrMethod { + /// Parses the field name or method call along with the `:` that follows it. + fn parse(input: ParseStream) -> syn::Result { + // If the ident is followed by a single `:` and not a `::`, then it's a field + // match and not a method call match. + let value = if input.peek2(Token![:]) && !input.peek2(Token![::]) { + input.parse().map(FieldOrMethod::Field) + } else { + input.parse().map(FieldOrMethod::Method) + }?; + input.parse::()?; + Ok(value) + } +} + +/// Either field or method call matcher. E.g.: +/// * `field: starts_with("something")` or `field: _` +/// * `property(arg1, arg2): starts_with("something") +struct FieldOrMethodPattern { + ref_token: Option, + field_or_method: FieldOrMethod, + /// When `None`, it represents `_` which matches anything, meaning we should + /// ignore it. + matcher: Option, +} + +impl Parse for FieldOrMethodPattern { + fn parse(input: ParseStream) -> syn::Result { + let field_or_method: FieldOrMethod = input.parse()?; + let underscore = input.parse::>()?; + match underscore { + Some(underscore) if matches!(field_or_method, FieldOrMethod::Method(_)) => compile_err( + underscore.spans[0], + "Don't match a method call against `_`. Just omit it instead.", + ), + Some(_) => Ok(FieldOrMethodPattern { field_or_method, ref_token: None, matcher: None }), + None => Ok(FieldOrMethodPattern { + field_or_method, + ref_token: input.parse()?, + matcher: Some(input.parse()?), + }), + } + } +} + +/// Parses a struct's fields or method calls into a match expression. +fn parse_braced_pattern_args( + struct_name: TokenStream, + group_content: TokenStream, +) -> syn::Result { + let (patterns, dot_dot) = parse_list_terminated_pattern.parse2(group_content)?; + let mut field_names = vec![]; + let field_patterns: Vec = patterns + .into_iter() + .filter_map(|FieldOrMethodPattern { ref_token, field_or_method, matcher }| { + match field_or_method { + FieldOrMethod::Field(ident) => { + field_names.push(ident.clone()); + matcher.map(|matcher| { + quote! { field!(#struct_name . #ident, #ref_token #matcher) } + }) + } + FieldOrMethod::Method(call) => { + Some(quote! { property!(#struct_name . #call, #ref_token #matcher) }) + } + } + }) + .collect(); + + if field_patterns.is_empty() { + // It is possible that the logic above didn't generate any field matchers + // (e.g., for patterns like `AnEnum::Foo { a_field : _ }`). + // In this case we verify that the enum has the correct case, but don't + // verify the payload. + let full_pattern = quote! { #struct_name { #(#field_names: _,)* #dot_dot}}; + + Ok(quote! { + googletest::matchers::__internal_unstable_do_not_depend_on_these::pattern_only( + |actual| { matches!(actual, #full_pattern) }, + concat!("is ", stringify!(#full_pattern)), + concat!("is not ", stringify!(#full_pattern)) + ) + }) + } else { + // We have created at least one field matcher. Each field matcher will verify + // not only its part of the payload, but also that the enum has the + // correct case. + let matcher = quote! { + googletest::matchers::__internal_unstable_do_not_depend_on_these::is( + stringify!(#struct_name), + all!(#(#field_patterns),* ) + ) + }; + + // Do a match to ensure: + // - Fields are exhaustively listed unless the pattern ended with `..` and has + // any fields in the pattern. + // - `UNDEFINED_SYMBOL { .. }` fails to compile. + // + // The requisite that some fields are in the pattern is there because + // `matches_pattern!` also uses the brace notation for tuple structs when + // asserting on method calls on tuple structs. i.e. + // + // ``` + // struct Struct(u32); + // ... + // matches_pattern!(foo, Struct { bar(): eq(1) }) + // ``` + // and we can't emit an exhaustiveness check based on the `matches_pattern!`. + if field_names.is_empty() && dot_dot.is_none() && + // If there are no fields, then this check means that there are method patterns, and we can + // no longer be confident that this is a braced struct rather than a tuple struct. + !field_patterns.is_empty() + { + Ok(matcher) + } else { + Ok(quote! { + googletest::matchers::__internal_unstable_do_not_depend_on_these::compile_assert_and_match( + |actual| { + // Exhaustively check that all field names are specified. + match actual { + #struct_name { #(#field_names: _,)* #dot_dot } => {}, + // The pattern below is unreachable if the type is a struct (as opposed to + // an enum). Since the macro can't know which it is, we always include it + // and just tell the compiler not to complain. + #[allow(unreachable_patterns)] + _ => {}, + } + }, + #matcher) + }) + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// General-purpose helpers + +/// Returns the parsed struct pattern body along with a `..` if it appears at +/// the end of the body. +/// +/// This is like `Punctuated::parse_terminated`, but additionally allows for an +/// optional `..`, which cannot be followed by a comma. +fn parse_list_terminated_pattern( + input: ParseStream<'_>, +) -> syn::Result<(Vec, Option)> { + let mut patterns = vec![]; + while !input.is_empty() { + // Check for trailing `..`. + let dot_dot = input.parse::>()?; + if dot_dot.is_some() { + // Must be at the end of the group content. + return if input.is_empty() { + Ok((patterns, dot_dot)) + } else { + compile_err(input.span(), "`..` must be at the end of the struct pattern") + }; + } + + // Otherwise, parse the field/method patterns. + patterns.push(input.parse::()?); + if input.is_empty() { + break; + } + input.parse::()?; + } + Ok((patterns, None)) +} + +fn compile_err(span: Span, message: &str) -> syn::Result { + Err(syn::Error::new(span, message)) +} diff --git a/third_party/crates/googletest_macro-0.14.3/src/verify_pred.rs b/third_party/crates/googletest_macro-0.14.3/src/verify_pred.rs new file mode 100644 index 0000000..a4ebaa1 --- /dev/null +++ b/third_party/crates/googletest_macro-0.14.3/src/verify_pred.rs @@ -0,0 +1,183 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use quote::quote; +use syn::{parse_macro_input, punctuated::Punctuated, spanned::Spanned, token::Comma, Expr, Ident}; + +struct AccumulatePartsState { + var_num: usize, + error_message_ident: Ident, + statements: Vec, +} + +fn expr_to_string(expr: &Expr) -> String { + quote!(#expr).to_string() +} + +impl AccumulatePartsState { + fn new() -> Self { + Self { + var_num: 0, + error_message_ident: Ident::new( + "__googletest__verify_pred__error_message", + ::proc_macro2::Span::call_site(), + ), + statements: vec![], + } + } + + /// Takes an expression with chained field accesses and method calls and + /// accumulates intermediate expressions used for computing `verify_pred!`'s + /// expression, including intermediate variable assignments to evaluate + /// parts of the expression exactly once, and the format string used to + /// output intermediate values on condition failure. It returns the new form + /// of the input expression with parts of it potentially replaced by the + /// intermediate variables. + fn accumulate_parts(&mut self, expr: Expr) -> Expr { + // Literals don't need to be printed or stored in intermediate variables. + if is_literal(&expr) { + return expr; + } + let expr_string = expr_to_string(&expr); + let new_expr = match expr { + Expr::Group(mut group) => { + // This is an invisible group added for correct precedence in the AST. Just pass + // through without having a separate printing result. + *group.expr = self.accumulate_parts(*group.expr); + return Expr::Group(group); + } + Expr::Field(mut field) => { + // Don't assign field access to an intermediate variable to avoid moving out of + // non-`Copy` fields. + *field.base = self.accumulate_parts(*field.base); + Expr::Field(field) + } + Expr::Call(mut call) => { + // Cache args into intermediate variables. + call.args = self.define_variables_for_args(call.args); + // Cache function value into an intermediate variable. + self.define_variable(&Expr::Call(call)) + } + Expr::MethodCall(mut method_call) => { + *method_call.receiver = self.accumulate_parts(*method_call.receiver); + // Cache args into intermediate variables. + method_call.args = self.define_variables_for_args(method_call.args); + // Cache method value into an intermediate variable. + self.define_variable(&Expr::MethodCall(method_call)) + } + Expr::Binary(mut binary) => { + *binary.left = self.accumulate_parts(*binary.left); + *binary.right = self.accumulate_parts(*binary.right); + Expr::Binary(binary) + } + Expr::Unary(mut unary) => { + *unary.expr = self.accumulate_parts(*unary.expr); + Expr::Unary(unary) + } + // A path expression doesn't need to be stored in an intermediate variable. + // This avoids moving out of an existing variable. + Expr::Path(_) => expr, + // By default, assume it's some expression that needs to be cached to avoid + // double-evaluation. + _ => self.define_variable(&expr), + }; + let error_message_ident = &self.error_message_ident; + self.statements.push(quote! { + ::googletest::fmt::internal::__googletest__write_expr_value!( + &mut #error_message_ident, + #expr_string, + #new_expr, + ); + }); + new_expr + } + + // Defines a variable for each argument expression so that it's evaluated + // exactly once. + fn define_variables_for_args( + &mut self, + args: Punctuated, + ) -> Punctuated { + args.into_pairs() + .map(|mut pair| { + // Don't need to assign literals to intermediate variables. + if is_literal(pair.value()) { + return pair; + } + + let var_expr = self.define_variable(pair.value()); + let error_message_ident = &self.error_message_ident; + let expr_string = expr_to_string(pair.value()); + self.statements.push(quote! { + ::googletest::fmt::internal::__googletest__write_expr_value!( + &mut #error_message_ident, + #expr_string, + #var_expr, + ); + }); + + *pair.value_mut() = var_expr; + pair + }) + .collect() + } + + /// Defines a new variable assigned to the expression and returns the + /// variable as an expression to be used in place of the passed-in + /// expression. + fn define_variable(&mut self, value: &Expr) -> Expr { + let var_name = + Ident::new(&format!("__googletest__verify_pred__var{}", self.var_num), value.span()); + self.var_num += 1; + self.statements.push(quote! { + #[allow(non_snake_case)] + let mut #var_name = #value; + }); + syn::parse::(quote!(#var_name).into()).unwrap() + } +} + +// Whether it's a literal or unary operator applied to a literal (1, -1). +fn is_literal(expr: &Expr) -> bool { + match expr { + Expr::Lit(_) => true, + Expr::Unary(unary) => matches!(&*unary.expr, Expr::Lit(_)), + _ => false, + } +} + +pub fn verify_pred_impl(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let parsed = parse_macro_input!(input as Expr); + let error_message = quote!(#parsed).to_string() + " was false with"; + + let mut state = AccumulatePartsState::new(); + let pred_value = state.accumulate_parts(parsed); + let AccumulatePartsState { error_message_ident, mut statements, .. } = state; + + let _ = statements.pop(); // The last statement prints the full expression itself. + quote! { + { + let mut #error_message_ident = #error_message.to_string(); + #(#statements)* + if (#pred_value) { + Ok(()) + } else { + ::core::result::Result::Err( + ::googletest::internal::test_outcome::TestAssertionFailure::create( + #error_message_ident)) + } + } + } + .into() +} diff --git a/third_party/crates/num-traits-0.2.19/.cargo-checksum.json b/third_party/crates/num-traits-0.2.19/.cargo-checksum.json new file mode 100644 index 0000000..0c99818 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"e34bdc2d65c741cc679d2e5c72c8ed8d952d2e20c5712a3c950af4235363ad7b","Cargo.toml":"dd701a0da6cce7e6e047cc944b2330f3c5e48cd64329e10942ea7d9299d435be","Cargo.toml.orig":"172863fc77ccbdd76c7d65660c1030e553cc3629931808f18f4cd4cc73a08c53","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb","README.md":"3fa9b7a124e7bc011758919f6d13fbdce58027852abc6c0851ddf6addfedabd5","RELEASES.md":"eb3298353c12e0cd48c79437199bd7f159c4250da06f8362afcf71004e98fde5","build.rs":"d3969209fc1c9d201c66ed11820d0b328600d75b3971f8ceebeab04900bc0587","src/bounds.rs":"a79325f6a92269ad7df3d11b9ff700d2d13fb1160e28f0c091a644efc4badc2b","src/cast.rs":"d2c9337163c3d594a701bfce21aec56571625fa0117f970f14fd82dcc4504bf6","src/float.rs":"2caf20ca3227422dcd8981accc79898e889bbda97fce1fc7f881c98e7091d8c3","src/identities.rs":"c26d7f2fd6636721930d11d345e0c7103b0a4ac242dc4035dbc956a45eba422f","src/int.rs":"ca214bab6624cb7f19bcb439958aa34596e0b13d8fc9b0090a7b37e89946d124","src/lib.rs":"00973c9c52fd607d6e4b1da4ce4d5caa7d1bc39c8cd6f5d64878075766c8ad33","src/macros.rs":"ee96613a2c73a3bef10ec7ae4d359dbf5f0b41f83e8a87c3d62ccc18dd27e498","src/ops/bytes.rs":"24ade942062566d686c6de5674b5668ea3cd688c7c88bd8e5df7ea7f96d70c6e","src/ops/checked.rs":"01e6379bf1d8eeca9dcf8bb5397e419e898e4043b57b0e2470e225bc27e81e6a","src/ops/euclid.rs":"89c09c2df7fcc4bdba32533d4e7254dceb424085239544ea2a9ae8a606bf7383","src/ops/inv.rs":"dd80b9bd48d815f17855a25842287942317fa49d1fdcdd655b61bd20ef927cda","src/ops/mod.rs":"2b3c396af44cd240205ba8b560625fa00c07cf387139d2c49eeb7869545d976d","src/ops/mul_add.rs":"15bd64d9420c86300c5ea7f57aa736af2ef968e4e5eaaae03f62fd277f124569","src/ops/overflowing.rs":"01f4cd27f8b0e257687170cc537188029e08e5d13e0c552b01153be5d66d5716","src/ops/saturating.rs":"165993c829c10c4f60e32c8cf34434b669ef54284d7f73dc7ec58a22ba65e6fc","src/ops/wrapping.rs":"39d7bc7e074ba7590cd29b40206baed9cb30ae70dca2b7ceb460c6ca7eaad2a8","src/pow.rs":"92c12990d2396b2dabd4ba80e80ad706c0c8fd0f1b967ab3bdd9cb738b150702","src/real.rs":"b45bc1ca54549595c8d4f91a0e192769c1d924b40eda3446af4780f7a07ac8ed","src/sign.rs":"7ca11eebee94b553a33a9e53b7663ba5173db297dee523d1a2600fbbc80ef850","tests/cast.rs":"6fcc0d6653253182e979e42542fe971829cd24ab2c3a21a668e935c23d39f7c0"},"package":"071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"} \ No newline at end of file diff --git a/third_party/crates/num-traits-0.2.19/.cargo_vcs_info.json b/third_party/crates/num-traits-0.2.19/.cargo_vcs_info.json new file mode 100644 index 0000000..7ee50d6 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "7ec3d41d39b28190ec1d42db38021107b3951f3a" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/third_party/crates/num-traits-0.2.19/BUILD.bazel b/third_party/crates/num-traits-0.2.19/BUILD.bazel new file mode 100644 index 0000000..44be860 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/BUILD.bazel @@ -0,0 +1,190 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load( + "@rules_rust//cargo:defs.bzl", + "cargo_build_script", + "cargo_toml_env_vars", +) +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_library( + name = "num_traits", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "default", + "std", + ], + crate_root = "src/lib.rs", + edition = "2021", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=num-traits", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.2.19", + deps = [ + ":build_script_build", + ], +) + +cargo_build_script( + name = "_bs", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + build_script_env_files = [ + ":cargo_toml_env_vars", + ], + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + "**/*.rs", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "default", + "std", + ], + crate_name = "build_script_build", + crate_root = "build.rs", + data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + edition = "2021", + emit_warnings = False, + pkg_name = "num-traits", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=num-traits", + "manual", + "noclippy", + "norustfmt", + ], + version = "0.2.19", + visibility = ["//visibility:private"], + deps = [ + "//third_party/crates/autocfg-1.5.1:autocfg", + ], +) + +alias( + name = "build_script_build", + actual = ":_bs", + tags = ["manual"], +) diff --git a/third_party/crates/num-traits-0.2.19/Cargo.toml b/third_party/crates/num-traits-0.2.19/Cargo.toml new file mode 100644 index 0000000..97ceb1b --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/Cargo.toml @@ -0,0 +1,54 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.60" +name = "num-traits" +version = "0.2.19" +authors = ["The Rust Project Developers"] +build = "build.rs" +exclude = [ + "/ci/*", + "/.github/*", +] +description = "Numeric traits for generic mathematics" +homepage = "https://github.com/rust-num/num-traits" +documentation = "https://docs.rs/num-traits" +readme = "README.md" +keywords = [ + "mathematics", + "numerics", +] +categories = [ + "algorithms", + "science", + "no-std", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-num/num-traits" + +[package.metadata.docs.rs] +features = ["std"] +rustdoc-args = ["--generate-link-to-definition"] + +[dependencies.libm] +version = "0.2.0" +optional = true + +[build-dependencies.autocfg] +version = "1" + +[features] +default = ["std"] +i128 = [] +libm = ["dep:libm"] +std = [] diff --git a/third_party/crates/num-traits-0.2.19/Cargo.toml.orig b/third_party/crates/num-traits-0.2.19/Cargo.toml.orig new file mode 100644 index 0000000..226d416 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/Cargo.toml.orig @@ -0,0 +1,34 @@ +[package] +authors = ["The Rust Project Developers"] +description = "Numeric traits for generic mathematics" +documentation = "https://docs.rs/num-traits" +homepage = "https://github.com/rust-num/num-traits" +keywords = ["mathematics", "numerics"] +categories = ["algorithms", "science", "no-std"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-num/num-traits" +name = "num-traits" +version = "0.2.19" +readme = "README.md" +build = "build.rs" +exclude = ["/ci/*", "/.github/*"] +edition = "2021" +rust-version = "1.60" + +[package.metadata.docs.rs] +features = ["std"] +rustdoc-args = ["--generate-link-to-definition"] + +[dependencies] +libm = { version = "0.2.0", optional = true } + +[features] +default = ["std"] +libm = ["dep:libm"] +std = [] + +# vestigial features, now always in effect +i128 = [] + +[build-dependencies] +autocfg = "1" diff --git a/third_party/crates/num-traits-0.2.19/LICENSE-APACHE b/third_party/crates/num-traits-0.2.19/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/crates/num-traits-0.2.19/LICENSE-MIT b/third_party/crates/num-traits-0.2.19/LICENSE-MIT new file mode 100644 index 0000000..39d4bdb --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2014 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third_party/crates/num-traits-0.2.19/README.md b/third_party/crates/num-traits-0.2.19/README.md new file mode 100644 index 0000000..fd42070 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/README.md @@ -0,0 +1,58 @@ +# num-traits + +[![crate](https://img.shields.io/crates/v/num-traits.svg)](https://crates.io/crates/num-traits) +[![documentation](https://docs.rs/num-traits/badge.svg)](https://docs.rs/num-traits) +[![minimum rustc 1.60](https://img.shields.io/badge/rustc-1.60+-red.svg)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) +[![build status](https://github.com/rust-num/num-traits/workflows/master/badge.svg)](https://github.com/rust-num/num-traits/actions) + +Numeric traits for generic mathematics in Rust. + +## Usage + +Add this to your `Cargo.toml`: + +```toml +[dependencies] +num-traits = "0.2" +``` + +## Features + +This crate can be used without the standard library (`#![no_std]`) by disabling +the default `std` feature. Use this in `Cargo.toml`: + +```toml +[dependencies.num-traits] +version = "0.2" +default-features = false +# features = ["libm"] # <--- Uncomment if you wish to use `Float` and `Real` without `std` +``` + +The `Float` and `Real` traits are only available when either `std` or `libm` is enabled. + +The `FloatCore` trait is always available. `MulAdd` and `MulAddAssign` for `f32` +and `f64` also require `std` or `libm`, as do implementations of signed and floating- +point exponents in `Pow`. + +## Releases + +Release notes are available in [RELEASES.md](RELEASES.md). + +## Compatibility + +The `num-traits` crate is tested for rustc 1.60 and greater. + +## License + +Licensed under either of + + * [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + * [MIT license](http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be +dual licensed as above, without any additional terms or conditions. diff --git a/third_party/crates/num-traits-0.2.19/RELEASES.md b/third_party/crates/num-traits-0.2.19/RELEASES.md new file mode 100644 index 0000000..5f23f56 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/RELEASES.md @@ -0,0 +1,307 @@ +# Release 0.2.19 (2024-05-03) + +- [Upgrade to 2021 edition, **MSRV 1.60**][310] +- [The new `Float::clamp` limits values by minimum and maximum][305] + +**Contributors**: @cuviper, @michaelciraci + +[305]: https://github.com/rust-num/num-traits/pull/305 +[310]: https://github.com/rust-num/num-traits/pull/310 + +# Release 0.2.18 (2024-02-07) + +- [The new `Euclid::div_rem_euclid` and `CheckedEuclid::checked_div_rem_euclid` methods][291] + compute and return the quotient and remainder at the same time. +- [The new `TotalOrder` trait implements the IEEE 754 `totalOrder` predicate.][295] +- [The new `ConstZero` and `ConstOne` traits offered associated constants][303], + extending the non-const `Zero` and `One` traits for types that have constant values. + +**Contributors**: @andrewjradcliffe, @cuviper, @tarcieri, @tdelabro, @waywardmonkeys + +[291]: https://github.com/rust-num/num-traits/pull/291 +[295]: https://github.com/rust-num/num-traits/pull/295 +[303]: https://github.com/rust-num/num-traits/pull/303 + +# Release 0.2.17 (2023-10-07) + +- [Fix a doc warning about custom classes with newer rustdoc.][286] + +**Contributors**: @robamu + +[286]: https://github.com/rust-num/num-traits/pull/286 + +# Release 0.2.16 (2023-07-20) + +- [Upgrade to 2018 edition, **MSRV 1.31**][240] +- [The new `ToBytes` and `FromBytes` traits][224] convert to and from byte + representations of a value, with little, big, and native-endian options. +- [The new `Float::is_subnormal` method checks for subnormal values][279], with + a non-zero magnitude that is less than the normal minimum positive value. +- Several other improvements to documentation and testing. + +**Contributors**: @ctrlcctrlv, @cuviper, @flier, @GuillaumeGomez, @kaidokert, +@rs017991, @vicsn + +[224]: https://github.com/rust-num/num-traits/pull/224 +[240]: https://github.com/rust-num/num-traits/pull/240 +[279]: https://github.com/rust-num/num-traits/pull/279 + +# Release 0.2.15 (2022-05-02) + +- [The new `Euclid` trait calculates Euclidean division][195], where the + remainder is always positive or zero. +- [The new `LowerBounded` and `UpperBounded` traits][210] separately describe + types with lower and upper bounds. These traits are automatically implemented + for all fully-`Bounded` types. +- [The new `Float::copysign` method copies the sign of the argument][207] to + to the magnitude of `self`. +- [The new `PrimInt::leading_ones` and `trailing_ones` methods][205] are the + complement of the existing methods that count zero bits. +- [The new `PrimInt::reverse_bits` method reverses the order of all bits][202] + of a primitive integer. +- [Improved `Num::from_str_radix` for floats][201], also [ignoring case][214]. +- [`Float` and `FloatCore` use more from `libm`][196] when that is enabled. + +**Contributors**: @alion02, @clarfonthey, @cuviper, @ElectronicRU, +@ibraheemdev, @SparrowLii, @sshilovsky, @tspiteri, @XAMPPRocky, @Xiretza + +[195]: https://github.com/rust-num/num-traits/pull/195 +[196]: https://github.com/rust-num/num-traits/pull/196 +[201]: https://github.com/rust-num/num-traits/pull/201 +[202]: https://github.com/rust-num/num-traits/pull/202 +[205]: https://github.com/rust-num/num-traits/pull/205 +[207]: https://github.com/rust-num/num-traits/pull/207 +[210]: https://github.com/rust-num/num-traits/pull/210 +[214]: https://github.com/rust-num/num-traits/pull/214 + +# Release 0.2.14 (2020-10-29) + +- Clarify the license specification as "MIT OR Apache-2.0". + +**Contributors**: @cuviper + +# Release 0.2.13 (2020-10-29) + +- [The new `OverflowingAdd`, `OverflowingSub`, and `OverflowingMul` traits][180] + return a tuple with the operation result and a `bool` indicating overflow. +- [The "i128" feature now overrides compiler probes for that support][185]. + This may fix scenarios where `autocfg` probing doesn't work properly. +- [Casts from large `f64` values to `f32` now saturate to infinity][186]. They + previously returned `None` because that was once thought to be undefined + behavior, but [rust#15536] resolved that such casts are fine. +- [`Num::from_str_radix` documents requirements for radix support][192], which + are now more relaxed than previously implied. It is suggested to accept at + least `2..=36` without panicking, but `Err` may be returned otherwise. + +**Contributors**: @cuviper, @Enet4, @KaczuH, @martin-t, @newpavlov + +[180]: https://github.com/rust-num/num-traits/pull/180 +[185]: https://github.com/rust-num/num-traits/pull/185 +[186]: https://github.com/rust-num/num-traits/pull/186 +[192]: https://github.com/rust-num/num-traits/issues/192 +[rust#15536]: https://github.com/rust-lang/rust/issues/15536 + +# Release 0.2.12 (2020-06-11) + +- [The new `WrappingNeg` trait][153] will wrap the result if it exceeds the + boundary of the type, e.g. `i32::MIN.wrapping_neg() == i32::MIN`. +- [The new `SaturatingAdd`, `SaturatingSub`, and `SaturatingMul` traits][165] + will saturate at the numeric bounds if the operation would overflow. These + soft-deprecate the existing `Saturating` trait that only has addition and + subtraction methods. +- [Added new constants for logarithms, `FloatConst::{LOG10_2, LOG2_10}`][171]. + +**Contributors**: @cuviper, @ocstl, @trepetti, @vallentin + +[153]: https://github.com/rust-num/num-traits/pull/153 +[165]: https://github.com/rust-num/num-traits/pull/165 +[171]: https://github.com/rust-num/num-traits/pull/171 + +# Release 0.2.11 (2020-01-09) + +- [Added the full circle constant τ as `FloatConst::TAU`][145]. +- [Updated the `autocfg` build dependency to 1.0][148]. + +**Contributors**: @cuviper, @m-ou-se + +[145]: https://github.com/rust-num/num-traits/pull/145 +[148]: https://github.com/rust-num/num-traits/pull/148 + +# Release 0.2.10 (2019-11-22) + +- [Updated the `libm` dependency to 0.2][144]. + +**Contributors**: @CryZe + +[144]: https://github.com/rust-num/num-traits/pull/144 + +# Release 0.2.9 (2019-11-12) + +- [A new optional `libm` dependency][99] enables the `Float` and `Real` traits + in `no_std` builds. +- [The new `clamp_min` and `clamp_max`][122] limit minimum and maximum values + while preserving input `NAN`s. +- [Fixed a panic in floating point `from_str_radix` on invalid signs][126]. +- Miscellaneous documentation updates. + +**Contributors**: @cuviper, @dingelish, @HeroicKatora, @jturner314, @ocstl, +@Shnatsel, @termoshtt, @waywardmonkeys, @yoanlcq + +[99]: https://github.com/rust-num/num-traits/pull/99 +[122]: https://github.com/rust-num/num-traits/pull/122 +[126]: https://github.com/rust-num/num-traits/pull/126 + +# Release 0.2.8 (2019-05-21) + +- [Fixed feature detection on `no_std` targets][116]. + +**Contributors**: @cuviper + +[116]: https://github.com/rust-num/num-traits/pull/116 + +# Release 0.2.7 (2019-05-20) + +- [Documented when `CheckedShl` and `CheckedShr` return `None`][90]. +- [The new `Zero::set_zero` and `One::set_one`][104] will set values to their + identities in place, possibly optimized better than direct assignment. +- [Documented general features and intentions of `PrimInt`][108]. + +**Contributors**: @cuviper, @dvdhrm, @ignatenkobrain, @lcnr, @samueltardieu + +[90]: https://github.com/rust-num/num-traits/pull/90 +[104]: https://github.com/rust-num/num-traits/pull/104 +[108]: https://github.com/rust-num/num-traits/pull/108 + +# Release 0.2.6 (2018-09-13) + +- [Documented that `pow(0, 0)` returns `1`][79]. Mathematically, this is not + strictly defined, but the current behavior is a pragmatic choice that has + precedent in Rust `core` for the primitives and in many other languages. +- [The new `WrappingShl` and `WrappingShr` traits][81] will wrap the shift count + if it exceeds the bit size of the type. + +**Contributors**: @cuviper, @edmccard, @meltinglava + +[79]: https://github.com/rust-num/num-traits/pull/79 +[81]: https://github.com/rust-num/num-traits/pull/81 + +# Release 0.2.5 (2018-06-20) + +- [Documentation for `mul_add` now clarifies that it's not always faster.][70] +- [The default methods in `FromPrimitive` and `ToPrimitive` are more robust.][73] + +**Contributors**: @cuviper, @frewsxcv + +[70]: https://github.com/rust-num/num-traits/pull/70 +[73]: https://github.com/rust-num/num-traits/pull/73 + +# Release 0.2.4 (2018-05-11) + +- [Support for 128-bit integers is now automatically detected and enabled.][69] + Setting the `i128` crate feature now causes the build script to panic if such + support is not detected. + +**Contributors**: @cuviper + +[69]: https://github.com/rust-num/num-traits/pull/69 + +# Release 0.2.3 (2018-05-10) + +- [The new `CheckedNeg` and `CheckedRem` traits][63] perform checked `Neg` and + `Rem`, returning `Some(output)` or `None` on overflow. +- [The `no_std` implementation of `FloatCore::to_degrees` for `f32`][61] now + uses a constant for greater accuracy, mirroring [rust#47919]. (With `std` it + just calls the inherent `f32::to_degrees` in the standard library.) +- [The new `MulAdd` and `MulAddAssign` traits][59] perform a fused multiply- + add. For integer types this is just a convenience, but for floating point + types this produces a more accurate result than the separate operations. +- [All applicable traits are now implemented for 128-bit integers][60] starting + with Rust 1.26, enabled by the new `i128` crate feature. The `FromPrimitive` + and `ToPrimitive` traits now also have corresponding 128-bit methods, which + default to converting via 64-bit integers for compatibility. + +**Contributors**: @cuviper, @LEXUGE, @regexident, @vks + +[59]: https://github.com/rust-num/num-traits/pull/59 +[60]: https://github.com/rust-num/num-traits/pull/60 +[61]: https://github.com/rust-num/num-traits/pull/61 +[63]: https://github.com/rust-num/num-traits/pull/63 +[rust#47919]: https://github.com/rust-lang/rust/pull/47919 + +# Release 0.2.2 (2018-03-18) + +- [Casting from floating point to integers now returns `None` on overflow][52], + avoiding [rustc's undefined behavior][rust-10184]. This applies to the `cast` + function and the traits `NumCast`, `FromPrimitive`, and `ToPrimitive`. + +**Contributors**: @apopiak, @cuviper, @dbarella + +[52]: https://github.com/rust-num/num-traits/pull/52 +[rust-10184]: https://github.com/rust-lang/rust/issues/10184 + + +# Release 0.2.1 (2018-03-01) + +- [The new `FloatCore` trait][32] offers a subset of `Float` for `#![no_std]` use. + [This includes everything][41] except the transcendental functions and FMA. +- [The new `Inv` trait][37] returns the multiplicative inverse, or reciprocal. +- [The new `Pow` trait][37] performs exponentiation, much like the existing `pow` + function, but with generic exponent types. +- [The new `One::is_one` method][39] tests if a value equals 1. Implementers + should override this method if there's a more efficient way to check for 1, + rather than comparing with a temporary `one()`. + +**Contributors**: @clarcharr, @cuviper, @vks + +[32]: https://github.com/rust-num/num-traits/pull/32 +[37]: https://github.com/rust-num/num-traits/pull/37 +[39]: https://github.com/rust-num/num-traits/pull/39 +[41]: https://github.com/rust-num/num-traits/pull/41 + + +# Release 0.2.0 (2018-02-06) + +- **breaking change**: [There is now a `std` feature][30], enabled by default, along + with the implication that building *without* this feature makes this a + `#![no_std]` crate. + - The `Float` and `Real` traits are only available when `std` is enabled. + - Otherwise, the API is unchanged, and num-traits 0.1.43 now re-exports its + items from num-traits 0.2 for compatibility (the [semver-trick]). + +**Contributors**: @cuviper, @termoshtt, @vks + +[semver-trick]: https://github.com/dtolnay/semver-trick +[30]: https://github.com/rust-num/num-traits/pull/30 + + +# Release 0.1.43 (2018-02-06) + +- All items are now [re-exported from num-traits 0.2][31] for compatibility. + +[31]: https://github.com/rust-num/num-traits/pull/31 + + +# Release 0.1.42 (2018-01-22) + +- [num-traits now has its own source repository][num-356] at [rust-num/num-traits][home]. +- [`ParseFloatError` now implements `Display`][22]. +- [The new `AsPrimitive` trait][17] implements generic casting with the `as` operator. +- [The new `CheckedShl` and `CheckedShr` traits][21] implement generic + support for the `checked_shl` and `checked_shr` methods on primitive integers. +- [The new `Real` trait][23] offers a subset of `Float` functionality that may be applicable to more + types, with a blanket implementation for all existing `T: Float` types. + +Thanks to @cuviper, @Enet4, @fabianschuiki, @svartalf, and @yoanlcq for their contributions! + +[home]: https://github.com/rust-num/num-traits +[num-356]: https://github.com/rust-num/num/pull/356 +[17]: https://github.com/rust-num/num-traits/pull/17 +[21]: https://github.com/rust-num/num-traits/pull/21 +[22]: https://github.com/rust-num/num-traits/pull/22 +[23]: https://github.com/rust-num/num-traits/pull/23 + + +# Prior releases + +No prior release notes were kept. Thanks all the same to the many +contributors that have made this crate what it is! diff --git a/third_party/crates/num-traits-0.2.19/build.rs b/third_party/crates/num-traits-0.2.19/build.rs new file mode 100644 index 0000000..98b06be --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/build.rs @@ -0,0 +1,7 @@ +fn main() { + let ac = autocfg::new(); + + ac.emit_expression_cfg("1f64.total_cmp(&2f64)", "has_total_cmp"); // 1.62 + + autocfg::rerun_path("build.rs"); +} diff --git a/third_party/crates/num-traits-0.2.19/src/bounds.rs b/third_party/crates/num-traits-0.2.19/src/bounds.rs new file mode 100644 index 0000000..acc990e --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/bounds.rs @@ -0,0 +1,148 @@ +use core::num::Wrapping; +use core::{f32, f64}; +use core::{i128, i16, i32, i64, i8, isize}; +use core::{u128, u16, u32, u64, u8, usize}; + +/// Numbers which have upper and lower bounds +pub trait Bounded { + // FIXME (#5527): These should be associated constants + /// Returns the smallest finite number this type can represent + fn min_value() -> Self; + /// Returns the largest finite number this type can represent + fn max_value() -> Self; +} + +/// Numbers which have lower bounds +pub trait LowerBounded { + /// Returns the smallest finite number this type can represent + fn min_value() -> Self; +} + +// FIXME: With a major version bump, this should be a supertrait instead +impl LowerBounded for T { + fn min_value() -> T { + Bounded::min_value() + } +} + +/// Numbers which have upper bounds +pub trait UpperBounded { + /// Returns the largest finite number this type can represent + fn max_value() -> Self; +} + +// FIXME: With a major version bump, this should be a supertrait instead +impl UpperBounded for T { + fn max_value() -> T { + Bounded::max_value() + } +} + +macro_rules! bounded_impl { + ($t:ty, $min:expr, $max:expr) => { + impl Bounded for $t { + #[inline] + fn min_value() -> $t { + $min + } + + #[inline] + fn max_value() -> $t { + $max + } + } + }; +} + +bounded_impl!(usize, usize::MIN, usize::MAX); +bounded_impl!(u8, u8::MIN, u8::MAX); +bounded_impl!(u16, u16::MIN, u16::MAX); +bounded_impl!(u32, u32::MIN, u32::MAX); +bounded_impl!(u64, u64::MIN, u64::MAX); +bounded_impl!(u128, u128::MIN, u128::MAX); + +bounded_impl!(isize, isize::MIN, isize::MAX); +bounded_impl!(i8, i8::MIN, i8::MAX); +bounded_impl!(i16, i16::MIN, i16::MAX); +bounded_impl!(i32, i32::MIN, i32::MAX); +bounded_impl!(i64, i64::MIN, i64::MAX); +bounded_impl!(i128, i128::MIN, i128::MAX); + +impl Bounded for Wrapping { + fn min_value() -> Self { + Wrapping(T::min_value()) + } + fn max_value() -> Self { + Wrapping(T::max_value()) + } +} + +bounded_impl!(f32, f32::MIN, f32::MAX); + +macro_rules! for_each_tuple_ { + ( $m:ident !! ) => ( + $m! { } + ); + ( $m:ident !! $h:ident, $($t:ident,)* ) => ( + $m! { $h $($t)* } + for_each_tuple_! { $m !! $($t,)* } + ); +} +macro_rules! for_each_tuple { + ($m:ident) => { + for_each_tuple_! { $m !! A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, } + }; +} + +macro_rules! bounded_tuple { + ( $($name:ident)* ) => ( + impl<$($name: Bounded,)*> Bounded for ($($name,)*) { + #[inline] + fn min_value() -> Self { + ($($name::min_value(),)*) + } + #[inline] + fn max_value() -> Self { + ($($name::max_value(),)*) + } + } + ); +} + +for_each_tuple!(bounded_tuple); +bounded_impl!(f64, f64::MIN, f64::MAX); + +#[test] +fn wrapping_bounded() { + macro_rules! test_wrapping_bounded { + ($($t:ty)+) => { + $( + assert_eq!( as Bounded>::min_value().0, <$t>::min_value()); + assert_eq!( as Bounded>::max_value().0, <$t>::max_value()); + )+ + }; + } + + test_wrapping_bounded!(usize u8 u16 u32 u64 isize i8 i16 i32 i64); +} + +#[test] +fn wrapping_bounded_i128() { + macro_rules! test_wrapping_bounded { + ($($t:ty)+) => { + $( + assert_eq!( as Bounded>::min_value().0, <$t>::min_value()); + assert_eq!( as Bounded>::max_value().0, <$t>::max_value()); + )+ + }; + } + + test_wrapping_bounded!(u128 i128); +} + +#[test] +fn wrapping_is_bounded() { + fn require_bounded(_: &T) {} + require_bounded(&Wrapping(42_u32)); + require_bounded(&Wrapping(-42)); +} diff --git a/third_party/crates/num-traits-0.2.19/src/cast.rs b/third_party/crates/num-traits-0.2.19/src/cast.rs new file mode 100644 index 0000000..a3e4dd8 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/cast.rs @@ -0,0 +1,770 @@ +use core::mem::size_of; +use core::num::Wrapping; +use core::{f32, f64}; +use core::{i128, i16, i32, i64, i8, isize}; +use core::{u128, u16, u32, u64, u8, usize}; + +/// A generic trait for converting a value to a number. +/// +/// A value can be represented by the target type when it lies within +/// the range of scalars supported by the target type. +/// For example, a negative integer cannot be represented by an unsigned +/// integer type, and an `i64` with a very high magnitude might not be +/// convertible to an `i32`. +/// On the other hand, conversions with possible precision loss or truncation +/// are admitted, like an `f32` with a decimal part to an integer type, or +/// even a large `f64` saturating to `f32` infinity. +pub trait ToPrimitive { + /// Converts the value of `self` to an `isize`. If the value cannot be + /// represented by an `isize`, then `None` is returned. + #[inline] + fn to_isize(&self) -> Option { + self.to_i64().as_ref().and_then(ToPrimitive::to_isize) + } + + /// Converts the value of `self` to an `i8`. If the value cannot be + /// represented by an `i8`, then `None` is returned. + #[inline] + fn to_i8(&self) -> Option { + self.to_i64().as_ref().and_then(ToPrimitive::to_i8) + } + + /// Converts the value of `self` to an `i16`. If the value cannot be + /// represented by an `i16`, then `None` is returned. + #[inline] + fn to_i16(&self) -> Option { + self.to_i64().as_ref().and_then(ToPrimitive::to_i16) + } + + /// Converts the value of `self` to an `i32`. If the value cannot be + /// represented by an `i32`, then `None` is returned. + #[inline] + fn to_i32(&self) -> Option { + self.to_i64().as_ref().and_then(ToPrimitive::to_i32) + } + + /// Converts the value of `self` to an `i64`. If the value cannot be + /// represented by an `i64`, then `None` is returned. + fn to_i64(&self) -> Option; + + /// Converts the value of `self` to an `i128`. If the value cannot be + /// represented by an `i128` (`i64` under the default implementation), then + /// `None` is returned. + /// + /// The default implementation converts through `to_i64()`. Types implementing + /// this trait should override this method if they can represent a greater range. + #[inline] + fn to_i128(&self) -> Option { + self.to_i64().map(From::from) + } + + /// Converts the value of `self` to a `usize`. If the value cannot be + /// represented by a `usize`, then `None` is returned. + #[inline] + fn to_usize(&self) -> Option { + self.to_u64().as_ref().and_then(ToPrimitive::to_usize) + } + + /// Converts the value of `self` to a `u8`. If the value cannot be + /// represented by a `u8`, then `None` is returned. + #[inline] + fn to_u8(&self) -> Option { + self.to_u64().as_ref().and_then(ToPrimitive::to_u8) + } + + /// Converts the value of `self` to a `u16`. If the value cannot be + /// represented by a `u16`, then `None` is returned. + #[inline] + fn to_u16(&self) -> Option { + self.to_u64().as_ref().and_then(ToPrimitive::to_u16) + } + + /// Converts the value of `self` to a `u32`. If the value cannot be + /// represented by a `u32`, then `None` is returned. + #[inline] + fn to_u32(&self) -> Option { + self.to_u64().as_ref().and_then(ToPrimitive::to_u32) + } + + /// Converts the value of `self` to a `u64`. If the value cannot be + /// represented by a `u64`, then `None` is returned. + fn to_u64(&self) -> Option; + + /// Converts the value of `self` to a `u128`. If the value cannot be + /// represented by a `u128` (`u64` under the default implementation), then + /// `None` is returned. + /// + /// The default implementation converts through `to_u64()`. Types implementing + /// this trait should override this method if they can represent a greater range. + #[inline] + fn to_u128(&self) -> Option { + self.to_u64().map(From::from) + } + + /// Converts the value of `self` to an `f32`. Overflows may map to positive + /// or negative inifinity, otherwise `None` is returned if the value cannot + /// be represented by an `f32`. + #[inline] + fn to_f32(&self) -> Option { + self.to_f64().as_ref().and_then(ToPrimitive::to_f32) + } + + /// Converts the value of `self` to an `f64`. Overflows may map to positive + /// or negative inifinity, otherwise `None` is returned if the value cannot + /// be represented by an `f64`. + /// + /// The default implementation tries to convert through `to_i64()`, and + /// failing that through `to_u64()`. Types implementing this trait should + /// override this method if they can represent a greater range. + #[inline] + fn to_f64(&self) -> Option { + match self.to_i64() { + Some(i) => i.to_f64(), + None => self.to_u64().as_ref().and_then(ToPrimitive::to_f64), + } + } +} + +macro_rules! impl_to_primitive_int_to_int { + ($SrcT:ident : $( $(#[$cfg:meta])* fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> Option<$DstT> { + let min = $DstT::MIN as $SrcT; + let max = $DstT::MAX as $SrcT; + if size_of::<$SrcT>() <= size_of::<$DstT>() || (min <= *self && *self <= max) { + Some(*self as $DstT) + } else { + None + } + } + )*} +} + +macro_rules! impl_to_primitive_int_to_uint { + ($SrcT:ident : $( $(#[$cfg:meta])* fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> Option<$DstT> { + let max = $DstT::MAX as $SrcT; + if 0 <= *self && (size_of::<$SrcT>() <= size_of::<$DstT>() || *self <= max) { + Some(*self as $DstT) + } else { + None + } + } + )*} +} + +macro_rules! impl_to_primitive_int { + ($T:ident) => { + impl ToPrimitive for $T { + impl_to_primitive_int_to_int! { $T: + fn to_isize -> isize; + fn to_i8 -> i8; + fn to_i16 -> i16; + fn to_i32 -> i32; + fn to_i64 -> i64; + fn to_i128 -> i128; + } + + impl_to_primitive_int_to_uint! { $T: + fn to_usize -> usize; + fn to_u8 -> u8; + fn to_u16 -> u16; + fn to_u32 -> u32; + fn to_u64 -> u64; + fn to_u128 -> u128; + } + + #[inline] + fn to_f32(&self) -> Option { + Some(*self as f32) + } + #[inline] + fn to_f64(&self) -> Option { + Some(*self as f64) + } + } + }; +} + +impl_to_primitive_int!(isize); +impl_to_primitive_int!(i8); +impl_to_primitive_int!(i16); +impl_to_primitive_int!(i32); +impl_to_primitive_int!(i64); +impl_to_primitive_int!(i128); + +macro_rules! impl_to_primitive_uint_to_int { + ($SrcT:ident : $( $(#[$cfg:meta])* fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> Option<$DstT> { + let max = $DstT::MAX as $SrcT; + if size_of::<$SrcT>() < size_of::<$DstT>() || *self <= max { + Some(*self as $DstT) + } else { + None + } + } + )*} +} + +macro_rules! impl_to_primitive_uint_to_uint { + ($SrcT:ident : $( $(#[$cfg:meta])* fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> Option<$DstT> { + let max = $DstT::MAX as $SrcT; + if size_of::<$SrcT>() <= size_of::<$DstT>() || *self <= max { + Some(*self as $DstT) + } else { + None + } + } + )*} +} + +macro_rules! impl_to_primitive_uint { + ($T:ident) => { + impl ToPrimitive for $T { + impl_to_primitive_uint_to_int! { $T: + fn to_isize -> isize; + fn to_i8 -> i8; + fn to_i16 -> i16; + fn to_i32 -> i32; + fn to_i64 -> i64; + fn to_i128 -> i128; + } + + impl_to_primitive_uint_to_uint! { $T: + fn to_usize -> usize; + fn to_u8 -> u8; + fn to_u16 -> u16; + fn to_u32 -> u32; + fn to_u64 -> u64; + fn to_u128 -> u128; + } + + #[inline] + fn to_f32(&self) -> Option { + Some(*self as f32) + } + #[inline] + fn to_f64(&self) -> Option { + Some(*self as f64) + } + } + }; +} + +impl_to_primitive_uint!(usize); +impl_to_primitive_uint!(u8); +impl_to_primitive_uint!(u16); +impl_to_primitive_uint!(u32); +impl_to_primitive_uint!(u64); +impl_to_primitive_uint!(u128); + +macro_rules! impl_to_primitive_float_to_float { + ($SrcT:ident : $( fn $method:ident -> $DstT:ident ; )*) => {$( + #[inline] + fn $method(&self) -> Option<$DstT> { + // We can safely cast all values, whether NaN, +-inf, or finite. + // Finite values that are reducing size may saturate to +-inf. + Some(*self as $DstT) + } + )*} +} + +macro_rules! float_to_int_unchecked { + // SAFETY: Must not be NaN or infinite; must be representable as the integer after truncating. + // We already checked that the float is in the exclusive range `(MIN-1, MAX+1)`. + ($float:expr => $int:ty) => { + unsafe { $float.to_int_unchecked::<$int>() } + }; +} + +macro_rules! impl_to_primitive_float_to_signed_int { + ($f:ident : $( $(#[$cfg:meta])* fn $method:ident -> $i:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> Option<$i> { + // Float as int truncates toward zero, so we want to allow values + // in the exclusive range `(MIN-1, MAX+1)`. + if size_of::<$f>() > size_of::<$i>() { + // With a larger size, we can represent the range exactly. + const MIN_M1: $f = $i::MIN as $f - 1.0; + const MAX_P1: $f = $i::MAX as $f + 1.0; + if *self > MIN_M1 && *self < MAX_P1 { + return Some(float_to_int_unchecked!(*self => $i)); + } + } else { + // We can't represent `MIN-1` exactly, but there's no fractional part + // at this magnitude, so we can just use a `MIN` inclusive boundary. + const MIN: $f = $i::MIN as $f; + // We can't represent `MAX` exactly, but it will round up to exactly + // `MAX+1` (a power of two) when we cast it. + const MAX_P1: $f = $i::MAX as $f; + if *self >= MIN && *self < MAX_P1 { + return Some(float_to_int_unchecked!(*self => $i)); + } + } + None + } + )*} +} + +macro_rules! impl_to_primitive_float_to_unsigned_int { + ($f:ident : $( $(#[$cfg:meta])* fn $method:ident -> $u:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> Option<$u> { + // Float as int truncates toward zero, so we want to allow values + // in the exclusive range `(-1, MAX+1)`. + if size_of::<$f>() > size_of::<$u>() { + // With a larger size, we can represent the range exactly. + const MAX_P1: $f = $u::MAX as $f + 1.0; + if *self > -1.0 && *self < MAX_P1 { + return Some(float_to_int_unchecked!(*self => $u)); + } + } else { + // We can't represent `MAX` exactly, but it will round up to exactly + // `MAX+1` (a power of two) when we cast it. + // (`u128::MAX as f32` is infinity, but this is still ok.) + const MAX_P1: $f = $u::MAX as $f; + if *self > -1.0 && *self < MAX_P1 { + return Some(float_to_int_unchecked!(*self => $u)); + } + } + None + } + )*} +} + +macro_rules! impl_to_primitive_float { + ($T:ident) => { + impl ToPrimitive for $T { + impl_to_primitive_float_to_signed_int! { $T: + fn to_isize -> isize; + fn to_i8 -> i8; + fn to_i16 -> i16; + fn to_i32 -> i32; + fn to_i64 -> i64; + fn to_i128 -> i128; + } + + impl_to_primitive_float_to_unsigned_int! { $T: + fn to_usize -> usize; + fn to_u8 -> u8; + fn to_u16 -> u16; + fn to_u32 -> u32; + fn to_u64 -> u64; + fn to_u128 -> u128; + } + + impl_to_primitive_float_to_float! { $T: + fn to_f32 -> f32; + fn to_f64 -> f64; + } + } + }; +} + +impl_to_primitive_float!(f32); +impl_to_primitive_float!(f64); + +/// A generic trait for converting a number to a value. +/// +/// A value can be represented by the target type when it lies within +/// the range of scalars supported by the target type. +/// For example, a negative integer cannot be represented by an unsigned +/// integer type, and an `i64` with a very high magnitude might not be +/// convertible to an `i32`. +/// On the other hand, conversions with possible precision loss or truncation +/// are admitted, like an `f32` with a decimal part to an integer type, or +/// even a large `f64` saturating to `f32` infinity. +pub trait FromPrimitive: Sized { + /// Converts an `isize` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_isize(n: isize) -> Option { + n.to_i64().and_then(FromPrimitive::from_i64) + } + + /// Converts an `i8` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_i8(n: i8) -> Option { + FromPrimitive::from_i64(From::from(n)) + } + + /// Converts an `i16` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_i16(n: i16) -> Option { + FromPrimitive::from_i64(From::from(n)) + } + + /// Converts an `i32` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_i32(n: i32) -> Option { + FromPrimitive::from_i64(From::from(n)) + } + + /// Converts an `i64` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + fn from_i64(n: i64) -> Option; + + /// Converts an `i128` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + /// + /// The default implementation converts through `from_i64()`. Types implementing + /// this trait should override this method if they can represent a greater range. + #[inline] + fn from_i128(n: i128) -> Option { + n.to_i64().and_then(FromPrimitive::from_i64) + } + + /// Converts a `usize` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_usize(n: usize) -> Option { + n.to_u64().and_then(FromPrimitive::from_u64) + } + + /// Converts an `u8` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_u8(n: u8) -> Option { + FromPrimitive::from_u64(From::from(n)) + } + + /// Converts an `u16` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_u16(n: u16) -> Option { + FromPrimitive::from_u64(From::from(n)) + } + + /// Converts an `u32` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_u32(n: u32) -> Option { + FromPrimitive::from_u64(From::from(n)) + } + + /// Converts an `u64` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + fn from_u64(n: u64) -> Option; + + /// Converts an `u128` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + /// + /// The default implementation converts through `from_u64()`. Types implementing + /// this trait should override this method if they can represent a greater range. + #[inline] + fn from_u128(n: u128) -> Option { + n.to_u64().and_then(FromPrimitive::from_u64) + } + + /// Converts a `f32` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + #[inline] + fn from_f32(n: f32) -> Option { + FromPrimitive::from_f64(From::from(n)) + } + + /// Converts a `f64` to return an optional value of this type. If the + /// value cannot be represented by this type, then `None` is returned. + /// + /// The default implementation tries to convert through `from_i64()`, and + /// failing that through `from_u64()`. Types implementing this trait should + /// override this method if they can represent a greater range. + #[inline] + fn from_f64(n: f64) -> Option { + match n.to_i64() { + Some(i) => FromPrimitive::from_i64(i), + None => n.to_u64().and_then(FromPrimitive::from_u64), + } + } +} + +macro_rules! impl_from_primitive { + ($T:ty, $to_ty:ident) => { + #[allow(deprecated)] + impl FromPrimitive for $T { + #[inline] + fn from_isize(n: isize) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_i8(n: i8) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_i16(n: i16) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_i32(n: i32) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_i64(n: i64) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_i128(n: i128) -> Option<$T> { + n.$to_ty() + } + + #[inline] + fn from_usize(n: usize) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_u8(n: u8) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_u16(n: u16) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_u32(n: u32) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_u64(n: u64) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_u128(n: u128) -> Option<$T> { + n.$to_ty() + } + + #[inline] + fn from_f32(n: f32) -> Option<$T> { + n.$to_ty() + } + #[inline] + fn from_f64(n: f64) -> Option<$T> { + n.$to_ty() + } + } + }; +} + +impl_from_primitive!(isize, to_isize); +impl_from_primitive!(i8, to_i8); +impl_from_primitive!(i16, to_i16); +impl_from_primitive!(i32, to_i32); +impl_from_primitive!(i64, to_i64); +impl_from_primitive!(i128, to_i128); +impl_from_primitive!(usize, to_usize); +impl_from_primitive!(u8, to_u8); +impl_from_primitive!(u16, to_u16); +impl_from_primitive!(u32, to_u32); +impl_from_primitive!(u64, to_u64); +impl_from_primitive!(u128, to_u128); +impl_from_primitive!(f32, to_f32); +impl_from_primitive!(f64, to_f64); + +macro_rules! impl_to_primitive_wrapping { + ($( $(#[$cfg:meta])* fn $method:ident -> $i:ident ; )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(&self) -> Option<$i> { + (self.0).$method() + } + )*} +} + +impl ToPrimitive for Wrapping { + impl_to_primitive_wrapping! { + fn to_isize -> isize; + fn to_i8 -> i8; + fn to_i16 -> i16; + fn to_i32 -> i32; + fn to_i64 -> i64; + fn to_i128 -> i128; + + fn to_usize -> usize; + fn to_u8 -> u8; + fn to_u16 -> u16; + fn to_u32 -> u32; + fn to_u64 -> u64; + fn to_u128 -> u128; + + fn to_f32 -> f32; + fn to_f64 -> f64; + } +} + +macro_rules! impl_from_primitive_wrapping { + ($( $(#[$cfg:meta])* fn $method:ident ( $i:ident ); )*) => {$( + #[inline] + $(#[$cfg])* + fn $method(n: $i) -> Option { + T::$method(n).map(Wrapping) + } + )*} +} + +impl FromPrimitive for Wrapping { + impl_from_primitive_wrapping! { + fn from_isize(isize); + fn from_i8(i8); + fn from_i16(i16); + fn from_i32(i32); + fn from_i64(i64); + fn from_i128(i128); + + fn from_usize(usize); + fn from_u8(u8); + fn from_u16(u16); + fn from_u32(u32); + fn from_u64(u64); + fn from_u128(u128); + + fn from_f32(f32); + fn from_f64(f64); + } +} + +/// Cast from one machine scalar to another. +/// +/// # Examples +/// +/// ``` +/// # use num_traits as num; +/// let twenty: f32 = num::cast(0x14).unwrap(); +/// assert_eq!(twenty, 20f32); +/// ``` +/// +#[inline] +pub fn cast(n: T) -> Option { + NumCast::from(n) +} + +/// An interface for casting between machine scalars. +pub trait NumCast: Sized + ToPrimitive { + /// Creates a number from another value that can be converted into + /// a primitive via the `ToPrimitive` trait. If the source value cannot be + /// represented by the target type, then `None` is returned. + /// + /// A value can be represented by the target type when it lies within + /// the range of scalars supported by the target type. + /// For example, a negative integer cannot be represented by an unsigned + /// integer type, and an `i64` with a very high magnitude might not be + /// convertible to an `i32`. + /// On the other hand, conversions with possible precision loss or truncation + /// are admitted, like an `f32` with a decimal part to an integer type, or + /// even a large `f64` saturating to `f32` infinity. + fn from(n: T) -> Option; +} + +macro_rules! impl_num_cast { + ($T:ty, $conv:ident) => { + impl NumCast for $T { + #[inline] + #[allow(deprecated)] + fn from(n: N) -> Option<$T> { + // `$conv` could be generated using `concat_idents!`, but that + // macro seems to be broken at the moment + n.$conv() + } + } + }; +} + +impl_num_cast!(u8, to_u8); +impl_num_cast!(u16, to_u16); +impl_num_cast!(u32, to_u32); +impl_num_cast!(u64, to_u64); +impl_num_cast!(u128, to_u128); +impl_num_cast!(usize, to_usize); +impl_num_cast!(i8, to_i8); +impl_num_cast!(i16, to_i16); +impl_num_cast!(i32, to_i32); +impl_num_cast!(i64, to_i64); +impl_num_cast!(i128, to_i128); +impl_num_cast!(isize, to_isize); +impl_num_cast!(f32, to_f32); +impl_num_cast!(f64, to_f64); + +impl NumCast for Wrapping { + fn from(n: U) -> Option { + T::from(n).map(Wrapping) + } +} + +/// A generic interface for casting between machine scalars with the +/// `as` operator, which admits narrowing and precision loss. +/// Implementers of this trait `AsPrimitive` should behave like a primitive +/// numeric type (e.g. a newtype around another primitive), and the +/// intended conversion must never fail. +/// +/// # Examples +/// +/// ``` +/// # use num_traits::AsPrimitive; +/// let three: i32 = (3.14159265f32).as_(); +/// assert_eq!(three, 3); +/// ``` +/// +/// # Safety +/// +/// **In Rust versions before 1.45.0**, some uses of the `as` operator were not entirely safe. +/// In particular, it was undefined behavior if +/// a truncated floating point value could not fit in the target integer +/// type ([#10184](https://github.com/rust-lang/rust/issues/10184)). +/// +/// ```ignore +/// # use num_traits::AsPrimitive; +/// let x: u8 = (1.04E+17).as_(); // UB +/// ``` +/// +pub trait AsPrimitive: 'static + Copy +where + T: 'static + Copy, +{ + /// Convert a value to another, using the `as` operator. + fn as_(self) -> T; +} + +macro_rules! impl_as_primitive { + (@ $T: ty => $(#[$cfg:meta])* impl $U: ty ) => { + $(#[$cfg])* + impl AsPrimitive<$U> for $T { + #[inline] fn as_(self) -> $U { self as $U } + } + }; + (@ $T: ty => { $( $U: ty ),* } ) => {$( + impl_as_primitive!(@ $T => impl $U); + )*}; + ($T: ty => { $( $U: ty ),* } ) => { + impl_as_primitive!(@ $T => { $( $U ),* }); + impl_as_primitive!(@ $T => { u8, u16, u32, u64, u128, usize }); + impl_as_primitive!(@ $T => { i8, i16, i32, i64, i128, isize }); + }; +} + +impl_as_primitive!(u8 => { char, f32, f64 }); +impl_as_primitive!(i8 => { f32, f64 }); +impl_as_primitive!(u16 => { f32, f64 }); +impl_as_primitive!(i16 => { f32, f64 }); +impl_as_primitive!(u32 => { f32, f64 }); +impl_as_primitive!(i32 => { f32, f64 }); +impl_as_primitive!(u64 => { f32, f64 }); +impl_as_primitive!(i64 => { f32, f64 }); +impl_as_primitive!(u128 => { f32, f64 }); +impl_as_primitive!(i128 => { f32, f64 }); +impl_as_primitive!(usize => { f32, f64 }); +impl_as_primitive!(isize => { f32, f64 }); +impl_as_primitive!(f32 => { f32, f64 }); +impl_as_primitive!(f64 => { f32, f64 }); +impl_as_primitive!(char => { char }); +impl_as_primitive!(bool => {}); diff --git a/third_party/crates/num-traits-0.2.19/src/float.rs b/third_party/crates/num-traits-0.2.19/src/float.rs new file mode 100644 index 0000000..d0d21a5 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/float.rs @@ -0,0 +1,2513 @@ +use core::cmp::Ordering; +use core::num::FpCategory; +use core::ops::{Add, Div, Neg}; + +use core::f32; +use core::f64; + +use crate::{Num, NumCast, ToPrimitive}; + +/// Generic trait for floating point numbers that works with `no_std`. +/// +/// This trait implements a subset of the `Float` trait. +pub trait FloatCore: Num + NumCast + Neg + PartialOrd + Copy { + /// Returns positive infinity. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T) { + /// assert!(T::infinity() == x); + /// } + /// + /// check(f32::INFINITY); + /// check(f64::INFINITY); + /// ``` + fn infinity() -> Self; + + /// Returns negative infinity. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T) { + /// assert!(T::neg_infinity() == x); + /// } + /// + /// check(f32::NEG_INFINITY); + /// check(f64::NEG_INFINITY); + /// ``` + fn neg_infinity() -> Self; + + /// Returns NaN. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// + /// fn check() { + /// let n = T::nan(); + /// assert!(n != n); + /// } + /// + /// check::(); + /// check::(); + /// ``` + fn nan() -> Self; + + /// Returns `-0.0`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(n: T) { + /// let z = T::neg_zero(); + /// assert!(z.is_zero()); + /// assert!(T::one() / z == n); + /// } + /// + /// check(f32::NEG_INFINITY); + /// check(f64::NEG_INFINITY); + /// ``` + fn neg_zero() -> Self; + + /// Returns the smallest finite value that this type can represent. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T) { + /// assert!(T::min_value() == x); + /// } + /// + /// check(f32::MIN); + /// check(f64::MIN); + /// ``` + fn min_value() -> Self; + + /// Returns the smallest positive, normalized value that this type can represent. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T) { + /// assert!(T::min_positive_value() == x); + /// } + /// + /// check(f32::MIN_POSITIVE); + /// check(f64::MIN_POSITIVE); + /// ``` + fn min_positive_value() -> Self; + + /// Returns epsilon, a small positive value. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T) { + /// assert!(T::epsilon() == x); + /// } + /// + /// check(f32::EPSILON); + /// check(f64::EPSILON); + /// ``` + fn epsilon() -> Self; + + /// Returns the largest finite value that this type can represent. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T) { + /// assert!(T::max_value() == x); + /// } + /// + /// check(f32::MAX); + /// check(f64::MAX); + /// ``` + fn max_value() -> Self; + + /// Returns `true` if the number is NaN. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, p: bool) { + /// assert!(x.is_nan() == p); + /// } + /// + /// check(f32::NAN, true); + /// check(f32::INFINITY, false); + /// check(f64::NAN, true); + /// check(0.0f64, false); + /// ``` + #[inline] + #[allow(clippy::eq_op)] + fn is_nan(self) -> bool { + self != self + } + + /// Returns `true` if the number is infinite. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, p: bool) { + /// assert!(x.is_infinite() == p); + /// } + /// + /// check(f32::INFINITY, true); + /// check(f32::NEG_INFINITY, true); + /// check(f32::NAN, false); + /// check(f64::INFINITY, true); + /// check(f64::NEG_INFINITY, true); + /// check(0.0f64, false); + /// ``` + #[inline] + fn is_infinite(self) -> bool { + self == Self::infinity() || self == Self::neg_infinity() + } + + /// Returns `true` if the number is neither infinite or NaN. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, p: bool) { + /// assert!(x.is_finite() == p); + /// } + /// + /// check(f32::INFINITY, false); + /// check(f32::MAX, true); + /// check(f64::NEG_INFINITY, false); + /// check(f64::MIN_POSITIVE, true); + /// check(f64::NAN, false); + /// ``` + #[inline] + fn is_finite(self) -> bool { + !(self.is_nan() || self.is_infinite()) + } + + /// Returns `true` if the number is neither zero, infinite, subnormal or NaN. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, p: bool) { + /// assert!(x.is_normal() == p); + /// } + /// + /// check(f32::INFINITY, false); + /// check(f32::MAX, true); + /// check(f64::NEG_INFINITY, false); + /// check(f64::MIN_POSITIVE, true); + /// check(0.0f64, false); + /// ``` + #[inline] + fn is_normal(self) -> bool { + self.classify() == FpCategory::Normal + } + + /// Returns `true` if the number is [subnormal]. + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::f64; + /// + /// let min = f64::MIN_POSITIVE; // 2.2250738585072014e-308_f64 + /// let max = f64::MAX; + /// let lower_than_min = 1.0e-308_f64; + /// let zero = 0.0_f64; + /// + /// assert!(!min.is_subnormal()); + /// assert!(!max.is_subnormal()); + /// + /// assert!(!zero.is_subnormal()); + /// assert!(!f64::NAN.is_subnormal()); + /// assert!(!f64::INFINITY.is_subnormal()); + /// // Values between `0` and `min` are Subnormal. + /// assert!(lower_than_min.is_subnormal()); + /// ``` + /// [subnormal]: https://en.wikipedia.org/wiki/Subnormal_number + #[inline] + fn is_subnormal(self) -> bool { + self.classify() == FpCategory::Subnormal + } + + /// Returns the floating point category of the number. If only one property + /// is going to be tested, it is generally faster to use the specific + /// predicate instead. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// use std::num::FpCategory; + /// + /// fn check(x: T, c: FpCategory) { + /// assert!(x.classify() == c); + /// } + /// + /// check(f32::INFINITY, FpCategory::Infinite); + /// check(f32::MAX, FpCategory::Normal); + /// check(f64::NAN, FpCategory::Nan); + /// check(f64::MIN_POSITIVE, FpCategory::Normal); + /// check(f64::MIN_POSITIVE / 2.0, FpCategory::Subnormal); + /// check(0.0f64, FpCategory::Zero); + /// ``` + fn classify(self) -> FpCategory; + + /// Returns the largest integer less than or equal to a number. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T) { + /// assert!(x.floor() == y); + /// } + /// + /// check(f32::INFINITY, f32::INFINITY); + /// check(0.9f32, 0.0); + /// check(1.0f32, 1.0); + /// check(1.1f32, 1.0); + /// check(-0.0f64, 0.0); + /// check(-0.9f64, -1.0); + /// check(-1.0f64, -1.0); + /// check(-1.1f64, -2.0); + /// check(f64::MIN, f64::MIN); + /// ``` + #[inline] + fn floor(self) -> Self { + let f = self.fract(); + if f.is_nan() || f.is_zero() { + self + } else if self < Self::zero() { + self - f - Self::one() + } else { + self - f + } + } + + /// Returns the smallest integer greater than or equal to a number. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T) { + /// assert!(x.ceil() == y); + /// } + /// + /// check(f32::INFINITY, f32::INFINITY); + /// check(0.9f32, 1.0); + /// check(1.0f32, 1.0); + /// check(1.1f32, 2.0); + /// check(-0.0f64, 0.0); + /// check(-0.9f64, -0.0); + /// check(-1.0f64, -1.0); + /// check(-1.1f64, -1.0); + /// check(f64::MIN, f64::MIN); + /// ``` + #[inline] + fn ceil(self) -> Self { + let f = self.fract(); + if f.is_nan() || f.is_zero() { + self + } else if self > Self::zero() { + self - f + Self::one() + } else { + self - f + } + } + + /// Returns the nearest integer to a number. Round half-way cases away from `0.0`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T) { + /// assert!(x.round() == y); + /// } + /// + /// check(f32::INFINITY, f32::INFINITY); + /// check(0.4f32, 0.0); + /// check(0.5f32, 1.0); + /// check(0.6f32, 1.0); + /// check(-0.4f64, 0.0); + /// check(-0.5f64, -1.0); + /// check(-0.6f64, -1.0); + /// check(f64::MIN, f64::MIN); + /// ``` + #[inline] + fn round(self) -> Self { + let one = Self::one(); + let h = Self::from(0.5).expect("Unable to cast from 0.5"); + let f = self.fract(); + if f.is_nan() || f.is_zero() { + self + } else if self > Self::zero() { + if f < h { + self - f + } else { + self - f + one + } + } else if -f < h { + self - f + } else { + self - f - one + } + } + + /// Return the integer part of a number. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T) { + /// assert!(x.trunc() == y); + /// } + /// + /// check(f32::INFINITY, f32::INFINITY); + /// check(0.9f32, 0.0); + /// check(1.0f32, 1.0); + /// check(1.1f32, 1.0); + /// check(-0.0f64, 0.0); + /// check(-0.9f64, -0.0); + /// check(-1.0f64, -1.0); + /// check(-1.1f64, -1.0); + /// check(f64::MIN, f64::MIN); + /// ``` + #[inline] + fn trunc(self) -> Self { + let f = self.fract(); + if f.is_nan() { + self + } else { + self - f + } + } + + /// Returns the fractional part of a number. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T) { + /// assert!(x.fract() == y); + /// } + /// + /// check(f32::MAX, 0.0); + /// check(0.75f32, 0.75); + /// check(1.0f32, 0.0); + /// check(1.25f32, 0.25); + /// check(-0.0f64, 0.0); + /// check(-0.75f64, -0.75); + /// check(-1.0f64, 0.0); + /// check(-1.25f64, -0.25); + /// check(f64::MIN, 0.0); + /// ``` + #[inline] + fn fract(self) -> Self { + if self.is_zero() { + Self::zero() + } else { + self % Self::one() + } + } + + /// Computes the absolute value of `self`. Returns `FloatCore::nan()` if the + /// number is `FloatCore::nan()`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T) { + /// assert!(x.abs() == y); + /// } + /// + /// check(f32::INFINITY, f32::INFINITY); + /// check(1.0f32, 1.0); + /// check(0.0f64, 0.0); + /// check(-0.0f64, 0.0); + /// check(-1.0f64, 1.0); + /// check(f64::MIN, f64::MAX); + /// ``` + #[inline] + fn abs(self) -> Self { + if self.is_sign_positive() { + return self; + } + if self.is_sign_negative() { + return -self; + } + Self::nan() + } + + /// Returns a number that represents the sign of `self`. + /// + /// - `1.0` if the number is positive, `+0.0` or `FloatCore::infinity()` + /// - `-1.0` if the number is negative, `-0.0` or `FloatCore::neg_infinity()` + /// - `FloatCore::nan()` if the number is `FloatCore::nan()` + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T) { + /// assert!(x.signum() == y); + /// } + /// + /// check(f32::INFINITY, 1.0); + /// check(3.0f32, 1.0); + /// check(0.0f32, 1.0); + /// check(-0.0f64, -1.0); + /// check(-3.0f64, -1.0); + /// check(f64::MIN, -1.0); + /// ``` + #[inline] + fn signum(self) -> Self { + if self.is_nan() { + Self::nan() + } else if self.is_sign_negative() { + -Self::one() + } else { + Self::one() + } + } + + /// Returns `true` if `self` is positive, including `+0.0` and + /// `FloatCore::infinity()`, and `FloatCore::nan()`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, p: bool) { + /// assert!(x.is_sign_positive() == p); + /// } + /// + /// check(f32::INFINITY, true); + /// check(f32::MAX, true); + /// check(0.0f32, true); + /// check(-0.0f64, false); + /// check(f64::NEG_INFINITY, false); + /// check(f64::MIN_POSITIVE, true); + /// check(f64::NAN, true); + /// check(-f64::NAN, false); + /// ``` + #[inline] + fn is_sign_positive(self) -> bool { + !self.is_sign_negative() + } + + /// Returns `true` if `self` is negative, including `-0.0` and + /// `FloatCore::neg_infinity()`, and `-FloatCore::nan()`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, p: bool) { + /// assert!(x.is_sign_negative() == p); + /// } + /// + /// check(f32::INFINITY, false); + /// check(f32::MAX, false); + /// check(0.0f32, false); + /// check(-0.0f64, true); + /// check(f64::NEG_INFINITY, true); + /// check(f64::MIN_POSITIVE, false); + /// check(f64::NAN, false); + /// check(-f64::NAN, true); + /// ``` + #[inline] + fn is_sign_negative(self) -> bool { + let (_, _, sign) = self.integer_decode(); + sign < 0 + } + + /// Returns the minimum of the two numbers. + /// + /// If one of the arguments is NaN, then the other argument is returned. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T, min: T) { + /// assert!(x.min(y) == min); + /// } + /// + /// check(1.0f32, 2.0, 1.0); + /// check(f32::NAN, 2.0, 2.0); + /// check(1.0f64, -2.0, -2.0); + /// check(1.0f64, f64::NAN, 1.0); + /// ``` + #[inline] + fn min(self, other: Self) -> Self { + if self.is_nan() { + return other; + } + if other.is_nan() { + return self; + } + if self < other { + self + } else { + other + } + } + + /// Returns the maximum of the two numbers. + /// + /// If one of the arguments is NaN, then the other argument is returned. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T, max: T) { + /// assert!(x.max(y) == max); + /// } + /// + /// check(1.0f32, 2.0, 2.0); + /// check(1.0f32, f32::NAN, 1.0); + /// check(-1.0f64, 2.0, 2.0); + /// check(-1.0f64, f64::NAN, -1.0); + /// ``` + #[inline] + fn max(self, other: Self) -> Self { + if self.is_nan() { + return other; + } + if other.is_nan() { + return self; + } + if self > other { + self + } else { + other + } + } + + /// A value bounded by a minimum and a maximum + /// + /// If input is less than min then this returns min. + /// If input is greater than max then this returns max. + /// Otherwise this returns input. + /// + /// **Panics** in debug mode if `!(min <= max)`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// + /// fn check(val: T, min: T, max: T, expected: T) { + /// assert!(val.clamp(min, max) == expected); + /// } + /// + /// + /// check(1.0f32, 0.0, 2.0, 1.0); + /// check(1.0f32, 2.0, 3.0, 2.0); + /// check(3.0f32, 0.0, 2.0, 2.0); + /// + /// check(1.0f64, 0.0, 2.0, 1.0); + /// check(1.0f64, 2.0, 3.0, 2.0); + /// check(3.0f64, 0.0, 2.0, 2.0); + /// ``` + fn clamp(self, min: Self, max: Self) -> Self { + crate::clamp(self, min, max) + } + + /// Returns the reciprocal (multiplicative inverse) of the number. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, y: T) { + /// assert!(x.recip() == y); + /// assert!(y.recip() == x); + /// } + /// + /// check(f32::INFINITY, 0.0); + /// check(2.0f32, 0.5); + /// check(-0.25f64, -4.0); + /// check(-0.0f64, f64::NEG_INFINITY); + /// ``` + #[inline] + fn recip(self) -> Self { + Self::one() / self + } + + /// Raise a number to an integer power. + /// + /// Using this function is generally faster than using `powf` + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// + /// fn check(x: T, exp: i32, powi: T) { + /// assert!(x.powi(exp) == powi); + /// } + /// + /// check(9.0f32, 2, 81.0); + /// check(1.0f32, -2, 1.0); + /// check(10.0f64, 20, 1e20); + /// check(4.0f64, -2, 0.0625); + /// check(-1.0f64, std::i32::MIN, 1.0); + /// ``` + #[inline] + fn powi(mut self, mut exp: i32) -> Self { + if exp < 0 { + exp = exp.wrapping_neg(); + self = self.recip(); + } + // It should always be possible to convert a positive `i32` to a `usize`. + // Note, `i32::MIN` will wrap and still be negative, so we need to convert + // to `u32` without sign-extension before growing to `usize`. + super::pow(self, (exp as u32).to_usize().unwrap()) + } + + /// Converts to degrees, assuming the number is in radians. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(rad: T, deg: T) { + /// assert!(rad.to_degrees() == deg); + /// } + /// + /// check(0.0f32, 0.0); + /// check(f32::consts::PI, 180.0); + /// check(f64::consts::FRAC_PI_4, 45.0); + /// check(f64::INFINITY, f64::INFINITY); + /// ``` + fn to_degrees(self) -> Self; + + /// Converts to radians, assuming the number is in degrees. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(deg: T, rad: T) { + /// assert!(deg.to_radians() == rad); + /// } + /// + /// check(0.0f32, 0.0); + /// check(180.0, f32::consts::PI); + /// check(45.0, f64::consts::FRAC_PI_4); + /// check(f64::INFINITY, f64::INFINITY); + /// ``` + fn to_radians(self) -> Self; + + /// Returns the mantissa, base 2 exponent, and sign as integers, respectively. + /// The original number can be recovered by `sign * mantissa * 2 ^ exponent`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::float::FloatCore; + /// use std::{f32, f64}; + /// + /// fn check(x: T, m: u64, e: i16, s:i8) { + /// let (mantissa, exponent, sign) = x.integer_decode(); + /// assert_eq!(mantissa, m); + /// assert_eq!(exponent, e); + /// assert_eq!(sign, s); + /// } + /// + /// check(2.0f32, 1 << 23, -22, 1); + /// check(-2.0f32, 1 << 23, -22, -1); + /// check(f32::INFINITY, 1 << 23, 105, 1); + /// check(f64::NEG_INFINITY, 1 << 52, 972, -1); + /// ``` + fn integer_decode(self) -> (u64, i16, i8); +} + +impl FloatCore for f32 { + constant! { + infinity() -> f32::INFINITY; + neg_infinity() -> f32::NEG_INFINITY; + nan() -> f32::NAN; + neg_zero() -> -0.0; + min_value() -> f32::MIN; + min_positive_value() -> f32::MIN_POSITIVE; + epsilon() -> f32::EPSILON; + max_value() -> f32::MAX; + } + + #[inline] + fn integer_decode(self) -> (u64, i16, i8) { + integer_decode_f32(self) + } + + forward! { + Self::is_nan(self) -> bool; + Self::is_infinite(self) -> bool; + Self::is_finite(self) -> bool; + Self::is_normal(self) -> bool; + Self::is_subnormal(self) -> bool; + Self::clamp(self, min: Self, max: Self) -> Self; + Self::classify(self) -> FpCategory; + Self::is_sign_positive(self) -> bool; + Self::is_sign_negative(self) -> bool; + Self::min(self, other: Self) -> Self; + Self::max(self, other: Self) -> Self; + Self::recip(self) -> Self; + Self::to_degrees(self) -> Self; + Self::to_radians(self) -> Self; + } + + #[cfg(feature = "std")] + forward! { + Self::floor(self) -> Self; + Self::ceil(self) -> Self; + Self::round(self) -> Self; + Self::trunc(self) -> Self; + Self::fract(self) -> Self; + Self::abs(self) -> Self; + Self::signum(self) -> Self; + Self::powi(self, n: i32) -> Self; + } + + #[cfg(all(not(feature = "std"), feature = "libm"))] + forward! { + libm::floorf as floor(self) -> Self; + libm::ceilf as ceil(self) -> Self; + libm::roundf as round(self) -> Self; + libm::truncf as trunc(self) -> Self; + libm::fabsf as abs(self) -> Self; + } + + #[cfg(all(not(feature = "std"), feature = "libm"))] + #[inline] + fn fract(self) -> Self { + self - libm::truncf(self) + } +} + +impl FloatCore for f64 { + constant! { + infinity() -> f64::INFINITY; + neg_infinity() -> f64::NEG_INFINITY; + nan() -> f64::NAN; + neg_zero() -> -0.0; + min_value() -> f64::MIN; + min_positive_value() -> f64::MIN_POSITIVE; + epsilon() -> f64::EPSILON; + max_value() -> f64::MAX; + } + + #[inline] + fn integer_decode(self) -> (u64, i16, i8) { + integer_decode_f64(self) + } + + forward! { + Self::is_nan(self) -> bool; + Self::is_infinite(self) -> bool; + Self::is_finite(self) -> bool; + Self::is_normal(self) -> bool; + Self::is_subnormal(self) -> bool; + Self::clamp(self, min: Self, max: Self) -> Self; + Self::classify(self) -> FpCategory; + Self::is_sign_positive(self) -> bool; + Self::is_sign_negative(self) -> bool; + Self::min(self, other: Self) -> Self; + Self::max(self, other: Self) -> Self; + Self::recip(self) -> Self; + Self::to_degrees(self) -> Self; + Self::to_radians(self) -> Self; + } + + #[cfg(feature = "std")] + forward! { + Self::floor(self) -> Self; + Self::ceil(self) -> Self; + Self::round(self) -> Self; + Self::trunc(self) -> Self; + Self::fract(self) -> Self; + Self::abs(self) -> Self; + Self::signum(self) -> Self; + Self::powi(self, n: i32) -> Self; + } + + #[cfg(all(not(feature = "std"), feature = "libm"))] + forward! { + libm::floor as floor(self) -> Self; + libm::ceil as ceil(self) -> Self; + libm::round as round(self) -> Self; + libm::trunc as trunc(self) -> Self; + libm::fabs as abs(self) -> Self; + } + + #[cfg(all(not(feature = "std"), feature = "libm"))] + #[inline] + fn fract(self) -> Self { + self - libm::trunc(self) + } +} + +// FIXME: these doctests aren't actually helpful, because they're using and +// testing the inherent methods directly, not going through `Float`. + +/// Generic trait for floating point numbers +/// +/// This trait is only available with the `std` feature, or with the `libm` feature otherwise. +#[cfg(any(feature = "std", feature = "libm"))] +pub trait Float: Num + Copy + NumCast + PartialOrd + Neg { + /// Returns the `NaN` value. + /// + /// ``` + /// use num_traits::Float; + /// + /// let nan: f32 = Float::nan(); + /// + /// assert!(nan.is_nan()); + /// ``` + fn nan() -> Self; + /// Returns the infinite value. + /// + /// ``` + /// use num_traits::Float; + /// use std::f32; + /// + /// let infinity: f32 = Float::infinity(); + /// + /// assert!(infinity.is_infinite()); + /// assert!(!infinity.is_finite()); + /// assert!(infinity > f32::MAX); + /// ``` + fn infinity() -> Self; + /// Returns the negative infinite value. + /// + /// ``` + /// use num_traits::Float; + /// use std::f32; + /// + /// let neg_infinity: f32 = Float::neg_infinity(); + /// + /// assert!(neg_infinity.is_infinite()); + /// assert!(!neg_infinity.is_finite()); + /// assert!(neg_infinity < f32::MIN); + /// ``` + fn neg_infinity() -> Self; + /// Returns `-0.0`. + /// + /// ``` + /// use num_traits::{Zero, Float}; + /// + /// let inf: f32 = Float::infinity(); + /// let zero: f32 = Zero::zero(); + /// let neg_zero: f32 = Float::neg_zero(); + /// + /// assert_eq!(zero, neg_zero); + /// assert_eq!(7.0f32/inf, zero); + /// assert_eq!(zero * 10.0, zero); + /// ``` + fn neg_zero() -> Self; + + /// Returns the smallest finite value that this type can represent. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x: f64 = Float::min_value(); + /// + /// assert_eq!(x, f64::MIN); + /// ``` + fn min_value() -> Self; + + /// Returns the smallest positive, normalized value that this type can represent. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x: f64 = Float::min_positive_value(); + /// + /// assert_eq!(x, f64::MIN_POSITIVE); + /// ``` + fn min_positive_value() -> Self; + + /// Returns epsilon, a small positive value. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x: f64 = Float::epsilon(); + /// + /// assert_eq!(x, f64::EPSILON); + /// ``` + /// + /// # Panics + /// + /// The default implementation will panic if `f32::EPSILON` cannot + /// be cast to `Self`. + fn epsilon() -> Self { + Self::from(f32::EPSILON).expect("Unable to cast from f32::EPSILON") + } + + /// Returns the largest finite value that this type can represent. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x: f64 = Float::max_value(); + /// assert_eq!(x, f64::MAX); + /// ``` + fn max_value() -> Self; + + /// Returns `true` if this value is `NaN` and false otherwise. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let nan = f64::NAN; + /// let f = 7.0; + /// + /// assert!(nan.is_nan()); + /// assert!(!f.is_nan()); + /// ``` + fn is_nan(self) -> bool; + + /// Returns `true` if this value is positive infinity or negative infinity and + /// false otherwise. + /// + /// ``` + /// use num_traits::Float; + /// use std::f32; + /// + /// let f = 7.0f32; + /// let inf: f32 = Float::infinity(); + /// let neg_inf: f32 = Float::neg_infinity(); + /// let nan: f32 = f32::NAN; + /// + /// assert!(!f.is_infinite()); + /// assert!(!nan.is_infinite()); + /// + /// assert!(inf.is_infinite()); + /// assert!(neg_inf.is_infinite()); + /// ``` + fn is_infinite(self) -> bool; + + /// Returns `true` if this number is neither infinite nor `NaN`. + /// + /// ``` + /// use num_traits::Float; + /// use std::f32; + /// + /// let f = 7.0f32; + /// let inf: f32 = Float::infinity(); + /// let neg_inf: f32 = Float::neg_infinity(); + /// let nan: f32 = f32::NAN; + /// + /// assert!(f.is_finite()); + /// + /// assert!(!nan.is_finite()); + /// assert!(!inf.is_finite()); + /// assert!(!neg_inf.is_finite()); + /// ``` + fn is_finite(self) -> bool; + + /// Returns `true` if the number is neither zero, infinite, + /// [subnormal][subnormal], or `NaN`. + /// + /// ``` + /// use num_traits::Float; + /// use std::f32; + /// + /// let min = f32::MIN_POSITIVE; // 1.17549435e-38f32 + /// let max = f32::MAX; + /// let lower_than_min = 1.0e-40_f32; + /// let zero = 0.0f32; + /// + /// assert!(min.is_normal()); + /// assert!(max.is_normal()); + /// + /// assert!(!zero.is_normal()); + /// assert!(!f32::NAN.is_normal()); + /// assert!(!f32::INFINITY.is_normal()); + /// // Values between `0` and `min` are Subnormal. + /// assert!(!lower_than_min.is_normal()); + /// ``` + /// [subnormal]: http://en.wikipedia.org/wiki/Subnormal_number + fn is_normal(self) -> bool; + + /// Returns `true` if the number is [subnormal]. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let min = f64::MIN_POSITIVE; // 2.2250738585072014e-308_f64 + /// let max = f64::MAX; + /// let lower_than_min = 1.0e-308_f64; + /// let zero = 0.0_f64; + /// + /// assert!(!min.is_subnormal()); + /// assert!(!max.is_subnormal()); + /// + /// assert!(!zero.is_subnormal()); + /// assert!(!f64::NAN.is_subnormal()); + /// assert!(!f64::INFINITY.is_subnormal()); + /// // Values between `0` and `min` are Subnormal. + /// assert!(lower_than_min.is_subnormal()); + /// ``` + /// [subnormal]: https://en.wikipedia.org/wiki/Subnormal_number + #[inline] + fn is_subnormal(self) -> bool { + self.classify() == FpCategory::Subnormal + } + + /// Returns the floating point category of the number. If only one property + /// is going to be tested, it is generally faster to use the specific + /// predicate instead. + /// + /// ``` + /// use num_traits::Float; + /// use std::num::FpCategory; + /// use std::f32; + /// + /// let num = 12.4f32; + /// let inf = f32::INFINITY; + /// + /// assert_eq!(num.classify(), FpCategory::Normal); + /// assert_eq!(inf.classify(), FpCategory::Infinite); + /// ``` + fn classify(self) -> FpCategory; + + /// Returns the largest integer less than or equal to a number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let f = 3.99; + /// let g = 3.0; + /// + /// assert_eq!(f.floor(), 3.0); + /// assert_eq!(g.floor(), 3.0); + /// ``` + fn floor(self) -> Self; + + /// Returns the smallest integer greater than or equal to a number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let f = 3.01; + /// let g = 4.0; + /// + /// assert_eq!(f.ceil(), 4.0); + /// assert_eq!(g.ceil(), 4.0); + /// ``` + fn ceil(self) -> Self; + + /// Returns the nearest integer to a number. Round half-way cases away from + /// `0.0`. + /// + /// ``` + /// use num_traits::Float; + /// + /// let f = 3.3; + /// let g = -3.3; + /// + /// assert_eq!(f.round(), 3.0); + /// assert_eq!(g.round(), -3.0); + /// ``` + fn round(self) -> Self; + + /// Return the integer part of a number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let f = 3.3; + /// let g = -3.7; + /// + /// assert_eq!(f.trunc(), 3.0); + /// assert_eq!(g.trunc(), -3.0); + /// ``` + fn trunc(self) -> Self; + + /// Returns the fractional part of a number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 3.5; + /// let y = -3.5; + /// let abs_difference_x = (x.fract() - 0.5).abs(); + /// let abs_difference_y = (y.fract() - (-0.5)).abs(); + /// + /// assert!(abs_difference_x < 1e-10); + /// assert!(abs_difference_y < 1e-10); + /// ``` + fn fract(self) -> Self; + + /// Computes the absolute value of `self`. Returns `Float::nan()` if the + /// number is `Float::nan()`. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x = 3.5; + /// let y = -3.5; + /// + /// let abs_difference_x = (x.abs() - x).abs(); + /// let abs_difference_y = (y.abs() - (-y)).abs(); + /// + /// assert!(abs_difference_x < 1e-10); + /// assert!(abs_difference_y < 1e-10); + /// + /// assert!(f64::NAN.abs().is_nan()); + /// ``` + fn abs(self) -> Self; + + /// Returns a number that represents the sign of `self`. + /// + /// - `1.0` if the number is positive, `+0.0` or `Float::infinity()` + /// - `-1.0` if the number is negative, `-0.0` or `Float::neg_infinity()` + /// - `Float::nan()` if the number is `Float::nan()` + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let f = 3.5; + /// + /// assert_eq!(f.signum(), 1.0); + /// assert_eq!(f64::NEG_INFINITY.signum(), -1.0); + /// + /// assert!(f64::NAN.signum().is_nan()); + /// ``` + fn signum(self) -> Self; + + /// Returns `true` if `self` is positive, including `+0.0`, + /// `Float::infinity()`, and `Float::nan()`. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let nan: f64 = f64::NAN; + /// let neg_nan: f64 = -f64::NAN; + /// + /// let f = 7.0; + /// let g = -7.0; + /// + /// assert!(f.is_sign_positive()); + /// assert!(!g.is_sign_positive()); + /// assert!(nan.is_sign_positive()); + /// assert!(!neg_nan.is_sign_positive()); + /// ``` + fn is_sign_positive(self) -> bool; + + /// Returns `true` if `self` is negative, including `-0.0`, + /// `Float::neg_infinity()`, and `-Float::nan()`. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let nan: f64 = f64::NAN; + /// let neg_nan: f64 = -f64::NAN; + /// + /// let f = 7.0; + /// let g = -7.0; + /// + /// assert!(!f.is_sign_negative()); + /// assert!(g.is_sign_negative()); + /// assert!(!nan.is_sign_negative()); + /// assert!(neg_nan.is_sign_negative()); + /// ``` + fn is_sign_negative(self) -> bool; + + /// Fused multiply-add. Computes `(self * a) + b` with only one rounding + /// error, yielding a more accurate result than an unfused multiply-add. + /// + /// Using `mul_add` can be more performant than an unfused multiply-add if + /// the target architecture has a dedicated `fma` CPU instruction. + /// + /// ``` + /// use num_traits::Float; + /// + /// let m = 10.0; + /// let x = 4.0; + /// let b = 60.0; + /// + /// // 100.0 + /// let abs_difference = (m.mul_add(x, b) - (m*x + b)).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn mul_add(self, a: Self, b: Self) -> Self; + /// Take the reciprocal (inverse) of a number, `1/x`. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 2.0; + /// let abs_difference = (x.recip() - (1.0/x)).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn recip(self) -> Self; + + /// Raise a number to an integer power. + /// + /// Using this function is generally faster than using `powf` + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 2.0; + /// let abs_difference = (x.powi(2) - x*x).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn powi(self, n: i32) -> Self; + + /// Raise a number to a floating point power. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 2.0; + /// let abs_difference = (x.powf(2.0) - x*x).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn powf(self, n: Self) -> Self; + + /// Take the square root of a number. + /// + /// Returns NaN if `self` is a negative number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let positive = 4.0; + /// let negative = -4.0; + /// + /// let abs_difference = (positive.sqrt() - 2.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// assert!(negative.sqrt().is_nan()); + /// ``` + fn sqrt(self) -> Self; + + /// Returns `e^(self)`, (the exponential function). + /// + /// ``` + /// use num_traits::Float; + /// + /// let one = 1.0; + /// // e^1 + /// let e = one.exp(); + /// + /// // ln(e) - 1 == 0 + /// let abs_difference = (e.ln() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn exp(self) -> Self; + + /// Returns `2^(self)`. + /// + /// ``` + /// use num_traits::Float; + /// + /// let f = 2.0; + /// + /// // 2^2 - 4 == 0 + /// let abs_difference = (f.exp2() - 4.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn exp2(self) -> Self; + + /// Returns the natural logarithm of the number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let one = 1.0; + /// // e^1 + /// let e = one.exp(); + /// + /// // ln(e) - 1 == 0 + /// let abs_difference = (e.ln() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn ln(self) -> Self; + + /// Returns the logarithm of the number with respect to an arbitrary base. + /// + /// ``` + /// use num_traits::Float; + /// + /// let ten = 10.0; + /// let two = 2.0; + /// + /// // log10(10) - 1 == 0 + /// let abs_difference_10 = (ten.log(10.0) - 1.0).abs(); + /// + /// // log2(2) - 1 == 0 + /// let abs_difference_2 = (two.log(2.0) - 1.0).abs(); + /// + /// assert!(abs_difference_10 < 1e-10); + /// assert!(abs_difference_2 < 1e-10); + /// ``` + fn log(self, base: Self) -> Self; + + /// Returns the base 2 logarithm of the number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let two = 2.0; + /// + /// // log2(2) - 1 == 0 + /// let abs_difference = (two.log2() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn log2(self) -> Self; + + /// Returns the base 10 logarithm of the number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let ten = 10.0; + /// + /// // log10(10) - 1 == 0 + /// let abs_difference = (ten.log10() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn log10(self) -> Self; + + /// Converts radians to degrees. + /// + /// ``` + /// use std::f64::consts; + /// + /// let angle = consts::PI; + /// + /// let abs_difference = (angle.to_degrees() - 180.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + #[inline] + fn to_degrees(self) -> Self { + let halfpi = Self::zero().acos(); + let ninety = Self::from(90u8).unwrap(); + self * ninety / halfpi + } + + /// Converts degrees to radians. + /// + /// ``` + /// use std::f64::consts; + /// + /// let angle = 180.0_f64; + /// + /// let abs_difference = (angle.to_radians() - consts::PI).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + #[inline] + fn to_radians(self) -> Self { + let halfpi = Self::zero().acos(); + let ninety = Self::from(90u8).unwrap(); + self * halfpi / ninety + } + + /// Returns the maximum of the two numbers. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 1.0; + /// let y = 2.0; + /// + /// assert_eq!(x.max(y), y); + /// ``` + fn max(self, other: Self) -> Self; + + /// Returns the minimum of the two numbers. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 1.0; + /// let y = 2.0; + /// + /// assert_eq!(x.min(y), x); + /// ``` + fn min(self, other: Self) -> Self; + + /// Clamps a value between a min and max. + /// + /// **Panics** in debug mode if `!(min <= max)`. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 1.0; + /// let y = 2.0; + /// let z = 3.0; + /// + /// assert_eq!(x.clamp(y, z), 2.0); + /// ``` + fn clamp(self, min: Self, max: Self) -> Self { + crate::clamp(self, min, max) + } + + /// The positive difference of two numbers. + /// + /// * If `self <= other`: `0:0` + /// * Else: `self - other` + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 3.0; + /// let y = -3.0; + /// + /// let abs_difference_x = (x.abs_sub(1.0) - 2.0).abs(); + /// let abs_difference_y = (y.abs_sub(1.0) - 0.0).abs(); + /// + /// assert!(abs_difference_x < 1e-10); + /// assert!(abs_difference_y < 1e-10); + /// ``` + fn abs_sub(self, other: Self) -> Self; + + /// Take the cubic root of a number. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 8.0; + /// + /// // x^(1/3) - 2 == 0 + /// let abs_difference = (x.cbrt() - 2.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn cbrt(self) -> Self; + + /// Calculate the length of the hypotenuse of a right-angle triangle given + /// legs of length `x` and `y`. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 2.0; + /// let y = 3.0; + /// + /// // sqrt(x^2 + y^2) + /// let abs_difference = (x.hypot(y) - (x.powi(2) + y.powi(2)).sqrt()).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn hypot(self, other: Self) -> Self; + + /// Computes the sine of a number (in radians). + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x = f64::consts::PI/2.0; + /// + /// let abs_difference = (x.sin() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn sin(self) -> Self; + + /// Computes the cosine of a number (in radians). + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x = 2.0*f64::consts::PI; + /// + /// let abs_difference = (x.cos() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn cos(self) -> Self; + + /// Computes the tangent of a number (in radians). + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x = f64::consts::PI/4.0; + /// let abs_difference = (x.tan() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-14); + /// ``` + fn tan(self) -> Self; + + /// Computes the arcsine of a number. Return value is in radians in + /// the range [-pi/2, pi/2] or NaN if the number is outside the range + /// [-1, 1]. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let f = f64::consts::PI / 2.0; + /// + /// // asin(sin(pi/2)) + /// let abs_difference = (f.sin().asin() - f64::consts::PI / 2.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn asin(self) -> Self; + + /// Computes the arccosine of a number. Return value is in radians in + /// the range [0, pi] or NaN if the number is outside the range + /// [-1, 1]. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let f = f64::consts::PI / 4.0; + /// + /// // acos(cos(pi/4)) + /// let abs_difference = (f.cos().acos() - f64::consts::PI / 4.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn acos(self) -> Self; + + /// Computes the arctangent of a number. Return value is in radians in the + /// range [-pi/2, pi/2]; + /// + /// ``` + /// use num_traits::Float; + /// + /// let f = 1.0; + /// + /// // atan(tan(1)) + /// let abs_difference = (f.tan().atan() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn atan(self) -> Self; + + /// Computes the four quadrant arctangent of `self` (`y`) and `other` (`x`). + /// + /// * `x = 0`, `y = 0`: `0` + /// * `x >= 0`: `arctan(y/x)` -> `[-pi/2, pi/2]` + /// * `y >= 0`: `arctan(y/x) + pi` -> `(pi/2, pi]` + /// * `y < 0`: `arctan(y/x) - pi` -> `(-pi, -pi/2)` + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let pi = f64::consts::PI; + /// // All angles from horizontal right (+x) + /// // 45 deg counter-clockwise + /// let x1 = 3.0; + /// let y1 = -3.0; + /// + /// // 135 deg clockwise + /// let x2 = -3.0; + /// let y2 = 3.0; + /// + /// let abs_difference_1 = (y1.atan2(x1) - (-pi/4.0)).abs(); + /// let abs_difference_2 = (y2.atan2(x2) - 3.0*pi/4.0).abs(); + /// + /// assert!(abs_difference_1 < 1e-10); + /// assert!(abs_difference_2 < 1e-10); + /// ``` + fn atan2(self, other: Self) -> Self; + + /// Simultaneously computes the sine and cosine of the number, `x`. Returns + /// `(sin(x), cos(x))`. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x = f64::consts::PI/4.0; + /// let f = x.sin_cos(); + /// + /// let abs_difference_0 = (f.0 - x.sin()).abs(); + /// let abs_difference_1 = (f.1 - x.cos()).abs(); + /// + /// assert!(abs_difference_0 < 1e-10); + /// assert!(abs_difference_0 < 1e-10); + /// ``` + fn sin_cos(self) -> (Self, Self); + + /// Returns `e^(self) - 1` in a way that is accurate even if the + /// number is close to zero. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 7.0; + /// + /// // e^(ln(7)) - 1 + /// let abs_difference = (x.ln().exp_m1() - 6.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn exp_m1(self) -> Self; + + /// Returns `ln(1+n)` (natural logarithm) more accurately than if + /// the operations were performed separately. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let x = f64::consts::E - 1.0; + /// + /// // ln(1 + (e - 1)) == ln(e) == 1 + /// let abs_difference = (x.ln_1p() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn ln_1p(self) -> Self; + + /// Hyperbolic sine function. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let e = f64::consts::E; + /// let x = 1.0; + /// + /// let f = x.sinh(); + /// // Solving sinh() at 1 gives `(e^2-1)/(2e)` + /// let g = (e*e - 1.0)/(2.0*e); + /// let abs_difference = (f - g).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn sinh(self) -> Self; + + /// Hyperbolic cosine function. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let e = f64::consts::E; + /// let x = 1.0; + /// let f = x.cosh(); + /// // Solving cosh() at 1 gives this result + /// let g = (e*e + 1.0)/(2.0*e); + /// let abs_difference = (f - g).abs(); + /// + /// // Same result + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn cosh(self) -> Self; + + /// Hyperbolic tangent function. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let e = f64::consts::E; + /// let x = 1.0; + /// + /// let f = x.tanh(); + /// // Solving tanh() at 1 gives `(1 - e^(-2))/(1 + e^(-2))` + /// let g = (1.0 - e.powi(-2))/(1.0 + e.powi(-2)); + /// let abs_difference = (f - g).abs(); + /// + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn tanh(self) -> Self; + + /// Inverse hyperbolic sine function. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 1.0; + /// let f = x.sinh().asinh(); + /// + /// let abs_difference = (f - x).abs(); + /// + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn asinh(self) -> Self; + + /// Inverse hyperbolic cosine function. + /// + /// ``` + /// use num_traits::Float; + /// + /// let x = 1.0; + /// let f = x.cosh().acosh(); + /// + /// let abs_difference = (f - x).abs(); + /// + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn acosh(self) -> Self; + + /// Inverse hyperbolic tangent function. + /// + /// ``` + /// use num_traits::Float; + /// use std::f64; + /// + /// let e = f64::consts::E; + /// let f = e.tanh().atanh(); + /// + /// let abs_difference = (f - e).abs(); + /// + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn atanh(self) -> Self; + + /// Returns the mantissa, base 2 exponent, and sign as integers, respectively. + /// The original number can be recovered by `sign * mantissa * 2 ^ exponent`. + /// + /// ``` + /// use num_traits::Float; + /// + /// let num = 2.0f32; + /// + /// // (8388608, -22, 1) + /// let (mantissa, exponent, sign) = Float::integer_decode(num); + /// let sign_f = sign as f32; + /// let mantissa_f = mantissa as f32; + /// let exponent_f = num.powf(exponent as f32); + /// + /// // 1 * 8388608 * 2^(-22) == 2 + /// let abs_difference = (sign_f * mantissa_f * exponent_f - num).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn integer_decode(self) -> (u64, i16, i8); + + /// Returns a number composed of the magnitude of `self` and the sign of + /// `sign`. + /// + /// Equal to `self` if the sign of `self` and `sign` are the same, otherwise + /// equal to `-self`. If `self` is a `NAN`, then a `NAN` with the sign of + /// `sign` is returned. + /// + /// # Examples + /// + /// ``` + /// use num_traits::Float; + /// + /// let f = 3.5_f32; + /// + /// assert_eq!(f.copysign(0.42), 3.5_f32); + /// assert_eq!(f.copysign(-0.42), -3.5_f32); + /// assert_eq!((-f).copysign(0.42), 3.5_f32); + /// assert_eq!((-f).copysign(-0.42), -3.5_f32); + /// + /// assert!(f32::nan().copysign(1.0).is_nan()); + /// ``` + fn copysign(self, sign: Self) -> Self { + if self.is_sign_negative() == sign.is_sign_negative() { + self + } else { + self.neg() + } + } +} + +#[cfg(feature = "std")] +macro_rules! float_impl_std { + ($T:ident $decode:ident) => { + impl Float for $T { + constant! { + nan() -> $T::NAN; + infinity() -> $T::INFINITY; + neg_infinity() -> $T::NEG_INFINITY; + neg_zero() -> -0.0; + min_value() -> $T::MIN; + min_positive_value() -> $T::MIN_POSITIVE; + epsilon() -> $T::EPSILON; + max_value() -> $T::MAX; + } + + #[inline] + #[allow(deprecated)] + fn abs_sub(self, other: Self) -> Self { + <$T>::abs_sub(self, other) + } + + #[inline] + fn integer_decode(self) -> (u64, i16, i8) { + $decode(self) + } + + forward! { + Self::is_nan(self) -> bool; + Self::is_infinite(self) -> bool; + Self::is_finite(self) -> bool; + Self::is_normal(self) -> bool; + Self::is_subnormal(self) -> bool; + Self::classify(self) -> FpCategory; + Self::clamp(self, min: Self, max: Self) -> Self; + Self::floor(self) -> Self; + Self::ceil(self) -> Self; + Self::round(self) -> Self; + Self::trunc(self) -> Self; + Self::fract(self) -> Self; + Self::abs(self) -> Self; + Self::signum(self) -> Self; + Self::is_sign_positive(self) -> bool; + Self::is_sign_negative(self) -> bool; + Self::mul_add(self, a: Self, b: Self) -> Self; + Self::recip(self) -> Self; + Self::powi(self, n: i32) -> Self; + Self::powf(self, n: Self) -> Self; + Self::sqrt(self) -> Self; + Self::exp(self) -> Self; + Self::exp2(self) -> Self; + Self::ln(self) -> Self; + Self::log(self, base: Self) -> Self; + Self::log2(self) -> Self; + Self::log10(self) -> Self; + Self::to_degrees(self) -> Self; + Self::to_radians(self) -> Self; + Self::max(self, other: Self) -> Self; + Self::min(self, other: Self) -> Self; + Self::cbrt(self) -> Self; + Self::hypot(self, other: Self) -> Self; + Self::sin(self) -> Self; + Self::cos(self) -> Self; + Self::tan(self) -> Self; + Self::asin(self) -> Self; + Self::acos(self) -> Self; + Self::atan(self) -> Self; + Self::atan2(self, other: Self) -> Self; + Self::sin_cos(self) -> (Self, Self); + Self::exp_m1(self) -> Self; + Self::ln_1p(self) -> Self; + Self::sinh(self) -> Self; + Self::cosh(self) -> Self; + Self::tanh(self) -> Self; + Self::asinh(self) -> Self; + Self::acosh(self) -> Self; + Self::atanh(self) -> Self; + Self::copysign(self, sign: Self) -> Self; + } + } + }; +} + +#[cfg(all(not(feature = "std"), feature = "libm"))] +macro_rules! float_impl_libm { + ($T:ident $decode:ident) => { + constant! { + nan() -> $T::NAN; + infinity() -> $T::INFINITY; + neg_infinity() -> $T::NEG_INFINITY; + neg_zero() -> -0.0; + min_value() -> $T::MIN; + min_positive_value() -> $T::MIN_POSITIVE; + epsilon() -> $T::EPSILON; + max_value() -> $T::MAX; + } + + #[inline] + fn integer_decode(self) -> (u64, i16, i8) { + $decode(self) + } + + #[inline] + fn fract(self) -> Self { + self - Float::trunc(self) + } + + #[inline] + fn log(self, base: Self) -> Self { + self.ln() / base.ln() + } + + forward! { + Self::is_nan(self) -> bool; + Self::is_infinite(self) -> bool; + Self::is_finite(self) -> bool; + Self::is_normal(self) -> bool; + Self::is_subnormal(self) -> bool; + Self::clamp(self, min: Self, max: Self) -> Self; + Self::classify(self) -> FpCategory; + Self::is_sign_positive(self) -> bool; + Self::is_sign_negative(self) -> bool; + Self::min(self, other: Self) -> Self; + Self::max(self, other: Self) -> Self; + Self::recip(self) -> Self; + Self::to_degrees(self) -> Self; + Self::to_radians(self) -> Self; + } + + forward! { + FloatCore::signum(self) -> Self; + FloatCore::powi(self, n: i32) -> Self; + } + }; +} + +fn integer_decode_f32(f: f32) -> (u64, i16, i8) { + let bits: u32 = f.to_bits(); + let sign: i8 = if bits >> 31 == 0 { 1 } else { -1 }; + let mut exponent: i16 = ((bits >> 23) & 0xff) as i16; + let mantissa = if exponent == 0 { + (bits & 0x7fffff) << 1 + } else { + (bits & 0x7fffff) | 0x800000 + }; + // Exponent bias + mantissa shift + exponent -= 127 + 23; + (mantissa as u64, exponent, sign) +} + +fn integer_decode_f64(f: f64) -> (u64, i16, i8) { + let bits: u64 = f.to_bits(); + let sign: i8 = if bits >> 63 == 0 { 1 } else { -1 }; + let mut exponent: i16 = ((bits >> 52) & 0x7ff) as i16; + let mantissa = if exponent == 0 { + (bits & 0xfffffffffffff) << 1 + } else { + (bits & 0xfffffffffffff) | 0x10000000000000 + }; + // Exponent bias + mantissa shift + exponent -= 1023 + 52; + (mantissa, exponent, sign) +} + +#[cfg(feature = "std")] +float_impl_std!(f32 integer_decode_f32); +#[cfg(feature = "std")] +float_impl_std!(f64 integer_decode_f64); + +#[cfg(all(not(feature = "std"), feature = "libm"))] +impl Float for f32 { + float_impl_libm!(f32 integer_decode_f32); + + #[inline] + #[allow(deprecated)] + fn abs_sub(self, other: Self) -> Self { + libm::fdimf(self, other) + } + + forward! { + libm::floorf as floor(self) -> Self; + libm::ceilf as ceil(self) -> Self; + libm::roundf as round(self) -> Self; + libm::truncf as trunc(self) -> Self; + libm::fabsf as abs(self) -> Self; + libm::fmaf as mul_add(self, a: Self, b: Self) -> Self; + libm::powf as powf(self, n: Self) -> Self; + libm::sqrtf as sqrt(self) -> Self; + libm::expf as exp(self) -> Self; + libm::exp2f as exp2(self) -> Self; + libm::logf as ln(self) -> Self; + libm::log2f as log2(self) -> Self; + libm::log10f as log10(self) -> Self; + libm::cbrtf as cbrt(self) -> Self; + libm::hypotf as hypot(self, other: Self) -> Self; + libm::sinf as sin(self) -> Self; + libm::cosf as cos(self) -> Self; + libm::tanf as tan(self) -> Self; + libm::asinf as asin(self) -> Self; + libm::acosf as acos(self) -> Self; + libm::atanf as atan(self) -> Self; + libm::atan2f as atan2(self, other: Self) -> Self; + libm::sincosf as sin_cos(self) -> (Self, Self); + libm::expm1f as exp_m1(self) -> Self; + libm::log1pf as ln_1p(self) -> Self; + libm::sinhf as sinh(self) -> Self; + libm::coshf as cosh(self) -> Self; + libm::tanhf as tanh(self) -> Self; + libm::asinhf as asinh(self) -> Self; + libm::acoshf as acosh(self) -> Self; + libm::atanhf as atanh(self) -> Self; + libm::copysignf as copysign(self, other: Self) -> Self; + } +} + +#[cfg(all(not(feature = "std"), feature = "libm"))] +impl Float for f64 { + float_impl_libm!(f64 integer_decode_f64); + + #[inline] + #[allow(deprecated)] + fn abs_sub(self, other: Self) -> Self { + libm::fdim(self, other) + } + + forward! { + libm::floor as floor(self) -> Self; + libm::ceil as ceil(self) -> Self; + libm::round as round(self) -> Self; + libm::trunc as trunc(self) -> Self; + libm::fabs as abs(self) -> Self; + libm::fma as mul_add(self, a: Self, b: Self) -> Self; + libm::pow as powf(self, n: Self) -> Self; + libm::sqrt as sqrt(self) -> Self; + libm::exp as exp(self) -> Self; + libm::exp2 as exp2(self) -> Self; + libm::log as ln(self) -> Self; + libm::log2 as log2(self) -> Self; + libm::log10 as log10(self) -> Self; + libm::cbrt as cbrt(self) -> Self; + libm::hypot as hypot(self, other: Self) -> Self; + libm::sin as sin(self) -> Self; + libm::cos as cos(self) -> Self; + libm::tan as tan(self) -> Self; + libm::asin as asin(self) -> Self; + libm::acos as acos(self) -> Self; + libm::atan as atan(self) -> Self; + libm::atan2 as atan2(self, other: Self) -> Self; + libm::sincos as sin_cos(self) -> (Self, Self); + libm::expm1 as exp_m1(self) -> Self; + libm::log1p as ln_1p(self) -> Self; + libm::sinh as sinh(self) -> Self; + libm::cosh as cosh(self) -> Self; + libm::tanh as tanh(self) -> Self; + libm::asinh as asinh(self) -> Self; + libm::acosh as acosh(self) -> Self; + libm::atanh as atanh(self) -> Self; + libm::copysign as copysign(self, sign: Self) -> Self; + } +} + +macro_rules! float_const_impl { + ($(#[$doc:meta] $constant:ident,)+) => ( + #[allow(non_snake_case)] + pub trait FloatConst { + $(#[$doc] fn $constant() -> Self;)+ + #[doc = "Return the full circle constant `τ`."] + #[inline] + fn TAU() -> Self where Self: Sized + Add { + Self::PI() + Self::PI() + } + #[doc = "Return `log10(2.0)`."] + #[inline] + fn LOG10_2() -> Self where Self: Sized + Div { + Self::LN_2() / Self::LN_10() + } + #[doc = "Return `log2(10.0)`."] + #[inline] + fn LOG2_10() -> Self where Self: Sized + Div { + Self::LN_10() / Self::LN_2() + } + } + float_const_impl! { @float f32, $($constant,)+ } + float_const_impl! { @float f64, $($constant,)+ } + ); + (@float $T:ident, $($constant:ident,)+) => ( + impl FloatConst for $T { + constant! { + $( $constant() -> $T::consts::$constant; )+ + TAU() -> 6.28318530717958647692528676655900577; + LOG10_2() -> 0.301029995663981195213738894724493027; + LOG2_10() -> 3.32192809488736234787031942948939018; + } + } + ); +} + +float_const_impl! { + #[doc = "Return Euler’s number."] + E, + #[doc = "Return `1.0 / π`."] + FRAC_1_PI, + #[doc = "Return `1.0 / sqrt(2.0)`."] + FRAC_1_SQRT_2, + #[doc = "Return `2.0 / π`."] + FRAC_2_PI, + #[doc = "Return `2.0 / sqrt(π)`."] + FRAC_2_SQRT_PI, + #[doc = "Return `π / 2.0`."] + FRAC_PI_2, + #[doc = "Return `π / 3.0`."] + FRAC_PI_3, + #[doc = "Return `π / 4.0`."] + FRAC_PI_4, + #[doc = "Return `π / 6.0`."] + FRAC_PI_6, + #[doc = "Return `π / 8.0`."] + FRAC_PI_8, + #[doc = "Return `ln(10.0)`."] + LN_10, + #[doc = "Return `ln(2.0)`."] + LN_2, + #[doc = "Return `log10(e)`."] + LOG10_E, + #[doc = "Return `log2(e)`."] + LOG2_E, + #[doc = "Return Archimedes’ constant `π`."] + PI, + #[doc = "Return `sqrt(2.0)`."] + SQRT_2, +} + +/// Trait for floating point numbers that provide an implementation +/// of the `totalOrder` predicate as defined in the IEEE 754 (2008 revision) +/// floating point standard. +pub trait TotalOrder { + /// Return the ordering between `self` and `other`. + /// + /// Unlike the standard partial comparison between floating point numbers, + /// this comparison always produces an ordering in accordance to + /// the `totalOrder` predicate as defined in the IEEE 754 (2008 revision) + /// floating point standard. The values are ordered in the following sequence: + /// + /// - negative quiet NaN + /// - negative signaling NaN + /// - negative infinity + /// - negative numbers + /// - negative subnormal numbers + /// - negative zero + /// - positive zero + /// - positive subnormal numbers + /// - positive numbers + /// - positive infinity + /// - positive signaling NaN + /// - positive quiet NaN. + /// + /// The ordering established by this function does not always agree with the + /// [`PartialOrd`] and [`PartialEq`] implementations. For example, + /// they consider negative and positive zero equal, while `total_cmp` + /// doesn't. + /// + /// The interpretation of the signaling NaN bit follows the definition in + /// the IEEE 754 standard, which may not match the interpretation by some of + /// the older, non-conformant (e.g. MIPS) hardware implementations. + /// + /// # Examples + /// ``` + /// use num_traits::float::TotalOrder; + /// use std::cmp::Ordering; + /// use std::{f32, f64}; + /// + /// fn check_eq(x: T, y: T) { + /// assert_eq!(x.total_cmp(&y), Ordering::Equal); + /// } + /// + /// check_eq(f64::NAN, f64::NAN); + /// check_eq(f32::NAN, f32::NAN); + /// + /// fn check_lt(x: T, y: T) { + /// assert_eq!(x.total_cmp(&y), Ordering::Less); + /// } + /// + /// check_lt(-f64::NAN, f64::NAN); + /// check_lt(f64::INFINITY, f64::NAN); + /// check_lt(-0.0_f64, 0.0_f64); + /// ``` + fn total_cmp(&self, other: &Self) -> Ordering; +} +macro_rules! totalorder_impl { + ($T:ident, $I:ident, $U:ident, $bits:expr) => { + impl TotalOrder for $T { + #[inline] + #[cfg(has_total_cmp)] + fn total_cmp(&self, other: &Self) -> Ordering { + // Forward to the core implementation + Self::total_cmp(&self, other) + } + #[inline] + #[cfg(not(has_total_cmp))] + fn total_cmp(&self, other: &Self) -> Ordering { + // Backport the core implementation (since 1.62) + let mut left = self.to_bits() as $I; + let mut right = other.to_bits() as $I; + + left ^= (((left >> ($bits - 1)) as $U) >> 1) as $I; + right ^= (((right >> ($bits - 1)) as $U) >> 1) as $I; + + left.cmp(&right) + } + } + }; +} +totalorder_impl!(f64, i64, u64, 64); +totalorder_impl!(f32, i32, u32, 32); + +#[cfg(test)] +mod tests { + use core::f64::consts; + + const DEG_RAD_PAIRS: [(f64, f64); 7] = [ + (0.0, 0.), + (22.5, consts::FRAC_PI_8), + (30.0, consts::FRAC_PI_6), + (45.0, consts::FRAC_PI_4), + (60.0, consts::FRAC_PI_3), + (90.0, consts::FRAC_PI_2), + (180.0, consts::PI), + ]; + + #[test] + fn convert_deg_rad() { + use crate::float::FloatCore; + + for &(deg, rad) in &DEG_RAD_PAIRS { + assert!((FloatCore::to_degrees(rad) - deg).abs() < 1e-6); + assert!((FloatCore::to_radians(deg) - rad).abs() < 1e-6); + + let (deg, rad) = (deg as f32, rad as f32); + assert!((FloatCore::to_degrees(rad) - deg).abs() < 1e-5); + assert!((FloatCore::to_radians(deg) - rad).abs() < 1e-5); + } + } + + #[cfg(any(feature = "std", feature = "libm"))] + #[test] + fn convert_deg_rad_std() { + for &(deg, rad) in &DEG_RAD_PAIRS { + use crate::Float; + + assert!((Float::to_degrees(rad) - deg).abs() < 1e-6); + assert!((Float::to_radians(deg) - rad).abs() < 1e-6); + + let (deg, rad) = (deg as f32, rad as f32); + assert!((Float::to_degrees(rad) - deg).abs() < 1e-5); + assert!((Float::to_radians(deg) - rad).abs() < 1e-5); + } + } + + #[test] + fn to_degrees_rounding() { + use crate::float::FloatCore; + + assert_eq!( + FloatCore::to_degrees(1_f32), + 57.2957795130823208767981548141051703 + ); + } + + #[test] + #[cfg(any(feature = "std", feature = "libm"))] + fn extra_logs() { + use crate::float::{Float, FloatConst}; + + fn check(diff: F) { + let _2 = F::from(2.0).unwrap(); + assert!((F::LOG10_2() - F::log10(_2)).abs() < diff); + assert!((F::LOG10_2() - F::LN_2() / F::LN_10()).abs() < diff); + + let _10 = F::from(10.0).unwrap(); + assert!((F::LOG2_10() - F::log2(_10)).abs() < diff); + assert!((F::LOG2_10() - F::LN_10() / F::LN_2()).abs() < diff); + } + + check::(1e-6); + check::(1e-12); + } + + #[test] + #[cfg(any(feature = "std", feature = "libm"))] + fn copysign() { + use crate::float::Float; + test_copysign_generic(2.0_f32, -2.0_f32, f32::nan()); + test_copysign_generic(2.0_f64, -2.0_f64, f64::nan()); + test_copysignf(2.0_f32, -2.0_f32, f32::nan()); + } + + #[cfg(any(feature = "std", feature = "libm"))] + fn test_copysignf(p: f32, n: f32, nan: f32) { + use crate::float::Float; + use core::ops::Neg; + + assert!(p.is_sign_positive()); + assert!(n.is_sign_negative()); + assert!(nan.is_nan()); + + assert_eq!(p, Float::copysign(p, p)); + assert_eq!(p.neg(), Float::copysign(p, n)); + + assert_eq!(n, Float::copysign(n, n)); + assert_eq!(n.neg(), Float::copysign(n, p)); + + assert!(Float::copysign(nan, p).is_sign_positive()); + assert!(Float::copysign(nan, n).is_sign_negative()); + } + + #[cfg(any(feature = "std", feature = "libm"))] + fn test_copysign_generic(p: F, n: F, nan: F) { + assert!(p.is_sign_positive()); + assert!(n.is_sign_negative()); + assert!(nan.is_nan()); + assert!(!nan.is_subnormal()); + + assert_eq!(p, p.copysign(p)); + assert_eq!(p.neg(), p.copysign(n)); + + assert_eq!(n, n.copysign(n)); + assert_eq!(n.neg(), n.copysign(p)); + + assert!(nan.copysign(p).is_sign_positive()); + assert!(nan.copysign(n).is_sign_negative()); + } + + #[cfg(any(feature = "std", feature = "libm"))] + fn test_subnormal() { + let min_positive = F::min_positive_value(); + let lower_than_min = min_positive / F::from(2.0f32).unwrap(); + assert!(!min_positive.is_subnormal()); + assert!(lower_than_min.is_subnormal()); + } + + #[test] + #[cfg(any(feature = "std", feature = "libm"))] + fn subnormal() { + test_subnormal::(); + test_subnormal::(); + } + + #[test] + fn total_cmp() { + use crate::float::TotalOrder; + use core::cmp::Ordering; + use core::{f32, f64}; + + fn check_eq(x: T, y: T) { + assert_eq!(x.total_cmp(&y), Ordering::Equal); + } + fn check_lt(x: T, y: T) { + assert_eq!(x.total_cmp(&y), Ordering::Less); + } + fn check_gt(x: T, y: T) { + assert_eq!(x.total_cmp(&y), Ordering::Greater); + } + + check_eq(f64::NAN, f64::NAN); + check_eq(f32::NAN, f32::NAN); + + check_lt(-0.0_f64, 0.0_f64); + check_lt(-0.0_f32, 0.0_f32); + + // x87 registers don't preserve the exact value of signaling NaN: + // https://github.com/rust-lang/rust/issues/115567 + #[cfg(not(target_arch = "x86"))] + { + let s_nan = f64::from_bits(0x7ff4000000000000); + let q_nan = f64::from_bits(0x7ff8000000000000); + check_lt(s_nan, q_nan); + + let neg_s_nan = f64::from_bits(0xfff4000000000000); + let neg_q_nan = f64::from_bits(0xfff8000000000000); + check_lt(neg_q_nan, neg_s_nan); + + let s_nan = f32::from_bits(0x7fa00000); + let q_nan = f32::from_bits(0x7fc00000); + check_lt(s_nan, q_nan); + + let neg_s_nan = f32::from_bits(0xffa00000); + let neg_q_nan = f32::from_bits(0xffc00000); + check_lt(neg_q_nan, neg_s_nan); + } + + check_lt(-f64::NAN, f64::NEG_INFINITY); + check_gt(1.0_f64, -f64::NAN); + check_lt(f64::INFINITY, f64::NAN); + check_gt(f64::NAN, 1.0_f64); + + check_lt(-f32::NAN, f32::NEG_INFINITY); + check_gt(1.0_f32, -f32::NAN); + check_lt(f32::INFINITY, f32::NAN); + check_gt(f32::NAN, 1.0_f32); + } +} diff --git a/third_party/crates/num-traits-0.2.19/src/identities.rs b/third_party/crates/num-traits-0.2.19/src/identities.rs new file mode 100644 index 0000000..c30cd1d --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/identities.rs @@ -0,0 +1,238 @@ +use core::num::Wrapping; +use core::ops::{Add, Mul}; + +/// Defines an additive identity element for `Self`. +/// +/// # Laws +/// +/// ```text +/// a + 0 = a ∀ a ∈ Self +/// 0 + a = a ∀ a ∈ Self +/// ``` +pub trait Zero: Sized + Add { + /// Returns the additive identity element of `Self`, `0`. + /// # Purity + /// + /// This function should return the same result at all times regardless of + /// external mutable state, for example values stored in TLS or in + /// `static mut`s. + // This cannot be an associated constant, because of bignums. + fn zero() -> Self; + + /// Sets `self` to the additive identity element of `Self`, `0`. + fn set_zero(&mut self) { + *self = Zero::zero(); + } + + /// Returns `true` if `self` is equal to the additive identity. + fn is_zero(&self) -> bool; +} + +/// Defines an associated constant representing the additive identity element +/// for `Self`. +pub trait ConstZero: Zero { + /// The additive identity element of `Self`, `0`. + const ZERO: Self; +} + +macro_rules! zero_impl { + ($t:ty, $v:expr) => { + impl Zero for $t { + #[inline] + fn zero() -> $t { + $v + } + #[inline] + fn is_zero(&self) -> bool { + *self == $v + } + } + + impl ConstZero for $t { + const ZERO: Self = $v; + } + }; +} + +zero_impl!(usize, 0); +zero_impl!(u8, 0); +zero_impl!(u16, 0); +zero_impl!(u32, 0); +zero_impl!(u64, 0); +zero_impl!(u128, 0); + +zero_impl!(isize, 0); +zero_impl!(i8, 0); +zero_impl!(i16, 0); +zero_impl!(i32, 0); +zero_impl!(i64, 0); +zero_impl!(i128, 0); + +zero_impl!(f32, 0.0); +zero_impl!(f64, 0.0); + +impl Zero for Wrapping +where + Wrapping: Add>, +{ + fn is_zero(&self) -> bool { + self.0.is_zero() + } + + fn set_zero(&mut self) { + self.0.set_zero(); + } + + fn zero() -> Self { + Wrapping(T::zero()) + } +} + +impl ConstZero for Wrapping +where + Wrapping: Add>, +{ + const ZERO: Self = Wrapping(T::ZERO); +} + +/// Defines a multiplicative identity element for `Self`. +/// +/// # Laws +/// +/// ```text +/// a * 1 = a ∀ a ∈ Self +/// 1 * a = a ∀ a ∈ Self +/// ``` +pub trait One: Sized + Mul { + /// Returns the multiplicative identity element of `Self`, `1`. + /// + /// # Purity + /// + /// This function should return the same result at all times regardless of + /// external mutable state, for example values stored in TLS or in + /// `static mut`s. + // This cannot be an associated constant, because of bignums. + fn one() -> Self; + + /// Sets `self` to the multiplicative identity element of `Self`, `1`. + fn set_one(&mut self) { + *self = One::one(); + } + + /// Returns `true` if `self` is equal to the multiplicative identity. + /// + /// For performance reasons, it's best to implement this manually. + /// After a semver bump, this method will be required, and the + /// `where Self: PartialEq` bound will be removed. + #[inline] + fn is_one(&self) -> bool + where + Self: PartialEq, + { + *self == Self::one() + } +} + +/// Defines an associated constant representing the multiplicative identity +/// element for `Self`. +pub trait ConstOne: One { + /// The multiplicative identity element of `Self`, `1`. + const ONE: Self; +} + +macro_rules! one_impl { + ($t:ty, $v:expr) => { + impl One for $t { + #[inline] + fn one() -> $t { + $v + } + #[inline] + fn is_one(&self) -> bool { + *self == $v + } + } + + impl ConstOne for $t { + const ONE: Self = $v; + } + }; +} + +one_impl!(usize, 1); +one_impl!(u8, 1); +one_impl!(u16, 1); +one_impl!(u32, 1); +one_impl!(u64, 1); +one_impl!(u128, 1); + +one_impl!(isize, 1); +one_impl!(i8, 1); +one_impl!(i16, 1); +one_impl!(i32, 1); +one_impl!(i64, 1); +one_impl!(i128, 1); + +one_impl!(f32, 1.0); +one_impl!(f64, 1.0); + +impl One for Wrapping +where + Wrapping: Mul>, +{ + fn set_one(&mut self) { + self.0.set_one(); + } + + fn one() -> Self { + Wrapping(T::one()) + } +} + +impl ConstOne for Wrapping +where + Wrapping: Mul>, +{ + const ONE: Self = Wrapping(T::ONE); +} + +// Some helper functions provided for backwards compatibility. + +/// Returns the additive identity, `0`. +#[inline(always)] +pub fn zero() -> T { + Zero::zero() +} + +/// Returns the multiplicative identity, `1`. +#[inline(always)] +pub fn one() -> T { + One::one() +} + +#[test] +fn wrapping_identities() { + macro_rules! test_wrapping_identities { + ($($t:ty)+) => { + $( + assert_eq!(zero::<$t>(), zero::>().0); + assert_eq!(one::<$t>(), one::>().0); + assert_eq!((0 as $t).is_zero(), Wrapping(0 as $t).is_zero()); + assert_eq!((1 as $t).is_zero(), Wrapping(1 as $t).is_zero()); + )+ + }; + } + + test_wrapping_identities!(isize i8 i16 i32 i64 usize u8 u16 u32 u64); +} + +#[test] +fn wrapping_is_zero() { + fn require_zero(_: &T) {} + require_zero(&Wrapping(42)); +} +#[test] +fn wrapping_is_one() { + fn require_one(_: &T) {} + require_one(&Wrapping(42)); +} diff --git a/third_party/crates/num-traits-0.2.19/src/int.rs b/third_party/crates/num-traits-0.2.19/src/int.rs new file mode 100644 index 0000000..c6284bf --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/int.rs @@ -0,0 +1,562 @@ +use core::ops::{BitAnd, BitOr, BitXor, Not, Shl, Shr}; + +use crate::bounds::Bounded; +use crate::ops::checked::*; +use crate::ops::saturating::Saturating; +use crate::{Num, NumCast}; + +/// Generic trait for primitive integers. +/// +/// The `PrimInt` trait is an abstraction over the builtin primitive integer types (e.g., `u8`, +/// `u32`, `isize`, `i128`, ...). It inherits the basic numeric traits and extends them with +/// bitwise operators and non-wrapping arithmetic. +/// +/// The trait explicitly inherits `Copy`, `Eq`, `Ord`, and `Sized`. The intention is that all +/// types implementing this trait behave like primitive types that are passed by value by default +/// and behave like builtin integers. Furthermore, the types are expected to expose the integer +/// value in binary representation and support bitwise operators. The standard bitwise operations +/// (e.g., bitwise-and, bitwise-or, right-shift, left-shift) are inherited and the trait extends +/// these with introspective queries (e.g., `PrimInt::count_ones()`, `PrimInt::leading_zeros()`), +/// bitwise combinators (e.g., `PrimInt::rotate_left()`), and endianness converters (e.g., +/// `PrimInt::to_be()`). +/// +/// All `PrimInt` types are expected to be fixed-width binary integers. The width can be queried +/// via `T::zero().count_zeros()`. The trait currently lacks a way to query the width at +/// compile-time. +/// +/// While a default implementation for all builtin primitive integers is provided, the trait is in +/// no way restricted to these. Other integer types that fulfil the requirements are free to +/// implement the trait was well. +/// +/// This trait and many of the method names originate in the unstable `core::num::Int` trait from +/// the rust standard library. The original trait was never stabilized and thus removed from the +/// standard library. +pub trait PrimInt: + Sized + + Copy + + Num + + NumCast + + Bounded + + PartialOrd + + Ord + + Eq + + Not + + BitAnd + + BitOr + + BitXor + + Shl + + Shr + + CheckedAdd + + CheckedSub + + CheckedMul + + CheckedDiv + + Saturating +{ + /// Returns the number of ones in the binary representation of `self`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0b01001100u8; + /// + /// assert_eq!(n.count_ones(), 3); + /// ``` + fn count_ones(self) -> u32; + + /// Returns the number of zeros in the binary representation of `self`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0b01001100u8; + /// + /// assert_eq!(n.count_zeros(), 5); + /// ``` + fn count_zeros(self) -> u32; + + /// Returns the number of leading ones in the binary representation + /// of `self`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0xF00Du16; + /// + /// assert_eq!(n.leading_ones(), 4); + /// ``` + fn leading_ones(self) -> u32 { + (!self).leading_zeros() + } + + /// Returns the number of leading zeros in the binary representation + /// of `self`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0b0101000u16; + /// + /// assert_eq!(n.leading_zeros(), 10); + /// ``` + fn leading_zeros(self) -> u32; + + /// Returns the number of trailing ones in the binary representation + /// of `self`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0xBEEFu16; + /// + /// assert_eq!(n.trailing_ones(), 4); + /// ``` + fn trailing_ones(self) -> u32 { + (!self).trailing_zeros() + } + + /// Returns the number of trailing zeros in the binary representation + /// of `self`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0b0101000u16; + /// + /// assert_eq!(n.trailing_zeros(), 3); + /// ``` + fn trailing_zeros(self) -> u32; + + /// Shifts the bits to the left by a specified amount, `n`, wrapping + /// the truncated bits to the end of the resulting integer. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFu64; + /// let m = 0x3456789ABCDEF012u64; + /// + /// assert_eq!(n.rotate_left(12), m); + /// ``` + fn rotate_left(self, n: u32) -> Self; + + /// Shifts the bits to the right by a specified amount, `n`, wrapping + /// the truncated bits to the beginning of the resulting integer. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFu64; + /// let m = 0xDEF0123456789ABCu64; + /// + /// assert_eq!(n.rotate_right(12), m); + /// ``` + fn rotate_right(self, n: u32) -> Self; + + /// Shifts the bits to the left by a specified amount, `n`, filling + /// zeros in the least significant bits. + /// + /// This is bitwise equivalent to signed `Shl`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFu64; + /// let m = 0x3456789ABCDEF000u64; + /// + /// assert_eq!(n.signed_shl(12), m); + /// ``` + fn signed_shl(self, n: u32) -> Self; + + /// Shifts the bits to the right by a specified amount, `n`, copying + /// the "sign bit" in the most significant bits even for unsigned types. + /// + /// This is bitwise equivalent to signed `Shr`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0xFEDCBA9876543210u64; + /// let m = 0xFFFFEDCBA9876543u64; + /// + /// assert_eq!(n.signed_shr(12), m); + /// ``` + fn signed_shr(self, n: u32) -> Self; + + /// Shifts the bits to the left by a specified amount, `n`, filling + /// zeros in the least significant bits. + /// + /// This is bitwise equivalent to unsigned `Shl`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFi64; + /// let m = 0x3456789ABCDEF000i64; + /// + /// assert_eq!(n.unsigned_shl(12), m); + /// ``` + fn unsigned_shl(self, n: u32) -> Self; + + /// Shifts the bits to the right by a specified amount, `n`, filling + /// zeros in the most significant bits. + /// + /// This is bitwise equivalent to unsigned `Shr`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = -8i8; // 0b11111000 + /// let m = 62i8; // 0b00111110 + /// + /// assert_eq!(n.unsigned_shr(2), m); + /// ``` + fn unsigned_shr(self, n: u32) -> Self; + + /// Reverses the byte order of the integer. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFu64; + /// let m = 0xEFCDAB8967452301u64; + /// + /// assert_eq!(n.swap_bytes(), m); + /// ``` + fn swap_bytes(self) -> Self; + + /// Reverses the order of bits in the integer. + /// + /// The least significant bit becomes the most significant bit, second least-significant bit + /// becomes second most-significant bit, etc. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x12345678u32; + /// let m = 0x1e6a2c48u32; + /// + /// assert_eq!(n.reverse_bits(), m); + /// assert_eq!(0u32.reverse_bits(), 0); + /// ``` + fn reverse_bits(self) -> Self { + reverse_bits_fallback(self) + } + + /// Convert an integer from big endian to the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are swapped. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFu64; + /// + /// if cfg!(target_endian = "big") { + /// assert_eq!(u64::from_be(n), n) + /// } else { + /// assert_eq!(u64::from_be(n), n.swap_bytes()) + /// } + /// ``` + fn from_be(x: Self) -> Self; + + /// Convert an integer from little endian to the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are swapped. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFu64; + /// + /// if cfg!(target_endian = "little") { + /// assert_eq!(u64::from_le(n), n) + /// } else { + /// assert_eq!(u64::from_le(n), n.swap_bytes()) + /// } + /// ``` + fn from_le(x: Self) -> Self; + + /// Convert `self` to big endian from the target's endianness. + /// + /// On big endian this is a no-op. On little endian the bytes are swapped. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFu64; + /// + /// if cfg!(target_endian = "big") { + /// assert_eq!(n.to_be(), n) + /// } else { + /// assert_eq!(n.to_be(), n.swap_bytes()) + /// } + /// ``` + fn to_be(self) -> Self; + + /// Convert `self` to little endian from the target's endianness. + /// + /// On little endian this is a no-op. On big endian the bytes are swapped. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// let n = 0x0123456789ABCDEFu64; + /// + /// if cfg!(target_endian = "little") { + /// assert_eq!(n.to_le(), n) + /// } else { + /// assert_eq!(n.to_le(), n.swap_bytes()) + /// } + /// ``` + fn to_le(self) -> Self; + + /// Raises self to the power of `exp`, using exponentiation by squaring. + /// + /// # Examples + /// + /// ``` + /// use num_traits::PrimInt; + /// + /// assert_eq!(2i32.pow(4), 16); + /// ``` + fn pow(self, exp: u32) -> Self; +} + +fn one_per_byte() -> P { + // i8, u8: return 0x01 + // i16, u16: return 0x0101 = (0x01 << 8) | 0x01 + // i32, u32: return 0x01010101 = (0x0101 << 16) | 0x0101 + // ... + let mut ret = P::one(); + let mut shift = 8; + let mut b = ret.count_zeros() >> 3; + while b != 0 { + ret = (ret << shift) | ret; + shift <<= 1; + b >>= 1; + } + ret +} + +fn reverse_bits_fallback(i: P) -> P { + let rep_01: P = one_per_byte(); + let rep_03 = (rep_01 << 1) | rep_01; + let rep_05 = (rep_01 << 2) | rep_01; + let rep_0f = (rep_03 << 2) | rep_03; + let rep_33 = (rep_03 << 4) | rep_03; + let rep_55 = (rep_05 << 4) | rep_05; + + // code above only used to determine rep_0f, rep_33, rep_55; + // optimizer should be able to do it in compile time + let mut ret = i.swap_bytes(); + ret = ((ret & rep_0f) << 4) | ((ret >> 4) & rep_0f); + ret = ((ret & rep_33) << 2) | ((ret >> 2) & rep_33); + ret = ((ret & rep_55) << 1) | ((ret >> 1) & rep_55); + ret +} + +macro_rules! prim_int_impl { + ($T:ty, $S:ty, $U:ty) => { + impl PrimInt for $T { + #[inline] + fn count_ones(self) -> u32 { + <$T>::count_ones(self) + } + + #[inline] + fn count_zeros(self) -> u32 { + <$T>::count_zeros(self) + } + + #[inline] + fn leading_ones(self) -> u32 { + <$T>::leading_ones(self) + } + + #[inline] + fn leading_zeros(self) -> u32 { + <$T>::leading_zeros(self) + } + + #[inline] + fn trailing_ones(self) -> u32 { + <$T>::trailing_ones(self) + } + + #[inline] + fn trailing_zeros(self) -> u32 { + <$T>::trailing_zeros(self) + } + + #[inline] + fn rotate_left(self, n: u32) -> Self { + <$T>::rotate_left(self, n) + } + + #[inline] + fn rotate_right(self, n: u32) -> Self { + <$T>::rotate_right(self, n) + } + + #[inline] + fn signed_shl(self, n: u32) -> Self { + ((self as $S) << n) as $T + } + + #[inline] + fn signed_shr(self, n: u32) -> Self { + ((self as $S) >> n) as $T + } + + #[inline] + fn unsigned_shl(self, n: u32) -> Self { + ((self as $U) << n) as $T + } + + #[inline] + fn unsigned_shr(self, n: u32) -> Self { + ((self as $U) >> n) as $T + } + + #[inline] + fn swap_bytes(self) -> Self { + <$T>::swap_bytes(self) + } + + #[inline] + fn reverse_bits(self) -> Self { + <$T>::reverse_bits(self) + } + + #[inline] + fn from_be(x: Self) -> Self { + <$T>::from_be(x) + } + + #[inline] + fn from_le(x: Self) -> Self { + <$T>::from_le(x) + } + + #[inline] + fn to_be(self) -> Self { + <$T>::to_be(self) + } + + #[inline] + fn to_le(self) -> Self { + <$T>::to_le(self) + } + + #[inline] + fn pow(self, exp: u32) -> Self { + <$T>::pow(self, exp) + } + } + }; +} + +// prim_int_impl!(type, signed, unsigned); +prim_int_impl!(u8, i8, u8); +prim_int_impl!(u16, i16, u16); +prim_int_impl!(u32, i32, u32); +prim_int_impl!(u64, i64, u64); +prim_int_impl!(u128, i128, u128); +prim_int_impl!(usize, isize, usize); +prim_int_impl!(i8, i8, u8); +prim_int_impl!(i16, i16, u16); +prim_int_impl!(i32, i32, u32); +prim_int_impl!(i64, i64, u64); +prim_int_impl!(i128, i128, u128); +prim_int_impl!(isize, isize, usize); + +#[cfg(test)] +mod tests { + use crate::int::PrimInt; + + #[test] + pub fn reverse_bits() { + use core::{i16, i32, i64, i8}; + + assert_eq!( + PrimInt::reverse_bits(0x0123_4567_89ab_cdefu64), + 0xf7b3_d591_e6a2_c480 + ); + + assert_eq!(PrimInt::reverse_bits(0i8), 0); + assert_eq!(PrimInt::reverse_bits(-1i8), -1); + assert_eq!(PrimInt::reverse_bits(1i8), i8::MIN); + assert_eq!(PrimInt::reverse_bits(i8::MIN), 1); + assert_eq!(PrimInt::reverse_bits(-2i8), i8::MAX); + assert_eq!(PrimInt::reverse_bits(i8::MAX), -2); + + assert_eq!(PrimInt::reverse_bits(0i16), 0); + assert_eq!(PrimInt::reverse_bits(-1i16), -1); + assert_eq!(PrimInt::reverse_bits(1i16), i16::MIN); + assert_eq!(PrimInt::reverse_bits(i16::MIN), 1); + assert_eq!(PrimInt::reverse_bits(-2i16), i16::MAX); + assert_eq!(PrimInt::reverse_bits(i16::MAX), -2); + + assert_eq!(PrimInt::reverse_bits(0i32), 0); + assert_eq!(PrimInt::reverse_bits(-1i32), -1); + assert_eq!(PrimInt::reverse_bits(1i32), i32::MIN); + assert_eq!(PrimInt::reverse_bits(i32::MIN), 1); + assert_eq!(PrimInt::reverse_bits(-2i32), i32::MAX); + assert_eq!(PrimInt::reverse_bits(i32::MAX), -2); + + assert_eq!(PrimInt::reverse_bits(0i64), 0); + assert_eq!(PrimInt::reverse_bits(-1i64), -1); + assert_eq!(PrimInt::reverse_bits(1i64), i64::MIN); + assert_eq!(PrimInt::reverse_bits(i64::MIN), 1); + assert_eq!(PrimInt::reverse_bits(-2i64), i64::MAX); + assert_eq!(PrimInt::reverse_bits(i64::MAX), -2); + } + + #[test] + pub fn reverse_bits_i128() { + use core::i128; + + assert_eq!(PrimInt::reverse_bits(0i128), 0); + assert_eq!(PrimInt::reverse_bits(-1i128), -1); + assert_eq!(PrimInt::reverse_bits(1i128), i128::MIN); + assert_eq!(PrimInt::reverse_bits(i128::MIN), 1); + assert_eq!(PrimInt::reverse_bits(-2i128), i128::MAX); + assert_eq!(PrimInt::reverse_bits(i128::MAX), -2); + } +} diff --git a/third_party/crates/num-traits-0.2.19/src/lib.rs b/third_party/crates/num-traits-0.2.19/src/lib.rs new file mode 100644 index 0000000..d392e92 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/lib.rs @@ -0,0 +1,635 @@ +// Copyright 2013-2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Numeric traits for generic mathematics +//! +//! ## Compatibility +//! +//! The `num-traits` crate is tested for rustc 1.60 and greater. + +#![doc(html_root_url = "https://docs.rs/num-traits/0.2")] +#![deny(unconditional_recursion)] +#![no_std] + +// Need to explicitly bring the crate in for inherent float methods +#[cfg(feature = "std")] +extern crate std; + +use core::fmt; +use core::num::Wrapping; +use core::ops::{Add, Div, Mul, Rem, Sub}; +use core::ops::{AddAssign, DivAssign, MulAssign, RemAssign, SubAssign}; + +pub use crate::bounds::Bounded; +#[cfg(any(feature = "std", feature = "libm"))] +pub use crate::float::Float; +pub use crate::float::FloatConst; +// pub use real::{FloatCore, Real}; // NOTE: Don't do this, it breaks `use num_traits::*;`. +pub use crate::cast::{cast, AsPrimitive, FromPrimitive, NumCast, ToPrimitive}; +pub use crate::identities::{one, zero, ConstOne, ConstZero, One, Zero}; +pub use crate::int::PrimInt; +pub use crate::ops::bytes::{FromBytes, ToBytes}; +pub use crate::ops::checked::{ + CheckedAdd, CheckedDiv, CheckedMul, CheckedNeg, CheckedRem, CheckedShl, CheckedShr, CheckedSub, +}; +pub use crate::ops::euclid::{CheckedEuclid, Euclid}; +pub use crate::ops::inv::Inv; +pub use crate::ops::mul_add::{MulAdd, MulAddAssign}; +pub use crate::ops::saturating::{Saturating, SaturatingAdd, SaturatingMul, SaturatingSub}; +pub use crate::ops::wrapping::{ + WrappingAdd, WrappingMul, WrappingNeg, WrappingShl, WrappingShr, WrappingSub, +}; +pub use crate::pow::{checked_pow, pow, Pow}; +pub use crate::sign::{abs, abs_sub, signum, Signed, Unsigned}; + +#[macro_use] +mod macros; + +pub mod bounds; +pub mod cast; +pub mod float; +pub mod identities; +pub mod int; +pub mod ops; +pub mod pow; +pub mod real; +pub mod sign; + +/// The base trait for numeric types, covering `0` and `1` values, +/// comparisons, basic numeric operations, and string conversion. +pub trait Num: PartialEq + Zero + One + NumOps { + type FromStrRadixErr; + + /// Convert from a string and radix (typically `2..=36`). + /// + /// # Examples + /// + /// ```rust + /// use num_traits::Num; + /// + /// let result = ::from_str_radix("27", 10); + /// assert_eq!(result, Ok(27)); + /// + /// let result = ::from_str_radix("foo", 10); + /// assert!(result.is_err()); + /// ``` + /// + /// # Supported radices + /// + /// The exact range of supported radices is at the discretion of each type implementation. For + /// primitive integers, this is implemented by the inherent `from_str_radix` methods in the + /// standard library, which **panic** if the radix is not in the range from 2 to 36. The + /// implementation in this crate for primitive floats is similar. + /// + /// For third-party types, it is suggested that implementations should follow suit and at least + /// accept `2..=36` without panicking, but an `Err` may be returned for any unsupported radix. + /// It's possible that a type might not even support the common radix 10, nor any, if string + /// parsing doesn't make sense for that type. + fn from_str_radix(str: &str, radix: u32) -> Result; +} + +/// Generic trait for types implementing basic numeric operations +/// +/// This is automatically implemented for types which implement the operators. +pub trait NumOps: + Add + + Sub + + Mul + + Div + + Rem +{ +} + +impl NumOps for T where + T: Add + + Sub + + Mul + + Div + + Rem +{ +} + +/// The trait for `Num` types which also implement numeric operations taking +/// the second operand by reference. +/// +/// This is automatically implemented for types which implement the operators. +pub trait NumRef: Num + for<'r> NumOps<&'r Self> {} +impl NumRef for T where T: Num + for<'r> NumOps<&'r T> {} + +/// The trait for `Num` references which implement numeric operations, taking the +/// second operand either by value or by reference. +/// +/// This is automatically implemented for all types which implement the operators. It covers +/// every type implementing the operations though, regardless of it being a reference or +/// related to `Num`. +pub trait RefNum: NumOps + for<'r> NumOps<&'r Base, Base> {} +impl RefNum for T where T: NumOps + for<'r> NumOps<&'r Base, Base> {} + +/// Generic trait for types implementing numeric assignment operators (like `+=`). +/// +/// This is automatically implemented for types which implement the operators. +pub trait NumAssignOps: + AddAssign + SubAssign + MulAssign + DivAssign + RemAssign +{ +} + +impl NumAssignOps for T where + T: AddAssign + SubAssign + MulAssign + DivAssign + RemAssign +{ +} + +/// The trait for `Num` types which also implement assignment operators. +/// +/// This is automatically implemented for types which implement the operators. +pub trait NumAssign: Num + NumAssignOps {} +impl NumAssign for T where T: Num + NumAssignOps {} + +/// The trait for `NumAssign` types which also implement assignment operations +/// taking the second operand by reference. +/// +/// This is automatically implemented for types which implement the operators. +pub trait NumAssignRef: NumAssign + for<'r> NumAssignOps<&'r Self> {} +impl NumAssignRef for T where T: NumAssign + for<'r> NumAssignOps<&'r T> {} + +macro_rules! int_trait_impl { + ($name:ident for $($t:ty)*) => ($( + impl $name for $t { + type FromStrRadixErr = ::core::num::ParseIntError; + #[inline] + fn from_str_radix(s: &str, radix: u32) + -> Result + { + <$t>::from_str_radix(s, radix) + } + } + )*) +} +int_trait_impl!(Num for usize u8 u16 u32 u64 u128); +int_trait_impl!(Num for isize i8 i16 i32 i64 i128); + +impl Num for Wrapping +where + Wrapping: NumOps, +{ + type FromStrRadixErr = T::FromStrRadixErr; + fn from_str_radix(str: &str, radix: u32) -> Result { + T::from_str_radix(str, radix).map(Wrapping) + } +} + +#[derive(Debug)] +pub enum FloatErrorKind { + Empty, + Invalid, +} +// FIXME: core::num::ParseFloatError is stable in 1.0, but opaque to us, +// so there's not really any way for us to reuse it. +#[derive(Debug)] +pub struct ParseFloatError { + pub kind: FloatErrorKind, +} + +impl fmt::Display for ParseFloatError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let description = match self.kind { + FloatErrorKind::Empty => "cannot parse float from empty string", + FloatErrorKind::Invalid => "invalid float literal", + }; + + description.fmt(f) + } +} + +fn str_to_ascii_lower_eq_str(a: &str, b: &str) -> bool { + a.len() == b.len() + && a.bytes().zip(b.bytes()).all(|(a, b)| { + let a_to_ascii_lower = a | (((b'A' <= a && a <= b'Z') as u8) << 5); + a_to_ascii_lower == b + }) +} + +// FIXME: The standard library from_str_radix on floats was deprecated, so we're stuck +// with this implementation ourselves until we want to make a breaking change. +// (would have to drop it from `Num` though) +macro_rules! float_trait_impl { + ($name:ident for $($t:ident)*) => ($( + impl $name for $t { + type FromStrRadixErr = ParseFloatError; + + fn from_str_radix(src: &str, radix: u32) + -> Result + { + use self::FloatErrorKind::*; + use self::ParseFloatError as PFE; + + // Special case radix 10 to use more accurate standard library implementation + if radix == 10 { + return src.parse().map_err(|_| PFE { + kind: if src.is_empty() { Empty } else { Invalid }, + }); + } + + // Special values + if str_to_ascii_lower_eq_str(src, "inf") + || str_to_ascii_lower_eq_str(src, "infinity") + { + return Ok(core::$t::INFINITY); + } else if str_to_ascii_lower_eq_str(src, "-inf") + || str_to_ascii_lower_eq_str(src, "-infinity") + { + return Ok(core::$t::NEG_INFINITY); + } else if str_to_ascii_lower_eq_str(src, "nan") { + return Ok(core::$t::NAN); + } else if str_to_ascii_lower_eq_str(src, "-nan") { + return Ok(-core::$t::NAN); + } + + fn slice_shift_char(src: &str) -> Option<(char, &str)> { + let mut chars = src.chars(); + Some((chars.next()?, chars.as_str())) + } + + let (is_positive, src) = match slice_shift_char(src) { + None => return Err(PFE { kind: Empty }), + Some(('-', "")) => return Err(PFE { kind: Empty }), + Some(('-', src)) => (false, src), + Some((_, _)) => (true, src), + }; + + // The significand to accumulate + let mut sig = if is_positive { 0.0 } else { -0.0 }; + // Necessary to detect overflow + let mut prev_sig = sig; + let mut cs = src.chars().enumerate(); + // Exponent prefix and exponent index offset + let mut exp_info = None::<(char, usize)>; + + // Parse the integer part of the significand + for (i, c) in cs.by_ref() { + match c.to_digit(radix) { + Some(digit) => { + // shift significand one digit left + sig *= radix as $t; + + // add/subtract current digit depending on sign + if is_positive { + sig += (digit as isize) as $t; + } else { + sig -= (digit as isize) as $t; + } + + // Detect overflow by comparing to last value, except + // if we've not seen any non-zero digits. + if prev_sig != 0.0 { + if is_positive && sig <= prev_sig + { return Ok(core::$t::INFINITY); } + if !is_positive && sig >= prev_sig + { return Ok(core::$t::NEG_INFINITY); } + + // Detect overflow by reversing the shift-and-add process + if is_positive && (prev_sig != (sig - digit as $t) / radix as $t) + { return Ok(core::$t::INFINITY); } + if !is_positive && (prev_sig != (sig + digit as $t) / radix as $t) + { return Ok(core::$t::NEG_INFINITY); } + } + prev_sig = sig; + }, + None => match c { + 'e' | 'E' | 'p' | 'P' => { + exp_info = Some((c, i + 1)); + break; // start of exponent + }, + '.' => { + break; // start of fractional part + }, + _ => { + return Err(PFE { kind: Invalid }); + }, + }, + } + } + + // If we are not yet at the exponent parse the fractional + // part of the significand + if exp_info.is_none() { + let mut power = 1.0; + for (i, c) in cs.by_ref() { + match c.to_digit(radix) { + Some(digit) => { + // Decrease power one order of magnitude + power /= radix as $t; + // add/subtract current digit depending on sign + sig = if is_positive { + sig + (digit as $t) * power + } else { + sig - (digit as $t) * power + }; + // Detect overflow by comparing to last value + if is_positive && sig < prev_sig + { return Ok(core::$t::INFINITY); } + if !is_positive && sig > prev_sig + { return Ok(core::$t::NEG_INFINITY); } + prev_sig = sig; + }, + None => match c { + 'e' | 'E' | 'p' | 'P' => { + exp_info = Some((c, i + 1)); + break; // start of exponent + }, + _ => { + return Err(PFE { kind: Invalid }); + }, + }, + } + } + } + + // Parse and calculate the exponent + let exp = match exp_info { + Some((c, offset)) => { + let base = match c { + 'E' | 'e' if radix == 10 => 10.0, + 'P' | 'p' if radix == 16 => 2.0, + _ => return Err(PFE { kind: Invalid }), + }; + + // Parse the exponent as decimal integer + let src = &src[offset..]; + let (is_positive, exp) = match slice_shift_char(src) { + Some(('-', src)) => (false, src.parse::()), + Some(('+', src)) => (true, src.parse::()), + Some((_, _)) => (true, src.parse::()), + None => return Err(PFE { kind: Invalid }), + }; + + #[cfg(feature = "std")] + fn pow(base: $t, exp: usize) -> $t { + Float::powi(base, exp as i32) + } + // otherwise uses the generic `pow` from the root + + match (is_positive, exp) { + (true, Ok(exp)) => pow(base, exp), + (false, Ok(exp)) => 1.0 / pow(base, exp), + (_, Err(_)) => return Err(PFE { kind: Invalid }), + } + }, + None => 1.0, // no exponent + }; + + Ok(sig * exp) + } + } + )*) +} +float_trait_impl!(Num for f32 f64); + +/// A value bounded by a minimum and a maximum +/// +/// If input is less than min then this returns min. +/// If input is greater than max then this returns max. +/// Otherwise this returns input. +/// +/// **Panics** in debug mode if `!(min <= max)`. +#[inline] +pub fn clamp(input: T, min: T, max: T) -> T { + debug_assert!(min <= max, "min must be less than or equal to max"); + if input < min { + min + } else if input > max { + max + } else { + input + } +} + +/// A value bounded by a minimum value +/// +/// If input is less than min then this returns min. +/// Otherwise this returns input. +/// `clamp_min(std::f32::NAN, 1.0)` preserves `NAN` different from `f32::min(std::f32::NAN, 1.0)`. +/// +/// **Panics** in debug mode if `!(min == min)`. (This occurs if `min` is `NAN`.) +#[inline] +#[allow(clippy::eq_op)] +pub fn clamp_min(input: T, min: T) -> T { + debug_assert!(min == min, "min must not be NAN"); + if input < min { + min + } else { + input + } +} + +/// A value bounded by a maximum value +/// +/// If input is greater than max then this returns max. +/// Otherwise this returns input. +/// `clamp_max(std::f32::NAN, 1.0)` preserves `NAN` different from `f32::max(std::f32::NAN, 1.0)`. +/// +/// **Panics** in debug mode if `!(max == max)`. (This occurs if `max` is `NAN`.) +#[inline] +#[allow(clippy::eq_op)] +pub fn clamp_max(input: T, max: T) -> T { + debug_assert!(max == max, "max must not be NAN"); + if input > max { + max + } else { + input + } +} + +#[test] +fn clamp_test() { + // Int test + assert_eq!(1, clamp(1, -1, 2)); + assert_eq!(-1, clamp(-2, -1, 2)); + assert_eq!(2, clamp(3, -1, 2)); + assert_eq!(1, clamp_min(1, -1)); + assert_eq!(-1, clamp_min(-2, -1)); + assert_eq!(-1, clamp_max(1, -1)); + assert_eq!(-2, clamp_max(-2, -1)); + + // Float test + assert_eq!(1.0, clamp(1.0, -1.0, 2.0)); + assert_eq!(-1.0, clamp(-2.0, -1.0, 2.0)); + assert_eq!(2.0, clamp(3.0, -1.0, 2.0)); + assert_eq!(1.0, clamp_min(1.0, -1.0)); + assert_eq!(-1.0, clamp_min(-2.0, -1.0)); + assert_eq!(-1.0, clamp_max(1.0, -1.0)); + assert_eq!(-2.0, clamp_max(-2.0, -1.0)); + assert!(clamp(::core::f32::NAN, -1.0, 1.0).is_nan()); + assert!(clamp_min(::core::f32::NAN, 1.0).is_nan()); + assert!(clamp_max(::core::f32::NAN, 1.0).is_nan()); +} + +#[test] +#[should_panic] +#[cfg(debug_assertions)] +fn clamp_nan_min() { + clamp(0., ::core::f32::NAN, 1.); +} + +#[test] +#[should_panic] +#[cfg(debug_assertions)] +fn clamp_nan_max() { + clamp(0., -1., ::core::f32::NAN); +} + +#[test] +#[should_panic] +#[cfg(debug_assertions)] +fn clamp_nan_min_max() { + clamp(0., ::core::f32::NAN, ::core::f32::NAN); +} + +#[test] +#[should_panic] +#[cfg(debug_assertions)] +fn clamp_min_nan_min() { + clamp_min(0., ::core::f32::NAN); +} + +#[test] +#[should_panic] +#[cfg(debug_assertions)] +fn clamp_max_nan_max() { + clamp_max(0., ::core::f32::NAN); +} + +#[test] +fn from_str_radix_unwrap() { + // The Result error must impl Debug to allow unwrap() + + let i: i32 = Num::from_str_radix("0", 10).unwrap(); + assert_eq!(i, 0); + + let f: f32 = Num::from_str_radix("0.0", 10).unwrap(); + assert_eq!(f, 0.0); +} + +#[test] +fn from_str_radix_multi_byte_fail() { + // Ensure parsing doesn't panic, even on invalid sign characters + assert!(f32::from_str_radix("™0.2", 10).is_err()); + + // Even when parsing the exponent sign + assert!(f32::from_str_radix("0.2E™1", 10).is_err()); +} + +#[test] +fn from_str_radix_ignore_case() { + assert_eq!( + f32::from_str_radix("InF", 16).unwrap(), + ::core::f32::INFINITY + ); + assert_eq!( + f32::from_str_radix("InfinitY", 16).unwrap(), + ::core::f32::INFINITY + ); + assert_eq!( + f32::from_str_radix("-InF", 8).unwrap(), + ::core::f32::NEG_INFINITY + ); + assert_eq!( + f32::from_str_radix("-InfinitY", 8).unwrap(), + ::core::f32::NEG_INFINITY + ); + assert!(f32::from_str_radix("nAn", 4).unwrap().is_nan()); + assert!(f32::from_str_radix("-nAn", 4).unwrap().is_nan()); +} + +#[test] +fn wrapping_is_num() { + fn require_num(_: &T) {} + require_num(&Wrapping(42_u32)); + require_num(&Wrapping(-42)); +} + +#[test] +fn wrapping_from_str_radix() { + macro_rules! test_wrapping_from_str_radix { + ($($t:ty)+) => { + $( + for &(s, r) in &[("42", 10), ("42", 2), ("-13.0", 10), ("foo", 10)] { + let w = Wrapping::<$t>::from_str_radix(s, r).map(|w| w.0); + assert_eq!(w, <$t as Num>::from_str_radix(s, r)); + } + )+ + }; + } + + test_wrapping_from_str_radix!(usize u8 u16 u32 u64 isize i8 i16 i32 i64); +} + +#[test] +fn check_num_ops() { + fn compute(x: T, y: T) -> T { + x * y / y % y + y - y + } + assert_eq!(compute(1, 2), 1) +} + +#[test] +fn check_numref_ops() { + fn compute(x: T, y: &T) -> T { + x * y / y % y + y - y + } + assert_eq!(compute(1, &2), 1) +} + +#[test] +fn check_refnum_ops() { + fn compute(x: &T, y: T) -> T + where + for<'a> &'a T: RefNum, + { + &(&(&(&(x * y) / y) % y) + y) - y + } + assert_eq!(compute(&1, 2), 1) +} + +#[test] +fn check_refref_ops() { + fn compute(x: &T, y: &T) -> T + where + for<'a> &'a T: RefNum, + { + &(&(&(&(x * y) / y) % y) + y) - y + } + assert_eq!(compute(&1, &2), 1) +} + +#[test] +fn check_numassign_ops() { + fn compute(mut x: T, y: T) -> T { + x *= y; + x /= y; + x %= y; + x += y; + x -= y; + x + } + assert_eq!(compute(1, 2), 1) +} + +#[test] +fn check_numassignref_ops() { + fn compute(mut x: T, y: &T) -> T { + x *= y; + x /= y; + x %= y; + x += y; + x -= y; + x + } + assert_eq!(compute(1, &2), 1) +} diff --git a/third_party/crates/num-traits-0.2.19/src/macros.rs b/third_party/crates/num-traits-0.2.19/src/macros.rs new file mode 100644 index 0000000..b97758e --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/macros.rs @@ -0,0 +1,44 @@ +// not all are used in all features configurations +#![allow(unused)] + +/// Forward a method to an inherent method or a base trait method. +macro_rules! forward { + ($( Self :: $method:ident ( self $( , $arg:ident : $ty:ty )* ) -> $ret:ty ; )*) + => {$( + #[inline] + fn $method(self $( , $arg : $ty )* ) -> $ret { + Self::$method(self $( , $arg )* ) + } + )*}; + ($( $base:ident :: $method:ident ( self $( , $arg:ident : $ty:ty )* ) -> $ret:ty ; )*) + => {$( + #[inline] + fn $method(self $( , $arg : $ty )* ) -> $ret { + ::$method(self $( , $arg )* ) + } + )*}; + ($( $base:ident :: $method:ident ( $( $arg:ident : $ty:ty ),* ) -> $ret:ty ; )*) + => {$( + #[inline] + fn $method( $( $arg : $ty ),* ) -> $ret { + ::$method( $( $arg ),* ) + } + )*}; + ($( $imp:path as $method:ident ( self $( , $arg:ident : $ty:ty )* ) -> $ret:ty ; )*) + => {$( + #[inline] + fn $method(self $( , $arg : $ty )* ) -> $ret { + $imp(self $( , $arg )* ) + } + )*}; +} + +macro_rules! constant { + ($( $method:ident () -> $ret:expr ; )*) + => {$( + #[inline] + fn $method() -> Self { + $ret + } + )*}; +} diff --git a/third_party/crates/num-traits-0.2.19/src/ops/bytes.rs b/third_party/crates/num-traits-0.2.19/src/ops/bytes.rs new file mode 100644 index 0000000..f6a8030 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/bytes.rs @@ -0,0 +1,317 @@ +use core::borrow::{Borrow, BorrowMut}; +use core::cmp::{Eq, Ord, PartialEq, PartialOrd}; +use core::fmt::Debug; +use core::hash::Hash; + +pub trait NumBytes: + Debug + + AsRef<[u8]> + + AsMut<[u8]> + + PartialEq + + Eq + + PartialOrd + + Ord + + Hash + + Borrow<[u8]> + + BorrowMut<[u8]> +{ +} + +impl NumBytes for T where + T: Debug + + AsRef<[u8]> + + AsMut<[u8]> + + PartialEq + + Eq + + PartialOrd + + Ord + + Hash + + Borrow<[u8]> + + BorrowMut<[u8]> + + ?Sized +{ +} + +pub trait ToBytes { + type Bytes: NumBytes; + + /// Return the memory representation of this number as a byte array in big-endian byte order. + /// + /// # Examples + /// + /// ``` + /// use num_traits::ToBytes; + /// + /// let bytes = ToBytes::to_be_bytes(&0x12345678u32); + /// assert_eq!(bytes, [0x12, 0x34, 0x56, 0x78]); + /// ``` + fn to_be_bytes(&self) -> Self::Bytes; + + /// Return the memory representation of this number as a byte array in little-endian byte order. + /// + /// # Examples + /// + /// ``` + /// use num_traits::ToBytes; + /// + /// let bytes = ToBytes::to_le_bytes(&0x12345678u32); + /// assert_eq!(bytes, [0x78, 0x56, 0x34, 0x12]); + /// ``` + fn to_le_bytes(&self) -> Self::Bytes; + + /// Return the memory representation of this number as a byte array in native byte order. + /// + /// As the target platform's native endianness is used, + /// portable code should use [`to_be_bytes`] or [`to_le_bytes`], as appropriate, instead. + /// + /// [`to_be_bytes`]: #method.to_be_bytes + /// [`to_le_bytes`]: #method.to_le_bytes + /// + /// # Examples + /// + /// ``` + /// use num_traits::ToBytes; + /// + /// #[cfg(target_endian = "big")] + /// let expected = [0x12, 0x34, 0x56, 0x78]; + /// + /// #[cfg(target_endian = "little")] + /// let expected = [0x78, 0x56, 0x34, 0x12]; + /// + /// let bytes = ToBytes::to_ne_bytes(&0x12345678u32); + /// assert_eq!(bytes, expected) + /// ``` + fn to_ne_bytes(&self) -> Self::Bytes { + #[cfg(target_endian = "big")] + let bytes = self.to_be_bytes(); + #[cfg(target_endian = "little")] + let bytes = self.to_le_bytes(); + bytes + } +} + +pub trait FromBytes: Sized { + type Bytes: NumBytes + ?Sized; + + /// Create a number from its representation as a byte array in big endian. + /// + /// # Examples + /// + /// ``` + /// use num_traits::FromBytes; + /// + /// let value: u32 = FromBytes::from_be_bytes(&[0x12, 0x34, 0x56, 0x78]); + /// assert_eq!(value, 0x12345678); + /// ``` + fn from_be_bytes(bytes: &Self::Bytes) -> Self; + + /// Create a number from its representation as a byte array in little endian. + /// + /// # Examples + /// + /// ``` + /// use num_traits::FromBytes; + /// + /// let value: u32 = FromBytes::from_le_bytes(&[0x78, 0x56, 0x34, 0x12]); + /// assert_eq!(value, 0x12345678); + /// ``` + fn from_le_bytes(bytes: &Self::Bytes) -> Self; + + /// Create a number from its memory representation as a byte array in native endianness. + /// + /// As the target platform's native endianness is used, + /// portable code likely wants to use [`from_be_bytes`] or [`from_le_bytes`], as appropriate instead. + /// + /// [`from_be_bytes`]: #method.from_be_bytes + /// [`from_le_bytes`]: #method.from_le_bytes + /// + /// # Examples + /// + /// ``` + /// use num_traits::FromBytes; + /// + /// #[cfg(target_endian = "big")] + /// let bytes = [0x12, 0x34, 0x56, 0x78]; + /// + /// #[cfg(target_endian = "little")] + /// let bytes = [0x78, 0x56, 0x34, 0x12]; + /// + /// let value: u32 = FromBytes::from_ne_bytes(&bytes); + /// assert_eq!(value, 0x12345678) + /// ``` + fn from_ne_bytes(bytes: &Self::Bytes) -> Self { + #[cfg(target_endian = "big")] + let this = Self::from_be_bytes(bytes); + #[cfg(target_endian = "little")] + let this = Self::from_le_bytes(bytes); + this + } +} + +macro_rules! float_to_from_bytes_impl { + ($T:ty, $L:expr) => { + impl ToBytes for $T { + type Bytes = [u8; $L]; + + #[inline] + fn to_be_bytes(&self) -> Self::Bytes { + <$T>::to_be_bytes(*self) + } + + #[inline] + fn to_le_bytes(&self) -> Self::Bytes { + <$T>::to_le_bytes(*self) + } + + #[inline] + fn to_ne_bytes(&self) -> Self::Bytes { + <$T>::to_ne_bytes(*self) + } + } + + impl FromBytes for $T { + type Bytes = [u8; $L]; + + #[inline] + fn from_be_bytes(bytes: &Self::Bytes) -> Self { + <$T>::from_be_bytes(*bytes) + } + + #[inline] + fn from_le_bytes(bytes: &Self::Bytes) -> Self { + <$T>::from_le_bytes(*bytes) + } + + #[inline] + fn from_ne_bytes(bytes: &Self::Bytes) -> Self { + <$T>::from_ne_bytes(*bytes) + } + } + }; +} + +macro_rules! int_to_from_bytes_impl { + ($T:ty, $L:expr) => { + impl ToBytes for $T { + type Bytes = [u8; $L]; + + #[inline] + fn to_be_bytes(&self) -> Self::Bytes { + <$T>::to_be_bytes(*self) + } + + #[inline] + fn to_le_bytes(&self) -> Self::Bytes { + <$T>::to_le_bytes(*self) + } + + #[inline] + fn to_ne_bytes(&self) -> Self::Bytes { + <$T>::to_ne_bytes(*self) + } + } + + impl FromBytes for $T { + type Bytes = [u8; $L]; + + #[inline] + fn from_be_bytes(bytes: &Self::Bytes) -> Self { + <$T>::from_be_bytes(*bytes) + } + + #[inline] + fn from_le_bytes(bytes: &Self::Bytes) -> Self { + <$T>::from_le_bytes(*bytes) + } + + #[inline] + fn from_ne_bytes(bytes: &Self::Bytes) -> Self { + <$T>::from_ne_bytes(*bytes) + } + } + }; +} + +int_to_from_bytes_impl!(u8, 1); +int_to_from_bytes_impl!(u16, 2); +int_to_from_bytes_impl!(u32, 4); +int_to_from_bytes_impl!(u64, 8); +int_to_from_bytes_impl!(u128, 16); +#[cfg(target_pointer_width = "64")] +int_to_from_bytes_impl!(usize, 8); +#[cfg(target_pointer_width = "32")] +int_to_from_bytes_impl!(usize, 4); + +int_to_from_bytes_impl!(i8, 1); +int_to_from_bytes_impl!(i16, 2); +int_to_from_bytes_impl!(i32, 4); +int_to_from_bytes_impl!(i64, 8); +int_to_from_bytes_impl!(i128, 16); +#[cfg(target_pointer_width = "64")] +int_to_from_bytes_impl!(isize, 8); +#[cfg(target_pointer_width = "32")] +int_to_from_bytes_impl!(isize, 4); + +float_to_from_bytes_impl!(f32, 4); +float_to_from_bytes_impl!(f64, 8); + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! check_to_from_bytes { + ($( $ty:ty )+) => {$({ + let n = 1; + let be = <$ty as ToBytes>::to_be_bytes(&n); + let le = <$ty as ToBytes>::to_le_bytes(&n); + let ne = <$ty as ToBytes>::to_ne_bytes(&n); + + assert_eq!(*be.last().unwrap(), 1); + assert_eq!(*le.first().unwrap(), 1); + if cfg!(target_endian = "big") { + assert_eq!(*ne.last().unwrap(), 1); + } else { + assert_eq!(*ne.first().unwrap(), 1); + } + + assert_eq!(<$ty as FromBytes>::from_be_bytes(&be), n); + assert_eq!(<$ty as FromBytes>::from_le_bytes(&le), n); + if cfg!(target_endian = "big") { + assert_eq!(<$ty as FromBytes>::from_ne_bytes(&be), n); + } else { + assert_eq!(<$ty as FromBytes>::from_ne_bytes(&le), n); + } + })+} + } + + #[test] + fn convert_between_int_and_bytes() { + check_to_from_bytes!(u8 u16 u32 u64 u128 usize); + check_to_from_bytes!(i8 i16 i32 i64 i128 isize); + } + + #[test] + fn convert_between_float_and_bytes() { + macro_rules! check_to_from_bytes { + ($( $ty:ty )+) => {$( + let n: $ty = 3.14; + + let be = <$ty as ToBytes>::to_be_bytes(&n); + let le = <$ty as ToBytes>::to_le_bytes(&n); + let ne = <$ty as ToBytes>::to_ne_bytes(&n); + + assert_eq!(<$ty as FromBytes>::from_be_bytes(&be), n); + assert_eq!(<$ty as FromBytes>::from_le_bytes(&le), n); + if cfg!(target_endian = "big") { + assert_eq!(ne, be); + assert_eq!(<$ty as FromBytes>::from_ne_bytes(&be), n); + } else { + assert_eq!(ne, le); + assert_eq!(<$ty as FromBytes>::from_ne_bytes(&le), n); + } + )+} + } + + check_to_from_bytes!(f32 f64); + } +} diff --git a/third_party/crates/num-traits-0.2.19/src/ops/checked.rs b/third_party/crates/num-traits-0.2.19/src/ops/checked.rs new file mode 100644 index 0000000..da1eb3e --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/checked.rs @@ -0,0 +1,261 @@ +use core::ops::{Add, Div, Mul, Rem, Shl, Shr, Sub}; + +/// Performs addition that returns `None` instead of wrapping around on +/// overflow. +pub trait CheckedAdd: Sized + Add { + /// Adds two numbers, checking for overflow. If overflow happens, `None` is + /// returned. + fn checked_add(&self, v: &Self) -> Option; +} + +macro_rules! checked_impl { + ($trait_name:ident, $method:ident, $t:ty) => { + impl $trait_name for $t { + #[inline] + fn $method(&self, v: &$t) -> Option<$t> { + <$t>::$method(*self, *v) + } + } + }; +} + +checked_impl!(CheckedAdd, checked_add, u8); +checked_impl!(CheckedAdd, checked_add, u16); +checked_impl!(CheckedAdd, checked_add, u32); +checked_impl!(CheckedAdd, checked_add, u64); +checked_impl!(CheckedAdd, checked_add, usize); +checked_impl!(CheckedAdd, checked_add, u128); + +checked_impl!(CheckedAdd, checked_add, i8); +checked_impl!(CheckedAdd, checked_add, i16); +checked_impl!(CheckedAdd, checked_add, i32); +checked_impl!(CheckedAdd, checked_add, i64); +checked_impl!(CheckedAdd, checked_add, isize); +checked_impl!(CheckedAdd, checked_add, i128); + +/// Performs subtraction that returns `None` instead of wrapping around on underflow. +pub trait CheckedSub: Sized + Sub { + /// Subtracts two numbers, checking for underflow. If underflow happens, + /// `None` is returned. + fn checked_sub(&self, v: &Self) -> Option; +} + +checked_impl!(CheckedSub, checked_sub, u8); +checked_impl!(CheckedSub, checked_sub, u16); +checked_impl!(CheckedSub, checked_sub, u32); +checked_impl!(CheckedSub, checked_sub, u64); +checked_impl!(CheckedSub, checked_sub, usize); +checked_impl!(CheckedSub, checked_sub, u128); + +checked_impl!(CheckedSub, checked_sub, i8); +checked_impl!(CheckedSub, checked_sub, i16); +checked_impl!(CheckedSub, checked_sub, i32); +checked_impl!(CheckedSub, checked_sub, i64); +checked_impl!(CheckedSub, checked_sub, isize); +checked_impl!(CheckedSub, checked_sub, i128); + +/// Performs multiplication that returns `None` instead of wrapping around on underflow or +/// overflow. +pub trait CheckedMul: Sized + Mul { + /// Multiplies two numbers, checking for underflow or overflow. If underflow + /// or overflow happens, `None` is returned. + fn checked_mul(&self, v: &Self) -> Option; +} + +checked_impl!(CheckedMul, checked_mul, u8); +checked_impl!(CheckedMul, checked_mul, u16); +checked_impl!(CheckedMul, checked_mul, u32); +checked_impl!(CheckedMul, checked_mul, u64); +checked_impl!(CheckedMul, checked_mul, usize); +checked_impl!(CheckedMul, checked_mul, u128); + +checked_impl!(CheckedMul, checked_mul, i8); +checked_impl!(CheckedMul, checked_mul, i16); +checked_impl!(CheckedMul, checked_mul, i32); +checked_impl!(CheckedMul, checked_mul, i64); +checked_impl!(CheckedMul, checked_mul, isize); +checked_impl!(CheckedMul, checked_mul, i128); + +/// Performs division that returns `None` instead of panicking on division by zero and instead of +/// wrapping around on underflow and overflow. +pub trait CheckedDiv: Sized + Div { + /// Divides two numbers, checking for underflow, overflow and division by + /// zero. If any of that happens, `None` is returned. + fn checked_div(&self, v: &Self) -> Option; +} + +checked_impl!(CheckedDiv, checked_div, u8); +checked_impl!(CheckedDiv, checked_div, u16); +checked_impl!(CheckedDiv, checked_div, u32); +checked_impl!(CheckedDiv, checked_div, u64); +checked_impl!(CheckedDiv, checked_div, usize); +checked_impl!(CheckedDiv, checked_div, u128); + +checked_impl!(CheckedDiv, checked_div, i8); +checked_impl!(CheckedDiv, checked_div, i16); +checked_impl!(CheckedDiv, checked_div, i32); +checked_impl!(CheckedDiv, checked_div, i64); +checked_impl!(CheckedDiv, checked_div, isize); +checked_impl!(CheckedDiv, checked_div, i128); + +/// Performs an integral remainder that returns `None` instead of panicking on division by zero and +/// instead of wrapping around on underflow and overflow. +pub trait CheckedRem: Sized + Rem { + /// Finds the remainder of dividing two numbers, checking for underflow, overflow and division + /// by zero. If any of that happens, `None` is returned. + /// + /// # Examples + /// + /// ``` + /// use num_traits::CheckedRem; + /// use std::i32::MIN; + /// + /// assert_eq!(CheckedRem::checked_rem(&10, &7), Some(3)); + /// assert_eq!(CheckedRem::checked_rem(&10, &-7), Some(3)); + /// assert_eq!(CheckedRem::checked_rem(&-10, &7), Some(-3)); + /// assert_eq!(CheckedRem::checked_rem(&-10, &-7), Some(-3)); + /// + /// assert_eq!(CheckedRem::checked_rem(&10, &0), None); + /// + /// assert_eq!(CheckedRem::checked_rem(&MIN, &1), Some(0)); + /// assert_eq!(CheckedRem::checked_rem(&MIN, &-1), None); + /// ``` + fn checked_rem(&self, v: &Self) -> Option; +} + +checked_impl!(CheckedRem, checked_rem, u8); +checked_impl!(CheckedRem, checked_rem, u16); +checked_impl!(CheckedRem, checked_rem, u32); +checked_impl!(CheckedRem, checked_rem, u64); +checked_impl!(CheckedRem, checked_rem, usize); +checked_impl!(CheckedRem, checked_rem, u128); + +checked_impl!(CheckedRem, checked_rem, i8); +checked_impl!(CheckedRem, checked_rem, i16); +checked_impl!(CheckedRem, checked_rem, i32); +checked_impl!(CheckedRem, checked_rem, i64); +checked_impl!(CheckedRem, checked_rem, isize); +checked_impl!(CheckedRem, checked_rem, i128); + +macro_rules! checked_impl_unary { + ($trait_name:ident, $method:ident, $t:ty) => { + impl $trait_name for $t { + #[inline] + fn $method(&self) -> Option<$t> { + <$t>::$method(*self) + } + } + }; +} + +/// Performs negation that returns `None` if the result can't be represented. +pub trait CheckedNeg: Sized { + /// Negates a number, returning `None` for results that can't be represented, like signed `MIN` + /// values that can't be positive, or non-zero unsigned values that can't be negative. + /// + /// # Examples + /// + /// ``` + /// use num_traits::CheckedNeg; + /// use std::i32::MIN; + /// + /// assert_eq!(CheckedNeg::checked_neg(&1_i32), Some(-1)); + /// assert_eq!(CheckedNeg::checked_neg(&-1_i32), Some(1)); + /// assert_eq!(CheckedNeg::checked_neg(&MIN), None); + /// + /// assert_eq!(CheckedNeg::checked_neg(&0_u32), Some(0)); + /// assert_eq!(CheckedNeg::checked_neg(&1_u32), None); + /// ``` + fn checked_neg(&self) -> Option; +} + +checked_impl_unary!(CheckedNeg, checked_neg, u8); +checked_impl_unary!(CheckedNeg, checked_neg, u16); +checked_impl_unary!(CheckedNeg, checked_neg, u32); +checked_impl_unary!(CheckedNeg, checked_neg, u64); +checked_impl_unary!(CheckedNeg, checked_neg, usize); +checked_impl_unary!(CheckedNeg, checked_neg, u128); + +checked_impl_unary!(CheckedNeg, checked_neg, i8); +checked_impl_unary!(CheckedNeg, checked_neg, i16); +checked_impl_unary!(CheckedNeg, checked_neg, i32); +checked_impl_unary!(CheckedNeg, checked_neg, i64); +checked_impl_unary!(CheckedNeg, checked_neg, isize); +checked_impl_unary!(CheckedNeg, checked_neg, i128); + +/// Performs a left shift that returns `None` on shifts larger than +/// or equal to the type width. +pub trait CheckedShl: Sized + Shl { + /// Checked shift left. Computes `self << rhs`, returning `None` + /// if `rhs` is larger than or equal to the number of bits in `self`. + /// + /// ``` + /// use num_traits::CheckedShl; + /// + /// let x: u16 = 0x0001; + /// + /// assert_eq!(CheckedShl::checked_shl(&x, 0), Some(0x0001)); + /// assert_eq!(CheckedShl::checked_shl(&x, 1), Some(0x0002)); + /// assert_eq!(CheckedShl::checked_shl(&x, 15), Some(0x8000)); + /// assert_eq!(CheckedShl::checked_shl(&x, 16), None); + /// ``` + fn checked_shl(&self, rhs: u32) -> Option; +} + +macro_rules! checked_shift_impl { + ($trait_name:ident, $method:ident, $t:ty) => { + impl $trait_name for $t { + #[inline] + fn $method(&self, rhs: u32) -> Option<$t> { + <$t>::$method(*self, rhs) + } + } + }; +} + +checked_shift_impl!(CheckedShl, checked_shl, u8); +checked_shift_impl!(CheckedShl, checked_shl, u16); +checked_shift_impl!(CheckedShl, checked_shl, u32); +checked_shift_impl!(CheckedShl, checked_shl, u64); +checked_shift_impl!(CheckedShl, checked_shl, usize); +checked_shift_impl!(CheckedShl, checked_shl, u128); + +checked_shift_impl!(CheckedShl, checked_shl, i8); +checked_shift_impl!(CheckedShl, checked_shl, i16); +checked_shift_impl!(CheckedShl, checked_shl, i32); +checked_shift_impl!(CheckedShl, checked_shl, i64); +checked_shift_impl!(CheckedShl, checked_shl, isize); +checked_shift_impl!(CheckedShl, checked_shl, i128); + +/// Performs a right shift that returns `None` on shifts larger than +/// or equal to the type width. +pub trait CheckedShr: Sized + Shr { + /// Checked shift right. Computes `self >> rhs`, returning `None` + /// if `rhs` is larger than or equal to the number of bits in `self`. + /// + /// ``` + /// use num_traits::CheckedShr; + /// + /// let x: u16 = 0x8000; + /// + /// assert_eq!(CheckedShr::checked_shr(&x, 0), Some(0x8000)); + /// assert_eq!(CheckedShr::checked_shr(&x, 1), Some(0x4000)); + /// assert_eq!(CheckedShr::checked_shr(&x, 15), Some(0x0001)); + /// assert_eq!(CheckedShr::checked_shr(&x, 16), None); + /// ``` + fn checked_shr(&self, rhs: u32) -> Option; +} + +checked_shift_impl!(CheckedShr, checked_shr, u8); +checked_shift_impl!(CheckedShr, checked_shr, u16); +checked_shift_impl!(CheckedShr, checked_shr, u32); +checked_shift_impl!(CheckedShr, checked_shr, u64); +checked_shift_impl!(CheckedShr, checked_shr, usize); +checked_shift_impl!(CheckedShr, checked_shr, u128); + +checked_shift_impl!(CheckedShr, checked_shr, i8); +checked_shift_impl!(CheckedShr, checked_shr, i16); +checked_shift_impl!(CheckedShr, checked_shr, i32); +checked_shift_impl!(CheckedShr, checked_shr, i64); +checked_shift_impl!(CheckedShr, checked_shr, isize); +checked_shift_impl!(CheckedShr, checked_shr, i128); diff --git a/third_party/crates/num-traits-0.2.19/src/ops/euclid.rs b/third_party/crates/num-traits-0.2.19/src/ops/euclid.rs new file mode 100644 index 0000000..fa7b317 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/euclid.rs @@ -0,0 +1,278 @@ +use core::ops::{Div, Rem}; + +pub trait Euclid: Sized + Div + Rem { + /// Calculates Euclidean division, the matching method for `rem_euclid`. + /// + /// This computes the integer `n` such that + /// `self = n * v + self.rem_euclid(v)`. + /// In other words, the result is `self / v` rounded to the integer `n` + /// such that `self >= n * v`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::Euclid; + /// + /// let a: i32 = 7; + /// let b: i32 = 4; + /// assert_eq!(Euclid::div_euclid(&a, &b), 1); // 7 > 4 * 1 + /// assert_eq!(Euclid::div_euclid(&-a, &b), -2); // -7 >= 4 * -2 + /// assert_eq!(Euclid::div_euclid(&a, &-b), -1); // 7 >= -4 * -1 + /// assert_eq!(Euclid::div_euclid(&-a, &-b), 2); // -7 >= -4 * 2 + /// ``` + fn div_euclid(&self, v: &Self) -> Self; + + /// Calculates the least nonnegative remainder of `self (mod v)`. + /// + /// In particular, the return value `r` satisfies `0.0 <= r < v.abs()` in + /// most cases. However, due to a floating point round-off error it can + /// result in `r == v.abs()`, violating the mathematical definition, if + /// `self` is much smaller than `v.abs()` in magnitude and `self < 0.0`. + /// This result is not an element of the function's codomain, but it is the + /// closest floating point number in the real numbers and thus fulfills the + /// property `self == self.div_euclid(v) * v + self.rem_euclid(v)` + /// approximatively. + /// + /// # Examples + /// + /// ``` + /// use num_traits::Euclid; + /// + /// let a: i32 = 7; + /// let b: i32 = 4; + /// assert_eq!(Euclid::rem_euclid(&a, &b), 3); + /// assert_eq!(Euclid::rem_euclid(&-a, &b), 1); + /// assert_eq!(Euclid::rem_euclid(&a, &-b), 3); + /// assert_eq!(Euclid::rem_euclid(&-a, &-b), 1); + /// ``` + fn rem_euclid(&self, v: &Self) -> Self; + + /// Returns both the quotient and remainder from Euclidean division. + /// + /// By default, it internally calls both `Euclid::div_euclid` and `Euclid::rem_euclid`, + /// but it can be overridden in order to implement some optimization. + /// + /// # Examples + /// + /// ``` + /// # use num_traits::Euclid; + /// let x = 5u8; + /// let y = 3u8; + /// + /// let div = Euclid::div_euclid(&x, &y); + /// let rem = Euclid::rem_euclid(&x, &y); + /// + /// assert_eq!((div, rem), Euclid::div_rem_euclid(&x, &y)); + /// ``` + fn div_rem_euclid(&self, v: &Self) -> (Self, Self) { + (self.div_euclid(v), self.rem_euclid(v)) + } +} + +macro_rules! euclid_forward_impl { + ($($t:ty)*) => {$( + impl Euclid for $t { + #[inline] + fn div_euclid(&self, v: &$t) -> Self { + <$t>::div_euclid(*self, *v) + } + + #[inline] + fn rem_euclid(&self, v: &$t) -> Self { + <$t>::rem_euclid(*self, *v) + } + } + )*} +} + +euclid_forward_impl!(isize i8 i16 i32 i64 i128); +euclid_forward_impl!(usize u8 u16 u32 u64 u128); + +#[cfg(feature = "std")] +euclid_forward_impl!(f32 f64); + +#[cfg(not(feature = "std"))] +impl Euclid for f32 { + #[inline] + fn div_euclid(&self, v: &f32) -> f32 { + let q = ::trunc(self / v); + if self % v < 0.0 { + return if *v > 0.0 { q - 1.0 } else { q + 1.0 }; + } + q + } + + #[inline] + fn rem_euclid(&self, v: &f32) -> f32 { + let r = self % v; + if r < 0.0 { + r + ::abs(*v) + } else { + r + } + } +} + +#[cfg(not(feature = "std"))] +impl Euclid for f64 { + #[inline] + fn div_euclid(&self, v: &f64) -> f64 { + let q = ::trunc(self / v); + if self % v < 0.0 { + return if *v > 0.0 { q - 1.0 } else { q + 1.0 }; + } + q + } + + #[inline] + fn rem_euclid(&self, v: &f64) -> f64 { + let r = self % v; + if r < 0.0 { + r + ::abs(*v) + } else { + r + } + } +} + +pub trait CheckedEuclid: Euclid { + /// Performs euclid division that returns `None` instead of panicking on division by zero + /// and instead of wrapping around on underflow and overflow. + fn checked_div_euclid(&self, v: &Self) -> Option; + + /// Finds the euclid remainder of dividing two numbers, checking for underflow, overflow and + /// division by zero. If any of that happens, `None` is returned. + fn checked_rem_euclid(&self, v: &Self) -> Option; + + /// Returns both the quotient and remainder from checked Euclidean division. + /// + /// By default, it internally calls both `CheckedEuclid::checked_div_euclid` and `CheckedEuclid::checked_rem_euclid`, + /// but it can be overridden in order to implement some optimization. + /// # Examples + /// + /// ``` + /// # use num_traits::CheckedEuclid; + /// let x = 5u8; + /// let y = 3u8; + /// + /// let div = CheckedEuclid::checked_div_euclid(&x, &y); + /// let rem = CheckedEuclid::checked_rem_euclid(&x, &y); + /// + /// assert_eq!(Some((div.unwrap(), rem.unwrap())), CheckedEuclid::checked_div_rem_euclid(&x, &y)); + /// ``` + fn checked_div_rem_euclid(&self, v: &Self) -> Option<(Self, Self)> { + Some((self.checked_div_euclid(v)?, self.checked_rem_euclid(v)?)) + } +} + +macro_rules! checked_euclid_forward_impl { + ($($t:ty)*) => {$( + impl CheckedEuclid for $t { + #[inline] + fn checked_div_euclid(&self, v: &$t) -> Option { + <$t>::checked_div_euclid(*self, *v) + } + + #[inline] + fn checked_rem_euclid(&self, v: &$t) -> Option { + <$t>::checked_rem_euclid(*self, *v) + } + } + )*} +} + +checked_euclid_forward_impl!(isize i8 i16 i32 i64 i128); +checked_euclid_forward_impl!(usize u8 u16 u32 u64 u128); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn euclid_unsigned() { + macro_rules! test_euclid { + ($($t:ident)+) => { + $( + { + let x: $t = 10; + let y: $t = 3; + let div = Euclid::div_euclid(&x, &y); + let rem = Euclid::rem_euclid(&x, &y); + assert_eq!(div, 3); + assert_eq!(rem, 1); + assert_eq!((div, rem), Euclid::div_rem_euclid(&x, &y)); + } + )+ + }; + } + + test_euclid!(usize u8 u16 u32 u64); + } + + #[test] + fn euclid_signed() { + macro_rules! test_euclid { + ($($t:ident)+) => { + $( + { + let x: $t = 10; + let y: $t = -3; + assert_eq!(Euclid::div_euclid(&x, &y), -3); + assert_eq!(Euclid::div_euclid(&-x, &y), 4); + assert_eq!(Euclid::rem_euclid(&x, &y), 1); + assert_eq!(Euclid::rem_euclid(&-x, &y), 2); + assert_eq!((Euclid::div_euclid(&x, &y), Euclid::rem_euclid(&x, &y)), Euclid::div_rem_euclid(&x, &y)); + let x: $t = $t::min_value() + 1; + let y: $t = -1; + assert_eq!(Euclid::div_euclid(&x, &y), $t::max_value()); + } + )+ + }; + } + + test_euclid!(isize i8 i16 i32 i64 i128); + } + + #[test] + fn euclid_float() { + macro_rules! test_euclid { + ($($t:ident)+) => { + $( + { + let x: $t = 12.1; + let y: $t = 3.2; + assert!(Euclid::div_euclid(&x, &y) * y + Euclid::rem_euclid(&x, &y) - x + <= 46.4 * <$t as crate::float::FloatCore>::epsilon()); + assert!(Euclid::div_euclid(&x, &-y) * -y + Euclid::rem_euclid(&x, &-y) - x + <= 46.4 * <$t as crate::float::FloatCore>::epsilon()); + assert!(Euclid::div_euclid(&-x, &y) * y + Euclid::rem_euclid(&-x, &y) + x + <= 46.4 * <$t as crate::float::FloatCore>::epsilon()); + assert!(Euclid::div_euclid(&-x, &-y) * -y + Euclid::rem_euclid(&-x, &-y) + x + <= 46.4 * <$t as crate::float::FloatCore>::epsilon()); + assert_eq!((Euclid::div_euclid(&x, &y), Euclid::rem_euclid(&x, &y)), Euclid::div_rem_euclid(&x, &y)); + } + )+ + }; + } + + test_euclid!(f32 f64); + } + + #[test] + fn euclid_checked() { + macro_rules! test_euclid_checked { + ($($t:ident)+) => { + $( + { + assert_eq!(CheckedEuclid::checked_div_euclid(&$t::min_value(), &-1), None); + assert_eq!(CheckedEuclid::checked_rem_euclid(&$t::min_value(), &-1), None); + assert_eq!(CheckedEuclid::checked_div_euclid(&1, &0), None); + assert_eq!(CheckedEuclid::checked_rem_euclid(&1, &0), None); + } + )+ + }; + } + + test_euclid_checked!(isize i8 i16 i32 i64 i128); + } +} diff --git a/third_party/crates/num-traits-0.2.19/src/ops/inv.rs b/third_party/crates/num-traits-0.2.19/src/ops/inv.rs new file mode 100644 index 0000000..7087d09 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/inv.rs @@ -0,0 +1,47 @@ +/// Unary operator for retrieving the multiplicative inverse, or reciprocal, of a value. +pub trait Inv { + /// The result after applying the operator. + type Output; + + /// Returns the multiplicative inverse of `self`. + /// + /// # Examples + /// + /// ``` + /// use std::f64::INFINITY; + /// use num_traits::Inv; + /// + /// assert_eq!(7.0.inv() * 7.0, 1.0); + /// assert_eq!((-0.0).inv(), -INFINITY); + /// ``` + fn inv(self) -> Self::Output; +} + +impl Inv for f32 { + type Output = f32; + #[inline] + fn inv(self) -> f32 { + 1.0 / self + } +} +impl Inv for f64 { + type Output = f64; + #[inline] + fn inv(self) -> f64 { + 1.0 / self + } +} +impl<'a> Inv for &'a f32 { + type Output = f32; + #[inline] + fn inv(self) -> f32 { + 1.0 / *self + } +} +impl<'a> Inv for &'a f64 { + type Output = f64; + #[inline] + fn inv(self) -> f64 { + 1.0 / *self + } +} diff --git a/third_party/crates/num-traits-0.2.19/src/ops/mod.rs b/third_party/crates/num-traits-0.2.19/src/ops/mod.rs new file mode 100644 index 0000000..2128d86 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/mod.rs @@ -0,0 +1,8 @@ +pub mod bytes; +pub mod checked; +pub mod euclid; +pub mod inv; +pub mod mul_add; +pub mod overflowing; +pub mod saturating; +pub mod wrapping; diff --git a/third_party/crates/num-traits-0.2.19/src/ops/mul_add.rs b/third_party/crates/num-traits-0.2.19/src/ops/mul_add.rs new file mode 100644 index 0000000..51beb55 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/mul_add.rs @@ -0,0 +1,149 @@ +/// Fused multiply-add. Computes `(self * a) + b` with only one rounding +/// error, yielding a more accurate result than an unfused multiply-add. +/// +/// Using `mul_add` can be more performant than an unfused multiply-add if +/// the target architecture has a dedicated `fma` CPU instruction. +/// +/// Note that `A` and `B` are `Self` by default, but this is not mandatory. +/// +/// # Example +/// +/// ``` +/// use std::f32; +/// +/// let m = 10.0_f32; +/// let x = 4.0_f32; +/// let b = 60.0_f32; +/// +/// // 100.0 +/// let abs_difference = (m.mul_add(x, b) - (m*x + b)).abs(); +/// +/// assert!(abs_difference <= 100.0 * f32::EPSILON); +/// ``` +pub trait MulAdd { + /// The resulting type after applying the fused multiply-add. + type Output; + + /// Performs the fused multiply-add operation `(self * a) + b` + fn mul_add(self, a: A, b: B) -> Self::Output; +} + +/// The fused multiply-add assignment operation `*self = (*self * a) + b` +pub trait MulAddAssign { + /// Performs the fused multiply-add assignment operation `*self = (*self * a) + b` + fn mul_add_assign(&mut self, a: A, b: B); +} + +#[cfg(any(feature = "std", feature = "libm"))] +impl MulAdd for f32 { + type Output = Self; + + #[inline] + fn mul_add(self, a: Self, b: Self) -> Self::Output { + ::mul_add(self, a, b) + } +} + +#[cfg(any(feature = "std", feature = "libm"))] +impl MulAdd for f64 { + type Output = Self; + + #[inline] + fn mul_add(self, a: Self, b: Self) -> Self::Output { + ::mul_add(self, a, b) + } +} + +macro_rules! mul_add_impl { + ($trait_name:ident for $($t:ty)*) => {$( + impl $trait_name for $t { + type Output = Self; + + #[inline] + fn mul_add(self, a: Self, b: Self) -> Self::Output { + (self * a) + b + } + } + )*} +} + +mul_add_impl!(MulAdd for isize i8 i16 i32 i64 i128); +mul_add_impl!(MulAdd for usize u8 u16 u32 u64 u128); + +#[cfg(any(feature = "std", feature = "libm"))] +impl MulAddAssign for f32 { + #[inline] + fn mul_add_assign(&mut self, a: Self, b: Self) { + *self = ::mul_add(*self, a, b) + } +} + +#[cfg(any(feature = "std", feature = "libm"))] +impl MulAddAssign for f64 { + #[inline] + fn mul_add_assign(&mut self, a: Self, b: Self) { + *self = ::mul_add(*self, a, b) + } +} + +macro_rules! mul_add_assign_impl { + ($trait_name:ident for $($t:ty)*) => {$( + impl $trait_name for $t { + #[inline] + fn mul_add_assign(&mut self, a: Self, b: Self) { + *self = (*self * a) + b + } + } + )*} +} + +mul_add_assign_impl!(MulAddAssign for isize i8 i16 i32 i64 i128); +mul_add_assign_impl!(MulAddAssign for usize u8 u16 u32 u64 u128); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mul_add_integer() { + macro_rules! test_mul_add { + ($($t:ident)+) => { + $( + { + let m: $t = 2; + let x: $t = 3; + let b: $t = 4; + + assert_eq!(MulAdd::mul_add(m, x, b), (m*x + b)); + } + )+ + }; + } + + test_mul_add!(usize u8 u16 u32 u64 isize i8 i16 i32 i64); + } + + #[test] + #[cfg(feature = "std")] + fn mul_add_float() { + macro_rules! test_mul_add { + ($($t:ident)+) => { + $( + { + use core::$t; + + let m: $t = 12.0; + let x: $t = 3.4; + let b: $t = 5.6; + + let abs_difference = (MulAdd::mul_add(m, x, b) - (m*x + b)).abs(); + + assert!(abs_difference <= 46.4 * $t::EPSILON); + } + )+ + }; + } + + test_mul_add!(f32 f64); + } +} diff --git a/third_party/crates/num-traits-0.2.19/src/ops/overflowing.rs b/third_party/crates/num-traits-0.2.19/src/ops/overflowing.rs new file mode 100644 index 0000000..c7a35a5 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/overflowing.rs @@ -0,0 +1,96 @@ +use core::ops::{Add, Mul, Sub}; +use core::{i128, i16, i32, i64, i8, isize}; +use core::{u128, u16, u32, u64, u8, usize}; + +macro_rules! overflowing_impl { + ($trait_name:ident, $method:ident, $t:ty) => { + impl $trait_name for $t { + #[inline] + fn $method(&self, v: &Self) -> (Self, bool) { + <$t>::$method(*self, *v) + } + } + }; +} + +/// Performs addition with a flag for overflow. +pub trait OverflowingAdd: Sized + Add { + /// Returns a tuple of the sum along with a boolean indicating whether an arithmetic overflow would occur. + /// If an overflow would have occurred then the wrapped value is returned. + fn overflowing_add(&self, v: &Self) -> (Self, bool); +} + +overflowing_impl!(OverflowingAdd, overflowing_add, u8); +overflowing_impl!(OverflowingAdd, overflowing_add, u16); +overflowing_impl!(OverflowingAdd, overflowing_add, u32); +overflowing_impl!(OverflowingAdd, overflowing_add, u64); +overflowing_impl!(OverflowingAdd, overflowing_add, usize); +overflowing_impl!(OverflowingAdd, overflowing_add, u128); + +overflowing_impl!(OverflowingAdd, overflowing_add, i8); +overflowing_impl!(OverflowingAdd, overflowing_add, i16); +overflowing_impl!(OverflowingAdd, overflowing_add, i32); +overflowing_impl!(OverflowingAdd, overflowing_add, i64); +overflowing_impl!(OverflowingAdd, overflowing_add, isize); +overflowing_impl!(OverflowingAdd, overflowing_add, i128); + +/// Performs substraction with a flag for overflow. +pub trait OverflowingSub: Sized + Sub { + /// Returns a tuple of the difference along with a boolean indicating whether an arithmetic overflow would occur. + /// If an overflow would have occurred then the wrapped value is returned. + fn overflowing_sub(&self, v: &Self) -> (Self, bool); +} + +overflowing_impl!(OverflowingSub, overflowing_sub, u8); +overflowing_impl!(OverflowingSub, overflowing_sub, u16); +overflowing_impl!(OverflowingSub, overflowing_sub, u32); +overflowing_impl!(OverflowingSub, overflowing_sub, u64); +overflowing_impl!(OverflowingSub, overflowing_sub, usize); +overflowing_impl!(OverflowingSub, overflowing_sub, u128); + +overflowing_impl!(OverflowingSub, overflowing_sub, i8); +overflowing_impl!(OverflowingSub, overflowing_sub, i16); +overflowing_impl!(OverflowingSub, overflowing_sub, i32); +overflowing_impl!(OverflowingSub, overflowing_sub, i64); +overflowing_impl!(OverflowingSub, overflowing_sub, isize); +overflowing_impl!(OverflowingSub, overflowing_sub, i128); + +/// Performs multiplication with a flag for overflow. +pub trait OverflowingMul: Sized + Mul { + /// Returns a tuple of the product along with a boolean indicating whether an arithmetic overflow would occur. + /// If an overflow would have occurred then the wrapped value is returned. + fn overflowing_mul(&self, v: &Self) -> (Self, bool); +} + +overflowing_impl!(OverflowingMul, overflowing_mul, u8); +overflowing_impl!(OverflowingMul, overflowing_mul, u16); +overflowing_impl!(OverflowingMul, overflowing_mul, u32); +overflowing_impl!(OverflowingMul, overflowing_mul, u64); +overflowing_impl!(OverflowingMul, overflowing_mul, usize); +overflowing_impl!(OverflowingMul, overflowing_mul, u128); + +overflowing_impl!(OverflowingMul, overflowing_mul, i8); +overflowing_impl!(OverflowingMul, overflowing_mul, i16); +overflowing_impl!(OverflowingMul, overflowing_mul, i32); +overflowing_impl!(OverflowingMul, overflowing_mul, i64); +overflowing_impl!(OverflowingMul, overflowing_mul, isize); +overflowing_impl!(OverflowingMul, overflowing_mul, i128); + +#[test] +fn test_overflowing_traits() { + fn overflowing_add(a: T, b: T) -> (T, bool) { + a.overflowing_add(&b) + } + fn overflowing_sub(a: T, b: T) -> (T, bool) { + a.overflowing_sub(&b) + } + fn overflowing_mul(a: T, b: T) -> (T, bool) { + a.overflowing_mul(&b) + } + assert_eq!(overflowing_add(5i16, 2), (7, false)); + assert_eq!(overflowing_add(i16::MAX, 1), (i16::MIN, true)); + assert_eq!(overflowing_sub(5i16, 2), (3, false)); + assert_eq!(overflowing_sub(i16::MIN, 1), (i16::MAX, true)); + assert_eq!(overflowing_mul(5i16, 2), (10, false)); + assert_eq!(overflowing_mul(1_000_000_000i32, 10), (1410065408, true)); +} diff --git a/third_party/crates/num-traits-0.2.19/src/ops/saturating.rs b/third_party/crates/num-traits-0.2.19/src/ops/saturating.rs new file mode 100644 index 0000000..16a0045 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/saturating.rs @@ -0,0 +1,130 @@ +use core::ops::{Add, Mul, Sub}; + +/// Saturating math operations. Deprecated, use `SaturatingAdd`, `SaturatingSub` and +/// `SaturatingMul` instead. +pub trait Saturating { + /// Saturating addition operator. + /// Returns a+b, saturating at the numeric bounds instead of overflowing. + fn saturating_add(self, v: Self) -> Self; + + /// Saturating subtraction operator. + /// Returns a-b, saturating at the numeric bounds instead of overflowing. + fn saturating_sub(self, v: Self) -> Self; +} + +macro_rules! deprecated_saturating_impl { + ($trait_name:ident for $($t:ty)*) => {$( + impl $trait_name for $t { + #[inline] + fn saturating_add(self, v: Self) -> Self { + Self::saturating_add(self, v) + } + + #[inline] + fn saturating_sub(self, v: Self) -> Self { + Self::saturating_sub(self, v) + } + } + )*} +} + +deprecated_saturating_impl!(Saturating for isize i8 i16 i32 i64 i128); +deprecated_saturating_impl!(Saturating for usize u8 u16 u32 u64 u128); + +macro_rules! saturating_impl { + ($trait_name:ident, $method:ident, $t:ty) => { + impl $trait_name for $t { + #[inline] + fn $method(&self, v: &Self) -> Self { + <$t>::$method(*self, *v) + } + } + }; +} + +/// Performs addition that saturates at the numeric bounds instead of overflowing. +pub trait SaturatingAdd: Sized + Add { + /// Saturating addition. Computes `self + other`, saturating at the relevant high or low boundary of + /// the type. + fn saturating_add(&self, v: &Self) -> Self; +} + +saturating_impl!(SaturatingAdd, saturating_add, u8); +saturating_impl!(SaturatingAdd, saturating_add, u16); +saturating_impl!(SaturatingAdd, saturating_add, u32); +saturating_impl!(SaturatingAdd, saturating_add, u64); +saturating_impl!(SaturatingAdd, saturating_add, usize); +saturating_impl!(SaturatingAdd, saturating_add, u128); + +saturating_impl!(SaturatingAdd, saturating_add, i8); +saturating_impl!(SaturatingAdd, saturating_add, i16); +saturating_impl!(SaturatingAdd, saturating_add, i32); +saturating_impl!(SaturatingAdd, saturating_add, i64); +saturating_impl!(SaturatingAdd, saturating_add, isize); +saturating_impl!(SaturatingAdd, saturating_add, i128); + +/// Performs subtraction that saturates at the numeric bounds instead of overflowing. +pub trait SaturatingSub: Sized + Sub { + /// Saturating subtraction. Computes `self - other`, saturating at the relevant high or low boundary of + /// the type. + fn saturating_sub(&self, v: &Self) -> Self; +} + +saturating_impl!(SaturatingSub, saturating_sub, u8); +saturating_impl!(SaturatingSub, saturating_sub, u16); +saturating_impl!(SaturatingSub, saturating_sub, u32); +saturating_impl!(SaturatingSub, saturating_sub, u64); +saturating_impl!(SaturatingSub, saturating_sub, usize); +saturating_impl!(SaturatingSub, saturating_sub, u128); + +saturating_impl!(SaturatingSub, saturating_sub, i8); +saturating_impl!(SaturatingSub, saturating_sub, i16); +saturating_impl!(SaturatingSub, saturating_sub, i32); +saturating_impl!(SaturatingSub, saturating_sub, i64); +saturating_impl!(SaturatingSub, saturating_sub, isize); +saturating_impl!(SaturatingSub, saturating_sub, i128); + +/// Performs multiplication that saturates at the numeric bounds instead of overflowing. +pub trait SaturatingMul: Sized + Mul { + /// Saturating multiplication. Computes `self * other`, saturating at the relevant high or low boundary of + /// the type. + fn saturating_mul(&self, v: &Self) -> Self; +} + +saturating_impl!(SaturatingMul, saturating_mul, u8); +saturating_impl!(SaturatingMul, saturating_mul, u16); +saturating_impl!(SaturatingMul, saturating_mul, u32); +saturating_impl!(SaturatingMul, saturating_mul, u64); +saturating_impl!(SaturatingMul, saturating_mul, usize); +saturating_impl!(SaturatingMul, saturating_mul, u128); + +saturating_impl!(SaturatingMul, saturating_mul, i8); +saturating_impl!(SaturatingMul, saturating_mul, i16); +saturating_impl!(SaturatingMul, saturating_mul, i32); +saturating_impl!(SaturatingMul, saturating_mul, i64); +saturating_impl!(SaturatingMul, saturating_mul, isize); +saturating_impl!(SaturatingMul, saturating_mul, i128); + +// TODO: add SaturatingNeg for signed integer primitives once the saturating_neg() API is stable. + +#[test] +fn test_saturating_traits() { + fn saturating_add(a: T, b: T) -> T { + a.saturating_add(&b) + } + fn saturating_sub(a: T, b: T) -> T { + a.saturating_sub(&b) + } + fn saturating_mul(a: T, b: T) -> T { + a.saturating_mul(&b) + } + assert_eq!(saturating_add(255, 1), 255u8); + assert_eq!(saturating_add(127, 1), 127i8); + assert_eq!(saturating_add(-128, -1), -128i8); + assert_eq!(saturating_sub(0, 1), 0u8); + assert_eq!(saturating_sub(-128, 1), -128i8); + assert_eq!(saturating_sub(127, -1), 127i8); + assert_eq!(saturating_mul(255, 2), 255u8); + assert_eq!(saturating_mul(127, 2), 127i8); + assert_eq!(saturating_mul(-128, 2), -128i8); +} diff --git a/third_party/crates/num-traits-0.2.19/src/ops/wrapping.rs b/third_party/crates/num-traits-0.2.19/src/ops/wrapping.rs new file mode 100644 index 0000000..3a8b331 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/ops/wrapping.rs @@ -0,0 +1,327 @@ +use core::num::Wrapping; +use core::ops::{Add, Mul, Neg, Shl, Shr, Sub}; + +macro_rules! wrapping_impl { + ($trait_name:ident, $method:ident, $t:ty) => { + impl $trait_name for $t { + #[inline] + fn $method(&self, v: &Self) -> Self { + <$t>::$method(*self, *v) + } + } + }; + ($trait_name:ident, $method:ident, $t:ty, $rhs:ty) => { + impl $trait_name<$rhs> for $t { + #[inline] + fn $method(&self, v: &$rhs) -> Self { + <$t>::$method(*self, *v) + } + } + }; +} + +/// Performs addition that wraps around on overflow. +pub trait WrappingAdd: Sized + Add { + /// Wrapping (modular) addition. Computes `self + other`, wrapping around at the boundary of + /// the type. + fn wrapping_add(&self, v: &Self) -> Self; +} + +wrapping_impl!(WrappingAdd, wrapping_add, u8); +wrapping_impl!(WrappingAdd, wrapping_add, u16); +wrapping_impl!(WrappingAdd, wrapping_add, u32); +wrapping_impl!(WrappingAdd, wrapping_add, u64); +wrapping_impl!(WrappingAdd, wrapping_add, usize); +wrapping_impl!(WrappingAdd, wrapping_add, u128); + +wrapping_impl!(WrappingAdd, wrapping_add, i8); +wrapping_impl!(WrappingAdd, wrapping_add, i16); +wrapping_impl!(WrappingAdd, wrapping_add, i32); +wrapping_impl!(WrappingAdd, wrapping_add, i64); +wrapping_impl!(WrappingAdd, wrapping_add, isize); +wrapping_impl!(WrappingAdd, wrapping_add, i128); + +/// Performs subtraction that wraps around on overflow. +pub trait WrappingSub: Sized + Sub { + /// Wrapping (modular) subtraction. Computes `self - other`, wrapping around at the boundary + /// of the type. + fn wrapping_sub(&self, v: &Self) -> Self; +} + +wrapping_impl!(WrappingSub, wrapping_sub, u8); +wrapping_impl!(WrappingSub, wrapping_sub, u16); +wrapping_impl!(WrappingSub, wrapping_sub, u32); +wrapping_impl!(WrappingSub, wrapping_sub, u64); +wrapping_impl!(WrappingSub, wrapping_sub, usize); +wrapping_impl!(WrappingSub, wrapping_sub, u128); + +wrapping_impl!(WrappingSub, wrapping_sub, i8); +wrapping_impl!(WrappingSub, wrapping_sub, i16); +wrapping_impl!(WrappingSub, wrapping_sub, i32); +wrapping_impl!(WrappingSub, wrapping_sub, i64); +wrapping_impl!(WrappingSub, wrapping_sub, isize); +wrapping_impl!(WrappingSub, wrapping_sub, i128); + +/// Performs multiplication that wraps around on overflow. +pub trait WrappingMul: Sized + Mul { + /// Wrapping (modular) multiplication. Computes `self * other`, wrapping around at the boundary + /// of the type. + fn wrapping_mul(&self, v: &Self) -> Self; +} + +wrapping_impl!(WrappingMul, wrapping_mul, u8); +wrapping_impl!(WrappingMul, wrapping_mul, u16); +wrapping_impl!(WrappingMul, wrapping_mul, u32); +wrapping_impl!(WrappingMul, wrapping_mul, u64); +wrapping_impl!(WrappingMul, wrapping_mul, usize); +wrapping_impl!(WrappingMul, wrapping_mul, u128); + +wrapping_impl!(WrappingMul, wrapping_mul, i8); +wrapping_impl!(WrappingMul, wrapping_mul, i16); +wrapping_impl!(WrappingMul, wrapping_mul, i32); +wrapping_impl!(WrappingMul, wrapping_mul, i64); +wrapping_impl!(WrappingMul, wrapping_mul, isize); +wrapping_impl!(WrappingMul, wrapping_mul, i128); + +macro_rules! wrapping_unary_impl { + ($trait_name:ident, $method:ident, $t:ty) => { + impl $trait_name for $t { + #[inline] + fn $method(&self) -> $t { + <$t>::$method(*self) + } + } + }; +} + +/// Performs a negation that does not panic. +pub trait WrappingNeg: Sized { + /// Wrapping (modular) negation. Computes `-self`, + /// wrapping around at the boundary of the type. + /// + /// Since unsigned types do not have negative equivalents + /// all applications of this function will wrap (except for `-0`). + /// For values smaller than the corresponding signed type's maximum + /// the result is the same as casting the corresponding signed value. + /// Any larger values are equivalent to `MAX + 1 - (val - MAX - 1)` where + /// `MAX` is the corresponding signed type's maximum. + /// + /// ``` + /// use num_traits::WrappingNeg; + /// + /// assert_eq!(100i8.wrapping_neg(), -100); + /// assert_eq!((-100i8).wrapping_neg(), 100); + /// assert_eq!((-128i8).wrapping_neg(), -128); // wrapped! + /// ``` + fn wrapping_neg(&self) -> Self; +} + +wrapping_unary_impl!(WrappingNeg, wrapping_neg, u8); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, u16); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, u32); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, u64); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, usize); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, u128); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, i8); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, i16); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, i32); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, i64); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, isize); +wrapping_unary_impl!(WrappingNeg, wrapping_neg, i128); + +macro_rules! wrapping_shift_impl { + ($trait_name:ident, $method:ident, $t:ty) => { + impl $trait_name for $t { + #[inline] + fn $method(&self, rhs: u32) -> $t { + <$t>::$method(*self, rhs) + } + } + }; +} + +/// Performs a left shift that does not panic. +pub trait WrappingShl: Sized + Shl { + /// Panic-free bitwise shift-left; yields `self << mask(rhs)`, + /// where `mask` removes any high order bits of `rhs` that would + /// cause the shift to exceed the bitwidth of the type. + /// + /// ``` + /// use num_traits::WrappingShl; + /// + /// let x: u16 = 0x0001; + /// + /// assert_eq!(WrappingShl::wrapping_shl(&x, 0), 0x0001); + /// assert_eq!(WrappingShl::wrapping_shl(&x, 1), 0x0002); + /// assert_eq!(WrappingShl::wrapping_shl(&x, 15), 0x8000); + /// assert_eq!(WrappingShl::wrapping_shl(&x, 16), 0x0001); + /// ``` + fn wrapping_shl(&self, rhs: u32) -> Self; +} + +wrapping_shift_impl!(WrappingShl, wrapping_shl, u8); +wrapping_shift_impl!(WrappingShl, wrapping_shl, u16); +wrapping_shift_impl!(WrappingShl, wrapping_shl, u32); +wrapping_shift_impl!(WrappingShl, wrapping_shl, u64); +wrapping_shift_impl!(WrappingShl, wrapping_shl, usize); +wrapping_shift_impl!(WrappingShl, wrapping_shl, u128); + +wrapping_shift_impl!(WrappingShl, wrapping_shl, i8); +wrapping_shift_impl!(WrappingShl, wrapping_shl, i16); +wrapping_shift_impl!(WrappingShl, wrapping_shl, i32); +wrapping_shift_impl!(WrappingShl, wrapping_shl, i64); +wrapping_shift_impl!(WrappingShl, wrapping_shl, isize); +wrapping_shift_impl!(WrappingShl, wrapping_shl, i128); + +/// Performs a right shift that does not panic. +pub trait WrappingShr: Sized + Shr { + /// Panic-free bitwise shift-right; yields `self >> mask(rhs)`, + /// where `mask` removes any high order bits of `rhs` that would + /// cause the shift to exceed the bitwidth of the type. + /// + /// ``` + /// use num_traits::WrappingShr; + /// + /// let x: u16 = 0x8000; + /// + /// assert_eq!(WrappingShr::wrapping_shr(&x, 0), 0x8000); + /// assert_eq!(WrappingShr::wrapping_shr(&x, 1), 0x4000); + /// assert_eq!(WrappingShr::wrapping_shr(&x, 15), 0x0001); + /// assert_eq!(WrappingShr::wrapping_shr(&x, 16), 0x8000); + /// ``` + fn wrapping_shr(&self, rhs: u32) -> Self; +} + +wrapping_shift_impl!(WrappingShr, wrapping_shr, u8); +wrapping_shift_impl!(WrappingShr, wrapping_shr, u16); +wrapping_shift_impl!(WrappingShr, wrapping_shr, u32); +wrapping_shift_impl!(WrappingShr, wrapping_shr, u64); +wrapping_shift_impl!(WrappingShr, wrapping_shr, usize); +wrapping_shift_impl!(WrappingShr, wrapping_shr, u128); + +wrapping_shift_impl!(WrappingShr, wrapping_shr, i8); +wrapping_shift_impl!(WrappingShr, wrapping_shr, i16); +wrapping_shift_impl!(WrappingShr, wrapping_shr, i32); +wrapping_shift_impl!(WrappingShr, wrapping_shr, i64); +wrapping_shift_impl!(WrappingShr, wrapping_shr, isize); +wrapping_shift_impl!(WrappingShr, wrapping_shr, i128); + +// Well this is a bit funny, but all the more appropriate. +impl WrappingAdd for Wrapping +where + Wrapping: Add>, +{ + fn wrapping_add(&self, v: &Self) -> Self { + Wrapping(self.0.wrapping_add(&v.0)) + } +} +impl WrappingSub for Wrapping +where + Wrapping: Sub>, +{ + fn wrapping_sub(&self, v: &Self) -> Self { + Wrapping(self.0.wrapping_sub(&v.0)) + } +} +impl WrappingMul for Wrapping +where + Wrapping: Mul>, +{ + fn wrapping_mul(&self, v: &Self) -> Self { + Wrapping(self.0.wrapping_mul(&v.0)) + } +} +impl WrappingNeg for Wrapping +where + Wrapping: Neg>, +{ + fn wrapping_neg(&self) -> Self { + Wrapping(self.0.wrapping_neg()) + } +} +impl WrappingShl for Wrapping +where + Wrapping: Shl>, +{ + fn wrapping_shl(&self, rhs: u32) -> Self { + Wrapping(self.0.wrapping_shl(rhs)) + } +} +impl WrappingShr for Wrapping +where + Wrapping: Shr>, +{ + fn wrapping_shr(&self, rhs: u32) -> Self { + Wrapping(self.0.wrapping_shr(rhs)) + } +} + +#[test] +fn test_wrapping_traits() { + fn wrapping_add(a: T, b: T) -> T { + a.wrapping_add(&b) + } + fn wrapping_sub(a: T, b: T) -> T { + a.wrapping_sub(&b) + } + fn wrapping_mul(a: T, b: T) -> T { + a.wrapping_mul(&b) + } + fn wrapping_neg(a: T) -> T { + a.wrapping_neg() + } + fn wrapping_shl(a: T, b: u32) -> T { + a.wrapping_shl(b) + } + fn wrapping_shr(a: T, b: u32) -> T { + a.wrapping_shr(b) + } + assert_eq!(wrapping_add(255, 1), 0u8); + assert_eq!(wrapping_sub(0, 1), 255u8); + assert_eq!(wrapping_mul(255, 2), 254u8); + assert_eq!(wrapping_neg(255), 1u8); + assert_eq!(wrapping_shl(255, 8), 255u8); + assert_eq!(wrapping_shr(255, 8), 255u8); + assert_eq!(wrapping_add(255, 1), (Wrapping(255u8) + Wrapping(1u8)).0); + assert_eq!(wrapping_sub(0, 1), (Wrapping(0u8) - Wrapping(1u8)).0); + assert_eq!(wrapping_mul(255, 2), (Wrapping(255u8) * Wrapping(2u8)).0); + assert_eq!(wrapping_neg(255), (-Wrapping(255u8)).0); + assert_eq!(wrapping_shl(255, 8), (Wrapping(255u8) << 8).0); + assert_eq!(wrapping_shr(255, 8), (Wrapping(255u8) >> 8).0); +} + +#[test] +fn wrapping_is_wrappingadd() { + fn require_wrappingadd(_: &T) {} + require_wrappingadd(&Wrapping(42)); +} + +#[test] +fn wrapping_is_wrappingsub() { + fn require_wrappingsub(_: &T) {} + require_wrappingsub(&Wrapping(42)); +} + +#[test] +fn wrapping_is_wrappingmul() { + fn require_wrappingmul(_: &T) {} + require_wrappingmul(&Wrapping(42)); +} + +#[test] +fn wrapping_is_wrappingneg() { + fn require_wrappingneg(_: &T) {} + require_wrappingneg(&Wrapping(42)); +} + +#[test] +fn wrapping_is_wrappingshl() { + fn require_wrappingshl(_: &T) {} + require_wrappingshl(&Wrapping(42)); +} + +#[test] +fn wrapping_is_wrappingshr() { + fn require_wrappingshr(_: &T) {} + require_wrappingshr(&Wrapping(42)); +} diff --git a/third_party/crates/num-traits-0.2.19/src/pow.rs b/third_party/crates/num-traits-0.2.19/src/pow.rs new file mode 100644 index 0000000..ef51c95 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/pow.rs @@ -0,0 +1,242 @@ +use crate::{CheckedMul, One}; +use core::num::Wrapping; +use core::ops::Mul; + +/// Binary operator for raising a value to a power. +pub trait Pow { + /// The result after applying the operator. + type Output; + + /// Returns `self` to the power `rhs`. + /// + /// # Examples + /// + /// ``` + /// use num_traits::Pow; + /// assert_eq!(Pow::pow(10u32, 2u32), 100); + /// ``` + fn pow(self, rhs: RHS) -> Self::Output; +} + +macro_rules! pow_impl { + ($t:ty) => { + pow_impl!($t, u8); + pow_impl!($t, usize); + + // FIXME: these should be possible + // pow_impl!($t, u16); + // pow_impl!($t, u32); + // pow_impl!($t, u64); + }; + ($t:ty, $rhs:ty) => { + pow_impl!($t, $rhs, usize, pow); + }; + ($t:ty, $rhs:ty, $desired_rhs:ty, $method:expr) => { + impl Pow<$rhs> for $t { + type Output = $t; + #[inline] + fn pow(self, rhs: $rhs) -> $t { + ($method)(self, <$desired_rhs>::from(rhs)) + } + } + + impl<'a> Pow<&'a $rhs> for $t { + type Output = $t; + #[inline] + fn pow(self, rhs: &'a $rhs) -> $t { + ($method)(self, <$desired_rhs>::from(*rhs)) + } + } + + impl<'a> Pow<$rhs> for &'a $t { + type Output = $t; + #[inline] + fn pow(self, rhs: $rhs) -> $t { + ($method)(*self, <$desired_rhs>::from(rhs)) + } + } + + impl<'a, 'b> Pow<&'a $rhs> for &'b $t { + type Output = $t; + #[inline] + fn pow(self, rhs: &'a $rhs) -> $t { + ($method)(*self, <$desired_rhs>::from(*rhs)) + } + } + }; +} + +pow_impl!(u8, u8, u32, u8::pow); +pow_impl!(u8, u16, u32, u8::pow); +pow_impl!(u8, u32, u32, u8::pow); +pow_impl!(u8, usize); +pow_impl!(i8, u8, u32, i8::pow); +pow_impl!(i8, u16, u32, i8::pow); +pow_impl!(i8, u32, u32, i8::pow); +pow_impl!(i8, usize); +pow_impl!(u16, u8, u32, u16::pow); +pow_impl!(u16, u16, u32, u16::pow); +pow_impl!(u16, u32, u32, u16::pow); +pow_impl!(u16, usize); +pow_impl!(i16, u8, u32, i16::pow); +pow_impl!(i16, u16, u32, i16::pow); +pow_impl!(i16, u32, u32, i16::pow); +pow_impl!(i16, usize); +pow_impl!(u32, u8, u32, u32::pow); +pow_impl!(u32, u16, u32, u32::pow); +pow_impl!(u32, u32, u32, u32::pow); +pow_impl!(u32, usize); +pow_impl!(i32, u8, u32, i32::pow); +pow_impl!(i32, u16, u32, i32::pow); +pow_impl!(i32, u32, u32, i32::pow); +pow_impl!(i32, usize); +pow_impl!(u64, u8, u32, u64::pow); +pow_impl!(u64, u16, u32, u64::pow); +pow_impl!(u64, u32, u32, u64::pow); +pow_impl!(u64, usize); +pow_impl!(i64, u8, u32, i64::pow); +pow_impl!(i64, u16, u32, i64::pow); +pow_impl!(i64, u32, u32, i64::pow); +pow_impl!(i64, usize); + +pow_impl!(u128, u8, u32, u128::pow); +pow_impl!(u128, u16, u32, u128::pow); +pow_impl!(u128, u32, u32, u128::pow); +pow_impl!(u128, usize); + +pow_impl!(i128, u8, u32, i128::pow); +pow_impl!(i128, u16, u32, i128::pow); +pow_impl!(i128, u32, u32, i128::pow); +pow_impl!(i128, usize); + +pow_impl!(usize, u8, u32, usize::pow); +pow_impl!(usize, u16, u32, usize::pow); +pow_impl!(usize, u32, u32, usize::pow); +pow_impl!(usize, usize); +pow_impl!(isize, u8, u32, isize::pow); +pow_impl!(isize, u16, u32, isize::pow); +pow_impl!(isize, u32, u32, isize::pow); +pow_impl!(isize, usize); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); +pow_impl!(Wrapping); + +// FIXME: these should be possible +// pow_impl!(u8, u64); +// pow_impl!(i16, u64); +// pow_impl!(i8, u64); +// pow_impl!(u16, u64); +// pow_impl!(u32, u64); +// pow_impl!(i32, u64); +// pow_impl!(u64, u64); +// pow_impl!(i64, u64); +// pow_impl!(usize, u64); +// pow_impl!(isize, u64); + +#[cfg(any(feature = "std", feature = "libm"))] +mod float_impls { + use super::Pow; + use crate::Float; + + pow_impl!(f32, i8, i32, ::powi); + pow_impl!(f32, u8, i32, ::powi); + pow_impl!(f32, i16, i32, ::powi); + pow_impl!(f32, u16, i32, ::powi); + pow_impl!(f32, i32, i32, ::powi); + pow_impl!(f64, i8, i32, ::powi); + pow_impl!(f64, u8, i32, ::powi); + pow_impl!(f64, i16, i32, ::powi); + pow_impl!(f64, u16, i32, ::powi); + pow_impl!(f64, i32, i32, ::powi); + pow_impl!(f32, f32, f32, ::powf); + pow_impl!(f64, f32, f64, ::powf); + pow_impl!(f64, f64, f64, ::powf); +} + +/// Raises a value to the power of exp, using exponentiation by squaring. +/// +/// Note that `0⁰` (`pow(0, 0)`) returns `1`. Mathematically this is undefined. +/// +/// # Example +/// +/// ```rust +/// use num_traits::pow; +/// +/// assert_eq!(pow(2i8, 4), 16); +/// assert_eq!(pow(6u8, 3), 216); +/// assert_eq!(pow(0u8, 0), 1); // Be aware if this case affects you +/// ``` +#[inline] +pub fn pow>(mut base: T, mut exp: usize) -> T { + if exp == 0 { + return T::one(); + } + + while exp & 1 == 0 { + base = base.clone() * base; + exp >>= 1; + } + if exp == 1 { + return base; + } + + let mut acc = base.clone(); + while exp > 1 { + exp >>= 1; + base = base.clone() * base; + if exp & 1 == 1 { + acc = acc * base.clone(); + } + } + acc +} + +/// Raises a value to the power of exp, returning `None` if an overflow occurred. +/// +/// Note that `0⁰` (`checked_pow(0, 0)`) returns `Some(1)`. Mathematically this is undefined. +/// +/// Otherwise same as the `pow` function. +/// +/// # Example +/// +/// ```rust +/// use num_traits::checked_pow; +/// +/// assert_eq!(checked_pow(2i8, 4), Some(16)); +/// assert_eq!(checked_pow(7i8, 8), None); +/// assert_eq!(checked_pow(7u32, 8), Some(5_764_801)); +/// assert_eq!(checked_pow(0u32, 0), Some(1)); // Be aware if this case affect you +/// ``` +#[inline] +pub fn checked_pow(mut base: T, mut exp: usize) -> Option { + if exp == 0 { + return Some(T::one()); + } + + while exp & 1 == 0 { + base = base.checked_mul(&base)?; + exp >>= 1; + } + if exp == 1 { + return Some(base); + } + + let mut acc = base.clone(); + while exp > 1 { + exp >>= 1; + base = base.checked_mul(&base)?; + if exp & 1 == 1 { + acc = acc.checked_mul(&base)?; + } + } + Some(acc) +} diff --git a/third_party/crates/num-traits-0.2.19/src/real.rs b/third_party/crates/num-traits-0.2.19/src/real.rs new file mode 100644 index 0000000..25ec873 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/real.rs @@ -0,0 +1,834 @@ +#![cfg(any(feature = "std", feature = "libm"))] + +use core::ops::Neg; + +use crate::{Float, Num, NumCast}; + +// NOTE: These doctests have the same issue as those in src/float.rs. +// They're testing the inherent methods directly, and not those of `Real`. + +/// A trait for real number types that do not necessarily have +/// floating-point-specific characteristics such as NaN and infinity. +/// +/// See [this Wikipedia article](https://en.wikipedia.org/wiki/Real_data_type) +/// for a list of data types that could meaningfully implement this trait. +/// +/// This trait is only available with the `std` feature, or with the `libm` feature otherwise. +pub trait Real: Num + Copy + NumCast + PartialOrd + Neg { + /// Returns the smallest finite value that this type can represent. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x: f64 = Real::min_value(); + /// + /// assert_eq!(x, f64::MIN); + /// ``` + fn min_value() -> Self; + + /// Returns the smallest positive, normalized value that this type can represent. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x: f64 = Real::min_positive_value(); + /// + /// assert_eq!(x, f64::MIN_POSITIVE); + /// ``` + fn min_positive_value() -> Self; + + /// Returns epsilon, a small positive value. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x: f64 = Real::epsilon(); + /// + /// assert_eq!(x, f64::EPSILON); + /// ``` + /// + /// # Panics + /// + /// The default implementation will panic if `f32::EPSILON` cannot + /// be cast to `Self`. + fn epsilon() -> Self; + + /// Returns the largest finite value that this type can represent. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x: f64 = Real::max_value(); + /// assert_eq!(x, f64::MAX); + /// ``` + fn max_value() -> Self; + + /// Returns the largest integer less than or equal to a number. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let f = 3.99; + /// let g = 3.0; + /// + /// assert_eq!(f.floor(), 3.0); + /// assert_eq!(g.floor(), 3.0); + /// ``` + fn floor(self) -> Self; + + /// Returns the smallest integer greater than or equal to a number. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let f = 3.01; + /// let g = 4.0; + /// + /// assert_eq!(f.ceil(), 4.0); + /// assert_eq!(g.ceil(), 4.0); + /// ``` + fn ceil(self) -> Self; + + /// Returns the nearest integer to a number. Round half-way cases away from + /// `0.0`. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let f = 3.3; + /// let g = -3.3; + /// + /// assert_eq!(f.round(), 3.0); + /// assert_eq!(g.round(), -3.0); + /// ``` + fn round(self) -> Self; + + /// Return the integer part of a number. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let f = 3.3; + /// let g = -3.7; + /// + /// assert_eq!(f.trunc(), 3.0); + /// assert_eq!(g.trunc(), -3.0); + /// ``` + fn trunc(self) -> Self; + + /// Returns the fractional part of a number. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 3.5; + /// let y = -3.5; + /// let abs_difference_x = (x.fract() - 0.5).abs(); + /// let abs_difference_y = (y.fract() - (-0.5)).abs(); + /// + /// assert!(abs_difference_x < 1e-10); + /// assert!(abs_difference_y < 1e-10); + /// ``` + fn fract(self) -> Self; + + /// Computes the absolute value of `self`. Returns `Float::nan()` if the + /// number is `Float::nan()`. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x = 3.5; + /// let y = -3.5; + /// + /// let abs_difference_x = (x.abs() - x).abs(); + /// let abs_difference_y = (y.abs() - (-y)).abs(); + /// + /// assert!(abs_difference_x < 1e-10); + /// assert!(abs_difference_y < 1e-10); + /// + /// assert!(::num_traits::Float::is_nan(f64::NAN.abs())); + /// ``` + fn abs(self) -> Self; + + /// Returns a number that represents the sign of `self`. + /// + /// - `1.0` if the number is positive, `+0.0` or `Float::infinity()` + /// - `-1.0` if the number is negative, `-0.0` or `Float::neg_infinity()` + /// - `Float::nan()` if the number is `Float::nan()` + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let f = 3.5; + /// + /// assert_eq!(f.signum(), 1.0); + /// assert_eq!(f64::NEG_INFINITY.signum(), -1.0); + /// + /// assert!(f64::NAN.signum().is_nan()); + /// ``` + fn signum(self) -> Self; + + /// Returns `true` if `self` is positive, including `+0.0`, + /// `Float::infinity()`, and with newer versions of Rust `f64::NAN`. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let neg_nan: f64 = -f64::NAN; + /// + /// let f = 7.0; + /// let g = -7.0; + /// + /// assert!(f.is_sign_positive()); + /// assert!(!g.is_sign_positive()); + /// assert!(!neg_nan.is_sign_positive()); + /// ``` + fn is_sign_positive(self) -> bool; + + /// Returns `true` if `self` is negative, including `-0.0`, + /// `Float::neg_infinity()`, and with newer versions of Rust `-f64::NAN`. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let nan: f64 = f64::NAN; + /// + /// let f = 7.0; + /// let g = -7.0; + /// + /// assert!(!f.is_sign_negative()); + /// assert!(g.is_sign_negative()); + /// assert!(!nan.is_sign_negative()); + /// ``` + fn is_sign_negative(self) -> bool; + + /// Fused multiply-add. Computes `(self * a) + b` with only one rounding + /// error, yielding a more accurate result than an unfused multiply-add. + /// + /// Using `mul_add` can be more performant than an unfused multiply-add if + /// the target architecture has a dedicated `fma` CPU instruction. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let m = 10.0; + /// let x = 4.0; + /// let b = 60.0; + /// + /// // 100.0 + /// let abs_difference = (m.mul_add(x, b) - (m*x + b)).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn mul_add(self, a: Self, b: Self) -> Self; + + /// Take the reciprocal (inverse) of a number, `1/x`. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 2.0; + /// let abs_difference = (x.recip() - (1.0/x)).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn recip(self) -> Self; + + /// Raise a number to an integer power. + /// + /// Using this function is generally faster than using `powf` + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 2.0; + /// let abs_difference = (x.powi(2) - x*x).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn powi(self, n: i32) -> Self; + + /// Raise a number to a real number power. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 2.0; + /// let abs_difference = (x.powf(2.0) - x*x).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn powf(self, n: Self) -> Self; + + /// Take the square root of a number. + /// + /// Returns NaN if `self` is a negative floating-point number. + /// + /// # Panics + /// + /// If the implementing type doesn't support NaN, this method should panic if `self < 0`. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let positive = 4.0; + /// let negative = -4.0; + /// + /// let abs_difference = (positive.sqrt() - 2.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// assert!(::num_traits::Float::is_nan(negative.sqrt())); + /// ``` + fn sqrt(self) -> Self; + + /// Returns `e^(self)`, (the exponential function). + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let one = 1.0; + /// // e^1 + /// let e = one.exp(); + /// + /// // ln(e) - 1 == 0 + /// let abs_difference = (e.ln() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn exp(self) -> Self; + + /// Returns `2^(self)`. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let f = 2.0; + /// + /// // 2^2 - 4 == 0 + /// let abs_difference = (f.exp2() - 4.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn exp2(self) -> Self; + + /// Returns the natural logarithm of the number. + /// + /// # Panics + /// + /// If `self <= 0` and this type does not support a NaN representation, this function should panic. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let one = 1.0; + /// // e^1 + /// let e = one.exp(); + /// + /// // ln(e) - 1 == 0 + /// let abs_difference = (e.ln() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn ln(self) -> Self; + + /// Returns the logarithm of the number with respect to an arbitrary base. + /// + /// # Panics + /// + /// If `self <= 0` and this type does not support a NaN representation, this function should panic. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let ten = 10.0; + /// let two = 2.0; + /// + /// // log10(10) - 1 == 0 + /// let abs_difference_10 = (ten.log(10.0) - 1.0).abs(); + /// + /// // log2(2) - 1 == 0 + /// let abs_difference_2 = (two.log(2.0) - 1.0).abs(); + /// + /// assert!(abs_difference_10 < 1e-10); + /// assert!(abs_difference_2 < 1e-10); + /// ``` + fn log(self, base: Self) -> Self; + + /// Returns the base 2 logarithm of the number. + /// + /// # Panics + /// + /// If `self <= 0` and this type does not support a NaN representation, this function should panic. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let two = 2.0; + /// + /// // log2(2) - 1 == 0 + /// let abs_difference = (two.log2() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn log2(self) -> Self; + + /// Returns the base 10 logarithm of the number. + /// + /// # Panics + /// + /// If `self <= 0` and this type does not support a NaN representation, this function should panic. + /// + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let ten = 10.0; + /// + /// // log10(10) - 1 == 0 + /// let abs_difference = (ten.log10() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn log10(self) -> Self; + + /// Converts radians to degrees. + /// + /// ``` + /// use std::f64::consts; + /// + /// let angle = consts::PI; + /// + /// let abs_difference = (angle.to_degrees() - 180.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn to_degrees(self) -> Self; + + /// Converts degrees to radians. + /// + /// ``` + /// use std::f64::consts; + /// + /// let angle = 180.0_f64; + /// + /// let abs_difference = (angle.to_radians() - consts::PI).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn to_radians(self) -> Self; + + /// Returns the maximum of the two numbers. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 1.0; + /// let y = 2.0; + /// + /// assert_eq!(x.max(y), y); + /// ``` + fn max(self, other: Self) -> Self; + + /// Returns the minimum of the two numbers. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 1.0; + /// let y = 2.0; + /// + /// assert_eq!(x.min(y), x); + /// ``` + fn min(self, other: Self) -> Self; + + /// The positive difference of two numbers. + /// + /// * If `self <= other`: `0:0` + /// * Else: `self - other` + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 3.0; + /// let y = -3.0; + /// + /// let abs_difference_x = (x.abs_sub(1.0) - 2.0).abs(); + /// let abs_difference_y = (y.abs_sub(1.0) - 0.0).abs(); + /// + /// assert!(abs_difference_x < 1e-10); + /// assert!(abs_difference_y < 1e-10); + /// ``` + fn abs_sub(self, other: Self) -> Self; + + /// Take the cubic root of a number. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 8.0; + /// + /// // x^(1/3) - 2 == 0 + /// let abs_difference = (x.cbrt() - 2.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn cbrt(self) -> Self; + + /// Calculate the length of the hypotenuse of a right-angle triangle given + /// legs of length `x` and `y`. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 2.0; + /// let y = 3.0; + /// + /// // sqrt(x^2 + y^2) + /// let abs_difference = (x.hypot(y) - (x.powi(2) + y.powi(2)).sqrt()).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn hypot(self, other: Self) -> Self; + + /// Computes the sine of a number (in radians). + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x = f64::consts::PI/2.0; + /// + /// let abs_difference = (x.sin() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn sin(self) -> Self; + + /// Computes the cosine of a number (in radians). + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x = 2.0*f64::consts::PI; + /// + /// let abs_difference = (x.cos() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn cos(self) -> Self; + + /// Computes the tangent of a number (in radians). + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x = f64::consts::PI/4.0; + /// let abs_difference = (x.tan() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-14); + /// ``` + fn tan(self) -> Self; + + /// Computes the arcsine of a number. Return value is in radians in + /// the range [-pi/2, pi/2] or NaN if the number is outside the range + /// [-1, 1]. + /// + /// # Panics + /// + /// If this type does not support a NaN representation, this function should panic + /// if the number is outside the range [-1, 1]. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let f = f64::consts::PI / 2.0; + /// + /// // asin(sin(pi/2)) + /// let abs_difference = (f.sin().asin() - f64::consts::PI / 2.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn asin(self) -> Self; + + /// Computes the arccosine of a number. Return value is in radians in + /// the range [0, pi] or NaN if the number is outside the range + /// [-1, 1]. + /// + /// # Panics + /// + /// If this type does not support a NaN representation, this function should panic + /// if the number is outside the range [-1, 1]. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let f = f64::consts::PI / 4.0; + /// + /// // acos(cos(pi/4)) + /// let abs_difference = (f.cos().acos() - f64::consts::PI / 4.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn acos(self) -> Self; + + /// Computes the arctangent of a number. Return value is in radians in the + /// range [-pi/2, pi/2]; + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let f = 1.0; + /// + /// // atan(tan(1)) + /// let abs_difference = (f.tan().atan() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn atan(self) -> Self; + + /// Computes the four quadrant arctangent of `self` (`y`) and `other` (`x`). + /// + /// * `x = 0`, `y = 0`: `0` + /// * `x >= 0`: `arctan(y/x)` -> `[-pi/2, pi/2]` + /// * `y >= 0`: `arctan(y/x) + pi` -> `(pi/2, pi]` + /// * `y < 0`: `arctan(y/x) - pi` -> `(-pi, -pi/2)` + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let pi = f64::consts::PI; + /// // All angles from horizontal right (+x) + /// // 45 deg counter-clockwise + /// let x1 = 3.0; + /// let y1 = -3.0; + /// + /// // 135 deg clockwise + /// let x2 = -3.0; + /// let y2 = 3.0; + /// + /// let abs_difference_1 = (y1.atan2(x1) - (-pi/4.0)).abs(); + /// let abs_difference_2 = (y2.atan2(x2) - 3.0*pi/4.0).abs(); + /// + /// assert!(abs_difference_1 < 1e-10); + /// assert!(abs_difference_2 < 1e-10); + /// ``` + fn atan2(self, other: Self) -> Self; + + /// Simultaneously computes the sine and cosine of the number, `x`. Returns + /// `(sin(x), cos(x))`. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x = f64::consts::PI/4.0; + /// let f = x.sin_cos(); + /// + /// let abs_difference_0 = (f.0 - x.sin()).abs(); + /// let abs_difference_1 = (f.1 - x.cos()).abs(); + /// + /// assert!(abs_difference_0 < 1e-10); + /// assert!(abs_difference_0 < 1e-10); + /// ``` + fn sin_cos(self) -> (Self, Self); + + /// Returns `e^(self) - 1` in a way that is accurate even if the + /// number is close to zero. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 7.0; + /// + /// // e^(ln(7)) - 1 + /// let abs_difference = (x.ln().exp_m1() - 6.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn exp_m1(self) -> Self; + + /// Returns `ln(1+n)` (natural logarithm) more accurately than if + /// the operations were performed separately. + /// + /// # Panics + /// + /// If this type does not support a NaN representation, this function should panic + /// if `self-1 <= 0`. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let x = f64::consts::E - 1.0; + /// + /// // ln(1 + (e - 1)) == ln(e) == 1 + /// let abs_difference = (x.ln_1p() - 1.0).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn ln_1p(self) -> Self; + + /// Hyperbolic sine function. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let e = f64::consts::E; + /// let x = 1.0; + /// + /// let f = x.sinh(); + /// // Solving sinh() at 1 gives `(e^2-1)/(2e)` + /// let g = (e*e - 1.0)/(2.0*e); + /// let abs_difference = (f - g).abs(); + /// + /// assert!(abs_difference < 1e-10); + /// ``` + fn sinh(self) -> Self; + + /// Hyperbolic cosine function. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let e = f64::consts::E; + /// let x = 1.0; + /// let f = x.cosh(); + /// // Solving cosh() at 1 gives this result + /// let g = (e*e + 1.0)/(2.0*e); + /// let abs_difference = (f - g).abs(); + /// + /// // Same result + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn cosh(self) -> Self; + + /// Hyperbolic tangent function. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let e = f64::consts::E; + /// let x = 1.0; + /// + /// let f = x.tanh(); + /// // Solving tanh() at 1 gives `(1 - e^(-2))/(1 + e^(-2))` + /// let g = (1.0 - e.powi(-2))/(1.0 + e.powi(-2)); + /// let abs_difference = (f - g).abs(); + /// + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn tanh(self) -> Self; + + /// Inverse hyperbolic sine function. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 1.0; + /// let f = x.sinh().asinh(); + /// + /// let abs_difference = (f - x).abs(); + /// + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn asinh(self) -> Self; + + /// Inverse hyperbolic cosine function. + /// + /// ``` + /// use num_traits::real::Real; + /// + /// let x = 1.0; + /// let f = x.cosh().acosh(); + /// + /// let abs_difference = (f - x).abs(); + /// + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn acosh(self) -> Self; + + /// Inverse hyperbolic tangent function. + /// + /// ``` + /// use num_traits::real::Real; + /// use std::f64; + /// + /// let e = f64::consts::E; + /// let f = e.tanh().atanh(); + /// + /// let abs_difference = (f - e).abs(); + /// + /// assert!(abs_difference < 1.0e-10); + /// ``` + fn atanh(self) -> Self; +} + +impl Real for T { + forward! { + Float::min_value() -> Self; + Float::min_positive_value() -> Self; + Float::epsilon() -> Self; + Float::max_value() -> Self; + } + forward! { + Float::floor(self) -> Self; + Float::ceil(self) -> Self; + Float::round(self) -> Self; + Float::trunc(self) -> Self; + Float::fract(self) -> Self; + Float::abs(self) -> Self; + Float::signum(self) -> Self; + Float::is_sign_positive(self) -> bool; + Float::is_sign_negative(self) -> bool; + Float::mul_add(self, a: Self, b: Self) -> Self; + Float::recip(self) -> Self; + Float::powi(self, n: i32) -> Self; + Float::powf(self, n: Self) -> Self; + Float::sqrt(self) -> Self; + Float::exp(self) -> Self; + Float::exp2(self) -> Self; + Float::ln(self) -> Self; + Float::log(self, base: Self) -> Self; + Float::log2(self) -> Self; + Float::log10(self) -> Self; + Float::to_degrees(self) -> Self; + Float::to_radians(self) -> Self; + Float::max(self, other: Self) -> Self; + Float::min(self, other: Self) -> Self; + Float::abs_sub(self, other: Self) -> Self; + Float::cbrt(self) -> Self; + Float::hypot(self, other: Self) -> Self; + Float::sin(self) -> Self; + Float::cos(self) -> Self; + Float::tan(self) -> Self; + Float::asin(self) -> Self; + Float::acos(self) -> Self; + Float::atan(self) -> Self; + Float::atan2(self, other: Self) -> Self; + Float::sin_cos(self) -> (Self, Self); + Float::exp_m1(self) -> Self; + Float::ln_1p(self) -> Self; + Float::sinh(self) -> Self; + Float::cosh(self) -> Self; + Float::tanh(self) -> Self; + Float::asinh(self) -> Self; + Float::acosh(self) -> Self; + Float::atanh(self) -> Self; + } +} diff --git a/third_party/crates/num-traits-0.2.19/src/sign.rs b/third_party/crates/num-traits-0.2.19/src/sign.rs new file mode 100644 index 0000000..a0d6b0f --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/src/sign.rs @@ -0,0 +1,216 @@ +use core::num::Wrapping; +use core::ops::Neg; + +use crate::float::FloatCore; +use crate::Num; + +/// Useful functions for signed numbers (i.e. numbers that can be negative). +pub trait Signed: Sized + Num + Neg { + /// Computes the absolute value. + /// + /// For `f32` and `f64`, `NaN` will be returned if the number is `NaN`. + /// + /// For signed integers, `::MIN` will be returned if the number is `::MIN`. + fn abs(&self) -> Self; + + /// The positive difference of two numbers. + /// + /// Returns `zero` if the number is less than or equal to `other`, otherwise the difference + /// between `self` and `other` is returned. + fn abs_sub(&self, other: &Self) -> Self; + + /// Returns the sign of the number. + /// + /// For `f32` and `f64`: + /// + /// * `1.0` if the number is positive, `+0.0` or `INFINITY` + /// * `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY` + /// * `NaN` if the number is `NaN` + /// + /// For signed integers: + /// + /// * `0` if the number is zero + /// * `1` if the number is positive + /// * `-1` if the number is negative + fn signum(&self) -> Self; + + /// Returns true if the number is positive and false if the number is zero or negative. + fn is_positive(&self) -> bool; + + /// Returns true if the number is negative and false if the number is zero or positive. + fn is_negative(&self) -> bool; +} + +macro_rules! signed_impl { + ($($t:ty)*) => ($( + impl Signed for $t { + #[inline] + fn abs(&self) -> $t { + if self.is_negative() { -*self } else { *self } + } + + #[inline] + fn abs_sub(&self, other: &$t) -> $t { + if *self <= *other { 0 } else { *self - *other } + } + + #[inline] + fn signum(&self) -> $t { + match *self { + n if n > 0 => 1, + 0 => 0, + _ => -1, + } + } + + #[inline] + fn is_positive(&self) -> bool { *self > 0 } + + #[inline] + fn is_negative(&self) -> bool { *self < 0 } + } + )*) +} + +signed_impl!(isize i8 i16 i32 i64 i128); + +impl Signed for Wrapping +where + Wrapping: Num + Neg>, +{ + #[inline] + fn abs(&self) -> Self { + Wrapping(self.0.abs()) + } + + #[inline] + fn abs_sub(&self, other: &Self) -> Self { + Wrapping(self.0.abs_sub(&other.0)) + } + + #[inline] + fn signum(&self) -> Self { + Wrapping(self.0.signum()) + } + + #[inline] + fn is_positive(&self) -> bool { + self.0.is_positive() + } + + #[inline] + fn is_negative(&self) -> bool { + self.0.is_negative() + } +} + +macro_rules! signed_float_impl { + ($t:ty) => { + impl Signed for $t { + /// Computes the absolute value. Returns `NAN` if the number is `NAN`. + #[inline] + fn abs(&self) -> $t { + FloatCore::abs(*self) + } + + /// The positive difference of two numbers. Returns `0.0` if the number is + /// less than or equal to `other`, otherwise the difference between`self` + /// and `other` is returned. + #[inline] + fn abs_sub(&self, other: &$t) -> $t { + if *self <= *other { + 0. + } else { + *self - *other + } + } + + /// # Returns + /// + /// - `1.0` if the number is positive, `+0.0` or `INFINITY` + /// - `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY` + /// - `NAN` if the number is NaN + #[inline] + fn signum(&self) -> $t { + FloatCore::signum(*self) + } + + /// Returns `true` if the number is positive, including `+0.0` and `INFINITY` + #[inline] + fn is_positive(&self) -> bool { + FloatCore::is_sign_positive(*self) + } + + /// Returns `true` if the number is negative, including `-0.0` and `NEG_INFINITY` + #[inline] + fn is_negative(&self) -> bool { + FloatCore::is_sign_negative(*self) + } + } + }; +} + +signed_float_impl!(f32); +signed_float_impl!(f64); + +/// Computes the absolute value. +/// +/// For `f32` and `f64`, `NaN` will be returned if the number is `NaN` +/// +/// For signed integers, `::MIN` will be returned if the number is `::MIN`. +#[inline(always)] +pub fn abs(value: T) -> T { + value.abs() +} + +/// The positive difference of two numbers. +/// +/// Returns zero if `x` is less than or equal to `y`, otherwise the difference +/// between `x` and `y` is returned. +#[inline(always)] +pub fn abs_sub(x: T, y: T) -> T { + x.abs_sub(&y) +} + +/// Returns the sign of the number. +/// +/// For `f32` and `f64`: +/// +/// * `1.0` if the number is positive, `+0.0` or `INFINITY` +/// * `-1.0` if the number is negative, `-0.0` or `NEG_INFINITY` +/// * `NaN` if the number is `NaN` +/// +/// For signed integers: +/// +/// * `0` if the number is zero +/// * `1` if the number is positive +/// * `-1` if the number is negative +#[inline(always)] +pub fn signum(value: T) -> T { + value.signum() +} + +/// A trait for values which cannot be negative +pub trait Unsigned: Num {} + +macro_rules! empty_trait_impl { + ($name:ident for $($t:ty)*) => ($( + impl $name for $t {} + )*) +} + +empty_trait_impl!(Unsigned for usize u8 u16 u32 u64 u128); + +impl Unsigned for Wrapping where Wrapping: Num {} + +#[test] +fn unsigned_wrapping_is_unsigned() { + fn require_unsigned(_: &T) {} + require_unsigned(&Wrapping(42_u32)); +} + +#[test] +fn signed_wrapping_is_signed() { + fn require_signed(_: &T) {} + require_signed(&Wrapping(-42)); +} diff --git a/third_party/crates/num-traits-0.2.19/tests/cast.rs b/third_party/crates/num-traits-0.2.19/tests/cast.rs new file mode 100644 index 0000000..4f01d74 --- /dev/null +++ b/third_party/crates/num-traits-0.2.19/tests/cast.rs @@ -0,0 +1,387 @@ +//! Tests of `num_traits::cast`. + +#![cfg_attr(not(feature = "std"), no_std)] + +use num_traits::cast::*; +use num_traits::Bounded; + +use core::{f32, f64}; +use core::{i128, i16, i32, i64, i8, isize}; +use core::{u128, u16, u32, u64, u8, usize}; + +use core::fmt::Debug; +use core::mem; +use core::num::Wrapping; + +#[test] +fn to_primitive_float() { + let f32_toolarge = 1e39f64; + assert_eq!(f32_toolarge.to_f32(), Some(f32::INFINITY)); + assert_eq!((-f32_toolarge).to_f32(), Some(f32::NEG_INFINITY)); + assert_eq!((f32::MAX as f64).to_f32(), Some(f32::MAX)); + assert_eq!((-f32::MAX as f64).to_f32(), Some(-f32::MAX)); + assert_eq!(f64::INFINITY.to_f32(), Some(f32::INFINITY)); + assert_eq!((f64::NEG_INFINITY).to_f32(), Some(f32::NEG_INFINITY)); + assert!((f64::NAN).to_f32().map_or(false, |f| f.is_nan())); +} + +#[test] +fn wrapping_to_primitive() { + macro_rules! test_wrapping_to_primitive { + ($($t:ty)+) => { + $({ + let i: $t = 0; + let w = Wrapping(i); + assert_eq!(i.to_u8(), w.to_u8()); + assert_eq!(i.to_u16(), w.to_u16()); + assert_eq!(i.to_u32(), w.to_u32()); + assert_eq!(i.to_u64(), w.to_u64()); + assert_eq!(i.to_usize(), w.to_usize()); + assert_eq!(i.to_i8(), w.to_i8()); + assert_eq!(i.to_i16(), w.to_i16()); + assert_eq!(i.to_i32(), w.to_i32()); + assert_eq!(i.to_i64(), w.to_i64()); + assert_eq!(i.to_isize(), w.to_isize()); + assert_eq!(i.to_f32(), w.to_f32()); + assert_eq!(i.to_f64(), w.to_f64()); + })+ + }; + } + + test_wrapping_to_primitive!(usize u8 u16 u32 u64 isize i8 i16 i32 i64); +} + +#[test] +fn wrapping_is_toprimitive() { + fn require_toprimitive(_: &T) {} + require_toprimitive(&Wrapping(42)); +} + +#[test] +fn wrapping_is_fromprimitive() { + fn require_fromprimitive(_: &T) {} + require_fromprimitive(&Wrapping(42)); +} + +#[test] +fn wrapping_is_numcast() { + fn require_numcast(_: &T) {} + require_numcast(&Wrapping(42)); +} + +#[test] +fn as_primitive() { + let x: f32 = (1.625f64).as_(); + assert_eq!(x, 1.625f32); + + let x: f32 = (3.14159265358979323846f64).as_(); + assert_eq!(x, 3.1415927f32); + + let x: u8 = (768i16).as_(); + assert_eq!(x, 0); +} + +#[test] +fn float_to_integer_checks_overflow() { + // This will overflow an i32 + let source: f64 = 1.0e+123f64; + + // Expect the overflow to be caught + assert_eq!(cast::(source), None); +} + +#[test] +fn cast_to_int_checks_overflow() { + let big_f: f64 = 1.0e123; + let normal_f: f64 = 1.0; + let small_f: f64 = -1.0e123; + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + + assert_eq!(Some(normal_f as isize), cast::(normal_f)); + assert_eq!(Some(normal_f as i8), cast::(normal_f)); + assert_eq!(Some(normal_f as i16), cast::(normal_f)); + assert_eq!(Some(normal_f as i32), cast::(normal_f)); + assert_eq!(Some(normal_f as i64), cast::(normal_f)); + + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); +} + +#[test] +fn cast_to_unsigned_int_checks_overflow() { + let big_f: f64 = 1.0e123; + let normal_f: f64 = 1.0; + let small_f: f64 = -1.0e123; + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + + assert_eq!(Some(normal_f as usize), cast::(normal_f)); + assert_eq!(Some(normal_f as u8), cast::(normal_f)); + assert_eq!(Some(normal_f as u16), cast::(normal_f)); + assert_eq!(Some(normal_f as u32), cast::(normal_f)); + assert_eq!(Some(normal_f as u64), cast::(normal_f)); + + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); +} + +#[test] +fn cast_to_i128_checks_overflow() { + let big_f: f64 = 1.0e123; + let normal_f: f64 = 1.0; + let small_f: f64 = -1.0e123; + assert_eq!(None, cast::(big_f)); + assert_eq!(None, cast::(big_f)); + + assert_eq!(Some(normal_f as i128), cast::(normal_f)); + assert_eq!(Some(normal_f as u128), cast::(normal_f)); + + assert_eq!(None, cast::(small_f)); + assert_eq!(None, cast::(small_f)); +} + +#[cfg(feature = "std")] +fn dbg(args: ::core::fmt::Arguments<'_>) { + println!("{}", args); +} + +#[cfg(not(feature = "std"))] +fn dbg(_: ::core::fmt::Arguments) {} + +// Rust 1.8 doesn't handle cfg on macros correctly +macro_rules! dbg { ($($tok:tt)*) => { dbg(format_args!($($tok)*)) } } + +macro_rules! float_test_edge { + ($f:ident -> $($t:ident)+) => { $({ + dbg!("testing cast edge cases for {} -> {}", stringify!($f), stringify!($t)); + + let small = if $t::MIN == 0 || mem::size_of::<$t>() < mem::size_of::<$f>() { + $t::MIN as $f - 1.0 + } else { + ($t::MIN as $f).raw_inc().floor() + }; + let fmin = small.raw_dec(); + dbg!(" testing min {}\n\tvs. {:.0}\n\tand {:.0}", $t::MIN, fmin, small); + assert_eq!(Some($t::MIN), cast::<$f, $t>($t::MIN as $f)); + assert_eq!(Some($t::MIN), cast::<$f, $t>(fmin)); + assert_eq!(None, cast::<$f, $t>(small)); + + let (max, large) = if mem::size_of::<$t>() < mem::size_of::<$f>() { + ($t::MAX, $t::MAX as $f + 1.0) + } else { + let large = $t::MAX as $f; // rounds up! + let max = large.raw_dec() as $t; // the next smallest possible + assert_eq!(max.count_ones(), $f::MANTISSA_DIGITS); + (max, large) + }; + let fmax = large.raw_dec(); + dbg!(" testing max {}\n\tvs. {:.0}\n\tand {:.0}", max, fmax, large); + assert_eq!(Some(max), cast::<$f, $t>(max as $f)); + assert_eq!(Some(max), cast::<$f, $t>(fmax)); + assert_eq!(None, cast::<$f, $t>(large)); + + dbg!(" testing non-finite values"); + assert_eq!(None, cast::<$f, $t>($f::NAN)); + assert_eq!(None, cast::<$f, $t>($f::INFINITY)); + assert_eq!(None, cast::<$f, $t>($f::NEG_INFINITY)); + })+} +} + +trait RawOffset: Sized { + fn raw_inc(self) -> Self; + fn raw_dec(self) -> Self; +} + +impl RawOffset for f32 { + fn raw_inc(self) -> Self { + Self::from_bits(self.to_bits() + 1) + } + + fn raw_dec(self) -> Self { + Self::from_bits(self.to_bits() - 1) + } +} + +impl RawOffset for f64 { + fn raw_inc(self) -> Self { + Self::from_bits(self.to_bits() + 1) + } + + fn raw_dec(self) -> Self { + Self::from_bits(self.to_bits() - 1) + } +} + +#[test] +fn cast_float_to_int_edge_cases() { + float_test_edge!(f32 -> isize i8 i16 i32 i64); + float_test_edge!(f32 -> usize u8 u16 u32 u64); + float_test_edge!(f64 -> isize i8 i16 i32 i64); + float_test_edge!(f64 -> usize u8 u16 u32 u64); +} + +#[test] +fn cast_float_to_i128_edge_cases() { + float_test_edge!(f32 -> i128 u128); + float_test_edge!(f64 -> i128 u128); +} + +macro_rules! int_test_edge { + ($f:ident -> { $($t:ident)+ } with $BigS:ident $BigU:ident ) => { $({ + #[allow(arithmetic_overflow)] // https://github.com/rust-lang/rust/issues/109731 + fn test_edge() { + dbg!("testing cast edge cases for {} -> {}", stringify!($f), stringify!($t)); + + match ($f::MIN as $BigS).cmp(&($t::MIN as $BigS)) { + Greater => { + assert_eq!(Some($f::MIN as $t), cast::<$f, $t>($f::MIN)); + } + Equal => { + assert_eq!(Some($t::MIN), cast::<$f, $t>($f::MIN)); + } + Less => { + let min = $t::MIN as $f; + assert_eq!(Some($t::MIN), cast::<$f, $t>(min)); + assert_eq!(None, cast::<$f, $t>(min - 1)); + } + } + + match ($f::MAX as $BigU).cmp(&($t::MAX as $BigU)) { + Greater => { + let max = $t::MAX as $f; + assert_eq!(Some($t::MAX), cast::<$f, $t>(max)); + assert_eq!(None, cast::<$f, $t>(max + 1)); + } + Equal => { + assert_eq!(Some($t::MAX), cast::<$f, $t>($f::MAX)); + } + Less => { + assert_eq!(Some($f::MAX as $t), cast::<$f, $t>($f::MAX)); + } + } + } + test_edge(); + })+} +} + +#[test] +fn cast_int_to_int_edge_cases() { + use core::cmp::Ordering::*; + + macro_rules! test_edge { + ($( $from:ident )+) => { $({ + int_test_edge!($from -> { isize i8 i16 i32 i64 } with i64 u64); + int_test_edge!($from -> { usize u8 u16 u32 u64 } with i64 u64); + })+} + } + + test_edge!(isize i8 i16 i32 i64); + test_edge!(usize u8 u16 u32 u64); +} + +#[test] +fn cast_int_to_128_edge_cases() { + use core::cmp::Ordering::*; + + macro_rules! test_edge { + ($( $t:ident )+) => { + $( + int_test_edge!($t -> { i128 u128 } with i128 u128); + )+ + int_test_edge!(i128 -> { $( $t )+ } with i128 u128); + int_test_edge!(u128 -> { $( $t )+ } with i128 u128); + } + } + + test_edge!(isize i8 i16 i32 i64 i128); + test_edge!(usize u8 u16 u32 u64 u128); +} + +#[test] +fn newtype_from_primitive() { + #[derive(PartialEq, Debug)] + struct New(T); + + // minimal impl + impl FromPrimitive for New { + fn from_i64(n: i64) -> Option { + T::from_i64(n).map(New) + } + + fn from_u64(n: u64) -> Option { + T::from_u64(n).map(New) + } + } + + macro_rules! assert_eq_from { + ($( $from:ident )+) => {$( + assert_eq!(T::$from(Bounded::min_value()).map(New), + New::::$from(Bounded::min_value())); + assert_eq!(T::$from(Bounded::max_value()).map(New), + New::::$from(Bounded::max_value())); + )+} + } + + fn check() { + assert_eq_from!(from_i8 from_i16 from_i32 from_i64 from_isize); + assert_eq_from!(from_u8 from_u16 from_u32 from_u64 from_usize); + assert_eq_from!(from_f32 from_f64); + } + + macro_rules! check { + ($( $ty:ty )+) => {$( check::<$ty>(); )+} + } + check!(i8 i16 i32 i64 isize); + check!(u8 u16 u32 u64 usize); +} + +#[test] +fn newtype_to_primitive() { + #[derive(PartialEq, Debug)] + struct New(T); + + // minimal impl + impl ToPrimitive for New { + fn to_i64(&self) -> Option { + self.0.to_i64() + } + + fn to_u64(&self) -> Option { + self.0.to_u64() + } + } + + macro_rules! assert_eq_to { + ($( $to:ident )+) => {$( + assert_eq!(T::$to(&Bounded::min_value()), + New::::$to(&New(Bounded::min_value()))); + assert_eq!(T::$to(&Bounded::max_value()), + New::::$to(&New(Bounded::max_value()))); + )+} + } + + fn check() { + assert_eq_to!(to_i8 to_i16 to_i32 to_i64 to_isize); + assert_eq_to!(to_u8 to_u16 to_u32 to_u64 to_usize); + assert_eq_to!(to_f32 to_f64); + } + + macro_rules! check { + ($( $ty:ty )+) => {$( check::<$ty>(); )+} + } + check!(i8 i16 i32 i64 isize); + check!(u8 u16 u32 u64 usize); +} diff --git a/third_party/crates/regex-1.13.1/.cargo-checksum.json b/third_party/crates/regex-1.13.1/.cargo-checksum.json new file mode 100644 index 0000000..d32b0ee --- /dev/null +++ b/third_party/crates/regex-1.13.1/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"26ae919976d9001afc67c8e1e4e4eedd691565299de17c61cdeb3f8aa210a649","CHANGELOG.md":"c8b5fe791459cf72a0a26b76bf3d736670968011b377f99a414fbad27583c61d","Cargo.lock":"56fe5fef9799c00ffc8e9ca37577c0b78708fe3b0e4a9a7abfe9931b08111d24","Cargo.toml":"d6695aba00bda6017a25464bdf2c8e8c0d7e769389416d03d53de58cb62dc873","Cargo.toml.orig":"837906b45d826e12c4ca04a5e11c729e40e76e137020907bb8e0dfe375e619d3","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb","README.md":"ce1a8faf72cf8f0e28a28658c59033b02726e3caecd9a3d358b4c6f6cd3e6c24","UNICODE.md":"91ee848bf40a67626940d242d3ef05e90c7d5ef72d23bcf626033b5394aee0ea","bench/README.md":"0aee42206b0e5edcb400a11faa2c536f512bcc6086e5ffdda001b9bfe4d19808","src/builders.rs":"d08f5867d8b994395546e318860d05e00cd70347223505b43d578b8d1477fe8f","src/bytes.rs":"5c28739462f73ac36110723c264f3308d36ecbb0d5897df2fd9cdde4f451db93","src/error.rs":"362c126a701852b355906acdb2c19ee31230570a408bbe52deb2803a1dc77039","src/find_byte.rs":"e17cd3b765467685946707840b92ea4e37d3c11081fbf316174a15858cd4bd99","src/lib.rs":"e87192b2e862d3a01777307584c7c898b189c4d78e6f7a800f52d7e84c3aab1c","src/pattern.rs":"53971d02dde4f8e69055c36e7c56c6c872f0302161bf0977a02b97dc8a152d46","src/regex/bytes.rs":"ffbaa431baaa6ad23ac796e3f46e691b0a6567fed632c460fe052e5abe875c36","src/regex/mod.rs":"c220b6dd7a5e1945f8e743d1dcd796c5f782c91b0c34eb9915c588174a517fe8","src/regex/string.rs":"93bcceb4b9fb1f6240c792fd81eeb53f82624cc5cb16401cbaec96e69753fd85","src/regexset/bytes.rs":"25c8d896e4b9caf627cce46e3c305d2e640aeeacea96c40526699f86960d1868","src/regexset/mod.rs":"c220b6dd7a5e1945f8e743d1dcd796c5f782c91b0c34eb9915c588174a517fe8","src/regexset/string.rs":"ac3fc9c8d2d58379e63bcd92ab2f8ee1c32a1210dceec63925d0c23f1d9dfedd","testdata/anchored.toml":"7a1b5cd81deed2099796a451bf764a3f9bd21f0d60c0fa46accd3a35666866f2","testdata/bytes.toml":"1d84179165fd25f3b94bd2bfbeb43fc8a162041f7bf98b717e0f85cef7fb652b","testdata/crazy.toml":"a146e2d2e23f1a57168979d9b1fc193c2ba38dca66294b61140d6d2a2958ec86","testdata/crlf.toml":"d19cf22756434d145dd20946c00af01c102a556a252070405c3c8294129d9ece","testdata/earliest.toml":"d561e643623ee1889b5b049fdcf3c7cb71b0c746d7eb822ddbd09d0acda2620b","testdata/empty.toml":"738dbe92fbd8971385a1cf3affb0e956e5b692c858b9b48439d718f10801c08e","testdata/expensive.toml":"5ce2f60209c99cdd2cdcb9d3069d1d5ca13d5e08a85e913efe57267b2f5f0e9d","testdata/flags.toml":"9a7e001808195c84f2a7d3e18bc0a82c7386e60f03a616e99af00c3f7f2c3fd4","testdata/iter.toml":"6875460302974a5b3073a7304a865c45aba9653c54afea2c4d26e1ea248a81f7","testdata/leftmost-all.toml":"903bfbeff888b7664296f4d5aa367ce53d1dafe249ab0a3359223ae94d596396","testdata/line-terminator.toml":"02148068137b69d95587966917bdf0697bf7eb41ad6d47387f2eb30f67d04fd9","testdata/misc.toml":"32c9591655c6fb118dfefcb4de49a04820a63cb960533dfc2538cdaabf4f4047","testdata/multiline.toml":"eb07cf5427e6ddbcf61f4cc64c2d74ff41b5ef75ef857959651b20196f3cd157","testdata/no-unicode.toml":"d209da04506900fd5f69e48170cddaad0702355ac6176c3a75ab3ff96974457c","testdata/overlapping.toml":"5d96497a7233566d40b05ba22047e483fa8662e45515a9be86da45cf6c28703a","testdata/regex-lite.toml":"fecca7cc8c9cea2e1f84f846a89fd9b3ca7011c83698211a2eeda8924deb900c","testdata/regression.toml":"0203facd366c50adb2120d87ea5da3a4d3effcb7cdd446c0d3f08ea472c64989","testdata/set.toml":"dfd265dc1aee80026e881616840df0236ae9abf12467d7ec0e141a52c236128c","testdata/substring.toml":"48122d9f3477ed81f95e3ad42c06e9bb25f849b66994601a75ceae0693b81866","testdata/unicode.toml":"7e4b013039b0cdd85fa73f32d15d096182fe901643d4e40c0910087a736cd46d","testdata/utf8.toml":"2eabce0582bcacb2073e08bbe7ca413f096d14d06e917b107949691e24f84b20","testdata/word-boundary-special.toml":"7d0ea2f796478d1ca2a6954430cb1cfbd04031a182f8611cb50a7c73e443ce33","testdata/word-boundary.toml":"51bc1c498ab825420340a2dd3e6623de4054937ba6d5020ff8cd14b1c1e45271","tests/lib.rs":"9bffc95568c09ac95b6a3e7ca64b6e858a0552d0c0b0fca2c447da3b9c0a45a2","tests/misc.rs":"7d40d6690846dd54c80ffb6b544fe577db5a61381c566763161b3a92a0051475","tests/regression.rs":"3490aac99fdbf3f0949ba1f338d5184a84b505ebd96d0b6d6145c610587aa60b","tests/regression_fuzz.rs":"57e0bcba0fdfa7797865e35ae547cd7fe1c6132b80a7bfdfb06eb053a568b00d","tests/replace.rs":"78ff9bf7f78783ad83a78041bb7ee0705c7efc85b4d12301581d0ce5b2a59325","tests/searcher.rs":"04152e5c86431deec0c196d2564a11bc4ec36f14c77e8c16a2f9d1cbc9fc574e","tests/suite_bytes.rs":"75fb0a332527c36d31e126f6032c8ccf1f81ba47ee785affa834404bc1a79f4c","tests/suite_bytes_set.rs":"db85513e87429fc68904270a0f414e75ae0b7c6b7deb1c66f05eb4f98b09c67a","tests/suite_string.rs":"249d707dba99d23ada40558a7526f028c1d3fdf715d3866a106f3435da01bf66","tests/suite_string_set.rs":"c839fb3c08a23348230591a49118406373353bc6c9a87528f36e4e9635e7b9ac"},"package":"f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"} \ No newline at end of file diff --git a/third_party/crates/regex-1.13.1/.cargo_vcs_info.json b/third_party/crates/regex-1.13.1/.cargo_vcs_info.json new file mode 100644 index 0000000..80a0aeb --- /dev/null +++ b/third_party/crates/regex-1.13.1/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "2b527599eb9eea0dcc288c704584f242f26a5c61" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/third_party/crates/regex-1.13.1/BUILD.bazel b/third_party/crates/regex-1.13.1/BUILD.bazel new file mode 100644 index 0000000..cbf5d5d --- /dev/null +++ b/third_party/crates/regex-1.13.1/BUILD.bazel @@ -0,0 +1,135 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_library( + name = "regex", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "default", + "perf", + "perf-backtrack", + "perf-cache", + "perf-dfa", + "perf-inline", + "perf-literal", + "perf-onepass", + "std", + "unicode", + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", + ], + crate_root = "src/lib.rs", + edition = "2021", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=regex", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "1.13.1", + deps = [ + "//third_party/crates/aho-corasick-1.1.5:aho_corasick", + "//third_party/crates/memchr-2.8.3:memchr", + "//third_party/crates/regex-automata-0.4.18:regex_automata", + "//third_party/crates/regex-syntax-0.8.11:regex_syntax", + ], +) diff --git a/third_party/crates/regex-1.13.1/CHANGELOG.md b/third_party/crates/regex-1.13.1/CHANGELOG.md new file mode 100644 index 0000000..fa074bc --- /dev/null +++ b/third_party/crates/regex-1.13.1/CHANGELOG.md @@ -0,0 +1,1813 @@ +1.13.1 (2026-07-15) +=================== +This is a release that fixes a bug where incorrect regex match offsets could be +reported. Note that this doesn't impact whether a match occurs or not, just +where it occurs. The match offsets are still valid for slicing, they just may +not refer to the correct leftmost-first match. See +[#1364](https://github.com/rust-lang/regex/pull/1364) for (many) more details. + +Bug fixes: + +* [#1354](https://github.com/rust-lang/regex/issues/1354): +Fixes previously unsound reverse suffix and inner optimizations. + + +1.13.0 (2026-07-09) +=================== +This release includes a new API, a `regex!` macro, for lazy compilation of +a regex from a string literal. If you use regexes a lot, it's likely you've +already written one exactly like it. The new macro can be used like this: + +```rust +use regex::regex; + +fn is_match(line: &str) -> bool { + // The regex will be compiled approximately once and reused automatically. + // This avoids the footgun of using `Regex::new` here, which would + // guarantee that it would be compiled every time this routine is called. + // This would likely make this routine much slower than it needs to be. + regex!(r"bar|baz").is_match(line) +} + +let hay = "\ +path/to/foo:54:Blue Harvest +path/to/bar:90:Something, Something, Something, Dark Side +path/to/baz:3:It's a Trap! +"; + +let matches = hay.lines().filter(|line| is_match(line)).count(); +assert_eq!(matches, 2); +``` + +Improvements: + +* [#709](https://github.com/rust-lang/regex/issues/709): +Add a new `regex!` macro for efficient and automatic reuse of a compiled regex. + + +1.12.4 (2026-06-09) +=================== +This release includes a performance optimization for compilation of regexes +with very large character classes. + +Improvements: + +* [#1308](https://github.com/rust-lang/regex/pull/1308): +Avoid re-canonicalizing the entire interval set when pushing new class ranges. + + +1.12.3 (2026-02-03) +=================== +This release excludes some unnecessary things from the archive published to +crates.io. Specifically, fuzzing data and various shell scripts are now +excluded. If you run into problems, please file an issue. + +Improvements: + +* [#1319](https://github.com/rust-lang/regex/pull/1319): +Switch from a Cargo `exclude` list to an `include` list, and exclude some +unnecessary stuff. + + +1.12.2 (2025-10-13) +=================== +This release fixes a `cargo doc` breakage on nightly when `--cfg docsrs` is +enabled. This caused documentation to fail to build on docs.rs. + +Bug fixes: + +* [BUG #1305](https://github.com/rust-lang/regex/issues/1305): +Switches the `doc_auto_cfg` feature to `doc_cfg` on nightly for docs.rs builds. + + +1.12.1 (2025-10-10) +=================== +This release makes a bug fix in the new `regex::Captures::get_match` API +introduced in `1.12.0`. There was an oversight with the lifetime parameter +for the `Match` returned. This is technically a breaking change, but given +that it was caught almost immediately and I've yanked the `1.12.0` release, +I think this is fine. + + +1.12.0 (2025-10-10) +=================== +This release contains a smattering of bug fixes, a fix for excessive memory +consumption in some cases and a new `regex::Captures::get_match` API. + +Improvements: + +* [FEATURE #1146](https://github.com/rust-lang/regex/issues/1146): +Add `Capture::get_match` for returning the overall match without `unwrap()`. + +Bug fixes: + +* [BUG #1083](https://github.com/rust-lang/regex/issues/1083): +Fixes a panic in the lazy DFA (can only occur for especially large regexes). +* [BUG #1116](https://github.com/rust-lang/regex/issues/1116): +Fixes a memory usage regression for large regexes (introduced in `regex 1.9`). +* [BUG #1195](https://github.com/rust-lang/regex/issues/1195): +Fix universal start states in sparse DFA. +* [BUG #1295](https://github.com/rust-lang/regex/pull/1295): +Fixes a panic when deserializing a corrupted dense DFA. +* [BUG 8f5d9479](https://github.com/rust-lang/regex/commit/8f5d9479d0f1da5726488a530d7fd66a73d05b80): +Make `regex_automata::meta::Regex::find` consistently return `None` when +`WhichCaptures::None` is used. + + +1.11.3 (2025-09-25) +=================== +This is a small patch release with an improvement in memory usage in some +cases. + +Improvements: + +* [BUG #1297](https://github.com/rust-lang/regex/issues/1297): +Improve memory usage by trimming excess memory capacity in some spots. + + +1.11.2 (2025-08-24) +=================== +This is a new patch release of `regex` with some minor fixes. A larger number +of typo or lint fix patches were merged. Also, we now finally recommend using +`std::sync::LazyLock`. + +Improvements: + +* [BUG #1217](https://github.com/rust-lang/regex/issues/1217): +Switch recommendation from `once_cell` to `std::sync::LazyLock`. +* [BUG #1225](https://github.com/rust-lang/regex/issues/1225): +Add `DFA::set_prefilter` to `regex-automata`. + +Bug fixes: + +* [BUG #1165](https://github.com/rust-lang/regex/pull/1150): +Remove `std` dependency from `perf-literal-multisubstring` crate feature. +* [BUG #1165](https://github.com/rust-lang/regex/pull/1165): +Clarify the meaning of `(?R)$` in the documentation. +* [BUG #1281](https://github.com/rust-lang/regex/pull/1281): +Remove `fuzz/` and `record/` directories from published crate on crates.io. + + +1.11.1 (2024-10-24) +=================== +This is a new patch release of `regex` that fixes compilation on nightly +Rust when the unstable `pattern` crate feature is enabled. Users on nightly +Rust without this feature enabled are unaffected. + +Bug fixes: + +* [BUG #1231](https://github.com/rust-lang/regex/issues/1231): +Fix the `Pattern` trait implementation as a result of nightly API breakage. + + +1.11.0 (2024-09-29) +=================== +This is a new minor release of `regex` that brings in an update to the +Unicode Character Database. Specifically, this updates the Unicode data +used by `regex` internally to the version 16 release. + +New features: + +* [FEATURE #1228](https://github.com/rust-lang/regex/pull/1228): +Add new `regex::SetMatches::matched_all` method. +* [FEATURE #1229](https://github.com/rust-lang/regex/pull/1229): +Update to Unicode Character Database (UCD) version 16. + + +1.10.6 (2024-08-02) +=================== +This is a new patch release with a fix for the `unstable` crate feature that +enables `std::str::Pattern` trait integration. + +Bug fixes: + +* [BUG #1219](https://github.com/rust-lang/regex/pull/1219): +Fix the `Pattern` trait implementation as a result of nightly API breakage. + + +1.10.5 (2024-06-09) +=================== +This is a new patch release with some minor fixes. + +Bug fixes: + +* [BUG #1203](https://github.com/rust-lang/regex/pull/1203): +Escape invalid UTF-8 when in the `Debug` impl of `regex::bytes::Match`. + + +1.10.4 (2024-03-22) +=================== +This is a new patch release with some minor fixes. + +* [BUG #1169](https://github.com/rust-lang/regex/issues/1169): +Fixes a bug with compiling a reverse NFA automaton in `regex-automata`. +* [BUG #1178](https://github.com/rust-lang/regex/pull/1178): +Clarifies that when `Cow::Borrowed` is returned from replace APIs, it is +equivalent to the input. + + +1.10.3 (2024-01-21) +=================== +This is a new patch release that fixes the feature configuration of optional +dependencies, and fixes an unsound use of bounds check elision. + +Bug fixes: + +* [BUG #1147](https://github.com/rust-lang/regex/issues/1147): +Set `default-features=false` for the `memchr` and `aho-corasick` dependencies. +* [BUG #1154](https://github.com/rust-lang/regex/pull/1154): +Fix unsound bounds check elision. + + +1.10.2 (2023-10-16) +=================== +This is a new patch release that fixes a search regression where incorrect +matches could be reported. + +Bug fixes: + +* [BUG #1110](https://github.com/rust-lang/regex/issues/1110): +Revert broadening of reverse suffix literal optimization introduced in 1.10.1. + + +1.10.1 (2023-10-14) +=================== +This is a new patch release with a minor increase in the number of valid +patterns and a broadening of some literal optimizations. + +New features: + +* [FEATURE 04f5d7be](https://github.com/rust-lang/regex/commit/04f5d7be4efc542864cc400f5d43fbea4eb9bab6): +Loosen ASCII-compatible rules such that regexes like `(?-u:☃)` are now allowed. + +Performance improvements: + +* [PERF 8a8d599f](https://github.com/rust-lang/regex/commit/8a8d599f9d2f2d78e9ad84e4084788c2d563afa5): +Broader the reverse suffix optimization to apply in more cases. + + +1.10.0 (2023-10-09) +=================== +This is a new minor release of `regex` that adds support for start and end +word boundary assertions. That is, `\<` and `\>`. The minimum supported Rust +version has also been raised to 1.65, which was released about one year ago. + +The new word boundary assertions are: + +* `\<` or `\b{start}`: a Unicode start-of-word boundary (`\W|\A` on the left, +`\w` on the right). +* `\>` or `\b{end}`: a Unicode end-of-word boundary (`\w` on the left, `\W|\z` +on the right). +* `\b{start-half}`: half of a Unicode start-of-word boundary (`\W|\A` on the +left). +* `\b{end-half}`: half of a Unicode end-of-word boundary (`\W|\z` on the +right). + +The `\<` and `\>` are GNU extensions to POSIX regexes. They have been added +to the `regex` crate because they enjoy somewhat broad support in other regex +engines as well (for example, vim). The `\b{start}` and `\b{end}` assertions +are aliases for `\<` and `\>`, respectively. + +The `\b{start-half}` and `\b{end-half}` assertions are not found in any +other regex engine (although regex engines with general look-around support +can certainly express them). They were added principally to support the +implementation of word matching in grep programs, where one generally wants to +be a bit more flexible in what is considered a word boundary. + +New features: + +* [FEATURE #469](https://github.com/rust-lang/regex/issues/469): +Add support for `\<` and `\>` word boundary assertions. +* [FEATURE(regex-automata) #1031](https://github.com/rust-lang/regex/pull/1031): +DFAs now have a `start_state` method that doesn't use an `Input`. + +Performance improvements: + +* [PERF #1051](https://github.com/rust-lang/regex/pull/1051): +Unicode character class operations have been optimized in `regex-syntax`. +* [PERF #1090](https://github.com/rust-lang/regex/issues/1090): +Make patterns containing lots of literal characters use less memory. + +Bug fixes: + +* [BUG #1046](https://github.com/rust-lang/regex/issues/1046): +Fix a bug that could result in incorrect match spans when using a Unicode word +boundary and searching non-ASCII strings. +* [BUG(regex-syntax) #1047](https://github.com/rust-lang/regex/issues/1047): +Fix panics that can occur in `Ast->Hir` translation (not reachable from `regex` +crate). +* [BUG(regex-syntax) #1088](https://github.com/rust-lang/regex/issues/1088): +Remove guarantees in the API that connect the `u` flag with a specific HIR +representation. + +`regex-automata` breaking change release: + +This release includes a `regex-automata 0.4.0` breaking change release, which +was necessary in order to support the new word boundary assertions. For +example, the `Look` enum has new variants and the `LookSet` type now uses `u32` +instead of `u16` to represent a bitset of look-around assertions. These are +overall very minor changes, and most users of `regex-automata` should be able +to move to `0.4` from `0.3` without any changes at all. + +`regex-syntax` breaking change release: + +This release also includes a `regex-syntax 0.8.0` breaking change release, +which, like `regex-automata`, was necessary in order to support the new word +boundary assertions. This release also includes some changes to the `Ast` +type to reduce heap usage in some cases. If you are using the `Ast` type +directly, your code may require some minor modifications. Otherwise, users of +`regex-syntax 0.7` should be able to migrate to `0.8` without any code changes. + +`regex-lite` release: + +The `regex-lite 0.1.1` release contains support for the new word boundary +assertions. There are no breaking changes. + + +1.9.6 (2023-09-30) +================== +This is a patch release that fixes a panic that can occur when the default +regex size limit is increased to a large number. + +* [BUG aa4e4c71](https://github.com/rust-lang/regex/commit/aa4e4c7120b0090ce0624e3c42a2ed06dd8b918a): +Fix a bug where computing the maximum haystack length for the bounded +backtracker could result underflow and thus provoke a panic later in a search +due to a broken invariant. + + +1.9.5 (2023-09-02) +================== +This is a patch release that hopefully mostly fixes a performance bug that +occurs when sharing a regex across multiple threads. + +Issue [#934](https://github.com/rust-lang/regex/issues/934) +explains this in more detail. It is [also noted in the crate +documentation](https://docs.rs/regex/latest/regex/#sharing-a-regex-across-threads-can-result-in-contention). +The bug can appear when sharing a regex across multiple threads simultaneously, +as might be the case when using a regex from a `OnceLock`, `lazy_static` or +similar primitive. Usually high contention only results when using many threads +to execute searches on small haystacks. + +One can avoid the contention problem entirely through one of two methods. +The first is to use lower level APIs from `regex-automata` that require passing +state explicitly, such as [`meta::Regex::search_with`](https://docs.rs/regex-automata/latest/regex_automata/meta/struct.Regex.html#method.search_with). +The second is to clone a regex and send it to other threads explicitly. This +will not use any additional memory usage compared to sharing the regex. The +only downside of this approach is that it may be less convenient, for example, +it won't work with things like `OnceLock` or `lazy_static` or `once_cell`. + +With that said, as of this release, the contention performance problems have +been greatly reduced. This was achieved by changing the free-list so that it +was sharded across threads, and that ensuring each sharded mutex occupies a +single cache line to mitigate false sharing. So while contention may still +impact performance in some cases, it should be a lot better now. + +Because of the changes to how the free-list works, please report any issues you +find with this release. That not only includes search time regressions but also +significant regressions in memory usage. Reporting improvements is also welcome +as well! If possible, provide a reproduction. + +Bug fixes: + +* [BUG #934](https://github.com/rust-lang/regex/issues/934): +Fix a performance bug where high contention on a single regex led to massive +slow-downs. + + +1.9.4 (2023-08-26) +================== +This is a patch release that fixes a bug where `RegexSet::is_match(..)` could +incorrectly return false (even when `RegexSet::matches(..).matched_any()` +returns true). + +Bug fixes: + +* [BUG #1070](https://github.com/rust-lang/regex/issues/1070): +Fix a bug where a prefilter was incorrectly configured for a `RegexSet`. + + +1.9.3 (2023-08-05) +================== +This is a patch release that fixes a bug where some searches could result in +incorrect match offsets being reported. It is difficult to characterize the +types of regexes susceptible to this bug. They generally involve patterns +that contain no prefix or suffix literals, but have an inner literal along with +a regex prefix that can conditionally match. + +Bug fixes: + +* [BUG #1060](https://github.com/rust-lang/regex/issues/1060): +Fix a bug with the reverse inner literal optimization reporting incorrect match +offsets. + + +1.9.2 (2023-08-05) +================== +This is a patch release that fixes another memory usage regression. This +particular regression occurred only when using a `RegexSet`. In some cases, +much more heap memory (by one or two orders of magnitude) was allocated than in +versions prior to 1.9.0. + +Bug fixes: + +* [BUG #1059](https://github.com/rust-lang/regex/issues/1059): +Fix a memory usage regression when using a `RegexSet`. + + +1.9.1 (2023-07-07) +================== +This is a patch release which fixes a memory usage regression. In the regex +1.9 release, one of the internal engines used a more aggressive allocation +strategy than what was done previously. This patch release reverts to the +prior on-demand strategy. + +Bug fixes: + +* [BUG #1027](https://github.com/rust-lang/regex/issues/1027): +Change the allocation strategy for the backtracker to be less aggressive. + + +1.9.0 (2023-07-05) +================== +This release marks the end of a [years long rewrite of the regex crate +internals](https://github.com/rust-lang/regex/issues/656). Since this is +such a big release, please report any issues or regressions you find. We would +also love to hear about improvements as well. + +In addition to many internal improvements that should hopefully result in +"my regex searches are faster," there have also been a few API additions: + +* A new `Captures::extract` method for quickly accessing the substrings +that match each capture group in a regex. +* A new inline flag, `R`, which enables CRLF mode. This makes `.` match any +Unicode scalar value except for `\r` and `\n`, and also makes `(?m:^)` and +`(?m:$)` match after and before both `\r` and `\n`, respectively, but never +between a `\r` and `\n`. +* `RegexBuilder::line_terminator` was added to further customize the line +terminator used by `(?m:^)` and `(?m:$)` to be any arbitrary byte. +* The `std` Cargo feature is now actually optional. That is, the `regex` crate +can be used without the standard library. +* Because `regex 1.9` may make binary size and compile times even worse, a +new experimental crate called `regex-lite` has been published. It prioritizes +binary size and compile times over functionality (like Unicode) and +performance. It shares no code with the `regex` crate. + +New features: + +* [FEATURE #244](https://github.com/rust-lang/regex/issues/244): +One can opt into CRLF mode via the `R` flag. +e.g., `(?mR:$)` matches just before `\r\n`. +* [FEATURE #259](https://github.com/rust-lang/regex/issues/259): +Multi-pattern searches with offsets can be done with `regex-automata 0.3`. +* [FEATURE #476](https://github.com/rust-lang/regex/issues/476): +`std` is now an optional feature. `regex` may be used with only `alloc`. +* [FEATURE #644](https://github.com/rust-lang/regex/issues/644): +`RegexBuilder::line_terminator` configures how `(?m:^)` and `(?m:$)` behave. +* [FEATURE #675](https://github.com/rust-lang/regex/issues/675): +Anchored search APIs are now available in `regex-automata 0.3`. +* [FEATURE #824](https://github.com/rust-lang/regex/issues/824): +Add new `Captures::extract` method for easier capture group access. +* [FEATURE #961](https://github.com/rust-lang/regex/issues/961): +Add `regex-lite` crate with smaller binary sizes and faster compile times. +* [FEATURE #1022](https://github.com/rust-lang/regex/pull/1022): +Add `TryFrom` implementations for the `Regex` type. + +Performance improvements: + +* [PERF #68](https://github.com/rust-lang/regex/issues/68): +Added a one-pass DFA engine for faster capture group matching. +* [PERF #510](https://github.com/rust-lang/regex/issues/510): +Inner literals are now used to accelerate searches, e.g., `\w+@\w+` will scan +for `@`. +* [PERF #787](https://github.com/rust-lang/regex/issues/787), +[PERF #891](https://github.com/rust-lang/regex/issues/891): +Makes literal optimizations apply to regexes of the form `\b(foo|bar|quux)\b`. + +(There are many more performance improvements as well, but not all of them have +specific issues devoted to them.) + +Bug fixes: + +* [BUG #429](https://github.com/rust-lang/regex/issues/429): +Fix matching bugs related to `\B` and inconsistencies across internal engines. +* [BUG #517](https://github.com/rust-lang/regex/issues/517): +Fix matching bug with capture groups. +* [BUG #579](https://github.com/rust-lang/regex/issues/579): +Fix matching bug with word boundaries. +* [BUG #779](https://github.com/rust-lang/regex/issues/779): +Fix bug where some regexes like `(re)+` were not equivalent to `(re)(re)*`. +* [BUG #850](https://github.com/rust-lang/regex/issues/850): +Fix matching bug inconsistency between NFA and DFA engines. +* [BUG #921](https://github.com/rust-lang/regex/issues/921): +Fix matching bug where literal extraction got confused by `$`. +* [BUG #976](https://github.com/rust-lang/regex/issues/976): +Add documentation to replacement routines about dealing with fallibility. +* [BUG #1002](https://github.com/rust-lang/regex/issues/1002): +Use corpus rejection in fuzz testing. + + +1.8.4 (2023-06-05) +================== +This is a patch release that fixes a bug where `(?-u:\B)` was allowed in +Unicode regexes, despite the fact that the current matching engines can report +match offsets between the code units of a single UTF-8 encoded codepoint. That +in turn means that match offsets that split a codepoint could be reported, +which in turn results in panicking when one uses them to slice a `&str`. + +This bug occurred in the transition to `regex 1.8` because the underlying +syntactical error that prevented this regex from compiling was intentionally +removed. That's because `(?-u:\B)` will be permitted in Unicode regexes in +`regex 1.9`, but the matching engines will guarantee to never report match +offsets that split a codepoint. When the underlying syntactical error was +removed, no code was added to ensure that `(?-u:\B)` didn't compile in the +`regex 1.8` transition release. This release, `regex 1.8.4`, adds that code +such that `Regex::new(r"(?-u:\B)")` returns to the `regex <1.8` behavior of +not compiling. (A `bytes::Regex` can still of course compile it.) + +Bug fixes: + +* [BUG #1006](https://github.com/rust-lang/regex/issues/1006): +Fix a bug where `(?-u:\B)` was allowed in Unicode regexes, and in turn could +lead to match offsets that split a codepoint in `&str`. + + +1.8.3 (2023-05-25) +================== +This is a patch release that fixes a bug where the regex would report a +match at every position even when it shouldn't. This could occur in a very +small subset of regexes, usually an alternation of simple literals that +have particular properties. (See the issue linked below for a more precise +description.) + +Bug fixes: + +* [BUG #999](https://github.com/rust-lang/regex/issues/999): +Fix a bug where a match at every position is erroneously reported. + + +1.8.2 (2023-05-22) +================== +This is a patch release that fixes a bug where regex compilation could panic +in debug mode for regexes with large counted repetitions. For example, +`a{2147483516}{2147483416}{5}` resulted in an integer overflow that wrapped +in release mode but panicking in debug mode. Despite the unintended wrapping +arithmetic in release mode, it didn't cause any other logical bugs since the +errant code was for new analysis that wasn't used yet. + +Bug fixes: + +* [BUG #995](https://github.com/rust-lang/regex/issues/995): +Fix a bug where regex compilation with large counted repetitions could panic. + + +1.8.1 (2023-04-21) +================== +This is a patch release that fixes a bug where a regex match could be reported +where none was found. Specifically, the bug occurs when a pattern contains some +literal prefixes that could be extracted _and_ an optional word boundary in the +prefix. + +Bug fixes: + +* [BUG #981](https://github.com/rust-lang/regex/issues/981): +Fix a bug where a word boundary could interact with prefix literal +optimizations and lead to a false positive match. + + +1.8.0 (2023-04-20) +================== +This is a sizeable release that will be soon followed by another sizeable +release. Both of them will combined close over 40 existing issues and PRs. + +This first release, despite its size, essentially represents preparatory work +for the second release, which will be even bigger. Namely, this release: + +* Increases the MSRV to Rust 1.60.0, which was released about 1 year ago. +* Upgrades its dependency on `aho-corasick` to the recently released 1.0 +version. +* Upgrades its dependency on `regex-syntax` to the simultaneously released +`0.7` version. The changes to `regex-syntax` principally revolve around a +rewrite of its literal extraction code and a number of simplifications and +optimizations to its high-level intermediate representation (HIR). + +The second release, which will follow ~shortly after the release above, will +contain a soup-to-nuts rewrite of every regex engine. This will be done by +bringing [`regex-automata`](https://github.com/BurntSushi/regex-automata) into +this repository, and then changing the `regex` crate to be nothing but an API +shim layer on top of `regex-automata`'s API. + +These tandem releases are the culmination of about 3 +years of on-and-off work that [began in earnest in March +2020](https://github.com/rust-lang/regex/issues/656). + +Because of the scale of changes involved in these releases, I would love to +hear about your experience. Especially if you notice undocumented changes in +behavior or performance changes (positive *or* negative). + +Most changes in the first release are listed below. For more details, please +see the commit log, which reflects a linear and decently documented history +of all changes. + +New features: + +* [FEATURE #501](https://github.com/rust-lang/regex/issues/501): +Permit many more characters to be escaped, even if they have no significance. +More specifically, any ASCII character except for `[0-9A-Za-z<>]` can now be +escaped. Also, a new routine, `is_escapeable_character`, has been added to +`regex-syntax` to query whether a character is escapable or not. +* [FEATURE #547](https://github.com/rust-lang/regex/issues/547): +Add `Regex::captures_at`. This fills a hole in the API, but doesn't otherwise +introduce any new expressive power. +* [FEATURE #595](https://github.com/rust-lang/regex/issues/595): +Capture group names are now Unicode-aware. They can now begin with either a `_` +or any "alphabetic" codepoint. After the first codepoint, subsequent codepoints +can be any sequence of alphanumeric codepoints, along with `_`, `.`, `[` and +`]`. Note that replacement syntax has not changed. +* [FEATURE #810](https://github.com/rust-lang/regex/issues/810): +Add `Match::is_empty` and `Match::len` APIs. +* [FEATURE #905](https://github.com/rust-lang/regex/issues/905): +Add an `impl Default for RegexSet`, with the default being the empty set. +* [FEATURE #908](https://github.com/rust-lang/regex/issues/908): +A new method, `Regex::static_captures_len`, has been added which returns the +number of capture groups in the pattern if and only if every possible match +always contains the same number of matching groups. +* [FEATURE #955](https://github.com/rust-lang/regex/issues/955): +Named captures can now be written as `(?re)` in addition to +`(?Pre)`. +* FEATURE: `regex-syntax` now supports empty character classes. +* FEATURE: `regex-syntax` now has an optional `std` feature. (This will come +to `regex` in the second release.) +* FEATURE: The `Hir` type in `regex-syntax` has had a number of simplifications +made to it. +* FEATURE: `regex-syntax` has support for a new `R` flag for enabling CRLF +mode. This will be supported in `regex` proper in the second release. +* FEATURE: `regex-syntax` now has proper support for "regex that never +matches" via `Hir::fail()`. +* FEATURE: The `hir::literal` module of `regex-syntax` has been completely +re-worked. It now has more documentation, examples and advice. +* FEATURE: The `allow_invalid_utf8` option in `regex-syntax` has been renamed +to `utf8`, and the meaning of the boolean has been flipped. + +Performance improvements: + +* PERF: The upgrade to `aho-corasick 1.0` may improve performance in some +cases. It's difficult to characterize exactly which patterns this might impact, +but if there are a small number of longish (>= 4 bytes) prefix literals, then +it might be faster than before. + +Bug fixes: + +* [BUG #514](https://github.com/rust-lang/regex/issues/514): +Improve `Debug` impl for `Match` so that it doesn't show the entire haystack. +* BUGS [#516](https://github.com/rust-lang/regex/issues/516), +[#731](https://github.com/rust-lang/regex/issues/731): +Fix a number of issues with printing `Hir` values as regex patterns. +* [BUG #610](https://github.com/rust-lang/regex/issues/610): +Add explicit example of `foo|bar` in the regex syntax docs. +* [BUG #625](https://github.com/rust-lang/regex/issues/625): +Clarify that `SetMatches::len` does not (regrettably) refer to the number of +matches in the set. +* [BUG #660](https://github.com/rust-lang/regex/issues/660): +Clarify "verbose mode" in regex syntax documentation. +* BUG [#738](https://github.com/rust-lang/regex/issues/738), +[#950](https://github.com/rust-lang/regex/issues/950): +Fix `CaptureLocations::get` so that it never panics. +* [BUG #747](https://github.com/rust-lang/regex/issues/747): +Clarify documentation for `Regex::shortest_match`. +* [BUG #835](https://github.com/rust-lang/regex/issues/835): +Fix `\p{Sc}` so that it is equivalent to `\p{Currency_Symbol}`. +* [BUG #846](https://github.com/rust-lang/regex/issues/846): +Add more clarifying documentation to the `CompiledTooBig` error variant. +* [BUG #854](https://github.com/rust-lang/regex/issues/854): +Clarify that `regex::Regex` searches as if the haystack is a sequence of +Unicode scalar values. +* [BUG #884](https://github.com/rust-lang/regex/issues/884): +Replace `__Nonexhaustive` variants with `#[non_exhaustive]` attribute. +* [BUG #893](https://github.com/rust-lang/regex/pull/893): +Optimize case folding since it can get quite slow in some pathological cases. +* [BUG #895](https://github.com/rust-lang/regex/issues/895): +Reject `(?-u:\W)` in `regex::Regex` APIs. +* [BUG #942](https://github.com/rust-lang/regex/issues/942): +Add a missing `void` keyword to indicate "no parameters" in C API. +* [BUG #965](https://github.com/rust-lang/regex/issues/965): +Fix `\p{Lc}` so that it is equivalent to `\p{Cased_Letter}`. +* [BUG #975](https://github.com/rust-lang/regex/issues/975): +Clarify documentation for `\pX` syntax. + + +1.7.3 (2023-03-24) +================== +This is a small release that fixes a bug in `Regex::shortest_match_at` that +could cause it to panic, even when the offset given is valid. + +Bug fixes: + +* [BUG #969](https://github.com/rust-lang/regex/issues/969): + Fix a bug in how the reverse DFA was called for `Regex::shortest_match_at`. + + +1.7.2 (2023-03-21) +================== +This is a small release that fixes a failing test on FreeBSD. + +Bug fixes: + +* [BUG #967](https://github.com/rust-lang/regex/issues/967): + Fix "no stack overflow" test which can fail due to the small stack size. + + +1.7.1 (2023-01-09) +================== +This release was done principally to try and fix the doc.rs rendering for the +regex crate. + +Performance improvements: + +* [PERF #930](https://github.com/rust-lang/regex/pull/930): + Optimize `replacen`. This also applies to `replace`, but not `replace_all`. + +Bug fixes: + +* [BUG #945](https://github.com/rust-lang/regex/issues/945): + Maybe fix rustdoc rendering by just bumping a new release? + + +1.7.0 (2022-11-05) +================== +This release principally includes an upgrade to Unicode 15. + +New features: + +* [FEATURE #832](https://github.com/rust-lang/regex/issues/916): + Upgrade to Unicode 15. + + +1.6.0 (2022-07-05) +================== +This release principally includes an upgrade to Unicode 14. + +New features: + +* [FEATURE #832](https://github.com/rust-lang/regex/pull/832): + Clarify that `Captures::len` includes all groups, not just matching groups. +* [FEATURE #857](https://github.com/rust-lang/regex/pull/857): + Add an `ExactSizeIterator` impl for `SubCaptureMatches`. +* [FEATURE #861](https://github.com/rust-lang/regex/pull/861): + Improve `RegexSet` documentation examples. +* [FEATURE #877](https://github.com/rust-lang/regex/issues/877): + Upgrade to Unicode 14. + +Bug fixes: + +* [BUG #792](https://github.com/rust-lang/regex/issues/792): + Fix error message rendering bug. + + +1.5.6 (2022-05-20) +================== +This release includes a few bug fixes, including a bug that produced incorrect +matches when a non-greedy `?` operator was used. + +* [BUG #680](https://github.com/rust-lang/regex/issues/680): + Fixes a bug where `[[:alnum:][:^ascii:]]` dropped `[:alnum:]` from the class. +* [BUG #859](https://github.com/rust-lang/regex/issues/859): + Fixes a bug where `Hir::is_match_empty` returned `false` for `\b`. +* [BUG #862](https://github.com/rust-lang/regex/issues/862): + Fixes a bug where 'ab??' matches 'ab' instead of 'a' in 'ab'. + + +1.5.5 (2022-03-08) +================== +This releases fixes a security bug in the regex compiler. This bug permits a +vector for a denial-of-service attack in cases where the regex being compiled +is untrusted. There are no known problems where the regex is itself trusted, +including in cases of untrusted haystacks. + +* [SECURITY #GHSA-m5pq-gvj9-9vr8](https://github.com/rust-lang/regex/security/advisories/GHSA-m5pq-gvj9-9vr8): + Fixes a bug in the regex compiler where empty sub-expressions subverted the + existing mitigations in place to enforce a size limit on compiled regexes. + The Rust Security Response WG published an advisory about this: + https://groups.google.com/g/rustlang-security-announcements/c/NcNNL1Jq7Yw + + +1.5.4 (2021-05-06) +================== +This release fixes another compilation failure when building regex. This time, +the fix is for when the `pattern` feature is enabled, which only works on +nightly Rust. CI has been updated to test this case. + +* [BUG #772](https://github.com/rust-lang/regex/pull/772): + Fix build when `pattern` feature is enabled. + + +1.5.3 (2021-05-01) +================== +This releases fixes a bug when building regex with only the `unicode-perl` +feature. It turns out that while CI was building this configuration, it wasn't +actually failing the overall build on a failed compilation. + +* [BUG #769](https://github.com/rust-lang/regex/issues/769): + Fix build in `regex-syntax` when only the `unicode-perl` feature is enabled. + + +1.5.2 (2021-05-01) +================== +This release fixes a performance bug when Unicode word boundaries are used. +Namely, for certain regexes on certain inputs, it's possible for the lazy DFA +to stop searching (causing a fallback to a slower engine) when it doesn't +actually need to. + +[PR #768](https://github.com/rust-lang/regex/pull/768) fixes the bug, which was +originally reported in +[ripgrep#1860](https://github.com/BurntSushi/ripgrep/issues/1860). + + +1.5.1 (2021-04-30) +================== +This is a patch release that fixes a compilation error when the `perf-literal` +feature is not enabled. + + +1.5.0 (2021-04-30) +================== +This release primarily updates to Rust 2018 (finally) and bumps the MSRV to +Rust 1.41 (from Rust 1.28). Rust 1.41 was chosen because it's still reasonably +old, and is what's in Debian stable at the time of writing. + +This release also drops this crate's own bespoke substring search algorithms +in favor of a new +[`memmem` implementation provided by the `memchr` crate](https://docs.rs/memchr/2.4.0/memchr/memmem/index.html). +This will change the performance profile of some regexes, sometimes getting a +little worse, and hopefully more frequently, getting a lot better. Please +report any serious performance regressions if you find them. + + +1.4.6 (2021-04-22) +================== +This is a small patch release that fixes the compiler's size check on how much +heap memory a regex uses. Previously, the compiler did not account for the +heap usage of Unicode character classes. Now it does. It's possible that this +may make some regexes fail to compile that previously did compile. If that +happens, please file an issue. + +* [BUG OSS-fuzz#33579](https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=33579): + Some regexes can use more heap memory than one would expect. + + +1.4.5 (2021-03-14) +================== +This is a small patch release that fixes a regression in the size of a `Regex` +in the 1.4.4 release. Prior to 1.4.4, a `Regex` was 552 bytes. In the 1.4.4 +release, it was 856 bytes due to internal changes. In this release, a `Regex` +is now 16 bytes. In general, the size of a `Regex` was never something that was +on my radar, but this increased size in the 1.4.4 release seems to have crossed +a threshold and resulted in stack overflows in some programs. + +* [BUG #750](https://github.com/rust-lang/regex/pull/750): + Fixes stack overflows seemingly caused by a large `Regex` size by decreasing + its size. + + +1.4.4 (2021-03-11) +================== +This is a small patch release that contains some bug fixes. Notably, it also +drops the `thread_local` (and `lazy_static`, via transitivity) dependencies. + +Bug fixes: + +* [BUG #362](https://github.com/rust-lang/regex/pull/362): + Memory leaks caused by an internal caching strategy should now be fixed. +* [BUG #576](https://github.com/rust-lang/regex/pull/576): + All regex types now implement `UnwindSafe` and `RefUnwindSafe`. +* [BUG #728](https://github.com/rust-lang/regex/pull/749): + Add missing `Replacer` impls for `Vec`, `String`, `Cow`, etc. + + +1.4.3 (2021-01-08) +================== +This is a small patch release that adds some missing standard trait +implementations for some types in the public API. + +Bug fixes: + +* [BUG #734](https://github.com/rust-lang/regex/pull/734): + Add `FusedIterator` and `ExactSizeIterator` impls to iterator types. +* [BUG #735](https://github.com/rust-lang/regex/pull/735): + Add missing `Debug` impls to public API types. + + +1.4.2 (2020-11-01) +================== +This is a small bug fix release that bans `\P{any}`. We previously banned empty +classes like `[^\w\W]`, but missed the `\P{any}` case. In the future, we hope +to permit empty classes. + +* [BUG #722](https://github.com/rust-lang/regex/issues/722): + Ban `\P{any}` to avoid a panic in the regex compiler. Found by OSS-Fuzz. + + +1.4.1 (2020-10-13) +================== +This is a small bug fix release that makes `\p{cf}` work. Previously, it would +report "property not found" even though `cf` is a valid abbreviation for the +`Format` general category. + +* [BUG #719](https://github.com/rust-lang/regex/issues/719): + Fixes bug that prevented `\p{cf}` from working. + + +1.4.0 (2020-10-11) +================== +This releases has a few minor documentation fixes as well as some very minor +API additions. The MSRV remains at Rust 1.28 for now, but this is intended to +increase to at least Rust 1.41.1 soon. + +This release also adds support for OSS-Fuzz. Kudos to +[@DavidKorczynski](https://github.com/DavidKorczynski) +for doing the heavy lifting for that! + +New features: + +* [FEATURE #649](https://github.com/rust-lang/regex/issues/649): + Support `[`, `]` and `.` in capture group names. +* [FEATURE #687](https://github.com/rust-lang/regex/issues/687): + Add `is_empty` predicate to `RegexSet`. +* [FEATURE #689](https://github.com/rust-lang/regex/issues/689): + Implement `Clone` for `SubCaptureMatches`. +* [FEATURE #715](https://github.com/rust-lang/regex/issues/715): + Add `empty` constructor to `RegexSet` for convenience. + +Bug fixes: + +* [BUG #694](https://github.com/rust-lang/regex/issues/694): + Fix doc example for `Replacer::replace_append`. +* [BUG #698](https://github.com/rust-lang/regex/issues/698): + Clarify docs for `s` flag when using a `bytes::Regex`. +* [BUG #711](https://github.com/rust-lang/regex/issues/711): + Clarify `is_match` docs to indicate that it can match anywhere in string. + + +1.3.9 (2020-05-28) +================== +This release fixes a MSRV (Minimum Support Rust Version) regression in the +1.3.8 release. Namely, while 1.3.8 compiles on Rust 1.28, it actually does not +compile on other Rust versions, such as Rust 1.39. + +Bug fixes: + +* [BUG #685](https://github.com/rust-lang/regex/issues/685): + Remove use of `doc_comment` crate, which cannot be used before Rust 1.43. + + +1.3.8 (2020-05-28) +================== +This release contains a couple of important bug fixes driven +by better support for empty-subexpressions in regexes. For +example, regexes like `b|` are now allowed. Major thanks to +[@sliquister](https://github.com/sliquister) for implementing support for this +in [#677](https://github.com/rust-lang/regex/pull/677). + +Bug fixes: + +* [BUG #523](https://github.com/rust-lang/regex/pull/523): + Add note to documentation that spaces can be escaped in `x` mode. +* [BUG #524](https://github.com/rust-lang/regex/issues/524): + Add support for empty sub-expressions, including empty alternations. +* [BUG #659](https://github.com/rust-lang/regex/issues/659): + Fix match bug caused by an empty sub-expression miscompilation. + + +1.3.7 (2020-04-17) +================== +This release contains a small bug fix that fixes how `regex` forwards crate +features to `regex-syntax`. In particular, this will reduce recompilations in +some cases. + +Bug fixes: + +* [BUG #665](https://github.com/rust-lang/regex/pull/665): + Fix feature forwarding to `regex-syntax`. + + +1.3.6 (2020-03-24) +================== +This release contains a sizable (~30%) performance improvement when compiling +some kinds of large regular expressions. + +Performance improvements: + +* [PERF #657](https://github.com/rust-lang/regex/pull/657): + Improvement performance of compiling large regular expressions. + + +1.3.5 (2020-03-12) +================== +This release updates this crate to Unicode 13. + +New features: + +* [FEATURE #653](https://github.com/rust-lang/regex/pull/653): + Update `regex-syntax` to Unicode 13. + + +1.3.4 (2020-01-30) +================== +This is a small bug fix release that fixes a bug related to the scoping of +flags in a regex. Namely, before this fix, a regex like `((?i)a)b)` would +match `aB` despite the fact that `b` should not be matched case insensitively. + +Bug fixes: + +* [BUG #640](https://github.com/rust-lang/regex/issues/640): + Fix bug related to the scoping of flags in a regex. + + +1.3.3 (2020-01-09) +================== +This is a small maintenance release that upgrades the dependency on +`thread_local` from `0.3` to `1.0`. The minimum supported Rust version remains +at Rust 1.28. + + +1.3.2 (2020-01-09) +================== +This is a small maintenance release with some house cleaning and bug fixes. + +New features: + +* [FEATURE #631](https://github.com/rust-lang/regex/issues/631): + Add a `Match::range` method an a `From for Range` impl. + +Bug fixes: + +* [BUG #521](https://github.com/rust-lang/regex/issues/521): + Corrects `/-/.splitn("a", 2)` to return `["a"]` instead of `["a", ""]`. +* [BUG #594](https://github.com/rust-lang/regex/pull/594): + Improve error reporting when writing `\p\`. +* [BUG #627](https://github.com/rust-lang/regex/issues/627): + Corrects `/-/.split("a-")` to return `["a", ""]` instead of `["a"]`. +* [BUG #633](https://github.com/rust-lang/regex/pull/633): + Squash deprecation warnings for the `std::error::Error::description` method. + + +1.3.1 (2019-09-04) +================== +This is a maintenance release with no changes in order to try to work around +a [docs.rs/Cargo issue](https://github.com/rust-lang/docs.rs/issues/400). + + +1.3.0 (2019-09-03) +================== +This release adds a plethora of new crate features that permit users of regex +to shrink its size considerably, in exchange for giving up either functionality +(such as Unicode support) or runtime performance. When all such features are +disabled, the dependency tree for `regex` shrinks to exactly 1 crate +(`regex-syntax`). More information about the new crate features can be +[found in the docs](https://docs.rs/regex/*/#crate-features). + +Note that while this is a new minor version release, the minimum supported +Rust version for this crate remains at `1.28.0`. + +New features: + +* [FEATURE #474](https://github.com/rust-lang/regex/issues/474): + The `use_std` feature has been deprecated in favor of the `std` feature. + The `use_std` feature will be removed in regex 2. Until then, `use_std` will + remain as an alias for the `std` feature. +* [FEATURE #583](https://github.com/rust-lang/regex/issues/583): + Add a substantial number of crate features shrinking `regex`. + + +1.2.1 (2019-08-03) +================== +This release does a bit of house cleaning. Namely: + +* This repository is now using rustfmt. +* License headers have been removed from all files, in following suit with the + Rust project. +* Teddy has been removed from the `regex` crate, and is now part of the + `aho-corasick` crate. + [See `aho-corasick`'s new `packed` submodule for details](https://docs.rs/aho-corasick/0.7.6/aho_corasick/packed/index.html). +* The `utf8-ranges` crate has been deprecated, with its functionality moving + into the + [`utf8` sub-module of `regex-syntax`](https://docs.rs/regex-syntax/0.6.11/regex_syntax/utf8/index.html). +* The `ucd-util` dependency has been dropped, in favor of implementing what + little we need inside of `regex-syntax` itself. + +In general, this is part of an ongoing (long term) effort to make optimizations +in the regex engine easier to reason about. The current code is too convoluted, +and thus it is very easy to introduce new bugs. This simplification effort is +the primary motivation behind re-working the `aho-corasick` crate to not only +bundle algorithms like Teddy, but to also provide regex-like match semantics +automatically. + +Moving forward, the plan is to join up with the `bstr` and `regex-automata` +crates, with the former providing more sophisticated substring search +algorithms (thereby deleting existing code in `regex`) and the latter providing +ahead-of-time compiled DFAs for cases where they are inexpensive to compute. + + +1.2.0 (2019-07-20) +================== +This release updates regex's minimum supported Rust version to 1.28, which was +release almost 1 year ago. This release also updates regex's Unicode data +tables to 12.1.0. + + +1.1.9 (2019-07-06) +================== +This release contains a bug fix that caused regex's tests to fail, due to a +dependency on an unreleased behavior in regex-syntax. + +* [BUG #593](https://github.com/rust-lang/regex/issues/593): + Move an integration-style test on error messages into regex-syntax. + + +1.1.8 (2019-07-04) +================== +This release contains a few small internal refactorings. One of which fixes +an instance of undefined behavior in a part of the SIMD code. + +Bug fixes: + +* [BUG #545](https://github.com/rust-lang/regex/issues/545): + Improves error messages when a repetition operator is used without a number. +* [BUG #588](https://github.com/rust-lang/regex/issues/588): + Removes use of a repr(Rust) union used for type punning in the Teddy matcher. +* [BUG #591](https://github.com/rust-lang/regex/issues/591): + Update docs for running benchmarks and improve failure modes. + + +1.1.7 (2019-06-09) +================== +This release fixes up a few warnings as a result of recent deprecations. + + +1.1.6 (2019-04-16) +================== +This release fixes a regression introduced by a bug fix (for +[BUG #557](https://github.com/rust-lang/regex/issues/557)) which could cause +the regex engine to enter an infinite loop. This bug was originally +[reported against ripgrep](https://github.com/BurntSushi/ripgrep/issues/1247). + + +1.1.5 (2019-04-01) +================== +This release fixes a bug in regex's dependency specification where it requires +a newer version of regex-syntax, but this wasn't communicated correctly in the +Cargo.toml. This would have been caught by a minimal version check, but this +check was disabled because the `rand` crate itself advertises incorrect +dependency specifications. + +Bug fixes: + +* [BUG #570](https://github.com/rust-lang/regex/pull/570): + Fix regex-syntax minimal version. + + +1.1.4 (2019-03-31) +================== +This release fixes a backwards compatibility regression where Regex was no +longer UnwindSafe. This was caused by the upgrade to aho-corasick 0.7, whose +AhoCorasick type was itself not UnwindSafe. This has been fixed in aho-corasick +0.7.4, which we now require. + +Bug fixes: + +* [BUG #568](https://github.com/rust-lang/regex/pull/568): + Fix an API regression where Regex was no longer UnwindSafe. + + +1.1.3 (2019-03-30) +================== +This releases fixes a few bugs and adds a performance improvement when a regex +is a simple alternation of literals. + +Performance improvements: + +* [OPT #566](https://github.com/rust-lang/regex/pull/566): + Upgrades `aho-corasick` to 0.7 and uses it for `foo|bar|...|quux` regexes. + +Bug fixes: + +* [BUG #527](https://github.com/rust-lang/regex/issues/527): + Fix a bug where the parser would panic on patterns like `((?x))`. +* [BUG #555](https://github.com/rust-lang/regex/issues/555): + Fix a bug where the parser would panic on patterns like `(?m){1,1}`. +* [BUG #557](https://github.com/rust-lang/regex/issues/557): + Fix a bug where captures could lead to an incorrect match. + + +1.1.2 (2019-02-27) +================== +This release fixes a bug found in the fix introduced in 1.1.1. + +Bug fixes: + +* [BUG edf45e6f](https://github.com/rust-lang/regex/commit/edf45e6f): + Fix bug introduced in reverse suffix literal matcher in the 1.1.1 release. + + +1.1.1 (2019-02-27) +================== +This is a small release with one fix for a bug caused by literal optimizations. + +Bug fixes: + +* [BUG 661bf53d](https://github.com/rust-lang/regex/commit/661bf53d): + Fixes a bug in the reverse suffix literal optimization. This was originally + reported + [against ripgrep](https://github.com/BurntSushi/ripgrep/issues/1203). + + +1.1.0 (2018-11-30) +================== +This is a small release with a couple small enhancements. This release also +increases the minimal supported Rust version (MSRV) to 1.24.1 (from 1.20.0). In +accordance with this crate's MSRV policy, this release bumps the minor version +number. + +Performance improvements: + +* [OPT #511](https://github.com/rust-lang/regex/pull/511), + [OPT #540](https://github.com/rust-lang/regex/pull/540): + Improve lazy DFA construction for large regex sets. + +New features: + +* [FEATURE #538](https://github.com/rust-lang/regex/pull/538): + Add Emoji and "break" Unicode properties. See [UNICODE.md](UNICODE.md). + +Bug fixes: + +* [BUG #530](https://github.com/rust-lang/regex/pull/530): + Add Unicode license (for data tables). +* Various typo/doc fixups. + + +1.0.6 (2018-11-06) +================== +This is a small release. + +Performance improvements: + +* [OPT #513](https://github.com/rust-lang/regex/pull/513): + Improve performance of compiling large Unicode classes by 8-10%. + +Bug fixes: + +* [BUG #533](https://github.com/rust-lang/regex/issues/533): + Fix definition of `[[:blank:]]` class that regressed in `regex-syntax 0.5`. + + +1.0.5 (2018-09-06) +================== +This is a small release with an API enhancement. + +New features: + +* [FEATURE #509](https://github.com/rust-lang/regex/pull/509): + Generalize impls of the `Replacer` trait. + + +1.0.4 (2018-08-25) +================== +This is a small release that bumps the quickcheck dependency. + + +1.0.3 (2018-08-24) +================== +This is a small bug fix release. + +Bug fixes: + +* [BUG #504](https://github.com/rust-lang/regex/pull/504): + Fix for Cargo's "minimal version" support. +* [BUG 1e39165f](https://github.com/rust-lang/regex/commit/1e39165f): + Fix doc examples for byte regexes. + + +1.0.2 (2018-07-18) +================== +This release exposes some new lower level APIs on `Regex` that permit +amortizing allocation and controlling the location at which a search is +performed in a more granular way. Most users of the regex crate will not +need or want to use these APIs. + +New features: + +* [FEATURE #493](https://github.com/rust-lang/regex/pull/493): + Add a few lower level APIs for amortizing allocation and more fine-grained + searching. + +Bug fixes: + +* [BUG 3981d2ad](https://github.com/rust-lang/regex/commit/3981d2ad): + Correct outdated documentation on `RegexBuilder::dot_matches_new_line`. +* [BUG 7ebe4ae0](https://github.com/rust-lang/regex/commit/7ebe4ae0): + Correct outdated documentation on `Parser::allow_invalid_utf8` in the + `regex-syntax` crate. +* [BUG 24c7770b](https://github.com/rust-lang/regex/commit/24c7770b): + Fix a bug in the HIR printer where it wouldn't correctly escape meta + characters in character classes. + + +1.0.1 (2018-06-19) +================== +This release upgrades regex's Unicode tables to Unicode 11, and enables SIMD +optimizations automatically on Rust stable (1.27 or newer). + +New features: + +* [FEATURE #486](https://github.com/rust-lang/regex/pull/486): + Implement `size_hint` on `RegexSet` match iterators. +* [FEATURE #488](https://github.com/rust-lang/regex/pull/488): + Update Unicode tables for Unicode 11. +* [FEATURE #490](https://github.com/rust-lang/regex/pull/490): + SIMD optimizations are now enabled automatically in Rust stable, for versions + 1.27 and up. No compilation flags or features need to be set. CPU support + SIMD is detected automatically at runtime. + +Bug fixes: + +* [BUG #482](https://github.com/rust-lang/regex/pull/482): + Present a better compilation error when the `use_std` feature isn't used. + + +1.0.0 (2018-05-01) +================== +This release marks the 1.0 release of regex. + +While this release includes some breaking changes, most users of older versions +of the regex library should be able to migrate to 1.0 by simply bumping the +version number. The important changes are as follows: + +* We adopt Rust 1.20 as the new minimum supported version of Rust for regex. + We also tentatively adopt a policy that permits bumping the minimum supported + version of Rust in minor version releases of regex, but no patch releases. + That is, with respect to semver, we do not strictly consider bumping the + minimum version of Rust to be a breaking change, but adopt a conservative + stance as a compromise. +* Octal syntax in regular expressions has been disabled by default. This + permits better error messages that inform users that backreferences aren't + available. Octal syntax can be re-enabled via the corresponding option on + `RegexBuilder`. +* `(?-u:\B)` is no longer allowed in Unicode regexes since it can match at + invalid UTF-8 code unit boundaries. `(?-u:\b)` is still allowed in Unicode + regexes. +* The `From` impl has been removed. This formally removes + the public dependency on `regex-syntax`. +* A new feature, `use_std`, has been added and enabled by default. Disabling + the feature will result in a compilation error. In the future, this may + permit us to support `no_std` environments (w/ `alloc`) in a backwards + compatible way. + +For more information and discussion, please see +[1.0 release tracking issue](https://github.com/rust-lang/regex/issues/457). + + +0.2.11 (2018-05-01) +=================== +This release primarily contains bug fixes. Some of them resolve bugs where +the parser could panic. + +New features: + +* [FEATURE #459](https://github.com/rust-lang/regex/pull/459): + Include C++'s standard regex library and Boost's regex library in the + benchmark harness. We now include D/libphobos, C++/std, C++/boost, Oniguruma, + PCRE1, PCRE2, RE2 and Tcl in the harness. + +Bug fixes: + +* [BUG #445](https://github.com/rust-lang/regex/issues/445): + Clarify order of indices returned by RegexSet match iterator. +* [BUG #461](https://github.com/rust-lang/regex/issues/461): + Improve error messages for invalid regexes like `[\d-a]`. +* [BUG #464](https://github.com/rust-lang/regex/issues/464): + Fix a bug in the error message pretty printer that could cause a panic when + a regex contained a literal `\n` character. +* [BUG #465](https://github.com/rust-lang/regex/issues/465): + Fix a panic in the parser that was caused by applying a repetition operator + to `(?flags)`. +* [BUG #466](https://github.com/rust-lang/regex/issues/466): + Fix a bug where `\pC` was not recognized as an alias for `\p{Other}`. +* [BUG #470](https://github.com/rust-lang/regex/pull/470): + Fix a bug where literal searches did more work than necessary for anchored + regexes. + + +0.2.10 (2018-03-16) +=================== +This release primarily updates the regex crate to changes made in `std::arch` +on nightly Rust. + +New features: + +* [FEATURE #458](https://github.com/rust-lang/regex/pull/458): + The `Hir` type in `regex-syntax` now has a printer. + + +0.2.9 (2018-03-12) +================== +This release introduces a new nightly only feature, `unstable`, which enables +SIMD optimizations for certain types of regexes. No additional compile time +options are necessary, and the regex crate will automatically choose the +best CPU features at run time. As a result, the `simd` (nightly only) crate +dependency has been dropped. + +New features: + +* [FEATURE #456](https://github.com/rust-lang/regex/pull/456): + The regex crate now includes AVX2 optimizations in addition to the extant + SSSE3 optimization. + +Bug fixes: + +* [BUG #455](https://github.com/rust-lang/regex/pull/455): + Fix a bug where `(?x)[ / - ]` failed to parse. + + +0.2.8 (2018-03-12) +================== +Bug fixes: + +* [BUG #454](https://github.com/rust-lang/regex/pull/454): + Fix a bug in the nest limit checker being too aggressive. + + +0.2.7 (2018-03-07) +================== +This release includes a ground-up rewrite of the regex-syntax crate, which has +been in development for over a year. +731 +New features: + +* Error messages for invalid regexes have been greatly improved. You get these + automatically; you don't need to do anything. In addition to better + formatting, error messages will now explicitly call out the use of look + around. When regex 1.0 is released, this will happen for backreferences as + well. +* Full support for intersection, difference and symmetric difference of + character classes. These can be used via the `&&`, `--` and `~~` binary + operators within classes. +* A Unicode Level 1 conformant implementation of `\p{..}` character classes. + Things like `\p{scx:Hira}`, `\p{age:3.2}` or `\p{Changes_When_Casefolded}` + now work. All property name and value aliases are supported, and properties + are selected via loose matching. e.g., `\p{Greek}` is the same as + `\p{G r E e K}`. +* A new `UNICODE.md` document has been added to this repository that + exhaustively documents support for UTS#18. +* Empty sub-expressions are now permitted in most places. That is, `()+` is + now a valid regex. +* Almost everything in regex-syntax now uses constant stack space, even when + performing analysis that requires structural induction. This reduces the risk + of a user provided regular expression causing a stack overflow. +* [FEATURE #174](https://github.com/rust-lang/regex/issues/174): + The `Ast` type in `regex-syntax` now contains span information. +* [FEATURE #424](https://github.com/rust-lang/regex/issues/424): + Support `\u`, `\u{...}`, `\U` and `\U{...}` syntax for specifying code points + in a regular expression. +* [FEATURE #449](https://github.com/rust-lang/regex/pull/449): + Add a `Replace::by_ref` adapter for use of a replacer without consuming it. + +Bug fixes: + +* [BUG #446](https://github.com/rust-lang/regex/issues/446): + We re-enable the Boyer-Moore literal matcher. + + +0.2.6 (2018-02-08) +================== +Bug fixes: + +* [BUG #446](https://github.com/rust-lang/regex/issues/446): + Fixes a bug in the new Boyer-Moore searcher that results in a match failure. + We fix this bug by temporarily disabling Boyer-Moore. + + +0.2.5 (2017-12-30) +================== +Bug fixes: + +* [BUG #437](https://github.com/rust-lang/regex/issues/437): + Fixes a bug in the new Boyer-Moore searcher that results in a panic. + + +0.2.4 (2017-12-30) +================== +New features: + +* [FEATURE #348](https://github.com/rust-lang/regex/pull/348): + Improve performance for capture searches on anchored regex. + (Contributed by @ethanpailes. Nice work!) +* [FEATURE #419](https://github.com/rust-lang/regex/pull/419): + Expand literal searching to include Tuned Boyer-Moore in some cases. + (Contributed by @ethanpailes. Nice work!) + +Bug fixes: + +* [BUG](https://github.com/rust-lang/regex/pull/436): + The regex compiler plugin has been removed. +* [BUG](https://github.com/rust-lang/regex/pull/436): + `simd` has been bumped to `0.2.1`, which fixes a Rust nightly build error. +* [BUG](https://github.com/rust-lang/regex/pull/436): + Bring the benchmark harness up to date. + + +0.2.3 (2017-11-30) +================== +New features: + +* [FEATURE #374](https://github.com/rust-lang/regex/pull/374): + Add `impl From for &str`. +* [FEATURE #380](https://github.com/rust-lang/regex/pull/380): + Derive `Clone` and `PartialEq` on `Error`. +* [FEATURE #400](https://github.com/rust-lang/regex/pull/400): + Update to Unicode 10. + +Bug fixes: + +* [BUG #375](https://github.com/rust-lang/regex/issues/375): + Fix a bug that prevented the bounded backtracker from terminating. +* [BUG #393](https://github.com/rust-lang/regex/issues/393), + [BUG #394](https://github.com/rust-lang/regex/issues/394): + Fix bug with `replace` methods for empty matches. + + +0.2.2 (2017-05-21) +================== +New features: + +* [FEATURE #341](https://github.com/rust-lang/regex/issues/341): + Support nested character classes and intersection operation. + For example, `[\p{Greek}&&\pL]` matches greek letters and + `[[0-9]&&[^4]]` matches every decimal digit except `4`. + (Much thanks to @robinst, who contributed this awesome feature.) + +Bug fixes: + +* [BUG #321](https://github.com/rust-lang/regex/issues/321): + Fix bug in literal extraction and UTF-8 decoding. +* [BUG #326](https://github.com/rust-lang/regex/issues/326): + Add documentation tip about the `(?x)` flag. +* [BUG #333](https://github.com/rust-lang/regex/issues/333): + Show additional replacement example using curly braces. +* [BUG #334](https://github.com/rust-lang/regex/issues/334): + Fix bug when resolving captures after a match. +* [BUG #338](https://github.com/rust-lang/regex/issues/338): + Add example that uses `Captures::get` to API documentation. +* [BUG #353](https://github.com/rust-lang/regex/issues/353): + Fix RegexSet bug that caused match failure in some cases. +* [BUG #354](https://github.com/rust-lang/regex/pull/354): + Fix panic in parser when `(?x)` is used. +* [BUG #358](https://github.com/rust-lang/regex/issues/358): + Fix literal optimization bug with RegexSet. +* [BUG #359](https://github.com/rust-lang/regex/issues/359): + Fix example code in README. +* [BUG #365](https://github.com/rust-lang/regex/pull/365): + Fix bug in `rure_captures_len` in the C binding. +* [BUG #367](https://github.com/rust-lang/regex/issues/367): + Fix byte class bug that caused a panic. + + +0.2.1 +===== +One major bug with `replace_all` has been fixed along with a couple of other +touch-ups. + +* [BUG #312](https://github.com/rust-lang/regex/issues/312): + Fix documentation for `NoExpand` to reference correct lifetime parameter. +* [BUG #314](https://github.com/rust-lang/regex/issues/314): + Fix a bug with `replace_all` when replacing a match with the empty string. +* [BUG #316](https://github.com/rust-lang/regex/issues/316): + Note a missing breaking change from the `0.2.0` CHANGELOG entry. + (`RegexBuilder::compile` was renamed to `RegexBuilder::build`.) +* [BUG #324](https://github.com/rust-lang/regex/issues/324): + Compiling `regex` should only require one version of `memchr` crate. + + +0.2.0 +===== +This is a new major release of the regex crate, and is an implementation of the +[regex 1.0 RFC](https://github.com/rust-lang/rfcs/blob/master/text/1620-regex-1.0.md). +We are releasing a `0.2` first, and if there are no major problems, we will +release a `1.0` shortly. For `0.2`, the minimum *supported* Rust version is +1.12. + +There are a number of **breaking changes** in `0.2`. They are split into two +types. The first type correspond to breaking changes in regular expression +syntax. The second type correspond to breaking changes in the API. + +Breaking changes for regex syntax: + +* POSIX character classes now require double bracketing. Previously, the regex + `[:upper:]` would parse as the `upper` POSIX character class. Now it parses + as the character class containing the characters `:upper:`. The fix to this + change is to use `[[:upper:]]` instead. Note that variants like + `[[:upper:][:blank:]]` continue to work. +* The character `[` must always be escaped inside a character class. +* The characters `&`, `-` and `~` must be escaped if any one of them are + repeated consecutively. For example, `[&]`, `[\&]`, `[\&\&]`, `[&-&]` are all + equivalent while `[&&]` is illegal. (The motivation for this and the prior + change is to provide a backwards compatible path for adding character class + set notation.) +* A `bytes::Regex` now has Unicode mode enabled by default (like the main + `Regex` type). This means regexes compiled with `bytes::Regex::new` that + don't have the Unicode flag set should add `(?-u)` to recover the original + behavior. + +Breaking changes for the regex API: + +* `find` and `find_iter` now **return `Match` values instead of + `(usize, usize)`.** `Match` values have `start` and `end` methods, which + return the match offsets. `Match` values also have an `as_str` method, + which returns the text of the match itself. +* The `Captures` type now only provides a single iterator over all capturing + matches, which should replace uses of `iter` and `iter_pos`. Uses of + `iter_named` should use the `capture_names` method on `Regex`. +* The `at` method on the `Captures` type has been renamed to `get`, and it + now returns a `Match`. Similarly, the `name` method on `Captures` now returns + a `Match`. +* The `replace` methods now return `Cow` values. The `Cow::Borrowed` variant + is returned when no replacements are made. +* The `Replacer` trait has been completely overhauled. This should only + impact clients that implement this trait explicitly. Standard uses of + the `replace` methods should continue to work unchanged. If you implement + the `Replacer` trait, please consult the new documentation. +* The `quote` free function has been renamed to `escape`. +* The `Regex::with_size_limit` method has been removed. It is replaced by + `RegexBuilder::size_limit`. +* The `RegexBuilder` type has switched from owned `self` method receivers to + `&mut self` method receivers. Most uses will continue to work unchanged, but + some code may require naming an intermediate variable to hold the builder. +* The `compile` method on `RegexBuilder` has been renamed to `build`. +* The free `is_match` function has been removed. It is replaced by compiling + a `Regex` and calling its `is_match` method. +* The `PartialEq` and `Eq` impls on `Regex` have been dropped. If you relied + on these impls, the fix is to define a wrapper type around `Regex`, impl + `Deref` on it and provide the necessary impls. +* The `is_empty` method on `Captures` has been removed. This always returns + `false`, so its use is superfluous. +* The `Syntax` variant of the `Error` type now contains a string instead of + a `regex_syntax::Error`. If you were examining syntax errors more closely, + you'll need to explicitly use the `regex_syntax` crate to re-parse the regex. +* The `InvalidSet` variant of the `Error` type has been removed since it is + no longer used. +* Most of the iterator types have been renamed to match conventions. If you + were using these iterator types explicitly, please consult the documentation + for its new name. For example, `RegexSplits` has been renamed to `Split`. + +A number of bugs have been fixed: + +* [BUG #151](https://github.com/rust-lang/regex/issues/151): + The `Replacer` trait has been changed to permit the caller to control + allocation. +* [BUG #165](https://github.com/rust-lang/regex/issues/165): + Remove the free `is_match` function. +* [BUG #166](https://github.com/rust-lang/regex/issues/166): + Expose more knobs (available in `0.1`) and remove `with_size_limit`. +* [BUG #168](https://github.com/rust-lang/regex/issues/168): + Iterators produced by `Captures` now have the correct lifetime parameters. +* [BUG #175](https://github.com/rust-lang/regex/issues/175): + Fix a corner case in the parsing of POSIX character classes. +* [BUG #178](https://github.com/rust-lang/regex/issues/178): + Drop the `PartialEq` and `Eq` impls on `Regex`. +* [BUG #179](https://github.com/rust-lang/regex/issues/179): + Remove `is_empty` from `Captures` since it always returns false. +* [BUG #276](https://github.com/rust-lang/regex/issues/276): + Position of named capture can now be retrieved from a `Captures`. +* [BUG #296](https://github.com/rust-lang/regex/issues/296): + Remove winapi/kernel32-sys dependency on UNIX. +* [BUG #307](https://github.com/rust-lang/regex/issues/307): + Fix error on emscripten. + + +0.1.80 +====== +* [PR #292](https://github.com/rust-lang/regex/pull/292): + Fixes bug #291, which was introduced by PR #290. + +0.1.79 +====== +* Require regex-syntax 0.3.8. + +0.1.78 +====== +* [PR #290](https://github.com/rust-lang/regex/pull/290): + Fixes bug #289, which caused some regexes with a certain combination + of literals to match incorrectly. + +0.1.77 +====== +* [PR #281](https://github.com/rust-lang/regex/pull/281): + Fixes bug #280 by disabling all literal optimizations when a pattern + is partially anchored. + +0.1.76 +====== +* Tweak criteria for using the Teddy literal matcher. + +0.1.75 +====== +* [PR #275](https://github.com/rust-lang/regex/pull/275): + Improves match verification performance in the Teddy SIMD searcher. +* [PR #278](https://github.com/rust-lang/regex/pull/278): + Replaces slow substring loop in the Teddy SIMD searcher with Aho-Corasick. +* Implemented DoubleEndedIterator on regex set match iterators. + +0.1.74 +====== +* Release regex-syntax 0.3.5 with a minor bug fix. +* Fix bug #272. +* Fix bug #277. +* [PR #270](https://github.com/rust-lang/regex/pull/270): + Fixes bugs #264, #268 and an unreported where the DFA cache size could be + drastically underestimated in some cases (leading to high unexpected memory + usage). + +0.1.73 +====== +* Release `regex-syntax 0.3.4`. +* Bump `regex-syntax` dependency version for `regex` to `0.3.4`. + +0.1.72 +====== +* [PR #262](https://github.com/rust-lang/regex/pull/262): + Fixes a number of small bugs caught by fuzz testing (AFL). + +0.1.71 +====== +* [PR #236](https://github.com/rust-lang/regex/pull/236): + Fix a bug in how suffix literals were extracted, which could lead + to invalid match behavior in some cases. + +0.1.70 +====== +* [PR #231](https://github.com/rust-lang/regex/pull/231): + Add SIMD accelerated multiple pattern search. +* [PR #228](https://github.com/rust-lang/regex/pull/228): + Reintroduce the reverse suffix literal optimization. +* [PR #226](https://github.com/rust-lang/regex/pull/226): + Implements NFA state compression in the lazy DFA. +* [PR #223](https://github.com/rust-lang/regex/pull/223): + A fully anchored RegexSet can now short-circuit. + +0.1.69 +====== +* [PR #216](https://github.com/rust-lang/regex/pull/216): + Tweak the threshold for running backtracking. +* [PR #217](https://github.com/rust-lang/regex/pull/217): + Add upper limit (from the DFA) to capture search (for the NFA). +* [PR #218](https://github.com/rust-lang/regex/pull/218): + Add rure, a C API. + +0.1.68 +====== +* [PR #210](https://github.com/rust-lang/regex/pull/210): + Fixed a performance bug in `bytes::Regex::replace` where `extend` was used + instead of `extend_from_slice`. +* [PR #211](https://github.com/rust-lang/regex/pull/211): + Fixed a bug in the handling of word boundaries in the DFA. +* [PR #213](https://github.com/rust-lang/pull/213): + Added RE2 and Tcl to the benchmark harness. Also added a CLI utility from + running regexes using any of the following regex engines: PCRE1, PCRE2, + Oniguruma, RE2, Tcl and of course Rust's own regexes. + +0.1.67 +====== +* [PR #201](https://github.com/rust-lang/regex/pull/201): + Fix undefined behavior in the `regex!` compiler plugin macro. +* [PR #205](https://github.com/rust-lang/regex/pull/205): + More improvements to DFA performance. Competitive with RE2. See PR for + benchmarks. +* [PR #209](https://github.com/rust-lang/regex/pull/209): + Release 0.1.66 was semver incompatible since it required a newer version + of Rust than previous releases. This PR fixes that. (And `0.1.66` was + yanked.) + +0.1.66 +====== +* Speculative support for Unicode word boundaries was added to the DFA. This + should remove the last common case that disqualified use of the DFA. +* An optimization that scanned for suffix literals and then matched the regular + expression in reverse was removed because it had worst case quadratic time + complexity. It was replaced with a more limited optimization where, given any + regex of the form `re$`, it will be matched in reverse from the end of the + haystack. +* [PR #202](https://github.com/rust-lang/regex/pull/202): + The inner loop of the DFA was heavily optimized to improve cache locality + and reduce the overall number of instructions run on each iteration. This + represents the first use of `unsafe` in `regex` (to elide bounds checks). +* [PR #200](https://github.com/rust-lang/regex/pull/200): + Use of the `mempool` crate (which used thread local storage) was replaced + with a faster version of a similar API in @Amanieu's `thread_local` crate. + It should reduce contention when using a regex from multiple threads + simultaneously. +* PCRE2 JIT benchmarks were added. A benchmark comparison can be found + [here](https://gist.github.com/anonymous/14683c01993e91689f7206a18675901b). + (Includes a comparison with PCRE1's JIT and Oniguruma.) +* A bug where word boundaries weren't being matched correctly in the DFA was + fixed. This only affected use of `bytes::Regex`. +* [#160](https://github.com/rust-lang/regex/issues/160): + `Captures` now has a `Debug` impl. diff --git a/third_party/crates/regex-1.13.1/Cargo.lock b/third_party/crates/regex-1.13.1/Cargo.lock new file mode 100644 index 0000000..43c4ef2 --- /dev/null +++ b/third_party/crates/regex-1.13.1/Cargo.lock @@ -0,0 +1,383 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "log", + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "indexmap" +version = "2.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +dependencies = [ + "log", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quickcheck" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +dependencies = [ + "rand", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "regex" +version = "1.13.1" +dependencies = [ + "aho-corasick", + "anyhow", + "doc-comment", + "env_logger", + "memchr", + "quickcheck", + "regex-automata", + "regex-syntax", + "regex-test", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "log", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "regex-test" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da40f0939bc4c598b4326abdbb363a8987aa43d0526e5624aefcf3ed90344e62" +dependencies = [ + "anyhow", + "bstr", + "serde", + "toml", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +dependencies = [ + "memchr", +] diff --git a/third_party/crates/regex-1.13.1/Cargo.toml b/third_party/crates/regex-1.13.1/Cargo.toml new file mode 100644 index 0000000..44ba433 --- /dev/null +++ b/third_party/crates/regex-1.13.1/Cargo.toml @@ -0,0 +1,216 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.65" +name = "regex" +version = "1.13.1" +authors = [ + "The Rust Project Developers", + "Andrew Gallant ", +] +build = false +include = [ + "/CHANGELOG.md", + "/Cargo.toml", + "/LICENSE-MIT", + "/LICENSE-APACHE", + "/README.md", + "/UNICODE.md", + "bench/README.md", + "src/**/*.rs", + "testdata/**.toml", + "tests/**/*.rs", + "bench/**/*.rs", + "benches/**/*.rs", + "LICENSE-UNICODE", + "!/tests/fuzz/mod.rs", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +An implementation of regular expressions for Rust. This implementation uses +finite automata and guarantees linear time matching on all inputs. +""" +homepage = "https://github.com/rust-lang/regex" +documentation = "https://docs.rs/regex" +readme = "README.md" +categories = ["text-processing"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/regex" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs_regex", +] + +[features] +default = [ + "std", + "perf", + "unicode", + "regex-syntax/default", +] +logging = [ + "aho-corasick?/logging", + "memchr?/logging", + "regex-automata/logging", +] +pattern = [] +perf = [ + "perf-cache", + "perf-dfa", + "perf-onepass", + "perf-backtrack", + "perf-inline", + "perf-literal", +] +perf-backtrack = ["regex-automata/nfa-backtrack"] +perf-cache = [] +perf-dfa = ["regex-automata/hybrid"] +perf-dfa-full = [ + "regex-automata/dfa-build", + "regex-automata/dfa-search", +] +perf-inline = ["regex-automata/perf-inline"] +perf-literal = [ + "dep:aho-corasick", + "dep:memchr", + "regex-automata/perf-literal", +] +perf-onepass = ["regex-automata/dfa-onepass"] +std = [ + "aho-corasick?/std", + "memchr?/std", + "regex-automata/std", + "regex-syntax/std", +] +unicode = [ + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", + "regex-automata/unicode", + "regex-syntax/unicode", +] +unicode-age = [ + "regex-automata/unicode-age", + "regex-syntax/unicode-age", +] +unicode-bool = [ + "regex-automata/unicode-bool", + "regex-syntax/unicode-bool", +] +unicode-case = [ + "regex-automata/unicode-case", + "regex-syntax/unicode-case", +] +unicode-gencat = [ + "regex-automata/unicode-gencat", + "regex-syntax/unicode-gencat", +] +unicode-perl = [ + "regex-automata/unicode-perl", + "regex-automata/unicode-word-boundary", + "regex-syntax/unicode-perl", +] +unicode-script = [ + "regex-automata/unicode-script", + "regex-syntax/unicode-script", +] +unicode-segment = [ + "regex-automata/unicode-segment", + "regex-syntax/unicode-segment", +] +unstable = ["pattern"] +use_std = ["std"] + +[lib] +name = "regex" +path = "src/lib.rs" + +[[test]] +name = "integration" +path = "tests/lib.rs" + +[dependencies.aho-corasick] +version = "1.0.0" +optional = true +default-features = false + +[dependencies.memchr] +version = "2.6.0" +optional = true +default-features = false + +[dependencies.regex-automata] +version = "0.4.16" +features = [ + "alloc", + "syntax", + "meta", + "nfa-pikevm", +] +default-features = false + +[dependencies.regex-syntax] +version = "0.8.11" +default-features = false + +[dev-dependencies.anyhow] +version = "1.0.69" + +[dev-dependencies.doc-comment] +version = "0.3" + +[dev-dependencies.env_logger] +version = "0.9.3" +features = [ + "atty", + "humantime", + "termcolor", +] +default-features = false + +[dev-dependencies.quickcheck] +version = "1.0.3" +default-features = false + +[dev-dependencies.regex-test] +version = "0.1.0" + +[lints.rust.unexpected_cfgs] +level = "allow" +priority = 0 +check-cfg = ["cfg(docsrs_regex)"] + +[profile.bench] +debug = 2 + +[profile.dev] +opt-level = 3 +debug = 2 + +[profile.release] +debug = 2 + +[profile.test] +opt-level = 3 +debug = 2 diff --git a/third_party/crates/regex-1.13.1/Cargo.toml.orig b/third_party/crates/regex-1.13.1/Cargo.toml.orig new file mode 100644 index 0000000..d035527 --- /dev/null +++ b/third_party/crates/regex-1.13.1/Cargo.toml.orig @@ -0,0 +1,287 @@ +[package] +name = "regex" +version = "1.13.1" #:version +authors = ["The Rust Project Developers", "Andrew Gallant "] +license = "MIT OR Apache-2.0" +readme = "README.md" +repository = "https://github.com/rust-lang/regex" +documentation = "https://docs.rs/regex" +homepage = "https://github.com/rust-lang/regex" +description = """ +An implementation of regular expressions for Rust. This implementation uses +finite automata and guarantees linear time matching on all inputs. +""" +categories = ["text-processing"] +autotests = false +include.workspace = true +edition = "2021" +rust-version = "1.65" + +[workspace] +members = [ + "regex-automata", + "regex-capi", + "regex-cli", + "regex-lite", + "regex-syntax", + "regex-test", +] + +[workspace.package] +include = [ + "/CHANGELOG.md", + "/Cargo.toml", + "/LICENSE-MIT", + "/LICENSE-APACHE", + "/README.md", + "/UNICODE.md", + "bench/README.md", + "src/**/*.rs", + "testdata/**.toml", + "tests/**/*.rs", + "bench/**/*.rs", + "benches/**/*.rs", + "LICENSE-UNICODE", + "!/tests/fuzz/mod.rs", +] + +# Features are documented in the "Crate features" section of the crate docs: +# https://docs.rs/regex/*/#crate-features +[features] +default = ["std", "perf", "unicode", "regex-syntax/default"] + +# ECOSYSTEM FEATURES + +# The 'std' feature permits the regex crate to use the standard library. This +# is intended to support future use cases where the regex crate may be able +# to compile without std, and instead just rely on 'core' and 'alloc' (for +# example). Currently, this isn't supported, and removing the 'std' feature +# will prevent regex from compiling. +std = [ + "aho-corasick?/std", + "memchr?/std", + "regex-automata/std", + "regex-syntax/std", +] +# This feature enables the 'log' crate to emit messages. This is usually +# only useful for folks working on the regex crate itself, but can be useful +# if you're trying hard to do some performance hacking on regex patterns +# themselves. Note that you'll need to pair this with a crate like 'env_logger' +# to actually emit the log messages somewhere. +logging = [ + "aho-corasick?/logging", + "memchr?/logging", + "regex-automata/logging", +] +# The 'use_std' feature is DEPRECATED. It will be removed in regex 2. Until +# then, it is an alias for the 'std' feature. +use_std = ["std"] + + +# PERFORMANCE FEATURES + +# Enables all default performance features. Note that this specifically does +# not include perf-dfa-full, because it leads to higher compile times and +# bigger binaries, and the runtime performance improvement is not obviously +# worth it. +perf = [ + "perf-cache", + "perf-dfa", + "perf-onepass", + "perf-backtrack", + "perf-inline", + "perf-literal", +] +# Enables use of a lazy DFA when possible. +perf-dfa = ["regex-automata/hybrid"] +# Enables use of a fully compiled DFA when possible. +perf-dfa-full = ["regex-automata/dfa-build", "regex-automata/dfa-search"] +# Enables use of the one-pass regex matcher, which speeds up capture searches +# even beyond the backtracker. +perf-onepass = ["regex-automata/dfa-onepass"] +# Enables use of a bounded backtracker, which speeds up capture searches. +perf-backtrack = ["regex-automata/nfa-backtrack"] +# Enables aggressive use of inlining. +perf-inline = ["regex-automata/perf-inline"] +# Enables literal optimizations. +perf-literal = [ + "dep:aho-corasick", + "dep:memchr", + "regex-automata/perf-literal", +] +# Enables fast caching. (If disabled, caching is still used, but is slower.) +# Currently, this feature has no effect. It used to remove the thread_local +# dependency and use a slower internal cache, but now the default cache has +# been improved and thread_local is no longer a dependency at all. +perf-cache = [] + + +# UNICODE DATA FEATURES + +# Enables all Unicode features. This expands if new Unicode features are added. +unicode = [ + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", + "regex-automata/unicode", + "regex-syntax/unicode", +] +# Enables use of the `Age` property, e.g., `\p{Age:3.0}`. +unicode-age = [ + "regex-automata/unicode-age", + "regex-syntax/unicode-age", +] +# Enables use of a smattering of boolean properties, e.g., `\p{Emoji}`. +unicode-bool = [ + "regex-automata/unicode-bool", + "regex-syntax/unicode-bool", +] +# Enables Unicode-aware case insensitive matching, e.g., `(?i)β`. +unicode-case = [ + "regex-automata/unicode-case", + "regex-syntax/unicode-case", +] +# Enables Unicode general categories, e.g., `\p{Letter}` or `\pL`. +unicode-gencat = [ + "regex-automata/unicode-gencat", + "regex-syntax/unicode-gencat", +] +# Enables Unicode-aware Perl classes corresponding to `\w`, `\s` and `\d`. +unicode-perl = [ + "regex-automata/unicode-perl", + "regex-automata/unicode-word-boundary", + "regex-syntax/unicode-perl", +] +# Enables Unicode scripts and script extensions, e.g., `\p{Greek}`. +unicode-script = [ + "regex-automata/unicode-script", + "regex-syntax/unicode-script", +] +# Enables Unicode segmentation properties, e.g., `\p{gcb=Extend}`. +unicode-segment = [ + "regex-automata/unicode-segment", + "regex-syntax/unicode-segment", +] + + +# UNSTABLE FEATURES (requires Rust nightly) + +# A blanket feature that governs whether unstable features are enabled or not. +# Unstable features are disabled by default, and typically rely on unstable +# features in rustc itself. +unstable = ["pattern"] + +# Enable to use the unstable pattern traits defined in std. This is enabled +# by default if the unstable feature is enabled. +pattern = [] + +# For very fast multi-prefix literal matching. +[dependencies.aho-corasick] +version = "1.0.0" +optional = true +default-features = false + +# For skipping along search text quickly when a leading byte is known. +[dependencies.memchr] +version = "2.6.0" +optional = true +default-features = false + +# For the actual regex engines. +[dependencies.regex-automata] +path = "regex-automata" +version = "0.4.16" +default-features = false +features = ["alloc", "syntax", "meta", "nfa-pikevm"] + +# For parsing regular expressions. +[dependencies.regex-syntax] +path = "regex-syntax" +version = "0.8.11" +default-features = false + +[dev-dependencies] +# For property based tests. +quickcheck = { version = "1.0.3", default-features = false } +# To check README's example +doc-comment = "0.3" +# For easy error handling in integration tests. +anyhow = "1.0.69" +# A library for testing regex engines. +regex-test = { path = "regex-test", version = "0.1.0" } + +[dev-dependencies.env_logger] +# Note that this is currently using an older version because of the dependency +# tree explosion that happened in 0.10. +version = "0.9.3" +default-features = false +features = ["atty", "humantime", "termcolor"] + +# This test suite reads a whole boatload of tests from the top-level testdata +# directory, and then runs them against the regex crate API. +# +# regex-automata has its own version of them, and runs them against each +# internal regex engine individually. +# +# This means that if you're seeing a failure in this test suite, you should +# try running regex-automata's tests: +# +# cargo test --manifest-path regex-automata/Cargo.toml --test integration +# +# That *might* give you a more targeted test failure. i.e., "only the +# PikeVM fails this test." Which gives you a narrower place to search. If +# regex-automata's test suite passes, then the bug might be in the integration +# of the regex crate and regex-automata. But generally speaking, a failure +# in this test suite *should* mean there is a corresponding failure in +# regex-automata's test suite. +[[test]] +path = "tests/lib.rs" +name = "integration" + +[package.metadata.docs.rs] +# We want to document all features. +all-features = true +# Since this crate's feature setup is pretty complicated, it is worth opting +# into a nightly unstable option to show the features that need to be enabled +# for public API items. To do that, we set 'docsrs_regex', and when that's +# enabled, we enable the 'doc_cfg' feature. +# +# To test this locally, run: +# +# RUSTDOCFLAGS="--cfg docsrs_regex" cargo +nightly doc --all-features +# +# Note that we use `docsrs_regex` instead of the more standard `docsrs` because +# other crates use that same `cfg` knob. And since we are enabling a nightly +# feature, they sometimes break. By using our "own" `cfg` knob, we are closer +# to being masters of our own destiny. +rustdoc-args = ["--cfg", "docsrs_regex"] + +# This squashes the (AFAIK) erroneous warning that `docsrs_regex` is not a +# valid `cfg` knob. +[lints.rust] +unexpected_cfgs = { level = "allow", check-cfg = ['cfg(docsrs_regex)'] } + +[profile.release] +debug = true + +[profile.bench] +debug = true + +[profile.dev] +# Running tests takes too long in debug mode, so we forcefully always build +# with optimizations. Unfortunate, but, ¯\_(ツ)_/¯. +# +# It's counter-intuitive that this needs to be set on dev *and* test, but +# it's because the tests that take a long time to run are run as integration +# tests in a separate crate. The test.opt-level setting won't apply there, so +# we need to set the opt-level across the entire build. +opt-level = 3 +debug = true + +[profile.test] +opt-level = 3 +debug = true diff --git a/third_party/crates/regex-1.13.1/LICENSE-APACHE b/third_party/crates/regex-1.13.1/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/third_party/crates/regex-1.13.1/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/crates/regex-1.13.1/LICENSE-MIT b/third_party/crates/regex-1.13.1/LICENSE-MIT new file mode 100644 index 0000000..39d4bdb --- /dev/null +++ b/third_party/crates/regex-1.13.1/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2014 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third_party/crates/regex-1.13.1/README.md b/third_party/crates/regex-1.13.1/README.md new file mode 100644 index 0000000..6517f29 --- /dev/null +++ b/third_party/crates/regex-1.13.1/README.md @@ -0,0 +1,339 @@ +regex +===== +This crate provides routines for searching strings for matches of a [regular +expression] (aka "regex"). The regex syntax supported by this crate is similar +to other regex engines, but it lacks several features that are not known how to +implement efficiently. This includes, but is not limited to, look-around and +backreferences. In exchange, all regex searches in this crate have worst case +`O(m * n)` time complexity, where `m` is proportional to the size of the regex +and `n` is proportional to the size of the string being searched. + +[regular expression]: https://en.wikipedia.org/wiki/Regular_expression + +[![Build status](https://github.com/rust-lang/regex/workflows/ci/badge.svg)](https://github.com/rust-lang/regex/actions) +[![Crates.io](https://img.shields.io/crates/v/regex.svg)](https://crates.io/crates/regex) + +### Documentation + +[Module documentation with examples](https://docs.rs/regex). +The module documentation also includes a comprehensive description of the +syntax supported. + +Documentation with examples for the various matching functions and iterators +can be found on the +[`Regex` type](https://docs.rs/regex/*/regex/struct.Regex.html). + +### Usage + +To bring this crate into your repository, either add `regex` to your +`Cargo.toml`, or run `cargo add regex`. + +Here's a simple example that matches a date in YYYY-MM-DD format and prints the +year, month and day: + +```rust +use regex::Regex; + +fn main() { + let re = Regex::new(r"(?x) +(?P\d{4}) # the year +- +(?P\d{2}) # the month +- +(?P\d{2}) # the day +").unwrap(); + + let caps = re.captures("2010-03-14").unwrap(); + assert_eq!("2010", &caps["year"]); + assert_eq!("03", &caps["month"]); + assert_eq!("14", &caps["day"]); +} +``` + +If you have lots of dates in text that you'd like to iterate over, then it's +easy to adapt the above example with an iterator: + +```rust +use regex::Regex; + +fn main() { + let re = Regex::new(r"(\d{4})-(\d{2})-(\d{2})").unwrap(); + let hay = "On 2010-03-14, foo happened. On 2014-10-14, bar happened."; + + let mut dates = vec![]; + for (_, [year, month, day]) in re.captures_iter(hay).map(|c| c.extract()) { + dates.push((year, month, day)); + } + assert_eq!(dates, vec![ + ("2010", "03", "14"), + ("2014", "10", "14"), + ]); +} +``` + +### Usage: Avoid compiling the same regex in a loop + +It is an anti-pattern to compile the same regular expression in a loop since +compilation is typically expensive. (It takes anywhere from a few microseconds +to a few **milliseconds** depending on the size of the regex.) Not only is +compilation itself expensive, but this also prevents optimizations that reuse +allocations internally to the matching engines. + +In Rust, it can sometimes be a pain to pass regular expressions around if +they're used from inside a helper function. Instead, we recommend using +[`std::sync::LazyLock`], or the [`once_cell`] crate, +if you can't use the standard library. + +This example shows how to use `std::sync::LazyLock`: + +```rust +use std::sync::LazyLock; + +use regex::Regex; + +fn some_helper_function(haystack: &str) -> bool { + static RE: LazyLock = LazyLock::new(|| Regex::new(r"...").unwrap()); + RE.is_match(haystack) +} + +fn main() { + assert!(some_helper_function("abc")); + assert!(!some_helper_function("ac")); +} +``` + +Specifically, in this example, the regex will be compiled when it is used for +the first time. On subsequent uses, it will reuse the previous compilation. + +The [`regex!`] macro can also be used, which handles lazy compilation. + +[`std::sync::LazyLock`]: https://doc.rust-lang.org/std/sync/struct.LazyLock.html +[`once_cell`]: https://crates.io/crates/once_cell +[`regex!`]: https://docs.rs/regex/*/regex/macro.regex.html + +### Usage: match regular expressions on `&[u8]` + +The main API of this crate (`regex::Regex`) requires the caller to pass a +`&str` for searching. In Rust, an `&str` is required to be valid UTF-8, which +means the main API can't be used for searching arbitrary bytes. + +To match on arbitrary bytes, use the `regex::bytes::Regex` API. The API is +identical to the main API, except that it takes an `&[u8]` to search on instead +of an `&str`. The `&[u8]` APIs also permit disabling Unicode mode in the regex +even when the pattern would match invalid UTF-8. For example, `(?-u:.)` is +not allowed in `regex::Regex` but is allowed in `regex::bytes::Regex` since +`(?-u:.)` matches any byte except for `\n`. Conversely, `.` will match the +UTF-8 encoding of any Unicode scalar value except for `\n`. + +This example shows how to find all null-terminated strings in a slice of bytes: + +```rust +use regex::bytes::Regex; + +let re = Regex::new(r"(?-u)(?[^\x00]+)\x00").unwrap(); +let text = b"foo\xFFbar\x00baz\x00"; + +// Extract all of the strings without the null terminator from each match. +// The unwrap is OK here since a match requires the `cstr` capture to match. +let cstrs: Vec<&[u8]> = + re.captures_iter(text) + .map(|c| c.name("cstr").unwrap().as_bytes()) + .collect(); +assert_eq!(vec![&b"foo\xFFbar"[..], &b"baz"[..]], cstrs); +``` + +Notice here that the `[^\x00]+` will match any *byte* except for `NUL`, +including bytes like `\xFF` which are not valid UTF-8. When using the main API, +`[^\x00]+` would instead match any valid UTF-8 sequence except for `NUL`. + +### Usage: match multiple regular expressions simultaneously + +This demonstrates how to use a `RegexSet` to match multiple (possibly +overlapping) regular expressions in a single scan of the search text: + +```rust +use regex::RegexSet; + +let set = RegexSet::new(&[ + r"\w+", + r"\d+", + r"\pL+", + r"foo", + r"bar", + r"barfoo", + r"foobar", +]).unwrap(); + +// Iterate over and collect all of the matches. +let matches: Vec<_> = set.matches("foobar").into_iter().collect(); +assert_eq!(matches, vec![0, 2, 3, 4, 6]); + +// You can also test whether a particular regex matched: +let matches = set.matches("foobar"); +assert!(!matches.matched(5)); +assert!(matches.matched(6)); +``` + + +### Usage: regex internals as a library + +The [`regex-automata` directory](./regex-automata/) contains a crate that +exposes all the internal matching engines used by the `regex` crate. The +idea is that the `regex` crate exposes a simple API for 99% of use cases, but +`regex-automata` exposes oodles of customizable behaviors. + +[Documentation for `regex-automata`.](https://docs.rs/regex-automata) + + +### Usage: a regular expression parser + +This repository contains a crate that provides a well tested regular expression +parser, abstract syntax and a high-level intermediate representation for +convenient analysis. It provides no facilities for compilation or execution. +This may be useful if you're implementing your own regex engine or otherwise +need to do analysis on the syntax of a regular expression. It is otherwise not +recommended for general use. + +[Documentation for `regex-syntax`.](https://docs.rs/regex-syntax) + + +### Crate features + +This crate comes with several features that permit tweaking the trade-off +between binary size, compilation time and runtime performance. Users of this +crate can selectively disable Unicode tables, or choose from a variety of +optimizations performed by this crate to disable. + +When all of these features are disabled, runtime match performance may be much +worse, but if you're matching on short strings, or if high performance isn't +necessary, then such a configuration is perfectly serviceable. To disable +all such features, use the following `Cargo.toml` dependency configuration: + +```toml +[dependencies.regex] +version = "1.3" +default-features = false +# Unless you have a specific reason not to, it's good sense to enable standard +# library support. It enables several optimizations and avoids spin locks. It +# also shouldn't meaningfully impact compile times or binary size. +features = ["std"] +``` + +This will reduce the dependency tree of `regex` down to two crates: +`regex-syntax` and `regex-automata`. + +The full set of features one can disable are +[in the "Crate features" section of the documentation](https://docs.rs/regex/1.*/#crate-features). + + +### Performance + +One of the goals of this crate is for the regex engine to be "fast." What that +is a somewhat nebulous goal, it is usually interpreted in one of two ways. +First, it means that all searches take worst case `O(m * n)` time, where +`m` is proportional to `len(regex)` and `n` is proportional to `len(haystack)`. +Second, it means that even aside from the time complexity constraint, regex +searches are "fast" in practice. + +While the first interpretation is pretty unambiguous, the second one remains +nebulous. While nebulous, it guides this crate's architecture and the sorts of +the trade-offs it makes. For example, here are some general architectural +statements that follow as a result of the goal to be "fast": + +* When given the choice between faster regex searches and faster _Rust compile +times_, this crate will generally choose faster regex searches. +* When given the choice between faster regex searches and faster _regex compile +times_, this crate will generally choose faster regex searches. That is, it is +generally acceptable for `Regex::new` to get a little slower if it means that +searches get faster. (This is a somewhat delicate balance to strike, because +the speed of `Regex::new` needs to remain somewhat reasonable. But this is why +one should avoid re-compiling the same regex over and over again.) +* When given the choice between faster regex searches and simpler API +design, this crate will generally choose faster regex searches. For example, +if one didn't care about performance, we could like get rid of both of +the `Regex::is_match` and `Regex::find` APIs and instead just rely on +`Regex::captures`. + +There are perhaps more ways that being "fast" influences things. + +While this repository used to provide its own benchmark suite, it has since +been moved to [rebar](https://github.com/BurntSushi/rebar). The benchmarks are +quite extensive, and there are many more than what is shown in rebar's README +(which is just limited to a "curated" set meant to compare performance between +regex engines). To run all of this crate's benchmarks, first start by cloning +and installing `rebar`: + +```text +$ git clone https://github.com/BurntSushi/rebar +$ cd rebar +$ cargo install --path ./ +``` + +Then build the benchmark harness for just this crate: + +```text +$ rebar build -e '^rust/regex$' +``` + +Run all benchmarks for this crate as tests (each benchmark is executed once to +ensure it works): + +```text +$ rebar measure -e '^rust/regex$' -t +``` + +Record measurements for all benchmarks and save them to a CSV file: + +```text +$ rebar measure -e '^rust/regex$' | tee results.csv +``` + +Explore benchmark timings: + +```text +$ rebar cmp results.csv +``` + +See the `rebar` documentation for more details on how it works and how to +compare results with other regex engines. + + +### Hacking + +The `regex` crate is, for the most part, a pretty thin wrapper around the +[`meta::Regex`](https://docs.rs/regex-automata/latest/regex_automata/meta/struct.Regex.html) +from the +[`regex-automata` crate](https://docs.rs/regex-automata/latest/regex_automata/). +Therefore, if you're looking to work on the internals of this crate, you'll +likely either want to look in `regex-syntax` (for parsing) or `regex-automata` +(for construction of finite automata and the search routines). + +My [blog on regex internals](https://burntsushi.net/regex-internals/) +goes into more depth. + + +### Minimum Rust version policy + +This crate's minimum supported `rustc` version is `1.65.0`. + +The policy is that the minimum Rust version required to use this crate can be +increased in minor version updates. For example, if regex 1.0 requires Rust +1.20.0, then regex 1.0.z for all values of `z` will also require Rust 1.20.0 or +newer. However, regex 1.y for `y > 0` may require a newer minimum version of +Rust. + + +### License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + https://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + https://opensource.org/licenses/MIT) + +at your option. + +The data in `regex-syntax/src/unicode_tables/` is licensed under the Unicode +License Agreement +([LICENSE-UNICODE](https://www.unicode.org/copyright.html#License)). diff --git a/third_party/crates/regex-1.13.1/UNICODE.md b/third_party/crates/regex-1.13.1/UNICODE.md new file mode 100644 index 0000000..2b62567 --- /dev/null +++ b/third_party/crates/regex-1.13.1/UNICODE.md @@ -0,0 +1,258 @@ +# Unicode conformance + +This document describes the regex crate's conformance to Unicode's +[UTS#18](https://unicode.org/reports/tr18/) +report, which lays out 3 levels of support: Basic, Extended and Tailored. + +Full support for Level 1 ("Basic Unicode Support") is provided with two +exceptions: + +1. Line boundaries are not Unicode aware. Namely, only the `\n` + (`END OF LINE`) character is recognized as a line boundary by default. + One can opt into `\r\n|\r|\n` being a line boundary via CRLF mode. +2. The compatibility properties specified by + [RL1.2a](https://unicode.org/reports/tr18/#RL1.2a) + are ASCII-only definitions. + +Little to no support is provided for either Level 2 or Level 3. For the most +part, this is because the features are either complex/hard to implement, or at +the very least, very difficult to implement without sacrificing performance. +For example, tackling canonical equivalence such that matching worked as one +would expect regardless of normalization form would be a significant +undertaking. This is at least partially a result of the fact that this regex +engine is based on finite automata, which admits less flexibility normally +associated with backtracking implementations. + + +## RL1.1 Hex Notation + +[UTS#18 RL1.1](https://unicode.org/reports/tr18/#Hex_notation) + +Hex Notation refers to the ability to specify a Unicode code point in a regular +expression via its hexadecimal code point representation. This is useful in +environments that have poor Unicode font rendering or if you need to express a +code point that is not normally displayable. All forms of hexadecimal notation +are supported + + \x7F hex character code (exactly two digits) + \x{10FFFF} any hex character code corresponding to a Unicode code point + \u007F hex character code (exactly four digits) + \u{7F} any hex character code corresponding to a Unicode code point + \U0000007F hex character code (exactly eight digits) + \U{7F} any hex character code corresponding to a Unicode code point + +Briefly, the `\x{...}`, `\u{...}` and `\U{...}` are all exactly equivalent ways +of expressing hexadecimal code points. Any number of digits can be written +within the brackets. In contrast, `\xNN`, `\uNNNN`, `\UNNNNNNNN` are all +fixed-width variants of the same idea. + +Note that when Unicode mode is disabled, any non-ASCII Unicode codepoint is +banned. Additionally, the `\xNN` syntax represents arbitrary bytes when Unicode +mode is disabled. That is, the regex `\xFF` matches the Unicode codepoint +U+00FF (encoded as `\xC3\xBF` in UTF-8) while the regex `(?-u)\xFF` matches +the literal byte `\xFF`. + + +## RL1.2 Properties + +[UTS#18 RL1.2](https://unicode.org/reports/tr18/#Categories) + +Full support for Unicode property syntax is provided. Unicode properties +provide a convenient way to construct character classes of groups of code +points specified by Unicode. The regex crate does not provide exhaustive +support, but covers a useful subset. In particular: + +* [General categories](https://unicode.org/reports/tr18/#General_Category_Property) +* [Scripts and Script Extensions](https://unicode.org/reports/tr18/#Script_Property) +* [Age](https://unicode.org/reports/tr18/#Age) +* A smattering of boolean properties, including all of those specified by + [RL1.2](https://unicode.org/reports/tr18/#RL1.2) explicitly. + +In all cases, property name and value abbreviations are supported, and all +names/values are matched loosely without regard for case, whitespace or +underscores. Property name aliases can be found in Unicode's +[`PropertyAliases.txt`](https://www.unicode.org/Public/UCD/latest/ucd/PropertyAliases.txt) +file, while property value aliases can be found in Unicode's +[`PropertyValueAliases.txt`](https://www.unicode.org/Public/UCD/latest/ucd/PropertyValueAliases.txt) +file. + +The syntax supported is also consistent with the UTS#18 recommendation: + +* `\p{Greek}` selects the `Greek` script. Equivalent expressions follow: + `\p{sc:Greek}`, `\p{Script:Greek}`, `\p{Sc=Greek}`, `\p{script=Greek}`, + `\P{sc!=Greek}`. Similarly for `General_Category` (or `gc` for short) and + `Script_Extensions` (or `scx` for short). +* `\p{age:3.2}` selects all code points in Unicode 3.2. +* `\p{Alphabetic}` selects the "alphabetic" property and can be abbreviated + via `\p{alpha}` (for example). +* Single letter variants for properties with single letter abbreviations. + For example, `\p{Letter}` can be equivalently written as `\pL`. + +The following is a list of all properties supported by the regex crate (starred +properties correspond to properties required by RL1.2): + +* `General_Category` \* (including `Any`, `ASCII` and `Assigned`) +* `Script` \* +* `Script_Extensions` \* +* `Age` +* `ASCII_Hex_Digit` +* `Alphabetic` \* +* `Bidi_Control` +* `Case_Ignorable` +* `Cased` +* `Changes_When_Casefolded` +* `Changes_When_Casemapped` +* `Changes_When_Lowercased` +* `Changes_When_Titlecased` +* `Changes_When_Uppercased` +* `Dash` +* `Default_Ignorable_Code_Point` \* +* `Deprecated` +* `Diacritic` +* `Emoji` +* `Emoji_Presentation` +* `Emoji_Modifier` +* `Emoji_Modifier_Base` +* `Emoji_Component` +* `Extended_Pictographic` +* `Extender` +* `Grapheme_Base` +* `Grapheme_Cluster_Break` +* `Grapheme_Extend` +* `Hex_Digit` +* `IDS_Binary_Operator` +* `IDS_Trinary_Operator` +* `ID_Continue` +* `ID_Start` +* `Join_Control` +* `Logical_Order_Exception` +* `Lowercase` \* +* `Math` +* `Noncharacter_Code_Point` \* +* `Pattern_Syntax` +* `Pattern_White_Space` +* `Prepended_Concatenation_Mark` +* `Quotation_Mark` +* `Radical` +* `Regional_Indicator` +* `Sentence_Break` +* `Sentence_Terminal` +* `Soft_Dotted` +* `Terminal_Punctuation` +* `Unified_Ideograph` +* `Uppercase` \* +* `Variation_Selector` +* `White_Space` \* +* `Word_Break` +* `XID_Continue` +* `XID_Start` + + +## RL1.2a Compatibility Properties + +[UTS#18 RL1.2a](https://unicode.org/reports/tr18/#RL1.2a) + +The regex crate only provides ASCII definitions of the +[compatibility properties documented in UTS#18 Annex C](https://unicode.org/reports/tr18/#Compatibility_Properties) +(sans the `\X` class, for matching grapheme clusters, which isn't provided +at all). This is because it seems to be consistent with most other regular +expression engines, and in particular, because these are often referred to as +"ASCII" or "POSIX" character classes. + +Note that the `\w`, `\s` and `\d` character classes **are** Unicode aware. +Their traditional ASCII definition can be used by disabling Unicode. That is, +`[[:word:]]` and `(?-u)\w` are equivalent. + + +## RL1.3 Subtraction and Intersection + +[UTS#18 RL1.3](https://unicode.org/reports/tr18/#Subtraction_and_Intersection) + +The regex crate provides full support for nested character classes, along with +union, intersection (`&&`), difference (`--`) and symmetric difference (`~~`) +operations on arbitrary character classes. + +For example, to match all non-ASCII letters, you could use either +`[\p{Letter}--\p{Ascii}]` (difference) or `[\p{Letter}&&[^\p{Ascii}]]` +(intersecting the negation). + + +## RL1.4 Simple Word Boundaries + +[UTS#18 RL1.4](https://unicode.org/reports/tr18/#Simple_Word_Boundaries) + +The regex crate provides basic Unicode aware word boundary assertions. A word +boundary assertion can be written as `\b`, or `\B` as its negation. A word +boundary negation corresponds to a zero-width match, where its adjacent +characters correspond to word and non-word, or non-word and word characters. + +Conformance in this case chooses to define word character in the same way that +the `\w` character class is defined: a code point that is a member of one of +the following classes: + +* `\p{Alphabetic}` +* `\p{Join_Control}` +* `\p{gc:Mark}` +* `\p{gc:Decimal_Number}` +* `\p{gc:Connector_Punctuation}` + +In particular, this differs slightly from the +[prescription given in RL1.4](https://unicode.org/reports/tr18/#Simple_Word_Boundaries) +but is permissible according to +[UTS#18 Annex C](https://unicode.org/reports/tr18/#Compatibility_Properties). +Namely, it is convenient and simpler to have `\w` and `\b` be in sync with +one another. + +Finally, Unicode word boundaries can be disabled, which will cause ASCII word +boundaries to be used instead. That is, `\b` is a Unicode word boundary while +`(?-u)\b` is an ASCII-only word boundary. This can occasionally be beneficial +if performance is important, since the implementation of Unicode word +boundaries is currently suboptimal on non-ASCII text. + + +## RL1.5 Simple Loose Matches + +[UTS#18 RL1.5](https://unicode.org/reports/tr18/#Simple_Loose_Matches) + +The regex crate provides full support for case-insensitive matching in +accordance with RL1.5. That is, it uses the "simple" case folding mapping. The +"simple" mapping was chosen because of a key convenient property: every +"simple" mapping is a mapping from exactly one code point to exactly one other +code point. This makes case-insensitive matching of character classes, for +example, straight-forward to implement. + +When case-insensitive mode is enabled (e.g., `(?i)[a]` is equivalent to `a|A`), +then all characters classes are case folded as well. + + +## RL1.6 Line Boundaries + +[UTS#18 RL1.6](https://unicode.org/reports/tr18/#Line_Boundaries) + +The regex crate only provides support for recognizing the `\n` (`END OF LINE`) +character as a line boundary by default. One can also opt into treating +`\r\n|\r|\n` as a line boundary via CRLF mode. This choice was made mostly for +implementation convenience, and to avoid performance cliffs that Unicode word +boundaries are subject to. + + +## RL1.7 Code Points + +[UTS#18 RL1.7](https://unicode.org/reports/tr18/#Supplementary_Characters) + +The regex crate provides full support for Unicode code point matching. Namely, +the fundamental atom of any match is always a single code point. + +Given Rust's strong ties to UTF-8, the following guarantees are also provided: + +* All matches are reported on valid UTF-8 code unit boundaries. That is, any + match range returned by the public regex API is guaranteed to successfully + slice the string that was searched. +* By consequence of the above, it is impossible to match surrogate code points. + No support for UTF-16 is provided, so this is never necessary. + +Note that when Unicode mode is disabled, the fundamental atom of matching is +no longer a code point but a single byte. When Unicode mode is disabled, many +Unicode features are disabled as well. For example, `(?-u)\pL` is not a valid +regex but `\pL(?-u)\xFF` (matches any Unicode `Letter` followed by the literal +byte `\xFF`) is, for example. diff --git a/third_party/crates/regex-1.13.1/bench/README.md b/third_party/crates/regex-1.13.1/bench/README.md new file mode 100644 index 0000000..3cc6a1a --- /dev/null +++ b/third_party/crates/regex-1.13.1/bench/README.md @@ -0,0 +1,2 @@ +Benchmarks for this crate have been moved into the rebar project: +https://github.com/BurntSushi/rebar diff --git a/third_party/crates/regex-1.13.1/src/builders.rs b/third_party/crates/regex-1.13.1/src/builders.rs new file mode 100644 index 0000000..3bb08de --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/builders.rs @@ -0,0 +1,2539 @@ +#![allow(warnings)] + +// This module defines an internal builder that encapsulates all interaction +// with meta::Regex construction, and then 4 public API builders that wrap +// around it. The docs are essentially repeated on each of the 4 public +// builders, with tweaks to the examples as needed. +// +// The reason why there are so many builders is partially because of a misstep +// in the initial API design: the builder constructor takes in the pattern +// strings instead of using the `build` method to accept the pattern strings. +// This means `new` has a different signature for each builder. It probably +// would have been nicer to to use one builder with `fn new()`, and then add +// `build(pat)` and `build_many(pats)` constructors. +// +// The other reason is because I think the `bytes` module should probably +// have its own builder type. That way, it is completely isolated from the +// top-level API. +// +// If I could do it again, I'd probably have a `regex::Builder` and a +// `regex::bytes::Builder`. Each would have `build` and `build_set` (or +// `build_many`) methods for constructing a single pattern `Regex` and a +// multi-pattern `RegexSet`, respectively. + +use alloc::{ + string::{String, ToString}, + sync::Arc, + vec, + vec::Vec, +}; + +use regex_automata::{ + meta, nfa::thompson::WhichCaptures, util::syntax, MatchKind, +}; + +use crate::error::Error; + +/// A builder for constructing a `Regex`, `bytes::Regex`, `RegexSet` or a +/// `bytes::RegexSet`. +/// +/// This is essentially the implementation of the four different builder types +/// in the public API: `RegexBuilder`, `bytes::RegexBuilder`, `RegexSetBuilder` +/// and `bytes::RegexSetBuilder`. +#[derive(Clone, Debug)] +struct Builder { + pats: Vec, + metac: meta::Config, + syntaxc: syntax::Config, +} + +impl Default for Builder { + fn default() -> Builder { + let metac = meta::Config::new() + .nfa_size_limit(Some(10 * (1 << 20))) + .hybrid_cache_capacity(2 * (1 << 20)); + Builder { pats: vec![], metac, syntaxc: syntax::Config::default() } + } +} + +impl Builder { + fn new(patterns: I) -> Builder + where + S: AsRef, + I: IntoIterator, + { + let mut b = Builder::default(); + b.pats.extend(patterns.into_iter().map(|p| p.as_ref().to_string())); + b + } + + fn build_one_string(&self) -> Result { + assert_eq!(1, self.pats.len()); + let metac = self + .metac + .clone() + .match_kind(MatchKind::LeftmostFirst) + .utf8_empty(true); + let syntaxc = self.syntaxc.clone().utf8(true); + let pattern = Arc::from(self.pats[0].as_str()); + meta::Builder::new() + .configure(metac) + .syntax(syntaxc) + .build(&pattern) + .map(|meta| crate::Regex { meta, pattern }) + .map_err(Error::from_meta_build_error) + } + + fn build_one_bytes(&self) -> Result { + assert_eq!(1, self.pats.len()); + let metac = self + .metac + .clone() + .match_kind(MatchKind::LeftmostFirst) + .utf8_empty(false); + let syntaxc = self.syntaxc.clone().utf8(false); + let pattern = Arc::from(self.pats[0].as_str()); + meta::Builder::new() + .configure(metac) + .syntax(syntaxc) + .build(&pattern) + .map(|meta| crate::bytes::Regex { meta, pattern }) + .map_err(Error::from_meta_build_error) + } + + fn build_many_string(&self) -> Result { + let metac = self + .metac + .clone() + .match_kind(MatchKind::All) + .utf8_empty(true) + .which_captures(WhichCaptures::None); + let syntaxc = self.syntaxc.clone().utf8(true); + let patterns = Arc::from(self.pats.as_slice()); + meta::Builder::new() + .configure(metac) + .syntax(syntaxc) + .build_many(&patterns) + .map(|meta| crate::RegexSet { meta, patterns }) + .map_err(Error::from_meta_build_error) + } + + fn build_many_bytes(&self) -> Result { + let metac = self + .metac + .clone() + .match_kind(MatchKind::All) + .utf8_empty(false) + .which_captures(WhichCaptures::None); + let syntaxc = self.syntaxc.clone().utf8(false); + let patterns = Arc::from(self.pats.as_slice()); + meta::Builder::new() + .configure(metac) + .syntax(syntaxc) + .build_many(&patterns) + .map(|meta| crate::bytes::RegexSet { meta, patterns }) + .map_err(Error::from_meta_build_error) + } + + fn case_insensitive(&mut self, yes: bool) -> &mut Builder { + self.syntaxc = self.syntaxc.case_insensitive(yes); + self + } + + fn multi_line(&mut self, yes: bool) -> &mut Builder { + self.syntaxc = self.syntaxc.multi_line(yes); + self + } + + fn dot_matches_new_line(&mut self, yes: bool) -> &mut Builder { + self.syntaxc = self.syntaxc.dot_matches_new_line(yes); + self + } + + fn crlf(&mut self, yes: bool) -> &mut Builder { + self.syntaxc = self.syntaxc.crlf(yes); + self + } + + fn line_terminator(&mut self, byte: u8) -> &mut Builder { + self.metac = self.metac.clone().line_terminator(byte); + self.syntaxc = self.syntaxc.line_terminator(byte); + self + } + + fn swap_greed(&mut self, yes: bool) -> &mut Builder { + self.syntaxc = self.syntaxc.swap_greed(yes); + self + } + + fn ignore_whitespace(&mut self, yes: bool) -> &mut Builder { + self.syntaxc = self.syntaxc.ignore_whitespace(yes); + self + } + + fn unicode(&mut self, yes: bool) -> &mut Builder { + self.syntaxc = self.syntaxc.unicode(yes); + self + } + + fn octal(&mut self, yes: bool) -> &mut Builder { + self.syntaxc = self.syntaxc.octal(yes); + self + } + + fn size_limit(&mut self, limit: usize) -> &mut Builder { + self.metac = self.metac.clone().nfa_size_limit(Some(limit)); + self + } + + fn dfa_size_limit(&mut self, limit: usize) -> &mut Builder { + self.metac = self.metac.clone().hybrid_cache_capacity(limit); + self + } + + fn nest_limit(&mut self, limit: u32) -> &mut Builder { + self.syntaxc = self.syntaxc.nest_limit(limit); + self + } +} + +pub(crate) mod string { + use crate::{error::Error, Regex, RegexSet}; + + use super::Builder; + + /// A configurable builder for a [`Regex`]. + /// + /// This builder can be used to programmatically set flags such as `i` + /// (case insensitive) and `x` (for verbose mode). This builder can also be + /// used to configure things like the line terminator and a size limit on + /// the compiled regular expression. + #[derive(Clone, Debug)] + pub struct RegexBuilder { + builder: Builder, + } + + impl RegexBuilder { + /// Create a new builder with a default configuration for the given + /// pattern. + /// + /// If the pattern is invalid or exceeds the configured size limits, + /// then an error will be returned when [`RegexBuilder::build`] is + /// called. + pub fn new(pattern: &str) -> RegexBuilder { + RegexBuilder { builder: Builder::new([pattern]) } + } + + /// Compiles the pattern given to `RegexBuilder::new` with the + /// configuration set on this builder. + /// + /// If the pattern isn't a valid regex or if a configured size limit + /// was exceeded, then an error is returned. + pub fn build(&self) -> Result { + self.builder.build_one_string() + } + + /// This configures Unicode mode for the entire pattern. + /// + /// Enabling Unicode mode does a number of things: + /// + /// * Most fundamentally, it causes the fundamental atom of matching + /// to be a single codepoint. When Unicode mode is disabled, it's a + /// single byte. For example, when Unicode mode is enabled, `.` will + /// match `💩` once, where as it will match 4 times when Unicode mode + /// is disabled. (Since the UTF-8 encoding of `💩` is 4 bytes long.) + /// * Case insensitive matching uses Unicode simple case folding rules. + /// * Unicode character classes like `\p{Letter}` and `\p{Greek}` are + /// available. + /// * Perl character classes are Unicode aware. That is, `\w`, `\s` and + /// `\d`. + /// * The word boundary assertions, `\b` and `\B`, use the Unicode + /// definition of a word character. + /// + /// Note that if Unicode mode is disabled, then the regex will fail to + /// compile if it could match invalid UTF-8. For example, when Unicode + /// mode is disabled, then since `.` matches any byte (except for + /// `\n`), then it can match invalid UTF-8 and thus building a regex + /// from it will fail. Another example is `\w` and `\W`. Since `\w` can + /// only match ASCII bytes when Unicode mode is disabled, it's allowed. + /// But `\W` can match more than ASCII bytes, including invalid UTF-8, + /// and so it is not allowed. This restriction can be lifted only by + /// using a [`bytes::Regex`](crate::bytes::Regex). + /// + /// For more details on the Unicode support in this crate, see the + /// [Unicode section](crate#unicode) in this crate's top-level + /// documentation. + /// + /// The default for this is `true`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"\w") + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally greek letters would be included in \w, but since + /// // Unicode mode is disabled, it only matches ASCII letters. + /// assert!(!re.is_match("δ")); + /// + /// let re = RegexBuilder::new(r"s") + /// .case_insensitive(true) + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally 'ſ' is included when searching for 's' case + /// // insensitively due to Unicode's simple case folding rules. But + /// // when Unicode mode is disabled, only ASCII case insensitive rules + /// // are used. + /// assert!(!re.is_match("ſ")); + /// ``` + pub fn unicode(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.unicode(yes); + self + } + + /// This configures whether to enable case insensitive matching for the + /// entire pattern. + /// + /// This setting can also be configured using the inline flag `i` + /// in the pattern. For example, `(?i:foo)` matches `foo` case + /// insensitively while `(?-i:foo)` matches `foo` case sensitively. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"foo(?-i:bar)quux") + /// .case_insensitive(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match("FoObarQuUx")); + /// // Even though case insensitive matching is enabled in the builder, + /// // it can be locally disabled within the pattern. In this case, + /// // `bar` is matched case sensitively. + /// assert!(!re.is_match("fooBARquux")); + /// ``` + pub fn case_insensitive(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.case_insensitive(yes); + self + } + + /// This configures multi-line mode for the entire pattern. + /// + /// Enabling multi-line mode changes the behavior of the `^` and `$` + /// anchor assertions. Instead of only matching at the beginning and + /// end of a haystack, respectively, multi-line mode causes them to + /// match at the beginning and end of a line *in addition* to the + /// beginning and end of a haystack. More precisely, `^` will match at + /// the position immediately following a `\n` and `$` will match at the + /// position immediately preceding a `\n`. + /// + /// The behavior of this option can be impacted by other settings too: + /// + /// * The [`RegexBuilder::line_terminator`] option changes `\n` above + /// to any ASCII byte. + /// * The [`RegexBuilder::crlf`] option changes the line terminator to + /// be either `\r` or `\n`, but never at the position between a `\r` + /// and `\n`. + /// + /// This setting can also be configured using the inline flag `m` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"^foo$") + /// .multi_line(true) + /// .build() + /// .unwrap(); + /// assert_eq!(Some(1..4), re.find("\nfoo\n").map(|m| m.range())); + /// ``` + pub fn multi_line(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.multi_line(yes); + self + } + + /// This configures dot-matches-new-line mode for the entire pattern. + /// + /// Perhaps surprisingly, the default behavior for `.` is not to match + /// any character, but rather, to match any character except for the + /// line terminator (which is `\n` by default). When this mode is + /// enabled, the behavior changes such that `.` truly matches any + /// character. + /// + /// This setting can also be configured using the inline flag `s` in + /// the pattern. For example, `(?s:.)` and `\p{any}` are equivalent + /// regexes. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"foo.bar") + /// .dot_matches_new_line(true) + /// .build() + /// .unwrap(); + /// let hay = "foo\nbar"; + /// assert_eq!(Some("foo\nbar"), re.find(hay).map(|m| m.as_str())); + /// ``` + pub fn dot_matches_new_line( + &mut self, + yes: bool, + ) -> &mut RegexBuilder { + self.builder.dot_matches_new_line(yes); + self + } + + /// This configures CRLF mode for the entire pattern. + /// + /// When CRLF mode is enabled, both `\r` ("carriage return" or CR for + /// short) and `\n` ("line feed" or LF for short) are treated as line + /// terminators. This results in the following: + /// + /// * Unless dot-matches-new-line mode is enabled, `.` will now match + /// any character except for `\n` and `\r`. + /// * When multi-line mode is enabled, `^` will match immediately + /// following a `\n` or a `\r`. Similarly, `$` will match immediately + /// preceding a `\n` or a `\r`. Neither `^` nor `$` will ever match + /// between `\r` and `\n`. + /// + /// This setting can also be configured using the inline flag `R` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"^foo$") + /// .multi_line(true) + /// .crlf(true) + /// .build() + /// .unwrap(); + /// let hay = "\r\nfoo\r\n"; + /// // If CRLF mode weren't enabled here, then '$' wouldn't match + /// // immediately after 'foo', and thus no match would be found. + /// assert_eq!(Some("foo"), re.find(hay).map(|m| m.as_str())); + /// ``` + /// + /// This example demonstrates that `^` will never match at a position + /// between `\r` and `\n`. (`$` will similarly not match between a `\r` + /// and a `\n`.) + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"^") + /// .multi_line(true) + /// .crlf(true) + /// .build() + /// .unwrap(); + /// let hay = "\r\n\r\n"; + /// let ranges: Vec<_> = re.find_iter(hay).map(|m| m.range()).collect(); + /// assert_eq!(ranges, vec![0..0, 2..2, 4..4]); + /// ``` + pub fn crlf(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.crlf(yes); + self + } + + /// Configures the line terminator to be used by the regex. + /// + /// The line terminator is relevant in two ways for a particular regex: + /// + /// * When dot-matches-new-line mode is *not* enabled (the default), + /// then `.` will match any character except for the configured line + /// terminator. + /// * When multi-line mode is enabled (not the default), then `^` and + /// `$` will match immediately after and before, respectively, a line + /// terminator. + /// + /// In both cases, if CRLF mode is enabled in a particular context, + /// then it takes precedence over any configured line terminator. + /// + /// This option cannot be configured from within the pattern. + /// + /// The default line terminator is `\n`. + /// + /// # Example + /// + /// This shows how to treat the NUL byte as a line terminator. This can + /// be a useful heuristic when searching binary data. + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"^foo$") + /// .multi_line(true) + /// .line_terminator(b'\x00') + /// .build() + /// .unwrap(); + /// let hay = "\x00foo\x00"; + /// assert_eq!(Some(1..4), re.find(hay).map(|m| m.range())); + /// ``` + /// + /// This example shows that the behavior of `.` is impacted by this + /// setting as well: + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r".") + /// .line_terminator(b'\x00') + /// .build() + /// .unwrap(); + /// assert!(re.is_match("\n")); + /// assert!(!re.is_match("\x00")); + /// ``` + /// + /// This shows that building a regex will fail if the byte given + /// is not ASCII and the pattern could result in matching invalid + /// UTF-8. This is because any singular non-ASCII byte is not valid + /// UTF-8, and it is not permitted for a [`Regex`] to match invalid + /// UTF-8. (It is permissible to use a non-ASCII byte when building a + /// [`bytes::Regex`](crate::bytes::Regex).) + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// assert!(RegexBuilder::new(r".").line_terminator(0x80).build().is_err()); + /// // Note that using a non-ASCII byte isn't enough on its own to + /// // cause regex compilation to fail. You actually have to make use + /// // of it in the regex in a way that leads to matching invalid + /// // UTF-8. If you don't, then regex compilation will succeed! + /// assert!(RegexBuilder::new(r"a").line_terminator(0x80).build().is_ok()); + /// ``` + pub fn line_terminator(&mut self, byte: u8) -> &mut RegexBuilder { + self.builder.line_terminator(byte); + self + } + + /// This configures swap-greed mode for the entire pattern. + /// + /// When swap-greed mode is enabled, patterns like `a+` will become + /// non-greedy and patterns like `a+?` will become greedy. In other + /// words, the meanings of `a+` and `a+?` are switched. + /// + /// This setting can also be configured using the inline flag `U` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"a+") + /// .swap_greed(true) + /// .build() + /// .unwrap(); + /// assert_eq!(Some("a"), re.find("aaa").map(|m| m.as_str())); + /// ``` + pub fn swap_greed(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.swap_greed(yes); + self + } + + /// This configures verbose mode for the entire pattern. + /// + /// When enabled, whitespace will treated as insignificant in the + /// pattern and `#` can be used to start a comment until the next new + /// line. + /// + /// Normally, in most places in a pattern, whitespace is treated + /// literally. For example ` +` will match one or more ASCII whitespace + /// characters. + /// + /// When verbose mode is enabled, `\#` can be used to match a literal + /// `#` and `\ ` can be used to match a literal ASCII whitespace + /// character. + /// + /// Verbose mode is useful for permitting regexes to be formatted and + /// broken up more nicely. This may make them more easily readable. + /// + /// This setting can also be configured using the inline flag `x` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// let pat = r" + /// \b + /// (?\p{Uppercase}\w*) # always start with uppercase letter + /// [\s--\n]+ # whitespace should separate names + /// (?: # middle name can be an initial! + /// (?:(?\p{Uppercase})\.|(?\p{Uppercase}\w*)) + /// [\s--\n]+ + /// )? + /// (?\p{Uppercase}\w*) + /// \b + /// "; + /// let re = RegexBuilder::new(pat) + /// .ignore_whitespace(true) + /// .build() + /// .unwrap(); + /// + /// let caps = re.captures("Harry Potter").unwrap(); + /// assert_eq!("Harry", &caps["first"]); + /// assert_eq!("Potter", &caps["last"]); + /// + /// let caps = re.captures("Harry J. Potter").unwrap(); + /// assert_eq!("Harry", &caps["first"]); + /// // Since a middle name/initial isn't required for an overall match, + /// // we can't assume that 'initial' or 'middle' will be populated! + /// assert_eq!(Some("J"), caps.name("initial").map(|m| m.as_str())); + /// assert_eq!(None, caps.name("middle").map(|m| m.as_str())); + /// assert_eq!("Potter", &caps["last"]); + /// + /// let caps = re.captures("Harry James Potter").unwrap(); + /// assert_eq!("Harry", &caps["first"]); + /// // Since a middle name/initial isn't required for an overall match, + /// // we can't assume that 'initial' or 'middle' will be populated! + /// assert_eq!(None, caps.name("initial").map(|m| m.as_str())); + /// assert_eq!(Some("James"), caps.name("middle").map(|m| m.as_str())); + /// assert_eq!("Potter", &caps["last"]); + /// ``` + pub fn ignore_whitespace(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.ignore_whitespace(yes); + self + } + + /// This configures octal mode for the entire pattern. + /// + /// Octal syntax is a little-known way of uttering Unicode codepoints + /// in a pattern. For example, `a`, `\x61`, `\u0061` and `\141` are all + /// equivalent patterns, where the last example shows octal syntax. + /// + /// While supporting octal syntax isn't in and of itself a problem, + /// it does make good error messages harder. That is, in PCRE based + /// regex engines, syntax like `\1` invokes a backreference, which is + /// explicitly unsupported this library. However, many users expect + /// backreferences to be supported. Therefore, when octal support + /// is disabled, the error message will explicitly mention that + /// backreferences aren't supported. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// // Normally this pattern would not compile, with an error message + /// // about backreferences not being supported. But with octal mode + /// // enabled, octal escape sequences work. + /// let re = RegexBuilder::new(r"\141") + /// .octal(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match("a")); + /// ``` + pub fn octal(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.octal(yes); + self + } + + /// Sets the approximate size limit, in bytes, of the compiled regex. + /// + /// This roughly corresponds to the number of heap memory, in + /// bytes, occupied by a single regex. If the regex would otherwise + /// approximately exceed this limit, then compiling that regex will + /// fail. + /// + /// The main utility of a method like this is to avoid compiling + /// regexes that use an unexpected amount of resources, such as + /// time and memory. Even if the memory usage of a large regex is + /// acceptable, its search time may not be. Namely, worst case time + /// complexity for search is `O(m * n)`, where `m ~ len(pattern)` and + /// `n ~ len(haystack)`. That is, search time depends, in part, on the + /// size of the compiled regex. This means that putting a limit on the + /// size of the regex limits how much a regex can impact search time. + /// + /// For more information about regex size limits, see the section on + /// [untrusted inputs](crate#untrusted-input) in the top-level crate + /// documentation. + /// + /// The default for this is some reasonable number that permits most + /// patterns to compile successfully. + /// + /// # Example + /// + /// ``` + /// # if !cfg!(target_pointer_width = "64") { return; } // see #1041 + /// use regex::RegexBuilder; + /// + /// // It may surprise you how big some seemingly small patterns can + /// // be! Since \w is Unicode aware, this generates a regex that can + /// // match approximately 140,000 distinct codepoints. + /// assert!(RegexBuilder::new(r"\w").size_limit(45_000).build().is_err()); + /// ``` + pub fn size_limit(&mut self, bytes: usize) -> &mut RegexBuilder { + self.builder.size_limit(bytes); + self + } + + /// Set the approximate capacity, in bytes, of the cache of transitions + /// used by the lazy DFA. + /// + /// While the lazy DFA isn't always used, in tends to be the most + /// commonly use regex engine in default configurations. It tends to + /// adopt the performance profile of a fully build DFA, but without the + /// downside of taking worst case exponential time to build. + /// + /// The downside is that it needs to keep a cache of transitions and + /// states that are built while running a search, and this cache + /// can fill up. When it fills up, the cache will reset itself. Any + /// previously generated states and transitions will then need to be + /// re-generated. If this happens too many times, then this library + /// will bail out of using the lazy DFA and switch to a different regex + /// engine. + /// + /// If your regex provokes this particular downside of the lazy DFA, + /// then it may be beneficial to increase its cache capacity. This will + /// potentially reduce the frequency of cache resetting (ideally to + /// `0`). While it won't fix all potential performance problems with + /// the lazy DFA, increasing the cache capacity does fix some. + /// + /// There is no easy way to determine, a priori, whether increasing + /// this cache capacity will help. In general, the larger your regex, + /// the more cache it's likely to use. But that isn't an ironclad rule. + /// For example, a regex like `[01]*1[01]{N}` would normally produce a + /// fully build DFA that is exponential in size with respect to `N`. + /// The lazy DFA will prevent exponential space blow-up, but it cache + /// is likely to fill up, even when it's large and even for smallish + /// values of `N`. + /// + /// If you aren't sure whether this helps or not, it is sensible to + /// set this to some arbitrarily large number in testing, such as + /// `usize::MAX`. Namely, this represents the amount of capacity that + /// *may* be used. It's probably not a good idea to use `usize::MAX` in + /// production though, since it implies there are no controls on heap + /// memory used by this library during a search. In effect, set it to + /// whatever you're willing to allocate for a single regex search. + pub fn dfa_size_limit(&mut self, bytes: usize) -> &mut RegexBuilder { + self.builder.dfa_size_limit(bytes); + self + } + + /// Set the nesting limit for this parser. + /// + /// The nesting limit controls how deep the abstract syntax tree is + /// allowed to be. If the AST exceeds the given limit (e.g., with too + /// many nested groups), then an error is returned by the parser. + /// + /// The purpose of this limit is to act as a heuristic to prevent stack + /// overflow for consumers that do structural induction on an AST using + /// explicit recursion. While this crate never does this (instead using + /// constant stack space and moving the call stack to the heap), other + /// crates may. + /// + /// This limit is not checked until the entire AST is parsed. + /// Therefore, if callers want to put a limit on the amount of heap + /// space used, then they should impose a limit on the length, in + /// bytes, of the concrete pattern string. In particular, this is + /// viable since this parser implementation will limit itself to heap + /// space proportional to the length of the pattern string. See also + /// the [untrusted inputs](crate#untrusted-input) section in the + /// top-level crate documentation for more information about this. + /// + /// Note that a nest limit of `0` will return a nest limit error for + /// most patterns but not all. For example, a nest limit of `0` permits + /// `a` but not `ab`, since `ab` requires an explicit concatenation, + /// which results in a nest depth of `1`. In general, a nest limit is + /// not something that manifests in an obvious way in the concrete + /// syntax, therefore, it should not be used in a granular way. + /// + /// # Example + /// + /// ``` + /// use regex::RegexBuilder; + /// + /// assert!(RegexBuilder::new(r"a").nest_limit(0).build().is_ok()); + /// assert!(RegexBuilder::new(r"ab").nest_limit(0).build().is_err()); + /// ``` + pub fn nest_limit(&mut self, limit: u32) -> &mut RegexBuilder { + self.builder.nest_limit(limit); + self + } + } + + /// A configurable builder for a [`RegexSet`]. + /// + /// This builder can be used to programmatically set flags such as + /// `i` (case insensitive) and `x` (for verbose mode). This builder + /// can also be used to configure things like the line terminator + /// and a size limit on the compiled regular expression. + #[derive(Clone, Debug)] + pub struct RegexSetBuilder { + builder: Builder, + } + + impl RegexSetBuilder { + /// Create a new builder with a default configuration for the given + /// patterns. + /// + /// If the patterns are invalid or exceed the configured size limits, + /// then an error will be returned when [`RegexSetBuilder::build`] is + /// called. + pub fn new(patterns: I) -> RegexSetBuilder + where + I: IntoIterator, + S: AsRef, + { + RegexSetBuilder { builder: Builder::new(patterns) } + } + + /// Compiles the patterns given to `RegexSetBuilder::new` with the + /// configuration set on this builder. + /// + /// If the patterns aren't valid regexes or if a configured size limit + /// was exceeded, then an error is returned. + pub fn build(&self) -> Result { + self.builder.build_many_string() + } + + /// This configures Unicode mode for the all of the patterns. + /// + /// Enabling Unicode mode does a number of things: + /// + /// * Most fundamentally, it causes the fundamental atom of matching + /// to be a single codepoint. When Unicode mode is disabled, it's a + /// single byte. For example, when Unicode mode is enabled, `.` will + /// match `💩` once, where as it will match 4 times when Unicode mode + /// is disabled. (Since the UTF-8 encoding of `💩` is 4 bytes long.) + /// * Case insensitive matching uses Unicode simple case folding rules. + /// * Unicode character classes like `\p{Letter}` and `\p{Greek}` are + /// available. + /// * Perl character classes are Unicode aware. That is, `\w`, `\s` and + /// `\d`. + /// * The word boundary assertions, `\b` and `\B`, use the Unicode + /// definition of a word character. + /// + /// Note that if Unicode mode is disabled, then the regex will fail to + /// compile if it could match invalid UTF-8. For example, when Unicode + /// mode is disabled, then since `.` matches any byte (except for + /// `\n`), then it can match invalid UTF-8 and thus building a regex + /// from it will fail. Another example is `\w` and `\W`. Since `\w` can + /// only match ASCII bytes when Unicode mode is disabled, it's allowed. + /// But `\W` can match more than ASCII bytes, including invalid UTF-8, + /// and so it is not allowed. This restriction can be lifted only by + /// using a [`bytes::RegexSet`](crate::bytes::RegexSet). + /// + /// For more details on the Unicode support in this crate, see the + /// [Unicode section](crate#unicode) in this crate's top-level + /// documentation. + /// + /// The default for this is `true`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"\w"]) + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally greek letters would be included in \w, but since + /// // Unicode mode is disabled, it only matches ASCII letters. + /// assert!(!re.is_match("δ")); + /// + /// let re = RegexSetBuilder::new([r"s"]) + /// .case_insensitive(true) + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally 'ſ' is included when searching for 's' case + /// // insensitively due to Unicode's simple case folding rules. But + /// // when Unicode mode is disabled, only ASCII case insensitive rules + /// // are used. + /// assert!(!re.is_match("ſ")); + /// ``` + pub fn unicode(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.unicode(yes); + self + } + + /// This configures whether to enable case insensitive matching for all + /// of the patterns. + /// + /// This setting can also be configured using the inline flag `i` + /// in the pattern. For example, `(?i:foo)` matches `foo` case + /// insensitively while `(?-i:foo)` matches `foo` case sensitively. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"foo(?-i:bar)quux"]) + /// .case_insensitive(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match("FoObarQuUx")); + /// // Even though case insensitive matching is enabled in the builder, + /// // it can be locally disabled within the pattern. In this case, + /// // `bar` is matched case sensitively. + /// assert!(!re.is_match("fooBARquux")); + /// ``` + pub fn case_insensitive(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.case_insensitive(yes); + self + } + + /// This configures multi-line mode for all of the patterns. + /// + /// Enabling multi-line mode changes the behavior of the `^` and `$` + /// anchor assertions. Instead of only matching at the beginning and + /// end of a haystack, respectively, multi-line mode causes them to + /// match at the beginning and end of a line *in addition* to the + /// beginning and end of a haystack. More precisely, `^` will match at + /// the position immediately following a `\n` and `$` will match at the + /// position immediately preceding a `\n`. + /// + /// The behavior of this option can be impacted by other settings too: + /// + /// * The [`RegexSetBuilder::line_terminator`] option changes `\n` + /// above to any ASCII byte. + /// * The [`RegexSetBuilder::crlf`] option changes the line terminator + /// to be either `\r` or `\n`, but never at the position between a `\r` + /// and `\n`. + /// + /// This setting can also be configured using the inline flag `m` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"^foo$"]) + /// .multi_line(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match("\nfoo\n")); + /// ``` + pub fn multi_line(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.multi_line(yes); + self + } + + /// This configures dot-matches-new-line mode for the entire pattern. + /// + /// Perhaps surprisingly, the default behavior for `.` is not to match + /// any character, but rather, to match any character except for the + /// line terminator (which is `\n` by default). When this mode is + /// enabled, the behavior changes such that `.` truly matches any + /// character. + /// + /// This setting can also be configured using the inline flag `s` in + /// the pattern. For example, `(?s:.)` and `\p{any}` are equivalent + /// regexes. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"foo.bar"]) + /// .dot_matches_new_line(true) + /// .build() + /// .unwrap(); + /// let hay = "foo\nbar"; + /// assert!(re.is_match(hay)); + /// ``` + pub fn dot_matches_new_line( + &mut self, + yes: bool, + ) -> &mut RegexSetBuilder { + self.builder.dot_matches_new_line(yes); + self + } + + /// This configures CRLF mode for all of the patterns. + /// + /// When CRLF mode is enabled, both `\r` ("carriage return" or CR for + /// short) and `\n` ("line feed" or LF for short) are treated as line + /// terminators. This results in the following: + /// + /// * Unless dot-matches-new-line mode is enabled, `.` will now match + /// any character except for `\n` and `\r`. + /// * When multi-line mode is enabled, `^` will match immediately + /// following a `\n` or a `\r`. Similarly, `$` will match immediately + /// preceding a `\n` or a `\r`. Neither `^` nor `$` will ever match + /// between `\r` and `\n`. + /// + /// This setting can also be configured using the inline flag `R` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"^foo$"]) + /// .multi_line(true) + /// .crlf(true) + /// .build() + /// .unwrap(); + /// let hay = "\r\nfoo\r\n"; + /// // If CRLF mode weren't enabled here, then '$' wouldn't match + /// // immediately after 'foo', and thus no match would be found. + /// assert!(re.is_match(hay)); + /// ``` + /// + /// This example demonstrates that `^` will never match at a position + /// between `\r` and `\n`. (`$` will similarly not match between a `\r` + /// and a `\n`.) + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"^\n"]) + /// .multi_line(true) + /// .crlf(true) + /// .build() + /// .unwrap(); + /// assert!(!re.is_match("\r\n")); + /// ``` + pub fn crlf(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.crlf(yes); + self + } + + /// Configures the line terminator to be used by the regex. + /// + /// The line terminator is relevant in two ways for a particular regex: + /// + /// * When dot-matches-new-line mode is *not* enabled (the default), + /// then `.` will match any character except for the configured line + /// terminator. + /// * When multi-line mode is enabled (not the default), then `^` and + /// `$` will match immediately after and before, respectively, a line + /// terminator. + /// + /// In both cases, if CRLF mode is enabled in a particular context, + /// then it takes precedence over any configured line terminator. + /// + /// This option cannot be configured from within the pattern. + /// + /// The default line terminator is `\n`. + /// + /// # Example + /// + /// This shows how to treat the NUL byte as a line terminator. This can + /// be a useful heuristic when searching binary data. + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"^foo$"]) + /// .multi_line(true) + /// .line_terminator(b'\x00') + /// .build() + /// .unwrap(); + /// let hay = "\x00foo\x00"; + /// assert!(re.is_match(hay)); + /// ``` + /// + /// This example shows that the behavior of `.` is impacted by this + /// setting as well: + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"."]) + /// .line_terminator(b'\x00') + /// .build() + /// .unwrap(); + /// assert!(re.is_match("\n")); + /// assert!(!re.is_match("\x00")); + /// ``` + /// + /// This shows that building a regex will fail if the byte given + /// is not ASCII and the pattern could result in matching invalid + /// UTF-8. This is because any singular non-ASCII byte is not valid + /// UTF-8, and it is not permitted for a [`RegexSet`] to match invalid + /// UTF-8. (It is permissible to use a non-ASCII byte when building a + /// [`bytes::RegexSet`](crate::bytes::RegexSet).) + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// assert!( + /// RegexSetBuilder::new([r"."]) + /// .line_terminator(0x80) + /// .build() + /// .is_err() + /// ); + /// // Note that using a non-ASCII byte isn't enough on its own to + /// // cause regex compilation to fail. You actually have to make use + /// // of it in the regex in a way that leads to matching invalid + /// // UTF-8. If you don't, then regex compilation will succeed! + /// assert!( + /// RegexSetBuilder::new([r"a"]) + /// .line_terminator(0x80) + /// .build() + /// .is_ok() + /// ); + /// ``` + pub fn line_terminator(&mut self, byte: u8) -> &mut RegexSetBuilder { + self.builder.line_terminator(byte); + self + } + + /// This configures swap-greed mode for all of the patterns. + /// + /// When swap-greed mode is enabled, patterns like `a+` will become + /// non-greedy and patterns like `a+?` will become greedy. In other + /// words, the meanings of `a+` and `a+?` are switched. + /// + /// This setting can also be configured using the inline flag `U` in + /// the pattern. + /// + /// Note that this is generally not useful for a `RegexSet` since a + /// `RegexSet` can only report whether a pattern matches or not. Since + /// greediness never impacts whether a match is found or not (only the + /// offsets of the match), it follows that whether parts of a pattern + /// are greedy or not doesn't matter for a `RegexSet`. + /// + /// The default for this is `false`. + pub fn swap_greed(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.swap_greed(yes); + self + } + + /// This configures verbose mode for all of the patterns. + /// + /// When enabled, whitespace will treated as insignificant in the + /// pattern and `#` can be used to start a comment until the next new + /// line. + /// + /// Normally, in most places in a pattern, whitespace is treated + /// literally. For example ` +` will match one or more ASCII whitespace + /// characters. + /// + /// When verbose mode is enabled, `\#` can be used to match a literal + /// `#` and `\ ` can be used to match a literal ASCII whitespace + /// character. + /// + /// Verbose mode is useful for permitting regexes to be formatted and + /// broken up more nicely. This may make them more easily readable. + /// + /// This setting can also be configured using the inline flag `x` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// let pat = r" + /// \b + /// (?\p{Uppercase}\w*) # always start with uppercase letter + /// [\s--\n]+ # whitespace should separate names + /// (?: # middle name can be an initial! + /// (?:(?\p{Uppercase})\.|(?\p{Uppercase}\w*)) + /// [\s--\n]+ + /// )? + /// (?\p{Uppercase}\w*) + /// \b + /// "; + /// let re = RegexSetBuilder::new([pat]) + /// .ignore_whitespace(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match("Harry Potter")); + /// assert!(re.is_match("Harry J. Potter")); + /// assert!(re.is_match("Harry James Potter")); + /// assert!(!re.is_match("harry J. Potter")); + /// ``` + pub fn ignore_whitespace( + &mut self, + yes: bool, + ) -> &mut RegexSetBuilder { + self.builder.ignore_whitespace(yes); + self + } + + /// This configures octal mode for all of the patterns. + /// + /// Octal syntax is a little-known way of uttering Unicode codepoints + /// in a pattern. For example, `a`, `\x61`, `\u0061` and `\141` are all + /// equivalent patterns, where the last example shows octal syntax. + /// + /// While supporting octal syntax isn't in and of itself a problem, + /// it does make good error messages harder. That is, in PCRE based + /// regex engines, syntax like `\1` invokes a backreference, which is + /// explicitly unsupported this library. However, many users expect + /// backreferences to be supported. Therefore, when octal support + /// is disabled, the error message will explicitly mention that + /// backreferences aren't supported. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// // Normally this pattern would not compile, with an error message + /// // about backreferences not being supported. But with octal mode + /// // enabled, octal escape sequences work. + /// let re = RegexSetBuilder::new([r"\141"]) + /// .octal(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match("a")); + /// ``` + pub fn octal(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.octal(yes); + self + } + + /// Sets the approximate size limit, in bytes, of the compiled regex. + /// + /// This roughly corresponds to the number of heap memory, in + /// bytes, occupied by a single regex. If the regex would otherwise + /// approximately exceed this limit, then compiling that regex will + /// fail. + /// + /// The main utility of a method like this is to avoid compiling + /// regexes that use an unexpected amount of resources, such as + /// time and memory. Even if the memory usage of a large regex is + /// acceptable, its search time may not be. Namely, worst case time + /// complexity for search is `O(m * n)`, where `m ~ len(pattern)` and + /// `n ~ len(haystack)`. That is, search time depends, in part, on the + /// size of the compiled regex. This means that putting a limit on the + /// size of the regex limits how much a regex can impact search time. + /// + /// For more information about regex size limits, see the section on + /// [untrusted inputs](crate#untrusted-input) in the top-level crate + /// documentation. + /// + /// The default for this is some reasonable number that permits most + /// patterns to compile successfully. + /// + /// # Example + /// + /// ``` + /// # if !cfg!(target_pointer_width = "64") { return; } // see #1041 + /// use regex::RegexSetBuilder; + /// + /// // It may surprise you how big some seemingly small patterns can + /// // be! Since \w is Unicode aware, this generates a regex that can + /// // match approximately 140,000 distinct codepoints. + /// assert!( + /// RegexSetBuilder::new([r"\w"]) + /// .size_limit(45_000) + /// .build() + /// .is_err() + /// ); + /// ``` + pub fn size_limit(&mut self, bytes: usize) -> &mut RegexSetBuilder { + self.builder.size_limit(bytes); + self + } + + /// Set the approximate capacity, in bytes, of the cache of transitions + /// used by the lazy DFA. + /// + /// While the lazy DFA isn't always used, in tends to be the most + /// commonly use regex engine in default configurations. It tends to + /// adopt the performance profile of a fully build DFA, but without the + /// downside of taking worst case exponential time to build. + /// + /// The downside is that it needs to keep a cache of transitions and + /// states that are built while running a search, and this cache + /// can fill up. When it fills up, the cache will reset itself. Any + /// previously generated states and transitions will then need to be + /// re-generated. If this happens too many times, then this library + /// will bail out of using the lazy DFA and switch to a different regex + /// engine. + /// + /// If your regex provokes this particular downside of the lazy DFA, + /// then it may be beneficial to increase its cache capacity. This will + /// potentially reduce the frequency of cache resetting (ideally to + /// `0`). While it won't fix all potential performance problems with + /// the lazy DFA, increasing the cache capacity does fix some. + /// + /// There is no easy way to determine, a priori, whether increasing + /// this cache capacity will help. In general, the larger your regex, + /// the more cache it's likely to use. But that isn't an ironclad rule. + /// For example, a regex like `[01]*1[01]{N}` would normally produce a + /// fully build DFA that is exponential in size with respect to `N`. + /// The lazy DFA will prevent exponential space blow-up, but it cache + /// is likely to fill up, even when it's large and even for smallish + /// values of `N`. + /// + /// If you aren't sure whether this helps or not, it is sensible to + /// set this to some arbitrarily large number in testing, such as + /// `usize::MAX`. Namely, this represents the amount of capacity that + /// *may* be used. It's probably not a good idea to use `usize::MAX` in + /// production though, since it implies there are no controls on heap + /// memory used by this library during a search. In effect, set it to + /// whatever you're willing to allocate for a single regex search. + pub fn dfa_size_limit( + &mut self, + bytes: usize, + ) -> &mut RegexSetBuilder { + self.builder.dfa_size_limit(bytes); + self + } + + /// Set the nesting limit for this parser. + /// + /// The nesting limit controls how deep the abstract syntax tree is + /// allowed to be. If the AST exceeds the given limit (e.g., with too + /// many nested groups), then an error is returned by the parser. + /// + /// The purpose of this limit is to act as a heuristic to prevent stack + /// overflow for consumers that do structural induction on an AST using + /// explicit recursion. While this crate never does this (instead using + /// constant stack space and moving the call stack to the heap), other + /// crates may. + /// + /// This limit is not checked until the entire AST is parsed. + /// Therefore, if callers want to put a limit on the amount of heap + /// space used, then they should impose a limit on the length, in + /// bytes, of the concrete pattern string. In particular, this is + /// viable since this parser implementation will limit itself to heap + /// space proportional to the length of the pattern string. See also + /// the [untrusted inputs](crate#untrusted-input) section in the + /// top-level crate documentation for more information about this. + /// + /// Note that a nest limit of `0` will return a nest limit error for + /// most patterns but not all. For example, a nest limit of `0` permits + /// `a` but not `ab`, since `ab` requires an explicit concatenation, + /// which results in a nest depth of `1`. In general, a nest limit is + /// not something that manifests in an obvious way in the concrete + /// syntax, therefore, it should not be used in a granular way. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSetBuilder; + /// + /// assert!(RegexSetBuilder::new([r"a"]).nest_limit(0).build().is_ok()); + /// assert!(RegexSetBuilder::new([r"ab"]).nest_limit(0).build().is_err()); + /// ``` + pub fn nest_limit(&mut self, limit: u32) -> &mut RegexSetBuilder { + self.builder.nest_limit(limit); + self + } + } +} + +pub(crate) mod bytes { + use crate::{ + bytes::{Regex, RegexSet}, + error::Error, + }; + + use super::Builder; + + /// A configurable builder for a [`Regex`]. + /// + /// This builder can be used to programmatically set flags such as `i` + /// (case insensitive) and `x` (for verbose mode). This builder can also be + /// used to configure things like the line terminator and a size limit on + /// the compiled regular expression. + #[derive(Clone, Debug)] + pub struct RegexBuilder { + builder: Builder, + } + + impl RegexBuilder { + /// Create a new builder with a default configuration for the given + /// pattern. + /// + /// If the pattern is invalid or exceeds the configured size limits, + /// then an error will be returned when [`RegexBuilder::build`] is + /// called. + pub fn new(pattern: &str) -> RegexBuilder { + RegexBuilder { builder: Builder::new([pattern]) } + } + + /// Compiles the pattern given to `RegexBuilder::new` with the + /// configuration set on this builder. + /// + /// If the pattern isn't a valid regex or if a configured size limit + /// was exceeded, then an error is returned. + pub fn build(&self) -> Result { + self.builder.build_one_bytes() + } + + /// This configures Unicode mode for the entire pattern. + /// + /// Enabling Unicode mode does a number of things: + /// + /// * Most fundamentally, it causes the fundamental atom of matching + /// to be a single codepoint. When Unicode mode is disabled, it's a + /// single byte. For example, when Unicode mode is enabled, `.` will + /// match `💩` once, where as it will match 4 times when Unicode mode + /// is disabled. (Since the UTF-8 encoding of `💩` is 4 bytes long.) + /// * Case insensitive matching uses Unicode simple case folding rules. + /// * Unicode character classes like `\p{Letter}` and `\p{Greek}` are + /// available. + /// * Perl character classes are Unicode aware. That is, `\w`, `\s` and + /// `\d`. + /// * The word boundary assertions, `\b` and `\B`, use the Unicode + /// definition of a word character. + /// + /// Note that unlike the top-level `Regex` for searching `&str`, it + /// is permitted to disable Unicode mode even if the resulting pattern + /// could match invalid UTF-8. For example, `(?-u:.)` is not a valid + /// pattern for a top-level `Regex`, but is valid for a `bytes::Regex`. + /// + /// For more details on the Unicode support in this crate, see the + /// [Unicode section](crate#unicode) in this crate's top-level + /// documentation. + /// + /// The default for this is `true`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"\w") + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally greek letters would be included in \w, but since + /// // Unicode mode is disabled, it only matches ASCII letters. + /// assert!(!re.is_match("δ".as_bytes())); + /// + /// let re = RegexBuilder::new(r"s") + /// .case_insensitive(true) + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally 'ſ' is included when searching for 's' case + /// // insensitively due to Unicode's simple case folding rules. But + /// // when Unicode mode is disabled, only ASCII case insensitive rules + /// // are used. + /// assert!(!re.is_match("ſ".as_bytes())); + /// ``` + /// + /// Since this builder is for constructing a [`bytes::Regex`](Regex), + /// one can disable Unicode mode even if it would match invalid UTF-8: + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r".") + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally greek letters would be included in \w, but since + /// // Unicode mode is disabled, it only matches ASCII letters. + /// assert!(re.is_match(b"\xFF")); + /// ``` + pub fn unicode(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.unicode(yes); + self + } + + /// This configures whether to enable case insensitive matching for the + /// entire pattern. + /// + /// This setting can also be configured using the inline flag `i` + /// in the pattern. For example, `(?i:foo)` matches `foo` case + /// insensitively while `(?-i:foo)` matches `foo` case sensitively. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"foo(?-i:bar)quux") + /// .case_insensitive(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match(b"FoObarQuUx")); + /// // Even though case insensitive matching is enabled in the builder, + /// // it can be locally disabled within the pattern. In this case, + /// // `bar` is matched case sensitively. + /// assert!(!re.is_match(b"fooBARquux")); + /// ``` + pub fn case_insensitive(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.case_insensitive(yes); + self + } + + /// This configures multi-line mode for the entire pattern. + /// + /// Enabling multi-line mode changes the behavior of the `^` and `$` + /// anchor assertions. Instead of only matching at the beginning and + /// end of a haystack, respectively, multi-line mode causes them to + /// match at the beginning and end of a line *in addition* to the + /// beginning and end of a haystack. More precisely, `^` will match at + /// the position immediately following a `\n` and `$` will match at the + /// position immediately preceding a `\n`. + /// + /// The behavior of this option can be impacted by other settings too: + /// + /// * The [`RegexBuilder::line_terminator`] option changes `\n` above + /// to any ASCII byte. + /// * The [`RegexBuilder::crlf`] option changes the line terminator to + /// be either `\r` or `\n`, but never at the position between a `\r` + /// and `\n`. + /// + /// This setting can also be configured using the inline flag `m` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"^foo$") + /// .multi_line(true) + /// .build() + /// .unwrap(); + /// assert_eq!(Some(1..4), re.find(b"\nfoo\n").map(|m| m.range())); + /// ``` + pub fn multi_line(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.multi_line(yes); + self + } + + /// This configures dot-matches-new-line mode for the entire pattern. + /// + /// Perhaps surprisingly, the default behavior for `.` is not to match + /// any character, but rather, to match any character except for the + /// line terminator (which is `\n` by default). When this mode is + /// enabled, the behavior changes such that `.` truly matches any + /// character. + /// + /// This setting can also be configured using the inline flag `s` in + /// the pattern. For example, `(?s:.)` and `\p{any}` are equivalent + /// regexes. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"foo.bar") + /// .dot_matches_new_line(true) + /// .build() + /// .unwrap(); + /// let hay = b"foo\nbar"; + /// assert_eq!(Some(&b"foo\nbar"[..]), re.find(hay).map(|m| m.as_bytes())); + /// ``` + pub fn dot_matches_new_line( + &mut self, + yes: bool, + ) -> &mut RegexBuilder { + self.builder.dot_matches_new_line(yes); + self + } + + /// This configures CRLF mode for the entire pattern. + /// + /// When CRLF mode is enabled, both `\r` ("carriage return" or CR for + /// short) and `\n` ("line feed" or LF for short) are treated as line + /// terminators. This results in the following: + /// + /// * Unless dot-matches-new-line mode is enabled, `.` will now match + /// any character except for `\n` and `\r`. + /// * When multi-line mode is enabled, `^` will match immediately + /// following a `\n` or a `\r`. Similarly, `$` will match immediately + /// preceding a `\n` or a `\r`. Neither `^` nor `$` will ever match + /// between `\r` and `\n`. + /// + /// This setting can also be configured using the inline flag `R` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"^foo$") + /// .multi_line(true) + /// .crlf(true) + /// .build() + /// .unwrap(); + /// let hay = b"\r\nfoo\r\n"; + /// // If CRLF mode weren't enabled here, then '$' wouldn't match + /// // immediately after 'foo', and thus no match would be found. + /// assert_eq!(Some(&b"foo"[..]), re.find(hay).map(|m| m.as_bytes())); + /// ``` + /// + /// This example demonstrates that `^` will never match at a position + /// between `\r` and `\n`. (`$` will similarly not match between a `\r` + /// and a `\n`.) + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"^") + /// .multi_line(true) + /// .crlf(true) + /// .build() + /// .unwrap(); + /// let hay = b"\r\n\r\n"; + /// let ranges: Vec<_> = re.find_iter(hay).map(|m| m.range()).collect(); + /// assert_eq!(ranges, vec![0..0, 2..2, 4..4]); + /// ``` + pub fn crlf(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.crlf(yes); + self + } + + /// Configures the line terminator to be used by the regex. + /// + /// The line terminator is relevant in two ways for a particular regex: + /// + /// * When dot-matches-new-line mode is *not* enabled (the default), + /// then `.` will match any character except for the configured line + /// terminator. + /// * When multi-line mode is enabled (not the default), then `^` and + /// `$` will match immediately after and before, respectively, a line + /// terminator. + /// + /// In both cases, if CRLF mode is enabled in a particular context, + /// then it takes precedence over any configured line terminator. + /// + /// This option cannot be configured from within the pattern. + /// + /// The default line terminator is `\n`. + /// + /// # Example + /// + /// This shows how to treat the NUL byte as a line terminator. This can + /// be a useful heuristic when searching binary data. + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"^foo$") + /// .multi_line(true) + /// .line_terminator(b'\x00') + /// .build() + /// .unwrap(); + /// let hay = b"\x00foo\x00"; + /// assert_eq!(Some(1..4), re.find(hay).map(|m| m.range())); + /// ``` + /// + /// This example shows that the behavior of `.` is impacted by this + /// setting as well: + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r".") + /// .line_terminator(b'\x00') + /// .build() + /// .unwrap(); + /// assert!(re.is_match(b"\n")); + /// assert!(!re.is_match(b"\x00")); + /// ``` + /// + /// This shows that building a regex will work even when the byte + /// given is not ASCII. This is unlike the top-level `Regex` API where + /// matching invalid UTF-8 is not allowed. + /// + /// Note though that you must disable Unicode mode. This is required + /// because Unicode mode requires matching one codepoint at a time, + /// and there is no way to match a non-ASCII byte as if it were a + /// codepoint. + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// assert!( + /// RegexBuilder::new(r".") + /// .unicode(false) + /// .line_terminator(0x80) + /// .build() + /// .is_ok(), + /// ); + /// ``` + pub fn line_terminator(&mut self, byte: u8) -> &mut RegexBuilder { + self.builder.line_terminator(byte); + self + } + + /// This configures swap-greed mode for the entire pattern. + /// + /// When swap-greed mode is enabled, patterns like `a+` will become + /// non-greedy and patterns like `a+?` will become greedy. In other + /// words, the meanings of `a+` and `a+?` are switched. + /// + /// This setting can also be configured using the inline flag `U` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let re = RegexBuilder::new(r"a+") + /// .swap_greed(true) + /// .build() + /// .unwrap(); + /// assert_eq!(Some(&b"a"[..]), re.find(b"aaa").map(|m| m.as_bytes())); + /// ``` + pub fn swap_greed(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.swap_greed(yes); + self + } + + /// This configures verbose mode for the entire pattern. + /// + /// When enabled, whitespace will treated as insignificant in the + /// pattern and `#` can be used to start a comment until the next new + /// line. + /// + /// Normally, in most places in a pattern, whitespace is treated + /// literally. For example ` +` will match one or more ASCII whitespace + /// characters. + /// + /// When verbose mode is enabled, `\#` can be used to match a literal + /// `#` and `\ ` can be used to match a literal ASCII whitespace + /// character. + /// + /// Verbose mode is useful for permitting regexes to be formatted and + /// broken up more nicely. This may make them more easily readable. + /// + /// This setting can also be configured using the inline flag `x` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// let pat = r" + /// \b + /// (?\p{Uppercase}\w*) # always start with uppercase letter + /// [\s--\n]+ # whitespace should separate names + /// (?: # middle name can be an initial! + /// (?:(?\p{Uppercase})\.|(?\p{Uppercase}\w*)) + /// [\s--\n]+ + /// )? + /// (?\p{Uppercase}\w*) + /// \b + /// "; + /// let re = RegexBuilder::new(pat) + /// .ignore_whitespace(true) + /// .build() + /// .unwrap(); + /// + /// let caps = re.captures(b"Harry Potter").unwrap(); + /// assert_eq!(&b"Harry"[..], &caps["first"]); + /// assert_eq!(&b"Potter"[..], &caps["last"]); + /// + /// let caps = re.captures(b"Harry J. Potter").unwrap(); + /// assert_eq!(&b"Harry"[..], &caps["first"]); + /// // Since a middle name/initial isn't required for an overall match, + /// // we can't assume that 'initial' or 'middle' will be populated! + /// assert_eq!( + /// Some(&b"J"[..]), + /// caps.name("initial").map(|m| m.as_bytes()), + /// ); + /// assert_eq!(None, caps.name("middle").map(|m| m.as_bytes())); + /// assert_eq!(&b"Potter"[..], &caps["last"]); + /// + /// let caps = re.captures(b"Harry James Potter").unwrap(); + /// assert_eq!(&b"Harry"[..], &caps["first"]); + /// // Since a middle name/initial isn't required for an overall match, + /// // we can't assume that 'initial' or 'middle' will be populated! + /// assert_eq!(None, caps.name("initial").map(|m| m.as_bytes())); + /// assert_eq!( + /// Some(&b"James"[..]), + /// caps.name("middle").map(|m| m.as_bytes()), + /// ); + /// assert_eq!(&b"Potter"[..], &caps["last"]); + /// ``` + pub fn ignore_whitespace(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.ignore_whitespace(yes); + self + } + + /// This configures octal mode for the entire pattern. + /// + /// Octal syntax is a little-known way of uttering Unicode codepoints + /// in a pattern. For example, `a`, `\x61`, `\u0061` and `\141` are all + /// equivalent patterns, where the last example shows octal syntax. + /// + /// While supporting octal syntax isn't in and of itself a problem, + /// it does make good error messages harder. That is, in PCRE based + /// regex engines, syntax like `\1` invokes a backreference, which is + /// explicitly unsupported this library. However, many users expect + /// backreferences to be supported. Therefore, when octal support + /// is disabled, the error message will explicitly mention that + /// backreferences aren't supported. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// // Normally this pattern would not compile, with an error message + /// // about backreferences not being supported. But with octal mode + /// // enabled, octal escape sequences work. + /// let re = RegexBuilder::new(r"\141") + /// .octal(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match(b"a")); + /// ``` + pub fn octal(&mut self, yes: bool) -> &mut RegexBuilder { + self.builder.octal(yes); + self + } + + /// Sets the approximate size limit, in bytes, of the compiled regex. + /// + /// This roughly corresponds to the number of heap memory, in + /// bytes, occupied by a single regex. If the regex would otherwise + /// approximately exceed this limit, then compiling that regex will + /// fail. + /// + /// The main utility of a method like this is to avoid compiling + /// regexes that use an unexpected amount of resources, such as + /// time and memory. Even if the memory usage of a large regex is + /// acceptable, its search time may not be. Namely, worst case time + /// complexity for search is `O(m * n)`, where `m ~ len(pattern)` and + /// `n ~ len(haystack)`. That is, search time depends, in part, on the + /// size of the compiled regex. This means that putting a limit on the + /// size of the regex limits how much a regex can impact search time. + /// + /// For more information about regex size limits, see the section on + /// [untrusted inputs](crate#untrusted-input) in the top-level crate + /// documentation. + /// + /// The default for this is some reasonable number that permits most + /// patterns to compile successfully. + /// + /// # Example + /// + /// ``` + /// # if !cfg!(target_pointer_width = "64") { return; } // see #1041 + /// use regex::bytes::RegexBuilder; + /// + /// // It may surprise you how big some seemingly small patterns can + /// // be! Since \w is Unicode aware, this generates a regex that can + /// // match approximately 140,000 distinct codepoints. + /// assert!(RegexBuilder::new(r"\w").size_limit(45_000).build().is_err()); + /// ``` + pub fn size_limit(&mut self, bytes: usize) -> &mut RegexBuilder { + self.builder.size_limit(bytes); + self + } + + /// Set the approximate capacity, in bytes, of the cache of transitions + /// used by the lazy DFA. + /// + /// While the lazy DFA isn't always used, in tends to be the most + /// commonly use regex engine in default configurations. It tends to + /// adopt the performance profile of a fully build DFA, but without the + /// downside of taking worst case exponential time to build. + /// + /// The downside is that it needs to keep a cache of transitions and + /// states that are built while running a search, and this cache + /// can fill up. When it fills up, the cache will reset itself. Any + /// previously generated states and transitions will then need to be + /// re-generated. If this happens too many times, then this library + /// will bail out of using the lazy DFA and switch to a different regex + /// engine. + /// + /// If your regex provokes this particular downside of the lazy DFA, + /// then it may be beneficial to increase its cache capacity. This will + /// potentially reduce the frequency of cache resetting (ideally to + /// `0`). While it won't fix all potential performance problems with + /// the lazy DFA, increasing the cache capacity does fix some. + /// + /// There is no easy way to determine, a priori, whether increasing + /// this cache capacity will help. In general, the larger your regex, + /// the more cache it's likely to use. But that isn't an ironclad rule. + /// For example, a regex like `[01]*1[01]{N}` would normally produce a + /// fully build DFA that is exponential in size with respect to `N`. + /// The lazy DFA will prevent exponential space blow-up, but it cache + /// is likely to fill up, even when it's large and even for smallish + /// values of `N`. + /// + /// If you aren't sure whether this helps or not, it is sensible to + /// set this to some arbitrarily large number in testing, such as + /// `usize::MAX`. Namely, this represents the amount of capacity that + /// *may* be used. It's probably not a good idea to use `usize::MAX` in + /// production though, since it implies there are no controls on heap + /// memory used by this library during a search. In effect, set it to + /// whatever you're willing to allocate for a single regex search. + pub fn dfa_size_limit(&mut self, bytes: usize) -> &mut RegexBuilder { + self.builder.dfa_size_limit(bytes); + self + } + + /// Set the nesting limit for this parser. + /// + /// The nesting limit controls how deep the abstract syntax tree is + /// allowed to be. If the AST exceeds the given limit (e.g., with too + /// many nested groups), then an error is returned by the parser. + /// + /// The purpose of this limit is to act as a heuristic to prevent stack + /// overflow for consumers that do structural induction on an AST using + /// explicit recursion. While this crate never does this (instead using + /// constant stack space and moving the call stack to the heap), other + /// crates may. + /// + /// This limit is not checked until the entire AST is parsed. + /// Therefore, if callers want to put a limit on the amount of heap + /// space used, then they should impose a limit on the length, in + /// bytes, of the concrete pattern string. In particular, this is + /// viable since this parser implementation will limit itself to heap + /// space proportional to the length of the pattern string. See also + /// the [untrusted inputs](crate#untrusted-input) section in the + /// top-level crate documentation for more information about this. + /// + /// Note that a nest limit of `0` will return a nest limit error for + /// most patterns but not all. For example, a nest limit of `0` permits + /// `a` but not `ab`, since `ab` requires an explicit concatenation, + /// which results in a nest depth of `1`. In general, a nest limit is + /// not something that manifests in an obvious way in the concrete + /// syntax, therefore, it should not be used in a granular way. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexBuilder; + /// + /// assert!(RegexBuilder::new(r"a").nest_limit(0).build().is_ok()); + /// assert!(RegexBuilder::new(r"ab").nest_limit(0).build().is_err()); + /// ``` + pub fn nest_limit(&mut self, limit: u32) -> &mut RegexBuilder { + self.builder.nest_limit(limit); + self + } + } + + /// A configurable builder for a [`RegexSet`]. + /// + /// This builder can be used to programmatically set flags such as `i` + /// (case insensitive) and `x` (for verbose mode). This builder can also be + /// used to configure things like the line terminator and a size limit on + /// the compiled regular expression. + #[derive(Clone, Debug)] + pub struct RegexSetBuilder { + builder: Builder, + } + + impl RegexSetBuilder { + /// Create a new builder with a default configuration for the given + /// patterns. + /// + /// If the patterns are invalid or exceed the configured size limits, + /// then an error will be returned when [`RegexSetBuilder::build`] is + /// called. + pub fn new(patterns: I) -> RegexSetBuilder + where + I: IntoIterator, + S: AsRef, + { + RegexSetBuilder { builder: Builder::new(patterns) } + } + + /// Compiles the patterns given to `RegexSetBuilder::new` with the + /// configuration set on this builder. + /// + /// If the patterns aren't valid regexes or if a configured size limit + /// was exceeded, then an error is returned. + pub fn build(&self) -> Result { + self.builder.build_many_bytes() + } + + /// This configures Unicode mode for the all of the patterns. + /// + /// Enabling Unicode mode does a number of things: + /// + /// * Most fundamentally, it causes the fundamental atom of matching + /// to be a single codepoint. When Unicode mode is disabled, it's a + /// single byte. For example, when Unicode mode is enabled, `.` will + /// match `💩` once, where as it will match 4 times when Unicode mode + /// is disabled. (Since the UTF-8 encoding of `💩` is 4 bytes long.) + /// * Case insensitive matching uses Unicode simple case folding rules. + /// * Unicode character classes like `\p{Letter}` and `\p{Greek}` are + /// available. + /// * Perl character classes are Unicode aware. That is, `\w`, `\s` and + /// `\d`. + /// * The word boundary assertions, `\b` and `\B`, use the Unicode + /// definition of a word character. + /// + /// Note that unlike the top-level `RegexSet` for searching `&str`, + /// it is permitted to disable Unicode mode even if the resulting + /// pattern could match invalid UTF-8. For example, `(?-u:.)` is not + /// a valid pattern for a top-level `RegexSet`, but is valid for a + /// `bytes::RegexSet`. + /// + /// For more details on the Unicode support in this crate, see the + /// [Unicode section](crate#unicode) in this crate's top-level + /// documentation. + /// + /// The default for this is `true`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"\w"]) + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally greek letters would be included in \w, but since + /// // Unicode mode is disabled, it only matches ASCII letters. + /// assert!(!re.is_match("δ".as_bytes())); + /// + /// let re = RegexSetBuilder::new([r"s"]) + /// .case_insensitive(true) + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally 'ſ' is included when searching for 's' case + /// // insensitively due to Unicode's simple case folding rules. But + /// // when Unicode mode is disabled, only ASCII case insensitive rules + /// // are used. + /// assert!(!re.is_match("ſ".as_bytes())); + /// ``` + /// + /// Since this builder is for constructing a + /// [`bytes::RegexSet`](RegexSet), one can disable Unicode mode even if + /// it would match invalid UTF-8: + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"."]) + /// .unicode(false) + /// .build() + /// .unwrap(); + /// // Normally greek letters would be included in \w, but since + /// // Unicode mode is disabled, it only matches ASCII letters. + /// assert!(re.is_match(b"\xFF")); + /// ``` + pub fn unicode(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.unicode(yes); + self + } + + /// This configures whether to enable case insensitive matching for all + /// of the patterns. + /// + /// This setting can also be configured using the inline flag `i` + /// in the pattern. For example, `(?i:foo)` matches `foo` case + /// insensitively while `(?-i:foo)` matches `foo` case sensitively. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"foo(?-i:bar)quux"]) + /// .case_insensitive(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match(b"FoObarQuUx")); + /// // Even though case insensitive matching is enabled in the builder, + /// // it can be locally disabled within the pattern. In this case, + /// // `bar` is matched case sensitively. + /// assert!(!re.is_match(b"fooBARquux")); + /// ``` + pub fn case_insensitive(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.case_insensitive(yes); + self + } + + /// This configures multi-line mode for all of the patterns. + /// + /// Enabling multi-line mode changes the behavior of the `^` and `$` + /// anchor assertions. Instead of only matching at the beginning and + /// end of a haystack, respectively, multi-line mode causes them to + /// match at the beginning and end of a line *in addition* to the + /// beginning and end of a haystack. More precisely, `^` will match at + /// the position immediately following a `\n` and `$` will match at the + /// position immediately preceding a `\n`. + /// + /// The behavior of this option can be impacted by other settings too: + /// + /// * The [`RegexSetBuilder::line_terminator`] option changes `\n` + /// above to any ASCII byte. + /// * The [`RegexSetBuilder::crlf`] option changes the line terminator + /// to be either `\r` or `\n`, but never at the position between a `\r` + /// and `\n`. + /// + /// This setting can also be configured using the inline flag `m` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"^foo$"]) + /// .multi_line(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match(b"\nfoo\n")); + /// ``` + pub fn multi_line(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.multi_line(yes); + self + } + + /// This configures dot-matches-new-line mode for the entire pattern. + /// + /// Perhaps surprisingly, the default behavior for `.` is not to match + /// any character, but rather, to match any character except for the + /// line terminator (which is `\n` by default). When this mode is + /// enabled, the behavior changes such that `.` truly matches any + /// character. + /// + /// This setting can also be configured using the inline flag `s` in + /// the pattern. For example, `(?s:.)` and `\p{any}` are equivalent + /// regexes. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"foo.bar"]) + /// .dot_matches_new_line(true) + /// .build() + /// .unwrap(); + /// let hay = b"foo\nbar"; + /// assert!(re.is_match(hay)); + /// ``` + pub fn dot_matches_new_line( + &mut self, + yes: bool, + ) -> &mut RegexSetBuilder { + self.builder.dot_matches_new_line(yes); + self + } + + /// This configures CRLF mode for all of the patterns. + /// + /// When CRLF mode is enabled, both `\r` ("carriage return" or CR for + /// short) and `\n` ("line feed" or LF for short) are treated as line + /// terminators. This results in the following: + /// + /// * Unless dot-matches-new-line mode is enabled, `.` will now match + /// any character except for `\n` and `\r`. + /// * When multi-line mode is enabled, `^` will match immediately + /// following a `\n` or a `\r`. Similarly, `$` will match immediately + /// preceding a `\n` or a `\r`. Neither `^` nor `$` will ever match + /// between `\r` and `\n`. + /// + /// This setting can also be configured using the inline flag `R` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"^foo$"]) + /// .multi_line(true) + /// .crlf(true) + /// .build() + /// .unwrap(); + /// let hay = b"\r\nfoo\r\n"; + /// // If CRLF mode weren't enabled here, then '$' wouldn't match + /// // immediately after 'foo', and thus no match would be found. + /// assert!(re.is_match(hay)); + /// ``` + /// + /// This example demonstrates that `^` will never match at a position + /// between `\r` and `\n`. (`$` will similarly not match between a `\r` + /// and a `\n`.) + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"^\n"]) + /// .multi_line(true) + /// .crlf(true) + /// .build() + /// .unwrap(); + /// assert!(!re.is_match(b"\r\n")); + /// ``` + pub fn crlf(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.crlf(yes); + self + } + + /// Configures the line terminator to be used by the regex. + /// + /// The line terminator is relevant in two ways for a particular regex: + /// + /// * When dot-matches-new-line mode is *not* enabled (the default), + /// then `.` will match any character except for the configured line + /// terminator. + /// * When multi-line mode is enabled (not the default), then `^` and + /// `$` will match immediately after and before, respectively, a line + /// terminator. + /// + /// In both cases, if CRLF mode is enabled in a particular context, + /// then it takes precedence over any configured line terminator. + /// + /// This option cannot be configured from within the pattern. + /// + /// The default line terminator is `\n`. + /// + /// # Example + /// + /// This shows how to treat the NUL byte as a line terminator. This can + /// be a useful heuristic when searching binary data. + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"^foo$"]) + /// .multi_line(true) + /// .line_terminator(b'\x00') + /// .build() + /// .unwrap(); + /// let hay = b"\x00foo\x00"; + /// assert!(re.is_match(hay)); + /// ``` + /// + /// This example shows that the behavior of `.` is impacted by this + /// setting as well: + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let re = RegexSetBuilder::new([r"."]) + /// .line_terminator(b'\x00') + /// .build() + /// .unwrap(); + /// assert!(re.is_match(b"\n")); + /// assert!(!re.is_match(b"\x00")); + /// ``` + /// + /// This shows that building a regex will work even when the byte given + /// is not ASCII. This is unlike the top-level `RegexSet` API where + /// matching invalid UTF-8 is not allowed. + /// + /// Note though that you must disable Unicode mode. This is required + /// because Unicode mode requires matching one codepoint at a time, + /// and there is no way to match a non-ASCII byte as if it were a + /// codepoint. + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// assert!( + /// RegexSetBuilder::new([r"."]) + /// .unicode(false) + /// .line_terminator(0x80) + /// .build() + /// .is_ok(), + /// ); + /// ``` + pub fn line_terminator(&mut self, byte: u8) -> &mut RegexSetBuilder { + self.builder.line_terminator(byte); + self + } + + /// This configures swap-greed mode for all of the patterns. + /// + /// When swap-greed mode is enabled, patterns like `a+` will become + /// non-greedy and patterns like `a+?` will become greedy. In other + /// words, the meanings of `a+` and `a+?` are switched. + /// + /// This setting can also be configured using the inline flag `U` in + /// the pattern. + /// + /// Note that this is generally not useful for a `RegexSet` since a + /// `RegexSet` can only report whether a pattern matches or not. Since + /// greediness never impacts whether a match is found or not (only the + /// offsets of the match), it follows that whether parts of a pattern + /// are greedy or not doesn't matter for a `RegexSet`. + /// + /// The default for this is `false`. + pub fn swap_greed(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.swap_greed(yes); + self + } + + /// This configures verbose mode for all of the patterns. + /// + /// When enabled, whitespace will treated as insignificant in the + /// pattern and `#` can be used to start a comment until the next new + /// line. + /// + /// Normally, in most places in a pattern, whitespace is treated + /// literally. For example ` +` will match one or more ASCII whitespace + /// characters. + /// + /// When verbose mode is enabled, `\#` can be used to match a literal + /// `#` and `\ ` can be used to match a literal ASCII whitespace + /// character. + /// + /// Verbose mode is useful for permitting regexes to be formatted and + /// broken up more nicely. This may make them more easily readable. + /// + /// This setting can also be configured using the inline flag `x` in + /// the pattern. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// let pat = r" + /// \b + /// (?\p{Uppercase}\w*) # always start with uppercase letter + /// [\s--\n]+ # whitespace should separate names + /// (?: # middle name can be an initial! + /// (?:(?\p{Uppercase})\.|(?\p{Uppercase}\w*)) + /// [\s--\n]+ + /// )? + /// (?\p{Uppercase}\w*) + /// \b + /// "; + /// let re = RegexSetBuilder::new([pat]) + /// .ignore_whitespace(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match(b"Harry Potter")); + /// assert!(re.is_match(b"Harry J. Potter")); + /// assert!(re.is_match(b"Harry James Potter")); + /// assert!(!re.is_match(b"harry J. Potter")); + /// ``` + pub fn ignore_whitespace( + &mut self, + yes: bool, + ) -> &mut RegexSetBuilder { + self.builder.ignore_whitespace(yes); + self + } + + /// This configures octal mode for all of the patterns. + /// + /// Octal syntax is a little-known way of uttering Unicode codepoints + /// in a pattern. For example, `a`, `\x61`, `\u0061` and `\141` are all + /// equivalent patterns, where the last example shows octal syntax. + /// + /// While supporting octal syntax isn't in and of itself a problem, + /// it does make good error messages harder. That is, in PCRE based + /// regex engines, syntax like `\1` invokes a backreference, which is + /// explicitly unsupported this library. However, many users expect + /// backreferences to be supported. Therefore, when octal support + /// is disabled, the error message will explicitly mention that + /// backreferences aren't supported. + /// + /// The default for this is `false`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// // Normally this pattern would not compile, with an error message + /// // about backreferences not being supported. But with octal mode + /// // enabled, octal escape sequences work. + /// let re = RegexSetBuilder::new([r"\141"]) + /// .octal(true) + /// .build() + /// .unwrap(); + /// assert!(re.is_match(b"a")); + /// ``` + pub fn octal(&mut self, yes: bool) -> &mut RegexSetBuilder { + self.builder.octal(yes); + self + } + + /// Sets the approximate size limit, in bytes, of the compiled regex. + /// + /// This roughly corresponds to the number of heap memory, in + /// bytes, occupied by a single regex. If the regex would otherwise + /// approximately exceed this limit, then compiling that regex will + /// fail. + /// + /// The main utility of a method like this is to avoid compiling + /// regexes that use an unexpected amount of resources, such as + /// time and memory. Even if the memory usage of a large regex is + /// acceptable, its search time may not be. Namely, worst case time + /// complexity for search is `O(m * n)`, where `m ~ len(pattern)` and + /// `n ~ len(haystack)`. That is, search time depends, in part, on the + /// size of the compiled regex. This means that putting a limit on the + /// size of the regex limits how much a regex can impact search time. + /// + /// For more information about regex size limits, see the section on + /// [untrusted inputs](crate#untrusted-input) in the top-level crate + /// documentation. + /// + /// The default for this is some reasonable number that permits most + /// patterns to compile successfully. + /// + /// # Example + /// + /// ``` + /// # if !cfg!(target_pointer_width = "64") { return; } // see #1041 + /// use regex::bytes::RegexSetBuilder; + /// + /// // It may surprise you how big some seemingly small patterns can + /// // be! Since \w is Unicode aware, this generates a regex that can + /// // match approximately 140,000 distinct codepoints. + /// assert!( + /// RegexSetBuilder::new([r"\w"]) + /// .size_limit(45_000) + /// .build() + /// .is_err() + /// ); + /// ``` + pub fn size_limit(&mut self, bytes: usize) -> &mut RegexSetBuilder { + self.builder.size_limit(bytes); + self + } + + /// Set the approximate capacity, in bytes, of the cache of transitions + /// used by the lazy DFA. + /// + /// While the lazy DFA isn't always used, in tends to be the most + /// commonly use regex engine in default configurations. It tends to + /// adopt the performance profile of a fully build DFA, but without the + /// downside of taking worst case exponential time to build. + /// + /// The downside is that it needs to keep a cache of transitions and + /// states that are built while running a search, and this cache + /// can fill up. When it fills up, the cache will reset itself. Any + /// previously generated states and transitions will then need to be + /// re-generated. If this happens too many times, then this library + /// will bail out of using the lazy DFA and switch to a different regex + /// engine. + /// + /// If your regex provokes this particular downside of the lazy DFA, + /// then it may be beneficial to increase its cache capacity. This will + /// potentially reduce the frequency of cache resetting (ideally to + /// `0`). While it won't fix all potential performance problems with + /// the lazy DFA, increasing the cache capacity does fix some. + /// + /// There is no easy way to determine, a priori, whether increasing + /// this cache capacity will help. In general, the larger your regex, + /// the more cache it's likely to use. But that isn't an ironclad rule. + /// For example, a regex like `[01]*1[01]{N}` would normally produce a + /// fully build DFA that is exponential in size with respect to `N`. + /// The lazy DFA will prevent exponential space blow-up, but it cache + /// is likely to fill up, even when it's large and even for smallish + /// values of `N`. + /// + /// If you aren't sure whether this helps or not, it is sensible to + /// set this to some arbitrarily large number in testing, such as + /// `usize::MAX`. Namely, this represents the amount of capacity that + /// *may* be used. It's probably not a good idea to use `usize::MAX` in + /// production though, since it implies there are no controls on heap + /// memory used by this library during a search. In effect, set it to + /// whatever you're willing to allocate for a single regex search. + pub fn dfa_size_limit( + &mut self, + bytes: usize, + ) -> &mut RegexSetBuilder { + self.builder.dfa_size_limit(bytes); + self + } + + /// Set the nesting limit for this parser. + /// + /// The nesting limit controls how deep the abstract syntax tree is + /// allowed to be. If the AST exceeds the given limit (e.g., with too + /// many nested groups), then an error is returned by the parser. + /// + /// The purpose of this limit is to act as a heuristic to prevent stack + /// overflow for consumers that do structural induction on an AST using + /// explicit recursion. While this crate never does this (instead using + /// constant stack space and moving the call stack to the heap), other + /// crates may. + /// + /// This limit is not checked until the entire AST is parsed. + /// Therefore, if callers want to put a limit on the amount of heap + /// space used, then they should impose a limit on the length, in + /// bytes, of the concrete pattern string. In particular, this is + /// viable since this parser implementation will limit itself to heap + /// space proportional to the length of the pattern string. See also + /// the [untrusted inputs](crate#untrusted-input) section in the + /// top-level crate documentation for more information about this. + /// + /// Note that a nest limit of `0` will return a nest limit error for + /// most patterns but not all. For example, a nest limit of `0` permits + /// `a` but not `ab`, since `ab` requires an explicit concatenation, + /// which results in a nest depth of `1`. In general, a nest limit is + /// not something that manifests in an obvious way in the concrete + /// syntax, therefore, it should not be used in a granular way. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSetBuilder; + /// + /// assert!(RegexSetBuilder::new([r"a"]).nest_limit(0).build().is_ok()); + /// assert!(RegexSetBuilder::new([r"ab"]).nest_limit(0).build().is_err()); + /// ``` + pub fn nest_limit(&mut self, limit: u32) -> &mut RegexSetBuilder { + self.builder.nest_limit(limit); + self + } + } +} diff --git a/third_party/crates/regex-1.13.1/src/bytes.rs b/third_party/crates/regex-1.13.1/src/bytes.rs new file mode 100644 index 0000000..75e8d81 --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/bytes.rs @@ -0,0 +1,95 @@ +/*! +Search for regex matches in `&[u8]` haystacks. + +This module provides a nearly identical API via [`Regex`] to the one found in +the top-level of this crate. There are two important differences: + +1. Matching is done on `&[u8]` instead of `&str`. Additionally, `Vec` +is used where `String` would have been used in the top-level API. +2. Unicode support can be disabled even when disabling it would result in +matching invalid UTF-8 bytes. + +# Example: match null terminated string + +This shows how to find all null-terminated strings in a slice of bytes. This +works even if a C string contains invalid UTF-8. + +```rust +use regex::bytes::Regex; + +let re = Regex::new(r"(?-u)(?[^\x00]+)\x00").unwrap(); +let hay = b"foo\x00qu\xFFux\x00baz\x00"; + +// Extract all of the strings without the NUL terminator from each match. +// The unwrap is OK here since a match requires the `cstr` capture to match. +let cstrs: Vec<&[u8]> = + re.captures_iter(hay) + .map(|c| c.name("cstr").unwrap().as_bytes()) + .collect(); +assert_eq!(cstrs, vec![&b"foo"[..], &b"qu\xFFux"[..], &b"baz"[..]]); +``` + +# Example: selectively enable Unicode support + +This shows how to match an arbitrary byte pattern followed by a UTF-8 encoded +string (e.g., to extract a title from a Matroska file): + +```rust +use regex::bytes::Regex; + +let re = Regex::new( + r"(?-u)\x7b\xa9(?:[\x80-\xfe]|[\x40-\xff].)(?u:(.*))" +).unwrap(); +let hay = b"\x12\xd0\x3b\x5f\x7b\xa9\x85\xe2\x98\x83\x80\x98\x54\x76\x68\x65"; + +// Notice that despite the `.*` at the end, it will only match valid UTF-8 +// because Unicode mode was enabled with the `u` flag. Without the `u` flag, +// the `.*` would match the rest of the bytes regardless of whether they were +// valid UTF-8. +let (_, [title]) = re.captures(hay).unwrap().extract(); +assert_eq!(title, b"\xE2\x98\x83"); +// We can UTF-8 decode the title now. And the unwrap here +// is correct because the existence of a match guarantees +// that `title` is valid UTF-8. +let title = std::str::from_utf8(title).unwrap(); +assert_eq!(title, "☃"); +``` + +In general, if the Unicode flag is enabled in a capture group and that capture +is part of the overall match, then the capture is *guaranteed* to be valid +UTF-8. + +# Syntax + +The supported syntax is pretty much the same as the syntax for Unicode +regular expressions with a few changes that make sense for matching arbitrary +bytes: + +1. The `u` flag can be disabled even when disabling it might cause the regex to +match invalid UTF-8. When the `u` flag is disabled, the regex is said to be in +"ASCII compatible" mode. +2. In ASCII compatible mode, Unicode character classes are not allowed. Literal +Unicode scalar values outside of character classes are allowed. +3. In ASCII compatible mode, Perl character classes (`\w`, `\d` and `\s`) +revert to their typical ASCII definition. `\w` maps to `[[:word:]]`, `\d` maps +to `[[:digit:]]` and `\s` maps to `[[:space:]]`. +4. In ASCII compatible mode, word boundaries use the ASCII compatible `\w` to +determine whether a byte is a word byte or not. +5. Hexadecimal notation can be used to specify arbitrary bytes instead of +Unicode codepoints. For example, in ASCII compatible mode, `\xFF` matches the +literal byte `\xFF`, while in Unicode mode, `\xFF` is the Unicode codepoint +`U+00FF` that matches its UTF-8 encoding of `\xC3\xBF`. Similarly for octal +notation when enabled. +6. In ASCII compatible mode, `.` matches any *byte* except for `\n`. When the +`s` flag is additionally enabled, `.` matches any byte. + +# Performance + +In general, one should expect performance on `&[u8]` to be roughly similar to +performance on `&str`. +*/ +pub use crate::{builders::bytes::*, regex::bytes::*, regexset::bytes::*}; + +// Re-export the public but hidden macro to make it usable as `bytes::regex`. +#[doc(inline)] +pub use crate::__bytes_regex as regex; diff --git a/third_party/crates/regex-1.13.1/src/error.rs b/third_party/crates/regex-1.13.1/src/error.rs new file mode 100644 index 0000000..9e90d56 --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/error.rs @@ -0,0 +1,101 @@ +use alloc::string::{String, ToString}; + +use regex_automata::meta; + +/// An error that occurred during parsing or compiling a regular expression. +#[non_exhaustive] +#[derive(Clone, PartialEq)] +pub enum Error { + /// A syntax error. + Syntax(String), + /// The compiled program exceeded the set size + /// limit. The argument is the size limit imposed by + /// [`RegexBuilder::size_limit`](crate::RegexBuilder::size_limit). Even + /// when not configured explicitly, it defaults to a reasonable limit. + /// + /// If you're getting this error, it occurred because your regex has been + /// compiled to an intermediate state that is too big. It is important to + /// note that exceeding this limit does _not_ mean the regex is too big to + /// _work_, but rather, the regex is big enough that it may wind up being + /// surprisingly slow when used in a search. In other words, this error is + /// meant to be a practical heuristic for avoiding a performance footgun, + /// and especially so for the case where the regex pattern is coming from + /// an untrusted source. + /// + /// There are generally two ways to move forward if you hit this error. + /// The first is to find some way to use a smaller regex. The second is to + /// increase the size limit via `RegexBuilder::size_limit`. However, if + /// your regex pattern is not from a trusted source, then neither of these + /// approaches may be appropriate. Instead, you'll have to determine just + /// how big of a regex you want to allow. + CompiledTooBig(usize), +} + +impl Error { + pub(crate) fn from_meta_build_error(err: meta::BuildError) -> Error { + if let Some(size_limit) = err.size_limit() { + Error::CompiledTooBig(size_limit) + } else if let Some(ref err) = err.syntax_error() { + Error::Syntax(err.to_string()) + } else { + // This is a little suspect. Technically there are more ways for + // a meta regex to fail to build other than "exceeded size limit" + // and "syntax error." For example, if there are too many states + // or even too many patterns. But in practice this is probably + // good enough. The worst thing that happens is that Error::Syntax + // represents an error that isn't technically a syntax error, but + // the actual message will still be shown. So... it's not too bad. + // + // We really should have made the Error type in the regex crate + // completely opaque. Rookie mistake. + Error::Syntax(err.to_string()) + } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for Error { + // TODO: Remove this method entirely on the next breaking semver release. + #[allow(deprecated)] + fn description(&self) -> &str { + match *self { + Error::Syntax(ref err) => err, + Error::CompiledTooBig(_) => "compiled program too big", + } + } +} + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match *self { + Error::Syntax(ref err) => err.fmt(f), + Error::CompiledTooBig(limit) => write!( + f, + "Compiled regex exceeds size limit of {limit} bytes.", + ), + } + } +} + +// We implement our own Debug implementation so that we show nicer syntax +// errors when people use `Regex::new(...).unwrap()`. It's a little weird, +// but the `Syntax` variant is already storing a `String` anyway, so we might +// as well format it nicely. +impl core::fmt::Debug for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match *self { + Error::Syntax(ref err) => { + let hr: String = core::iter::repeat('~').take(79).collect(); + writeln!(f, "Syntax(")?; + writeln!(f, "{hr}")?; + writeln!(f, "{err}")?; + writeln!(f, "{hr}")?; + write!(f, ")")?; + Ok(()) + } + Error::CompiledTooBig(limit) => { + f.debug_tuple("CompiledTooBig").field(&limit).finish() + } + } + } +} diff --git a/third_party/crates/regex-1.13.1/src/find_byte.rs b/third_party/crates/regex-1.13.1/src/find_byte.rs new file mode 100644 index 0000000..9c6915d --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/find_byte.rs @@ -0,0 +1,17 @@ +/// Searches for the given needle in the given haystack. +/// +/// If the perf-literal feature is enabled, then this uses the super optimized +/// memchr crate. Otherwise, it uses the naive byte-at-a-time implementation. +pub(crate) fn find_byte(needle: u8, haystack: &[u8]) -> Option { + #[cfg(not(feature = "perf-literal"))] + fn imp(needle: u8, haystack: &[u8]) -> Option { + haystack.iter().position(|&b| b == needle) + } + + #[cfg(feature = "perf-literal")] + fn imp(needle: u8, haystack: &[u8]) -> Option { + memchr::memchr(needle, haystack) + } + + imp(needle, haystack) +} diff --git a/third_party/crates/regex-1.13.1/src/lib.rs b/third_party/crates/regex-1.13.1/src/lib.rs new file mode 100644 index 0000000..3c5acc3 --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/lib.rs @@ -0,0 +1,1386 @@ +/*! +This crate provides routines for searching strings for matches of a [regular +expression] (aka "regex"). The regex syntax supported by this crate is similar +to other regex engines, but it lacks several features that are not known how to +implement efficiently. This includes, but is not limited to, look-around and +backreferences. In exchange, all regex searches in this crate have worst case +`O(m * n)` time complexity, where `m` is proportional to the size of the regex +and `n` is proportional to the size of the string being searched. + +[regular expression]: https://en.wikipedia.org/wiki/Regular_expression + +If you just want API documentation, then skip to the [`Regex`] type. Otherwise, +here's a quick example showing one way of parsing the output of a grep-like +program: + +```rust +use regex::Regex; + +let re = Regex::new(r"(?m)^([^:]+):([0-9]+):(.+)$").unwrap(); +let hay = "\ +path/to/foo:54:Blue Harvest +path/to/bar:90:Something, Something, Something, Dark Side +path/to/baz:3:It's a Trap! +"; + +let mut results = vec![]; +for (_, [path, lineno, line]) in re.captures_iter(hay).map(|c| c.extract()) { + results.push((path, lineno.parse::()?, line)); +} +assert_eq!(results, vec![ + ("path/to/foo", 54, "Blue Harvest"), + ("path/to/bar", 90, "Something, Something, Something, Dark Side"), + ("path/to/baz", 3, "It's a Trap!"), +]); +# Ok::<(), Box>(()) +``` + +Or, make use of the [`regex!`](crate::regex) macro to compile the regex once +and re-use that same regex automatically. This is useful inside functions. +For example: + +```rust +use regex::regex; + +fn is_match(line: &str) -> bool { + // The regex will be compiled approximately once and reused automatically. + // This avoids the footgun of using `Regex::new` here, which would + // guarantee that it would be compiled every time this routine is called. + // This would likely make this routine much slower than it needs to be. + regex!(r"bar|baz").is_match(line) +} + +let hay = "\ +path/to/foo:54:Blue Harvest +path/to/bar:90:Something, Something, Something, Dark Side +path/to/baz:3:It's a Trap! +"; + +let matches = hay.lines().filter(|line| is_match(line)).count(); +assert_eq!(matches, 2); +``` + +# Overview + +The primary type in this crate is a [`Regex`]. Its most important methods are +as follows: + +* [`Regex::new`] compiles a regex using the default configuration. A +[`RegexBuilder`] permits setting a non-default configuration. (For example, +case insensitive matching, verbose mode and others.) +* [`Regex::is_match`] reports whether a match exists in a particular haystack. +* [`Regex::find`] reports the byte offsets of a match in a haystack, if one +exists. [`Regex::find_iter`] returns an iterator over all such matches. +* [`Regex::captures`] returns a [`Captures`], which reports both the byte +offsets of a match in a haystack and the byte offsets of each matching capture +group from the regex in the haystack. +[`Regex::captures_iter`] returns an iterator over all such matches. + +There is also a [`RegexSet`], which permits searching for multiple regex +patterns simultaneously in a single search. However, it currently only reports +which patterns match and *not* the byte offsets of a match. + +Otherwise, this top-level crate documentation is organized as follows: + +* [Usage](#usage) shows how to add the `regex` crate to your Rust project. +* [Examples](#examples) provides a limited selection of regex search examples. +* [Performance](#performance) provides a brief summary of how to optimize regex +searching speed. +* [Unicode](#unicode) discusses support for non-ASCII patterns. +* [Syntax](#syntax) enumerates the specific regex syntax supported by this +crate. +* [Untrusted input](#untrusted-input) discusses how this crate deals with regex +patterns or haystacks that are untrusted. +* [Crate features](#crate-features) documents the Cargo features that can be +enabled or disabled for this crate. +* [Other crates](#other-crates) links to other crates in the `regex` family. + +# Usage + +The `regex` crate is [on crates.io](https://crates.io/crates/regex) and can be +used by adding `regex` to your dependencies in your project's `Cargo.toml`. +Or more simply, just run `cargo add regex`. + +Here is a complete example that creates a new Rust project, adds a dependency +on `regex`, creates the source code for a regex search and then runs the +program. + +First, create the project in a new directory: + +```text +$ mkdir regex-example +$ cd regex-example +$ cargo init +``` + +Second, add a dependency on `regex`: + +```text +$ cargo add regex +``` + +Third, edit `src/main.rs`. Delete what's there and replace it with this: + +``` +use regex::Regex; + +fn main() { + let re = Regex::new(r"Hello (?\w+)!").unwrap(); + let Some(caps) = re.captures("Hello Murphy!") else { + println!("no match!"); + return; + }; + println!("The name is: {}", &caps["name"]); +} +``` + +Fourth, run it with `cargo run`: + +```text +$ cargo run + Compiling memchr v2.5.0 + Compiling regex-syntax v0.7.1 + Compiling aho-corasick v1.0.1 + Compiling regex v1.8.1 + Compiling regex-example v0.1.0 (/tmp/regex-example) + Finished dev [unoptimized + debuginfo] target(s) in 4.22s + Running `target/debug/regex-example` +The name is: Murphy +``` + +The first time you run the program will show more output like above. But +subsequent runs shouldn't have to re-compile the dependencies. + +# Examples + +This section provides a few examples, in tutorial style, showing how to +search a haystack with a regex. There are more examples throughout the API +documentation. + +Before starting though, it's worth defining a few terms: + +* A **regex** is a Rust value whose type is `Regex`. We use `re` as a +variable name for a regex. +* A **pattern** is the string that is used to build a regex. We use `pat` as +a variable name for a pattern. +* A **haystack** is the string that is searched by a regex. We use `hay` as a +variable name for a haystack. + +Sometimes the words "regex" and "pattern" are used interchangeably. + +General use of regular expressions in this crate proceeds by compiling a +**pattern** into a **regex**, and then using that regex to search, split or +replace parts of a **haystack**. + +### Example: find a middle initial + +We'll start off with a very simple example: a regex that looks for a specific +name but uses a wildcard to match a middle initial. Our pattern serves as +something like a template that will match a particular name with *any* middle +initial. + +```rust +use regex::Regex; + +// We use 'unwrap()' here because it would be a bug in our program if the +// pattern failed to compile to a regex. Panicking in the presence of a bug +// is okay. +let re = Regex::new(r"Homer (.)\. Simpson").unwrap(); +let hay = "Homer J. Simpson"; +let Some(caps) = re.captures(hay) else { return }; +assert_eq!("J", &caps[1]); +``` + +There are a few things worth noticing here in our first example: + +* The `.` is a special pattern meta character that means "match any single +character except for new lines." (More precisely, in this crate, it means +"match any UTF-8 encoding of any Unicode scalar value other than `\n`.") +* We can match an actual `.` literally by escaping it, i.e., `\.`. +* We use Rust's [raw strings] to avoid needing to deal with escape sequences in +both the regex pattern syntax and in Rust's string literal syntax. If we didn't +use raw strings here, we would have had to use `\\.` to match a literal `.` +character. That is, `r"\."` and `"\\."` are equivalent patterns. +* We put our wildcard `.` instruction in parentheses. These parentheses have a +special meaning that says, "make whatever part of the haystack matches within +these parentheses available as a capturing group." After finding a match, we +access this capture group with `&caps[1]`. + +[raw strings]: https://doc.rust-lang.org/stable/reference/tokens.html#raw-string-literals + +Otherwise, we execute a search using `re.captures(hay)` and return from our +function if no match occurred. We then reference the middle initial by asking +for the part of the haystack that matched the capture group indexed at `1`. +(The capture group at index 0 is implicit and always corresponds to the entire +match. In this case, that's `Homer J. Simpson`.) + +### Example: named capture groups + +Continuing from our middle initial example above, we can tweak the pattern +slightly to give a name to the group that matches the middle initial: + +```rust +use regex::Regex; + +// Note that (?P.) is a different way to spell the same thing. +let re = Regex::new(r"Homer (?.)\. Simpson").unwrap(); +let hay = "Homer J. Simpson"; +let Some(caps) = re.captures(hay) else { return }; +assert_eq!("J", &caps["middle"]); +``` + +Giving a name to a group can be useful when there are multiple groups in +a pattern. It makes the code referring to those groups a bit easier to +understand. + +### Example: validating a particular date format + +This examples shows how to confirm whether a haystack, in its entirety, matches +a particular date format: + +```rust +use regex::Regex; + +let re = Regex::new(r"^\d{4}-\d{2}-\d{2}$").unwrap(); +assert!(re.is_match("2010-03-14")); +``` + +Notice the use of the `^` and `$` anchors. In this crate, every regex search is +run with an implicit `(?s:.)*?` at the beginning of its pattern, which allows +the regex to match anywhere in a haystack. Anchors, as above, can be used to +ensure that the full haystack matches a pattern. + +This crate is also Unicode aware by default, which means that `\d` might match +more than you might expect it to. For example: + +```rust +use regex::Regex; + +let re = Regex::new(r"^\d{4}-\d{2}-\d{2}$").unwrap(); +assert!(re.is_match("𝟚𝟘𝟙𝟘-𝟘𝟛-𝟙𝟜")); +``` + +To only match an ASCII decimal digit, all of the following are equivalent: + +* `[0-9]` +* `(?-u:\d)` +* `[[:digit:]]` +* `[\d&&\p{ascii}]` + +### Example: finding dates in a haystack + +In the previous example, we showed how one might validate that a haystack, +in its entirety, corresponded to a particular date format. But what if we wanted +to extract all things that look like dates in a specific format from a haystack? +To do this, we can use an iterator API to find all matches (notice that we've +removed the anchors and switched to looking for ASCII-only digits): + +```rust +use regex::Regex; + +let re = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}").unwrap(); +let hay = "What do 1865-04-14, 1881-07-02, 1901-09-06 and 1963-11-22 have in common?"; +// 'm' is a 'Match', and 'as_str()' returns the matching part of the haystack. +let dates: Vec<&str> = re.find_iter(hay).map(|m| m.as_str()).collect(); +assert_eq!(dates, vec![ + "1865-04-14", + "1881-07-02", + "1901-09-06", + "1963-11-22", +]); +``` + +We can also iterate over [`Captures`] values instead of [`Match`] values, and +that in turn permits accessing each component of the date via capturing groups: + +```rust +use regex::Regex; + +let re = Regex::new(r"(?[0-9]{4})-(?[0-9]{2})-(?[0-9]{2})").unwrap(); +let hay = "What do 1865-04-14, 1881-07-02, 1901-09-06 and 1963-11-22 have in common?"; +// 'm' is a 'Match', and 'as_str()' returns the matching part of the haystack. +let dates: Vec<(&str, &str, &str)> = re.captures_iter(hay).map(|caps| { + // The unwraps are okay because every capture group must match if the whole + // regex matches, and in this context, we know we have a match. + // + // Note that we use `caps.name("y").unwrap().as_str()` instead of + // `&caps["y"]` because the lifetime of the former is the same as the + // lifetime of `hay` above, but the lifetime of the latter is tied to the + // lifetime of `caps` due to how the `Index` trait is defined. + let year = caps.name("y").unwrap().as_str(); + let month = caps.name("m").unwrap().as_str(); + let day = caps.name("d").unwrap().as_str(); + (year, month, day) +}).collect(); +assert_eq!(dates, vec![ + ("1865", "04", "14"), + ("1881", "07", "02"), + ("1901", "09", "06"), + ("1963", "11", "22"), +]); +``` + +### Example: simpler capture group extraction + +One can use [`Captures::extract`] to make the code from the previous example a +bit simpler in this case: + +```rust +use regex::Regex; + +let re = Regex::new(r"([0-9]{4})-([0-9]{2})-([0-9]{2})").unwrap(); +let hay = "What do 1865-04-14, 1881-07-02, 1901-09-06 and 1963-11-22 have in common?"; +let dates: Vec<(&str, &str, &str)> = re.captures_iter(hay).map(|caps| { + let (_, [year, month, day]) = caps.extract(); + (year, month, day) +}).collect(); +assert_eq!(dates, vec![ + ("1865", "04", "14"), + ("1881", "07", "02"), + ("1901", "09", "06"), + ("1963", "11", "22"), +]); +``` + +`Captures::extract` works by ensuring that the number of matching groups match +the number of groups requested via the `[year, month, day]` syntax. If they do, +then the substrings for each corresponding capture group are automatically +returned in an appropriately sized array. Rust's syntax for pattern matching +arrays does the rest. + +### Example: replacement with named capture groups + +Building on the previous example, perhaps we'd like to rearrange the date +formats. This can be done by finding each match and replacing it with +something different. The [`Regex::replace_all`] routine provides a convenient +way to do this, including by supporting references to named groups in the +replacement string: + +```rust +use regex::Regex; + +let re = Regex::new(r"(?\d{4})-(?\d{2})-(?\d{2})").unwrap(); +let before = "1973-01-05, 1975-08-25 and 1980-10-18"; +let after = re.replace_all(before, "$m/$d/$y"); +assert_eq!(after, "01/05/1973, 08/25/1975 and 10/18/1980"); +``` + +The replace methods are actually polymorphic in the replacement, which +provides more flexibility than is seen here. (See the documentation for +[`Regex::replace`] for more details.) + +### Example: verbose mode + +When your regex gets complicated, you might consider using something other +than regex. But if you stick with regex, you can use the `x` flag to enable +insignificant whitespace mode or "verbose mode." In this mode, whitespace +is treated as insignificant and one may write comments. This may make your +patterns easier to comprehend. + +```rust +use regex::Regex; + +let re = Regex::new(r"(?x) + (?P\d{4}) # the year, including all Unicode digits + - + (?P\d{2}) # the month, including all Unicode digits + - + (?P\d{2}) # the day, including all Unicode digits +").unwrap(); + +let before = "1973-01-05, 1975-08-25 and 1980-10-18"; +let after = re.replace_all(before, "$m/$d/$y"); +assert_eq!(after, "01/05/1973, 08/25/1975 and 10/18/1980"); +``` + +If you wish to match against whitespace in this mode, you can still use `\s`, +`\n`, `\t`, etc. For escaping a single space character, you can escape it +directly with `\ `, use its hex character code `\x20` or temporarily disable +the `x` flag, e.g., `(?-x: )`. + +### Example: match multiple regular expressions simultaneously + +This demonstrates how to use a [`RegexSet`] to match multiple (possibly +overlapping) regexes in a single scan of a haystack: + +```rust +use regex::RegexSet; + +let set = RegexSet::new(&[ + r"\w+", + r"\d+", + r"\pL+", + r"foo", + r"bar", + r"barfoo", + r"foobar", +]).unwrap(); + +// Iterate over and collect all of the matches. Each match corresponds to the +// ID of the matching pattern. +let matches: Vec<_> = set.matches("foobar").into_iter().collect(); +assert_eq!(matches, vec![0, 2, 3, 4, 6]); + +// You can also test whether a particular regex matched: +let matches = set.matches("foobar"); +assert!(!matches.matched(5)); +assert!(matches.matched(6)); +``` + +# Performance + +This section briefly discusses a few concerns regarding the speed and resource +usage of regexes. + +### Only ask for what you need + +When running a search with a regex, there are generally three different types +of information one can ask for: + +1. Does a regex match in a haystack? +2. Where does a regex match in a haystack? +3. Where do each of the capturing groups match in a haystack? + +Generally speaking, this crate could provide a function to answer only #3, +which would subsume #1 and #2 automatically. However, it can be significantly +more expensive to compute the location of capturing group matches, so it's best +not to do it if you don't need to. + +Therefore, only ask for what you need. For example, don't use [`Regex::find`] +if you only need to test if a regex matches a haystack. Use [`Regex::is_match`] +instead. + +### Unicode can impact memory usage and search speed + +This crate has first class support for Unicode and it is **enabled by default**. +In many cases, the extra memory required to support it will be negligible and +it typically won't impact search speed. But it can in some cases. + +With respect to memory usage, the impact of Unicode principally manifests +through the use of Unicode character classes. Unicode character classes +tend to be quite large. For example, `\w` by default matches around 140,000 +distinct codepoints. This requires additional memory, and tends to slow down +regex compilation. While a `\w` here and there is unlikely to be noticed, +writing `\w{100}` will for example result in quite a large regex by default. +Indeed, `\w` is considerably larger than its ASCII-only version, so if your +requirements are satisfied by ASCII, it's probably a good idea to stick to +ASCII classes. The ASCII-only version of `\w` can be spelled in a number of +ways. All of the following are equivalent: + +* `[0-9A-Za-z_]` +* `(?-u:\w)` +* `[[:word:]]` +* `[\w&&\p{ascii}]` + +With respect to search speed, Unicode tends to be handled pretty well, even when +using large Unicode character classes. However, some of the faster internal +regex engines cannot handle a Unicode aware word boundary assertion. So if you +don't need Unicode-aware word boundary assertions, you might consider using +`(?-u:\b)` instead of `\b`, where the former uses an ASCII-only definition of +a word character. + +### Literals might accelerate searches + +This crate tends to be quite good at recognizing literals in a regex pattern +and using them to accelerate a search. If it is at all possible to include +some kind of literal in your pattern, then it might make search substantially +faster. For example, in the regex `\w+@\w+`, the engine will look for +occurrences of `@` and then try a reverse match for `\w+` to find the start +position. + +### Avoid re-compiling regexes, especially in a loop + +It is an anti-pattern to compile the same pattern in a loop since regex +compilation is typically expensive. (It takes anywhere from a few microseconds +to a few **milliseconds** depending on the size of the pattern.) Not only is +compilation itself expensive, but this also prevents optimizations that reuse +allocations internally to the regex engine. + +In Rust, it can sometimes be a pain to pass regular expressions around if +they're used from inside a helper function. Instead, we recommend using +[`std::sync::LazyLock`], or the [`once_cell`] crate, +if you can't use the standard library. + +This example shows how to use `std::sync::LazyLock`: + +```rust +use std::sync::LazyLock; + +use regex::Regex; + +fn some_helper_function(haystack: &str) -> bool { + static RE: LazyLock = LazyLock::new(|| Regex::new(r"...").unwrap()); + RE.is_match(haystack) +} + +fn main() { + assert!(some_helper_function("abc")); + assert!(!some_helper_function("ac")); +} +``` + +Specifically, in this example, the regex will be compiled when it is used for +the first time. On subsequent uses, it will reuse the previously built `Regex`. +Notice how one can define the `Regex` locally to a specific function. + +The [`regex!`] macro can also be used, which handles lazy compilation. + +[`std::sync::LazyLock`]: https://doc.rust-lang.org/std/sync/struct.LazyLock.html +[`once_cell`]: https://crates.io/crates/once_cell + +### Sharing a regex across threads can result in contention + +While a single `Regex` can be freely used from multiple threads simultaneously, +there is a small synchronization cost that must be paid. Generally speaking, +one shouldn't expect to observe this unless the principal task in each thread +is searching with the regex *and* most searches are on short haystacks. In this +case, internal contention on shared resources can spike and increase latency, +which in turn may slow down each individual search. + +One can work around this by cloning each `Regex` before sending it to another +thread. The cloned regexes will still share the same internal read-only portion +of its compiled state (it's reference counted), but each thread will get +optimized access to the mutable space that is used to run a search. In general, +there is no additional cost in memory to doing this. The only cost is the added +code complexity required to explicitly clone the regex. (If you share the same +`Regex` across multiple threads, each thread still gets its own mutable space, +but accessing that space is slower.) + +# Unicode + +This section discusses what kind of Unicode support this regex library has. +Before showing some examples, we'll summarize the relevant points: + +* This crate almost fully implements "Basic Unicode Support" (Level 1) as +specified by the [Unicode Technical Standard #18][UTS18]. The full details +of what is supported are documented in [UNICODE.md] in the root of the regex +crate repository. There is virtually no support for "Extended Unicode Support" +(Level 2) from UTS#18. +* The top-level [`Regex`] runs searches *as if* iterating over each of the +codepoints in the haystack. That is, the fundamental atom of matching is a +single codepoint. +* [`bytes::Regex`], in contrast, permits disabling Unicode mode for part of all +of your pattern in all cases. When Unicode mode is disabled, then a search is +run *as if* iterating over each byte in the haystack. That is, the fundamental +atom of matching is a single byte. (A top-level `Regex` also permits disabling +Unicode and thus matching *as if* it were one byte at a time, but only when +doing so wouldn't permit matching invalid UTF-8.) +* When Unicode mode is enabled (the default), `.` will match an entire Unicode +scalar value, even when it is encoded using multiple bytes. When Unicode mode +is disabled (e.g., `(?-u:.)`), then `.` will match a single byte in all cases. +* The character classes `\w`, `\d` and `\s` are all Unicode-aware by default. +Use `(?-u:\w)`, `(?-u:\d)` and `(?-u:\s)` to get their ASCII-only definitions. +* Similarly, `\b` and `\B` use a Unicode definition of a "word" character. +To get ASCII-only word boundaries, use `(?-u:\b)` and `(?-u:\B)`. This also +applies to the special word boundary assertions. (That is, `\b{start}`, +`\b{end}`, `\b{start-half}`, `\b{end-half}`.) +* `^` and `$` are **not** Unicode-aware in multi-line mode. Namely, they only +recognize `\n` (assuming CRLF mode is not enabled) and not any of the other +forms of line terminators defined by Unicode. +* Case insensitive searching is Unicode-aware and uses simple case folding. +* Unicode general categories, scripts and many boolean properties are available +by default via the `\p{property name}` syntax. +* In all cases, matches are reported using byte offsets. Or more precisely, +UTF-8 code unit offsets. This permits constant time indexing and slicing of the +haystack. + +[UTS18]: https://unicode.org/reports/tr18/ +[UNICODE.md]: https://github.com/rust-lang/regex/blob/master/UNICODE.md + +Patterns themselves are **only** interpreted as a sequence of Unicode scalar +values. This means you can use Unicode characters directly in your pattern: + +```rust +use regex::Regex; + +let re = Regex::new(r"(?i)Δ+").unwrap(); +let m = re.find("ΔδΔ").unwrap(); +assert_eq!((0, 6), (m.start(), m.end())); +// alternatively: +assert_eq!(0..6, m.range()); +``` + +As noted above, Unicode general categories, scripts, script extensions, ages +and a smattering of boolean properties are available as character classes. For +example, you can match a sequence of numerals, Greek or Cherokee letters: + +```rust +use regex::Regex; + +let re = Regex::new(r"[\pN\p{Greek}\p{Cherokee}]+").unwrap(); +let m = re.find("abcΔᎠβⅠᏴγδⅡxyz").unwrap(); +assert_eq!(3..23, m.range()); +``` + +While not specific to Unicode, this library also supports character class set +operations. Namely, one can nest character classes arbitrarily and perform set +operations on them. Those set operations are union (the default), intersection, +difference and symmetric difference. These set operations tend to be most +useful with Unicode character classes. For example, to match any codepoint +that is both in the `Greek` script and in the `Letter` general category: + +```rust +use regex::Regex; + +let re = Regex::new(r"[\p{Greek}&&\pL]+").unwrap(); +let subs: Vec<&str> = re.find_iter("ΔδΔ𐅌ΔδΔ").map(|m| m.as_str()).collect(); +assert_eq!(subs, vec!["ΔδΔ", "ΔδΔ"]); + +// If we just matches on Greek, then all codepoints would match! +let re = Regex::new(r"\p{Greek}+").unwrap(); +let subs: Vec<&str> = re.find_iter("ΔδΔ𐅌ΔδΔ").map(|m| m.as_str()).collect(); +assert_eq!(subs, vec!["ΔδΔ𐅌ΔδΔ"]); +``` + +### Opt out of Unicode support + +The [`bytes::Regex`] type that can be used to search `&[u8]` haystacks. By +default, haystacks are conventionally treated as UTF-8 just like it is with the +main `Regex` type. However, this behavior can be disabled by turning off the +`u` flag, even if doing so could result in matching invalid UTF-8. For example, +when the `u` flag is disabled, `.` will match any byte instead of any Unicode +scalar value. + +Disabling the `u` flag is also possible with the standard `&str`-based `Regex` +type, but it is only allowed where the UTF-8 invariant is maintained. For +example, `(?-u:\w)` is an ASCII-only `\w` character class and is legal in an +`&str`-based `Regex`, but `(?-u:\W)` will attempt to match *any byte* that +isn't in `(?-u:\w)`, which in turn includes bytes that are invalid UTF-8. +Similarly, `(?-u:\xFF)` will attempt to match the raw byte `\xFF` (instead of +`U+00FF`), which is invalid UTF-8 and therefore is illegal in `&str`-based +regexes. + +Finally, since Unicode support requires bundling large Unicode data +tables, this crate exposes knobs to disable the compilation of those +data tables, which can be useful for shrinking binary size and reducing +compilation times. For details on how to do that, see the section on [crate +features](#crate-features). + +# Syntax + +The syntax supported in this crate is documented below. + +Note that the regular expression parser and abstract syntax are exposed in +a separate crate, [`regex-syntax`](https://docs.rs/regex-syntax). + +### Matching one character + +

+.             any character except new line (includes new line with s flag)
+[0-9]         any ASCII digit
+\d            digit (\p{Nd})
+\D            not digit
+\pX           Unicode character class identified by a one-letter name
+\p{Greek}     Unicode character class (general category or script)
+\PX           Negated Unicode character class identified by a one-letter name
+\P{Greek}     negated Unicode character class (general category or script)
+
+ +### Character classes + +
+[xyz]         A character class matching either x, y or z (union).
+[^xyz]        A character class matching any character except x, y and z.
+[a-z]         A character class matching any character in range a-z.
+[[:alpha:]]   ASCII character class ([A-Za-z])
+[[:^alpha:]]  Negated ASCII character class ([^A-Za-z])
+[x[^xyz]]     Nested/grouping character class (matching any character except y and z)
+[a-y&&xyz]    Intersection (matching x or y)
+[0-9&&[^4]]   Subtraction using intersection and negation (matching 0-9 except 4)
+[0-9--4]      Direct subtraction (matching 0-9 except 4)
+[a-g~~b-h]    Symmetric difference (matching `a` and `h` only)
+[\[\]]        Escaping in character classes (matching [ or ])
+[a&&b]        An empty character class matching nothing
+
+ +Any named character class may appear inside a bracketed `[...]` character +class. For example, `[\p{Greek}[:digit:]]` matches any ASCII digit or any +codepoint in the `Greek` script. `[\p{Greek}&&\pL]` matches Greek letters. + +Precedence in character classes, from most binding to least: + +1. Ranges: `[a-cd]` == `[[a-c]d]` +2. Union: `[ab&&bc]` == `[[ab]&&[bc]]` +3. Intersection, difference, symmetric difference. All three have equivalent +precedence, and are evaluated in left-to-right order. For example, +`[\pL--\p{Greek}&&\p{Uppercase}]` == `[[\pL--\p{Greek}]&&\p{Uppercase}]`. +4. Negation: `[^a-z&&b]` == `[^[a-z&&b]]`. + +### Composites + +
+xy    concatenation (x followed by y)
+x|y   alternation (x or y, prefer x)
+
+ +This example shows how an alternation works, and what it means to prefer a +branch in the alternation over subsequent branches. + +``` +use regex::Regex; + +let haystack = "samwise"; +// If 'samwise' comes first in our alternation, then it is +// preferred as a match, even if the regex engine could +// technically detect that 'sam' led to a match earlier. +let re = Regex::new(r"samwise|sam").unwrap(); +assert_eq!("samwise", re.find(haystack).unwrap().as_str()); +// But if 'sam' comes first, then it will match instead. +// In this case, it is impossible for 'samwise' to match +// because 'sam' is a prefix of it. +let re = Regex::new(r"sam|samwise").unwrap(); +assert_eq!("sam", re.find(haystack).unwrap().as_str()); +``` + +### Repetitions + +
+x*        zero or more of x (greedy)
+x+        one or more of x (greedy)
+x?        zero or one of x (greedy)
+x*?       zero or more of x (ungreedy/lazy)
+x+?       one or more of x (ungreedy/lazy)
+x??       zero or one of x (ungreedy/lazy)
+x{n,m}    at least n x and at most m x (greedy)
+x{n,}     at least n x (greedy)
+x{n}      exactly n x
+x{n,m}?   at least n x and at most m x (ungreedy/lazy)
+x{n,}?    at least n x (ungreedy/lazy)
+x{n}?     exactly n x
+
+ +### Empty matches + +
+^               the beginning of a haystack (or start-of-line with multi-line mode)
+$               the end of a haystack (or end-of-line with multi-line mode)
+\A              only the beginning of a haystack (even with multi-line mode enabled)
+\z              only the end of a haystack (even with multi-line mode enabled)
+\b              a Unicode word boundary (\w on one side and \W, \A, or \z on other)
+\B              not a Unicode word boundary
+\b{start}, \<   a Unicode start-of-word boundary (\W|\A on the left, \w on the right)
+\b{end}, \>     a Unicode end-of-word boundary (\w on the left, \W|\z on the right))
+\b{start-half}  half of a Unicode start-of-word boundary (\W|\A on the left)
+\b{end-half}    half of a Unicode end-of-word boundary (\W|\z on the right)
+
+ +The empty regex is valid and matches the empty string. For example, the +empty regex matches `abc` at positions `0`, `1`, `2` and `3`. When using the +top-level [`Regex`] on `&str` haystacks, an empty match that splits a codepoint +is guaranteed to never be returned. However, such matches are permitted when +using a [`bytes::Regex`]. For example: + +```rust +let re = regex::Regex::new(r"").unwrap(); +let ranges: Vec<_> = re.find_iter("💩").map(|m| m.range()).collect(); +assert_eq!(ranges, vec![0..0, 4..4]); + +let re = regex::bytes::Regex::new(r"").unwrap(); +let ranges: Vec<_> = re.find_iter("💩".as_bytes()).map(|m| m.range()).collect(); +assert_eq!(ranges, vec![0..0, 1..1, 2..2, 3..3, 4..4]); +``` + +Note that an empty regex is distinct from a regex that can never match. +For example, the regex `[a&&b]` is a character class that represents the +intersection of `a` and `b`. That intersection is empty, which means the +character class is empty. Since nothing is in the empty set, `[a&&b]` matches +nothing, not even the empty string. + +### Grouping and flags + +
+(exp)          numbered capture group (indexed by opening parenthesis)
+(?P<name>exp)  named (also numbered) capture group (names must be alpha-numeric)
+(?<name>exp)   named (also numbered) capture group (names must be alpha-numeric)
+(?:exp)        non-capturing group
+(?flags)       set flags within current group
+(?flags:exp)   set flags for exp (non-capturing)
+
+ +Capture group names must be any sequence of alpha-numeric Unicode codepoints, +in addition to `.`, `_`, `[` and `]`. Names must start with either an `_` or +an alphabetic codepoint. Alphabetic codepoints correspond to the `Alphabetic` +Unicode property, while numeric codepoints correspond to the union of the +`Decimal_Number`, `Letter_Number` and `Other_Number` general categories. + +Flags are each a single character. For example, `(?x)` sets the flag `x` +and `(?-x)` clears the flag `x`. Multiple flags can be set or cleared at +the same time: `(?xy)` sets both the `x` and `y` flags and `(?x-y)` sets +the `x` flag and clears the `y` flag. + +All flags are by default disabled unless stated otherwise. They are: + +
+i     case-insensitive: letters match both upper and lower case
+m     multi-line mode: ^ and $ match begin/end of line
+s     allow . to match \n
+R     enables CRLF mode: when multi-line mode is enabled, \r\n is used
+U     swap the meaning of x* and x*?
+u     Unicode support (enabled by default)
+x     verbose mode, ignores whitespace and allow line comments (starting with `#`)
+
+ +Note that in verbose mode, whitespace is ignored everywhere, including within +character classes. To insert whitespace, use its escaped form or a hex literal. +For example, `\ ` or `\x20` for an ASCII space. + +Flags can be toggled within a pattern. Here's an example that matches +case-insensitively for the first part but case-sensitively for the second part: + +```rust +use regex::Regex; + +let re = Regex::new(r"(?i)a+(?-i)b+").unwrap(); +let m = re.find("AaAaAbbBBBb").unwrap(); +assert_eq!(m.as_str(), "AaAaAbb"); +``` + +Notice that the `a+` matches either `a` or `A`, but the `b+` only matches +`b`. + +Multi-line mode means `^` and `$` no longer match just at the beginning/end of +the input, but also at the beginning/end of lines: + +``` +use regex::Regex; + +let re = Regex::new(r"(?m)^line \d+").unwrap(); +let m = re.find("line one\nline 2\n").unwrap(); +assert_eq!(m.as_str(), "line 2"); +``` + +Note that `^` matches after new lines, even at the end of input: + +``` +use regex::Regex; + +let re = Regex::new(r"(?m)^").unwrap(); +let m = re.find_iter("test\n").last().unwrap(); +assert_eq!((m.start(), m.end()), (5, 5)); +``` + +When both CRLF mode and multi-line mode are enabled, then `^` and `$` will +match either `\r` or `\n`, but never in the middle of a `\r\n`: + +``` +use regex::Regex; + +let re = Regex::new(r"(?mR)^foo$").unwrap(); +let m = re.find("\r\nfoo\r\n").unwrap(); +assert_eq!(m.as_str(), "foo"); +``` + +Unicode mode can also be selectively disabled, although only when the result +*would not* match invalid UTF-8. One good example of this is using an ASCII +word boundary instead of a Unicode word boundary, which might make some regex +searches run faster: + +```rust +use regex::Regex; + +let re = Regex::new(r"(?-u:\b).+(?-u:\b)").unwrap(); +let m = re.find("$$abc$$").unwrap(); +assert_eq!(m.as_str(), "abc"); +``` + +### Escape sequences + +Note that this includes all possible escape sequences, even ones that are +documented elsewhere. + +
+\*              literal *, applies to all ASCII except [0-9A-Za-z<>]
+\a              bell (\x07)
+\f              form feed (\x0C)
+\t              horizontal tab
+\n              new line
+\r              carriage return
+\v              vertical tab (\x0B)
+\A              matches at the beginning of a haystack
+\z              matches at the end of a haystack
+\b              word boundary assertion
+\B              negated word boundary assertion
+\b{start}, \<   start-of-word boundary assertion
+\b{end}, \>     end-of-word boundary assertion
+\b{start-half}  half of a start-of-word boundary assertion
+\b{end-half}    half of a end-of-word boundary assertion
+\123            octal character code, up to three digits (when enabled)
+\x7F            hex character code (exactly two digits)
+\x{10FFFF}      any hex character code corresponding to a Unicode code point
+\u007F          hex character code (exactly four digits)
+\u{7F}          any hex character code corresponding to a Unicode code point
+\U0000007F      hex character code (exactly eight digits)
+\U{7F}          any hex character code corresponding to a Unicode code point
+\p{Letter}      Unicode character class
+\P{Letter}      negated Unicode character class
+\d, \s, \w      Perl character class
+\D, \S, \W      negated Perl character class
+
+ +### Perl character classes (Unicode friendly) + +These classes are based on the definitions provided in +[UTS#18](https://www.unicode.org/reports/tr18/#Compatibility_Properties): + +
+\d     digit (\p{Nd})
+\D     not digit
+\s     whitespace (\p{White_Space})
+\S     not whitespace
+\w     word character (\p{Alphabetic} + \p{M} + \d + \p{Pc} + \p{Join_Control})
+\W     not word character
+
+ +### ASCII character classes + +These classes are based on the definitions provided in +[UTS#18](https://www.unicode.org/reports/tr18/#Compatibility_Properties): + +
+[[:alnum:]]    alphanumeric ([0-9A-Za-z])
+[[:alpha:]]    alphabetic ([A-Za-z])
+[[:ascii:]]    ASCII ([\x00-\x7F])
+[[:blank:]]    blank ([\t ])
+[[:cntrl:]]    control ([\x00-\x1F\x7F])
+[[:digit:]]    digits ([0-9])
+[[:graph:]]    graphical ([!-~])
+[[:lower:]]    lower case ([a-z])
+[[:print:]]    printable ([ -~])
+[[:punct:]]    punctuation ([!-/:-@\[-`{-~])
+[[:space:]]    whitespace ([\t\n\v\f\r ])
+[[:upper:]]    upper case ([A-Z])
+[[:word:]]     word characters ([0-9A-Za-z_])
+[[:xdigit:]]   hex digit ([0-9A-Fa-f])
+
+ +# Untrusted input + +This crate is meant to be able to run regex searches on untrusted haystacks +without fear of [ReDoS]. This crate also, to a certain extent, supports +untrusted patterns. + +[ReDoS]: https://en.wikipedia.org/wiki/ReDoS + +This crate differs from most (but not all) other regex engines in that it +doesn't use unbounded backtracking to run a regex search. In those cases, +one generally cannot use untrusted patterns *or* untrusted haystacks because +it can be very difficult to know whether a particular pattern will result in +catastrophic backtracking or not. + +We'll first discuss how this crate deals with untrusted inputs and then wrap +it up with a realistic discussion about what practice really looks like. + +### Panics + +Outside of clearly documented cases, most APIs in this crate are intended to +never panic regardless of the inputs given to them. For example, `Regex::new`, +`Regex::is_match`, `Regex::find` and `Regex::captures` should never panic. That +is, it is an API promise that those APIs will never panic no matter what inputs +are given to them. With that said, regex engines are complicated beasts, and +providing a rock solid guarantee that these APIs literally never panic is +essentially equivalent to saying, "there are no bugs in this library." That is +a bold claim, and not really one that can be feasibly made with a straight +face. + +Don't get the wrong impression here. This crate is extensively tested, not just +with unit and integration tests, but also via fuzz testing. For example, this +crate is part of the [OSS-fuzz project]. Panics should be incredibly rare, but +it is possible for bugs to exist, and thus possible for a panic to occur. If +you need a rock solid guarantee against panics, then you should wrap calls into +this library with [`std::panic::catch_unwind`]. + +It's also worth pointing out that this library will *generally* panic when +other regex engines would commit undefined behavior. When undefined behavior +occurs, your program might continue as if nothing bad has happened, but it also +might mean your program is open to the worst kinds of exploits. In contrast, +the worst thing a panic can do is a denial of service. + +[OSS-fuzz project]: https://android.googlesource.com/platform/external/oss-fuzz/+/refs/tags/android-t-preview-1/projects/rust-regex/ +[`std::panic::catch_unwind`]: https://doc.rust-lang.org/std/panic/fn.catch_unwind.html + +### Untrusted patterns + +The principal way this crate deals with them is by limiting their size by +default. The size limit can be configured via [`RegexBuilder::size_limit`]. The +idea of a size limit is that compiling a pattern into a `Regex` will fail if it +becomes "too big." Namely, while *most* resources consumed by compiling a regex +are approximately proportional (albeit with some high constant factors in some +cases, such as with Unicode character classes) to the length of the pattern +itself, there is one particular exception to this: counted repetitions. Namely, +this pattern: + +```text +a{5}{5}{5}{5}{5}{5} +``` + +Is equivalent to this pattern: + +```text +a{15625} +``` + +In both of these cases, the actual pattern string is quite small, but the +resulting `Regex` value is quite large. Indeed, as the first pattern shows, +it isn't enough to locally limit the size of each repetition because they can +be stacked in a way that results in exponential growth. + +To provide a bit more context, a simplified view of regex compilation looks +like this: + +* The pattern string is parsed into a structured representation called an AST. +Counted repetitions are not expanded and Unicode character classes are not +looked up in this stage. That is, the size of the AST is proportional to the +size of the pattern with "reasonable" constant factors. In other words, one +can reasonably limit the memory used by an AST by limiting the length of the +pattern string. +* The AST is translated into an HIR. Counted repetitions are still *not* +expanded at this stage, but Unicode character classes are embedded into the +HIR. The memory usage of a HIR is still proportional to the length of the +original pattern string, but the constant factors---mostly as a result of +Unicode character classes---can be quite high. Still though, the memory used by +an HIR can be reasonably limited by limiting the length of the pattern string. +* The HIR is compiled into a [Thompson NFA]. This is the stage at which +something like `\w{5}` is rewritten to `\w\w\w\w\w`. Thus, this is the stage +at which [`RegexBuilder::size_limit`] is enforced. If the NFA exceeds the +configured size, then this stage will fail. + +[Thompson NFA]: https://en.wikipedia.org/wiki/Thompson%27s_construction + +The size limit helps avoid two different kinds of exorbitant resource usage: + +* It avoids permitting exponential memory usage based on the size of the +pattern string. +* It avoids long search times. This will be discussed in more detail in the +next section, but worst case search time *is* dependent on the size of the +regex. So keeping regexes limited to a reasonable size is also a way of keeping +search times reasonable. + +Finally, it's worth pointing out that regex compilation is guaranteed to take +worst case `O(m)` time, where `m` is proportional to the size of regex. The +size of the regex here is *after* the counted repetitions have been expanded. + +**Advice for those using untrusted regexes**: limit the pattern length to +something small and expand it as needed. Configure [`RegexBuilder::size_limit`] +to something small and then expand it as needed. + +### Untrusted haystacks + +The main way this crate guards against searches from taking a long time is by +using algorithms that guarantee a `O(m * n)` worst case time and space bound. +Namely: + +* `m` is proportional to the size of the regex, where the size of the regex +includes the expansion of all counted repetitions. (See the previous section on +untrusted patterns.) +* `n` is proportional to the length, in bytes, of the haystack. + +In other words, if you consider `m` to be a constant (for example, the regex +pattern is a literal in the source code), then the search can be said to run +in "linear time." Or equivalently, "linear time with respect to the size of the +haystack." + +But the `m` factor here is important not to ignore. If a regex is +particularly big, the search times can get quite slow. This is why, in part, +[`RegexBuilder::size_limit`] exists. + +**Advice for those searching untrusted haystacks**: As long as your regexes +are not enormous, you should expect to be able to search untrusted haystacks +without fear. If you aren't sure, you should benchmark it. Unlike backtracking +engines, if your regex is so big that it's likely to result in slow searches, +this is probably something you'll be able to observe regardless of what the +haystack is made up of. + +### Iterating over matches + +One thing that is perhaps easy to miss is that the worst case time +complexity bound of `O(m * n)` applies to methods like [`Regex::is_match`], +[`Regex::find`] and [`Regex::captures`]. It does **not** apply to +[`Regex::find_iter`] or [`Regex::captures_iter`]. Namely, since iterating over +all matches can execute many searches, and each search can scan the entire +haystack, the worst case time complexity for iterators is `O(m * n^2)`. + +One example of where this occurs is when a pattern consists of an alternation, +where an earlier branch of the alternation requires scanning the entire +haystack only to discover that there is no match. It also requires a later +branch of the alternation to have matched at the beginning of the search. For +example, consider the pattern `.*[^A-Z]|[A-Z]` and the haystack `AAAAA`. The +first search will scan to the end looking for matches of `.*[^A-Z]` even though +a finite automata engine (as in this crate) knows that `[A-Z]` has already +matched the first character of the haystack. This is due to the greedy nature +of regex searching. That first search will report a match at the first `A` only +after scanning to the end to discover that no other match exists. The next +search then begins at the second `A` and the behavior repeats. + +There is no way to avoid this. This means that if both patterns and haystacks +are untrusted and you're iterating over all matches, you're susceptible to +worst case quadratic time complexity. One possible way to mitigate this +is to drop down to the lower level `regex-automata` crate and use its +`meta::Regex` iterator APIs. There, you can configure the search to operate +in "earliest" mode by passing a `Input::new(haystack).earliest(true)` to +`meta::Regex::find_iter` (for example). By enabling this mode, you give up +the normal greedy match semantics of regex searches and instead ask the regex +engine to immediately stop as soon as a match has been found. Enabling this +mode will thus restore the worst case `O(m * n)` time complexity bound, but at +the cost of different semantics. + +### Untrusted inputs in practice + +While providing a `O(m * n)` worst case time bound on all searches goes a long +way toward preventing [ReDoS], that doesn't mean every search you can possibly +run will complete without burning CPU time. In general, there are a few ways +for the `m * n` time bound to still bite you: + +* You are searching an exceptionally long haystack. No matter how you slice +it, a longer haystack will take more time to search. This crate may often make +very quick work of even long haystacks because of its literal optimizations, +but those aren't available for all regexes. +* Unicode character classes can cause searches to be quite slow in some cases. +This is especially true when they are combined with counted repetitions. While +the regex size limit above will protect you from the most egregious cases, +the default size limit still permits pretty big regexes that can execute more +slowly than one might expect. +* While routines like [`Regex::find`] and [`Regex::captures`] guarantee +worst case `O(m * n)` search time, routines like [`Regex::find_iter`] and +[`Regex::captures_iter`] actually have worst case `O(m * n^2)` search time. +This is because `find_iter` runs many searches, and each search takes worst +case `O(m * n)` time. Thus, iteration of all matches in a haystack has +worst case `O(m * n^2)`. A good example of a pattern that exhibits this is +`(?:A+){1000}|` or even `.*[^A-Z]|[A-Z]`. + +In general, untrusted haystacks are easier to stomach than untrusted patterns. +Untrusted patterns give a lot more control to the caller to impact the +performance of a search. In many cases, a regex search will actually execute in +average case `O(n)` time (i.e., not dependent on the size of the regex), but +this can't be guaranteed in general. Therefore, permitting untrusted patterns +means that your only line of defense is to put a limit on how big `m` (and +perhaps also `n`) can be in `O(m * n)`. `n` is limited by simply inspecting +the length of the haystack while `m` is limited by *both* applying a limit to +the length of the pattern *and* a limit on the compiled size of the regex via +[`RegexBuilder::size_limit`]. + +It bears repeating: if you're accepting untrusted patterns, it would be a good +idea to start with conservative limits on `m` and `n`, and then carefully +increase them as needed. + +# Crate features + +By default, this crate tries pretty hard to make regex matching both as fast +as possible and as correct as it can be. This means that there is a lot of +code dedicated to performance, the handling of Unicode data and the Unicode +data itself. Overall, this leads to more dependencies, larger binaries and +longer compile times. This trade off may not be appropriate in all cases, and +indeed, even when all Unicode and performance features are disabled, one is +still left with a perfectly serviceable regex engine that will work well in +many cases. (Note that code is not arbitrarily reducible, and for this reason, +the [`regex-lite`](https://docs.rs/regex-lite) crate exists to provide an even +more minimal experience by cutting out Unicode and performance, but still +maintaining the linear search time bound.) + +This crate exposes a number of features for controlling that trade off. Some +of these features are strictly performance oriented, such that disabling them +won't result in a loss of functionality, but may result in worse performance. +Other features, such as the ones controlling the presence or absence of Unicode +data, can result in a loss of functionality. For example, if one disables the +`unicode-case` feature (described below), then compiling the regex `(?i)a` +will fail since Unicode case insensitivity is enabled by default. Instead, +callers must use `(?i-u)a` to disable Unicode case folding. Stated differently, +enabling or disabling any of the features below can only add or subtract from +the total set of valid regular expressions. Enabling or disabling a feature +will never modify the match semantics of a regular expression. + +Most features below are enabled by default. Features that aren't enabled by +default are noted. + +### Ecosystem features + +* **std** - + When enabled, this will cause `regex` to use the standard library. In terms + of APIs, `std` causes error types to implement the `std::error::Error` + trait. Enabling `std` will also result in performance optimizations, + including SIMD and faster synchronization primitives. Notably, **disabling + the `std` feature will result in the use of spin locks**. To use a regex + engine without `std` and without spin locks, you'll need to drop down to + the [`regex-automata`](https://docs.rs/regex-automata) crate. +* **logging** - + When enabled, the `log` crate is used to emit messages about regex + compilation and search strategies. This is **disabled by default**. This is + typically only useful to someone working on this crate's internals, but might + be useful if you're doing some rabbit hole performance hacking. Or if you're + just interested in the kinds of decisions being made by the regex engine. + +### Performance features + +**Note**: + To get performance benefits offered by the SIMD, `std` must be enabled. + None of the `perf-*` features will enable `std` implicitly. + +* **perf** - + Enables all performance related features except for `perf-dfa-full`. This + feature is enabled by default is intended to cover all reasonable features + that improve performance, even if more are added in the future. +* **perf-dfa** - + Enables the use of a lazy DFA for matching. The lazy DFA is used to compile + portions of a regex to a very fast DFA on an as-needed basis. This can + result in substantial speedups, usually by an order of magnitude on large + haystacks. The lazy DFA does not bring in any new dependencies, but it can + make compile times longer. +* **perf-dfa-full** - + Enables the use of a full DFA for matching. Full DFAs are problematic because + they have worst case `O(2^n)` construction time. For this reason, when this + feature is enabled, full DFAs are only used for very small regexes and a + very small space bound is used during determinization to avoid the DFA + from blowing up. This feature is not enabled by default, even as part of + `perf`, because it results in fairly sizeable increases in binary size and + compilation time. It can result in faster search times, but they tend to be + more modest and limited to non-Unicode regexes. +* **perf-onepass** - + Enables the use of a one-pass DFA for extracting the positions of capture + groups. This optimization applies to a subset of certain types of NFAs and + represents the fastest engine in this crate for dealing with capture groups. +* **perf-backtrack** - + Enables the use of a bounded backtracking algorithm for extracting the + positions of capture groups. This usually sits between the slowest engine + (the PikeVM) and the fastest engine (one-pass DFA) for extracting capture + groups. It's used whenever the regex is not one-pass and is small enough. +* **perf-inline** - + Enables the use of aggressive inlining inside match routines. This reduces + the overhead of each match. The aggressive inlining, however, increases + compile times and binary size. +* **perf-literal** - + Enables the use of literal optimizations for speeding up matches. In some + cases, literal optimizations can result in speedups of _several_ orders of + magnitude. Disabling this drops the `aho-corasick` and `memchr` dependencies. +* **perf-cache** - + This feature used to enable a faster internal cache at the cost of using + additional dependencies, but this is no longer an option. A fast internal + cache is now used unconditionally with no additional dependencies. This may + change in the future. + +### Unicode features + +* **unicode** - + Enables all Unicode features. This feature is enabled by default, and will + always cover all Unicode features, even if more are added in the future. +* **unicode-age** - + Provide the data for the + [Unicode `Age` property](https://www.unicode.org/reports/tr44/tr44-24.html#Character_Age). + This makes it possible to use classes like `\p{Age:6.0}` to refer to all + codepoints first introduced in Unicode 6.0 +* **unicode-bool** - + Provide the data for numerous Unicode boolean properties. The full list + is not included here, but contains properties like `Alphabetic`, `Emoji`, + `Lowercase`, `Math`, `Uppercase` and `White_Space`. +* **unicode-case** - + Provide the data for case insensitive matching using + [Unicode's "simple loose matches" specification](https://www.unicode.org/reports/tr18/#Simple_Loose_Matches). +* **unicode-gencat** - + Provide the data for + [Unicode general categories](https://www.unicode.org/reports/tr44/tr44-24.html#General_Category_Values). + This includes, but is not limited to, `Decimal_Number`, `Letter`, + `Math_Symbol`, `Number` and `Punctuation`. +* **unicode-perl** - + Provide the data for supporting the Unicode-aware Perl character classes, + corresponding to `\w`, `\s` and `\d`. This is also necessary for using + Unicode-aware word boundary assertions. Note that if this feature is + disabled, the `\s` and `\d` character classes are still available if the + `unicode-bool` and `unicode-gencat` features are enabled, respectively. +* **unicode-script** - + Provide the data for + [Unicode scripts and script extensions](https://www.unicode.org/reports/tr24/). + This includes, but is not limited to, `Arabic`, `Cyrillic`, `Hebrew`, + `Latin` and `Thai`. +* **unicode-segment** - + Provide the data necessary to provide the properties used to implement the + [Unicode text segmentation algorithms](https://www.unicode.org/reports/tr29/). + This enables using classes like `\p{gcb=Extend}`, `\p{wb=Katakana}` and + `\p{sb=ATerm}`. + +# Other crates + +This crate has two required dependencies and several optional dependencies. +This section briefly describes them with the goal of raising awareness of how +different components of this crate may be used independently. + +It is somewhat unusual for a regex engine to have dependencies, as most regex +libraries are self contained units with no dependencies other than a particular +environment's standard library. Indeed, for other similarly optimized regex +engines, most or all of the code in the dependencies of this crate would +normally just be inseparable or coupled parts of the crate itself. But since +Rust and its tooling ecosystem make the use of dependencies so easy, it made +sense to spend some effort de-coupling parts of this crate and making them +independently useful. + +We only briefly describe each crate here. + +* [`regex-lite`](https://docs.rs/regex-lite) is not a dependency of `regex`, +but rather, a standalone zero-dependency simpler version of `regex` that +prioritizes compile times and binary size. In exchange, it eschews Unicode +support and performance. Its match semantics are as identical as possible to +the `regex` crate, and for the things it supports, its APIs are identical to +the APIs in this crate. In other words, for a lot of use cases, it is a drop-in +replacement. +* [`regex-syntax`](https://docs.rs/regex-syntax) provides a regular expression +parser via `Ast` and `Hir` types. It also provides routines for extracting +literals from a pattern. Folks can use this crate to do analysis, or even to +build their own regex engine without having to worry about writing a parser. +* [`regex-automata`](https://docs.rs/regex-automata) provides the regex engines +themselves. One of the downsides of finite automata based regex engines is that +they often need multiple internal engines in order to have similar or better +performance than an unbounded backtracking engine in practice. `regex-automata` +in particular provides public APIs for a PikeVM, a bounded backtracker, a +one-pass DFA, a lazy DFA, a fully compiled DFA and a meta regex engine that +combines all them together. It also has native multi-pattern support and +provides a way to compile and serialize full DFAs such that they can be loaded +and searched in a no-std no-alloc environment. `regex-automata` itself doesn't +even have a required dependency on `regex-syntax`! +* [`memchr`](https://docs.rs/memchr) provides low level SIMD vectorized +routines for quickly finding the location of single bytes or even substrings +in a haystack. In other words, it provides fast `memchr` and `memmem` routines. +These are used by this crate in literal optimizations. +* [`aho-corasick`](https://docs.rs/aho-corasick) provides multi-substring +search. It also provides SIMD vectorized routines in the case where the number +of substrings to search for is relatively small. The `regex` crate also uses +this for literal optimizations. +*/ + +#![no_std] +#![deny(missing_docs)] +#![cfg_attr(feature = "pattern", feature(pattern))] +// This adds Cargo feature annotations to items in the rustdoc output. Which is +// sadly hugely beneficial for this crate due to the number of features. +#![cfg_attr(docsrs_regex, feature(doc_cfg))] +#![warn(missing_debug_implementations)] + +#[cfg(doctest)] +doc_comment::doctest!("../README.md"); + +extern crate alloc; +#[cfg(any(test, feature = "std"))] +extern crate std; + +pub use crate::error::Error; + +pub use crate::{builders::string::*, regex::string::*, regexset::string::*}; + +mod builders; +pub mod bytes; +mod error; +mod find_byte; +#[cfg(feature = "pattern")] +mod pattern; +mod regex; +mod regexset; + +/// Escapes all regular expression meta characters in `pattern`. +/// +/// The string returned may be safely used as a literal in a regular +/// expression. +pub fn escape(pattern: &str) -> alloc::string::String { + regex_syntax::escape(pattern) +} + +/// Public-but-unstable API for macro support. +#[doc(hidden)] +pub mod __private { + pub use regex_automata::util::lazy::Lazy; +} diff --git a/third_party/crates/regex-1.13.1/src/pattern.rs b/third_party/crates/regex-1.13.1/src/pattern.rs new file mode 100644 index 0000000..d7bf148 --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/pattern.rs @@ -0,0 +1,67 @@ +use core::str::pattern::{Pattern, SearchStep, Searcher, Utf8Pattern}; + +use crate::{Matches, Regex}; + +#[derive(Debug)] +pub struct RegexSearcher<'r, 't> { + haystack: &'t str, + it: Matches<'r, 't>, + last_step_end: usize, + next_match: Option<(usize, usize)>, +} + +impl<'r> Pattern for &'r Regex { + type Searcher<'t> = RegexSearcher<'r, 't>; + + fn into_searcher<'t>(self, haystack: &'t str) -> RegexSearcher<'r, 't> { + RegexSearcher { + haystack, + it: self.find_iter(haystack), + last_step_end: 0, + next_match: None, + } + } + + fn as_utf8_pattern<'p>(&'p self) -> Option> { + None + } +} + +unsafe impl<'r, 't> Searcher<'t> for RegexSearcher<'r, 't> { + #[inline] + fn haystack(&self) -> &'t str { + self.haystack + } + + #[inline] + fn next(&mut self) -> SearchStep { + if let Some((s, e)) = self.next_match { + self.next_match = None; + self.last_step_end = e; + return SearchStep::Match(s, e); + } + match self.it.next() { + None => { + if self.last_step_end < self.haystack().len() { + let last = self.last_step_end; + self.last_step_end = self.haystack().len(); + SearchStep::Reject(last, self.haystack().len()) + } else { + SearchStep::Done + } + } + Some(m) => { + let (s, e) = (m.start(), m.end()); + if s == self.last_step_end { + self.last_step_end = e; + SearchStep::Match(s, e) + } else { + self.next_match = Some((s, e)); + let last = self.last_step_end; + self.last_step_end = s; + SearchStep::Reject(last, s) + } + } + } + } +} diff --git a/third_party/crates/regex-1.13.1/src/regex/bytes.rs b/third_party/crates/regex-1.13.1/src/regex/bytes.rs new file mode 100644 index 0000000..5a3a73b --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/regex/bytes.rs @@ -0,0 +1,2775 @@ +use alloc::{borrow::Cow, string::String, sync::Arc, vec::Vec}; + +use regex_automata::{meta, util::captures, Input, PatternID}; + +use crate::{bytes::RegexBuilder, error::Error}; + +/// A convenient way to construct regex patterns from string literals. +/// +/// This macro can be used to construct reusable instances of [`Regex`] with +/// reduced boilerplate. The constructed `Regex` is stored in a static so the +/// pattern is compiled approximately once, even when called multiple times. +/// +/// There is *no compile-time checking of patterns* with `regex!`. Instead, +/// invalid patterns will panic the first time the regex is used. Invalid +/// patterns should still not be used with `regex!`; if compile-time checking +/// becomes feasible in the future, it may be added within a non-semver-breaking +/// release. In the meantime, consider enabling [`clippy::invalid_regex`]. +/// +/// # Examples +/// +/// ``` +/// use regex::bytes::{Regex, regex}; +/// +/// assert!(regex!("[a-z]").is_match(b"a")); +/// assert!(regex!("(inconceivable!|classic blunder)").is_match(b"inconceivable!")); +/// +/// let re: &Regex = regex!(r"(\d{3})-(\d{4})"); +/// assert_eq!(&re.captures(b"867-5309").unwrap()[1], b"867"); +/// ``` +/// +/// An invalid pattern will panic when it is first used: +/// +/// ```should_panic +/// use regex::bytes::regex; +/// +/// let re = regex!("invalid -> ("); // no panic here +/// re.is_match(b"invalid -> ("); // panic! +/// ``` +/// +/// [`clippy::invalid_regex`]: https://rust-lang.github.io/rust-clippy/master/#invalid_regex +// `macro_export` always makes the macro available at crate root. We hide +// from documentation there and instead re-export it in `crate::bytes` to get +// the desired behavior. +#[macro_export] +#[doc(hidden)] +macro_rules! __bytes_regex { + ($re:literal) => {{ + static REGEX: $crate::__private::Lazy<$crate::bytes::Regex> = + $crate::__private::Lazy::new(|| { + $crate::bytes::Regex::new($re).expect("invalid regex pattern") + }); + + // Coerce returned type from `&Lazy` to `&Regex` to avoid making the + // inner type public. + let re: &$crate::bytes::Regex = ®EX; + re + }}; +} + +/// A compiled regular expression for searching Unicode haystacks. +/// +/// A `Regex` can be used to search haystacks, split haystacks into substrings +/// or replace substrings in a haystack with a different substring. All +/// searching is done with an implicit `(?s:.)*?` at the beginning and end of +/// an pattern. To force an expression to match the whole string (or a prefix +/// or a suffix), you must use an anchor like `^` or `$` (or `\A` and `\z`). +/// +/// Like the `Regex` type in the parent module, matches with this regex return +/// byte offsets into the haystack. **Unlike** the parent `Regex` type, these +/// byte offsets may not correspond to UTF-8 sequence boundaries since the +/// regexes in this module can match arbitrary bytes. +/// +/// The only methods that allocate new byte strings are the string replacement +/// methods. All other methods (searching and splitting) return borrowed +/// references into the haystack given. +/// +/// # Example +/// +/// Find the offsets of a US phone number: +/// +/// ``` +/// use regex::bytes::Regex; +/// +/// let re = Regex::new("[0-9]{3}-[0-9]{3}-[0-9]{4}").unwrap(); +/// let m = re.find(b"phone: 111-222-3333").unwrap(); +/// assert_eq!(7..19, m.range()); +/// ``` +/// +/// # Example: extracting capture groups +/// +/// A common way to use regexes is with capture groups. That is, instead of +/// just looking for matches of an entire regex, parentheses are used to create +/// groups that represent part of the match. +/// +/// For example, consider a haystack with multiple lines, and each line has +/// three whitespace delimited fields where the second field is expected to be +/// a number and the third field a boolean. To make this convenient, we use +/// the [`Captures::extract`] API to put the strings that match each group +/// into a fixed size array: +/// +/// ``` +/// use regex::bytes::Regex; +/// +/// let hay = b" +/// rabbit 54 true +/// groundhog 2 true +/// does not match +/// fox 109 false +/// "; +/// let re = Regex::new(r"(?m)^\s*(\S+)\s+([0-9]+)\s+(true|false)\s*$").unwrap(); +/// let mut fields: Vec<(&[u8], i64, bool)> = vec![]; +/// for (_, [f1, f2, f3]) in re.captures_iter(hay).map(|caps| caps.extract()) { +/// // These unwraps are OK because our pattern is written in a way where +/// // all matches for f2 and f3 will be valid UTF-8. +/// let f2 = std::str::from_utf8(f2).unwrap(); +/// let f3 = std::str::from_utf8(f3).unwrap(); +/// fields.push((f1, f2.parse()?, f3.parse()?)); +/// } +/// assert_eq!(fields, vec![ +/// (&b"rabbit"[..], 54, true), +/// (&b"groundhog"[..], 2, true), +/// (&b"fox"[..], 109, false), +/// ]); +/// +/// # Ok::<(), Box>(()) +/// ``` +/// +/// # Example: matching invalid UTF-8 +/// +/// One of the reasons for searching `&[u8]` haystacks is that the `&[u8]` +/// might not be valid UTF-8. Indeed, with a `bytes::Regex`, patterns that +/// match invalid UTF-8 are explicitly allowed. Here's one example that looks +/// for valid UTF-8 fields that might be separated by invalid UTF-8. In this +/// case, we use `(?s-u:.)`, which matches any byte. Attempting to use it in a +/// top-level `Regex` will result in the regex failing to compile. Notice also +/// that we use `.` with Unicode mode enabled, in which case, only valid UTF-8 +/// is matched. In this way, we can build one pattern where some parts only +/// match valid UTF-8 while other parts are more permissive. +/// +/// ``` +/// use regex::bytes::Regex; +/// +/// // F0 9F 92 A9 is the UTF-8 encoding for a Pile of Poo. +/// let hay = b"\xFF\xFFfoo\xFF\xFF\xFF\xF0\x9F\x92\xA9\xFF"; +/// // An equivalent to '(?s-u:.)' is '(?-u:[\x00-\xFF])'. +/// let re = Regex::new(r"(?s)(?-u:.)*?(?.+)(?-u:.)*?(?.+)").unwrap(); +/// let caps = re.captures(hay).unwrap(); +/// assert_eq!(&caps["f1"], &b"foo"[..]); +/// assert_eq!(&caps["f2"], "💩".as_bytes()); +/// ``` +#[derive(Clone)] +pub struct Regex { + pub(crate) meta: meta::Regex, + pub(crate) pattern: Arc, +} + +impl core::fmt::Display for Regex { + /// Shows the original regular expression. + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +impl core::fmt::Debug for Regex { + /// Shows the original regular expression. + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_tuple("Regex").field(&self.as_str()).finish() + } +} + +impl core::str::FromStr for Regex { + type Err = Error; + + /// Attempts to parse a string into a regular expression + fn from_str(s: &str) -> Result { + Regex::new(s) + } +} + +impl TryFrom<&str> for Regex { + type Error = Error; + + /// Attempts to parse a string into a regular expression + fn try_from(s: &str) -> Result { + Regex::new(s) + } +} + +impl TryFrom for Regex { + type Error = Error; + + /// Attempts to parse a string into a regular expression + fn try_from(s: String) -> Result { + Regex::new(&s) + } +} + +/// Core regular expression methods. +impl Regex { + /// Compiles a regular expression. Once compiled, it can be used repeatedly + /// to search, split or replace substrings in a haystack. + /// + /// Note that regex compilation tends to be a somewhat expensive process, + /// and unlike higher level environments, compilation is not automatically + /// cached for you. One should endeavor to compile a regex once and then + /// reuse it. For example, it's a bad idea to compile the same regex + /// repeatedly in a loop. + /// + /// # Errors + /// + /// If an invalid pattern is given, then an error is returned. + /// An error is also returned if the pattern is valid, but would + /// produce a regex that is bigger than the configured size limit via + /// [`RegexBuilder::size_limit`]. (A reasonable size limit is enabled by + /// default.) + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// // An Invalid pattern because of an unclosed parenthesis + /// assert!(Regex::new(r"foo(bar").is_err()); + /// // An invalid pattern because the regex would be too big + /// // because Unicode tends to inflate things. + /// assert!(Regex::new(r"\w{1000}").is_err()); + /// // Disabling Unicode can make the regex much smaller, + /// // potentially by up to or more than an order of magnitude. + /// assert!(Regex::new(r"(?-u:\w){1000}").is_ok()); + /// ``` + pub fn new(re: &str) -> Result { + RegexBuilder::new(re).build() + } + + /// Returns true if and only if there is a match for the regex anywhere + /// in the haystack given. + /// + /// It is recommended to use this method if all you need to do is test + /// whether a match exists, since the underlying matching engine may be + /// able to do less work. + /// + /// # Example + /// + /// Test if some haystack contains at least one word with exactly 13 + /// Unicode word characters: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\b\w{13}\b").unwrap(); + /// let hay = b"I categorically deny having triskaidekaphobia."; + /// assert!(re.is_match(hay)); + /// ``` + #[inline] + pub fn is_match(&self, haystack: &[u8]) -> bool { + self.is_match_at(haystack, 0) + } + + /// This routine searches for the first match of this regex in the + /// haystack given, and if found, returns a [`Match`]. The `Match` + /// provides access to both the byte offsets of the match and the actual + /// substring that matched. + /// + /// Note that this should only be used if you want to find the entire + /// match. If instead you just want to test the existence of a match, + /// it's potentially faster to use `Regex::is_match(hay)` instead of + /// `Regex::find(hay).is_some()`. + /// + /// # Example + /// + /// Find the first word with exactly 13 Unicode word characters: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\b\w{13}\b").unwrap(); + /// let hay = b"I categorically deny having triskaidekaphobia."; + /// let mat = re.find(hay).unwrap(); + /// assert_eq!(2..15, mat.range()); + /// assert_eq!(b"categorically", mat.as_bytes()); + /// ``` + #[inline] + pub fn find<'h>(&self, haystack: &'h [u8]) -> Option> { + self.find_at(haystack, 0) + } + + /// Returns an iterator that yields successive non-overlapping matches in + /// the given haystack. The iterator yields values of type [`Match`]. + /// + /// # Time complexity + /// + /// Note that since `find_iter` runs potentially many searches on the + /// haystack and since each search has worst case `O(m * n)` time + /// complexity, the overall worst case time complexity for iteration is + /// `O(m * n^2)`. + /// + /// # Example + /// + /// Find every word with exactly 13 Unicode word characters: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\b\w{13}\b").unwrap(); + /// let hay = b"Retroactively relinquishing remunerations is reprehensible."; + /// let matches: Vec<_> = re.find_iter(hay).map(|m| m.as_bytes()).collect(); + /// assert_eq!(matches, vec![ + /// &b"Retroactively"[..], + /// &b"relinquishing"[..], + /// &b"remunerations"[..], + /// &b"reprehensible"[..], + /// ]); + /// ``` + #[inline] + pub fn find_iter<'r, 'h>(&'r self, haystack: &'h [u8]) -> Matches<'r, 'h> { + Matches { haystack, it: self.meta.find_iter(haystack) } + } + + /// This routine searches for the first match of this regex in the haystack + /// given, and if found, returns not only the overall match but also the + /// matches of each capture group in the regex. If no match is found, then + /// `None` is returned. + /// + /// Capture group `0` always corresponds to an implicit unnamed group that + /// includes the entire match. If a match is found, this group is always + /// present. Subsequent groups may be named and are numbered, starting + /// at 1, by the order in which the opening parenthesis appears in the + /// pattern. For example, in the pattern `(?.(?.))(?.)`, `a`, + /// `b` and `c` correspond to capture group indices `1`, `2` and `3`, + /// respectively. + /// + /// You should only use `captures` if you need access to the capture group + /// matches. Otherwise, [`Regex::find`] is generally faster for discovering + /// just the overall match. + /// + /// # Example + /// + /// Say you have some haystack with movie names and their release years, + /// like "'Citizen Kane' (1941)". It'd be nice if we could search for + /// strings looking like that, while also extracting the movie name and its + /// release year separately. The example below shows how to do that. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"'([^']+)'\s+\((\d{4})\)").unwrap(); + /// let hay = b"Not my favorite movie: 'Citizen Kane' (1941)."; + /// let caps = re.captures(hay).unwrap(); + /// assert_eq!(caps.get(0).unwrap().as_bytes(), b"'Citizen Kane' (1941)"); + /// assert_eq!(caps.get(1).unwrap().as_bytes(), b"Citizen Kane"); + /// assert_eq!(caps.get(2).unwrap().as_bytes(), b"1941"); + /// // You can also access the groups by index using the Index notation. + /// // Note that this will panic on an invalid index. In this case, these + /// // accesses are always correct because the overall regex will only + /// // match when these capture groups match. + /// assert_eq!(&caps[0], b"'Citizen Kane' (1941)"); + /// assert_eq!(&caps[1], b"Citizen Kane"); + /// assert_eq!(&caps[2], b"1941"); + /// ``` + /// + /// Note that the full match is at capture group `0`. Each subsequent + /// capture group is indexed by the order of its opening `(`. + /// + /// We can make this example a bit clearer by using *named* capture groups: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"'(?[^']+)'\s+\((?<year>\d{4})\)").unwrap(); + /// let hay = b"Not my favorite movie: 'Citizen Kane' (1941)."; + /// let caps = re.captures(hay).unwrap(); + /// assert_eq!(caps.get(0).unwrap().as_bytes(), b"'Citizen Kane' (1941)"); + /// assert_eq!(caps.name("title").unwrap().as_bytes(), b"Citizen Kane"); + /// assert_eq!(caps.name("year").unwrap().as_bytes(), b"1941"); + /// // You can also access the groups by name using the Index notation. + /// // Note that this will panic on an invalid group name. In this case, + /// // these accesses are always correct because the overall regex will + /// // only match when these capture groups match. + /// assert_eq!(&caps[0], b"'Citizen Kane' (1941)"); + /// assert_eq!(&caps["title"], b"Citizen Kane"); + /// assert_eq!(&caps["year"], b"1941"); + /// ``` + /// + /// Here we name the capture groups, which we can access with the `name` + /// method or the `Index` notation with a `&str`. Note that the named + /// capture groups are still accessible with `get` or the `Index` notation + /// with a `usize`. + /// + /// The `0`th capture group is always unnamed, so it must always be + /// accessed with `get(0)` or `[0]`. + /// + /// Finally, one other way to get the matched substrings is with the + /// [`Captures::extract`] API: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"'([^']+)'\s+\((\d{4})\)").unwrap(); + /// let hay = b"Not my favorite movie: 'Citizen Kane' (1941)."; + /// let (full, [title, year]) = re.captures(hay).unwrap().extract(); + /// assert_eq!(full, b"'Citizen Kane' (1941)"); + /// assert_eq!(title, b"Citizen Kane"); + /// assert_eq!(year, b"1941"); + /// ``` + #[inline] + pub fn captures<'h>(&self, haystack: &'h [u8]) -> Option<Captures<'h>> { + self.captures_at(haystack, 0) + } + + /// Returns an iterator that yields successive non-overlapping matches in + /// the given haystack. The iterator yields values of type [`Captures`]. + /// + /// This is the same as [`Regex::find_iter`], but instead of only providing + /// access to the overall match, each value yield includes access to the + /// matches of all capture groups in the regex. Reporting this extra match + /// data is potentially costly, so callers should only use `captures_iter` + /// over `find_iter` when they actually need access to the capture group + /// matches. + /// + /// # Time complexity + /// + /// Note that since `captures_iter` runs potentially many searches on the + /// haystack and since each search has worst case `O(m * n)` time + /// complexity, the overall worst case time complexity for iteration is + /// `O(m * n^2)`. + /// + /// # Example + /// + /// We can use this to find all movie titles and their release years in + /// some haystack, where the movie is formatted like "'Title' (xxxx)": + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"'([^']+)'\s+\(([0-9]{4})\)").unwrap(); + /// let hay = b"'Citizen Kane' (1941), 'The Wizard of Oz' (1939), 'M' (1931)."; + /// let mut movies = vec![]; + /// for (_, [title, year]) in re.captures_iter(hay).map(|c| c.extract()) { + /// // OK because [0-9]{4} can only match valid UTF-8. + /// let year = std::str::from_utf8(year).unwrap(); + /// movies.push((title, year.parse::<i64>()?)); + /// } + /// assert_eq!(movies, vec![ + /// (&b"Citizen Kane"[..], 1941), + /// (&b"The Wizard of Oz"[..], 1939), + /// (&b"M"[..], 1931), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Or with named groups: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"'(?<title>[^']+)'\s+\((?<year>[0-9]{4})\)").unwrap(); + /// let hay = b"'Citizen Kane' (1941), 'The Wizard of Oz' (1939), 'M' (1931)."; + /// let mut it = re.captures_iter(hay); + /// + /// let caps = it.next().unwrap(); + /// assert_eq!(&caps["title"], b"Citizen Kane"); + /// assert_eq!(&caps["year"], b"1941"); + /// + /// let caps = it.next().unwrap(); + /// assert_eq!(&caps["title"], b"The Wizard of Oz"); + /// assert_eq!(&caps["year"], b"1939"); + /// + /// let caps = it.next().unwrap(); + /// assert_eq!(&caps["title"], b"M"); + /// assert_eq!(&caps["year"], b"1931"); + /// ``` + #[inline] + pub fn captures_iter<'r, 'h>( + &'r self, + haystack: &'h [u8], + ) -> CaptureMatches<'r, 'h> { + CaptureMatches { haystack, it: self.meta.captures_iter(haystack) } + } + + /// Returns an iterator of substrings of the haystack given, delimited by a + /// match of the regex. Namely, each element of the iterator corresponds to + /// a part of the haystack that *isn't* matched by the regular expression. + /// + /// # Time complexity + /// + /// Since iterators over all matches requires running potentially many + /// searches on the haystack, and since each search has worst case + /// `O(m * n)` time complexity, the overall worst case time complexity for + /// this routine is `O(m * n^2)`. + /// + /// # Example + /// + /// To split a string delimited by arbitrary amounts of spaces or tabs: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"[ \t]+").unwrap(); + /// let hay = b"a b \t c\td e"; + /// let fields: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(fields, vec![ + /// &b"a"[..], &b"b"[..], &b"c"[..], &b"d"[..], &b"e"[..], + /// ]); + /// ``` + /// + /// # Example: more cases + /// + /// Basic usage: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r" ").unwrap(); + /// let hay = b"Mary had a little lamb"; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![ + /// &b"Mary"[..], &b"had"[..], &b"a"[..], &b"little"[..], &b"lamb"[..], + /// ]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = b""; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![&b""[..]]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = b"lionXXtigerXleopard"; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![ + /// &b"lion"[..], &b""[..], &b"tiger"[..], &b"leopard"[..], + /// ]); + /// + /// let re = Regex::new(r"::").unwrap(); + /// let hay = b"lion::tiger::leopard"; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![&b"lion"[..], &b"tiger"[..], &b"leopard"[..]]); + /// ``` + /// + /// If a haystack contains multiple contiguous matches, you will end up + /// with empty spans yielded by the iterator: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = b"XXXXaXXbXc"; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![ + /// &b""[..], &b""[..], &b""[..], &b""[..], + /// &b"a"[..], &b""[..], &b"b"[..], &b"c"[..], + /// ]); + /// + /// let re = Regex::new(r"/").unwrap(); + /// let hay = b"(///)"; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![&b"("[..], &b""[..], &b""[..], &b")"[..]]); + /// ``` + /// + /// Separators at the start or end of a haystack are neighbored by empty + /// substring. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"0").unwrap(); + /// let hay = b"010"; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![&b""[..], &b"1"[..], &b""[..]]); + /// ``` + /// + /// When the regex can match the empty string, it splits at every byte + /// position in the haystack. This includes between all UTF-8 code units. + /// (The top-level [`Regex::split`](crate::Regex::split) will only split + /// at valid UTF-8 boundaries.) + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"").unwrap(); + /// let hay = "☃".as_bytes(); + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![ + /// &[][..], &[b'\xE2'][..], &[b'\x98'][..], &[b'\x83'][..], &[][..], + /// ]); + /// ``` + /// + /// Contiguous separators (commonly shows up with whitespace), can lead to + /// possibly surprising behavior. For example, this code is correct: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r" ").unwrap(); + /// let hay = b" a b c"; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// assert_eq!(got, vec![ + /// &b""[..], &b""[..], &b""[..], &b""[..], + /// &b"a"[..], &b""[..], &b"b"[..], &b"c"[..], + /// ]); + /// ``` + /// + /// It does *not* give you `["a", "b", "c"]`. For that behavior, you'd want + /// to match contiguous space characters: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r" +").unwrap(); + /// let hay = b" a b c"; + /// let got: Vec<&[u8]> = re.split(hay).collect(); + /// // N.B. This does still include a leading empty span because ' +' + /// // matches at the beginning of the haystack. + /// assert_eq!(got, vec![&b""[..], &b"a"[..], &b"b"[..], &b"c"[..]]); + /// ``` + #[inline] + pub fn split<'r, 'h>(&'r self, haystack: &'h [u8]) -> Split<'r, 'h> { + Split { haystack, it: self.meta.split(haystack) } + } + + /// Returns an iterator of at most `limit` substrings of the haystack + /// given, delimited by a match of the regex. (A `limit` of `0` will return + /// no substrings.) Namely, each element of the iterator corresponds to a + /// part of the haystack that *isn't* matched by the regular expression. + /// The remainder of the haystack that is not split will be the last + /// element in the iterator. + /// + /// # Time complexity + /// + /// Since iterators over all matches requires running potentially many + /// searches on the haystack, and since each search has worst case + /// `O(m * n)` time complexity, the overall worst case time complexity for + /// this routine is `O(m * n^2)`. + /// + /// Although note that the worst case time here has an upper bound given + /// by the `limit` parameter. + /// + /// # Example + /// + /// Get the first two words in some haystack: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\W+").unwrap(); + /// let hay = b"Hey! How are you?"; + /// let fields: Vec<&[u8]> = re.splitn(hay, 3).collect(); + /// assert_eq!(fields, vec![&b"Hey"[..], &b"How"[..], &b"are you?"[..]]); + /// ``` + /// + /// # Examples: more cases + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r" ").unwrap(); + /// let hay = b"Mary had a little lamb"; + /// let got: Vec<&[u8]> = re.splitn(hay, 3).collect(); + /// assert_eq!(got, vec![&b"Mary"[..], &b"had"[..], &b"a little lamb"[..]]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = b""; + /// let got: Vec<&[u8]> = re.splitn(hay, 3).collect(); + /// assert_eq!(got, vec![&b""[..]]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = b"lionXXtigerXleopard"; + /// let got: Vec<&[u8]> = re.splitn(hay, 3).collect(); + /// assert_eq!(got, vec![&b"lion"[..], &b""[..], &b"tigerXleopard"[..]]); + /// + /// let re = Regex::new(r"::").unwrap(); + /// let hay = b"lion::tiger::leopard"; + /// let got: Vec<&[u8]> = re.splitn(hay, 2).collect(); + /// assert_eq!(got, vec![&b"lion"[..], &b"tiger::leopard"[..]]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = b"abcXdef"; + /// let got: Vec<&[u8]> = re.splitn(hay, 1).collect(); + /// assert_eq!(got, vec![&b"abcXdef"[..]]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = b"abcdef"; + /// let got: Vec<&[u8]> = re.splitn(hay, 2).collect(); + /// assert_eq!(got, vec![&b"abcdef"[..]]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = b"abcXdef"; + /// let got: Vec<&[u8]> = re.splitn(hay, 0).collect(); + /// assert!(got.is_empty()); + /// ``` + #[inline] + pub fn splitn<'r, 'h>( + &'r self, + haystack: &'h [u8], + limit: usize, + ) -> SplitN<'r, 'h> { + SplitN { haystack, it: self.meta.splitn(haystack, limit) } + } + + /// Replaces the leftmost-first match in the given haystack with the + /// replacement provided. The replacement can be a regular string (where + /// `$N` and `$name` are expanded to match capture groups) or a function + /// that takes a [`Captures`] and returns the replaced string. + /// + /// If no match is found, then the haystack is returned unchanged. In that + /// case, this implementation will likely return a `Cow::Borrowed` value + /// such that no allocation is performed. + /// + /// When a `Cow::Borrowed` is returned, the value returned is guaranteed + /// to be equivalent to the `haystack` given. + /// + /// # Replacement string syntax + /// + /// All instances of `$ref` in the replacement string are replaced with + /// the substring corresponding to the capture group identified by `ref`. + /// + /// `ref` may be an integer corresponding to the index of the capture group + /// (counted by order of opening parenthesis where `0` is the entire match) + /// or it can be a name (consisting of letters, digits or underscores) + /// corresponding to a named capture group. + /// + /// If `ref` isn't a valid capture group (whether the name doesn't exist or + /// isn't a valid index), then it is replaced with the empty string. + /// + /// The longest possible name is used. For example, `$1a` looks up the + /// capture group named `1a` and not the capture group at index `1`. To + /// exert more precise control over the name, use braces, e.g., `${1}a`. + /// + /// To write a literal `$` use `$$`. + /// + /// # Example + /// + /// Note that this function is polymorphic with respect to the replacement. + /// In typical usage, this can just be a normal string: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"[^01]+").unwrap(); + /// assert_eq!(re.replace(b"1078910", b""), &b"1010"[..]); + /// ``` + /// + /// But anything satisfying the [`Replacer`] trait will work. For example, + /// a closure of type `|&Captures| -> String` provides direct access to the + /// captures corresponding to a match. This allows one to access capturing + /// group matches easily: + /// + /// ``` + /// use regex::bytes::{Captures, Regex}; + /// + /// let re = Regex::new(r"([^,\s]+),\s+(\S+)").unwrap(); + /// let result = re.replace(b"Springsteen, Bruce", |caps: &Captures| { + /// let mut buf = vec![]; + /// buf.extend_from_slice(&caps[2]); + /// buf.push(b' '); + /// buf.extend_from_slice(&caps[1]); + /// buf + /// }); + /// assert_eq!(result, &b"Bruce Springsteen"[..]); + /// ``` + /// + /// But this is a bit cumbersome to use all the time. Instead, a simple + /// syntax is supported (as described above) that expands `$name` into the + /// corresponding capture group. Here's the last example, but using this + /// expansion technique with named capture groups: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(?<last>[^,\s]+),\s+(?<first>\S+)").unwrap(); + /// let result = re.replace(b"Springsteen, Bruce", b"$first $last"); + /// assert_eq!(result, &b"Bruce Springsteen"[..]); + /// ``` + /// + /// Note that using `$2` instead of `$first` or `$1` instead of `$last` + /// would produce the same result. To write a literal `$` use `$$`. + /// + /// Sometimes the replacement string requires use of curly braces to + /// delineate a capture group replacement when it is adjacent to some other + /// literal text. For example, if we wanted to join two words together with + /// an underscore: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(?<first>\w+)\s+(?<second>\w+)").unwrap(); + /// let result = re.replace(b"deep fried", b"${first}_$second"); + /// assert_eq!(result, &b"deep_fried"[..]); + /// ``` + /// + /// Without the curly braces, the capture group name `first_` would be + /// used, and since it doesn't exist, it would be replaced with the empty + /// string. + /// + /// Finally, sometimes you just want to replace a literal string with no + /// regard for capturing group expansion. This can be done by wrapping a + /// string with [`NoExpand`]: + /// + /// ``` + /// use regex::bytes::{NoExpand, Regex}; + /// + /// let re = Regex::new(r"(?<last>[^,\s]+),\s+(\S+)").unwrap(); + /// let result = re.replace(b"Springsteen, Bruce", NoExpand(b"$2 $last")); + /// assert_eq!(result, &b"$2 $last"[..]); + /// ``` + /// + /// Using `NoExpand` may also be faster, since the replacement string won't + /// need to be parsed for the `$` syntax. + #[inline] + pub fn replace<'h, R: Replacer>( + &self, + haystack: &'h [u8], + rep: R, + ) -> Cow<'h, [u8]> { + self.replacen(haystack, 1, rep) + } + + /// Replaces all non-overlapping matches in the haystack with the + /// replacement provided. This is the same as calling `replacen` with + /// `limit` set to `0`. + /// + /// If no match is found, then the haystack is returned unchanged. In that + /// case, this implementation will likely return a `Cow::Borrowed` value + /// such that no allocation is performed. + /// + /// When a `Cow::Borrowed` is returned, the value returned is guaranteed + /// to be equivalent to the `haystack` given. + /// + /// The documentation for [`Regex::replace`] goes into more detail about + /// what kinds of replacement strings are supported. + /// + /// # Time complexity + /// + /// Since iterators over all matches requires running potentially many + /// searches on the haystack, and since each search has worst case + /// `O(m * n)` time complexity, the overall worst case time complexity for + /// this routine is `O(m * n^2)`. + /// + /// # Fallibility + /// + /// If you need to write a replacement routine where any individual + /// replacement might "fail," doing so with this API isn't really feasible + /// because there's no way to stop the search process if a replacement + /// fails. Instead, if you need this functionality, you should consider + /// implementing your own replacement routine: + /// + /// ``` + /// use regex::bytes::{Captures, Regex}; + /// + /// fn replace_all<E>( + /// re: &Regex, + /// haystack: &[u8], + /// replacement: impl Fn(&Captures) -> Result<Vec<u8>, E>, + /// ) -> Result<Vec<u8>, E> { + /// let mut new = Vec::with_capacity(haystack.len()); + /// let mut last_match = 0; + /// for caps in re.captures_iter(haystack) { + /// let m = caps.get(0).unwrap(); + /// new.extend_from_slice(&haystack[last_match..m.start()]); + /// new.extend_from_slice(&replacement(&caps)?); + /// last_match = m.end(); + /// } + /// new.extend_from_slice(&haystack[last_match..]); + /// Ok(new) + /// } + /// + /// // Let's replace each word with the number of bytes in that word. + /// // But if we see a word that is "too long," we'll give up. + /// let re = Regex::new(r"\w+").unwrap(); + /// let replacement = |caps: &Captures| -> Result<Vec<u8>, &'static str> { + /// if caps[0].len() >= 5 { + /// return Err("word too long"); + /// } + /// Ok(caps[0].len().to_string().into_bytes()) + /// }; + /// assert_eq!( + /// Ok(b"2 3 3 3?".to_vec()), + /// replace_all(&re, b"hi how are you?", &replacement), + /// ); + /// assert!(replace_all(&re, b"hi there", &replacement).is_err()); + /// ``` + /// + /// # Example + /// + /// This example shows how to flip the order of whitespace (excluding line + /// terminators) delimited fields, and normalizes the whitespace that + /// delimits the fields: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(?m)^(\S+)[\s--\r\n]+(\S+)$").unwrap(); + /// let hay = b" + /// Greetings 1973 + /// Wild\t1973 + /// BornToRun\t\t\t\t1975 + /// Darkness 1978 + /// TheRiver 1980 + /// "; + /// let new = re.replace_all(hay, b"$2 $1"); + /// assert_eq!(new, &b" + /// 1973 Greetings + /// 1973 Wild + /// 1975 BornToRun + /// 1978 Darkness + /// 1980 TheRiver + /// "[..]); + /// ``` + #[inline] + pub fn replace_all<'h, R: Replacer>( + &self, + haystack: &'h [u8], + rep: R, + ) -> Cow<'h, [u8]> { + self.replacen(haystack, 0, rep) + } + + /// Replaces at most `limit` non-overlapping matches in the haystack with + /// the replacement provided. If `limit` is `0`, then all non-overlapping + /// matches are replaced. That is, `Regex::replace_all(hay, rep)` is + /// equivalent to `Regex::replacen(hay, 0, rep)`. + /// + /// If no match is found, then the haystack is returned unchanged. In that + /// case, this implementation will likely return a `Cow::Borrowed` value + /// such that no allocation is performed. + /// + /// When a `Cow::Borrowed` is returned, the value returned is guaranteed + /// to be equivalent to the `haystack` given. + /// + /// The documentation for [`Regex::replace`] goes into more detail about + /// what kinds of replacement strings are supported. + /// + /// # Time complexity + /// + /// Since iterators over all matches requires running potentially many + /// searches on the haystack, and since each search has worst case + /// `O(m * n)` time complexity, the overall worst case time complexity for + /// this routine is `O(m * n^2)`. + /// + /// Although note that the worst case time here has an upper bound given + /// by the `limit` parameter. + /// + /// # Fallibility + /// + /// See the corresponding section in the docs for [`Regex::replace_all`] + /// for tips on how to deal with a replacement routine that can fail. + /// + /// # Example + /// + /// This example shows how to flip the order of whitespace (excluding line + /// terminators) delimited fields, and normalizes the whitespace that + /// delimits the fields. But we only do it for the first two matches. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(?m)^(\S+)[\s--\r\n]+(\S+)$").unwrap(); + /// let hay = b" + /// Greetings 1973 + /// Wild\t1973 + /// BornToRun\t\t\t\t1975 + /// Darkness 1978 + /// TheRiver 1980 + /// "; + /// let new = re.replacen(hay, 2, b"$2 $1"); + /// assert_eq!(new, &b" + /// 1973 Greetings + /// 1973 Wild + /// BornToRun\t\t\t\t1975 + /// Darkness 1978 + /// TheRiver 1980 + /// "[..]); + /// ``` + #[inline] + pub fn replacen<'h, R: Replacer>( + &self, + haystack: &'h [u8], + limit: usize, + mut rep: R, + ) -> Cow<'h, [u8]> { + // If we know that the replacement doesn't have any capture expansions, + // then we can use the fast path. The fast path can make a tremendous + // difference: + // + // 1) We use `find_iter` instead of `captures_iter`. Not asking for + // captures generally makes the regex engines faster. + // 2) We don't need to look up all of the capture groups and do + // replacements inside the replacement string. We just push it + // at each match and be done with it. + if let Some(rep) = rep.no_expansion() { + let mut it = self.find_iter(haystack).enumerate().peekable(); + if it.peek().is_none() { + return Cow::Borrowed(haystack); + } + let mut new = Vec::with_capacity(haystack.len()); + let mut last_match = 0; + for (i, m) in it { + new.extend_from_slice(&haystack[last_match..m.start()]); + new.extend_from_slice(&rep); + last_match = m.end(); + if limit > 0 && i >= limit - 1 { + break; + } + } + new.extend_from_slice(&haystack[last_match..]); + return Cow::Owned(new); + } + + // The slower path, which we use if the replacement needs access to + // capture groups. + let mut it = self.captures_iter(haystack).enumerate().peekable(); + if it.peek().is_none() { + return Cow::Borrowed(haystack); + } + let mut new = Vec::with_capacity(haystack.len()); + let mut last_match = 0; + for (i, cap) in it { + // unwrap on 0 is OK because captures only reports matches + let m = cap.get(0).unwrap(); + new.extend_from_slice(&haystack[last_match..m.start()]); + rep.replace_append(&cap, &mut new); + last_match = m.end(); + if limit > 0 && i >= limit - 1 { + break; + } + } + new.extend_from_slice(&haystack[last_match..]); + Cow::Owned(new) + } +} + +/// A group of advanced or "lower level" search methods. Some methods permit +/// starting the search at a position greater than `0` in the haystack. Other +/// methods permit reusing allocations, for example, when extracting the +/// matches for capture groups. +impl Regex { + /// Returns the end byte offset of the first match in the haystack given. + /// + /// This method may have the same performance characteristics as + /// `is_match`. Behaviorally, it doesn't just report whether it match + /// occurs, but also the end offset for a match. In particular, the offset + /// returned *may be shorter* than the proper end of the leftmost-first + /// match that you would find via [`Regex::find`]. + /// + /// Note that it is not guaranteed that this routine finds the shortest or + /// "earliest" possible match. Instead, the main idea of this API is that + /// it returns the offset at the point at which the internal regex engine + /// has determined that a match has occurred. This may vary depending on + /// which internal regex engine is used, and thus, the offset itself may + /// change based on internal heuristics. + /// + /// # Example + /// + /// Typically, `a+` would match the entire first sequence of `a` in some + /// haystack, but `shortest_match` *may* give up as soon as it sees the + /// first `a`. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"a+").unwrap(); + /// let offset = re.shortest_match(b"aaaaa").unwrap(); + /// assert_eq!(offset, 1); + /// ``` + #[inline] + pub fn shortest_match(&self, haystack: &[u8]) -> Option<usize> { + self.shortest_match_at(haystack, 0) + } + + /// Returns the same as `shortest_match`, but starts the search at the + /// given offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only match + /// when `start == 0`. + /// + /// If a match is found, the offset returned is relative to the beginning + /// of the haystack, not the beginning of the search. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = b"eschew"; + /// // We get a match here, but it's probably not intended. + /// assert_eq!(re.shortest_match(&hay[2..]), Some(4)); + /// // No match because the assertions take the context into account. + /// assert_eq!(re.shortest_match_at(hay, 2), None); + /// ``` + #[inline] + pub fn shortest_match_at( + &self, + haystack: &[u8], + start: usize, + ) -> Option<usize> { + let input = + Input::new(haystack).earliest(true).span(start..haystack.len()); + self.meta.search_half(&input).map(|hm| hm.offset()) + } + + /// Returns the same as [`Regex::is_match`], but starts the search at the + /// given offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = b"eschew"; + /// // We get a match here, but it's probably not intended. + /// assert!(re.is_match(&hay[2..])); + /// // No match because the assertions take the context into account. + /// assert!(!re.is_match_at(hay, 2)); + /// ``` + #[inline] + pub fn is_match_at(&self, haystack: &[u8], start: usize) -> bool { + self.meta.is_match(Input::new(haystack).span(start..haystack.len())) + } + + /// Returns the same as [`Regex::find`], but starts the search at the given + /// offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = b"eschew"; + /// // We get a match here, but it's probably not intended. + /// assert_eq!(re.find(&hay[2..]).map(|m| m.range()), Some(0..4)); + /// // No match because the assertions take the context into account. + /// assert_eq!(re.find_at(hay, 2), None); + /// ``` + #[inline] + pub fn find_at<'h>( + &self, + haystack: &'h [u8], + start: usize, + ) -> Option<Match<'h>> { + let input = Input::new(haystack).span(start..haystack.len()); + self.meta.find(input).map(|m| Match::new(haystack, m.start(), m.end())) + } + + /// Returns the same as [`Regex::captures`], but starts the search at the + /// given offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = b"eschew"; + /// // We get a match here, but it's probably not intended. + /// assert_eq!(&re.captures(&hay[2..]).unwrap()[0], b"chew"); + /// // No match because the assertions take the context into account. + /// assert!(re.captures_at(hay, 2).is_none()); + /// ``` + #[inline] + pub fn captures_at<'h>( + &self, + haystack: &'h [u8], + start: usize, + ) -> Option<Captures<'h>> { + let input = Input::new(haystack).span(start..haystack.len()); + let mut caps = self.meta.create_captures(); + self.meta.captures(input, &mut caps); + if caps.is_match() { + let static_captures_len = self.static_captures_len(); + Some(Captures { haystack, caps, static_captures_len }) + } else { + None + } + } + + /// This is like [`Regex::captures`], but writes the byte offsets of each + /// capture group match into the locations given. + /// + /// A [`CaptureLocations`] stores the same byte offsets as a [`Captures`], + /// but does *not* store a reference to the haystack. This makes its API + /// a bit lower level and less convenient. But in exchange, callers + /// may allocate their own `CaptureLocations` and reuse it for multiple + /// searches. This may be helpful if allocating a `Captures` shows up in a + /// profile as too costly. + /// + /// To create a `CaptureLocations` value, use the + /// [`Regex::capture_locations`] method. + /// + /// This also returns the overall match if one was found. When a match is + /// found, its offsets are also always stored in `locs` at index `0`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"^([a-z]+)=(\S*)$").unwrap(); + /// let mut locs = re.capture_locations(); + /// assert!(re.captures_read(&mut locs, b"id=foo123").is_some()); + /// assert_eq!(Some((0, 9)), locs.get(0)); + /// assert_eq!(Some((0, 2)), locs.get(1)); + /// assert_eq!(Some((3, 9)), locs.get(2)); + /// ``` + #[inline] + pub fn captures_read<'h>( + &self, + locs: &mut CaptureLocations, + haystack: &'h [u8], + ) -> Option<Match<'h>> { + self.captures_read_at(locs, haystack, 0) + } + + /// Returns the same as [`Regex::captures_read`], but starts the search at + /// the given offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = b"eschew"; + /// let mut locs = re.capture_locations(); + /// // We get a match here, but it's probably not intended. + /// assert!(re.captures_read(&mut locs, &hay[2..]).is_some()); + /// // No match because the assertions take the context into account. + /// assert!(re.captures_read_at(&mut locs, hay, 2).is_none()); + /// ``` + #[inline] + pub fn captures_read_at<'h>( + &self, + locs: &mut CaptureLocations, + haystack: &'h [u8], + start: usize, + ) -> Option<Match<'h>> { + let input = Input::new(haystack).span(start..haystack.len()); + self.meta.search_captures(&input, &mut locs.0); + locs.0.get_match().map(|m| Match::new(haystack, m.start(), m.end())) + } + + /// An undocumented alias for `captures_read_at`. + /// + /// The `regex-capi` crate previously used this routine, so to avoid + /// breaking that crate, we continue to provide the name as an undocumented + /// alias. + #[doc(hidden)] + #[inline] + pub fn read_captures_at<'h>( + &self, + locs: &mut CaptureLocations, + haystack: &'h [u8], + start: usize, + ) -> Option<Match<'h>> { + self.captures_read_at(locs, haystack, start) + } +} + +/// Auxiliary methods. +impl Regex { + /// Returns the original string of this regex. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"foo\w+bar").unwrap(); + /// assert_eq!(re.as_str(), r"foo\w+bar"); + /// ``` + #[inline] + pub fn as_str(&self) -> &str { + &self.pattern + } + + /// Returns an iterator over the capture names in this regex. + /// + /// The iterator returned yields elements of type `Option<&str>`. That is, + /// the iterator yields values for all capture groups, even ones that are + /// unnamed. The order of the groups corresponds to the order of the group's + /// corresponding opening parenthesis. + /// + /// The first element of the iterator always yields the group corresponding + /// to the overall match, and this group is always unnamed. Therefore, the + /// iterator always yields at least one group. + /// + /// # Example + /// + /// This shows basic usage with a mix of named and unnamed capture groups: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(?<a>.(?<b>.))(.)(?:.)(?<c>.)").unwrap(); + /// let mut names = re.capture_names(); + /// assert_eq!(names.next(), Some(None)); + /// assert_eq!(names.next(), Some(Some("a"))); + /// assert_eq!(names.next(), Some(Some("b"))); + /// assert_eq!(names.next(), Some(None)); + /// // the '(?:.)' group is non-capturing and so doesn't appear here! + /// assert_eq!(names.next(), Some(Some("c"))); + /// assert_eq!(names.next(), None); + /// ``` + /// + /// The iterator always yields at least one element, even for regexes with + /// no capture groups and even for regexes that can never match: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"").unwrap(); + /// let mut names = re.capture_names(); + /// assert_eq!(names.next(), Some(None)); + /// assert_eq!(names.next(), None); + /// + /// let re = Regex::new(r"[a&&b]").unwrap(); + /// let mut names = re.capture_names(); + /// assert_eq!(names.next(), Some(None)); + /// assert_eq!(names.next(), None); + /// ``` + #[inline] + pub fn capture_names(&self) -> CaptureNames<'_> { + CaptureNames(self.meta.group_info().pattern_names(PatternID::ZERO)) + } + + /// Returns the number of captures groups in this regex. + /// + /// This includes all named and unnamed groups, including the implicit + /// unnamed group that is always present and corresponds to the entire + /// match. + /// + /// Since the implicit unnamed group is always included in this length, the + /// length returned is guaranteed to be greater than zero. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"foo").unwrap(); + /// assert_eq!(1, re.captures_len()); + /// + /// let re = Regex::new(r"(foo)").unwrap(); + /// assert_eq!(2, re.captures_len()); + /// + /// let re = Regex::new(r"(?<a>.(?<b>.))(.)(?:.)(?<c>.)").unwrap(); + /// assert_eq!(5, re.captures_len()); + /// + /// let re = Regex::new(r"[a&&b]").unwrap(); + /// assert_eq!(1, re.captures_len()); + /// ``` + #[inline] + pub fn captures_len(&self) -> usize { + self.meta.group_info().group_len(PatternID::ZERO) + } + + /// Returns the total number of capturing groups that appear in every + /// possible match. + /// + /// If the number of capture groups can vary depending on the match, then + /// this returns `None`. That is, a value is only returned when the number + /// of matching groups is invariant or "static." + /// + /// Note that like [`Regex::captures_len`], this **does** include the + /// implicit capturing group corresponding to the entire match. Therefore, + /// when a non-None value is returned, it is guaranteed to be at least `1`. + /// Stated differently, a return value of `Some(0)` is impossible. + /// + /// # Example + /// + /// This shows a few cases where a static number of capture groups is + /// available and a few cases where it is not. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let len = |pattern| { + /// Regex::new(pattern).map(|re| re.static_captures_len()) + /// }; + /// + /// assert_eq!(Some(1), len("a")?); + /// assert_eq!(Some(2), len("(a)")?); + /// assert_eq!(Some(2), len("(a)|(b)")?); + /// assert_eq!(Some(3), len("(a)(b)|(c)(d)")?); + /// assert_eq!(None, len("(a)|b")?); + /// assert_eq!(None, len("a|(b)")?); + /// assert_eq!(None, len("(b)*")?); + /// assert_eq!(Some(2), len("(b)+")?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn static_captures_len(&self) -> Option<usize> { + self.meta.static_captures_len() + } + + /// Returns a fresh allocated set of capture locations that can + /// be reused in multiple calls to [`Regex::captures_read`] or + /// [`Regex::captures_read_at`]. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(.)(.)(\w+)").unwrap(); + /// let mut locs = re.capture_locations(); + /// assert!(re.captures_read(&mut locs, b"Padron").is_some()); + /// assert_eq!(locs.get(0), Some((0, 6))); + /// assert_eq!(locs.get(1), Some((0, 1))); + /// assert_eq!(locs.get(2), Some((1, 2))); + /// assert_eq!(locs.get(3), Some((2, 6))); + /// ``` + #[inline] + pub fn capture_locations(&self) -> CaptureLocations { + CaptureLocations(self.meta.create_captures()) + } + + /// An alias for `capture_locations` to preserve backward compatibility. + /// + /// The `regex-capi` crate uses this method, so to avoid breaking that + /// crate, we continue to export it as an undocumented API. + #[doc(hidden)] + #[inline] + pub fn locations(&self) -> CaptureLocations { + self.capture_locations() + } +} + +/// Represents a single match of a regex in a haystack. +/// +/// A `Match` contains both the start and end byte offsets of the match and the +/// actual substring corresponding to the range of those byte offsets. It is +/// guaranteed that `start <= end`. When `start == end`, the match is empty. +/// +/// Unlike the top-level `Match` type, this `Match` type is produced by APIs +/// that search `&[u8]` haystacks. This means that the offsets in a `Match` can +/// point to anywhere in the haystack, including in a place that splits the +/// UTF-8 encoding of a Unicode scalar value. +/// +/// The lifetime parameter `'h` refers to the lifetime of the matched of the +/// haystack that this match was produced from. +/// +/// # Numbering +/// +/// The byte offsets in a `Match` form a half-open interval. That is, the +/// start of the range is inclusive and the end of the range is exclusive. +/// For example, given a haystack `abcFOOxyz` and a match of `FOO`, its byte +/// offset range starts at `3` and ends at `6`. `3` corresponds to `F` and +/// `6` corresponds to `x`, which is one past the end of the match. This +/// corresponds to the same kind of slicing that Rust uses. +/// +/// For more on why this was chosen over other schemes (aside from being +/// consistent with how Rust the language works), see [this discussion] and +/// [Dijkstra's note on a related topic][note]. +/// +/// [this discussion]: https://github.com/rust-lang/regex/discussions/866 +/// [note]: https://www.cs.utexas.edu/users/EWD/transcriptions/EWD08xx/EWD831.html +/// +/// # Example +/// +/// This example shows the value of each of the methods on `Match` for a +/// particular search. +/// +/// ``` +/// use regex::bytes::Regex; +/// +/// let re = Regex::new(r"\p{Greek}+").unwrap(); +/// let hay = "Greek: αβγδ".as_bytes(); +/// let m = re.find(hay).unwrap(); +/// assert_eq!(7, m.start()); +/// assert_eq!(15, m.end()); +/// assert!(!m.is_empty()); +/// assert_eq!(8, m.len()); +/// assert_eq!(7..15, m.range()); +/// assert_eq!("αβγδ".as_bytes(), m.as_bytes()); +/// ``` +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Match<'h> { + haystack: &'h [u8], + start: usize, + end: usize, +} + +impl<'h> Match<'h> { + /// Returns the byte offset of the start of the match in the haystack. The + /// start of the match corresponds to the position where the match begins + /// and includes the first byte in the match. + /// + /// It is guaranteed that `Match::start() <= Match::end()`. + /// + /// Unlike the top-level `Match` type, the start offset may appear anywhere + /// in the haystack. This includes between the code units of a UTF-8 + /// encoded Unicode scalar value. + #[inline] + pub fn start(&self) -> usize { + self.start + } + + /// Returns the byte offset of the end of the match in the haystack. The + /// end of the match corresponds to the byte immediately following the last + /// byte in the match. This means that `&slice[start..end]` works as one + /// would expect. + /// + /// It is guaranteed that `Match::start() <= Match::end()`. + /// + /// Unlike the top-level `Match` type, the start offset may appear anywhere + /// in the haystack. This includes between the code units of a UTF-8 + /// encoded Unicode scalar value. + #[inline] + pub fn end(&self) -> usize { + self.end + } + + /// Returns true if and only if this match has a length of zero. + /// + /// Note that an empty match can only occur when the regex itself can + /// match the empty string. Here are some examples of regexes that can + /// all match the empty string: `^`, `^$`, `\b`, `a?`, `a*`, `a{0}`, + /// `(foo|\d+|quux)?`. + #[inline] + pub fn is_empty(&self) -> bool { + self.start == self.end + } + + /// Returns the length, in bytes, of this match. + #[inline] + pub fn len(&self) -> usize { + self.end - self.start + } + + /// Returns the range over the starting and ending byte offsets of the + /// match in the haystack. + #[inline] + pub fn range(&self) -> core::ops::Range<usize> { + self.start..self.end + } + + /// Returns the substring of the haystack that matched. + #[inline] + pub fn as_bytes(&self) -> &'h [u8] { + &self.haystack[self.range()] + } + + /// Creates a new match from the given haystack and byte offsets. + #[inline] + fn new(haystack: &'h [u8], start: usize, end: usize) -> Match<'h> { + Match { haystack, start, end } + } +} + +impl<'h> core::fmt::Debug for Match<'h> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + use regex_automata::util::escape::DebugHaystack; + + let mut fmt = f.debug_struct("Match"); + fmt.field("start", &self.start) + .field("end", &self.end) + .field("bytes", &DebugHaystack(&self.as_bytes())); + + fmt.finish() + } +} + +impl<'h> From<Match<'h>> for &'h [u8] { + fn from(m: Match<'h>) -> &'h [u8] { + m.as_bytes() + } +} + +impl<'h> From<Match<'h>> for core::ops::Range<usize> { + fn from(m: Match<'h>) -> core::ops::Range<usize> { + m.range() + } +} + +/// Represents the capture groups for a single match. +/// +/// Capture groups refer to parts of a regex enclosed in parentheses. They +/// can be optionally named. The purpose of capture groups is to be able to +/// reference different parts of a match based on the original pattern. In +/// essence, a `Captures` is a container of [`Match`] values for each group +/// that participated in a regex match. Each `Match` can be looked up by either +/// its capture group index or name (if it has one). +/// +/// For example, say you want to match the individual letters in a 5-letter +/// word: +/// +/// ```text +/// (?<first>\w)(\w)(?:\w)\w(?<last>\w) +/// ``` +/// +/// This regex has 4 capture groups: +/// +/// * The group at index `0` corresponds to the overall match. It is always +/// present in every match and never has a name. +/// * The group at index `1` with name `first` corresponding to the first +/// letter. +/// * The group at index `2` with no name corresponding to the second letter. +/// * The group at index `3` with name `last` corresponding to the fifth and +/// last letter. +/// +/// Notice that `(?:\w)` was not listed above as a capture group despite it +/// being enclosed in parentheses. That's because `(?:pattern)` is a special +/// syntax that permits grouping but *without* capturing. The reason for not +/// treating it as a capture is that tracking and reporting capture groups +/// requires additional state that may lead to slower searches. So using as few +/// capture groups as possible can help performance. (Although the difference +/// in performance of a couple of capture groups is likely immaterial.) +/// +/// Values with this type are created by [`Regex::captures`] or +/// [`Regex::captures_iter`]. +/// +/// `'h` is the lifetime of the haystack that these captures were matched from. +/// +/// # Example +/// +/// ``` +/// use regex::bytes::Regex; +/// +/// let re = Regex::new(r"(?<first>\w)(\w)(?:\w)\w(?<last>\w)").unwrap(); +/// let caps = re.captures(b"toady").unwrap(); +/// assert_eq!(b"toady", &caps[0]); +/// assert_eq!(b"t", &caps["first"]); +/// assert_eq!(b"o", &caps[2]); +/// assert_eq!(b"y", &caps["last"]); +/// ``` +pub struct Captures<'h> { + haystack: &'h [u8], + caps: captures::Captures, + static_captures_len: Option<usize>, +} + +impl<'h> Captures<'h> { + /// Returns the `Match` associated with the capture group at index `i`. If + /// `i` does not correspond to a capture group, or if the capture group did + /// not participate in the match, then `None` is returned. + /// + /// When `i == 0`, this is guaranteed to return a non-`None` value. + /// + /// # Examples + /// + /// Get the substring that matched with a default of an empty string if the + /// group didn't participate in the match: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"[a-z]+(?:([0-9]+)|([A-Z]+))").unwrap(); + /// let caps = re.captures(b"abc123").unwrap(); + /// + /// let substr1 = caps.get(1).map_or(&b""[..], |m| m.as_bytes()); + /// let substr2 = caps.get(2).map_or(&b""[..], |m| m.as_bytes()); + /// assert_eq!(substr1, b"123"); + /// assert_eq!(substr2, b""); + /// ``` + #[inline] + pub fn get(&self, i: usize) -> Option<Match<'h>> { + self.caps + .get_group(i) + .map(|sp| Match::new(self.haystack, sp.start, sp.end)) + } + + /// Return the overall match for the capture. + /// + /// This returns the match for index `0`. That is it is equivalent to + /// `m.get(0).unwrap()` + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"[a-z]+([0-9]+)").unwrap(); + /// let caps = re.captures(b" abc123-def").unwrap(); + /// + /// assert_eq!(caps.get_match().as_bytes(), b"abc123"); + /// ``` + #[inline] + pub fn get_match(&self) -> Match<'h> { + self.get(0).unwrap() + } + + /// Returns the `Match` associated with the capture group named `name`. If + /// `name` isn't a valid capture group or it refers to a group that didn't + /// match, then `None` is returned. + /// + /// Note that unlike `caps["name"]`, this returns a `Match` whose lifetime + /// matches the lifetime of the haystack in this `Captures` value. + /// Conversely, the substring returned by `caps["name"]` has a lifetime + /// of the `Captures` value, which is likely shorter than the lifetime of + /// the haystack. In some cases, it may be necessary to use this method to + /// access the matching substring instead of the `caps["name"]` notation. + /// + /// # Examples + /// + /// Get the substring that matched with a default of an empty string if the + /// group didn't participate in the match: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new( + /// r"[a-z]+(?:(?<numbers>[0-9]+)|(?<letters>[A-Z]+))", + /// ).unwrap(); + /// let caps = re.captures(b"abc123").unwrap(); + /// + /// let numbers = caps.name("numbers").map_or(&b""[..], |m| m.as_bytes()); + /// let letters = caps.name("letters").map_or(&b""[..], |m| m.as_bytes()); + /// assert_eq!(numbers, b"123"); + /// assert_eq!(letters, b""); + /// ``` + #[inline] + pub fn name(&self, name: &str) -> Option<Match<'h>> { + self.caps + .get_group_by_name(name) + .map(|sp| Match::new(self.haystack, sp.start, sp.end)) + } + + /// This is a convenience routine for extracting the substrings + /// corresponding to matching capture groups. + /// + /// This returns a tuple where the first element corresponds to the full + /// substring of the haystack that matched the regex. The second element is + /// an array of substrings, with each corresponding to the substring that + /// matched for a particular capture group. + /// + /// # Panics + /// + /// This panics if the number of possible matching groups in this + /// `Captures` value is not fixed to `N` in all circumstances. + /// More precisely, this routine only works when `N` is equivalent to + /// [`Regex::static_captures_len`]. + /// + /// Stated more plainly, if the number of matching capture groups in a + /// regex can vary from match to match, then this function always panics. + /// + /// For example, `(a)(b)|(c)` could produce two matching capture groups + /// or one matching capture group for any given match. Therefore, one + /// cannot use `extract` with such a pattern. + /// + /// But a pattern like `(a)(b)|(c)(d)` can be used with `extract` because + /// the number of capture groups in every match is always equivalent, + /// even if the capture _indices_ in each match are not. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"([0-9]{4})-([0-9]{2})-([0-9]{2})").unwrap(); + /// let hay = b"On 2010-03-14, I became a Tennessee lamb."; + /// let Some((full, [year, month, day])) = + /// re.captures(hay).map(|caps| caps.extract()) else { return }; + /// assert_eq!(b"2010-03-14", full); + /// assert_eq!(b"2010", year); + /// assert_eq!(b"03", month); + /// assert_eq!(b"14", day); + /// ``` + /// + /// # Example: iteration + /// + /// This example shows how to use this method when iterating over all + /// `Captures` matches in a haystack. + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"([0-9]{4})-([0-9]{2})-([0-9]{2})").unwrap(); + /// let hay = b"1973-01-05, 1975-08-25 and 1980-10-18"; + /// + /// let mut dates: Vec<(&[u8], &[u8], &[u8])> = vec![]; + /// for (_, [y, m, d]) in re.captures_iter(hay).map(|c| c.extract()) { + /// dates.push((y, m, d)); + /// } + /// assert_eq!(dates, vec![ + /// (&b"1973"[..], &b"01"[..], &b"05"[..]), + /// (&b"1975"[..], &b"08"[..], &b"25"[..]), + /// (&b"1980"[..], &b"10"[..], &b"18"[..]), + /// ]); + /// ``` + /// + /// # Example: parsing different formats + /// + /// This API is particularly useful when you need to extract a particular + /// value that might occur in a different format. Consider, for example, + /// an identifier that might be in double quotes or single quotes: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r#"id:(?:"([^"]+)"|'([^']+)')"#).unwrap(); + /// let hay = br#"The first is id:"foo" and the second is id:'bar'."#; + /// let mut ids = vec![]; + /// for (_, [id]) in re.captures_iter(hay).map(|c| c.extract()) { + /// ids.push(id); + /// } + /// assert_eq!(ids, vec![b"foo", b"bar"]); + /// ``` + pub fn extract<const N: usize>(&self) -> (&'h [u8], [&'h [u8]; N]) { + let len = self + .static_captures_len + .expect("number of capture groups can vary in a match") + .checked_sub(1) + .expect("number of groups is always greater than zero"); + assert_eq!(N, len, "asked for {N} groups, but must ask for {len}"); + // The regex-automata variant of extract is a bit more permissive. + // It doesn't require the number of matching capturing groups to be + // static, and you can even request fewer groups than what's there. So + // this is guaranteed to never panic because we've asserted above that + // the user has requested precisely the number of groups that must be + // present in any match for this regex. + self.caps.extract_bytes(self.haystack) + } + + /// Expands all instances of `$ref` in `replacement` to the corresponding + /// capture group, and writes them to the `dst` buffer given. A `ref` can + /// be a capture group index or a name. If `ref` doesn't refer to a capture + /// group that participated in the match, then it is replaced with the + /// empty string. + /// + /// # Format + /// + /// The format of the replacement string supports two different kinds of + /// capture references: unbraced and braced. + /// + /// For the unbraced format, the format supported is `$ref` where `name` + /// can be any character in the class `[0-9A-Za-z_]`. `ref` is always + /// the longest possible parse. So for example, `$1a` corresponds to the + /// capture group named `1a` and not the capture group at index `1`. If + /// `ref` matches `^[0-9]+$`, then it is treated as a capture group index + /// itself and not a name. + /// + /// For the braced format, the format supported is `${ref}` where `ref` can + /// be any sequence of bytes except for `}`. If no closing brace occurs, + /// then it is not considered a capture reference. As with the unbraced + /// format, if `ref` matches `^[0-9]+$`, then it is treated as a capture + /// group index and not a name. + /// + /// The braced format is useful for exerting precise control over the name + /// of the capture reference. For example, `${1}a` corresponds to the + /// capture group reference `1` followed by the letter `a`, where as `$1a` + /// (as mentioned above) corresponds to the capture group reference `1a`. + /// The braced format is also useful for expressing capture group names + /// that use characters not supported by the unbraced format. For example, + /// `${foo[bar].baz}` refers to the capture group named `foo[bar].baz`. + /// + /// If a capture group reference is found and it does not refer to a valid + /// capture group, then it will be replaced with the empty string. + /// + /// To write a literal `$`, use `$$`. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new( + /// r"(?<day>[0-9]{2})-(?<month>[0-9]{2})-(?<year>[0-9]{4})", + /// ).unwrap(); + /// let hay = b"On 14-03-2010, I became a Tennessee lamb."; + /// let caps = re.captures(hay).unwrap(); + /// + /// let mut dst = vec![]; + /// caps.expand(b"year=$year, month=$month, day=$day", &mut dst); + /// assert_eq!(dst, b"year=2010, month=03, day=14"); + /// ``` + #[inline] + pub fn expand(&self, replacement: &[u8], dst: &mut Vec<u8>) { + self.caps.interpolate_bytes_into(self.haystack, replacement, dst); + } + + /// Returns an iterator over all capture groups. This includes both + /// matching and non-matching groups. + /// + /// The iterator always yields at least one matching group: the first group + /// (at index `0`) with no name. Subsequent groups are returned in the order + /// of their opening parenthesis in the regex. + /// + /// The elements yielded have type `Option<Match<'h>>`, where a non-`None` + /// value is present if the capture group matches. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(\w)(\d)?(\w)").unwrap(); + /// let caps = re.captures(b"AZ").unwrap(); + /// + /// let mut it = caps.iter(); + /// assert_eq!(it.next().unwrap().map(|m| m.as_bytes()), Some(&b"AZ"[..])); + /// assert_eq!(it.next().unwrap().map(|m| m.as_bytes()), Some(&b"A"[..])); + /// assert_eq!(it.next().unwrap().map(|m| m.as_bytes()), None); + /// assert_eq!(it.next().unwrap().map(|m| m.as_bytes()), Some(&b"Z"[..])); + /// assert_eq!(it.next(), None); + /// ``` + #[inline] + pub fn iter<'c>(&'c self) -> SubCaptureMatches<'c, 'h> { + SubCaptureMatches { haystack: self.haystack, it: self.caps.iter() } + } + + /// Returns the total number of capture groups. This includes both + /// matching and non-matching groups. + /// + /// The length returned is always equivalent to the number of elements + /// yielded by [`Captures::iter`]. Consequently, the length is always + /// greater than zero since every `Captures` value always includes the + /// match for the entire regex. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(\w)(\d)?(\w)").unwrap(); + /// let caps = re.captures(b"AZ").unwrap(); + /// assert_eq!(caps.len(), 4); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.caps.group_len() + } +} + +impl<'h> core::fmt::Debug for Captures<'h> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + /// A little helper type to provide a nice map-like debug + /// representation for our capturing group spans. + /// + /// regex-automata has something similar, but it includes the pattern + /// ID in its debug output, which is confusing. It also doesn't include + /// that strings that match because a regex-automata `Captures` doesn't + /// borrow the haystack. + struct CapturesDebugMap<'a> { + caps: &'a Captures<'a>, + } + + impl<'a> core::fmt::Debug for CapturesDebugMap<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut map = f.debug_map(); + let names = + self.caps.caps.group_info().pattern_names(PatternID::ZERO); + for (group_index, maybe_name) in names.enumerate() { + let key = Key(group_index, maybe_name); + match self.caps.get(group_index) { + None => map.entry(&key, &None::<()>), + Some(mat) => map.entry(&key, &Value(mat)), + }; + } + map.finish() + } + } + + struct Key<'a>(usize, Option<&'a str>); + + impl<'a> core::fmt::Debug for Key<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{}", self.0)?; + if let Some(name) = self.1 { + write!(f, "/{name:?}")?; + } + Ok(()) + } + } + + struct Value<'a>(Match<'a>); + + impl<'a> core::fmt::Debug for Value<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + use regex_automata::util::escape::DebugHaystack; + + write!( + f, + "{}..{}/{:?}", + self.0.start(), + self.0.end(), + DebugHaystack(self.0.as_bytes()) + ) + } + } + + f.debug_tuple("Captures") + .field(&CapturesDebugMap { caps: self }) + .finish() + } +} + +/// Get a matching capture group's haystack substring by index. +/// +/// The haystack substring returned can't outlive the `Captures` object if this +/// method is used, because of how `Index` is defined (normally `a[i]` is part +/// of `a` and can't outlive it). To work around this limitation, do that, use +/// [`Captures::get`] instead. +/// +/// `'h` is the lifetime of the matched haystack, but the lifetime of the +/// `&str` returned by this implementation is the lifetime of the `Captures` +/// value itself. +/// +/// # Panics +/// +/// If there is no matching group at the given index. +impl<'h> core::ops::Index<usize> for Captures<'h> { + type Output = [u8]; + + // The lifetime is written out to make it clear that the &str returned + // does NOT have a lifetime equivalent to 'h. + fn index<'a>(&'a self, i: usize) -> &'a [u8] { + self.get(i) + .map(|m| m.as_bytes()) + .unwrap_or_else(|| panic!("no group at index '{i}'")) + } +} + +/// Get a matching capture group's haystack substring by name. +/// +/// The haystack substring returned can't outlive the `Captures` object if this +/// method is used, because of how `Index` is defined (normally `a[i]` is part +/// of `a` and can't outlive it). To work around this limitation, do that, use +/// [`Captures::name`] instead. +/// +/// `'h` is the lifetime of the matched haystack, but the lifetime of the +/// `&str` returned by this implementation is the lifetime of the `Captures` +/// value itself. +/// +/// `'n` is the lifetime of the group name used to index the `Captures` value. +/// +/// # Panics +/// +/// If there is no matching group at the given name. +impl<'h, 'n> core::ops::Index<&'n str> for Captures<'h> { + type Output = [u8]; + + fn index<'a>(&'a self, name: &'n str) -> &'a [u8] { + self.name(name) + .map(|m| m.as_bytes()) + .unwrap_or_else(|| panic!("no group named '{name}'")) + } +} + +/// A low level representation of the byte offsets of each capture group. +/// +/// You can think of this as a lower level [`Captures`], where this type does +/// not support named capturing groups directly and it does not borrow the +/// haystack that these offsets were matched on. +/// +/// Primarily, this type is useful when using the lower level `Regex` APIs such +/// as [`Regex::captures_read`], which permits amortizing the allocation in +/// which capture match offsets are stored. +/// +/// In order to build a value of this type, you'll need to call the +/// [`Regex::capture_locations`] method. The value returned can then be reused +/// in subsequent searches for that regex. Using it for other regexes may +/// result in a panic or otherwise incorrect results. +/// +/// # Example +/// +/// This example shows how to create and use `CaptureLocations` in a search. +/// +/// ``` +/// use regex::bytes::Regex; +/// +/// let re = Regex::new(r"(?<first>\w+)\s+(?<last>\w+)").unwrap(); +/// let mut locs = re.capture_locations(); +/// let m = re.captures_read(&mut locs, b"Bruce Springsteen").unwrap(); +/// assert_eq!(0..17, m.range()); +/// assert_eq!(Some((0, 17)), locs.get(0)); +/// assert_eq!(Some((0, 5)), locs.get(1)); +/// assert_eq!(Some((6, 17)), locs.get(2)); +/// +/// // Asking for an invalid capture group always returns None. +/// assert_eq!(None, locs.get(3)); +/// # // literals are too big for 32-bit usize: #1041 +/// # #[cfg(target_pointer_width = "64")] +/// assert_eq!(None, locs.get(34973498648)); +/// # #[cfg(target_pointer_width = "64")] +/// assert_eq!(None, locs.get(9944060567225171988)); +/// ``` +#[derive(Clone, Debug)] +pub struct CaptureLocations(captures::Captures); + +/// A type alias for `CaptureLocations` for backwards compatibility. +/// +/// Previously, we exported `CaptureLocations` as `Locations` in an +/// undocumented API. To prevent breaking that code (e.g., in `regex-capi`), +/// we continue re-exporting the same undocumented API. +#[doc(hidden)] +pub type Locations = CaptureLocations; + +impl CaptureLocations { + /// Returns the start and end byte offsets of the capture group at index + /// `i`. This returns `None` if `i` is not a valid capture group or if the + /// capture group did not match. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(?<first>\w+)\s+(?<last>\w+)").unwrap(); + /// let mut locs = re.capture_locations(); + /// re.captures_read(&mut locs, b"Bruce Springsteen").unwrap(); + /// assert_eq!(Some((0, 17)), locs.get(0)); + /// assert_eq!(Some((0, 5)), locs.get(1)); + /// assert_eq!(Some((6, 17)), locs.get(2)); + /// ``` + #[inline] + pub fn get(&self, i: usize) -> Option<(usize, usize)> { + self.0.get_group(i).map(|sp| (sp.start, sp.end)) + } + + /// Returns the total number of capture groups (even if they didn't match). + /// That is, the length returned is unaffected by the result of a search. + /// + /// This is always at least `1` since every regex has at least `1` + /// capturing group that corresponds to the entire match. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"(?<first>\w+)\s+(?<last>\w+)").unwrap(); + /// let mut locs = re.capture_locations(); + /// assert_eq!(3, locs.len()); + /// re.captures_read(&mut locs, b"Bruce Springsteen").unwrap(); + /// assert_eq!(3, locs.len()); + /// ``` + /// + /// Notice that the length is always at least `1`, regardless of the regex: + /// + /// ``` + /// use regex::bytes::Regex; + /// + /// let re = Regex::new(r"").unwrap(); + /// let locs = re.capture_locations(); + /// assert_eq!(1, locs.len()); + /// + /// // [a&&b] is a regex that never matches anything. + /// let re = Regex::new(r"[a&&b]").unwrap(); + /// let locs = re.capture_locations(); + /// assert_eq!(1, locs.len()); + /// ``` + #[inline] + pub fn len(&self) -> usize { + // self.0.group_len() returns 0 if the underlying captures doesn't + // represent a match, but the behavior guaranteed for this method is + // that the length doesn't change based on a match or not. + self.0.group_info().group_len(PatternID::ZERO) + } + + /// An alias for the `get` method for backwards compatibility. + /// + /// Previously, we exported `get` as `pos` in an undocumented API. To + /// prevent breaking that code (e.g., in `regex-capi`), we continue + /// re-exporting the same undocumented API. + #[doc(hidden)] + #[inline] + pub fn pos(&self, i: usize) -> Option<(usize, usize)> { + self.get(i) + } +} + +/// An iterator over all non-overlapping matches in a haystack. +/// +/// This iterator yields [`Match`] values. The iterator stops when no more +/// matches can be found. +/// +/// `'r` is the lifetime of the compiled regular expression and `'h` is the +/// lifetime of the haystack. +/// +/// This iterator is created by [`Regex::find_iter`]. +/// +/// # Time complexity +/// +/// Note that since an iterator runs potentially many searches on the haystack +/// and since each search has worst case `O(m * n)` time complexity, the +/// overall worst case time complexity for iteration is `O(m * n^2)`. +#[derive(Debug)] +pub struct Matches<'r, 'h> { + haystack: &'h [u8], + it: meta::FindMatches<'r, 'h>, +} + +impl<'r, 'h> Iterator for Matches<'r, 'h> { + type Item = Match<'h>; + + #[inline] + fn next(&mut self) -> Option<Match<'h>> { + self.it + .next() + .map(|sp| Match::new(self.haystack, sp.start(), sp.end())) + } + + #[inline] + fn count(self) -> usize { + // This can actually be up to 2x faster than calling `next()` until + // completion, because counting matches when using a DFA only requires + // finding the end of each match. But returning a `Match` via `next()` + // requires the start of each match which, with a DFA, requires a + // reverse forward scan to find it. + self.it.count() + } +} + +impl<'r, 'h> core::iter::FusedIterator for Matches<'r, 'h> {} + +/// An iterator over all non-overlapping capture matches in a haystack. +/// +/// This iterator yields [`Captures`] values. The iterator stops when no more +/// matches can be found. +/// +/// `'r` is the lifetime of the compiled regular expression and `'h` is the +/// lifetime of the matched string. +/// +/// This iterator is created by [`Regex::captures_iter`]. +/// +/// # Time complexity +/// +/// Note that since an iterator runs potentially many searches on the haystack +/// and since each search has worst case `O(m * n)` time complexity, the +/// overall worst case time complexity for iteration is `O(m * n^2)`. +#[derive(Debug)] +pub struct CaptureMatches<'r, 'h> { + haystack: &'h [u8], + it: meta::CapturesMatches<'r, 'h>, +} + +impl<'r, 'h> Iterator for CaptureMatches<'r, 'h> { + type Item = Captures<'h>; + + #[inline] + fn next(&mut self) -> Option<Captures<'h>> { + let static_captures_len = self.it.regex().static_captures_len(); + self.it.next().map(|caps| Captures { + haystack: self.haystack, + caps, + static_captures_len, + }) + } + + #[inline] + fn count(self) -> usize { + // This can actually be up to 2x faster than calling `next()` until + // completion, because counting matches when using a DFA only requires + // finding the end of each match. But returning a `Match` via `next()` + // requires the start of each match which, with a DFA, requires a + // reverse forward scan to find it. + self.it.count() + } +} + +impl<'r, 'h> core::iter::FusedIterator for CaptureMatches<'r, 'h> {} + +/// An iterator over all substrings delimited by a regex match. +/// +/// `'r` is the lifetime of the compiled regular expression and `'h` is the +/// lifetime of the byte string being split. +/// +/// This iterator is created by [`Regex::split`]. +/// +/// # Time complexity +/// +/// Note that since an iterator runs potentially many searches on the haystack +/// and since each search has worst case `O(m * n)` time complexity, the +/// overall worst case time complexity for iteration is `O(m * n^2)`. +#[derive(Debug)] +pub struct Split<'r, 'h> { + haystack: &'h [u8], + it: meta::Split<'r, 'h>, +} + +impl<'r, 'h> Iterator for Split<'r, 'h> { + type Item = &'h [u8]; + + #[inline] + fn next(&mut self) -> Option<&'h [u8]> { + self.it.next().map(|span| &self.haystack[span]) + } +} + +impl<'r, 'h> core::iter::FusedIterator for Split<'r, 'h> {} + +/// An iterator over at most `N` substrings delimited by a regex match. +/// +/// The last substring yielded by this iterator will be whatever remains after +/// `N-1` splits. +/// +/// `'r` is the lifetime of the compiled regular expression and `'h` is the +/// lifetime of the byte string being split. +/// +/// This iterator is created by [`Regex::splitn`]. +/// +/// # Time complexity +/// +/// Note that since an iterator runs potentially many searches on the haystack +/// and since each search has worst case `O(m * n)` time complexity, the +/// overall worst case time complexity for iteration is `O(m * n^2)`. +/// +/// Although note that the worst case time here has an upper bound given +/// by the `limit` parameter to [`Regex::splitn`]. +#[derive(Debug)] +pub struct SplitN<'r, 'h> { + haystack: &'h [u8], + it: meta::SplitN<'r, 'h>, +} + +impl<'r, 'h> Iterator for SplitN<'r, 'h> { + type Item = &'h [u8]; + + #[inline] + fn next(&mut self) -> Option<&'h [u8]> { + self.it.next().map(|span| &self.haystack[span]) + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + self.it.size_hint() + } +} + +impl<'r, 'h> core::iter::FusedIterator for SplitN<'r, 'h> {} + +/// An iterator over the names of all capture groups in a regex. +/// +/// This iterator yields values of type `Option<&str>` in order of the opening +/// capture group parenthesis in the regex pattern. `None` is yielded for +/// groups with no name. The first element always corresponds to the implicit +/// and unnamed group for the overall match. +/// +/// `'r` is the lifetime of the compiled regular expression. +/// +/// This iterator is created by [`Regex::capture_names`]. +#[derive(Clone, Debug)] +pub struct CaptureNames<'r>(captures::GroupInfoPatternNames<'r>); + +impl<'r> Iterator for CaptureNames<'r> { + type Item = Option<&'r str>; + + #[inline] + fn next(&mut self) -> Option<Option<&'r str>> { + self.0.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + self.0.size_hint() + } + + #[inline] + fn count(self) -> usize { + self.0.count() + } +} + +impl<'r> ExactSizeIterator for CaptureNames<'r> {} + +impl<'r> core::iter::FusedIterator for CaptureNames<'r> {} + +/// An iterator over all group matches in a [`Captures`] value. +/// +/// This iterator yields values of type `Option<Match<'h>>`, where `'h` is the +/// lifetime of the haystack that the matches are for. The order of elements +/// yielded corresponds to the order of the opening parenthesis for the group +/// in the regex pattern. `None` is yielded for groups that did not participate +/// in the match. +/// +/// The first element always corresponds to the implicit group for the overall +/// match. Since this iterator is created by a [`Captures`] value, and a +/// `Captures` value is only created when a match occurs, it follows that the +/// first element yielded by this iterator is guaranteed to be non-`None`. +/// +/// The lifetime `'c` corresponds to the lifetime of the `Captures` value that +/// created this iterator, and the lifetime `'h` corresponds to the originally +/// matched haystack. +#[derive(Clone, Debug)] +pub struct SubCaptureMatches<'c, 'h> { + haystack: &'h [u8], + it: captures::CapturesPatternIter<'c>, +} + +impl<'c, 'h> Iterator for SubCaptureMatches<'c, 'h> { + type Item = Option<Match<'h>>; + + #[inline] + fn next(&mut self) -> Option<Option<Match<'h>>> { + self.it.next().map(|group| { + group.map(|sp| Match::new(self.haystack, sp.start, sp.end)) + }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + self.it.size_hint() + } + + #[inline] + fn count(self) -> usize { + self.it.count() + } +} + +impl<'c, 'h> ExactSizeIterator for SubCaptureMatches<'c, 'h> {} + +impl<'c, 'h> core::iter::FusedIterator for SubCaptureMatches<'c, 'h> {} + +/// A trait for types that can be used to replace matches in a haystack. +/// +/// In general, users of this crate shouldn't need to implement this trait, +/// since implementations are already provided for `&[u8]` along with other +/// variants of byte string types, as well as `FnMut(&Captures) -> Vec<u8>` (or +/// any `FnMut(&Captures) -> T` where `T: AsRef<[u8]>`). Those cover most use +/// cases, but callers can implement this trait directly if necessary. +/// +/// # Example +/// +/// This example shows a basic implementation of the `Replacer` trait. This can +/// be done much more simply using the replacement byte string interpolation +/// support (e.g., `$first $last`), but this approach avoids needing to parse +/// the replacement byte string at all. +/// +/// ``` +/// use regex::bytes::{Captures, Regex, Replacer}; +/// +/// struct NameSwapper; +/// +/// impl Replacer for NameSwapper { +/// fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { +/// dst.extend_from_slice(&caps["first"]); +/// dst.extend_from_slice(b" "); +/// dst.extend_from_slice(&caps["last"]); +/// } +/// } +/// +/// let re = Regex::new(r"(?<last>[^,\s]+),\s+(?<first>\S+)").unwrap(); +/// let result = re.replace(b"Springsteen, Bruce", NameSwapper); +/// assert_eq!(result, &b"Bruce Springsteen"[..]); +/// ``` +pub trait Replacer { + /// Appends possibly empty data to `dst` to replace the current match. + /// + /// The current match is represented by `caps`, which is guaranteed to have + /// a match at capture group `0`. + /// + /// For example, a no-op replacement would be + /// `dst.extend_from_slice(&caps[0])`. + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>); + + /// Return a fixed unchanging replacement byte string. + /// + /// When doing replacements, if access to [`Captures`] is not needed (e.g., + /// the replacement byte string does not need `$` expansion), then it can + /// be beneficial to avoid finding sub-captures. + /// + /// In general, this is called once for every call to a replacement routine + /// such as [`Regex::replace_all`]. + fn no_expansion<'r>(&'r mut self) -> Option<Cow<'r, [u8]>> { + None + } + + /// Returns a type that implements `Replacer`, but that borrows and wraps + /// this `Replacer`. + /// + /// This is useful when you want to take a generic `Replacer` (which might + /// not be cloneable) and use it without consuming it, so it can be used + /// more than once. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::{Regex, Replacer}; + /// + /// fn replace_all_twice<R: Replacer>( + /// re: Regex, + /// src: &[u8], + /// mut rep: R, + /// ) -> Vec<u8> { + /// let dst = re.replace_all(src, rep.by_ref()); + /// let dst = re.replace_all(&dst, rep.by_ref()); + /// dst.into_owned() + /// } + /// ``` + fn by_ref<'r>(&'r mut self) -> ReplacerRef<'r, Self> { + ReplacerRef(self) + } +} + +impl<'a, const N: usize> Replacer for &'a [u8; N] { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + caps.expand(&**self, dst); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, [u8]>> { + no_expansion(self) + } +} + +impl<const N: usize> Replacer for [u8; N] { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + caps.expand(&*self, dst); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, [u8]>> { + no_expansion(self) + } +} + +impl<'a> Replacer for &'a [u8] { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + caps.expand(*self, dst); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, [u8]>> { + no_expansion(self) + } +} + +impl<'a> Replacer for &'a Vec<u8> { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + caps.expand(*self, dst); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, [u8]>> { + no_expansion(self) + } +} + +impl Replacer for Vec<u8> { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + caps.expand(self, dst); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, [u8]>> { + no_expansion(self) + } +} + +impl<'a> Replacer for Cow<'a, [u8]> { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + caps.expand(self.as_ref(), dst); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, [u8]>> { + no_expansion(self) + } +} + +impl<'a> Replacer for &'a Cow<'a, [u8]> { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + caps.expand(self.as_ref(), dst); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, [u8]>> { + no_expansion(self) + } +} + +impl<F, T> Replacer for F +where + F: FnMut(&Captures<'_>) -> T, + T: AsRef<[u8]>, +{ + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + dst.extend_from_slice((*self)(caps).as_ref()); + } +} + +/// A by-reference adaptor for a [`Replacer`]. +/// +/// This permits reusing the same `Replacer` value in multiple calls to a +/// replacement routine like [`Regex::replace_all`]. +/// +/// This type is created by [`Replacer::by_ref`]. +#[derive(Debug)] +pub struct ReplacerRef<'a, R: ?Sized>(&'a mut R); + +impl<'a, R: Replacer + ?Sized + 'a> Replacer for ReplacerRef<'a, R> { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut Vec<u8>) { + self.0.replace_append(caps, dst) + } + + fn no_expansion<'r>(&'r mut self) -> Option<Cow<'r, [u8]>> { + self.0.no_expansion() + } +} + +/// A helper type for forcing literal string replacement. +/// +/// It can be used with routines like [`Regex::replace`] and +/// [`Regex::replace_all`] to do a literal string replacement without expanding +/// `$name` to their corresponding capture groups. This can be both convenient +/// (to avoid escaping `$`, for example) and faster (since capture groups +/// don't need to be found). +/// +/// `'s` is the lifetime of the literal string to use. +/// +/// # Example +/// +/// ``` +/// use regex::bytes::{NoExpand, Regex}; +/// +/// let re = Regex::new(r"(?<last>[^,\s]+),\s+(\S+)").unwrap(); +/// let result = re.replace(b"Springsteen, Bruce", NoExpand(b"$2 $last")); +/// assert_eq!(result, &b"$2 $last"[..]); +/// ``` +#[derive(Clone, Debug)] +pub struct NoExpand<'s>(pub &'s [u8]); + +impl<'s> Replacer for NoExpand<'s> { + fn replace_append(&mut self, _: &Captures<'_>, dst: &mut Vec<u8>) { + dst.extend_from_slice(self.0); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, [u8]>> { + Some(Cow::Borrowed(self.0)) + } +} + +/// Quickly checks the given replacement string for whether interpolation +/// should be done on it. It returns `None` if a `$` was found anywhere in the +/// given string, which suggests interpolation needs to be done. But if there's +/// no `$` anywhere, then interpolation definitely does not need to be done. In +/// that case, the given string is returned as a borrowed `Cow`. +/// +/// This is meant to be used to implement the `Replacer::no_expansion` method +/// in its various trait impls. +fn no_expansion<T: AsRef<[u8]>>(replacement: &T) -> Option<Cow<'_, [u8]>> { + let replacement = replacement.as_ref(); + match crate::find_byte::find_byte(b'$', replacement) { + Some(_) => None, + None => Some(Cow::Borrowed(replacement)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::format; + + #[test] + fn test_match_properties() { + let haystack = b"Hello, world!"; + let m = Match::new(haystack, 7, 12); + + assert_eq!(m.start(), 7); + assert_eq!(m.end(), 12); + assert_eq!(m.is_empty(), false); + assert_eq!(m.len(), 5); + assert_eq!(m.as_bytes(), b"world"); + } + + #[test] + fn test_empty_match() { + let haystack = b""; + let m = Match::new(haystack, 0, 0); + + assert_eq!(m.is_empty(), true); + assert_eq!(m.len(), 0); + } + + #[test] + fn test_debug_output_valid_utf8() { + let haystack = b"Hello, world!"; + let m = Match::new(haystack, 7, 12); + let debug_str = format!("{m:?}"); + + assert_eq!( + debug_str, + r#"Match { start: 7, end: 12, bytes: "world" }"# + ); + } + + #[test] + fn test_debug_output_invalid_utf8() { + let haystack = b"Hello, \xFFworld!"; + let m = Match::new(haystack, 7, 13); + let debug_str = format!("{m:?}"); + + assert_eq!( + debug_str, + r#"Match { start: 7, end: 13, bytes: "\xffworld" }"# + ); + } + + #[test] + fn test_debug_output_various_unicode() { + let haystack = + "Hello, 😊 world! 안녕하세요? مرحبا بالعالم!".as_bytes(); + let m = Match::new(haystack, 0, haystack.len()); + let debug_str = format!("{m:?}"); + + assert_eq!( + debug_str, + r#"Match { start: 0, end: 62, bytes: "Hello, 😊 world! 안녕하세요? مرحبا بالعالم!" }"# + ); + } + + #[test] + fn test_debug_output_ascii_escape() { + let haystack = b"Hello,\tworld!\nThis is a \x1b[31mtest\x1b[0m."; + let m = Match::new(haystack, 0, haystack.len()); + let debug_str = format!("{m:?}"); + + assert_eq!( + debug_str, + r#"Match { start: 0, end: 38, bytes: "Hello,\tworld!\nThis is a \u{1b}[31mtest\u{1b}[0m." }"# + ); + } + + #[test] + fn test_debug_output_match_in_middle() { + let haystack = b"The quick brown fox jumps over the lazy dog."; + let m = Match::new(haystack, 16, 19); + let debug_str = format!("{m:?}"); + + assert_eq!(debug_str, r#"Match { start: 16, end: 19, bytes: "fox" }"#); + } +} diff --git a/third_party/crates/regex-1.13.1/src/regex/mod.rs b/third_party/crates/regex-1.13.1/src/regex/mod.rs new file mode 100644 index 0000000..93fadec --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/regex/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod bytes; +pub(crate) mod string; diff --git a/third_party/crates/regex-1.13.1/src/regex/string.rs b/third_party/crates/regex-1.13.1/src/regex/string.rs new file mode 100644 index 0000000..627a6c1 --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/regex/string.rs @@ -0,0 +1,2674 @@ +use alloc::{borrow::Cow, string::String, sync::Arc}; + +use regex_automata::{meta, util::captures, Input, PatternID}; + +use crate::{error::Error, RegexBuilder}; + +/// A convenient way to construct regex patterns from string literals. +/// +/// This macro can be used to construct reusable instances of [`Regex`] with +/// reduced boilerplate. The constructed `Regex` is stored in a static so the +/// pattern is compiled approximately once, even when called multiple times. +/// +/// There is *no compile-time checking of patterns* with `regex!`. Instead, +/// invalid patterns will panic the first time the regex is used. Invalid +/// patterns should still not be used with `regex!`; if compile-time checking +/// becomes feasible in the future, it may be added within a non-semver-breaking +/// release. In the meantime, consider enabling [`clippy::invalid_regex`]. +/// +/// # Examples +/// +/// ``` +/// use regex::{Regex, regex}; +/// +/// assert!(regex!("[a-z]").is_match("a")); +/// assert!(regex!("(inconceivable!|classic blunder)").is_match("inconceivable!")); +/// +/// let re: &Regex = regex!(r"(\d{3})-(\d{4})"); +/// assert_eq!(&re.captures("867-5309").unwrap()[1], "867"); +/// ``` +/// +/// An invalid pattern will panic when it is first used: +/// +/// ```should_panic +/// use regex::regex; +/// +/// let re = regex!("invalid -> ("); // no panic here +/// re.is_match("invalid -> ("); // panic! +/// ``` +/// +/// [`clippy::invalid_regex`]: https://rust-lang.github.io/rust-clippy/master/#invalid_regex +#[macro_export] +macro_rules! regex { + ($re:literal) => {{ + static REGEX: $crate::__private::Lazy<$crate::Regex> = + $crate::__private::Lazy::new(|| { + $crate::Regex::new($re).expect("invalid regex pattern") + }); + + // Coerce returned type from `&Lazy<Regex>` to `&Regex` to avoid making the + // inner type public. + let re: &$crate::Regex = ®EX; + re + }}; +} + +/// A compiled regular expression for searching Unicode haystacks. +/// +/// A `Regex` can be used to search haystacks, split haystacks into substrings +/// or replace substrings in a haystack with a different substring. All +/// searching is done with an implicit `(?s:.)*?` at the beginning and end of +/// an pattern. To force an expression to match the whole string (or a prefix +/// or a suffix), you must use an anchor like `^` or `$` (or `\A` and `\z`). +/// +/// While this crate will handle Unicode strings (whether in the regular +/// expression or in the haystack), all positions returned are **byte +/// offsets**. Every byte offset is guaranteed to be at a Unicode code point +/// boundary. That is, all offsets returned by the `Regex` API are guaranteed +/// to be ranges that can slice a `&str` without panicking. If you want to +/// relax this requirement, then you must search `&[u8]` haystacks with a +/// [`bytes::Regex`](crate::bytes::Regex). +/// +/// The only methods that allocate new strings are the string replacement +/// methods. All other methods (searching and splitting) return borrowed +/// references into the haystack given. +/// +/// # Example +/// +/// Find the offsets of a US phone number: +/// +/// ``` +/// use regex::Regex; +/// +/// let re = Regex::new("[0-9]{3}-[0-9]{3}-[0-9]{4}").unwrap(); +/// let m = re.find("phone: 111-222-3333").unwrap(); +/// assert_eq!(7..19, m.range()); +/// ``` +/// +/// # Example: extracting capture groups +/// +/// A common way to use regexes is with capture groups. That is, instead of +/// just looking for matches of an entire regex, parentheses are used to create +/// groups that represent part of the match. +/// +/// For example, consider a haystack with multiple lines, and each line has +/// three whitespace delimited fields where the second field is expected to be +/// a number and the third field a boolean. To make this convenient, we use +/// the [`Captures::extract`] API to put the strings that match each group +/// into a fixed size array: +/// +/// ``` +/// use regex::Regex; +/// +/// let hay = " +/// rabbit 54 true +/// groundhog 2 true +/// does not match +/// fox 109 false +/// "; +/// let re = Regex::new(r"(?m)^\s*(\S+)\s+([0-9]+)\s+(true|false)\s*$").unwrap(); +/// let mut fields: Vec<(&str, i64, bool)> = vec![]; +/// for (_, [f1, f2, f3]) in re.captures_iter(hay).map(|caps| caps.extract()) { +/// fields.push((f1, f2.parse()?, f3.parse()?)); +/// } +/// assert_eq!(fields, vec![ +/// ("rabbit", 54, true), +/// ("groundhog", 2, true), +/// ("fox", 109, false), +/// ]); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: searching with the `Pattern` trait +/// +/// **Note**: This section requires that this crate is compiled with the +/// `pattern` Cargo feature enabled, which **requires nightly Rust**. +/// +/// Since `Regex` implements `Pattern` from the standard library, one can +/// use regexes with methods defined on `&str`. For example, `is_match`, +/// `find`, `find_iter` and `split` can, in some cases, be replaced with +/// `str::contains`, `str::find`, `str::match_indices` and `str::split`. +/// +/// Here are some examples: +/// +/// ```ignore +/// use regex::Regex; +/// +/// let re = Regex::new(r"\d+").unwrap(); +/// let hay = "a111b222c"; +/// +/// assert!(hay.contains(&re)); +/// assert_eq!(hay.find(&re), Some(1)); +/// assert_eq!(hay.match_indices(&re).collect::<Vec<_>>(), vec![ +/// (1, "111"), +/// (5, "222"), +/// ]); +/// assert_eq!(hay.split(&re).collect::<Vec<_>>(), vec!["a", "b", "c"]); +/// ``` +#[derive(Clone)] +pub struct Regex { + pub(crate) meta: meta::Regex, + pub(crate) pattern: Arc<str>, +} + +impl core::fmt::Display for Regex { + /// Shows the original regular expression. + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +impl core::fmt::Debug for Regex { + /// Shows the original regular expression. + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_tuple("Regex").field(&self.as_str()).finish() + } +} + +impl core::str::FromStr for Regex { + type Err = Error; + + /// Attempts to parse a string into a regular expression + fn from_str(s: &str) -> Result<Regex, Error> { + Regex::new(s) + } +} + +impl TryFrom<&str> for Regex { + type Error = Error; + + /// Attempts to parse a string into a regular expression + fn try_from(s: &str) -> Result<Regex, Error> { + Regex::new(s) + } +} + +impl TryFrom<String> for Regex { + type Error = Error; + + /// Attempts to parse a string into a regular expression + fn try_from(s: String) -> Result<Regex, Error> { + Regex::new(&s) + } +} + +/// Core regular expression methods. +impl Regex { + /// Compiles a regular expression. Once compiled, it can be used repeatedly + /// to search, split or replace substrings in a haystack. + /// + /// Note that regex compilation tends to be a somewhat expensive process, + /// and unlike higher level environments, compilation is not automatically + /// cached for you. One should endeavor to compile a regex once and then + /// reuse it. For example, it's a bad idea to compile the same regex + /// repeatedly in a loop. + /// + /// # Errors + /// + /// If an invalid pattern is given, then an error is returned. + /// An error is also returned if the pattern is valid, but would + /// produce a regex that is bigger than the configured size limit via + /// [`RegexBuilder::size_limit`]. (A reasonable size limit is enabled by + /// default.) + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// // An Invalid pattern because of an unclosed parenthesis + /// assert!(Regex::new(r"foo(bar").is_err()); + /// // An invalid pattern because the regex would be too big + /// // because Unicode tends to inflate things. + /// assert!(Regex::new(r"\w{1000}").is_err()); + /// // Disabling Unicode can make the regex much smaller, + /// // potentially by up to or more than an order of magnitude. + /// assert!(Regex::new(r"(?-u:\w){1000}").is_ok()); + /// ``` + pub fn new(re: &str) -> Result<Regex, Error> { + RegexBuilder::new(re).build() + } + + /// Returns true if and only if there is a match for the regex anywhere + /// in the haystack given. + /// + /// It is recommended to use this method if all you need to do is test + /// whether a match exists, since the underlying matching engine may be + /// able to do less work. + /// + /// # Example + /// + /// Test if some haystack contains at least one word with exactly 13 + /// Unicode word characters: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\b\w{13}\b").unwrap(); + /// let hay = "I categorically deny having triskaidekaphobia."; + /// assert!(re.is_match(hay)); + /// ``` + #[inline] + pub fn is_match(&self, haystack: &str) -> bool { + self.is_match_at(haystack, 0) + } + + /// This routine searches for the first match of this regex in the + /// haystack given, and if found, returns a [`Match`]. The `Match` + /// provides access to both the byte offsets of the match and the actual + /// substring that matched. + /// + /// Note that this should only be used if you want to find the entire + /// match. If instead you just want to test the existence of a match, + /// it's potentially faster to use `Regex::is_match(hay)` instead of + /// `Regex::find(hay).is_some()`. + /// + /// # Example + /// + /// Find the first word with exactly 13 Unicode word characters: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\b\w{13}\b").unwrap(); + /// let hay = "I categorically deny having triskaidekaphobia."; + /// let mat = re.find(hay).unwrap(); + /// assert_eq!(2..15, mat.range()); + /// assert_eq!("categorically", mat.as_str()); + /// ``` + #[inline] + pub fn find<'h>(&self, haystack: &'h str) -> Option<Match<'h>> { + self.find_at(haystack, 0) + } + + /// Returns an iterator that yields successive non-overlapping matches in + /// the given haystack. The iterator yields values of type [`Match`]. + /// + /// # Time complexity + /// + /// Note that since `find_iter` runs potentially many searches on the + /// haystack and since each search has worst case `O(m * n)` time + /// complexity, the overall worst case time complexity for iteration is + /// `O(m * n^2)`. + /// + /// # Example + /// + /// Find every word with exactly 13 Unicode word characters: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\b\w{13}\b").unwrap(); + /// let hay = "Retroactively relinquishing remunerations is reprehensible."; + /// let matches: Vec<_> = re.find_iter(hay).map(|m| m.as_str()).collect(); + /// assert_eq!(matches, vec![ + /// "Retroactively", + /// "relinquishing", + /// "remunerations", + /// "reprehensible", + /// ]); + /// ``` + #[inline] + pub fn find_iter<'r, 'h>(&'r self, haystack: &'h str) -> Matches<'r, 'h> { + Matches { haystack, it: self.meta.find_iter(haystack) } + } + + /// This routine searches for the first match of this regex in the haystack + /// given, and if found, returns not only the overall match but also the + /// matches of each capture group in the regex. If no match is found, then + /// `None` is returned. + /// + /// Capture group `0` always corresponds to an implicit unnamed group that + /// includes the entire match. If a match is found, this group is always + /// present. Subsequent groups may be named and are numbered, starting + /// at 1, by the order in which the opening parenthesis appears in the + /// pattern. For example, in the pattern `(?<a>.(?<b>.))(?<c>.)`, `a`, + /// `b` and `c` correspond to capture group indices `1`, `2` and `3`, + /// respectively. + /// + /// You should only use `captures` if you need access to the capture group + /// matches. Otherwise, [`Regex::find`] is generally faster for discovering + /// just the overall match. + /// + /// # Example + /// + /// Say you have some haystack with movie names and their release years, + /// like "'Citizen Kane' (1941)". It'd be nice if we could search for + /// substrings looking like that, while also extracting the movie name and + /// its release year separately. The example below shows how to do that. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"'([^']+)'\s+\((\d{4})\)").unwrap(); + /// let hay = "Not my favorite movie: 'Citizen Kane' (1941)."; + /// let caps = re.captures(hay).unwrap(); + /// assert_eq!(caps.get(0).unwrap().as_str(), "'Citizen Kane' (1941)"); + /// assert_eq!(caps.get(1).unwrap().as_str(), "Citizen Kane"); + /// assert_eq!(caps.get(2).unwrap().as_str(), "1941"); + /// // You can also access the groups by index using the Index notation. + /// // Note that this will panic on an invalid index. In this case, these + /// // accesses are always correct because the overall regex will only + /// // match when these capture groups match. + /// assert_eq!(&caps[0], "'Citizen Kane' (1941)"); + /// assert_eq!(&caps[1], "Citizen Kane"); + /// assert_eq!(&caps[2], "1941"); + /// ``` + /// + /// Note that the full match is at capture group `0`. Each subsequent + /// capture group is indexed by the order of its opening `(`. + /// + /// We can make this example a bit clearer by using *named* capture groups: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"'(?<title>[^']+)'\s+\((?<year>\d{4})\)").unwrap(); + /// let hay = "Not my favorite movie: 'Citizen Kane' (1941)."; + /// let caps = re.captures(hay).unwrap(); + /// assert_eq!(caps.get(0).unwrap().as_str(), "'Citizen Kane' (1941)"); + /// assert_eq!(caps.name("title").unwrap().as_str(), "Citizen Kane"); + /// assert_eq!(caps.name("year").unwrap().as_str(), "1941"); + /// // You can also access the groups by name using the Index notation. + /// // Note that this will panic on an invalid group name. In this case, + /// // these accesses are always correct because the overall regex will + /// // only match when these capture groups match. + /// assert_eq!(&caps[0], "'Citizen Kane' (1941)"); + /// assert_eq!(&caps["title"], "Citizen Kane"); + /// assert_eq!(&caps["year"], "1941"); + /// ``` + /// + /// Here we name the capture groups, which we can access with the `name` + /// method or the `Index` notation with a `&str`. Note that the named + /// capture groups are still accessible with `get` or the `Index` notation + /// with a `usize`. + /// + /// The `0`th capture group is always unnamed, so it must always be + /// accessed with `get(0)` or `[0]`. + /// + /// Finally, one other way to get the matched substrings is with the + /// [`Captures::extract`] API: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"'([^']+)'\s+\((\d{4})\)").unwrap(); + /// let hay = "Not my favorite movie: 'Citizen Kane' (1941)."; + /// let (full, [title, year]) = re.captures(hay).unwrap().extract(); + /// assert_eq!(full, "'Citizen Kane' (1941)"); + /// assert_eq!(title, "Citizen Kane"); + /// assert_eq!(year, "1941"); + /// ``` + #[inline] + pub fn captures<'h>(&self, haystack: &'h str) -> Option<Captures<'h>> { + self.captures_at(haystack, 0) + } + + /// Returns an iterator that yields successive non-overlapping matches in + /// the given haystack. The iterator yields values of type [`Captures`]. + /// + /// This is the same as [`Regex::find_iter`], but instead of only providing + /// access to the overall match, each value yield includes access to the + /// matches of all capture groups in the regex. Reporting this extra match + /// data is potentially costly, so callers should only use `captures_iter` + /// over `find_iter` when they actually need access to the capture group + /// matches. + /// + /// # Time complexity + /// + /// Note that since `captures_iter` runs potentially many searches on the + /// haystack and since each search has worst case `O(m * n)` time + /// complexity, the overall worst case time complexity for iteration is + /// `O(m * n^2)`. + /// + /// # Example + /// + /// We can use this to find all movie titles and their release years in + /// some haystack, where the movie is formatted like "'Title' (xxxx)": + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"'([^']+)'\s+\(([0-9]{4})\)").unwrap(); + /// let hay = "'Citizen Kane' (1941), 'The Wizard of Oz' (1939), 'M' (1931)."; + /// let mut movies = vec![]; + /// for (_, [title, year]) in re.captures_iter(hay).map(|c| c.extract()) { + /// movies.push((title, year.parse::<i64>()?)); + /// } + /// assert_eq!(movies, vec![ + /// ("Citizen Kane", 1941), + /// ("The Wizard of Oz", 1939), + /// ("M", 1931), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Or with named groups: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"'(?<title>[^']+)'\s+\((?<year>[0-9]{4})\)").unwrap(); + /// let hay = "'Citizen Kane' (1941), 'The Wizard of Oz' (1939), 'M' (1931)."; + /// let mut it = re.captures_iter(hay); + /// + /// let caps = it.next().unwrap(); + /// assert_eq!(&caps["title"], "Citizen Kane"); + /// assert_eq!(&caps["year"], "1941"); + /// + /// let caps = it.next().unwrap(); + /// assert_eq!(&caps["title"], "The Wizard of Oz"); + /// assert_eq!(&caps["year"], "1939"); + /// + /// let caps = it.next().unwrap(); + /// assert_eq!(&caps["title"], "M"); + /// assert_eq!(&caps["year"], "1931"); + /// ``` + #[inline] + pub fn captures_iter<'r, 'h>( + &'r self, + haystack: &'h str, + ) -> CaptureMatches<'r, 'h> { + CaptureMatches { haystack, it: self.meta.captures_iter(haystack) } + } + + /// Returns an iterator of substrings of the haystack given, delimited by a + /// match of the regex. Namely, each element of the iterator corresponds to + /// a part of the haystack that *isn't* matched by the regular expression. + /// + /// # Time complexity + /// + /// Since iterators over all matches requires running potentially many + /// searches on the haystack, and since each search has worst case + /// `O(m * n)` time complexity, the overall worst case time complexity for + /// this routine is `O(m * n^2)`. + /// + /// # Example + /// + /// To split a string delimited by arbitrary amounts of spaces or tabs: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"[ \t]+").unwrap(); + /// let hay = "a b \t c\td e"; + /// let fields: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(fields, vec!["a", "b", "c", "d", "e"]); + /// ``` + /// + /// # Example: more cases + /// + /// Basic usage: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r" ").unwrap(); + /// let hay = "Mary had a little lamb"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["Mary", "had", "a", "little", "lamb"]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = ""; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec![""]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = "lionXXtigerXleopard"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["lion", "", "tiger", "leopard"]); + /// + /// let re = Regex::new(r"::").unwrap(); + /// let hay = "lion::tiger::leopard"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["lion", "tiger", "leopard"]); + /// ``` + /// + /// If a haystack contains multiple contiguous matches, you will end up + /// with empty spans yielded by the iterator: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = "XXXXaXXbXc"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["", "", "", "", "a", "", "b", "c"]); + /// + /// let re = Regex::new(r"/").unwrap(); + /// let hay = "(///)"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["(", "", "", ")"]); + /// ``` + /// + /// Separators at the start or end of a haystack are neighbored by empty + /// substring. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"0").unwrap(); + /// let hay = "010"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["", "1", ""]); + /// ``` + /// + /// When the empty string is used as a regex, it splits at every valid + /// UTF-8 boundary by default (which includes the beginning and end of the + /// haystack): + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"").unwrap(); + /// let hay = "rust"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["", "r", "u", "s", "t", ""]); + /// + /// // Splitting by an empty string is UTF-8 aware by default! + /// let re = Regex::new(r"").unwrap(); + /// let hay = "☃"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["", "☃", ""]); + /// ``` + /// + /// Contiguous separators (commonly shows up with whitespace), can lead to + /// possibly surprising behavior. For example, this code is correct: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r" ").unwrap(); + /// let hay = " a b c"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// assert_eq!(got, vec!["", "", "", "", "a", "", "b", "c"]); + /// ``` + /// + /// It does *not* give you `["a", "b", "c"]`. For that behavior, you'd want + /// to match contiguous space characters: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r" +").unwrap(); + /// let hay = " a b c"; + /// let got: Vec<&str> = re.split(hay).collect(); + /// // N.B. This does still include a leading empty span because ' +' + /// // matches at the beginning of the haystack. + /// assert_eq!(got, vec!["", "a", "b", "c"]); + /// ``` + #[inline] + pub fn split<'r, 'h>(&'r self, haystack: &'h str) -> Split<'r, 'h> { + Split { haystack, it: self.meta.split(haystack) } + } + + /// Returns an iterator of at most `limit` substrings of the haystack + /// given, delimited by a match of the regex. (A `limit` of `0` will return + /// no substrings.) Namely, each element of the iterator corresponds to a + /// part of the haystack that *isn't* matched by the regular expression. + /// The remainder of the haystack that is not split will be the last + /// element in the iterator. + /// + /// # Time complexity + /// + /// Since iterators over all matches requires running potentially many + /// searches on the haystack, and since each search has worst case + /// `O(m * n)` time complexity, the overall worst case time complexity for + /// this routine is `O(m * n^2)`. + /// + /// Although note that the worst case time here has an upper bound given + /// by the `limit` parameter. + /// + /// # Example + /// + /// Get the first two words in some haystack: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\W+").unwrap(); + /// let hay = "Hey! How are you?"; + /// let fields: Vec<&str> = re.splitn(hay, 3).collect(); + /// assert_eq!(fields, vec!["Hey", "How", "are you?"]); + /// ``` + /// + /// # Examples: more cases + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r" ").unwrap(); + /// let hay = "Mary had a little lamb"; + /// let got: Vec<&str> = re.splitn(hay, 3).collect(); + /// assert_eq!(got, vec!["Mary", "had", "a little lamb"]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = ""; + /// let got: Vec<&str> = re.splitn(hay, 3).collect(); + /// assert_eq!(got, vec![""]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = "lionXXtigerXleopard"; + /// let got: Vec<&str> = re.splitn(hay, 3).collect(); + /// assert_eq!(got, vec!["lion", "", "tigerXleopard"]); + /// + /// let re = Regex::new(r"::").unwrap(); + /// let hay = "lion::tiger::leopard"; + /// let got: Vec<&str> = re.splitn(hay, 2).collect(); + /// assert_eq!(got, vec!["lion", "tiger::leopard"]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = "abcXdef"; + /// let got: Vec<&str> = re.splitn(hay, 1).collect(); + /// assert_eq!(got, vec!["abcXdef"]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = "abcdef"; + /// let got: Vec<&str> = re.splitn(hay, 2).collect(); + /// assert_eq!(got, vec!["abcdef"]); + /// + /// let re = Regex::new(r"X").unwrap(); + /// let hay = "abcXdef"; + /// let got: Vec<&str> = re.splitn(hay, 0).collect(); + /// assert!(got.is_empty()); + /// ``` + #[inline] + pub fn splitn<'r, 'h>( + &'r self, + haystack: &'h str, + limit: usize, + ) -> SplitN<'r, 'h> { + SplitN { haystack, it: self.meta.splitn(haystack, limit) } + } + + /// Replaces the leftmost-first match in the given haystack with the + /// replacement provided. The replacement can be a regular string (where + /// `$N` and `$name` are expanded to match capture groups) or a function + /// that takes a [`Captures`] and returns the replaced string. + /// + /// If no match is found, then the haystack is returned unchanged. In that + /// case, this implementation will likely return a `Cow::Borrowed` value + /// such that no allocation is performed. + /// + /// When a `Cow::Borrowed` is returned, the value returned is guaranteed + /// to be equivalent to the `haystack` given. + /// + /// # Replacement string syntax + /// + /// All instances of `$ref` in the replacement string are replaced with + /// the substring corresponding to the capture group identified by `ref`. + /// + /// `ref` may be an integer corresponding to the index of the capture group + /// (counted by order of opening parenthesis where `0` is the entire match) + /// or it can be a name (consisting of letters, digits or underscores) + /// corresponding to a named capture group. + /// + /// If `ref` isn't a valid capture group (whether the name doesn't exist or + /// isn't a valid index), then it is replaced with the empty string. + /// + /// The longest possible name is used. For example, `$1a` looks up the + /// capture group named `1a` and not the capture group at index `1`. To + /// exert more precise control over the name, use braces, e.g., `${1}a`. + /// + /// To write a literal `$` use `$$`. + /// + /// # Example + /// + /// Note that this function is polymorphic with respect to the replacement. + /// In typical usage, this can just be a normal string: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"[^01]+").unwrap(); + /// assert_eq!(re.replace("1078910", ""), "1010"); + /// ``` + /// + /// But anything satisfying the [`Replacer`] trait will work. For example, + /// a closure of type `|&Captures| -> String` provides direct access to the + /// captures corresponding to a match. This allows one to access capturing + /// group matches easily: + /// + /// ``` + /// use regex::{Captures, Regex}; + /// + /// let re = Regex::new(r"([^,\s]+),\s+(\S+)").unwrap(); + /// let result = re.replace("Springsteen, Bruce", |caps: &Captures| { + /// format!("{} {}", &caps[2], &caps[1]) + /// }); + /// assert_eq!(result, "Bruce Springsteen"); + /// ``` + /// + /// But this is a bit cumbersome to use all the time. Instead, a simple + /// syntax is supported (as described above) that expands `$name` into the + /// corresponding capture group. Here's the last example, but using this + /// expansion technique with named capture groups: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(?<last>[^,\s]+),\s+(?<first>\S+)").unwrap(); + /// let result = re.replace("Springsteen, Bruce", "$first $last"); + /// assert_eq!(result, "Bruce Springsteen"); + /// ``` + /// + /// Note that using `$2` instead of `$first` or `$1` instead of `$last` + /// would produce the same result. To write a literal `$` use `$$`. + /// + /// Sometimes the replacement string requires use of curly braces to + /// delineate a capture group replacement when it is adjacent to some other + /// literal text. For example, if we wanted to join two words together with + /// an underscore: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(?<first>\w+)\s+(?<second>\w+)").unwrap(); + /// let result = re.replace("deep fried", "${first}_$second"); + /// assert_eq!(result, "deep_fried"); + /// ``` + /// + /// Without the curly braces, the capture group name `first_` would be + /// used, and since it doesn't exist, it would be replaced with the empty + /// string. + /// + /// Finally, sometimes you just want to replace a literal string with no + /// regard for capturing group expansion. This can be done by wrapping a + /// string with [`NoExpand`]: + /// + /// ``` + /// use regex::{NoExpand, Regex}; + /// + /// let re = Regex::new(r"(?<last>[^,\s]+),\s+(\S+)").unwrap(); + /// let result = re.replace("Springsteen, Bruce", NoExpand("$2 $last")); + /// assert_eq!(result, "$2 $last"); + /// ``` + /// + /// Using `NoExpand` may also be faster, since the replacement string won't + /// need to be parsed for the `$` syntax. + #[inline] + pub fn replace<'h, R: Replacer>( + &self, + haystack: &'h str, + rep: R, + ) -> Cow<'h, str> { + self.replacen(haystack, 1, rep) + } + + /// Replaces all non-overlapping matches in the haystack with the + /// replacement provided. This is the same as calling `replacen` with + /// `limit` set to `0`. + /// + /// If no match is found, then the haystack is returned unchanged. In that + /// case, this implementation will likely return a `Cow::Borrowed` value + /// such that no allocation is performed. + /// + /// When a `Cow::Borrowed` is returned, the value returned is guaranteed + /// to be equivalent to the `haystack` given. + /// + /// The documentation for [`Regex::replace`] goes into more detail about + /// what kinds of replacement strings are supported. + /// + /// # Time complexity + /// + /// Since iterators over all matches requires running potentially many + /// searches on the haystack, and since each search has worst case + /// `O(m * n)` time complexity, the overall worst case time complexity for + /// this routine is `O(m * n^2)`. + /// + /// # Fallibility + /// + /// If you need to write a replacement routine where any individual + /// replacement might "fail," doing so with this API isn't really feasible + /// because there's no way to stop the search process if a replacement + /// fails. Instead, if you need this functionality, you should consider + /// implementing your own replacement routine: + /// + /// ``` + /// use regex::{Captures, Regex}; + /// + /// fn replace_all<E>( + /// re: &Regex, + /// haystack: &str, + /// replacement: impl Fn(&Captures) -> Result<String, E>, + /// ) -> Result<String, E> { + /// let mut new = String::with_capacity(haystack.len()); + /// let mut last_match = 0; + /// for caps in re.captures_iter(haystack) { + /// let m = caps.get(0).unwrap(); + /// new.push_str(&haystack[last_match..m.start()]); + /// new.push_str(&replacement(&caps)?); + /// last_match = m.end(); + /// } + /// new.push_str(&haystack[last_match..]); + /// Ok(new) + /// } + /// + /// // Let's replace each word with the number of bytes in that word. + /// // But if we see a word that is "too long," we'll give up. + /// let re = Regex::new(r"\w+").unwrap(); + /// let replacement = |caps: &Captures| -> Result<String, &'static str> { + /// if caps[0].len() >= 5 { + /// return Err("word too long"); + /// } + /// Ok(caps[0].len().to_string()) + /// }; + /// assert_eq!( + /// Ok("2 3 3 3?".to_string()), + /// replace_all(&re, "hi how are you?", &replacement), + /// ); + /// assert!(replace_all(&re, "hi there", &replacement).is_err()); + /// ``` + /// + /// # Example + /// + /// This example shows how to flip the order of whitespace (excluding line + /// terminators) delimited fields, and normalizes the whitespace that + /// delimits the fields: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(?m)^(\S+)[\s--\r\n]+(\S+)$").unwrap(); + /// let hay = " + /// Greetings 1973 + /// Wild\t1973 + /// BornToRun\t\t\t\t1975 + /// Darkness 1978 + /// TheRiver 1980 + /// "; + /// let new = re.replace_all(hay, "$2 $1"); + /// assert_eq!(new, " + /// 1973 Greetings + /// 1973 Wild + /// 1975 BornToRun + /// 1978 Darkness + /// 1980 TheRiver + /// "); + /// ``` + #[inline] + pub fn replace_all<'h, R: Replacer>( + &self, + haystack: &'h str, + rep: R, + ) -> Cow<'h, str> { + self.replacen(haystack, 0, rep) + } + + /// Replaces at most `limit` non-overlapping matches in the haystack with + /// the replacement provided. If `limit` is `0`, then all non-overlapping + /// matches are replaced. That is, `Regex::replace_all(hay, rep)` is + /// equivalent to `Regex::replacen(hay, 0, rep)`. + /// + /// If no match is found, then the haystack is returned unchanged. In that + /// case, this implementation will likely return a `Cow::Borrowed` value + /// such that no allocation is performed. + /// + /// When a `Cow::Borrowed` is returned, the value returned is guaranteed + /// to be equivalent to the `haystack` given. + /// + /// The documentation for [`Regex::replace`] goes into more detail about + /// what kinds of replacement strings are supported. + /// + /// # Time complexity + /// + /// Since iterators over all matches requires running potentially many + /// searches on the haystack, and since each search has worst case + /// `O(m * n)` time complexity, the overall worst case time complexity for + /// this routine is `O(m * n^2)`. + /// + /// Although note that the worst case time here has an upper bound given + /// by the `limit` parameter. + /// + /// # Fallibility + /// + /// See the corresponding section in the docs for [`Regex::replace_all`] + /// for tips on how to deal with a replacement routine that can fail. + /// + /// # Example + /// + /// This example shows how to flip the order of whitespace (excluding line + /// terminators) delimited fields, and normalizes the whitespace that + /// delimits the fields. But we only do it for the first two matches. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(?m)^(\S+)[\s--\r\n]+(\S+)$").unwrap(); + /// let hay = " + /// Greetings 1973 + /// Wild\t1973 + /// BornToRun\t\t\t\t1975 + /// Darkness 1978 + /// TheRiver 1980 + /// "; + /// let new = re.replacen(hay, 2, "$2 $1"); + /// assert_eq!(new, " + /// 1973 Greetings + /// 1973 Wild + /// BornToRun\t\t\t\t1975 + /// Darkness 1978 + /// TheRiver 1980 + /// "); + /// ``` + #[inline] + pub fn replacen<'h, R: Replacer>( + &self, + haystack: &'h str, + limit: usize, + mut rep: R, + ) -> Cow<'h, str> { + // If we know that the replacement doesn't have any capture expansions, + // then we can use the fast path. The fast path can make a tremendous + // difference: + // + // 1) We use `find_iter` instead of `captures_iter`. Not asking for + // captures generally makes the regex engines faster. + // 2) We don't need to look up all of the capture groups and do + // replacements inside the replacement string. We just push it + // at each match and be done with it. + if let Some(rep) = rep.no_expansion() { + let mut it = self.find_iter(haystack).enumerate().peekable(); + if it.peek().is_none() { + return Cow::Borrowed(haystack); + } + let mut new = String::with_capacity(haystack.len()); + let mut last_match = 0; + for (i, m) in it { + new.push_str(&haystack[last_match..m.start()]); + new.push_str(&rep); + last_match = m.end(); + if limit > 0 && i >= limit - 1 { + break; + } + } + new.push_str(&haystack[last_match..]); + return Cow::Owned(new); + } + + // The slower path, which we use if the replacement may need access to + // capture groups. + let mut it = self.captures_iter(haystack).enumerate().peekable(); + if it.peek().is_none() { + return Cow::Borrowed(haystack); + } + let mut new = String::with_capacity(haystack.len()); + let mut last_match = 0; + for (i, cap) in it { + // unwrap on 0 is OK because captures only reports matches + let m = cap.get(0).unwrap(); + new.push_str(&haystack[last_match..m.start()]); + rep.replace_append(&cap, &mut new); + last_match = m.end(); + if limit > 0 && i >= limit - 1 { + break; + } + } + new.push_str(&haystack[last_match..]); + Cow::Owned(new) + } +} + +/// A group of advanced or "lower level" search methods. Some methods permit +/// starting the search at a position greater than `0` in the haystack. Other +/// methods permit reusing allocations, for example, when extracting the +/// matches for capture groups. +impl Regex { + /// Returns the end byte offset of the first match in the haystack given. + /// + /// This method may have the same performance characteristics as + /// `is_match`. Behaviorally, it doesn't just report whether it match + /// occurs, but also the end offset for a match. In particular, the offset + /// returned *may be shorter* than the proper end of the leftmost-first + /// match that you would find via [`Regex::find`]. + /// + /// Note that it is not guaranteed that this routine finds the shortest or + /// "earliest" possible match. Instead, the main idea of this API is that + /// it returns the offset at the point at which the internal regex engine + /// has determined that a match has occurred. This may vary depending on + /// which internal regex engine is used, and thus, the offset itself may + /// change based on internal heuristics. + /// + /// # Example + /// + /// Typically, `a+` would match the entire first sequence of `a` in some + /// haystack, but `shortest_match` *may* give up as soon as it sees the + /// first `a`. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"a+").unwrap(); + /// let offset = re.shortest_match("aaaaa").unwrap(); + /// assert_eq!(offset, 1); + /// ``` + #[inline] + pub fn shortest_match(&self, haystack: &str) -> Option<usize> { + self.shortest_match_at(haystack, 0) + } + + /// Returns the same as [`Regex::shortest_match`], but starts the search at + /// the given offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only match + /// when `start == 0`. + /// + /// If a match is found, the offset returned is relative to the beginning + /// of the haystack, not the beginning of the search. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = "eschew"; + /// // We get a match here, but it's probably not intended. + /// assert_eq!(re.shortest_match(&hay[2..]), Some(4)); + /// // No match because the assertions take the context into account. + /// assert_eq!(re.shortest_match_at(hay, 2), None); + /// ``` + #[inline] + pub fn shortest_match_at( + &self, + haystack: &str, + start: usize, + ) -> Option<usize> { + let input = + Input::new(haystack).earliest(true).span(start..haystack.len()); + self.meta.search_half(&input).map(|hm| hm.offset()) + } + + /// Returns the same as [`Regex::is_match`], but starts the search at the + /// given offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = "eschew"; + /// // We get a match here, but it's probably not intended. + /// assert!(re.is_match(&hay[2..])); + /// // No match because the assertions take the context into account. + /// assert!(!re.is_match_at(hay, 2)); + /// ``` + #[inline] + pub fn is_match_at(&self, haystack: &str, start: usize) -> bool { + let input = + Input::new(haystack).earliest(true).span(start..haystack.len()); + self.meta.search_half(&input).is_some() + } + + /// Returns the same as [`Regex::find`], but starts the search at the given + /// offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = "eschew"; + /// // We get a match here, but it's probably not intended. + /// assert_eq!(re.find(&hay[2..]).map(|m| m.range()), Some(0..4)); + /// // No match because the assertions take the context into account. + /// assert_eq!(re.find_at(hay, 2), None); + /// ``` + #[inline] + pub fn find_at<'h>( + &self, + haystack: &'h str, + start: usize, + ) -> Option<Match<'h>> { + let input = Input::new(haystack).span(start..haystack.len()); + self.meta + .search(&input) + .map(|m| Match::new(haystack, m.start(), m.end())) + } + + /// Returns the same as [`Regex::captures`], but starts the search at the + /// given offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = "eschew"; + /// // We get a match here, but it's probably not intended. + /// assert_eq!(&re.captures(&hay[2..]).unwrap()[0], "chew"); + /// // No match because the assertions take the context into account. + /// assert!(re.captures_at(hay, 2).is_none()); + /// ``` + #[inline] + pub fn captures_at<'h>( + &self, + haystack: &'h str, + start: usize, + ) -> Option<Captures<'h>> { + let input = Input::new(haystack).span(start..haystack.len()); + let mut caps = self.meta.create_captures(); + self.meta.search_captures(&input, &mut caps); + if caps.is_match() { + let static_captures_len = self.static_captures_len(); + Some(Captures { haystack, caps, static_captures_len }) + } else { + None + } + } + + /// This is like [`Regex::captures`], but writes the byte offsets of each + /// capture group match into the locations given. + /// + /// A [`CaptureLocations`] stores the same byte offsets as a [`Captures`], + /// but does *not* store a reference to the haystack. This makes its API + /// a bit lower level and less convenient. But in exchange, callers + /// may allocate their own `CaptureLocations` and reuse it for multiple + /// searches. This may be helpful if allocating a `Captures` shows up in a + /// profile as too costly. + /// + /// To create a `CaptureLocations` value, use the + /// [`Regex::capture_locations`] method. + /// + /// This also returns the overall match if one was found. When a match is + /// found, its offsets are also always stored in `locs` at index `0`. + /// + /// # Panics + /// + /// This routine may panic if the given `CaptureLocations` was not created + /// by this regex. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"^([a-z]+)=(\S*)$").unwrap(); + /// let mut locs = re.capture_locations(); + /// assert!(re.captures_read(&mut locs, "id=foo123").is_some()); + /// assert_eq!(Some((0, 9)), locs.get(0)); + /// assert_eq!(Some((0, 2)), locs.get(1)); + /// assert_eq!(Some((3, 9)), locs.get(2)); + /// ``` + #[inline] + pub fn captures_read<'h>( + &self, + locs: &mut CaptureLocations, + haystack: &'h str, + ) -> Option<Match<'h>> { + self.captures_read_at(locs, haystack, 0) + } + + /// Returns the same as [`Regex::captures_read`], but starts the search at + /// the given offset. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// This routine may also panic if the given `CaptureLocations` was not + /// created by this regex. + /// + /// # Example + /// + /// This example shows the significance of `start` by demonstrating how it + /// can be used to permit look-around assertions in a regex to take the + /// surrounding context into account. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"\bchew\b").unwrap(); + /// let hay = "eschew"; + /// let mut locs = re.capture_locations(); + /// // We get a match here, but it's probably not intended. + /// assert!(re.captures_read(&mut locs, &hay[2..]).is_some()); + /// // No match because the assertions take the context into account. + /// assert!(re.captures_read_at(&mut locs, hay, 2).is_none()); + /// ``` + #[inline] + pub fn captures_read_at<'h>( + &self, + locs: &mut CaptureLocations, + haystack: &'h str, + start: usize, + ) -> Option<Match<'h>> { + let input = Input::new(haystack).span(start..haystack.len()); + self.meta.search_captures(&input, &mut locs.0); + locs.0.get_match().map(|m| Match::new(haystack, m.start(), m.end())) + } + + /// An undocumented alias for `captures_read_at`. + /// + /// The `regex-capi` crate previously used this routine, so to avoid + /// breaking that crate, we continue to provide the name as an undocumented + /// alias. + #[doc(hidden)] + #[inline] + pub fn read_captures_at<'h>( + &self, + locs: &mut CaptureLocations, + haystack: &'h str, + start: usize, + ) -> Option<Match<'h>> { + self.captures_read_at(locs, haystack, start) + } +} + +/// Auxiliary methods. +impl Regex { + /// Returns the original string of this regex. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"foo\w+bar").unwrap(); + /// assert_eq!(re.as_str(), r"foo\w+bar"); + /// ``` + #[inline] + pub fn as_str(&self) -> &str { + &self.pattern + } + + /// Returns an iterator over the capture names in this regex. + /// + /// The iterator returned yields elements of type `Option<&str>`. That is, + /// the iterator yields values for all capture groups, even ones that are + /// unnamed. The order of the groups corresponds to the order of the group's + /// corresponding opening parenthesis. + /// + /// The first element of the iterator always yields the group corresponding + /// to the overall match, and this group is always unnamed. Therefore, the + /// iterator always yields at least one group. + /// + /// # Example + /// + /// This shows basic usage with a mix of named and unnamed capture groups: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(?<a>.(?<b>.))(.)(?:.)(?<c>.)").unwrap(); + /// let mut names = re.capture_names(); + /// assert_eq!(names.next(), Some(None)); + /// assert_eq!(names.next(), Some(Some("a"))); + /// assert_eq!(names.next(), Some(Some("b"))); + /// assert_eq!(names.next(), Some(None)); + /// // the '(?:.)' group is non-capturing and so doesn't appear here! + /// assert_eq!(names.next(), Some(Some("c"))); + /// assert_eq!(names.next(), None); + /// ``` + /// + /// The iterator always yields at least one element, even for regexes with + /// no capture groups and even for regexes that can never match: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"").unwrap(); + /// let mut names = re.capture_names(); + /// assert_eq!(names.next(), Some(None)); + /// assert_eq!(names.next(), None); + /// + /// let re = Regex::new(r"[a&&b]").unwrap(); + /// let mut names = re.capture_names(); + /// assert_eq!(names.next(), Some(None)); + /// assert_eq!(names.next(), None); + /// ``` + #[inline] + pub fn capture_names(&self) -> CaptureNames<'_> { + CaptureNames(self.meta.group_info().pattern_names(PatternID::ZERO)) + } + + /// Returns the number of captures groups in this regex. + /// + /// This includes all named and unnamed groups, including the implicit + /// unnamed group that is always present and corresponds to the entire + /// match. + /// + /// Since the implicit unnamed group is always included in this length, the + /// length returned is guaranteed to be greater than zero. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"foo").unwrap(); + /// assert_eq!(1, re.captures_len()); + /// + /// let re = Regex::new(r"(foo)").unwrap(); + /// assert_eq!(2, re.captures_len()); + /// + /// let re = Regex::new(r"(?<a>.(?<b>.))(.)(?:.)(?<c>.)").unwrap(); + /// assert_eq!(5, re.captures_len()); + /// + /// let re = Regex::new(r"[a&&b]").unwrap(); + /// assert_eq!(1, re.captures_len()); + /// ``` + #[inline] + pub fn captures_len(&self) -> usize { + self.meta.group_info().group_len(PatternID::ZERO) + } + + /// Returns the total number of capturing groups that appear in every + /// possible match. + /// + /// If the number of capture groups can vary depending on the match, then + /// this returns `None`. That is, a value is only returned when the number + /// of matching groups is invariant or "static." + /// + /// Note that like [`Regex::captures_len`], this **does** include the + /// implicit capturing group corresponding to the entire match. Therefore, + /// when a non-None value is returned, it is guaranteed to be at least `1`. + /// Stated differently, a return value of `Some(0)` is impossible. + /// + /// # Example + /// + /// This shows a few cases where a static number of capture groups is + /// available and a few cases where it is not. + /// + /// ``` + /// use regex::Regex; + /// + /// let len = |pattern| { + /// Regex::new(pattern).map(|re| re.static_captures_len()) + /// }; + /// + /// assert_eq!(Some(1), len("a")?); + /// assert_eq!(Some(2), len("(a)")?); + /// assert_eq!(Some(2), len("(a)|(b)")?); + /// assert_eq!(Some(3), len("(a)(b)|(c)(d)")?); + /// assert_eq!(None, len("(a)|b")?); + /// assert_eq!(None, len("a|(b)")?); + /// assert_eq!(None, len("(b)*")?); + /// assert_eq!(Some(2), len("(b)+")?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn static_captures_len(&self) -> Option<usize> { + self.meta.static_captures_len() + } + + /// Returns a fresh allocated set of capture locations that can + /// be reused in multiple calls to [`Regex::captures_read`] or + /// [`Regex::captures_read_at`]. + /// + /// The returned locations can be used for any subsequent search for this + /// particular regex. There is no guarantee that it is correct to use for + /// other regexes, even if they have the same number of capture groups. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(.)(.)(\w+)").unwrap(); + /// let mut locs = re.capture_locations(); + /// assert!(re.captures_read(&mut locs, "Padron").is_some()); + /// assert_eq!(locs.get(0), Some((0, 6))); + /// assert_eq!(locs.get(1), Some((0, 1))); + /// assert_eq!(locs.get(2), Some((1, 2))); + /// assert_eq!(locs.get(3), Some((2, 6))); + /// ``` + #[inline] + pub fn capture_locations(&self) -> CaptureLocations { + CaptureLocations(self.meta.create_captures()) + } + + /// An alias for `capture_locations` to preserve backward compatibility. + /// + /// The `regex-capi` crate used this method, so to avoid breaking that + /// crate, we continue to export it as an undocumented API. + #[doc(hidden)] + #[inline] + pub fn locations(&self) -> CaptureLocations { + self.capture_locations() + } +} + +/// Represents a single match of a regex in a haystack. +/// +/// A `Match` contains both the start and end byte offsets of the match and the +/// actual substring corresponding to the range of those byte offsets. It is +/// guaranteed that `start <= end`. When `start == end`, the match is empty. +/// +/// Since this `Match` can only be produced by the top-level `Regex` APIs +/// that only support searching UTF-8 encoded strings, the byte offsets for a +/// `Match` are guaranteed to fall on valid UTF-8 codepoint boundaries. That +/// is, slicing a `&str` with [`Match::range`] is guaranteed to never panic. +/// +/// Values with this type are created by [`Regex::find`] or +/// [`Regex::find_iter`]. Other APIs can create `Match` values too. For +/// example, [`Captures::get`]. +/// +/// The lifetime parameter `'h` refers to the lifetime of the matched of the +/// haystack that this match was produced from. +/// +/// # Numbering +/// +/// The byte offsets in a `Match` form a half-open interval. That is, the +/// start of the range is inclusive and the end of the range is exclusive. +/// For example, given a haystack `abcFOOxyz` and a match of `FOO`, its byte +/// offset range starts at `3` and ends at `6`. `3` corresponds to `F` and +/// `6` corresponds to `x`, which is one past the end of the match. This +/// corresponds to the same kind of slicing that Rust uses. +/// +/// For more on why this was chosen over other schemes (aside from being +/// consistent with how Rust the language works), see [this discussion] and +/// [Dijkstra's note on a related topic][note]. +/// +/// [this discussion]: https://github.com/rust-lang/regex/discussions/866 +/// [note]: https://www.cs.utexas.edu/users/EWD/transcriptions/EWD08xx/EWD831.html +/// +/// # Example +/// +/// This example shows the value of each of the methods on `Match` for a +/// particular search. +/// +/// ``` +/// use regex::Regex; +/// +/// let re = Regex::new(r"\p{Greek}+").unwrap(); +/// let hay = "Greek: αβγδ"; +/// let m = re.find(hay).unwrap(); +/// assert_eq!(7, m.start()); +/// assert_eq!(15, m.end()); +/// assert!(!m.is_empty()); +/// assert_eq!(8, m.len()); +/// assert_eq!(7..15, m.range()); +/// assert_eq!("αβγδ", m.as_str()); +/// ``` +#[derive(Copy, Clone, Eq, PartialEq)] +pub struct Match<'h> { + haystack: &'h str, + start: usize, + end: usize, +} + +impl<'h> Match<'h> { + /// Returns the byte offset of the start of the match in the haystack. The + /// start of the match corresponds to the position where the match begins + /// and includes the first byte in the match. + /// + /// It is guaranteed that `Match::start() <= Match::end()`. + /// + /// This is guaranteed to fall on a valid UTF-8 codepoint boundary. That + /// is, it will never be an offset that appears between the UTF-8 code + /// units of a UTF-8 encoded Unicode scalar value. Consequently, it is + /// always safe to slice the corresponding haystack using this offset. + #[inline] + pub fn start(&self) -> usize { + self.start + } + + /// Returns the byte offset of the end of the match in the haystack. The + /// end of the match corresponds to the byte immediately following the last + /// byte in the match. This means that `&slice[start..end]` works as one + /// would expect. + /// + /// It is guaranteed that `Match::start() <= Match::end()`. + /// + /// This is guaranteed to fall on a valid UTF-8 codepoint boundary. That + /// is, it will never be an offset that appears between the UTF-8 code + /// units of a UTF-8 encoded Unicode scalar value. Consequently, it is + /// always safe to slice the corresponding haystack using this offset. + #[inline] + pub fn end(&self) -> usize { + self.end + } + + /// Returns true if and only if this match has a length of zero. + /// + /// Note that an empty match can only occur when the regex itself can + /// match the empty string. Here are some examples of regexes that can + /// all match the empty string: `^`, `^$`, `\b`, `a?`, `a*`, `a{0}`, + /// `(foo|\d+|quux)?`. + #[inline] + pub fn is_empty(&self) -> bool { + self.start == self.end + } + + /// Returns the length, in bytes, of this match. + #[inline] + pub fn len(&self) -> usize { + self.end - self.start + } + + /// Returns the range over the starting and ending byte offsets of the + /// match in the haystack. + /// + /// It is always correct to slice the original haystack searched with this + /// range. That is, because the offsets are guaranteed to fall on valid + /// UTF-8 boundaries, the range returned is always valid. + #[inline] + pub fn range(&self) -> core::ops::Range<usize> { + self.start..self.end + } + + /// Returns the substring of the haystack that matched. + #[inline] + pub fn as_str(&self) -> &'h str { + &self.haystack[self.range()] + } + + /// Creates a new match from the given haystack and byte offsets. + #[inline] + fn new(haystack: &'h str, start: usize, end: usize) -> Match<'h> { + Match { haystack, start, end } + } +} + +impl<'h> core::fmt::Debug for Match<'h> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_struct("Match") + .field("start", &self.start) + .field("end", &self.end) + .field("string", &self.as_str()) + .finish() + } +} + +impl<'h> From<Match<'h>> for &'h str { + fn from(m: Match<'h>) -> &'h str { + m.as_str() + } +} + +impl<'h> From<Match<'h>> for core::ops::Range<usize> { + fn from(m: Match<'h>) -> core::ops::Range<usize> { + m.range() + } +} + +/// Represents the capture groups for a single match. +/// +/// Capture groups refer to parts of a regex enclosed in parentheses. They +/// can be optionally named. The purpose of capture groups is to be able to +/// reference different parts of a match based on the original pattern. In +/// essence, a `Captures` is a container of [`Match`] values for each group +/// that participated in a regex match. Each `Match` can be looked up by either +/// its capture group index or name (if it has one). +/// +/// For example, say you want to match the individual letters in a 5-letter +/// word: +/// +/// ```text +/// (?<first>\w)(\w)(?:\w)\w(?<last>\w) +/// ``` +/// +/// This regex has 4 capture groups: +/// +/// * The group at index `0` corresponds to the overall match. It is always +/// present in every match and never has a name. +/// * The group at index `1` with name `first` corresponding to the first +/// letter. +/// * The group at index `2` with no name corresponding to the second letter. +/// * The group at index `3` with name `last` corresponding to the fifth and +/// last letter. +/// +/// Notice that `(?:\w)` was not listed above as a capture group despite it +/// being enclosed in parentheses. That's because `(?:pattern)` is a special +/// syntax that permits grouping but *without* capturing. The reason for not +/// treating it as a capture is that tracking and reporting capture groups +/// requires additional state that may lead to slower searches. So using as few +/// capture groups as possible can help performance. (Although the difference +/// in performance of a couple of capture groups is likely immaterial.) +/// +/// Values with this type are created by [`Regex::captures`] or +/// [`Regex::captures_iter`]. +/// +/// `'h` is the lifetime of the haystack that these captures were matched from. +/// +/// # Example +/// +/// ``` +/// use regex::Regex; +/// +/// let re = Regex::new(r"(?<first>\w)(\w)(?:\w)\w(?<last>\w)").unwrap(); +/// let caps = re.captures("toady").unwrap(); +/// assert_eq!("toady", &caps[0]); +/// assert_eq!("t", &caps["first"]); +/// assert_eq!("o", &caps[2]); +/// assert_eq!("y", &caps["last"]); +/// ``` +pub struct Captures<'h> { + haystack: &'h str, + caps: captures::Captures, + static_captures_len: Option<usize>, +} + +impl<'h> Captures<'h> { + /// Returns the `Match` associated with the capture group at index `i`. If + /// `i` does not correspond to a capture group, or if the capture group did + /// not participate in the match, then `None` is returned. + /// + /// When `i == 0`, this is guaranteed to return a non-`None` value. + /// + /// # Examples + /// + /// Get the substring that matched with a default of an empty string if the + /// group didn't participate in the match: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"[a-z]+(?:([0-9]+)|([A-Z]+))").unwrap(); + /// let caps = re.captures("abc123").unwrap(); + /// + /// let substr1 = caps.get(1).map_or("", |m| m.as_str()); + /// let substr2 = caps.get(2).map_or("", |m| m.as_str()); + /// assert_eq!(substr1, "123"); + /// assert_eq!(substr2, ""); + /// ``` + #[inline] + pub fn get(&self, i: usize) -> Option<Match<'h>> { + self.caps + .get_group(i) + .map(|sp| Match::new(self.haystack, sp.start, sp.end)) + } + + /// Return the overall match for the capture. + /// + /// This returns the match for index `0`. That is it is equivalent to + /// `m.get(0).unwrap()` + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"[a-z]+([0-9]+)").unwrap(); + /// let caps = re.captures(" abc123-def").unwrap(); + /// + /// assert_eq!(caps.get_match().as_str(), "abc123"); + /// + /// ``` + #[inline] + pub fn get_match(&self) -> Match<'h> { + self.get(0).unwrap() + } + + /// Returns the `Match` associated with the capture group named `name`. If + /// `name` isn't a valid capture group or it refers to a group that didn't + /// match, then `None` is returned. + /// + /// Note that unlike `caps["name"]`, this returns a `Match` whose lifetime + /// matches the lifetime of the haystack in this `Captures` value. + /// Conversely, the substring returned by `caps["name"]` has a lifetime + /// of the `Captures` value, which is likely shorter than the lifetime of + /// the haystack. In some cases, it may be necessary to use this method to + /// access the matching substring instead of the `caps["name"]` notation. + /// + /// # Examples + /// + /// Get the substring that matched with a default of an empty string if the + /// group didn't participate in the match: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new( + /// r"[a-z]+(?:(?<numbers>[0-9]+)|(?<letters>[A-Z]+))", + /// ).unwrap(); + /// let caps = re.captures("abc123").unwrap(); + /// + /// let numbers = caps.name("numbers").map_or("", |m| m.as_str()); + /// let letters = caps.name("letters").map_or("", |m| m.as_str()); + /// assert_eq!(numbers, "123"); + /// assert_eq!(letters, ""); + /// ``` + #[inline] + pub fn name(&self, name: &str) -> Option<Match<'h>> { + self.caps + .get_group_by_name(name) + .map(|sp| Match::new(self.haystack, sp.start, sp.end)) + } + + /// This is a convenience routine for extracting the substrings + /// corresponding to matching capture groups. + /// + /// This returns a tuple where the first element corresponds to the full + /// substring of the haystack that matched the regex. The second element is + /// an array of substrings, with each corresponding to the substring that + /// matched for a particular capture group. + /// + /// # Panics + /// + /// This panics if the number of possible matching groups in this + /// `Captures` value is not fixed to `N` in all circumstances. + /// More precisely, this routine only works when `N` is equivalent to + /// [`Regex::static_captures_len`]. + /// + /// Stated more plainly, if the number of matching capture groups in a + /// regex can vary from match to match, then this function always panics. + /// + /// For example, `(a)(b)|(c)` could produce two matching capture groups + /// or one matching capture group for any given match. Therefore, one + /// cannot use `extract` with such a pattern. + /// + /// But a pattern like `(a)(b)|(c)(d)` can be used with `extract` because + /// the number of capture groups in every match is always equivalent, + /// even if the capture _indices_ in each match are not. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"([0-9]{4})-([0-9]{2})-([0-9]{2})").unwrap(); + /// let hay = "On 2010-03-14, I became a Tennessee lamb."; + /// let Some((full, [year, month, day])) = + /// re.captures(hay).map(|caps| caps.extract()) else { return }; + /// assert_eq!("2010-03-14", full); + /// assert_eq!("2010", year); + /// assert_eq!("03", month); + /// assert_eq!("14", day); + /// ``` + /// + /// # Example: iteration + /// + /// This example shows how to use this method when iterating over all + /// `Captures` matches in a haystack. + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"([0-9]{4})-([0-9]{2})-([0-9]{2})").unwrap(); + /// let hay = "1973-01-05, 1975-08-25 and 1980-10-18"; + /// + /// let mut dates: Vec<(&str, &str, &str)> = vec![]; + /// for (_, [y, m, d]) in re.captures_iter(hay).map(|c| c.extract()) { + /// dates.push((y, m, d)); + /// } + /// assert_eq!(dates, vec![ + /// ("1973", "01", "05"), + /// ("1975", "08", "25"), + /// ("1980", "10", "18"), + /// ]); + /// ``` + /// + /// # Example: parsing different formats + /// + /// This API is particularly useful when you need to extract a particular + /// value that might occur in a different format. Consider, for example, + /// an identifier that might be in double quotes or single quotes: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r#"id:(?:"([^"]+)"|'([^']+)')"#).unwrap(); + /// let hay = r#"The first is id:"foo" and the second is id:'bar'."#; + /// let mut ids = vec![]; + /// for (_, [id]) in re.captures_iter(hay).map(|c| c.extract()) { + /// ids.push(id); + /// } + /// assert_eq!(ids, vec!["foo", "bar"]); + /// ``` + pub fn extract<const N: usize>(&self) -> (&'h str, [&'h str; N]) { + let len = self + .static_captures_len + .expect("number of capture groups can vary in a match") + .checked_sub(1) + .expect("number of groups is always greater than zero"); + assert_eq!(N, len, "asked for {N} groups, but must ask for {len}"); + // The regex-automata variant of extract is a bit more permissive. + // It doesn't require the number of matching capturing groups to be + // static, and you can even request fewer groups than what's there. So + // this is guaranteed to never panic because we've asserted above that + // the user has requested precisely the number of groups that must be + // present in any match for this regex. + self.caps.extract(self.haystack) + } + + /// Expands all instances of `$ref` in `replacement` to the corresponding + /// capture group, and writes them to the `dst` buffer given. A `ref` can + /// be a capture group index or a name. If `ref` doesn't refer to a capture + /// group that participated in the match, then it is replaced with the + /// empty string. + /// + /// # Format + /// + /// The format of the replacement string supports two different kinds of + /// capture references: unbraced and braced. + /// + /// For the unbraced format, the format supported is `$ref` where `name` + /// can be any character in the class `[0-9A-Za-z_]`. `ref` is always + /// the longest possible parse. So for example, `$1a` corresponds to the + /// capture group named `1a` and not the capture group at index `1`. If + /// `ref` matches `^[0-9]+$`, then it is treated as a capture group index + /// itself and not a name. + /// + /// For the braced format, the format supported is `${ref}` where `ref` can + /// be any sequence of bytes except for `}`. If no closing brace occurs, + /// then it is not considered a capture reference. As with the unbraced + /// format, if `ref` matches `^[0-9]+$`, then it is treated as a capture + /// group index and not a name. + /// + /// The braced format is useful for exerting precise control over the name + /// of the capture reference. For example, `${1}a` corresponds to the + /// capture group reference `1` followed by the letter `a`, where as `$1a` + /// (as mentioned above) corresponds to the capture group reference `1a`. + /// The braced format is also useful for expressing capture group names + /// that use characters not supported by the unbraced format. For example, + /// `${foo[bar].baz}` refers to the capture group named `foo[bar].baz`. + /// + /// If a capture group reference is found and it does not refer to a valid + /// capture group, then it will be replaced with the empty string. + /// + /// To write a literal `$`, use `$$`. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new( + /// r"(?<day>[0-9]{2})-(?<month>[0-9]{2})-(?<year>[0-9]{4})", + /// ).unwrap(); + /// let hay = "On 14-03-2010, I became a Tennessee lamb."; + /// let caps = re.captures(hay).unwrap(); + /// + /// let mut dst = String::new(); + /// caps.expand("year=$year, month=$month, day=$day", &mut dst); + /// assert_eq!(dst, "year=2010, month=03, day=14"); + /// ``` + #[inline] + pub fn expand(&self, replacement: &str, dst: &mut String) { + self.caps.interpolate_string_into(self.haystack, replacement, dst); + } + + /// Returns an iterator over all capture groups. This includes both + /// matching and non-matching groups. + /// + /// The iterator always yields at least one matching group: the first group + /// (at index `0`) with no name. Subsequent groups are returned in the order + /// of their opening parenthesis in the regex. + /// + /// The elements yielded have type `Option<Match<'h>>`, where a non-`None` + /// value is present if the capture group matches. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(\w)(\d)?(\w)").unwrap(); + /// let caps = re.captures("AZ").unwrap(); + /// + /// let mut it = caps.iter(); + /// assert_eq!(it.next().unwrap().map(|m| m.as_str()), Some("AZ")); + /// assert_eq!(it.next().unwrap().map(|m| m.as_str()), Some("A")); + /// assert_eq!(it.next().unwrap().map(|m| m.as_str()), None); + /// assert_eq!(it.next().unwrap().map(|m| m.as_str()), Some("Z")); + /// assert_eq!(it.next(), None); + /// ``` + #[inline] + pub fn iter<'c>(&'c self) -> SubCaptureMatches<'c, 'h> { + SubCaptureMatches { haystack: self.haystack, it: self.caps.iter() } + } + + /// Returns the total number of capture groups. This includes both + /// matching and non-matching groups. + /// + /// The length returned is always equivalent to the number of elements + /// yielded by [`Captures::iter`]. Consequently, the length is always + /// greater than zero since every `Captures` value always includes the + /// match for the entire regex. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(\w)(\d)?(\w)").unwrap(); + /// let caps = re.captures("AZ").unwrap(); + /// assert_eq!(caps.len(), 4); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.caps.group_len() + } +} + +impl<'h> core::fmt::Debug for Captures<'h> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + /// A little helper type to provide a nice map-like debug + /// representation for our capturing group spans. + /// + /// regex-automata has something similar, but it includes the pattern + /// ID in its debug output, which is confusing. It also doesn't include + /// that strings that match because a regex-automata `Captures` doesn't + /// borrow the haystack. + struct CapturesDebugMap<'a> { + caps: &'a Captures<'a>, + } + + impl<'a> core::fmt::Debug for CapturesDebugMap<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut map = f.debug_map(); + let names = + self.caps.caps.group_info().pattern_names(PatternID::ZERO); + for (group_index, maybe_name) in names.enumerate() { + let key = Key(group_index, maybe_name); + match self.caps.get(group_index) { + None => map.entry(&key, &None::<()>), + Some(mat) => map.entry(&key, &Value(mat)), + }; + } + map.finish() + } + } + + struct Key<'a>(usize, Option<&'a str>); + + impl<'a> core::fmt::Debug for Key<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{}", self.0)?; + if let Some(name) = self.1 { + write!(f, "/{name:?}")?; + } + Ok(()) + } + } + + struct Value<'a>(Match<'a>); + + impl<'a> core::fmt::Debug for Value<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "{}..{}/{:?}", + self.0.start(), + self.0.end(), + self.0.as_str() + ) + } + } + + f.debug_tuple("Captures") + .field(&CapturesDebugMap { caps: self }) + .finish() + } +} + +/// Get a matching capture group's haystack substring by index. +/// +/// The haystack substring returned can't outlive the `Captures` object if this +/// method is used, because of how `Index` is defined (normally `a[i]` is part +/// of `a` and can't outlive it). To work around this limitation, do that, use +/// [`Captures::get`] instead. +/// +/// `'h` is the lifetime of the matched haystack, but the lifetime of the +/// `&str` returned by this implementation is the lifetime of the `Captures` +/// value itself. +/// +/// # Panics +/// +/// If there is no matching group at the given index. +impl<'h> core::ops::Index<usize> for Captures<'h> { + type Output = str; + + // The lifetime is written out to make it clear that the &str returned + // does NOT have a lifetime equivalent to 'h. + fn index<'a>(&'a self, i: usize) -> &'a str { + self.get(i) + .map(|m| m.as_str()) + .unwrap_or_else(|| panic!("no group at index '{i}'")) + } +} + +/// Get a matching capture group's haystack substring by name. +/// +/// The haystack substring returned can't outlive the `Captures` object if this +/// method is used, because of how `Index` is defined (normally `a[i]` is part +/// of `a` and can't outlive it). To work around this limitation, do that, use +/// [`Captures::name`] instead. +/// +/// `'h` is the lifetime of the matched haystack, but the lifetime of the +/// `&str` returned by this implementation is the lifetime of the `Captures` +/// value itself. +/// +/// `'n` is the lifetime of the group name used to index the `Captures` value. +/// +/// # Panics +/// +/// If there is no matching group at the given name. +impl<'h, 'n> core::ops::Index<&'n str> for Captures<'h> { + type Output = str; + + fn index<'a>(&'a self, name: &'n str) -> &'a str { + self.name(name) + .map(|m| m.as_str()) + .unwrap_or_else(|| panic!("no group named '{name}'")) + } +} + +/// A low level representation of the byte offsets of each capture group. +/// +/// You can think of this as a lower level [`Captures`], where this type does +/// not support named capturing groups directly and it does not borrow the +/// haystack that these offsets were matched on. +/// +/// Primarily, this type is useful when using the lower level `Regex` APIs such +/// as [`Regex::captures_read`], which permits amortizing the allocation in +/// which capture match offsets are stored. +/// +/// In order to build a value of this type, you'll need to call the +/// [`Regex::capture_locations`] method. The value returned can then be reused +/// in subsequent searches for that regex. Using it for other regexes may +/// result in a panic or otherwise incorrect results. +/// +/// # Example +/// +/// This example shows how to create and use `CaptureLocations` in a search. +/// +/// ``` +/// use regex::Regex; +/// +/// let re = Regex::new(r"(?<first>\w+)\s+(?<last>\w+)").unwrap(); +/// let mut locs = re.capture_locations(); +/// let m = re.captures_read(&mut locs, "Bruce Springsteen").unwrap(); +/// assert_eq!(0..17, m.range()); +/// assert_eq!(Some((0, 17)), locs.get(0)); +/// assert_eq!(Some((0, 5)), locs.get(1)); +/// assert_eq!(Some((6, 17)), locs.get(2)); +/// +/// // Asking for an invalid capture group always returns None. +/// assert_eq!(None, locs.get(3)); +/// # // literals are too big for 32-bit usize: #1041 +/// # #[cfg(target_pointer_width = "64")] +/// assert_eq!(None, locs.get(34973498648)); +/// # #[cfg(target_pointer_width = "64")] +/// assert_eq!(None, locs.get(9944060567225171988)); +/// ``` +#[derive(Clone, Debug)] +pub struct CaptureLocations(captures::Captures); + +/// A type alias for `CaptureLocations` for backwards compatibility. +/// +/// Previously, we exported `CaptureLocations` as `Locations` in an +/// undocumented API. To prevent breaking that code (e.g., in `regex-capi`), +/// we continue re-exporting the same undocumented API. +#[doc(hidden)] +pub type Locations = CaptureLocations; + +impl CaptureLocations { + /// Returns the start and end byte offsets of the capture group at index + /// `i`. This returns `None` if `i` is not a valid capture group or if the + /// capture group did not match. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(?<first>\w+)\s+(?<last>\w+)").unwrap(); + /// let mut locs = re.capture_locations(); + /// re.captures_read(&mut locs, "Bruce Springsteen").unwrap(); + /// assert_eq!(Some((0, 17)), locs.get(0)); + /// assert_eq!(Some((0, 5)), locs.get(1)); + /// assert_eq!(Some((6, 17)), locs.get(2)); + /// ``` + #[inline] + pub fn get(&self, i: usize) -> Option<(usize, usize)> { + self.0.get_group(i).map(|sp| (sp.start, sp.end)) + } + + /// Returns the total number of capture groups (even if they didn't match). + /// That is, the length returned is unaffected by the result of a search. + /// + /// This is always at least `1` since every regex has at least `1` + /// capturing group that corresponds to the entire match. + /// + /// # Example + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"(?<first>\w+)\s+(?<last>\w+)").unwrap(); + /// let mut locs = re.capture_locations(); + /// assert_eq!(3, locs.len()); + /// re.captures_read(&mut locs, "Bruce Springsteen").unwrap(); + /// assert_eq!(3, locs.len()); + /// ``` + /// + /// Notice that the length is always at least `1`, regardless of the regex: + /// + /// ``` + /// use regex::Regex; + /// + /// let re = Regex::new(r"").unwrap(); + /// let locs = re.capture_locations(); + /// assert_eq!(1, locs.len()); + /// + /// // [a&&b] is a regex that never matches anything. + /// let re = Regex::new(r"[a&&b]").unwrap(); + /// let locs = re.capture_locations(); + /// assert_eq!(1, locs.len()); + /// ``` + #[inline] + pub fn len(&self) -> usize { + // self.0.group_len() returns 0 if the underlying captures doesn't + // represent a match, but the behavior guaranteed for this method is + // that the length doesn't change based on a match or not. + self.0.group_info().group_len(PatternID::ZERO) + } + + /// An alias for the `get` method for backwards compatibility. + /// + /// Previously, we exported `get` as `pos` in an undocumented API. To + /// prevent breaking that code (e.g., in `regex-capi`), we continue + /// re-exporting the same undocumented API. + #[doc(hidden)] + #[inline] + pub fn pos(&self, i: usize) -> Option<(usize, usize)> { + self.get(i) + } +} + +/// An iterator over all non-overlapping matches in a haystack. +/// +/// This iterator yields [`Match`] values. The iterator stops when no more +/// matches can be found. +/// +/// `'r` is the lifetime of the compiled regular expression and `'h` is the +/// lifetime of the haystack. +/// +/// This iterator is created by [`Regex::find_iter`]. +/// +/// # Time complexity +/// +/// Note that since an iterator runs potentially many searches on the haystack +/// and since each search has worst case `O(m * n)` time complexity, the +/// overall worst case time complexity for iteration is `O(m * n^2)`. +#[derive(Debug)] +pub struct Matches<'r, 'h> { + haystack: &'h str, + it: meta::FindMatches<'r, 'h>, +} + +impl<'r, 'h> Iterator for Matches<'r, 'h> { + type Item = Match<'h>; + + #[inline] + fn next(&mut self) -> Option<Match<'h>> { + self.it + .next() + .map(|sp| Match::new(self.haystack, sp.start(), sp.end())) + } + + #[inline] + fn count(self) -> usize { + // This can actually be up to 2x faster than calling `next()` until + // completion, because counting matches when using a DFA only requires + // finding the end of each match. But returning a `Match` via `next()` + // requires the start of each match which, with a DFA, requires a + // reverse forward scan to find it. + self.it.count() + } +} + +impl<'r, 'h> core::iter::FusedIterator for Matches<'r, 'h> {} + +/// An iterator over all non-overlapping capture matches in a haystack. +/// +/// This iterator yields [`Captures`] values. The iterator stops when no more +/// matches can be found. +/// +/// `'r` is the lifetime of the compiled regular expression and `'h` is the +/// lifetime of the matched string. +/// +/// This iterator is created by [`Regex::captures_iter`]. +/// +/// # Time complexity +/// +/// Note that since an iterator runs potentially many searches on the haystack +/// and since each search has worst case `O(m * n)` time complexity, the +/// overall worst case time complexity for iteration is `O(m * n^2)`. +#[derive(Debug)] +pub struct CaptureMatches<'r, 'h> { + haystack: &'h str, + it: meta::CapturesMatches<'r, 'h>, +} + +impl<'r, 'h> Iterator for CaptureMatches<'r, 'h> { + type Item = Captures<'h>; + + #[inline] + fn next(&mut self) -> Option<Captures<'h>> { + let static_captures_len = self.it.regex().static_captures_len(); + self.it.next().map(|caps| Captures { + haystack: self.haystack, + caps, + static_captures_len, + }) + } + + #[inline] + fn count(self) -> usize { + // This can actually be up to 2x faster than calling `next()` until + // completion, because counting matches when using a DFA only requires + // finding the end of each match. But returning a `Match` via `next()` + // requires the start of each match which, with a DFA, requires a + // reverse forward scan to find it. + self.it.count() + } +} + +impl<'r, 'h> core::iter::FusedIterator for CaptureMatches<'r, 'h> {} + +/// An iterator over all substrings delimited by a regex match. +/// +/// `'r` is the lifetime of the compiled regular expression and `'h` is the +/// lifetime of the byte string being split. +/// +/// This iterator is created by [`Regex::split`]. +/// +/// # Time complexity +/// +/// Note that since an iterator runs potentially many searches on the haystack +/// and since each search has worst case `O(m * n)` time complexity, the +/// overall worst case time complexity for iteration is `O(m * n^2)`. +#[derive(Debug)] +pub struct Split<'r, 'h> { + haystack: &'h str, + it: meta::Split<'r, 'h>, +} + +impl<'r, 'h> Iterator for Split<'r, 'h> { + type Item = &'h str; + + #[inline] + fn next(&mut self) -> Option<&'h str> { + self.it.next().map(|span| &self.haystack[span]) + } +} + +impl<'r, 'h> core::iter::FusedIterator for Split<'r, 'h> {} + +/// An iterator over at most `N` substrings delimited by a regex match. +/// +/// The last substring yielded by this iterator will be whatever remains after +/// `N-1` splits. +/// +/// `'r` is the lifetime of the compiled regular expression and `'h` is the +/// lifetime of the byte string being split. +/// +/// This iterator is created by [`Regex::splitn`]. +/// +/// # Time complexity +/// +/// Note that since an iterator runs potentially many searches on the haystack +/// and since each search has worst case `O(m * n)` time complexity, the +/// overall worst case time complexity for iteration is `O(m * n^2)`. +/// +/// Although note that the worst case time here has an upper bound given +/// by the `limit` parameter to [`Regex::splitn`]. +#[derive(Debug)] +pub struct SplitN<'r, 'h> { + haystack: &'h str, + it: meta::SplitN<'r, 'h>, +} + +impl<'r, 'h> Iterator for SplitN<'r, 'h> { + type Item = &'h str; + + #[inline] + fn next(&mut self) -> Option<&'h str> { + self.it.next().map(|span| &self.haystack[span]) + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + self.it.size_hint() + } +} + +impl<'r, 'h> core::iter::FusedIterator for SplitN<'r, 'h> {} + +/// An iterator over the names of all capture groups in a regex. +/// +/// This iterator yields values of type `Option<&str>` in order of the opening +/// capture group parenthesis in the regex pattern. `None` is yielded for +/// groups with no name. The first element always corresponds to the implicit +/// and unnamed group for the overall match. +/// +/// `'r` is the lifetime of the compiled regular expression. +/// +/// This iterator is created by [`Regex::capture_names`]. +#[derive(Clone, Debug)] +pub struct CaptureNames<'r>(captures::GroupInfoPatternNames<'r>); + +impl<'r> Iterator for CaptureNames<'r> { + type Item = Option<&'r str>; + + #[inline] + fn next(&mut self) -> Option<Option<&'r str>> { + self.0.next() + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + self.0.size_hint() + } + + #[inline] + fn count(self) -> usize { + self.0.count() + } +} + +impl<'r> ExactSizeIterator for CaptureNames<'r> {} + +impl<'r> core::iter::FusedIterator for CaptureNames<'r> {} + +/// An iterator over all group matches in a [`Captures`] value. +/// +/// This iterator yields values of type `Option<Match<'h>>`, where `'h` is the +/// lifetime of the haystack that the matches are for. The order of elements +/// yielded corresponds to the order of the opening parenthesis for the group +/// in the regex pattern. `None` is yielded for groups that did not participate +/// in the match. +/// +/// The first element always corresponds to the implicit group for the overall +/// match. Since this iterator is created by a [`Captures`] value, and a +/// `Captures` value is only created when a match occurs, it follows that the +/// first element yielded by this iterator is guaranteed to be non-`None`. +/// +/// The lifetime `'c` corresponds to the lifetime of the `Captures` value that +/// created this iterator, and the lifetime `'h` corresponds to the originally +/// matched haystack. +#[derive(Clone, Debug)] +pub struct SubCaptureMatches<'c, 'h> { + haystack: &'h str, + it: captures::CapturesPatternIter<'c>, +} + +impl<'c, 'h> Iterator for SubCaptureMatches<'c, 'h> { + type Item = Option<Match<'h>>; + + #[inline] + fn next(&mut self) -> Option<Option<Match<'h>>> { + self.it.next().map(|group| { + group.map(|sp| Match::new(self.haystack, sp.start, sp.end)) + }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option<usize>) { + self.it.size_hint() + } + + #[inline] + fn count(self) -> usize { + self.it.count() + } +} + +impl<'c, 'h> ExactSizeIterator for SubCaptureMatches<'c, 'h> {} + +impl<'c, 'h> core::iter::FusedIterator for SubCaptureMatches<'c, 'h> {} + +/// A trait for types that can be used to replace matches in a haystack. +/// +/// In general, users of this crate shouldn't need to implement this trait, +/// since implementations are already provided for `&str` along with other +/// variants of string types, as well as `FnMut(&Captures) -> String` (or any +/// `FnMut(&Captures) -> T` where `T: AsRef<str>`). Those cover most use cases, +/// but callers can implement this trait directly if necessary. +/// +/// # Example +/// +/// This example shows a basic implementation of the `Replacer` trait. This +/// can be done much more simply using the replacement string interpolation +/// support (e.g., `$first $last`), but this approach avoids needing to parse +/// the replacement string at all. +/// +/// ``` +/// use regex::{Captures, Regex, Replacer}; +/// +/// struct NameSwapper; +/// +/// impl Replacer for NameSwapper { +/// fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { +/// dst.push_str(&caps["first"]); +/// dst.push_str(" "); +/// dst.push_str(&caps["last"]); +/// } +/// } +/// +/// let re = Regex::new(r"(?<last>[^,\s]+),\s+(?<first>\S+)").unwrap(); +/// let result = re.replace("Springsteen, Bruce", NameSwapper); +/// assert_eq!(result, "Bruce Springsteen"); +/// ``` +pub trait Replacer { + /// Appends possibly empty data to `dst` to replace the current match. + /// + /// The current match is represented by `caps`, which is guaranteed to + /// have a match at capture group `0`. + /// + /// For example, a no-op replacement would be `dst.push_str(&caps[0])`. + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String); + + /// Return a fixed unchanging replacement string. + /// + /// When doing replacements, if access to [`Captures`] is not needed (e.g., + /// the replacement string does not need `$` expansion), then it can be + /// beneficial to avoid finding sub-captures. + /// + /// In general, this is called once for every call to a replacement routine + /// such as [`Regex::replace_all`]. + fn no_expansion<'r>(&'r mut self) -> Option<Cow<'r, str>> { + None + } + + /// Returns a type that implements `Replacer`, but that borrows and wraps + /// this `Replacer`. + /// + /// This is useful when you want to take a generic `Replacer` (which might + /// not be cloneable) and use it without consuming it, so it can be used + /// more than once. + /// + /// # Example + /// + /// ``` + /// use regex::{Regex, Replacer}; + /// + /// fn replace_all_twice<R: Replacer>( + /// re: Regex, + /// src: &str, + /// mut rep: R, + /// ) -> String { + /// let dst = re.replace_all(src, rep.by_ref()); + /// let dst = re.replace_all(&dst, rep.by_ref()); + /// dst.into_owned() + /// } + /// ``` + fn by_ref<'r>(&'r mut self) -> ReplacerRef<'r, Self> { + ReplacerRef(self) + } +} + +impl<'a> Replacer for &'a str { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { + caps.expand(*self, dst); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, str>> { + no_expansion(self) + } +} + +impl<'a> Replacer for &'a String { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { + self.as_str().replace_append(caps, dst) + } + + fn no_expansion(&mut self) -> Option<Cow<'_, str>> { + no_expansion(self) + } +} + +impl Replacer for String { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { + self.as_str().replace_append(caps, dst) + } + + fn no_expansion(&mut self) -> Option<Cow<'_, str>> { + no_expansion(self) + } +} + +impl<'a> Replacer for Cow<'a, str> { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { + self.as_ref().replace_append(caps, dst) + } + + fn no_expansion(&mut self) -> Option<Cow<'_, str>> { + no_expansion(self) + } +} + +impl<'a> Replacer for &'a Cow<'a, str> { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { + self.as_ref().replace_append(caps, dst) + } + + fn no_expansion(&mut self) -> Option<Cow<'_, str>> { + no_expansion(self) + } +} + +impl<F, T> Replacer for F +where + F: FnMut(&Captures<'_>) -> T, + T: AsRef<str>, +{ + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { + dst.push_str((*self)(caps).as_ref()); + } +} + +/// A by-reference adaptor for a [`Replacer`]. +/// +/// This permits reusing the same `Replacer` value in multiple calls to a +/// replacement routine like [`Regex::replace_all`]. +/// +/// This type is created by [`Replacer::by_ref`]. +#[derive(Debug)] +pub struct ReplacerRef<'a, R: ?Sized>(&'a mut R); + +impl<'a, R: Replacer + ?Sized + 'a> Replacer for ReplacerRef<'a, R> { + fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { + self.0.replace_append(caps, dst) + } + + fn no_expansion(&mut self) -> Option<Cow<'_, str>> { + self.0.no_expansion() + } +} + +/// A helper type for forcing literal string replacement. +/// +/// It can be used with routines like [`Regex::replace`] and +/// [`Regex::replace_all`] to do a literal string replacement without expanding +/// `$name` to their corresponding capture groups. This can be both convenient +/// (to avoid escaping `$`, for example) and faster (since capture groups +/// don't need to be found). +/// +/// `'s` is the lifetime of the literal string to use. +/// +/// # Example +/// +/// ``` +/// use regex::{NoExpand, Regex}; +/// +/// let re = Regex::new(r"(?<last>[^,\s]+),\s+(\S+)").unwrap(); +/// let result = re.replace("Springsteen, Bruce", NoExpand("$2 $last")); +/// assert_eq!(result, "$2 $last"); +/// ``` +#[derive(Clone, Debug)] +pub struct NoExpand<'s>(pub &'s str); + +impl<'s> Replacer for NoExpand<'s> { + fn replace_append(&mut self, _: &Captures<'_>, dst: &mut String) { + dst.push_str(self.0); + } + + fn no_expansion(&mut self) -> Option<Cow<'_, str>> { + Some(Cow::Borrowed(self.0)) + } +} + +/// Quickly checks the given replacement string for whether interpolation +/// should be done on it. It returns `None` if a `$` was found anywhere in the +/// given string, which suggests interpolation needs to be done. But if there's +/// no `$` anywhere, then interpolation definitely does not need to be done. In +/// that case, the given string is returned as a borrowed `Cow`. +/// +/// This is meant to be used to implement the [`Replacer::no_expansion`] method +/// in its various trait impls. +fn no_expansion<T: AsRef<str>>(replacement: &T) -> Option<Cow<'_, str>> { + let replacement = replacement.as_ref(); + match crate::find_byte::find_byte(b'$', replacement.as_bytes()) { + Some(_) => None, + None => Some(Cow::Borrowed(replacement)), + } +} diff --git a/third_party/crates/regex-1.13.1/src/regexset/bytes.rs b/third_party/crates/regex-1.13.1/src/regexset/bytes.rs new file mode 100644 index 0000000..76174af --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/regexset/bytes.rs @@ -0,0 +1,728 @@ +use alloc::string::String; + +use regex_automata::{meta, Input, PatternID, PatternSet, PatternSetIter}; + +use crate::{bytes::RegexSetBuilder, Error}; + +/// Match multiple, possibly overlapping, regexes in a single search. +/// +/// A regex set corresponds to the union of zero or more regular expressions. +/// That is, a regex set will match a haystack when at least one of its +/// constituent regexes matches. A regex set as its formulated here provides a +/// touch more power: it will also report *which* regular expressions in the +/// set match. Indeed, this is the key difference between regex sets and a +/// single `Regex` with many alternates, since only one alternate can match at +/// a time. +/// +/// For example, consider regular expressions to match email addresses and +/// domains: `[a-z]+@[a-z]+\.(com|org|net)` and `[a-z]+\.(com|org|net)`. If a +/// regex set is constructed from those regexes, then searching the haystack +/// `foo@example.com` will report both regexes as matching. Of course, one +/// could accomplish this by compiling each regex on its own and doing two +/// searches over the haystack. The key advantage of using a regex set is +/// that it will report the matching regexes using a *single pass through the +/// haystack*. If one has hundreds or thousands of regexes to match repeatedly +/// (like a URL router for a complex web application or a user agent matcher), +/// then a regex set *can* realize huge performance gains. +/// +/// Unlike the top-level [`RegexSet`](crate::RegexSet), this `RegexSet` +/// searches haystacks with type `&[u8]` instead of `&str`. Consequently, this +/// `RegexSet` is permitted to match invalid UTF-8. +/// +/// # Limitations +/// +/// Regex sets are limited to answering the following two questions: +/// +/// 1. Does any regex in the set match? +/// 2. If so, which regexes in the set match? +/// +/// As with the main [`Regex`][crate::bytes::Regex] type, it is cheaper to ask +/// (1) instead of (2) since the matching engines can stop after the first +/// match is found. +/// +/// You cannot directly extract [`Match`][crate::bytes::Match] or +/// [`Captures`][crate::bytes::Captures] objects from a regex set. If you need +/// these operations, the recommended approach is to compile each pattern in +/// the set independently and scan the exact same haystack a second time with +/// those independently compiled patterns: +/// +/// ``` +/// use regex::bytes::{Regex, RegexSet}; +/// +/// let patterns = ["foo", "bar"]; +/// // Both patterns will match different ranges of this string. +/// let hay = b"barfoo"; +/// +/// // Compile a set matching any of our patterns. +/// let set = RegexSet::new(patterns).unwrap(); +/// // Compile each pattern independently. +/// let regexes: Vec<_> = set +/// .patterns() +/// .iter() +/// .map(|pat| Regex::new(pat).unwrap()) +/// .collect(); +/// +/// // Match against the whole set first and identify the individual +/// // matching patterns. +/// let matches: Vec<&[u8]> = set +/// .matches(hay) +/// .into_iter() +/// // Dereference the match index to get the corresponding +/// // compiled pattern. +/// .map(|index| ®exes[index]) +/// // To get match locations or any other info, we then have to search the +/// // exact same haystack again, using our separately-compiled pattern. +/// .map(|re| re.find(hay).unwrap().as_bytes()) +/// .collect(); +/// +/// // Matches arrive in the order the constituent patterns were declared, +/// // not the order they appear in the haystack. +/// assert_eq!(vec![&b"foo"[..], &b"bar"[..]], matches); +/// ``` +/// +/// # Performance +/// +/// A `RegexSet` has the same performance characteristics as `Regex`. Namely, +/// search takes `O(m * n)` time, where `m` is proportional to the size of the +/// regex set and `n` is proportional to the length of the haystack. +/// +/// # Trait implementations +/// +/// The `Default` trait is implemented for `RegexSet`. The default value +/// is an empty set. An empty set can also be explicitly constructed via +/// [`RegexSet::empty`]. +/// +/// # Example +/// +/// This shows how the above two regexes (for matching email addresses and +/// domains) might work: +/// +/// ``` +/// use regex::bytes::RegexSet; +/// +/// let set = RegexSet::new(&[ +/// r"[a-z]+@[a-z]+\.(com|org|net)", +/// r"[a-z]+\.(com|org|net)", +/// ]).unwrap(); +/// +/// // Ask whether any regexes in the set match. +/// assert!(set.is_match(b"foo@example.com")); +/// +/// // Identify which regexes in the set match. +/// let matches: Vec<_> = set.matches(b"foo@example.com").into_iter().collect(); +/// assert_eq!(vec![0, 1], matches); +/// +/// // Try again, but with a haystack that only matches one of the regexes. +/// let matches: Vec<_> = set.matches(b"example.com").into_iter().collect(); +/// assert_eq!(vec![1], matches); +/// +/// // Try again, but with a haystack that doesn't match any regex in the set. +/// let matches: Vec<_> = set.matches(b"example").into_iter().collect(); +/// assert!(matches.is_empty()); +/// ``` +/// +/// Note that it would be possible to adapt the above example to using `Regex` +/// with an expression like: +/// +/// ```text +/// (?P<email>[a-z]+@(?P<email_domain>[a-z]+[.](com|org|net)))|(?P<domain>[a-z]+[.](com|org|net)) +/// ``` +/// +/// After a match, one could then inspect the capture groups to figure out +/// which alternates matched. The problem is that it is hard to make this +/// approach scale when there are many regexes since the overlap between each +/// alternate isn't always obvious to reason about. +#[derive(Clone)] +pub struct RegexSet { + pub(crate) meta: meta::Regex, + pub(crate) patterns: alloc::sync::Arc<[String]>, +} + +impl RegexSet { + /// Create a new regex set with the given regular expressions. + /// + /// This takes an iterator of `S`, where `S` is something that can produce + /// a `&str`. If any of the strings in the iterator are not valid regular + /// expressions, then an error is returned. + /// + /// # Example + /// + /// Create a new regex set from an iterator of strings: + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([r"\w+", r"\d+"]).unwrap(); + /// assert!(set.is_match(b"foo")); + /// ``` + pub fn new<I, S>(exprs: I) -> Result<RegexSet, Error> + where + S: AsRef<str>, + I: IntoIterator<Item = S>, + { + RegexSetBuilder::new(exprs).build() + } + + /// Create a new empty regex set. + /// + /// An empty regex never matches anything. + /// + /// This is a convenience function for `RegexSet::new([])`, but doesn't + /// require one to specify the type of the input. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::empty(); + /// assert!(set.is_empty()); + /// // an empty set matches nothing + /// assert!(!set.is_match(b"")); + /// ``` + pub fn empty() -> RegexSet { + let empty: [&str; 0] = []; + RegexSetBuilder::new(empty).build().unwrap() + } + + /// Returns true if and only if one of the regexes in this set matches + /// the haystack given. + /// + /// This method should be preferred if you only need to test whether any + /// of the regexes in the set should match, but don't care about *which* + /// regexes matched. This is because the underlying matching engine will + /// quit immediately after seeing the first match instead of continuing to + /// find all matches. + /// + /// Note that as with searches using [`Regex`](crate::bytes::Regex), the + /// expression is unanchored by default. That is, if the regex does not + /// start with `^` or `\A`, or end with `$` or `\z`, then it is permitted + /// to match anywhere in the haystack. + /// + /// # Example + /// + /// Tests whether a set matches somewhere in a haystack: + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([r"\w+", r"\d+"]).unwrap(); + /// assert!(set.is_match(b"foo")); + /// assert!(!set.is_match("☃".as_bytes())); + /// ``` + #[inline] + pub fn is_match(&self, haystack: &[u8]) -> bool { + self.is_match_at(haystack, 0) + } + + /// Returns true if and only if one of the regexes in this set matches the + /// haystack given, with the search starting at the offset given. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start`. Namely, consider a + /// haystack `foobar` and a desire to execute a search starting at offset + /// `3`. You could search a substring explicitly, but then the look-around + /// assertions won't work correctly. Instead, you can use this method to + /// specify the start position of a search. + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([r"\bbar\b", r"(?m)^bar$"]).unwrap(); + /// let hay = b"foobar"; + /// // We get a match here, but it's probably not intended. + /// assert!(set.is_match(&hay[3..])); + /// // No match because the assertions take the context into account. + /// assert!(!set.is_match_at(hay, 3)); + /// ``` + #[inline] + pub fn is_match_at(&self, haystack: &[u8], start: usize) -> bool { + self.meta.is_match(Input::new(haystack).span(start..haystack.len())) + } + + /// Returns the set of regexes that match in the given haystack. + /// + /// The set returned contains the index of each regex that matches in + /// the given haystack. The index is in correspondence with the order of + /// regular expressions given to `RegexSet`'s constructor. + /// + /// The set can also be used to iterate over the matched indices. The order + /// of iteration is always ascending with respect to the matching indices. + /// + /// Note that as with searches using [`Regex`](crate::bytes::Regex), the + /// expression is unanchored by default. That is, if the regex does not + /// start with `^` or `\A`, or end with `$` or `\z`, then it is permitted + /// to match anywhere in the haystack. + /// + /// # Example + /// + /// Tests which regular expressions match the given haystack: + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"\w+", + /// r"\d+", + /// r"\pL+", + /// r"foo", + /// r"bar", + /// r"barfoo", + /// r"foobar", + /// ]).unwrap(); + /// let matches: Vec<_> = set.matches(b"foobar").into_iter().collect(); + /// assert_eq!(matches, vec![0, 2, 3, 4, 6]); + /// + /// // You can also test whether a particular regex matched: + /// let matches = set.matches(b"foobar"); + /// assert!(!matches.matched(5)); + /// assert!(matches.matched(6)); + /// ``` + #[inline] + pub fn matches(&self, haystack: &[u8]) -> SetMatches { + self.matches_at(haystack, 0) + } + + /// Returns the set of regexes that match in the given haystack. + /// + /// The set returned contains the index of each regex that matches in + /// the given haystack. The index is in correspondence with the order of + /// regular expressions given to `RegexSet`'s constructor. + /// + /// The set can also be used to iterate over the matched indices. The order + /// of iteration is always ascending with respect to the matching indices. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// Tests which regular expressions match the given haystack: + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([r"\bbar\b", r"(?m)^bar$"]).unwrap(); + /// let hay = b"foobar"; + /// // We get matches here, but it's probably not intended. + /// let matches: Vec<_> = set.matches(&hay[3..]).into_iter().collect(); + /// assert_eq!(matches, vec![0, 1]); + /// // No matches because the assertions take the context into account. + /// let matches: Vec<_> = set.matches_at(hay, 3).into_iter().collect(); + /// assert_eq!(matches, vec![]); + /// ``` + #[inline] + pub fn matches_at(&self, haystack: &[u8], start: usize) -> SetMatches { + let input = Input::new(haystack).span(start..haystack.len()); + let mut patset = PatternSet::new(self.meta.pattern_len()); + self.meta.which_overlapping_matches(&input, &mut patset); + SetMatches(patset) + } + + /// Returns the same as matches, but starts the search at the given + /// offset and stores the matches into the slice given. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// `matches` must have a length that is at least the number of regexes + /// in this set. + /// + /// This method returns true if and only if at least one member of + /// `matches` is true after executing the set against `haystack`. + #[doc(hidden)] + #[inline] + pub fn matches_read_at( + &self, + matches: &mut [bool], + haystack: &[u8], + start: usize, + ) -> bool { + // This is pretty dumb. We should try to fix this, but the + // regex-automata API doesn't provide a way to store matches in an + // arbitrary &mut [bool]. Thankfully, this API is doc(hidden) and + // thus not public... But regex-capi currently uses it. We should + // fix regex-capi to use a PatternSet, maybe? Not sure... PatternSet + // is in regex-automata, not regex. So maybe we should just accept a + // 'SetMatches', which is basically just a newtype around PatternSet. + let mut patset = PatternSet::new(self.meta.pattern_len()); + let mut input = Input::new(haystack); + input.set_start(start); + self.meta.which_overlapping_matches(&input, &mut patset); + for pid in patset.iter() { + matches[pid] = true; + } + !patset.is_empty() + } + + /// An alias for `matches_read_at` to preserve backward compatibility. + /// + /// The `regex-capi` crate used this method, so to avoid breaking that + /// crate, we continue to export it as an undocumented API. + #[doc(hidden)] + #[inline] + pub fn read_matches_at( + &self, + matches: &mut [bool], + haystack: &[u8], + start: usize, + ) -> bool { + self.matches_read_at(matches, haystack, start) + } + + /// Returns the total number of regexes in this set. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// assert_eq!(0, RegexSet::empty().len()); + /// assert_eq!(1, RegexSet::new([r"[0-9]"]).unwrap().len()); + /// assert_eq!(2, RegexSet::new([r"[0-9]", r"[a-z]"]).unwrap().len()); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.meta.pattern_len() + } + + /// Returns `true` if this set contains no regexes. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// assert!(RegexSet::empty().is_empty()); + /// assert!(!RegexSet::new([r"[0-9]"]).unwrap().is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.meta.pattern_len() == 0 + } + + /// Returns the regex patterns that this regex set was constructed from. + /// + /// This function can be used to determine the pattern for a match. The + /// slice returned has exactly as many patterns givens to this regex set, + /// and the order of the slice is the same as the order of the patterns + /// provided to the set. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new(&[ + /// r"\w+", + /// r"\d+", + /// r"\pL+", + /// r"foo", + /// r"bar", + /// r"barfoo", + /// r"foobar", + /// ]).unwrap(); + /// let matches: Vec<_> = set + /// .matches(b"foobar") + /// .into_iter() + /// .map(|index| &set.patterns()[index]) + /// .collect(); + /// assert_eq!(matches, vec![r"\w+", r"\pL+", r"foo", r"bar", r"foobar"]); + /// ``` + #[inline] + pub fn patterns(&self) -> &[String] { + &self.patterns + } +} + +impl Default for RegexSet { + fn default() -> Self { + RegexSet::empty() + } +} + +/// A set of matches returned by a regex set. +/// +/// Values of this type are constructed by [`RegexSet::matches`]. +#[derive(Clone, Debug)] +pub struct SetMatches(PatternSet); + +impl SetMatches { + /// Whether this set contains any matches. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new(&[ + /// r"[a-z]+@[a-z]+\.(com|org|net)", + /// r"[a-z]+\.(com|org|net)", + /// ]).unwrap(); + /// let matches = set.matches(b"foo@example.com"); + /// assert!(matches.matched_any()); + /// ``` + #[inline] + pub fn matched_any(&self) -> bool { + !self.0.is_empty() + } + + /// Whether all patterns in this set matched. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new(&[ + /// r"^foo", + /// r"[a-z]+\.com", + /// ]).unwrap(); + /// let matches = set.matches(b"foo.example.com"); + /// assert!(matches.matched_all()); + /// ``` + pub fn matched_all(&self) -> bool { + self.0.is_full() + } + + /// Whether the regex at the given index matched. + /// + /// The index for a regex is determined by its insertion order upon the + /// initial construction of a `RegexSet`, starting at `0`. + /// + /// # Panics + /// + /// If `index` is greater than or equal to the number of regexes in the + /// original set that produced these matches. Equivalently, when `index` + /// is greater than or equal to [`SetMatches::len`]. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"[a-z]+@[a-z]+\.(com|org|net)", + /// r"[a-z]+\.(com|org|net)", + /// ]).unwrap(); + /// let matches = set.matches(b"example.com"); + /// assert!(!matches.matched(0)); + /// assert!(matches.matched(1)); + /// ``` + #[inline] + pub fn matched(&self, index: usize) -> bool { + self.0.contains(PatternID::new_unchecked(index)) + } + + /// The total number of regexes in the set that created these matches. + /// + /// **WARNING:** This always returns the same value as [`RegexSet::len`]. + /// In particular, it does *not* return the number of elements yielded by + /// [`SetMatches::iter`]. The only way to determine the total number of + /// matched regexes is to iterate over them. + /// + /// # Example + /// + /// Notice that this method returns the total number of regexes in the + /// original set, and *not* the total number of regexes that matched. + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"[a-z]+@[a-z]+\.(com|org|net)", + /// r"[a-z]+\.(com|org|net)", + /// ]).unwrap(); + /// let matches = set.matches(b"example.com"); + /// // Total number of patterns that matched. + /// assert_eq!(1, matches.iter().count()); + /// // Total number of patterns in the set. + /// assert_eq!(2, matches.len()); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.0.capacity() + } + + /// Returns an iterator over the indices of the regexes that matched. + /// + /// This will always produces matches in ascending order, where the index + /// yielded corresponds to the index of the regex that matched with respect + /// to its position when initially building the set. + /// + /// # Example + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"[0-9]", + /// r"[a-z]", + /// r"[A-Z]", + /// r"\p{Greek}", + /// ]).unwrap(); + /// let hay = "βa1".as_bytes(); + /// let matches: Vec<_> = set.matches(hay).iter().collect(); + /// assert_eq!(matches, vec![0, 1, 3]); + /// ``` + /// + /// Note that `SetMatches` also implements the `IntoIterator` trait, so + /// this method is not always needed. For example: + /// + /// ``` + /// use regex::bytes::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"[0-9]", + /// r"[a-z]", + /// r"[A-Z]", + /// r"\p{Greek}", + /// ]).unwrap(); + /// let hay = "βa1".as_bytes(); + /// let mut matches = vec![]; + /// for index in set.matches(hay) { + /// matches.push(index); + /// } + /// assert_eq!(matches, vec![0, 1, 3]); + /// ``` + #[inline] + pub fn iter(&self) -> SetMatchesIter<'_> { + SetMatchesIter(self.0.iter()) + } +} + +impl IntoIterator for SetMatches { + type IntoIter = SetMatchesIntoIter; + type Item = usize; + + fn into_iter(self) -> Self::IntoIter { + let it = 0..self.0.capacity(); + SetMatchesIntoIter { patset: self.0, it } + } +} + +impl<'a> IntoIterator for &'a SetMatches { + type IntoIter = SetMatchesIter<'a>; + type Item = usize; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// An owned iterator over the set of matches from a regex set. +/// +/// This will always produces matches in ascending order of index, where the +/// index corresponds to the index of the regex that matched with respect to +/// its position when initially building the set. +/// +/// This iterator is created by calling `SetMatches::into_iter` via the +/// `IntoIterator` trait. This is automatically done in `for` loops. +/// +/// # Example +/// +/// ``` +/// use regex::bytes::RegexSet; +/// +/// let set = RegexSet::new([ +/// r"[0-9]", +/// r"[a-z]", +/// r"[A-Z]", +/// r"\p{Greek}", +/// ]).unwrap(); +/// let hay = "βa1".as_bytes(); +/// let mut matches = vec![]; +/// for index in set.matches(hay) { +/// matches.push(index); +/// } +/// assert_eq!(matches, vec![0, 1, 3]); +/// ``` +#[derive(Debug)] +pub struct SetMatchesIntoIter { + patset: PatternSet, + it: core::ops::Range<usize>, +} + +impl Iterator for SetMatchesIntoIter { + type Item = usize; + + fn next(&mut self) -> Option<usize> { + loop { + let id = self.it.next()?; + if self.patset.contains(PatternID::new_unchecked(id)) { + return Some(id); + } + } + } + + fn size_hint(&self) -> (usize, Option<usize>) { + self.it.size_hint() + } +} + +impl DoubleEndedIterator for SetMatchesIntoIter { + fn next_back(&mut self) -> Option<usize> { + loop { + let id = self.it.next_back()?; + if self.patset.contains(PatternID::new_unchecked(id)) { + return Some(id); + } + } + } +} + +impl core::iter::FusedIterator for SetMatchesIntoIter {} + +/// A borrowed iterator over the set of matches from a regex set. +/// +/// The lifetime `'a` refers to the lifetime of the [`SetMatches`] value that +/// created this iterator. +/// +/// This will always produces matches in ascending order, where the index +/// corresponds to the index of the regex that matched with respect to its +/// position when initially building the set. +/// +/// This iterator is created by the [`SetMatches::iter`] method. +#[derive(Clone, Debug)] +pub struct SetMatchesIter<'a>(PatternSetIter<'a>); + +impl<'a> Iterator for SetMatchesIter<'a> { + type Item = usize; + + fn next(&mut self) -> Option<usize> { + self.0.next().map(|pid| pid.as_usize()) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + self.0.size_hint() + } +} + +impl<'a> DoubleEndedIterator for SetMatchesIter<'a> { + fn next_back(&mut self) -> Option<usize> { + self.0.next_back().map(|pid| pid.as_usize()) + } +} + +impl<'a> core::iter::FusedIterator for SetMatchesIter<'a> {} + +impl core::fmt::Debug for RegexSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "RegexSet({:?})", self.patterns()) + } +} diff --git a/third_party/crates/regex-1.13.1/src/regexset/mod.rs b/third_party/crates/regex-1.13.1/src/regexset/mod.rs new file mode 100644 index 0000000..93fadec --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/regexset/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod bytes; +pub(crate) mod string; diff --git a/third_party/crates/regex-1.13.1/src/regexset/string.rs b/third_party/crates/regex-1.13.1/src/regexset/string.rs new file mode 100644 index 0000000..5126a46 --- /dev/null +++ b/third_party/crates/regex-1.13.1/src/regexset/string.rs @@ -0,0 +1,724 @@ +use alloc::string::String; + +use regex_automata::{meta, Input, PatternID, PatternSet, PatternSetIter}; + +use crate::{Error, RegexSetBuilder}; + +/// Match multiple, possibly overlapping, regexes in a single search. +/// +/// A regex set corresponds to the union of zero or more regular expressions. +/// That is, a regex set will match a haystack when at least one of its +/// constituent regexes matches. A regex set as its formulated here provides a +/// touch more power: it will also report *which* regular expressions in the +/// set match. Indeed, this is the key difference between regex sets and a +/// single `Regex` with many alternates, since only one alternate can match at +/// a time. +/// +/// For example, consider regular expressions to match email addresses and +/// domains: `[a-z]+@[a-z]+\.(com|org|net)` and `[a-z]+\.(com|org|net)`. If a +/// regex set is constructed from those regexes, then searching the haystack +/// `foo@example.com` will report both regexes as matching. Of course, one +/// could accomplish this by compiling each regex on its own and doing two +/// searches over the haystack. The key advantage of using a regex set is +/// that it will report the matching regexes using a *single pass through the +/// haystack*. If one has hundreds or thousands of regexes to match repeatedly +/// (like a URL router for a complex web application or a user agent matcher), +/// then a regex set *can* realize huge performance gains. +/// +/// # Limitations +/// +/// Regex sets are limited to answering the following two questions: +/// +/// 1. Does any regex in the set match? +/// 2. If so, which regexes in the set match? +/// +/// As with the main [`Regex`][crate::Regex] type, it is cheaper to ask (1) +/// instead of (2) since the matching engines can stop after the first match +/// is found. +/// +/// You cannot directly extract [`Match`][crate::Match] or +/// [`Captures`][crate::Captures] objects from a regex set. If you need these +/// operations, the recommended approach is to compile each pattern in the set +/// independently and scan the exact same haystack a second time with those +/// independently compiled patterns: +/// +/// ``` +/// use regex::{Regex, RegexSet}; +/// +/// let patterns = ["foo", "bar"]; +/// // Both patterns will match different ranges of this string. +/// let hay = "barfoo"; +/// +/// // Compile a set matching any of our patterns. +/// let set = RegexSet::new(patterns).unwrap(); +/// // Compile each pattern independently. +/// let regexes: Vec<_> = set +/// .patterns() +/// .iter() +/// .map(|pat| Regex::new(pat).unwrap()) +/// .collect(); +/// +/// // Match against the whole set first and identify the individual +/// // matching patterns. +/// let matches: Vec<&str> = set +/// .matches(hay) +/// .into_iter() +/// // Dereference the match index to get the corresponding +/// // compiled pattern. +/// .map(|index| ®exes[index]) +/// // To get match locations or any other info, we then have to search the +/// // exact same haystack again, using our separately-compiled pattern. +/// .map(|re| re.find(hay).unwrap().as_str()) +/// .collect(); +/// +/// // Matches arrive in the order the constituent patterns were declared, +/// // not the order they appear in the haystack. +/// assert_eq!(vec!["foo", "bar"], matches); +/// ``` +/// +/// # Performance +/// +/// A `RegexSet` has the same performance characteristics as `Regex`. Namely, +/// search takes `O(m * n)` time, where `m` is proportional to the size of the +/// regex set and `n` is proportional to the length of the haystack. +/// +/// # Trait implementations +/// +/// The `Default` trait is implemented for `RegexSet`. The default value +/// is an empty set. An empty set can also be explicitly constructed via +/// [`RegexSet::empty`]. +/// +/// # Example +/// +/// This shows how the above two regexes (for matching email addresses and +/// domains) might work: +/// +/// ``` +/// use regex::RegexSet; +/// +/// let set = RegexSet::new(&[ +/// r"[a-z]+@[a-z]+\.(com|org|net)", +/// r"[a-z]+\.(com|org|net)", +/// ]).unwrap(); +/// +/// // Ask whether any regexes in the set match. +/// assert!(set.is_match("foo@example.com")); +/// +/// // Identify which regexes in the set match. +/// let matches: Vec<_> = set.matches("foo@example.com").into_iter().collect(); +/// assert_eq!(vec![0, 1], matches); +/// +/// // Try again, but with a haystack that only matches one of the regexes. +/// let matches: Vec<_> = set.matches("example.com").into_iter().collect(); +/// assert_eq!(vec![1], matches); +/// +/// // Try again, but with a haystack that doesn't match any regex in the set. +/// let matches: Vec<_> = set.matches("example").into_iter().collect(); +/// assert!(matches.is_empty()); +/// ``` +/// +/// Note that it would be possible to adapt the above example to using `Regex` +/// with an expression like: +/// +/// ```text +/// (?P<email>[a-z]+@(?P<email_domain>[a-z]+[.](com|org|net)))|(?P<domain>[a-z]+[.](com|org|net)) +/// ``` +/// +/// After a match, one could then inspect the capture groups to figure out +/// which alternates matched. The problem is that it is hard to make this +/// approach scale when there are many regexes since the overlap between each +/// alternate isn't always obvious to reason about. +#[derive(Clone)] +pub struct RegexSet { + pub(crate) meta: meta::Regex, + pub(crate) patterns: alloc::sync::Arc<[String]>, +} + +impl RegexSet { + /// Create a new regex set with the given regular expressions. + /// + /// This takes an iterator of `S`, where `S` is something that can produce + /// a `&str`. If any of the strings in the iterator are not valid regular + /// expressions, then an error is returned. + /// + /// # Example + /// + /// Create a new regex set from an iterator of strings: + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([r"\w+", r"\d+"]).unwrap(); + /// assert!(set.is_match("foo")); + /// ``` + pub fn new<I, S>(exprs: I) -> Result<RegexSet, Error> + where + S: AsRef<str>, + I: IntoIterator<Item = S>, + { + RegexSetBuilder::new(exprs).build() + } + + /// Create a new empty regex set. + /// + /// An empty regex never matches anything. + /// + /// This is a convenience function for `RegexSet::new([])`, but doesn't + /// require one to specify the type of the input. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::empty(); + /// assert!(set.is_empty()); + /// // an empty set matches nothing + /// assert!(!set.is_match("")); + /// ``` + pub fn empty() -> RegexSet { + let empty: [&str; 0] = []; + RegexSetBuilder::new(empty).build().unwrap() + } + + /// Returns true if and only if one of the regexes in this set matches + /// the haystack given. + /// + /// This method should be preferred if you only need to test whether any + /// of the regexes in the set should match, but don't care about *which* + /// regexes matched. This is because the underlying matching engine will + /// quit immediately after seeing the first match instead of continuing to + /// find all matches. + /// + /// Note that as with searches using [`Regex`](crate::Regex), the + /// expression is unanchored by default. That is, if the regex does not + /// start with `^` or `\A`, or end with `$` or `\z`, then it is permitted + /// to match anywhere in the haystack. + /// + /// # Example + /// + /// Tests whether a set matches somewhere in a haystack: + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([r"\w+", r"\d+"]).unwrap(); + /// assert!(set.is_match("foo")); + /// assert!(!set.is_match("☃")); + /// ``` + #[inline] + pub fn is_match(&self, haystack: &str) -> bool { + self.is_match_at(haystack, 0) + } + + /// Returns true if and only if one of the regexes in this set matches the + /// haystack given, with the search starting at the offset given. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// This example shows the significance of `start`. Namely, consider a + /// haystack `foobar` and a desire to execute a search starting at offset + /// `3`. You could search a substring explicitly, but then the look-around + /// assertions won't work correctly. Instead, you can use this method to + /// specify the start position of a search. + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([r"\bbar\b", r"(?m)^bar$"]).unwrap(); + /// let hay = "foobar"; + /// // We get a match here, but it's probably not intended. + /// assert!(set.is_match(&hay[3..])); + /// // No match because the assertions take the context into account. + /// assert!(!set.is_match_at(hay, 3)); + /// ``` + #[inline] + pub fn is_match_at(&self, haystack: &str, start: usize) -> bool { + self.meta.is_match(Input::new(haystack).span(start..haystack.len())) + } + + /// Returns the set of regexes that match in the given haystack. + /// + /// The set returned contains the index of each regex that matches in + /// the given haystack. The index is in correspondence with the order of + /// regular expressions given to `RegexSet`'s constructor. + /// + /// The set can also be used to iterate over the matched indices. The order + /// of iteration is always ascending with respect to the matching indices. + /// + /// Note that as with searches using [`Regex`](crate::Regex), the + /// expression is unanchored by default. That is, if the regex does not + /// start with `^` or `\A`, or end with `$` or `\z`, then it is permitted + /// to match anywhere in the haystack. + /// + /// # Example + /// + /// Tests which regular expressions match the given haystack: + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"\w+", + /// r"\d+", + /// r"\pL+", + /// r"foo", + /// r"bar", + /// r"barfoo", + /// r"foobar", + /// ]).unwrap(); + /// let matches: Vec<_> = set.matches("foobar").into_iter().collect(); + /// assert_eq!(matches, vec![0, 2, 3, 4, 6]); + /// + /// // You can also test whether a particular regex matched: + /// let matches = set.matches("foobar"); + /// assert!(!matches.matched(5)); + /// assert!(matches.matched(6)); + /// ``` + #[inline] + pub fn matches(&self, haystack: &str) -> SetMatches { + self.matches_at(haystack, 0) + } + + /// Returns the set of regexes that match in the given haystack. + /// + /// The set returned contains the index of each regex that matches in + /// the given haystack. The index is in correspondence with the order of + /// regular expressions given to `RegexSet`'s constructor. + /// + /// The set can also be used to iterate over the matched indices. The order + /// of iteration is always ascending with respect to the matching indices. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// # Panics + /// + /// This panics when `start >= haystack.len() + 1`. + /// + /// # Example + /// + /// Tests which regular expressions match the given haystack: + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([r"\bbar\b", r"(?m)^bar$"]).unwrap(); + /// let hay = "foobar"; + /// // We get matches here, but it's probably not intended. + /// let matches: Vec<_> = set.matches(&hay[3..]).into_iter().collect(); + /// assert_eq!(matches, vec![0, 1]); + /// // No matches because the assertions take the context into account. + /// let matches: Vec<_> = set.matches_at(hay, 3).into_iter().collect(); + /// assert_eq!(matches, vec![]); + /// ``` + #[inline] + pub fn matches_at(&self, haystack: &str, start: usize) -> SetMatches { + let input = Input::new(haystack).span(start..haystack.len()); + let mut patset = PatternSet::new(self.meta.pattern_len()); + self.meta.which_overlapping_matches(&input, &mut patset); + SetMatches(patset) + } + + /// Returns the same as matches, but starts the search at the given + /// offset and stores the matches into the slice given. + /// + /// The significance of the starting point is that it takes the surrounding + /// context into consideration. For example, the `\A` anchor can only + /// match when `start == 0`. + /// + /// `matches` must have a length that is at least the number of regexes + /// in this set. + /// + /// This method returns true if and only if at least one member of + /// `matches` is true after executing the set against `haystack`. + #[doc(hidden)] + #[inline] + pub fn matches_read_at( + &self, + matches: &mut [bool], + haystack: &str, + start: usize, + ) -> bool { + // This is pretty dumb. We should try to fix this, but the + // regex-automata API doesn't provide a way to store matches in an + // arbitrary &mut [bool]. Thankfully, this API is doc(hidden) and + // thus not public... But regex-capi currently uses it. We should + // fix regex-capi to use a PatternSet, maybe? Not sure... PatternSet + // is in regex-automata, not regex. So maybe we should just accept a + // 'SetMatches', which is basically just a newtype around PatternSet. + let mut patset = PatternSet::new(self.meta.pattern_len()); + let mut input = Input::new(haystack); + input.set_start(start); + self.meta.which_overlapping_matches(&input, &mut patset); + for pid in patset.iter() { + matches[pid] = true; + } + !patset.is_empty() + } + + /// An alias for `matches_read_at` to preserve backward compatibility. + /// + /// The `regex-capi` crate used this method, so to avoid breaking that + /// crate, we continue to export it as an undocumented API. + #[doc(hidden)] + #[inline] + pub fn read_matches_at( + &self, + matches: &mut [bool], + haystack: &str, + start: usize, + ) -> bool { + self.matches_read_at(matches, haystack, start) + } + + /// Returns the total number of regexes in this set. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSet; + /// + /// assert_eq!(0, RegexSet::empty().len()); + /// assert_eq!(1, RegexSet::new([r"[0-9]"]).unwrap().len()); + /// assert_eq!(2, RegexSet::new([r"[0-9]", r"[a-z]"]).unwrap().len()); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.meta.pattern_len() + } + + /// Returns `true` if this set contains no regexes. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSet; + /// + /// assert!(RegexSet::empty().is_empty()); + /// assert!(!RegexSet::new([r"[0-9]"]).unwrap().is_empty()); + /// ``` + #[inline] + pub fn is_empty(&self) -> bool { + self.meta.pattern_len() == 0 + } + + /// Returns the regex patterns that this regex set was constructed from. + /// + /// This function can be used to determine the pattern for a match. The + /// slice returned has exactly as many patterns givens to this regex set, + /// and the order of the slice is the same as the order of the patterns + /// provided to the set. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new(&[ + /// r"\w+", + /// r"\d+", + /// r"\pL+", + /// r"foo", + /// r"bar", + /// r"barfoo", + /// r"foobar", + /// ]).unwrap(); + /// let matches: Vec<_> = set + /// .matches("foobar") + /// .into_iter() + /// .map(|index| &set.patterns()[index]) + /// .collect(); + /// assert_eq!(matches, vec![r"\w+", r"\pL+", r"foo", r"bar", r"foobar"]); + /// ``` + #[inline] + pub fn patterns(&self) -> &[String] { + &self.patterns + } +} + +impl Default for RegexSet { + fn default() -> Self { + RegexSet::empty() + } +} + +/// A set of matches returned by a regex set. +/// +/// Values of this type are constructed by [`RegexSet::matches`]. +#[derive(Clone, Debug)] +pub struct SetMatches(PatternSet); + +impl SetMatches { + /// Whether this set contains any matches. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new(&[ + /// r"[a-z]+@[a-z]+\.(com|org|net)", + /// r"[a-z]+\.(com|org|net)", + /// ]).unwrap(); + /// let matches = set.matches("foo@example.com"); + /// assert!(matches.matched_any()); + /// ``` + #[inline] + pub fn matched_any(&self) -> bool { + !self.0.is_empty() + } + + /// Whether all patterns in this set matched. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new(&[ + /// r"^foo", + /// r"[a-z]+\.com", + /// ]).unwrap(); + /// let matches = set.matches("foo.example.com"); + /// assert!(matches.matched_all()); + /// ``` + pub fn matched_all(&self) -> bool { + self.0.is_full() + } + + /// Whether the regex at the given index matched. + /// + /// The index for a regex is determined by its insertion order upon the + /// initial construction of a `RegexSet`, starting at `0`. + /// + /// # Panics + /// + /// If `index` is greater than or equal to the number of regexes in the + /// original set that produced these matches. Equivalently, when `index` + /// is greater than or equal to [`SetMatches::len`]. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"[a-z]+@[a-z]+\.(com|org|net)", + /// r"[a-z]+\.(com|org|net)", + /// ]).unwrap(); + /// let matches = set.matches("example.com"); + /// assert!(!matches.matched(0)); + /// assert!(matches.matched(1)); + /// ``` + #[inline] + pub fn matched(&self, index: usize) -> bool { + self.0.contains(PatternID::new_unchecked(index)) + } + + /// The total number of regexes in the set that created these matches. + /// + /// **WARNING:** This always returns the same value as [`RegexSet::len`]. + /// In particular, it does *not* return the number of elements yielded by + /// [`SetMatches::iter`]. The only way to determine the total number of + /// matched regexes is to iterate over them. + /// + /// # Example + /// + /// Notice that this method returns the total number of regexes in the + /// original set, and *not* the total number of regexes that matched. + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"[a-z]+@[a-z]+\.(com|org|net)", + /// r"[a-z]+\.(com|org|net)", + /// ]).unwrap(); + /// let matches = set.matches("example.com"); + /// // Total number of patterns that matched. + /// assert_eq!(1, matches.iter().count()); + /// // Total number of patterns in the set. + /// assert_eq!(2, matches.len()); + /// ``` + #[inline] + pub fn len(&self) -> usize { + self.0.capacity() + } + + /// Returns an iterator over the indices of the regexes that matched. + /// + /// This will always produces matches in ascending order, where the index + /// yielded corresponds to the index of the regex that matched with respect + /// to its position when initially building the set. + /// + /// # Example + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"[0-9]", + /// r"[a-z]", + /// r"[A-Z]", + /// r"\p{Greek}", + /// ]).unwrap(); + /// let hay = "βa1"; + /// let matches: Vec<_> = set.matches(hay).iter().collect(); + /// assert_eq!(matches, vec![0, 1, 3]); + /// ``` + /// + /// Note that `SetMatches` also implements the `IntoIterator` trait, so + /// this method is not always needed. For example: + /// + /// ``` + /// use regex::RegexSet; + /// + /// let set = RegexSet::new([ + /// r"[0-9]", + /// r"[a-z]", + /// r"[A-Z]", + /// r"\p{Greek}", + /// ]).unwrap(); + /// let hay = "βa1"; + /// let mut matches = vec![]; + /// for index in set.matches(hay) { + /// matches.push(index); + /// } + /// assert_eq!(matches, vec![0, 1, 3]); + /// ``` + #[inline] + pub fn iter(&self) -> SetMatchesIter<'_> { + SetMatchesIter(self.0.iter()) + } +} + +impl IntoIterator for SetMatches { + type IntoIter = SetMatchesIntoIter; + type Item = usize; + + fn into_iter(self) -> Self::IntoIter { + let it = 0..self.0.capacity(); + SetMatchesIntoIter { patset: self.0, it } + } +} + +impl<'a> IntoIterator for &'a SetMatches { + type IntoIter = SetMatchesIter<'a>; + type Item = usize; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +/// An owned iterator over the set of matches from a regex set. +/// +/// This will always produces matches in ascending order of index, where the +/// index corresponds to the index of the regex that matched with respect to +/// its position when initially building the set. +/// +/// This iterator is created by calling `SetMatches::into_iter` via the +/// `IntoIterator` trait. This is automatically done in `for` loops. +/// +/// # Example +/// +/// ``` +/// use regex::RegexSet; +/// +/// let set = RegexSet::new([ +/// r"[0-9]", +/// r"[a-z]", +/// r"[A-Z]", +/// r"\p{Greek}", +/// ]).unwrap(); +/// let hay = "βa1"; +/// let mut matches = vec![]; +/// for index in set.matches(hay) { +/// matches.push(index); +/// } +/// assert_eq!(matches, vec![0, 1, 3]); +/// ``` +#[derive(Debug)] +pub struct SetMatchesIntoIter { + patset: PatternSet, + it: core::ops::Range<usize>, +} + +impl Iterator for SetMatchesIntoIter { + type Item = usize; + + fn next(&mut self) -> Option<usize> { + loop { + let id = self.it.next()?; + if self.patset.contains(PatternID::new_unchecked(id)) { + return Some(id); + } + } + } + + fn size_hint(&self) -> (usize, Option<usize>) { + self.it.size_hint() + } +} + +impl DoubleEndedIterator for SetMatchesIntoIter { + fn next_back(&mut self) -> Option<usize> { + loop { + let id = self.it.next_back()?; + if self.patset.contains(PatternID::new_unchecked(id)) { + return Some(id); + } + } + } +} + +impl core::iter::FusedIterator for SetMatchesIntoIter {} + +/// A borrowed iterator over the set of matches from a regex set. +/// +/// The lifetime `'a` refers to the lifetime of the [`SetMatches`] value that +/// created this iterator. +/// +/// This will always produces matches in ascending order, where the index +/// corresponds to the index of the regex that matched with respect to its +/// position when initially building the set. +/// +/// This iterator is created by the [`SetMatches::iter`] method. +#[derive(Clone, Debug)] +pub struct SetMatchesIter<'a>(PatternSetIter<'a>); + +impl<'a> Iterator for SetMatchesIter<'a> { + type Item = usize; + + fn next(&mut self) -> Option<usize> { + self.0.next().map(|pid| pid.as_usize()) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + self.0.size_hint() + } +} + +impl<'a> DoubleEndedIterator for SetMatchesIter<'a> { + fn next_back(&mut self) -> Option<usize> { + self.0.next_back().map(|pid| pid.as_usize()) + } +} + +impl<'a> core::iter::FusedIterator for SetMatchesIter<'a> {} + +impl core::fmt::Debug for RegexSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "RegexSet({:?})", self.patterns()) + } +} diff --git a/third_party/crates/regex-1.13.1/testdata/anchored.toml b/third_party/crates/regex-1.13.1/testdata/anchored.toml new file mode 100644 index 0000000..0f2248d --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/anchored.toml @@ -0,0 +1,127 @@ +# These tests are specifically geared toward searches with 'anchored = true'. +# While they are interesting in their own right, they are particularly +# important for testing the one-pass DFA since the one-pass DFA can't work in +# unanchored contexts. +# +# Note that "anchored" in this context does not mean "^". Anchored searches are +# searches whose matches must begin at the start of the search, which may not +# be at the start of the haystack. That's why anchored searches---and there are +# some examples below---can still report multiple matches. This occurs when the +# matches are adjacent to one another. + +[[test]] +name = "greedy" +regex = '(abc)+' +haystack = "abcabcabc" +matches = [ + [[0, 9], [6, 9]], +] +anchored = true + +# When a "earliest" search is used, greediness doesn't really exist because +# matches are reported as soon as they are known. +[[test]] +name = "greedy-earliest" +regex = '(abc)+' +haystack = "abcabcabc" +matches = [ + [[0, 3], [0, 3]], + [[3, 6], [3, 6]], + [[6, 9], [6, 9]], +] +anchored = true +search-kind = "earliest" + +[[test]] +name = "nongreedy" +regex = '(abc)+?' +haystack = "abcabcabc" +matches = [ + [[0, 3], [0, 3]], + [[3, 6], [3, 6]], + [[6, 9], [6, 9]], +] +anchored = true + +# When "all" semantics are used, non-greediness doesn't exist since the longest +# possible match is always taken. +[[test]] +name = "nongreedy-all" +regex = '(abc)+?' +haystack = "abcabcabc" +matches = [ + [[0, 9], [6, 9]], +] +anchored = true +match-kind = "all" + +[[test]] +name = "word-boundary-unicode-01" +regex = '\b\w+\b' +haystack = 'βββ☃' +matches = [[0, 6]] +anchored = true + +[[test]] +name = "word-boundary-nounicode-01" +regex = '\b\w+\b' +haystack = 'abcβ' +matches = [[0, 3]] +anchored = true +unicode = false + +# Tests that '.c' doesn't match 'abc' when performing an anchored search from +# the beginning of the haystack. This test found two different bugs in the +# PikeVM and the meta engine. +[[test]] +name = "no-match-at-start" +regex = '.c' +haystack = 'abc' +matches = [] +anchored = true + +# Like above, but at a non-zero start offset. +[[test]] +name = "no-match-at-start-bounds" +regex = '.c' +haystack = 'aabc' +bounds = [1, 4] +matches = [] +anchored = true + +# This is like no-match-at-start, but hits the "reverse inner" optimization +# inside the meta engine. (no-match-at-start hits the "reverse suffix" +# optimization.) +[[test]] +name = "no-match-at-start-reverse-inner" +regex = '.c[a-z]' +haystack = 'abcz' +matches = [] +anchored = true + +# Like above, but at a non-zero start offset. +[[test]] +name = "no-match-at-start-reverse-inner-bounds" +regex = '.c[a-z]' +haystack = 'aabcz' +bounds = [1, 5] +matches = [] +anchored = true + +# Same as no-match-at-start, but applies to the meta engine's "reverse +# anchored" optimization. +[[test]] +name = "no-match-at-start-reverse-anchored" +regex = '.c[a-z]$' +haystack = 'abcz' +matches = [] +anchored = true + +# Like above, but at a non-zero start offset. +[[test]] +name = "no-match-at-start-reverse-anchored-bounds" +regex = '.c[a-z]$' +haystack = 'aabcz' +bounds = [1, 5] +matches = [] +anchored = true diff --git a/third_party/crates/regex-1.13.1/testdata/bytes.toml b/third_party/crates/regex-1.13.1/testdata/bytes.toml new file mode 100644 index 0000000..346e369 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/bytes.toml @@ -0,0 +1,235 @@ +# These are tests specifically crafted for regexes that can match arbitrary +# bytes. In some cases, we also test the Unicode variant as well, just because +# it's good sense to do so. But also, these tests aren't really about Unicode, +# but whether matches are only reported at valid UTF-8 boundaries. For most +# tests in this entire collection, utf8 = true. But for these tests, we use +# utf8 = false. + +[[test]] +name = "word-boundary-ascii" +regex = ' \b' +haystack = " δ" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "word-boundary-unicode" +regex = ' \b' +haystack = " δ" +matches = [[0, 1]] +unicode = true +utf8 = false + +[[test]] +name = "word-boundary-ascii-not" +regex = ' \B' +haystack = " δ" +matches = [[0, 1]] +unicode = false +utf8 = false + +[[test]] +name = "word-boundary-unicode-not" +regex = ' \B' +haystack = " δ" +matches = [] +unicode = true +utf8 = false + +[[test]] +name = "perl-word-ascii" +regex = '\w+' +haystack = "aδ" +matches = [[0, 1]] +unicode = false +utf8 = false + +[[test]] +name = "perl-word-unicode" +regex = '\w+' +haystack = "aδ" +matches = [[0, 3]] +unicode = true +utf8 = false + +[[test]] +name = "perl-decimal-ascii" +regex = '\d+' +haystack = "1२३9" +matches = [[0, 1], [7, 8]] +unicode = false +utf8 = false + +[[test]] +name = "perl-decimal-unicode" +regex = '\d+' +haystack = "1२३9" +matches = [[0, 8]] +unicode = true +utf8 = false + +[[test]] +name = "perl-whitespace-ascii" +regex = '\s+' +haystack = " \u1680" +matches = [[0, 1]] +unicode = false +utf8 = false + +[[test]] +name = "perl-whitespace-unicode" +regex = '\s+' +haystack = " \u1680" +matches = [[0, 4]] +unicode = true +utf8 = false + +# The first `(.+)` matches two Unicode codepoints, but can't match the 5th +# byte, which isn't valid UTF-8. The second (byte based) `(.+)` takes over and +# matches. +[[test]] +name = "mixed-dot" +regex = '(.+)(?-u)(.+)' +haystack = '\xCE\x93\xCE\x94\xFF' +matches = [ + [[0, 5], [0, 4], [4, 5]], +] +unescape = true +unicode = true +utf8 = false + +[[test]] +name = "case-one-ascii" +regex = 'a' +haystack = "A" +matches = [[0, 1]] +case-insensitive = true +unicode = false +utf8 = false + +[[test]] +name = "case-one-unicode" +regex = 'a' +haystack = "A" +matches = [[0, 1]] +case-insensitive = true +unicode = true +utf8 = false + +[[test]] +name = "case-class-simple-ascii" +regex = '[a-z]+' +haystack = "AaAaA" +matches = [[0, 5]] +case-insensitive = true +unicode = false +utf8 = false + +[[test]] +name = "case-class-ascii" +regex = '[a-z]+' +haystack = "aA\u212AaA" +matches = [[0, 2], [5, 7]] +case-insensitive = true +unicode = false +utf8 = false + +[[test]] +name = "case-class-unicode" +regex = '[a-z]+' +haystack = "aA\u212AaA" +matches = [[0, 7]] +case-insensitive = true +unicode = true +utf8 = false + +[[test]] +name = "negate-ascii" +regex = '[^a]' +haystack = "δ" +matches = [[0, 1], [1, 2]] +unicode = false +utf8 = false + +[[test]] +name = "negate-unicode" +regex = '[^a]' +haystack = "δ" +matches = [[0, 2]] +unicode = true +utf8 = false + +# When utf8=true, this won't match, because the implicit '.*?' prefix is +# Unicode aware and will refuse to match through invalid UTF-8 bytes. +[[test]] +name = "dotstar-prefix-ascii" +regex = 'a' +haystack = '\xFFa' +matches = [[1, 2]] +unescape = true +unicode = false +utf8 = false + +[[test]] +name = "dotstar-prefix-unicode" +regex = 'a' +haystack = '\xFFa' +matches = [[1, 2]] +unescape = true +unicode = true +utf8 = false + +[[test]] +name = "null-bytes" +regex = '(?P<cstr>[^\x00]+)\x00' +haystack = 'foo\x00' +matches = [ + [[0, 4], [0, 3]], +] +unescape = true +unicode = false +utf8 = false + +[[test]] +name = "invalid-utf8-anchor-100" +regex = '\xCC?^' +haystack = '\x8d#;\x1a\xa4s3\x05foobarX\\\x0f0t\xe4\x9b\xa4' +matches = [[0, 0]] +unescape = true +unicode = false +utf8 = false + +[[test]] +name = "invalid-utf8-anchor-200" +regex = '^\xf7|4\xff\d\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a##########[] d\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a\x8a##########\[] #####\x80\S7|$' +haystack = '\x8d#;\x1a\xa4s3\x05foobarX\\\x0f0t\xe4\x9b\xa4' +matches = [[22, 22]] +unescape = true +unicode = false +utf8 = false + +[[test]] +name = "invalid-utf8-anchor-300" +regex = '^|ddp\xff\xffdddddlQd@\x80' +haystack = '\x8d#;\x1a\xa4s3\x05foobarX\\\x0f0t\xe4\x9b\xa4' +matches = [[0, 0]] +unescape = true +unicode = false +utf8 = false + +[[test]] +name = "word-boundary-ascii-100" +regex = '\Bx\B' +haystack = "áxβ" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "word-boundary-ascii-200" +regex = '\B' +haystack = "0\U0007EF5E" +matches = [[2, 2], [3, 3], [4, 4], [5, 5]] +unicode = false +utf8 = false diff --git a/third_party/crates/regex-1.13.1/testdata/crazy.toml b/third_party/crates/regex-1.13.1/testdata/crazy.toml new file mode 100644 index 0000000..aed46ea --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/crazy.toml @@ -0,0 +1,315 @@ +[[test]] +name = "nothing-empty" +regex = [] +haystack = "" +matches = [] + +[[test]] +name = "nothing-something" +regex = [] +haystack = "wat" +matches = [] + +[[test]] +name = "ranges" +regex = '(?-u)\b(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\b' +haystack = "num: 255" +matches = [[5, 8]] + +[[test]] +name = "ranges-not" +regex = '(?-u)\b(?:[0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\b' +haystack = "num: 256" +matches = [] + +[[test]] +name = "float1" +regex = '[-+]?[0-9]*\.?[0-9]+' +haystack = "0.1" +matches = [[0, 3]] + +[[test]] +name = "float2" +regex = '[-+]?[0-9]*\.?[0-9]+' +haystack = "0.1.2" +matches = [[0, 3]] +match-limit = 1 + +[[test]] +name = "float3" +regex = '[-+]?[0-9]*\.?[0-9]+' +haystack = "a1.2" +matches = [[1, 4]] + +[[test]] +name = "float4" +regex = '[-+]?[0-9]*\.?[0-9]+' +haystack = "1.a" +matches = [[0, 1]] + +[[test]] +name = "float5" +regex = '^[-+]?[0-9]*\.?[0-9]+$' +haystack = "1.a" +matches = [] + +[[test]] +name = "email" +regex = '(?i-u)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}\b' +haystack = "mine is jam.slam@gmail.com " +matches = [[8, 26]] + +[[test]] +name = "email-not" +regex = '(?i-u)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}\b' +haystack = "mine is jam.slam@gmail " +matches = [] + +[[test]] +name = "email-big" +regex = '''[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?''' +haystack = "mine is jam.slam@gmail.com " +matches = [[8, 26]] + +[[test]] +name = "date1" +regex = '^(?:19|20)\d\d[- /.](?:0[1-9]|1[012])[- /.](?:0[1-9]|[12][0-9]|3[01])$' +haystack = "1900-01-01" +matches = [[0, 10]] +unicode = false + +[[test]] +name = "date2" +regex = '^(?:19|20)\d\d[- /.](?:0[1-9]|1[012])[- /.](?:0[1-9]|[12][0-9]|3[01])$' +haystack = "1900-00-01" +matches = [] +unicode = false + +[[test]] +name = "date3" +regex = '^(?:19|20)\d\d[- /.](?:0[1-9]|1[012])[- /.](?:0[1-9]|[12][0-9]|3[01])$' +haystack = "1900-13-01" +matches = [] +unicode = false + +[[test]] +name = "start-end-empty" +regex = '^$' +haystack = "" +matches = [[0, 0]] + +[[test]] +name = "start-end-empty-rev" +regex = '$^' +haystack = "" +matches = [[0, 0]] + +[[test]] +name = "start-end-empty-many-1" +regex = '^$^$^$' +haystack = "" +matches = [[0, 0]] + +[[test]] +name = "start-end-empty-many-2" +regex = '^^^$$$' +haystack = "" +matches = [[0, 0]] + +[[test]] +name = "start-end-empty-rep" +regex = '(?:^$)*' +haystack = "a\nb\nc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +[[test]] +name = "start-end-empty-rep-rev" +regex = '(?:$^)*' +haystack = "a\nb\nc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +[[test]] +name = "neg-class-letter" +regex = '[^ac]' +haystack = "acx" +matches = [[2, 3]] + +[[test]] +name = "neg-class-letter-comma" +regex = '[^a,]' +haystack = "a,x" +matches = [[2, 3]] + +[[test]] +name = "neg-class-letter-space" +regex = '[^a[:space:]]' +haystack = "a x" +matches = [[2, 3]] + +[[test]] +name = "neg-class-comma" +regex = '[^,]' +haystack = ",,x" +matches = [[2, 3]] + +[[test]] +name = "neg-class-space" +regex = '[^[:space:]]' +haystack = " a" +matches = [[1, 2]] + +[[test]] +name = "neg-class-space-comma" +regex = '[^,[:space:]]' +haystack = ", a" +matches = [[2, 3]] + +[[test]] +name = "neg-class-comma-space" +regex = '[^[:space:],]' +haystack = " ,a" +matches = [[2, 3]] + +[[test]] +name = "neg-class-ascii" +regex = '[^[:alpha:]Z]' +haystack = "A1" +matches = [[1, 2]] + +[[test]] +name = "lazy-many-many" +regex = '(?:(?:.*)*?)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "lazy-many-optional" +regex = '(?:(?:.?)*?)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "lazy-one-many-many" +regex = '(?:(?:.*)+?)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "lazy-one-many-optional" +regex = '(?:(?:.?)+?)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "lazy-range-min-many" +regex = '(?:(?:.*){1,}?)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "lazy-range-many" +regex = '(?:(?:.*){1,2}?)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "greedy-many-many" +regex = '(?:(?:.*)*)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "greedy-many-optional" +regex = '(?:(?:.?)*)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "greedy-one-many-many" +regex = '(?:(?:.*)+)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "greedy-one-many-optional" +regex = '(?:(?:.?)+)=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "greedy-range-min-many" +regex = '(?:(?:.*){1,})=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "greedy-range-many" +regex = '(?:(?:.*){1,2})=' +haystack = "a=b" +matches = [[0, 2]] + +[[test]] +name = "empty1" +regex = '' +haystack = "" +matches = [[0, 0]] + +[[test]] +name = "empty2" +regex = '' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty3" +regex = '(?:)' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty4" +regex = '(?:)*' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty5" +regex = '(?:)+' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty6" +regex = '(?:)?' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty7" +regex = '(?:)(?:)' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty8" +regex = '(?:)+|z' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty9" +regex = 'z|(?:)+' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty10" +regex = '(?:)+|b' +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty11" +regex = 'b|(?:)+' +haystack = "abc" +matches = [[0, 0], [1, 2], [3, 3]] diff --git a/third_party/crates/regex-1.13.1/testdata/crlf.toml b/third_party/crates/regex-1.13.1/testdata/crlf.toml new file mode 100644 index 0000000..9e2d376 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/crlf.toml @@ -0,0 +1,117 @@ +# This is a basic test that checks ^ and $ treat \r\n as a single line +# terminator. If ^ and $ only treated \n as a line terminator, then this would +# only match 'xyz' at the end of the haystack. +[[test]] +name = "basic" +regex = '(?mR)^[a-z]+$' +haystack = "abc\r\ndef\r\nxyz" +matches = [[0, 3], [5, 8], [10, 13]] + +# Tests that a CRLF-aware '^$' assertion does not match between CR and LF. +[[test]] +name = "start-end-non-empty" +regex = '(?mR)^$' +haystack = "abc\r\ndef\r\nxyz" +matches = [] + +# Tests that a CRLF-aware '^$' assertion matches the empty string, just like +# a non-CRLF-aware '^$' assertion. +[[test]] +name = "start-end-empty" +regex = '(?mR)^$' +haystack = "" +matches = [[0, 0]] + +# Tests that a CRLF-aware '^$' assertion matches the empty string preceding +# and following a line terminator. +[[test]] +name = "start-end-before-after" +regex = '(?mR)^$' +haystack = "\r\n" +matches = [[0, 0], [2, 2]] + +# Tests that a CRLF-aware '^' assertion does not split a line terminator. +[[test]] +name = "start-no-split" +regex = '(?mR)^' +haystack = "abc\r\ndef\r\nxyz" +matches = [[0, 0], [5, 5], [10, 10]] + +# Same as above, but with adjacent runs of line terminators. +[[test]] +name = "start-no-split-adjacent" +regex = '(?mR)^' +haystack = "\r\n\r\n\r\n" +matches = [[0, 0], [2, 2], [4, 4], [6, 6]] + +# Same as above, but with adjacent runs of just carriage returns. +[[test]] +name = "start-no-split-adjacent-cr" +regex = '(?mR)^' +haystack = "\r\r\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +# Same as above, but with adjacent runs of just line feeds. +[[test]] +name = "start-no-split-adjacent-lf" +regex = '(?mR)^' +haystack = "\n\n\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +# Tests that a CRLF-aware '$' assertion does not split a line terminator. +[[test]] +name = "end-no-split" +regex = '(?mR)$' +haystack = "abc\r\ndef\r\nxyz" +matches = [[3, 3], [8, 8], [13, 13]] + +# Same as above, but with adjacent runs of line terminators. +[[test]] +name = "end-no-split-adjacent" +regex = '(?mR)$' +haystack = "\r\n\r\n\r\n" +matches = [[0, 0], [2, 2], [4, 4], [6, 6]] + +# Same as above, but with adjacent runs of just carriage returns. +[[test]] +name = "end-no-split-adjacent-cr" +regex = '(?mR)$' +haystack = "\r\r\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +# Same as above, but with adjacent runs of just line feeds. +[[test]] +name = "end-no-split-adjacent-lf" +regex = '(?mR)$' +haystack = "\n\n\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +# Tests that '.' does not match either \r or \n when CRLF mode is enabled. Note +# that this doesn't require multi-line mode to be enabled. +[[test]] +name = "dot-no-crlf" +regex = '(?R).' +haystack = "\r\n\r\n\r\n" +matches = [] + +# This is a test that caught a bug in the one-pass DFA where it (amazingly) was +# using 'is_end_lf' instead of 'is_end_crlf' here. It was probably a copy & +# paste bug. We insert an empty capture group here because it provokes the meta +# regex engine to first find a match and then trip over a panic because the +# one-pass DFA erroneously says there is no match. +[[test]] +name = "onepass-wrong-crlf-with-capture" +regex = '(?Rm:().$)' +haystack = "ZZ\r" +matches = [[[1, 2], [1, 1]]] + +# This is like onepass-wrong-crlf-with-capture above, except it sets up the +# test so that it can be run by the one-pass DFA directly. (i.e., Make it +# anchored and start the search at the right place.) +[[test]] +name = "onepass-wrong-crlf-anchored" +regex = '(?Rm:.$)' +haystack = "ZZ\r" +matches = [[1, 2]] +anchored = true +bounds = [1, 3] diff --git a/third_party/crates/regex-1.13.1/testdata/earliest.toml b/third_party/crates/regex-1.13.1/testdata/earliest.toml new file mode 100644 index 0000000..9516893 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/earliest.toml @@ -0,0 +1,52 @@ +[[test]] +name = "no-greedy-100" +regex = 'a+' +haystack = "aaa" +matches = [[0, 1], [1, 2], [2, 3]] +search-kind = "earliest" + +[[test]] +name = "no-greedy-200" +regex = 'abc+' +haystack = "zzzabccc" +matches = [[3, 6]] +search-kind = "earliest" + +[[test]] +name = "is-ungreedy" +regex = 'a+?' +haystack = "aaa" +matches = [[0, 1], [1, 2], [2, 3]] +search-kind = "earliest" + +[[test]] +name = "look-start-test" +regex = '^(abc|a)' +haystack = "abc" +matches = [ + [[0, 1], [0, 1]], +] +search-kind = "earliest" + +[[test]] +name = "look-end-test" +regex = '(abc|a)$' +haystack = "abc" +matches = [ + [[0, 3], [0, 3]], +] +search-kind = "earliest" + +[[test]] +name = "no-leftmost-first-100" +regex = 'abc|a' +haystack = "abc" +matches = [[0, 1]] +search-kind = "earliest" + +[[test]] +name = "no-leftmost-first-200" +regex = 'aba|a' +haystack = "aba" +matches = [[0, 1], [2, 3]] +search-kind = "earliest" diff --git a/third_party/crates/regex-1.13.1/testdata/empty.toml b/third_party/crates/regex-1.13.1/testdata/empty.toml new file mode 100644 index 0000000..7dfd802 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/empty.toml @@ -0,0 +1,113 @@ +[[test]] +name = "100" +regex = "|b" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "110" +regex = "b|" +haystack = "abc" +matches = [[0, 0], [1, 2], [3, 3]] + +[[test]] +name = "120" +regex = "|z" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "130" +regex = "z|" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "200" +regex = "|" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "210" +regex = "||" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "220" +regex = "||b" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "230" +regex = "b||" +haystack = "abc" +matches = [[0, 0], [1, 2], [3, 3]] + +[[test]] +name = "240" +regex = "||z" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "300" +regex = "(?:)|b" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "310" +regex = "b|(?:)" +haystack = "abc" +matches = [[0, 0], [1, 2], [3, 3]] + +[[test]] +name = "320" +regex = "(?:|)" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "330" +regex = "(?:|)|z" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "400" +regex = "a(?:)|b" +haystack = "abc" +matches = [[0, 1], [1, 2]] + +[[test]] +name = "500" +regex = "" +haystack = "" +matches = [[0, 0]] + +[[test]] +name = "510" +regex = "" +haystack = "a" +matches = [[0, 0], [1, 1]] + +[[test]] +name = "520" +regex = "" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "600" +regex = '(?:|a)*' +haystack = "aaa" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "610" +regex = '(?:|a)+' +haystack = "aaa" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] diff --git a/third_party/crates/regex-1.13.1/testdata/expensive.toml b/third_party/crates/regex-1.13.1/testdata/expensive.toml new file mode 100644 index 0000000..b70e42f --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/expensive.toml @@ -0,0 +1,23 @@ +# This file represent tests that may be expensive to run on some regex engines. +# For example, tests that build a full DFA ahead of time and minimize it can +# take a horrendously long time on regexes that are large (or result in an +# explosion in the number of states). We group these tests together so that +# such engines can simply skip these tests. + +# See: https://github.com/rust-lang/regex/issues/98 +[[test]] +name = "regression-many-repeat-no-stack-overflow" +regex = '^.{1,2500}' +haystack = "a" +matches = [[0, 1]] + +# This test is meant to blow the bounded backtracker's visited capacity. In +# order to do that, we need a somewhat sizeable regex. The purpose of this +# is to make sure there's at least one test that exercises this path in the +# backtracker. All other tests (at time of writing) are small enough that the +# backtracker can handle them fine. +[[test]] +name = "backtrack-blow-visited-capacity" +regex = '\pL{50}' +haystack = "abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxyZZ" +matches = [[0, 50], [50, 100], [100, 150]] diff --git a/third_party/crates/regex-1.13.1/testdata/flags.toml b/third_party/crates/regex-1.13.1/testdata/flags.toml new file mode 100644 index 0000000..30b412c --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/flags.toml @@ -0,0 +1,68 @@ +[[test]] +name = "1" +regex = "(?i)abc" +haystack = "ABC" +matches = [[0, 3]] + +[[test]] +name = "2" +regex = "(?i)a(?-i)bc" +haystack = "Abc" +matches = [[0, 3]] + +[[test]] +name = "3" +regex = "(?i)a(?-i)bc" +haystack = "ABC" +matches = [] + +[[test]] +name = "4" +regex = "(?is)a." +haystack = "A\n" +matches = [[0, 2]] + +[[test]] +name = "5" +regex = "(?is)a.(?-is)a." +haystack = "A\nab" +matches = [[0, 4]] + +[[test]] +name = "6" +regex = "(?is)a.(?-is)a." +haystack = "A\na\n" +matches = [] + +[[test]] +name = "7" +regex = "(?is)a.(?-is:a.)?" +haystack = "A\na\n" +matches = [[0, 2]] +match-limit = 1 + +[[test]] +name = "8" +regex = "(?U)a+" +haystack = "aa" +matches = [[0, 1]] +match-limit = 1 + +[[test]] +name = "9" +regex = "(?U)a+?" +haystack = "aa" +matches = [[0, 2]] + +[[test]] +name = "10" +regex = "(?U)(?-U)a+" +haystack = "aa" +matches = [[0, 2]] + +[[test]] +name = "11" +regex = '(?m)(?:^\d+$\n?)+' +haystack = "123\n456\n789" +matches = [[0, 11]] +unicode = false diff --git a/third_party/crates/regex-1.13.1/testdata/iter.toml b/third_party/crates/regex-1.13.1/testdata/iter.toml new file mode 100644 index 0000000..329b9f0 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/iter.toml @@ -0,0 +1,143 @@ +[[test]] +name = "1" +regex = "a" +haystack = "aaa" +matches = [[0, 1], [1, 2], [2, 3]] + +[[test]] +name = "2" +regex = "a" +haystack = "aba" +matches = [[0, 1], [2, 3]] + +[[test]] +name = "empty1" +regex = '' +haystack = '' +matches = [[0, 0]] + +[[test]] +name = "empty2" +regex = '' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty3" +regex = '(?:)' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty4" +regex = '(?:)*' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty5" +regex = '(?:)+' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty6" +regex = '(?:)?' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty7" +regex = '(?:)(?:)' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty8" +regex = '(?:)+|z' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty9" +regex = 'z|(?:)+' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty10" +regex = '(?:)+|b' +haystack = 'abc' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] + +[[test]] +name = "empty11" +regex = 'b|(?:)+' +haystack = 'abc' +matches = [[0, 0], [1, 2], [3, 3]] + +[[test]] +name = "start1" +regex = "^a" +haystack = "a" +matches = [[0, 1]] + +[[test]] +name = "start2" +regex = "^a" +haystack = "aa" +matches = [[0, 1]] + +[[test]] +name = "anchored1" +regex = "a" +haystack = "a" +matches = [[0, 1]] +anchored = true + +# This test is pretty subtle. It demonstrates the crucial difference between +# '^a' and 'a' compiled in 'anchored' mode. The former regex exclusively +# matches at the start of a haystack and nowhere else. The latter regex has +# no such restriction, but its automaton is constructed such that it lacks a +# `.*?` prefix. So it can actually produce matches at multiple locations. +# The anchored3 test drives this point home. +[[test]] +name = "anchored2" +regex = "a" +haystack = "aa" +matches = [[0, 1], [1, 2]] +anchored = true + +# Unlikely anchored2, this test stops matching anything after it sees `b` +# since it lacks a `.*?` prefix. Since it is looking for 'a' but sees 'b', it +# determines that there are no remaining matches. +[[test]] +name = "anchored3" +regex = "a" +haystack = "aaba" +matches = [[0, 1], [1, 2]] +anchored = true + +[[test]] +name = "nonempty-followedby-empty" +regex = 'abc|.*?' +haystack = "abczzz" +matches = [[0, 3], [4, 4], [5, 5], [6, 6]] + +[[test]] +name = "nonempty-followedby-oneempty" +regex = 'abc|.*?' +haystack = "abcz" +matches = [[0, 3], [4, 4]] + +[[test]] +name = "nonempty-followedby-onemixed" +regex = 'abc|.*?' +haystack = "abczabc" +matches = [[0, 3], [4, 7]] + +[[test]] +name = "nonempty-followedby-twomixed" +regex = 'abc|.*?' +haystack = "abczzabc" +matches = [[0, 3], [4, 4], [5, 8]] diff --git a/third_party/crates/regex-1.13.1/testdata/leftmost-all.toml b/third_party/crates/regex-1.13.1/testdata/leftmost-all.toml new file mode 100644 index 0000000..e3fd950 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/leftmost-all.toml @@ -0,0 +1,25 @@ +[[test]] +name = "alt" +regex = 'foo|foobar' +haystack = "foobar" +matches = [[0, 6]] +match-kind = "all" +search-kind = "leftmost" + +[[test]] +name = "multi" +regex = ['foo', 'foobar'] +haystack = "foobar" +matches = [ + { id = 1, span = [0, 6] }, +] +match-kind = "all" +search-kind = "leftmost" + +[[test]] +name = "dotall" +regex = '(?s:.)' +haystack = "foobar" +matches = [[5, 6]] +match-kind = "all" +search-kind = "leftmost" diff --git a/third_party/crates/regex-1.13.1/testdata/line-terminator.toml b/third_party/crates/regex-1.13.1/testdata/line-terminator.toml new file mode 100644 index 0000000..a398daf --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/line-terminator.toml @@ -0,0 +1,109 @@ +# This tests that we can switch the line terminator to the NUL byte. +[[test]] +name = "nul" +regex = '(?m)^[a-z]+$' +haystack = '\x00abc\x00' +matches = [[1, 4]] +unescape = true +line-terminator = '\x00' + +# This tests that '.' will not match the configured line terminator, but will +# match \n. +[[test]] +name = "dot-changes-with-line-terminator" +regex = '.' +haystack = '\x00\n' +matches = [[1, 2]] +unescape = true +line-terminator = '\x00' + +# This tests that when we switch the line terminator, \n is no longer +# recognized as the terminator. +[[test]] +name = "not-line-feed" +regex = '(?m)^[a-z]+$' +haystack = '\nabc\n' +matches = [] +unescape = true +line-terminator = '\x00' + +# This tests that we can set the line terminator to a non-ASCII byte and have +# it behave as expected. +[[test]] +name = "non-ascii" +regex = '(?m)^[a-z]+$' +haystack = '\xFFabc\xFF' +matches = [[1, 4]] +unescape = true +line-terminator = '\xFF' +utf8 = false + +# This tests a tricky case where the line terminator is set to \r. This ensures +# that the StartLF look-behind assertion is tracked when computing the start +# state. +[[test]] +name = "carriage" +regex = '(?m)^[a-z]+' +haystack = 'ABC\rabc' +matches = [[4, 7]] +bounds = [4, 7] +unescape = true +line-terminator = '\r' + +# This tests that we can set the line terminator to a byte corresponding to a +# word character, and things work as expected. +[[test]] +name = "word-byte" +regex = '(?m)^[a-z]+$' +haystack = 'ZabcZ' +matches = [[1, 4]] +unescape = true +line-terminator = 'Z' + +# This tests that we can set the line terminator to a byte corresponding to a +# non-word character, and things work as expected. +[[test]] +name = "non-word-byte" +regex = '(?m)^[a-z]+$' +haystack = '%abc%' +matches = [[1, 4]] +unescape = true +line-terminator = '%' + +# This combines "set line terminator to a word byte" with a word boundary +# assertion, which should result in no match even though ^/$ matches. +[[test]] +name = "word-boundary" +regex = '(?m)^\b[a-z]+\b$' +haystack = 'ZabcZ' +matches = [] +unescape = true +line-terminator = 'Z' + +# Like 'word-boundary', but does an anchored search at the point where ^ +# matches, but where \b should not. +[[test]] +name = "word-boundary-at" +regex = '(?m)^\b[a-z]+\b$' +haystack = 'ZabcZ' +matches = [] +bounds = [1, 4] +anchored = true +unescape = true +line-terminator = 'Z' + +# Like 'word-boundary-at', but flips the word boundary to a negation. This +# in particular tests a tricky case in DFA engines, where they must consider +# explicitly that a starting configuration from a custom line terminator may +# also required setting the "is from word byte" flag on a state. Otherwise, +# it's treated as "not from a word byte," which would result in \B not matching +# here when it should. +[[test]] +name = "not-word-boundary-at" +regex = '(?m)^\B[a-z]+\B$' +haystack = 'ZabcZ' +matches = [[1, 4]] +bounds = [1, 4] +anchored = true +unescape = true +line-terminator = 'Z' diff --git a/third_party/crates/regex-1.13.1/testdata/misc.toml b/third_party/crates/regex-1.13.1/testdata/misc.toml new file mode 100644 index 0000000..c65531f --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/misc.toml @@ -0,0 +1,99 @@ +[[test]] +name = "ascii-literal" +regex = "a" +haystack = "a" +matches = [[0, 1]] + +[[test]] +name = "ascii-literal-not" +regex = "a" +haystack = "z" +matches = [] + +[[test]] +name = "ascii-literal-anchored" +regex = "a" +haystack = "a" +matches = [[0, 1]] +anchored = true + +[[test]] +name = "ascii-literal-anchored-not" +regex = "a" +haystack = "z" +matches = [] +anchored = true + +[[test]] +name = "anchor-start-end-line" +regex = '(?m)^bar$' +haystack = "foo\nbar\nbaz" +matches = [[4, 7]] + +[[test]] +name = "prefix-literal-match" +regex = '^abc' +haystack = "abc" +matches = [[0, 3]] + +[[test]] +name = "prefix-literal-match-ascii" +regex = '^abc' +haystack = "abc" +matches = [[0, 3]] +unicode = false +utf8 = false + +[[test]] +name = "prefix-literal-no-match" +regex = '^abc' +haystack = "zabc" +matches = [] + +[[test]] +name = "one-literal-edge" +regex = 'abc' +haystack = "xxxxxab" +matches = [] + +[[test]] +name = "terminates" +regex = 'a$' +haystack = "a" +matches = [[0, 1]] + +[[test]] +name = "suffix-100" +regex = '.*abcd' +haystack = "abcd" +matches = [[0, 4]] + +[[test]] +name = "suffix-200" +regex = '.*(?:abcd)+' +haystack = "abcd" +matches = [[0, 4]] + +[[test]] +name = "suffix-300" +regex = '.*(?:abcd)+' +haystack = "abcdabcd" +matches = [[0, 8]] + +[[test]] +name = "suffix-400" +regex = '.*(?:abcd)+' +haystack = "abcdxabcd" +matches = [[0, 9]] + +[[test]] +name = "suffix-500" +regex = '.*x(?:abcd)+' +haystack = "abcdxabcd" +matches = [[0, 9]] + +[[test]] +name = "suffix-600" +regex = '[^abcd]*x(?:abcd)+' +haystack = "abcdxabcd" +matches = [[4, 9]] diff --git a/third_party/crates/regex-1.13.1/testdata/multiline.toml b/third_party/crates/regex-1.13.1/testdata/multiline.toml new file mode 100644 index 0000000..3acc901 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/multiline.toml @@ -0,0 +1,845 @@ +[[test]] +name = "basic1" +regex = '(?m)^[a-z]+$' +haystack = "abc\ndef\nxyz" +matches = [[0, 3], [4, 7], [8, 11]] + +[[test]] +name = "basic1-crlf" +regex = '(?Rm)^[a-z]+$' +haystack = "abc\ndef\nxyz" +matches = [[0, 3], [4, 7], [8, 11]] + +[[test]] +name = "basic1-crlf-cr" +regex = '(?Rm)^[a-z]+$' +haystack = "abc\rdef\rxyz" +matches = [[0, 3], [4, 7], [8, 11]] + +[[test]] +name = "basic2" +regex = '(?m)^$' +haystack = "abc\ndef\nxyz" +matches = [] + +[[test]] +name = "basic2-crlf" +regex = '(?Rm)^$' +haystack = "abc\ndef\nxyz" +matches = [] + +[[test]] +name = "basic2-crlf-cr" +regex = '(?Rm)^$' +haystack = "abc\rdef\rxyz" +matches = [] + +[[test]] +name = "basic3" +regex = '(?m)^' +haystack = "abc\ndef\nxyz" +matches = [[0, 0], [4, 4], [8, 8]] + +[[test]] +name = "basic3-crlf" +regex = '(?Rm)^' +haystack = "abc\ndef\nxyz" +matches = [[0, 0], [4, 4], [8, 8]] + +[[test]] +name = "basic3-crlf-cr" +regex = '(?Rm)^' +haystack = "abc\rdef\rxyz" +matches = [[0, 0], [4, 4], [8, 8]] + +[[test]] +name = "basic4" +regex = '(?m)$' +haystack = "abc\ndef\nxyz" +matches = [[3, 3], [7, 7], [11, 11]] + +[[test]] +name = "basic4-crlf" +regex = '(?Rm)$' +haystack = "abc\ndef\nxyz" +matches = [[3, 3], [7, 7], [11, 11]] + +[[test]] +name = "basic4-crlf-cr" +regex = '(?Rm)$' +haystack = "abc\rdef\rxyz" +matches = [[3, 3], [7, 7], [11, 11]] + +[[test]] +name = "basic5" +regex = '(?m)^[a-z]' +haystack = "abc\ndef\nxyz" +matches = [[0, 1], [4, 5], [8, 9]] + +[[test]] +name = "basic5-crlf" +regex = '(?Rm)^[a-z]' +haystack = "abc\ndef\nxyz" +matches = [[0, 1], [4, 5], [8, 9]] + +[[test]] +name = "basic5-crlf-cr" +regex = '(?Rm)^[a-z]' +haystack = "abc\rdef\rxyz" +matches = [[0, 1], [4, 5], [8, 9]] + +[[test]] +name = "basic6" +regex = '(?m)[a-z]^' +haystack = "abc\ndef\nxyz" +matches = [] + +[[test]] +name = "basic6-crlf" +regex = '(?Rm)[a-z]^' +haystack = "abc\ndef\nxyz" +matches = [] + +[[test]] +name = "basic6-crlf-cr" +regex = '(?Rm)[a-z]^' +haystack = "abc\rdef\rxyz" +matches = [] + +[[test]] +name = "basic7" +regex = '(?m)[a-z]$' +haystack = "abc\ndef\nxyz" +matches = [[2, 3], [6, 7], [10, 11]] + +[[test]] +name = "basic7-crlf" +regex = '(?Rm)[a-z]$' +haystack = "abc\ndef\nxyz" +matches = [[2, 3], [6, 7], [10, 11]] + +[[test]] +name = "basic7-crlf-cr" +regex = '(?Rm)[a-z]$' +haystack = "abc\rdef\rxyz" +matches = [[2, 3], [6, 7], [10, 11]] + +[[test]] +name = "basic8" +regex = '(?m)$[a-z]' +haystack = "abc\ndef\nxyz" +matches = [] + +[[test]] +name = "basic8-crlf" +regex = '(?Rm)$[a-z]' +haystack = "abc\ndef\nxyz" +matches = [] + +[[test]] +name = "basic8-crlf-cr" +regex = '(?Rm)$[a-z]' +haystack = "abc\rdef\rxyz" +matches = [] + +[[test]] +name = "basic9" +regex = '(?m)^$' +haystack = "" +matches = [[0, 0]] + +[[test]] +name = "basic9-crlf" +regex = '(?Rm)^$' +haystack = "" +matches = [[0, 0]] + +[[test]] +name = "repeat1" +regex = '(?m)(?:^$)*' +haystack = "a\nb\nc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +[[test]] +name = "repeat1-crlf" +regex = '(?Rm)(?:^$)*' +haystack = "a\nb\nc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +[[test]] +name = "repeat1-crlf-cr" +regex = '(?Rm)(?:^$)*' +haystack = "a\rb\rc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +[[test]] +name = "repeat1-no-multi" +regex = '(?:^$)*' +haystack = "a\nb\nc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +[[test]] +name = "repeat1-no-multi-crlf" +regex = '(?R)(?:^$)*' +haystack = "a\nb\nc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +[[test]] +name = "repeat1-no-multi-crlf-cr" +regex = '(?R)(?:^$)*' +haystack = "a\rb\rc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +[[test]] +name = "repeat2" +regex = '(?m)(?:^|a)+' +haystack = "a\naaa\n" +matches = [[0, 0], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat2-crlf" +regex = '(?Rm)(?:^|a)+' +haystack = "a\naaa\n" +matches = [[0, 0], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat2-crlf-cr" +regex = '(?Rm)(?:^|a)+' +haystack = "a\raaa\r" +matches = [[0, 0], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat2-no-multi" +regex = '(?:^|a)+' +haystack = "a\naaa\n" +matches = [[0, 0], [2, 5]] + +[[test]] +name = "repeat2-no-multi-crlf" +regex = '(?R)(?:^|a)+' +haystack = "a\naaa\n" +matches = [[0, 0], [2, 5]] + +[[test]] +name = "repeat2-no-multi-crlf-cr" +regex = '(?R)(?:^|a)+' +haystack = "a\raaa\r" +matches = [[0, 0], [2, 5]] + +[[test]] +name = "repeat3" +regex = '(?m)(?:^|a)*' +haystack = "a\naaa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat3-crlf" +regex = '(?Rm)(?:^|a)*' +haystack = "a\naaa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat3-crlf-cr" +regex = '(?Rm)(?:^|a)*' +haystack = "a\raaa\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat3-no-multi" +regex = '(?:^|a)*' +haystack = "a\naaa\n" +matches = [[0, 0], [1, 1], [2, 5], [6, 6]] + +[[test]] +name = "repeat3-no-multi-crlf" +regex = '(?R)(?:^|a)*' +haystack = "a\naaa\n" +matches = [[0, 0], [1, 1], [2, 5], [6, 6]] + +[[test]] +name = "repeat3-no-multi-crlf-cr" +regex = '(?R)(?:^|a)*' +haystack = "a\raaa\r" +matches = [[0, 0], [1, 1], [2, 5], [6, 6]] + +[[test]] +name = "repeat4" +regex = '(?m)(?:^|a+)' +haystack = "a\naaa\n" +matches = [[0, 0], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat4-crlf" +regex = '(?Rm)(?:^|a+)' +haystack = "a\naaa\n" +matches = [[0, 0], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat4-crlf-cr" +regex = '(?Rm)(?:^|a+)' +haystack = "a\raaa\r" +matches = [[0, 0], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat4-no-multi" +regex = '(?:^|a+)' +haystack = "a\naaa\n" +matches = [[0, 0], [2, 5]] + +[[test]] +name = "repeat4-no-multi-crlf" +regex = '(?R)(?:^|a+)' +haystack = "a\naaa\n" +matches = [[0, 0], [2, 5]] + +[[test]] +name = "repeat4-no-multi-crlf-cr" +regex = '(?R)(?:^|a+)' +haystack = "a\raaa\r" +matches = [[0, 0], [2, 5]] + +[[test]] +name = "repeat5" +regex = '(?m)(?:^|a*)' +haystack = "a\naaa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat5-crlf" +regex = '(?Rm)(?:^|a*)' +haystack = "a\naaa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat5-crlf-cr" +regex = '(?Rm)(?:^|a*)' +haystack = "a\raaa\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 5], [6, 6]] + +[[test]] +name = "repeat5-no-multi" +regex = '(?:^|a*)' +haystack = "a\naaa\n" +matches = [[0, 0], [1, 1], [2, 5], [6, 6]] + +[[test]] +name = "repeat5-no-multi-crlf" +regex = '(?R)(?:^|a*)' +haystack = "a\naaa\n" +matches = [[0, 0], [1, 1], [2, 5], [6, 6]] + +[[test]] +name = "repeat5-no-multi-crlf-cr" +regex = '(?R)(?:^|a*)' +haystack = "a\raaa\r" +matches = [[0, 0], [1, 1], [2, 5], [6, 6]] + +[[test]] +name = "repeat6" +regex = '(?m)(?:^[a-z])+' +haystack = "abc\ndef\nxyz" +matches = [[0, 1], [4, 5], [8, 9]] + +[[test]] +name = "repeat6-crlf" +regex = '(?Rm)(?:^[a-z])+' +haystack = "abc\ndef\nxyz" +matches = [[0, 1], [4, 5], [8, 9]] + +[[test]] +name = "repeat6-crlf-cr" +regex = '(?Rm)(?:^[a-z])+' +haystack = "abc\rdef\rxyz" +matches = [[0, 1], [4, 5], [8, 9]] + +[[test]] +name = "repeat6-no-multi" +regex = '(?:^[a-z])+' +haystack = "abc\ndef\nxyz" +matches = [[0, 1]] + +[[test]] +name = "repeat6-no-multi-crlf" +regex = '(?R)(?:^[a-z])+' +haystack = "abc\ndef\nxyz" +matches = [[0, 1]] + +[[test]] +name = "repeat6-no-multi-crlf-cr" +regex = '(?R)(?:^[a-z])+' +haystack = "abc\rdef\rxyz" +matches = [[0, 1]] + +[[test]] +name = "repeat7" +regex = '(?m)(?:^[a-z]{3}\n?)+' +haystack = "abc\ndef\nxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat7-crlf" +regex = '(?Rm)(?:^[a-z]{3}\n?)+' +haystack = "abc\ndef\nxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat7-crlf-cr" +regex = '(?Rm)(?:^[a-z]{3}\r?)+' +haystack = "abc\rdef\rxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat7-no-multi" +regex = '(?:^[a-z]{3}\n?)+' +haystack = "abc\ndef\nxyz" +matches = [[0, 4]] + +[[test]] +name = "repeat7-no-multi-crlf" +regex = '(?R)(?:^[a-z]{3}\n?)+' +haystack = "abc\ndef\nxyz" +matches = [[0, 4]] + +[[test]] +name = "repeat7-no-multi-crlf-cr" +regex = '(?R)(?:^[a-z]{3}\r?)+' +haystack = "abc\rdef\rxyz" +matches = [[0, 4]] + +[[test]] +name = "repeat8" +regex = '(?m)(?:^[a-z]{3}\n?)*' +haystack = "abc\ndef\nxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat8-crlf" +regex = '(?Rm)(?:^[a-z]{3}\n?)*' +haystack = "abc\ndef\nxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat8-crlf-cr" +regex = '(?Rm)(?:^[a-z]{3}\r?)*' +haystack = "abc\rdef\rxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat8-no-multi" +regex = '(?:^[a-z]{3}\n?)*' +haystack = "abc\ndef\nxyz" +matches = [[0, 4], [5, 5], [6, 6], [7, 7], [8, 8], [9, 9], [10, 10], [11, 11]] + +[[test]] +name = "repeat8-no-multi-crlf" +regex = '(?R)(?:^[a-z]{3}\n?)*' +haystack = "abc\ndef\nxyz" +matches = [[0, 4], [5, 5], [6, 6], [7, 7], [8, 8], [9, 9], [10, 10], [11, 11]] + +[[test]] +name = "repeat8-no-multi-crlf-cr" +regex = '(?R)(?:^[a-z]{3}\r?)*' +haystack = "abc\rdef\rxyz" +matches = [[0, 4], [5, 5], [6, 6], [7, 7], [8, 8], [9, 9], [10, 10], [11, 11]] + +[[test]] +name = "repeat9" +regex = '(?m)(?:\n?[a-z]{3}$)+' +haystack = "abc\ndef\nxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat9-crlf" +regex = '(?Rm)(?:\n?[a-z]{3}$)+' +haystack = "abc\ndef\nxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat9-crlf-cr" +regex = '(?Rm)(?:\r?[a-z]{3}$)+' +haystack = "abc\rdef\rxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat9-no-multi" +regex = '(?:\n?[a-z]{3}$)+' +haystack = "abc\ndef\nxyz" +matches = [[7, 11]] + +[[test]] +name = "repeat9-no-multi-crlf" +regex = '(?R)(?:\n?[a-z]{3}$)+' +haystack = "abc\ndef\nxyz" +matches = [[7, 11]] + +[[test]] +name = "repeat9-no-multi-crlf-cr" +regex = '(?R)(?:\r?[a-z]{3}$)+' +haystack = "abc\rdef\rxyz" +matches = [[7, 11]] + +[[test]] +name = "repeat10" +regex = '(?m)(?:\n?[a-z]{3}$)*' +haystack = "abc\ndef\nxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat10-crlf" +regex = '(?Rm)(?:\n?[a-z]{3}$)*' +haystack = "abc\ndef\nxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat10-crlf-cr" +regex = '(?Rm)(?:\r?[a-z]{3}$)*' +haystack = "abc\rdef\rxyz" +matches = [[0, 11]] + +[[test]] +name = "repeat10-no-multi" +regex = '(?:\n?[a-z]{3}$)*' +haystack = "abc\ndef\nxyz" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 11]] + +[[test]] +name = "repeat10-no-multi-crlf" +regex = '(?R)(?:\n?[a-z]{3}$)*' +haystack = "abc\ndef\nxyz" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 11]] + +[[test]] +name = "repeat10-no-multi-crlf-cr" +regex = '(?R)(?:\r?[a-z]{3}$)*' +haystack = "abc\rdef\rxyz" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 11]] + +[[test]] +name = "repeat11" +regex = '(?m)^*' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat11-crlf" +regex = '(?Rm)^*' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat11-crlf-cr" +regex = '(?Rm)^*' +haystack = "\raa\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat11-no-multi" +regex = '^*' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat11-no-multi-crlf" +regex = '(?R)^*' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat11-no-multi-crlf-cr" +regex = '(?R)^*' +haystack = "\raa\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat12" +regex = '(?m)^+' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [4, 4]] + +[[test]] +name = "repeat12-crlf" +regex = '(?Rm)^+' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [4, 4]] + +[[test]] +name = "repeat12-crlf-cr" +regex = '(?Rm)^+' +haystack = "\raa\r" +matches = [[0, 0], [1, 1], [4, 4]] + +[[test]] +name = "repeat12-no-multi" +regex = '^+' +haystack = "\naa\n" +matches = [[0, 0]] + +[[test]] +name = "repeat12-no-multi-crlf" +regex = '(?R)^+' +haystack = "\naa\n" +matches = [[0, 0]] + +[[test]] +name = "repeat12-no-multi-crlf-cr" +regex = '(?R)^+' +haystack = "\raa\r" +matches = [[0, 0]] + +[[test]] +name = "repeat13" +regex = '(?m)$*' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat13-crlf" +regex = '(?Rm)$*' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat13-crlf-cr" +regex = '(?Rm)$*' +haystack = "\raa\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat13-no-multi" +regex = '$*' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat13-no-multi-crlf" +regex = '(?R)$*' +haystack = "\naa\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat13-no-multi-crlf-cr" +regex = '(?R)$*' +haystack = "\raa\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +[[test]] +name = "repeat14" +regex = '(?m)$+' +haystack = "\naa\n" +matches = [[0, 0], [3, 3], [4, 4]] + +[[test]] +name = "repeat14-crlf" +regex = '(?Rm)$+' +haystack = "\naa\n" +matches = [[0, 0], [3, 3], [4, 4]] + +[[test]] +name = "repeat14-crlf-cr" +regex = '(?Rm)$+' +haystack = "\raa\r" +matches = [[0, 0], [3, 3], [4, 4]] + +[[test]] +name = "repeat14-no-multi" +regex = '$+' +haystack = "\naa\n" +matches = [[4, 4]] + +[[test]] +name = "repeat14-no-multi-crlf" +regex = '(?R)$+' +haystack = "\naa\n" +matches = [[4, 4]] + +[[test]] +name = "repeat14-no-multi-crlf-cr" +regex = '(?R)$+' +haystack = "\raa\r" +matches = [[4, 4]] + +[[test]] +name = "repeat15" +regex = '(?m)(?:$\n)+' +haystack = "\n\naaa\n\n" +matches = [[0, 2], [5, 7]] + +[[test]] +name = "repeat15-crlf" +regex = '(?Rm)(?:$\n)+' +haystack = "\n\naaa\n\n" +matches = [[0, 2], [5, 7]] + +[[test]] +name = "repeat15-crlf-cr" +regex = '(?Rm)(?:$\r)+' +haystack = "\r\raaa\r\r" +matches = [[0, 2], [5, 7]] + +[[test]] +name = "repeat15-no-multi" +regex = '(?:$\n)+' +haystack = "\n\naaa\n\n" +matches = [] + +[[test]] +name = "repeat15-no-multi-crlf" +regex = '(?R)(?:$\n)+' +haystack = "\n\naaa\n\n" +matches = [] + +[[test]] +name = "repeat15-no-multi-crlf-cr" +regex = '(?R)(?:$\r)+' +haystack = "\r\raaa\r\r" +matches = [] + +[[test]] +name = "repeat16" +regex = '(?m)(?:$\n)*' +haystack = "\n\naaa\n\n" +matches = [[0, 2], [3, 3], [4, 4], [5, 7]] + +[[test]] +name = "repeat16-crlf" +regex = '(?Rm)(?:$\n)*' +haystack = "\n\naaa\n\n" +matches = [[0, 2], [3, 3], [4, 4], [5, 7]] + +[[test]] +name = "repeat16-crlf-cr" +regex = '(?Rm)(?:$\r)*' +haystack = "\r\raaa\r\r" +matches = [[0, 2], [3, 3], [4, 4], [5, 7]] + +[[test]] +name = "repeat16-no-multi" +regex = '(?:$\n)*' +haystack = "\n\naaa\n\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]] + +[[test]] +name = "repeat16-no-multi-crlf" +regex = '(?R)(?:$\n)*' +haystack = "\n\naaa\n\n" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]] + +[[test]] +name = "repeat16-no-multi-crlf-cr" +regex = '(?R)(?:$\r)*' +haystack = "\r\raaa\r\r" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]] + +[[test]] +name = "repeat17" +regex = '(?m)(?:$\n^)+' +haystack = "\n\naaa\n\n" +matches = [[0, 2], [5, 7]] + +[[test]] +name = "repeat17-crlf" +regex = '(?Rm)(?:$\n^)+' +haystack = "\n\naaa\n\n" +matches = [[0, 2], [5, 7]] + +[[test]] +name = "repeat17-crlf-cr" +regex = '(?Rm)(?:$\r^)+' +haystack = "\r\raaa\r\r" +matches = [[0, 2], [5, 7]] + +[[test]] +name = "repeat17-no-multi" +regex = '(?:$\n^)+' +haystack = "\n\naaa\n\n" +matches = [] + +[[test]] +name = "repeat17-no-multi-crlf" +regex = '(?R)(?:$\n^)+' +haystack = "\n\naaa\n\n" +matches = [] + +[[test]] +name = "repeat17-no-multi-crlf-cr" +regex = '(?R)(?:$\r^)+' +haystack = "\r\raaa\r\r" +matches = [] + +[[test]] +name = "repeat18" +regex = '(?m)(?:^|$)+' +haystack = "\n\naaa\n\n" +matches = [[0, 0], [1, 1], [2, 2], [5, 5], [6, 6], [7, 7]] + +[[test]] +name = "repeat18-crlf" +regex = '(?Rm)(?:^|$)+' +haystack = "\n\naaa\n\n" +matches = [[0, 0], [1, 1], [2, 2], [5, 5], [6, 6], [7, 7]] + +[[test]] +name = "repeat18-crlf-cr" +regex = '(?Rm)(?:^|$)+' +haystack = "\r\raaa\r\r" +matches = [[0, 0], [1, 1], [2, 2], [5, 5], [6, 6], [7, 7]] + +[[test]] +name = "repeat18-no-multi" +regex = '(?:^|$)+' +haystack = "\n\naaa\n\n" +matches = [[0, 0], [7, 7]] + +[[test]] +name = "repeat18-no-multi-crlf" +regex = '(?R)(?:^|$)+' +haystack = "\n\naaa\n\n" +matches = [[0, 0], [7, 7]] + +[[test]] +name = "repeat18-no-multi-crlf-cr" +regex = '(?R)(?:^|$)+' +haystack = "\r\raaa\r\r" +matches = [[0, 0], [7, 7]] + +[[test]] +name = "match-line-100" +regex = '(?m)^.+$' +haystack = "aa\naaaaaaaaaaaaaaaaaaa\n" +matches = [[0, 2], [3, 22]] + +[[test]] +name = "match-line-100-crlf" +regex = '(?Rm)^.+$' +haystack = "aa\naaaaaaaaaaaaaaaaaaa\n" +matches = [[0, 2], [3, 22]] + +[[test]] +name = "match-line-100-crlf-cr" +regex = '(?Rm)^.+$' +haystack = "aa\raaaaaaaaaaaaaaaaaaa\r" +matches = [[0, 2], [3, 22]] + +[[test]] +name = "match-line-200" +regex = '(?m)^.+$' +haystack = "aa\naaaaaaaaaaaaaaaaaaa\n" +matches = [[0, 2], [3, 22]] +unicode = false +utf8 = false + +[[test]] +name = "match-line-200-crlf" +regex = '(?Rm)^.+$' +haystack = "aa\naaaaaaaaaaaaaaaaaaa\n" +matches = [[0, 2], [3, 22]] +unicode = false +utf8 = false + +[[test]] +name = "match-line-200-crlf-cr" +regex = '(?Rm)^.+$' +haystack = "aa\raaaaaaaaaaaaaaaaaaa\r" +matches = [[0, 2], [3, 22]] +unicode = false +utf8 = false diff --git a/third_party/crates/regex-1.13.1/testdata/no-unicode.toml b/third_party/crates/regex-1.13.1/testdata/no-unicode.toml new file mode 100644 index 0000000..0ddac4c --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/no-unicode.toml @@ -0,0 +1,222 @@ +[[test]] +name = "invalid-utf8-literal1" +regex = '\xFF' +haystack = '\xFF' +matches = [[0, 1]] +unicode = false +utf8 = false +unescape = true + + +[[test]] +name = "mixed" +regex = '(?:.+)(?-u)(?:.+)' +haystack = '\xCE\x93\xCE\x94\xFF' +matches = [[0, 5]] +utf8 = false +unescape = true + + +[[test]] +name = "case1" +regex = "a" +haystack = "A" +matches = [[0, 1]] +case-insensitive = true +unicode = false + +[[test]] +name = "case2" +regex = "[a-z]+" +haystack = "AaAaA" +matches = [[0, 5]] +case-insensitive = true +unicode = false + +[[test]] +name = "case3" +regex = "[a-z]+" +haystack = "aA\u212AaA" +matches = [[0, 7]] +case-insensitive = true + +[[test]] +name = "case4" +regex = "[a-z]+" +haystack = "aA\u212AaA" +matches = [[0, 2], [5, 7]] +case-insensitive = true +unicode = false + + +[[test]] +name = "negate1" +regex = "[^a]" +haystack = "δ" +matches = [[0, 2]] + +[[test]] +name = "negate2" +regex = "[^a]" +haystack = "δ" +matches = [[0, 1], [1, 2]] +unicode = false +utf8 = false + + +[[test]] +name = "dotstar-prefix1" +regex = "a" +haystack = '\xFFa' +matches = [[1, 2]] +unicode = false +utf8 = false +unescape = true + +[[test]] +name = "dotstar-prefix2" +regex = "a" +haystack = '\xFFa' +matches = [[1, 2]] +utf8 = false +unescape = true + + +[[test]] +name = "null-bytes1" +regex = '[^\x00]+\x00' +haystack = 'foo\x00' +matches = [[0, 4]] +unicode = false +utf8 = false +unescape = true + + +[[test]] +name = "word-ascii" +regex = '\w+' +haystack = "aδ" +matches = [[0, 1]] +unicode = false + +[[test]] +name = "word-unicode" +regex = '\w+' +haystack = "aδ" +matches = [[0, 3]] + +[[test]] +name = "decimal-ascii" +regex = '\d+' +haystack = "1२३9" +matches = [[0, 1], [7, 8]] +unicode = false + +[[test]] +name = "decimal-unicode" +regex = '\d+' +haystack = "1२३9" +matches = [[0, 8]] + +[[test]] +name = "space-ascii" +regex = '\s+' +haystack = " \u1680" +matches = [[0, 1]] +unicode = false + +[[test]] +name = "space-unicode" +regex = '\s+' +haystack = " \u1680" +matches = [[0, 4]] + + +[[test]] +# See: https://github.com/rust-lang/regex/issues/484 +name = "iter1-bytes" +regex = '' +haystack = "☃" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] +utf8 = false + +[[test]] +# See: https://github.com/rust-lang/regex/issues/484 +name = "iter1-utf8" +regex = '' +haystack = "☃" +matches = [[0, 0], [3, 3]] + +[[test]] +# See: https://github.com/rust-lang/regex/issues/484 +# Note that iter2-utf8 doesn't make sense here, since the input isn't UTF-8. +name = "iter2-bytes" +regex = '' +haystack = 'b\xFFr' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] +unescape = true +utf8 = false + + +# These test that unanchored prefixes can munch through invalid UTF-8 even when +# utf8 is enabled. +# +# This test actually reflects an interesting simplification in how the Thompson +# NFA is constructed. It used to be that the NFA could be built with an +# unanchored prefix that either matched any byte or _only_ matched valid UTF-8. +# But the latter turns out to be pretty precarious when it comes to prefilters, +# because if you search a haystack that contains invalid UTF-8 but have an +# unanchored prefix that requires UTF-8, then prefilters are no longer a valid +# optimization because you actually have to check that everything is valid +# UTF-8. +# +# Originally, I had thought that we needed a valid UTF-8 unanchored prefix in +# order to guarantee that we only match at valid UTF-8 boundaries. But this +# isn't actually true! There are really only two things to consider here: +# +# 1) Will a regex match split an encoded codepoint? No. Because by construction, +# we ensure that a MATCH state can only be reached by following valid UTF-8 (assuming +# all of the UTF-8 modes are enabled). +# +# 2) Will a regex match arbitrary bytes that aren't valid UTF-8? Again, no, +# assuming all of the UTF-8 modes are enabled. +[[test]] +name = "unanchored-invalid-utf8-match-100" +regex = '[a-z]' +haystack = '\xFFa\xFF' +matches = [[1, 2]] +unescape = true +utf8 = false + +# This test shows that we can still prevent a match from occurring by requiring +# that valid UTF-8 match by inserting our own unanchored prefix. Thus, if the +# behavior of not munching through invalid UTF-8 anywhere is needed, then it +# can be achieved thusly. +[[test]] +name = "unanchored-invalid-utf8-nomatch" +regex = '^(?s:.)*?[a-z]' +haystack = '\xFFa\xFF' +matches = [] +unescape = true +utf8 = false + +# This is a tricky test that makes sure we don't accidentally do a kind of +# unanchored search when we've requested that a regex engine not report +# empty matches that split a codepoint. This test caught a regression during +# development where the code for skipping over bad empty matches would do so +# even if the search should have been anchored. This is ultimately what led to +# making 'anchored' an 'Input' option, so that it was always clear what kind +# of search was being performed. (Before that, whether a search was anchored +# or not was a config knob on the regex engine.) This did wind up making DFAs +# a little more complex to configure (with their 'StartKind' knob), but it +# generally smoothed out everything else. +# +# Great example of a test whose failure motivated a sweeping API refactoring. +[[test]] +name = "anchored-iter-empty-utf8" +regex = '' +haystack = 'a☃z' +matches = [[0, 0], [1, 1]] +unescape = false +utf8 = true +anchored = true diff --git a/third_party/crates/regex-1.13.1/testdata/overlapping.toml b/third_party/crates/regex-1.13.1/testdata/overlapping.toml new file mode 100644 index 0000000..7bcd45a --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/overlapping.toml @@ -0,0 +1,280 @@ +# NOTE: We define a number of tests where the *match* kind is 'leftmost-first' +# but the *search* kind is 'overlapping'. This is a somewhat nonsensical +# combination and can produce odd results. Nevertheless, those results should +# be consistent so we test them here. (At the time of writing this note, I +# hadn't yet decided whether to make 'leftmost-first' with 'overlapping' result +# in unspecified behavior.) + +# This demonstrates how a full overlapping search is obvious quadratic. This +# regex reports a match for every substring in the haystack. +[[test]] +name = "ungreedy-dotstar-matches-everything-100" +regex = [".*?"] +haystack = "zzz" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [0, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [1, 2] }, + { id = 0, span = [0, 2] }, + { id = 0, span = [3, 3] }, + { id = 0, span = [2, 3] }, + { id = 0, span = [1, 3] }, + { id = 0, span = [0, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "greedy-dotstar-matches-everything-100" +regex = [".*"] +haystack = "zzz" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [0, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [1, 2] }, + { id = 0, span = [0, 2] }, + { id = 0, span = [3, 3] }, + { id = 0, span = [2, 3] }, + { id = 0, span = [1, 3] }, + { id = 0, span = [0, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "repetition-plus-leftmost-first-100" +regex = 'a+' +haystack = "aaa" +matches = [[0, 1], [1, 2], [0, 2], [2, 3], [1, 3], [0, 3]] +match-kind = "leftmost-first" +search-kind = "overlapping" + +[[test]] +name = "repetition-plus-leftmost-first-110" +regex = '☃+' +haystack = "☃☃☃" +matches = [[0, 3], [3, 6], [0, 6], [6, 9], [3, 9], [0, 9]] +match-kind = "leftmost-first" +search-kind = "overlapping" + +[[test]] +name = "repetition-plus-all-100" +regex = 'a+' +haystack = "aaa" +matches = [[0, 1], [1, 2], [0, 2], [2, 3], [1, 3], [0, 3]] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "repetition-plus-all-110" +regex = '☃+' +haystack = "☃☃☃" +matches = [[0, 3], [3, 6], [0, 6], [6, 9], [3, 9], [0, 9]] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "repetition-plus-leftmost-first-200" +regex = '(abc)+' +haystack = "zzabcabczzabc" +matches = [ + [[2, 5], [2, 5]], + [[5, 8], [5, 8]], + [[2, 8], [5, 8]], +] +match-kind = "leftmost-first" +search-kind = "overlapping" + +[[test]] +name = "repetition-plus-all-200" +regex = '(abc)+' +haystack = "zzabcabczzabc" +matches = [ + [[2, 5], [2, 5]], + [[5, 8], [5, 8]], + [[2, 8], [5, 8]], + [[10, 13], [10, 13]], +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "repetition-star-leftmost-first-100" +regex = 'a*' +haystack = "aaa" +matches = [ + [0, 0], + [1, 1], + [0, 1], + [2, 2], + [1, 2], + [0, 2], + [3, 3], + [2, 3], + [1, 3], + [0, 3], +] +match-kind = "leftmost-first" +search-kind = "overlapping" + +[[test]] +name = "repetition-star-all-100" +regex = 'a*' +haystack = "aaa" +matches = [ + [0, 0], + [1, 1], + [0, 1], + [2, 2], + [1, 2], + [0, 2], + [3, 3], + [2, 3], + [1, 3], + [0, 3], +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "repetition-star-leftmost-first-200" +regex = '(abc)*' +haystack = "zzabcabczzabc" +matches = [ + [[0, 0], []], +] +match-kind = "leftmost-first" +search-kind = "overlapping" + +[[test]] +name = "repetition-star-all-200" +regex = '(abc)*' +haystack = "zzabcabczzabc" +matches = [ + [[0, 0], []], + [[1, 1], []], + [[2, 2], []], + [[3, 3], []], + [[4, 4], []], + [[5, 5], []], + [[2, 5], [2, 5]], + [[6, 6], []], + [[7, 7], []], + [[8, 8], []], + [[5, 8], [5, 8]], + [[2, 8], [5, 8]], + [[9, 9], []], + [[10, 10], []], + [[11, 11], []], + [[12, 12], []], + [[13, 13], []], + [[10, 13], [10, 13]], +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "start-end-rep-leftmost-first" +regex = '(^$)*' +haystack = "abc" +matches = [ + [[0, 0], []], +] +match-kind = "leftmost-first" +search-kind = "overlapping" + +[[test]] +name = "start-end-rep-all" +regex = '(^$)*' +haystack = "abc" +matches = [ + [[0, 0], []], + [[1, 1], []], + [[2, 2], []], + [[3, 3], []], +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "alt-leftmost-first-100" +regex = 'abc|a' +haystack = "zzabcazzaabc" +matches = [[2, 3], [2, 5]] +match-kind = "leftmost-first" +search-kind = "overlapping" + +[[test]] +name = "alt-all-100" +regex = 'abc|a' +haystack = "zzabcazzaabc" +matches = [[2, 3], [2, 5], [5, 6], [8, 9], [9, 10], [9, 12]] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty-000" +regex = "" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty-alt-000" +regex = "|b" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [1, 2], [3, 3]] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty-alt-010" +regex = "b|" +haystack = "abc" +matches = [[0, 0], [1, 1], [2, 2], [1, 2], [3, 3]] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +# See: https://github.com/rust-lang/regex/issues/484 +name = "iter1-bytes" +regex = '' +haystack = "☃" +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] +utf8 = false +match-kind = "all" +search-kind = "overlapping" + +[[test]] +# See: https://github.com/rust-lang/regex/issues/484 +name = "iter1-utf8" +regex = '' +haystack = "☃" +matches = [[0, 0], [3, 3]] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "iter1-incomplete-utf8" +regex = '' +haystack = '\xE2\x98' # incomplete snowman +matches = [[0, 0], [1, 1], [2, 2]] +match-kind = "all" +search-kind = "overlapping" +unescape = true +utf8 = false + +[[test]] +name = "scratch" +regex = ['sam', 'samwise'] +haystack = "samwise" +matches = [ + { id = 0, span = [0, 3] }, +] +match-kind = "leftmost-first" +search-kind = "overlapping" diff --git a/third_party/crates/regex-1.13.1/testdata/regex-lite.toml b/third_party/crates/regex-1.13.1/testdata/regex-lite.toml new file mode 100644 index 0000000..1769d80 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/regex-lite.toml @@ -0,0 +1,98 @@ +# These tests are specifically written to test the regex-lite crate. While it +# largely has the same semantics as the regex crate, there are some differences +# around Unicode support and UTF-8. +# +# To be clear, regex-lite supports far fewer patterns because of its lack of +# Unicode support, nested character classes and character class set operations. +# What we're talking about here are the patterns that both crates support but +# where the semantics might differ. + +# regex-lite uses ASCII definitions for Perl character classes. +[[test]] +name = "perl-class-decimal" +regex = '\d' +haystack = '᠕' +matches = [] +unicode = true + +# regex-lite uses ASCII definitions for Perl character classes. +[[test]] +name = "perl-class-space" +regex = '\s' +haystack = "\u2000" +matches = [] +unicode = true + +# regex-lite uses ASCII definitions for Perl character classes. +[[test]] +name = "perl-class-word" +regex = '\w' +haystack = 'δ' +matches = [] +unicode = true + +# regex-lite uses the ASCII definition of word for word boundary assertions. +[[test]] +name = "word-boundary" +regex = '\b' +haystack = 'δ' +matches = [] +unicode = true + +# regex-lite uses the ASCII definition of word for negated word boundary +# assertions. But note that it should still not split codepoints! +[[test]] +name = "word-boundary-negated" +regex = '\B' +haystack = 'δ' +matches = [[0, 0], [2, 2]] +unicode = true + +# While we're here, the empty regex---which matches at every +# position---shouldn't split a codepoint either. +[[test]] +name = "empty-no-split-codepoint" +regex = '' +haystack = '💩' +matches = [[0, 0], [4, 4]] +unicode = true + +# A dot always matches a full codepoint. +[[test]] +name = "dot-always-matches-codepoint" +regex = '.' +haystack = '💩' +matches = [[0, 4]] +unicode = false + +# A negated character class also always matches a full codepoint. +[[test]] +name = "negated-class-always-matches-codepoint" +regex = '[^a]' +haystack = '💩' +matches = [[0, 4]] +unicode = false + +# regex-lite only supports ASCII-aware case insensitive matching. +[[test]] +name = "case-insensitive-is-ascii-only" +regex = 's' +haystack = 'ſ' +matches = [] +unicode = true +case-insensitive = true + +# Negated word boundaries shouldn't split a codepoint, but they will match +# between invalid UTF-8. +# +# This test is only valid for a 'bytes' API, but that doesn't (yet) exist in +# regex-lite. This can't happen in the main API because &str can't contain +# invalid UTF-8. +# [[test]] +# name = "word-boundary-invalid-utf8" +# regex = '\B' +# haystack = '\xFF\xFF\xFF\xFF' +# unescape = true +# matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] +# unicode = true +# utf8 = false diff --git a/third_party/crates/regex-1.13.1/testdata/regression.toml b/third_party/crates/regex-1.13.1/testdata/regression.toml new file mode 100644 index 0000000..9956dd7 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/regression.toml @@ -0,0 +1,985 @@ +# See: https://github.com/rust-lang/regex/issues/48 +[[test]] +name = "invalid-regex-no-crash-100" +regex = '(*)' +haystack = "" +matches = [] +compiles = false + +# See: https://github.com/rust-lang/regex/issues/48 +[[test]] +name = "invalid-regex-no-crash-200" +regex = '(?:?)' +haystack = "" +matches = [] +compiles = false + +# See: https://github.com/rust-lang/regex/issues/48 +[[test]] +name = "invalid-regex-no-crash-300" +regex = '(?)' +haystack = "" +matches = [] +compiles = false + +# See: https://github.com/rust-lang/regex/issues/48 +[[test]] +name = "invalid-regex-no-crash-400" +regex = '*' +haystack = "" +matches = [] +compiles = false + +# See: https://github.com/rust-lang/regex/issues/75 +[[test]] +name = "unsorted-binary-search-100" +regex = '(?i-u)[a_]+' +haystack = "A_" +matches = [[0, 2]] + +# See: https://github.com/rust-lang/regex/issues/75 +[[test]] +name = "unsorted-binary-search-200" +regex = '(?i-u)[A_]+' +haystack = "a_" +matches = [[0, 2]] + +# See: https://github.com/rust-lang/regex/issues/76 +[[test]] +name = "unicode-case-lower-nocase-flag" +regex = '(?i)\p{Ll}+' +haystack = "ΛΘΓΔα" +matches = [[0, 10]] + +# See: https://github.com/rust-lang/regex/issues/99 +[[test]] +name = "negated-char-class-100" +regex = '(?i)[^x]' +haystack = "x" +matches = [] + +# See: https://github.com/rust-lang/regex/issues/99 +[[test]] +name = "negated-char-class-200" +regex = '(?i)[^x]' +haystack = "X" +matches = [] + +# See: https://github.com/rust-lang/regex/issues/101 +[[test]] +name = "ascii-word-underscore" +regex = '[[:word:]]' +haystack = "_" +matches = [[0, 1]] + +# See: https://github.com/rust-lang/regex/issues/129 +[[test]] +name = "captures-repeat" +regex = '([a-f]){2}(?P<foo>[x-z])' +haystack = "abx" +matches = [ + [[0, 3], [1, 2], [2, 3]], +] + +# See: https://github.com/rust-lang/regex/issues/153 +[[test]] +name = "alt-in-alt-100" +regex = 'ab?|$' +haystack = "az" +matches = [[0, 1], [2, 2]] + +# See: https://github.com/rust-lang/regex/issues/153 +[[test]] +name = "alt-in-alt-200" +regex = '^(?:.*?)(?:\n|\r\n?|$)' +haystack = "ab\rcd" +matches = [[0, 3]] + +# See: https://github.com/rust-lang/regex/issues/169 +[[test]] +name = "leftmost-first-prefix" +regex = 'z*azb' +haystack = "azb" +matches = [[0, 3]] + +# See: https://github.com/rust-lang/regex/issues/191 +[[test]] +name = "many-alternates" +regex = '1|2|3|4|5|6|7|8|9|10|int' +haystack = "int" +matches = [[0, 3]] + +# See: https://github.com/rust-lang/regex/issues/204 +[[test]] +name = "word-boundary-alone-100" +regex = '\b' +haystack = "Should this (work?)" +matches = [[0, 0], [6, 6], [7, 7], [11, 11], [13, 13], [17, 17]] + +# See: https://github.com/rust-lang/regex/issues/204 +[[test]] +name = "word-boundary-alone-200" +regex = '\b' +haystack = "a b c" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] + +# See: https://github.com/rust-lang/regex/issues/264 +[[test]] +name = "word-boundary-ascii-no-capture" +regex = '\B' +haystack = "\U00028F3E" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] +unicode = false +utf8 = false + +# See: https://github.com/rust-lang/regex/issues/264 +[[test]] +name = "word-boundary-ascii-capture" +regex = '(?:\B)' +haystack = "\U00028F3E" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] +unicode = false +utf8 = false + +# See: https://github.com/rust-lang/regex/issues/268 +[[test]] +name = "partial-anchor" +regex = '^a|b' +haystack = "ba" +matches = [[0, 1]] + +# See: https://github.com/rust-lang/regex/issues/271 +[[test]] +name = "endl-or-word-boundary" +regex = '(?m:$)|(?-u:\b)' +haystack = "\U0006084E" +matches = [[4, 4]] + +# See: https://github.com/rust-lang/regex/issues/271 +[[test]] +name = "zero-or-end" +regex = '(?i-u:\x00)|$' +haystack = "\U000E682F" +matches = [[4, 4]] + +# See: https://github.com/rust-lang/regex/issues/271 +[[test]] +name = "y-or-endl" +regex = '(?i-u:y)|(?m:$)' +haystack = "\U000B4331" +matches = [[4, 4]] + +# See: https://github.com/rust-lang/regex/issues/271 +[[test]] +name = "word-boundary-start-x" +regex = '(?u:\b)^(?-u:X)' +haystack = "X" +matches = [[0, 1]] + +# See: https://github.com/rust-lang/regex/issues/271 +[[test]] +name = "word-boundary-ascii-start-x" +regex = '(?-u:\b)^(?-u:X)' +haystack = "X" +matches = [[0, 1]] + +# See: https://github.com/rust-lang/regex/issues/271 +[[test]] +name = "end-not-word-boundary" +regex = '$\B' +haystack = "\U0005C124\U000B576C" +matches = [[8, 8]] +unicode = false +utf8 = false + +# See: https://github.com/rust-lang/regex/issues/280 +[[test]] +name = "partial-anchor-alternate-begin" +regex = '^a|z' +haystack = "yyyyya" +matches = [] + +# See: https://github.com/rust-lang/regex/issues/280 +[[test]] +name = "partial-anchor-alternate-end" +regex = 'a$|z' +haystack = "ayyyyy" +matches = [] + +# See: https://github.com/rust-lang/regex/issues/289 +[[test]] +name = "lits-unambiguous-100" +regex = '(?:ABC|CDA|BC)X' +haystack = "CDAX" +matches = [[0, 4]] + +# See: https://github.com/rust-lang/regex/issues/291 +[[test]] +name = "lits-unambiguous-200" +regex = '((IMG|CAM|MG|MB2)_|(DSCN|CIMG))(?P<n>[0-9]+)$' +haystack = "CIMG2341" +matches = [ + [[0, 8], [0, 4], [], [0, 4], [4, 8]], +] + +# See: https://github.com/rust-lang/regex/issues/303 +# +# 2022-09-19: This has now been "properly" fixed in that empty character +# classes are fully supported as something that can never match. This test +# used to be marked as 'compiles = false', but now it works. +[[test]] +name = "negated-full-byte-range" +regex = '[^\x00-\xFF]' +haystack = "" +matches = [] +compiles = true +unicode = false +utf8 = false + +# See: https://github.com/rust-lang/regex/issues/321 +[[test]] +name = "strange-anchor-non-complete-prefix" +regex = 'a^{2}' +haystack = "" +matches = [] + +# See: https://github.com/rust-lang/regex/issues/321 +[[test]] +name = "strange-anchor-non-complete-suffix" +regex = '${2}a' +haystack = "" +matches = [] + +# See: https://github.com/rust-lang/regex/issues/334 +# See: https://github.com/rust-lang/regex/issues/557 +[[test]] +name = "captures-after-dfa-premature-end-100" +regex = 'a(b*(X|$))?' +haystack = "abcbX" +matches = [ + [[0, 1], [], []], +] + +# See: https://github.com/rust-lang/regex/issues/334 +# See: https://github.com/rust-lang/regex/issues/557 +[[test]] +name = "captures-after-dfa-premature-end-200" +regex = 'a(bc*(X|$))?' +haystack = "abcbX" +matches = [ + [[0, 1], [], []], +] + +# See: https://github.com/rust-lang/regex/issues/334 +# See: https://github.com/rust-lang/regex/issues/557 +[[test]] +name = "captures-after-dfa-premature-end-300" +regex = '(aa$)?' +haystack = "aaz" +matches = [ + [[0, 0], []], + [[1, 1], []], + [[2, 2], []], + [[3, 3], []], +] + +# Plucked from "Why aren’t regular expressions a lingua franca? an empirical +# study on the re-use and portability of regular expressions", The ACM Joint +# European Software Engineering Conference and Symposium on the Foundations of +# Software Engineering (ESEC/FSE), 2019. +# +# Link: https://dl.acm.org/doi/pdf/10.1145/3338906.3338909 +[[test]] +name = "captures-after-dfa-premature-end-400" +regex = '(a)\d*\.?\d+\b' +haystack = "a0.0c" +matches = [ + [[0, 2], [0, 1]], +] + +# See: https://github.com/rust-lang/regex/issues/437 +[[test]] +name = "literal-panic" +regex = 'typename type\-parameter\-[0-9]+\-[0-9]+::.+' +haystack = "test" +matches = [] + +# See: https://github.com/rust-lang/regex/issues/527 +[[test]] +name = "empty-flag-expr" +regex = '(?:(?:(?x)))' +haystack = "" +matches = [[0, 0]] + +# See: https://github.com/rust-lang/regex/issues/533 +#[[tests]] +#name = "blank-matches-nothing-between-space-and-tab" +#regex = '[[:blank:]]' +#input = '\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F' +#match = false +#unescape = true + +# See: https://github.com/rust-lang/regex/issues/533 +#[[tests]] +#name = "blank-matches-nothing-between-space-and-tab-inverted" +#regex = '^[[:^blank:]]+$' +#input = '\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F' +#match = true +#unescape = true + +# See: https://github.com/rust-lang/regex/issues/555 +[[test]] +name = "invalid-repetition" +regex = '(?m){1,1}' +haystack = "" +matches = [] +compiles = false + +# See: https://github.com/rust-lang/regex/issues/640 +[[test]] +name = "flags-are-unset" +regex = '(?:(?i)foo)|Bar' +haystack = "foo Foo bar Bar" +matches = [[0, 3], [4, 7], [12, 15]] + +# Note that 'Ј' is not 'j', but cyrillic Je +# https://en.wikipedia.org/wiki/Je_(Cyrillic) +# +# See: https://github.com/rust-lang/regex/issues/659 +[[test]] +name = "empty-group-with-unicode" +regex = '(?:)Ј01' +haystack = 'zЈ01' +matches = [[1, 5]] + +# See: https://github.com/rust-lang/regex/issues/579 +[[test]] +name = "word-boundary-weird" +regex = '\b..\b' +haystack = "I have 12, he has 2!" +matches = [[0, 2], [7, 9], [9, 11], [11, 13], [17, 19]] + +# See: https://github.com/rust-lang/regex/issues/579 +[[test]] +name = "word-boundary-weird-ascii" +regex = '\b..\b' +haystack = "I have 12, he has 2!" +matches = [[0, 2], [7, 9], [9, 11], [11, 13], [17, 19]] +unicode = false +utf8 = false + +# See: https://github.com/rust-lang/regex/issues/579 +[[test]] +name = "word-boundary-weird-minimal-ascii" +regex = '\b..\b' +haystack = "az,,b" +matches = [[0, 2], [2, 4]] +unicode = false +utf8 = false + +# See: https://github.com/BurntSushi/ripgrep/issues/1203 +[[test]] +name = "reverse-suffix-100" +regex = '[0-4][0-4][0-4]000' +haystack = "153.230000" +matches = [[4, 10]] + +# See: https://github.com/BurntSushi/ripgrep/issues/1203 +[[test]] +name = "reverse-suffix-200" +regex = '[0-9][0-9][0-9]000' +haystack = "153.230000\n" +matches = [[4, 10]] + +# This is a tricky case for the reverse suffix optimization, because it +# finds the 'foobar' match but the reverse scan must fail to find a match by +# correctly dealing with the word boundary following the 'foobar' literal when +# computing the start state. +# +# This test exists because I tried to break the following assumption that +# is currently in the code: that if a suffix is found and the reverse scan +# succeeds, then it's guaranteed that there is an overall match. Namely, the +# 'is_match' routine does *not* do another forward scan in this case because of +# this assumption. +[[test]] +name = "reverse-suffix-300" +regex = '\w+foobar\b' +haystack = "xyzfoobarZ" +matches = [] +unicode = false +utf8 = false + +# See: https://github.com/rust-lang/regex/issues/1354 +[[test]] +name = 'reverse-suffix-start-of-match-failure-010' +regex = '.abb|b' +haystack = 'zabb' +matches = [[0, 4]] + +# See: https://github.com/rust-lang/regex/issues/1354 +[[test]] +name = 'reverse-suffix-start-of-match-failure-011' +regex = '.bb|b' +haystack = 'zabb' +matches = [[1, 4]] + +# See: https://github.com/rust-lang/regex/issues/1354 +# See: https://github.com/rust-lang/regex/pull/1355 +[[test]] +name = 'reverse-suffix-start-of-match-failure-012' +regex = '.abb|b' +haystack = 'zzzabbz' +matches = [[2, 6]] + +# Like above, but use something other than a fixed `.`. Just as something more +# robust to potential future optimizations that could hide this bug. +# +# See: https://github.com/rust-lang/regex/issues/1354 +[[test]] +name = 'reverse-suffix-start-of-match-failure-020' +regex = '\s+\w+bb|b' +haystack = " zabb" +matches = [[0, 5]] + +# A trailing class can consume the entire suffix literal, but the variable +# length prefix before it can still make the first candidate non-leftmost. +[[test]] +name = 'reverse-suffix-start-of-match-failure-030' +regex = '(?:[a-wyz]{3}|[a-wyz]).b' +haystack = 'xaaabb' +matches = [[1, 6]] + +# A component immediately before the suffix can be disjoint from the suffix +# while an earlier candidate still occurs in the prefix before it. +[[test]] +name = 'reverse-suffix-start-of-match-failure-040' +regex = '(?:[ac-z]{2}b[ac-z])?[ac-z]b' +haystack = '!aabaab' +matches = [[1, 7]] + +# An unbounded trailing class can absorb the suffix without making the first +# literal candidate correspond to the leftmost match. +[[test]] +name = 'reverse-suffix-start-of-match-failure-050' +regex = '(?:[a-z]cb|c)[a-z]*b' +haystack = '!acbzb' +matches = [[1, 6]] + +# Rejecting the reverse suffix strategy falls back to Core. Its automatic +# prefix prefilter must not skip the leftmost match. +[[test]] +name = 'reverse-suffix-rejected-fallback-prefilter-010' +regex = '(?:ab|c)*c' +haystack = 'cabac' +matches = [[0, 1]] +match-limit = 1 + +# A required character-class separator lets the reverse suffix scan reject an +# earlier literal occurrence inside the prefix without rescanning the haystack. +[[test]] +name = 'reverse-suffix-required-class-separator-010' +regex = '\w+\s+Holmes' +haystack = 'abcHolmes xyz Holmes' +matches = [[10, 20]] + +# Checking only the component immediately before the separator is not enough. +# The earlier prefix can contain another complete separator alignment. +[[test]] +name = 'reverse-suffix-required-class-separator-020' +regex = '(?:.abc)?a+b+c' +haystack = 'xabcabc' +matches = [[0, 7]] + +# See: https://github.com/rust-lang/regex/issues/1354 +[[test]] +name = 'reverse-suffix-all-no-early-matches-010' +regex = '.abb|b' +haystack = 'xxfoobarbar' +match-kind = 'all' +matches = [[8, 9]] + +# See: https://github.com/BurntSushi/ripgrep/issues/1247 +[[test]] +name = "stops" +regex = '\bs(?:[ab])' +haystack = 's\xE4' +matches = [] +unescape = true +utf8 = false + +# See: https://github.com/BurntSushi/ripgrep/issues/1247 +[[test]] +name = "stops-ascii" +regex = '(?-u:\b)s(?:[ab])' +haystack = 's\xE4' +matches = [] +unescape = true +utf8 = false + +# See: https://github.com/rust-lang/regex/issues/850 +[[test]] +name = "adjacent-line-boundary-100" +regex = '(?m)^(?:[^ ]+?)$' +haystack = "line1\nline2" +matches = [[0, 5], [6, 11]] + +# Continued. +[[test]] +name = "adjacent-line-boundary-200" +regex = '(?m)^(?:[^ ]+?)$' +haystack = "A\nB" +matches = [[0, 1], [2, 3]] + +# There is no issue for this bug. +[[test]] +name = "anchored-prefix-100" +regex = '^a[[:^space:]]' +haystack = "a " +matches = [] + +# There is no issue for this bug. +[[test]] +name = "anchored-prefix-200" +regex = '^a[[:^space:]]' +haystack = "foo boo a" +matches = [] + +# There is no issue for this bug. +[[test]] +name = "anchored-prefix-300" +regex = '^-[a-z]' +haystack = "r-f" +matches = [] + +# Tests that a possible Aho-Corasick optimization works correctly. It only +# kicks in when we have a lot of literals. By "works correctly," we mean that +# leftmost-first match semantics are properly respected. That is, samwise +# should match, not sam. +# +# There is no issue for this bug. +[[test]] +name = "aho-corasick-100" +regex = 'samwise|sam|a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z|A|B|C|D|E|F|G|H|I|J|K|L|M|N|O|P|Q|R|S|T|U|V|W|X|Y|Z' +haystack = "samwise" +matches = [[0, 7]] + +# See: https://github.com/rust-lang/regex/issues/921 +[[test]] +name = "interior-anchor-capture" +regex = '(a$)b$' +haystack = 'ab' +matches = [] + +# I found this bug in the course of adding some of the regexes that Ruff uses +# to rebar. It turns out that the lazy DFA was finding a match that was being +# rejected by the one-pass DFA. Yikes. I then minimized the regex and haystack. +# +# Source: https://github.com/charliermarsh/ruff/blob/a919041ddaa64cdf6f216f90dd0480dab69fd3ba/crates/ruff/src/rules/pycodestyle/rules/whitespace_around_keywords.rs#L52 +[[test]] +name = "ruff-whitespace-around-keywords" +regex = '^(a|ab)$' +haystack = "ab" +anchored = true +unicode = false +utf8 = true +matches = [[[0, 2], [0, 2]]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-0" +regex = '(?:(?-u:\b)|(?u:h))+' +haystack = "h" +unicode = true +utf8 = false +matches = [[0, 0], [1, 1]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-1" +regex = '(?u:\B)' +haystack = "鋸" +unicode = true +utf8 = false +matches = [] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-2" +regex = '(?:(?u:\b)|(?s-u:.))+' +haystack = "oB" +unicode = true +utf8 = false +matches = [[0, 0], [1, 2]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-3" +regex = '(?:(?-u:\B)|(?su:.))+' +haystack = "\U000FEF80" +unicode = true +utf8 = false +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-3-utf8" +regex = '(?:(?-u:\B)|(?su:.))+' +haystack = "\U000FEF80" +unicode = true +utf8 = true +matches = [[0, 0], [4, 4]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-4" +regex = '(?m:$)(?m:^)(?su:.)' +haystack = "\n‣" +unicode = true +utf8 = false +matches = [[0, 1]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-5" +regex = '(?m:$)^(?m:^)' +haystack = "\n" +unicode = true +utf8 = false +matches = [[0, 0]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-6" +regex = '(?P<kp>(?iu:do)(?m:$))*' +haystack = "dodo" +unicode = true +utf8 = false +matches = [ + [[0, 0], []], + [[1, 1], []], + [[2, 4], [2, 4]], +] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-7" +regex = '(?u:\B)' +haystack = "䡁" +unicode = true +utf8 = false +matches = [] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-8" +regex = '(?:(?-u:\b)|(?u:[\u{0}-W]))+' +haystack = "0" +unicode = true +utf8 = false +matches = [[0, 0], [1, 1]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-9" +regex = '((?m:$)(?-u:\B)(?s-u:.)(?-u:\B)$)' +haystack = "\n\n" +unicode = true +utf8 = false +matches = [ + [[1, 2], [1, 2]], +] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-10" +regex = '(?m:$)(?m:$)^(?su:.)' +haystack = "\n\u0081¨\u200a" +unicode = true +utf8 = false +matches = [[0, 1]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-11" +regex = '(?-u:\B)(?m:^)' +haystack = "0\n" +unicode = true +utf8 = false +matches = [[2, 2]] + +# From: https://github.com/rust-lang/regex/issues/429 +[[test]] +name = "i429-12" +regex = '(?:(?u:\b)|(?-u:.))+' +haystack = "0" +unicode = true +utf8 = false +matches = [[0, 0], [1, 1]] + +# From: https://github.com/rust-lang/regex/issues/969 +[[test]] +name = "i969" +regex = 'c.*d\z' +haystack = "ababcd" +bounds = [4, 6] +search-kind = "earliest" +matches = [[4, 6]] + +# I found this during the regex-automata migration. This is the fowler basic +# 154 test, but without anchored = true and without a match limit. +# +# This test caught a subtle bug in the hybrid reverse DFA search, where it +# would skip over the termination condition if it entered a start state. This +# was a double bug. Firstly, the reverse DFA shouldn't have had start states +# specialized in the first place, and thus it shouldn't have possible to detect +# that the DFA had entered a start state. The second bug was that the start +# state handling was incorrect by jumping over the termination condition. +[[test]] +name = "fowler-basic154-unanchored" +regex = '''a([bc]*)c*''' +haystack = '''abc''' +matches = [[[0, 3], [1, 3]]] + +# From: https://github.com/rust-lang/regex/issues/981 +# +# This was never really a problem in the new architecture because the +# regex-automata engines are far more principled about how they deal with +# look-around. (This was one of the many reasons I wanted to re-work the +# original regex crate engines.) +[[test]] +name = "word-boundary-interact-poorly-with-literal-optimizations" +regex = '(?i:(?:\b|_)win(?:32|64|dows)?(?:\b|_))' +haystack = 'ubi-Darwin-x86_64.tar.gz' +matches = [] + +# This was found during fuzz testing of regex. It provoked a panic in the meta +# engine as a result of the reverse suffix optimization. Namely, it hit a case +# where a suffix match was found, a corresponding reverse match was found, but +# the forward search turned up no match. The forward search should always match +# if the suffix and reverse search match. +# +# This in turn uncovered an inconsistency between the PikeVM and the DFA (lazy +# and fully compiled) engines. It was caused by a mishandling of the collection +# of NFA state IDs in the generic determinization code (which is why both types +# of DFA were impacted). Namely, when a fail state was encountered (that's the +# `[^\s\S]` in the pattern below), then it would just stop collecting states. +# But that's not correct since a later state could lead to a match. +[[test]] +name = "impossible-branch" +regex = '.*[^\s\S]A|B' +haystack = "B" +matches = [[0, 1]] + +# This was found during fuzz testing in regex-lite. The regex crate never +# suffered from this bug, but it causes regex-lite to incorrectly compile +# captures. +[[test]] +name = "captures-wrong-order" +regex = '(a){0}(a)' +haystack = 'a' +matches = [[[0, 1], [], [0, 1]]] + +# This tests a bug in how quit states are handled in the DFA. At some point +# during development, the DFAs were tweaked slightly such that if they hit +# a quit state (which means, they hit a byte that the caller configured should +# stop the search), then it might not return an error necessarily. Namely, if a +# match had already been found, then it would be returned instead of an error. +# +# But this is actually wrong! Why? Because even though a match had been found, +# it wouldn't be fully correct to return it once a quit state has been seen +# because you can't determine whether the match offset returned is the correct +# greedy/leftmost-first match. Since you can't complete the search as requested +# by the caller, the DFA should just stop and return an error. +# +# Interestingly, this does seem to produce an unavoidable difference between +# 'try_is_match().unwrap()' and 'try_find().unwrap().is_some()' for the DFAs. +# The former will stop immediately once a match is known to occur and return +# 'Ok(true)', where as the latter could find the match but quit with an +# 'Err(..)' first. +# +# Thankfully, I believe this inconsistency between 'is_match()' and 'find()' +# cannot be observed in the higher level meta regex API because it specifically +# will try another engine that won't fail in the case of a DFA failing. +# +# This regression happened in the regex crate rewrite, but before anything got +# released. +[[test]] +name = "negated-unicode-word-boundary-dfa-fail" +regex = '\B.*' +haystack = "!\u02D7" +matches = [[0, 3]] + +# This failure was found in the *old* regex crate (prior to regex 1.9), but +# I didn't investigate why. My best guess is that it's a literal optimization +# bug. It didn't occur in the rewrite. +[[test]] +name = "missed-match" +regex = 'e..+e.ee>' +haystack = 'Zeee.eZZZZZZZZeee>eeeeeee>' +matches = [[1, 26]] + +# This test came from the 'ignore' crate and tripped a bug in how accelerated +# DFA states were handled in an overlapping search. +[[test]] +name = "regex-to-glob" +regex = ['(?-u)^path1/[^/]*$'] +haystack = "path1/foo" +matches = [[0, 9]] +utf8 = false +match-kind = "all" +search-kind = "overlapping" + +# See: https://github.com/rust-lang/regex/issues/1060 +[[test]] +name = "reverse-inner-plus-shorter-than-expected" +regex = '(?:(\d+)[:.])?(\d{1,2})[:.](\d{2})' +haystack = '102:12:39' +matches = [[[0, 9], [0, 3], [4, 6], [7, 9]]] + +# Like reverse-inner-plus-shorter-than-expected, but using a far simpler regex +# to demonstrate the extent of the rot. Sigh. +# +# See: https://github.com/rust-lang/regex/issues/1060 +[[test]] +name = "reverse-inner-short" +regex = '(?:([0-9][0-9][0-9]):)?([0-9][0-9]):([0-9][0-9])' +haystack = '102:12:39' +matches = [[[0, 9], [0, 3], [4, 6], [7, 9]]] + + +# See: https://github.com/rust-lang/regex/issues/1354 +[[test]] +name = 'reverse-inner-start-of-match-failure-010' +regex = '(?:..acbb|b)a(?:c|d)' +haystack = 'xzbacbbac' +matches = [[1, 9]] + +# Like above, but use something other than a fixed `.`. Just as something more +# robust to potential future optimizations that could hide this bug. +# +# See: https://github.com/rust-lang/regex/issues/1354 +[[test]] +name = 'reverse-inner-start-of-match-failure-011' +regex = '(?:\s+.\w+acbb|b)a(?:c|d)' +haystack = ' xzbacbbac' +matches = [[0, 10]] + +# See: https://github.com/rust-lang/regex/issues/1354 +[[test]] +name = 'reverse-inner-later-literal-010' +regex = '(?:..acbb|b)a(?:c|d)' +haystack = 'xzbacbbac' +matches = [[1, 9]] + +# A literal candidate can cross from the prefix into a different extracted +# literal even when the prefix cannot contain either complete literal. +[[test]] +name = 'reverse-inner-start-of-match-failure-020' +regex = '(?:[ac-z]{3}|[ac-z])(?:aba|baa)' +haystack = '!aaabaa' +matches = [[1, 7]] + +# Rejecting the reverse inner strategy falls back to Core. Its automatic +# prefix prefilter must not skip the leftmost match. +[[test]] +name = 'reverse-inner-rejected-fallback-prefilter-010' +regex = '(?:ab|c)*c[de]*' +haystack = 'cabac' +matches = [[0, 1]] +match-limit = 1 + +# A required class separator prevents the first inner literal occurrence from +# confirming inside the prefix. +[[test]] +name = 'reverse-inner-required-class-separator-010' +regex = '\w+\s+Holmes\s+\w+' +haystack = 'abcHolmes xyz Holmes tail' +matches = [[10, 25]] + +# The entire prefix before the separator must be disjoint from it. Checking +# only the immediately preceding component would incorrectly return 1..7. +[[test]] +name = 'reverse-inner-required-class-separator-020' +regex = '(?:.abc)?a+b+c.*' +haystack = 'xabcabc' +matches = [[0, 7]] + +# A required class separator at the beginning of the prefix makes an inner +# literal occurrence inside the bounded repetition safe to reject. +[[test]] +name = 'reverse-inner-leading-class-separator-010' +regex = '\s[A-Za-z]{0,12}ing\s' +haystack = ' xingabcing ' +matches = [[0, 12]] + +# This regression test was found via the RegexSet APIs. It triggered a +# particular code path where a regex was compiled with 'All' match semantics +# (to support overlapping search), but got funneled down into a standard +# leftmost search when calling 'is_match'. This is fine on its own, but the +# leftmost search will use a prefilter and that's where this went awry. +# +# Namely, since 'All' semantics were used, the aho-corasick prefilter was +# incorrectly compiled with 'Standard' semantics. This was wrong because +# 'Standard' immediately attempts to report a match at every position, even if +# that would mean reporting a match past the leftmost match before reporting +# the leftmost match. This breaks the prefilter contract of never having false +# negatives and leads overall to the engine not finding a match. +# +# See: https://github.com/rust-lang/regex/issues/1070 +[[test]] +name = "prefilter-with-aho-corasick-standard-semantics" +regex = '(?m)^ *v [0-9]' +haystack = 'v 0' +matches = [ + { id = 0, spans = [[0, 3]] }, +] +match-kind = "all" +search-kind = "overlapping" +unicode = true +utf8 = true + +# This tests that the PikeVM and the meta regex agree on a particular regex. +# This test previously failed when the ad hoc engines inside the meta engine +# did not handle quit states correctly. Namely, the Unicode word boundary here +# combined with a non-ASCII codepoint provokes the quit state. The ad hoc +# engines were previously returning a match even after entering the quit state +# if a match had been previously detected, but this is incorrect. The reason +# is that if a quit state is found, then the search must give up *immediately* +# because it prevents the search from finding the "proper" leftmost-first +# match. If it instead returns a match that has been found, it risks reporting +# an improper match, as it did in this case. +# +# See: https://github.com/rust-lang/regex/issues/1046 +[[test]] +name = "non-prefix-literal-quit-state" +regex = '.+\b\n' +haystack = "β77\n" +matches = [[0, 5]] + +# This is a regression test for some errant HIR interval set operations that +# were made in the regex-syntax 0.8.0 release and then reverted in 0.8.1. The +# issue here is that the HIR produced from the regex had out-of-order ranges. +# +# See: https://github.com/rust-lang/regex/issues/1103 +# Ref: https://github.com/rust-lang/regex/pull/1051 +# Ref: https://github.com/rust-lang/regex/pull/1102 +[[test]] +name = "hir-optimization-out-of-order-class" +regex = '^[[:alnum:]./-]+$' +haystack = "a-b" +matches = [[0, 3]] + +# This is a regression test for an improper reverse suffix optimization. This +# occurred when I "broadened" the applicability of the optimization to include +# multiple possible literal suffixes instead of only sticking to a non-empty +# longest common suffix. It turns out that, at least given how the reverse +# suffix optimization works, we need to stick to the longest common suffix for +# now. +# +# See: https://github.com/rust-lang/regex/issues/1110 +# See also: https://github.com/astral-sh/ruff/pull/7980 +[[test]] +name = 'improper-reverse-suffix-optimization' +regex = '(\\N\{[^}]+})|([{}])' +haystack = 'hiya \N{snowman} bye' +matches = [[[5, 16], [5, 16], []]] diff --git a/third_party/crates/regex-1.13.1/testdata/set.toml b/third_party/crates/regex-1.13.1/testdata/set.toml new file mode 100644 index 0000000..049e8a8 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/set.toml @@ -0,0 +1,641 @@ +# Basic multi-regex tests. + +[[test]] +name = "basic10" +regex = ["a", "a"] +haystack = "a" +matches = [ + { id = 0, span = [0, 1] }, + { id = 1, span = [0, 1] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic10-leftmost-first" +regex = ["a", "a"] +haystack = "a" +matches = [ + { id = 0, span = [0, 1] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "basic20" +regex = ["a", "a"] +haystack = "ba" +matches = [ + { id = 0, span = [1, 2] }, + { id = 1, span = [1, 2] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic30" +regex = ["a", "b"] +haystack = "a" +matches = [ + { id = 0, span = [0, 1] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic40" +regex = ["a", "b"] +haystack = "b" +matches = [ + { id = 1, span = [0, 1] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic50" +regex = ["a|b", "b|a"] +haystack = "b" +matches = [ + { id = 0, span = [0, 1] }, + { id = 1, span = [0, 1] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic60" +regex = ["foo", "oo"] +haystack = "foo" +matches = [ + { id = 0, span = [0, 3] }, + { id = 1, span = [1, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic60-leftmost-first" +regex = ["foo", "oo"] +haystack = "foo" +matches = [ + { id = 0, span = [0, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "basic61" +regex = ["oo", "foo"] +haystack = "foo" +matches = [ + { id = 1, span = [0, 3] }, + { id = 0, span = [1, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic61-leftmost-first" +regex = ["oo", "foo"] +haystack = "foo" +matches = [ + { id = 1, span = [0, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "basic70" +regex = ["abcd", "bcd", "cd", "d"] +haystack = "abcd" +matches = [ + { id = 0, span = [0, 4] }, + { id = 1, span = [1, 4] }, + { id = 2, span = [2, 4] }, + { id = 3, span = [3, 4] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic71" +regex = ["bcd", "cd", "d", "abcd"] +haystack = "abcd" +matches = [ + { id = 3, span = [0, 4] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "basic80" +regex = ["^foo", "bar$"] +haystack = "foo" +matches = [ + { id = 0, span = [0, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic81" +regex = ["^foo", "bar$"] +haystack = "foo bar" +matches = [ + { id = 0, span = [0, 3] }, + { id = 1, span = [4, 7] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic82" +regex = ["^foo", "bar$"] +haystack = "bar" +matches = [ + { id = 1, span = [0, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic90" +regex = ["[a-z]+$", "foo"] +haystack = "01234 foo" +matches = [ + { id = 0, span = [8, 9] }, + { id = 0, span = [7, 9] }, + { id = 0, span = [6, 9] }, + { id = 1, span = [6, 9] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic91" +regex = ["[a-z]+$", "foo"] +haystack = "foo 01234" +matches = [ + { id = 1, span = [0, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic100" +regex = [".*?", "a"] +haystack = "zzza" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [0, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [1, 2] }, + { id = 0, span = [0, 2] }, + { id = 0, span = [3, 3] }, + { id = 0, span = [2, 3] }, + { id = 0, span = [1, 3] }, + { id = 0, span = [0, 3] }, + { id = 0, span = [4, 4] }, + { id = 0, span = [3, 4] }, + { id = 0, span = [2, 4] }, + { id = 0, span = [1, 4] }, + { id = 0, span = [0, 4] }, + { id = 1, span = [3, 4] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic101" +regex = [".*", "a"] +haystack = "zzza" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [0, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [1, 2] }, + { id = 0, span = [0, 2] }, + { id = 0, span = [3, 3] }, + { id = 0, span = [2, 3] }, + { id = 0, span = [1, 3] }, + { id = 0, span = [0, 3] }, + { id = 0, span = [4, 4] }, + { id = 0, span = [3, 4] }, + { id = 0, span = [2, 4] }, + { id = 0, span = [1, 4] }, + { id = 0, span = [0, 4] }, + { id = 1, span = [3, 4] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic102" +regex = [".*", "a"] +haystack = "zzz" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [0, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [1, 2] }, + { id = 0, span = [0, 2] }, + { id = 0, span = [3, 3] }, + { id = 0, span = [2, 3] }, + { id = 0, span = [1, 3] }, + { id = 0, span = [0, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic110" +regex = ['\ba\b'] +haystack = "hello a bye" +matches = [ + { id = 0, span = [6, 7] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic111" +regex = ['\ba\b', '\be\b'] +haystack = "hello a bye e" +matches = [ + { id = 0, span = [6, 7] }, + { id = 1, span = [12, 13] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic120" +regex = ["a"] +haystack = "a" +matches = [ + { id = 0, span = [0, 1] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic121" +regex = [".*a"] +haystack = "a" +matches = [ + { id = 0, span = [0, 1] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic122" +regex = [".*a", "β"] +haystack = "β" +matches = [ + { id = 1, span = [0, 2] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "basic130" +regex = ["ab", "b"] +haystack = "ba" +matches = [ + { id = 1, span = [0, 1] }, +] +match-kind = "all" +search-kind = "overlapping" + +# These test cases where one of the regexes matches the empty string. + +[[test]] +name = "empty10" +regex = ["", "a"] +haystack = "abc" +matches = [ + { id = 0, span = [0, 0] }, + { id = 1, span = [0, 1] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [3, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty10-leftmost-first" +regex = ["", "a"] +haystack = "abc" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [3, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "empty11" +regex = ["a", ""] +haystack = "abc" +matches = [ + { id = 1, span = [0, 0] }, + { id = 0, span = [0, 1] }, + { id = 1, span = [1, 1] }, + { id = 1, span = [2, 2] }, + { id = 1, span = [3, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty11-leftmost-first" +regex = ["a", ""] +haystack = "abc" +matches = [ + { id = 0, span = [0, 1] }, + { id = 1, span = [2, 2] }, + { id = 1, span = [3, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "empty20" +regex = ["", "b"] +haystack = "abc" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 1, span = [1, 2] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [3, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty20-leftmost-first" +regex = ["", "b"] +haystack = "abc" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [3, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "empty21" +regex = ["b", ""] +haystack = "abc" +matches = [ + { id = 1, span = [0, 0] }, + { id = 1, span = [1, 1] }, + { id = 0, span = [1, 2] }, + { id = 1, span = [2, 2] }, + { id = 1, span = [3, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty21-leftmost-first" +regex = ["b", ""] +haystack = "abc" +matches = [ + { id = 1, span = [0, 0] }, + { id = 0, span = [1, 2] }, + { id = 1, span = [3, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "empty22" +regex = ["(?:)", "b"] +haystack = "abc" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 1, span = [1, 2] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [3, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty23" +regex = ["b", "(?:)"] +haystack = "abc" +matches = [ + { id = 1, span = [0, 0] }, + { id = 1, span = [1, 1] }, + { id = 0, span = [1, 2] }, + { id = 1, span = [2, 2] }, + { id = 1, span = [3, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty30" +regex = ["", "z"] +haystack = "abc" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [3, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty30-leftmost-first" +regex = ["", "z"] +haystack = "abc" +matches = [ + { id = 0, span = [0, 0] }, + { id = 0, span = [1, 1] }, + { id = 0, span = [2, 2] }, + { id = 0, span = [3, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "empty31" +regex = ["z", ""] +haystack = "abc" +matches = [ + { id = 1, span = [0, 0] }, + { id = 1, span = [1, 1] }, + { id = 1, span = [2, 2] }, + { id = 1, span = [3, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty31-leftmost-first" +regex = ["z", ""] +haystack = "abc" +matches = [ + { id = 1, span = [0, 0] }, + { id = 1, span = [1, 1] }, + { id = 1, span = [2, 2] }, + { id = 1, span = [3, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +[[test]] +name = "empty40" +regex = ["c(?:)", "b"] +haystack = "abc" +matches = [ + { id = 1, span = [1, 2] }, + { id = 0, span = [2, 3] }, +] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "empty40-leftmost-first" +regex = ["c(?:)", "b"] +haystack = "abc" +matches = [ + { id = 1, span = [1, 2] }, + { id = 0, span = [2, 3] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" + +# These test cases where there are no matches. + +[[test]] +name = "nomatch10" +regex = ["a", "a"] +haystack = "b" +matches = [] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "nomatch20" +regex = ["^foo", "bar$"] +haystack = "bar foo" +matches = [] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "nomatch30" +regex = [] +haystack = "a" +matches = [] +match-kind = "all" +search-kind = "overlapping" + +[[test]] +name = "nomatch40" +regex = ["^rooted$", '\.log$'] +haystack = "notrooted" +matches = [] +match-kind = "all" +search-kind = "overlapping" + +# These test multi-regex searches with capture groups. +# +# NOTE: I wrote these tests in the course of developing a first class API for +# overlapping capturing group matches, but ultimately removed that API because +# the semantics for overlapping matches aren't totally clear. However, I've +# left the tests because I believe the semantics for these patterns are clear +# and because we can still test our "which patterns matched" APIs with them. + +[[test]] +name = "caps-010" +regex = ['^(\w+) (\w+)$', '^(\S+) (\S+)$'] +haystack = "Bruce Springsteen" +matches = [ + { id = 0, spans = [[0, 17], [0, 5], [6, 17]] }, + { id = 1, spans = [[0, 17], [0, 5], [6, 17]] }, +] +match-kind = "all" +search-kind = "overlapping" +unicode = false +utf8 = false + +[[test]] +name = "caps-020" +regex = ['^(\w+) (\w+)$', '^[A-Z](\S+) [A-Z](\S+)$'] +haystack = "Bruce Springsteen" +matches = [ + { id = 0, spans = [[0, 17], [0, 5], [6, 17]] }, + { id = 1, spans = [[0, 17], [1, 5], [7, 17]] }, +] +match-kind = "all" +search-kind = "overlapping" +unicode = false +utf8 = false + +[[test]] +name = "caps-030" +regex = ['^(\w+) (\w+)$', '^([A-Z])(\S+) ([A-Z])(\S+)$'] +haystack = "Bruce Springsteen" +matches = [ + { id = 0, spans = [[0, 17], [0, 5], [6, 17]] }, + { id = 1, spans = [[0, 17], [0, 1], [1, 5], [6, 7], [7, 17]] }, +] +match-kind = "all" +search-kind = "overlapping" +unicode = false +utf8 = false + +[[test]] +name = "caps-110" +regex = ['(\w+) (\w+)', '(\S+) (\S+)'] +haystack = "Bruce Springsteen" +matches = [ + { id = 0, spans = [[0, 17], [0, 5], [6, 17]] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" +unicode = false +utf8 = false + +[[test]] +name = "caps-120" +regex = ['(\w+) (\w+)', '(\S+) (\S+)'] +haystack = "&ruce $pringsteen" +matches = [ + { id = 1, spans = [[0, 17], [0, 5], [6, 17]] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" +unicode = false +utf8 = false + +[[test]] +name = "caps-121" +regex = ['(\w+) (\w+)', '(\S+) (\S+)'] +haystack = "&ruce $pringsteen Foo Bar" +matches = [ + { id = 1, spans = [[0, 17], [0, 5], [6, 17]] }, + { id = 0, spans = [[18, 25], [18, 21], [22, 25]] }, +] +match-kind = "leftmost-first" +search-kind = "leftmost" +unicode = false +utf8 = false diff --git a/third_party/crates/regex-1.13.1/testdata/substring.toml b/third_party/crates/regex-1.13.1/testdata/substring.toml new file mode 100644 index 0000000..69595ce --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/substring.toml @@ -0,0 +1,36 @@ +# These tests check that regex engines perform as expected when the search is +# instructed to only search a substring of a haystack instead of the entire +# haystack. This tends to exercise interesting edge cases that are otherwise +# difficult to provoke. (But not necessarily impossible. Regex search iterators +# for example, make use of the "search just a substring" APIs by changing the +# starting position of a search to the end position of the previous match.) + +[[test]] +name = "unicode-word-start" +regex = '\b[0-9]+\b' +haystack = "β123" +bounds = { start = 2, end = 5 } +matches = [] + +[[test]] +name = "unicode-word-end" +regex = '\b[0-9]+\b' +haystack = "123β" +bounds = { start = 0, end = 3 } +matches = [] + +[[test]] +name = "ascii-word-start" +regex = '\b[0-9]+\b' +haystack = "β123" +bounds = { start = 2, end = 5 } +matches = [[2, 5]] +unicode = false + +[[test]] +name = "ascii-word-end" +regex = '\b[0-9]+\b' +haystack = "123β" +bounds = { start = 0, end = 3 } +matches = [[0, 3]] +unicode = false diff --git a/third_party/crates/regex-1.13.1/testdata/unicode.toml b/third_party/crates/regex-1.13.1/testdata/unicode.toml new file mode 100644 index 0000000..f4ac76b --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/unicode.toml @@ -0,0 +1,517 @@ +# Basic Unicode literal support. +[[test]] +name = "literal1" +regex = '☃' +haystack = "☃" +matches = [[0, 3]] + +[[test]] +name = "literal2" +regex = '☃+' +haystack = "☃" +matches = [[0, 3]] + +[[test]] +name = "literal3" +regex = '☃+' +haystack = "☃" +matches = [[0, 3]] +case-insensitive = true + +[[test]] +name = "literal4" +regex = 'Δ' +haystack = "δ" +matches = [[0, 2]] +case-insensitive = true + +# Unicode word boundaries. +[[test]] +name = "wb-100" +regex = '\d\b' +haystack = "6δ" +matches = [] + +[[test]] +name = "wb-200" +regex = '\d\b' +haystack = "6 " +matches = [[0, 1]] + +[[test]] +name = "wb-300" +regex = '\d\B' +haystack = "6δ" +matches = [[0, 1]] + +[[test]] +name = "wb-400" +regex = '\d\B' +haystack = "6 " +matches = [] + +# Unicode character class support. +[[test]] +name = "class1" +regex = '[☃Ⅰ]+' +haystack = "☃" +matches = [[0, 3]] + +[[test]] +name = "class2" +regex = '\pN' +haystack = "Ⅰ" +matches = [[0, 3]] + +[[test]] +name = "class3" +regex = '\pN+' +haystack = "Ⅰ1Ⅱ2" +matches = [[0, 8]] + +[[test]] +name = "class4" +regex = '\PN+' +haystack = "abⅠ" +matches = [[0, 2]] + +[[test]] +name = "class5" +regex = '[\PN]+' +haystack = "abⅠ" +matches = [[0, 2]] + +[[test]] +name = "class6" +regex = '[^\PN]+' +haystack = "abⅠ" +matches = [[2, 5]] + +[[test]] +name = "class7" +regex = '\p{Lu}+' +haystack = "ΛΘΓΔα" +matches = [[0, 8]] + +[[test]] +name = "class8" +regex = '\p{Lu}+' +haystack = "ΛΘΓΔα" +matches = [[0, 10]] +case-insensitive = true + +[[test]] +name = "class9" +regex = '\pL+' +haystack = "ΛΘΓΔα" +matches = [[0, 10]] + +[[test]] +name = "class10" +regex = '\p{Ll}+' +haystack = "ΛΘΓΔα" +matches = [[8, 10]] + +# Unicode aware "Perl" character classes. +[[test]] +name = "perl1" +regex = '\w+' +haystack = "dδd" +matches = [[0, 4]] + +[[test]] +name = "perl2" +regex = '\w+' +haystack = "⥡" +matches = [] + +[[test]] +name = "perl3" +regex = '\W+' +haystack = "⥡" +matches = [[0, 3]] + +[[test]] +name = "perl4" +regex = '\d+' +haystack = "1२३9" +matches = [[0, 8]] + +[[test]] +name = "perl5" +regex = '\d+' +haystack = "Ⅱ" +matches = [] + +[[test]] +name = "perl6" +regex = '\D+' +haystack = "Ⅱ" +matches = [[0, 3]] + +[[test]] +name = "perl7" +regex = '\s+' +haystack = " " +matches = [[0, 3]] + +[[test]] +name = "perl8" +regex = '\s+' +haystack = "☃" +matches = [] + +[[test]] +name = "perl9" +regex = '\S+' +haystack = "☃" +matches = [[0, 3]] + +# Specific tests for Unicode general category classes. +[[test]] +name = "class-gencat1" +regex = '\p{Cased_Letter}' +haystack = "Ａ" +matches = [[0, 3]] + +[[test]] +name = "class-gencat2" +regex = '\p{Close_Punctuation}' +haystack = "❯" +matches = [[0, 3]] + +[[test]] +name = "class-gencat3" +regex = '\p{Connector_Punctuation}' +haystack = "⁀" +matches = [[0, 3]] + +[[test]] +name = "class-gencat4" +regex = '\p{Control}' +haystack = "\u009F" +matches = [[0, 2]] + +[[test]] +name = "class-gencat5" +regex = '\p{Currency_Symbol}' +haystack = "￡" +matches = [[0, 3]] + +[[test]] +name = "class-gencat6" +regex = '\p{Dash_Punctuation}' +haystack = "〰" +matches = [[0, 3]] + +[[test]] +name = "class-gencat7" +regex = '\p{Decimal_Number}' +haystack = "𑓙" +matches = [[0, 4]] + +[[test]] +name = "class-gencat8" +regex = '\p{Enclosing_Mark}' +haystack = "\uA672" +matches = [[0, 3]] + +[[test]] +name = "class-gencat9" +regex = '\p{Final_Punctuation}' +haystack = "⸡" +matches = [[0, 3]] + +[[test]] +name = "class-gencat10" +regex = '\p{Format}' +haystack = "\U000E007F" +matches = [[0, 4]] + +[[test]] +name = "class-gencat11" +regex = '\p{Initial_Punctuation}' +haystack = "⸜" +matches = [[0, 3]] + +[[test]] +name = "class-gencat12" +regex = '\p{Letter}' +haystack = "Έ" +matches = [[0, 2]] + +[[test]] +name = "class-gencat13" +regex = '\p{Letter_Number}' +haystack = "ↂ" +matches = [[0, 3]] + +[[test]] +name = "class-gencat14" +regex = '\p{Line_Separator}' +haystack = "\u2028" +matches = [[0, 3]] + +[[test]] +name = "class-gencat15" +regex = '\p{Lowercase_Letter}' +haystack = "ϛ" +matches = [[0, 2]] + +[[test]] +name = "class-gencat16" +regex = '\p{Mark}' +haystack = "\U000E01EF" +matches = [[0, 4]] + +[[test]] +name = "class-gencat17" +regex = '\p{Math}' +haystack = "⋿" +matches = [[0, 3]] + +[[test]] +name = "class-gencat18" +regex = '\p{Modifier_Letter}' +haystack = "𖭃" +matches = [[0, 4]] + +[[test]] +name = "class-gencat19" +regex = '\p{Modifier_Symbol}' +haystack = "🏿" +matches = [[0, 4]] + +[[test]] +name = "class-gencat20" +regex = '\p{Nonspacing_Mark}' +haystack = "\U0001E94A" +matches = [[0, 4]] + +[[test]] +name = "class-gencat21" +regex = '\p{Number}' +haystack = "⓿" +matches = [[0, 3]] + +[[test]] +name = "class-gencat22" +regex = '\p{Open_Punctuation}' +haystack = "｟" +matches = [[0, 3]] + +[[test]] +name = "class-gencat23" +regex = '\p{Other}' +haystack = "\u0BC9" +matches = [[0, 3]] + +[[test]] +name = "class-gencat24" +regex = '\p{Other_Letter}' +haystack = "ꓷ" +matches = [[0, 3]] + +[[test]] +name = "class-gencat25" +regex = '\p{Other_Number}' +haystack = "㉏" +matches = [[0, 3]] + +[[test]] +name = "class-gencat26" +regex = '\p{Other_Punctuation}' +haystack = "𞥞" +matches = [[0, 4]] + +[[test]] +name = "class-gencat27" +regex = '\p{Other_Symbol}' +haystack = "⅌" +matches = [[0, 3]] + +[[test]] +name = "class-gencat28" +regex = '\p{Paragraph_Separator}' +haystack = "\u2029" +matches = [[0, 3]] + +[[test]] +name = "class-gencat29" +regex = '\p{Private_Use}' +haystack = "\U0010FFFD" +matches = [[0, 4]] + +[[test]] +name = "class-gencat30" +regex = '\p{Punctuation}' +haystack = "𑁍" +matches = [[0, 4]] + +[[test]] +name = "class-gencat31" +regex = '\p{Separator}' +haystack = "\u3000" +matches = [[0, 3]] + +[[test]] +name = "class-gencat32" +regex = '\p{Space_Separator}' +haystack = "\u205F" +matches = [[0, 3]] + +[[test]] +name = "class-gencat33" +regex = '\p{Spacing_Mark}' +haystack = "\U00016F7E" +matches = [[0, 4]] + +[[test]] +name = "class-gencat34" +regex = '\p{Symbol}' +haystack = "⯈" +matches = [[0, 3]] + +[[test]] +name = "class-gencat35" +regex = '\p{Titlecase_Letter}' +haystack = "ῼ" +matches = [[0, 3]] + +[[test]] +name = "class-gencat36" +regex = '\p{Unassigned}' +haystack = "\U0010FFFF" +matches = [[0, 4]] + +[[test]] +name = "class-gencat37" +regex = '\p{Uppercase_Letter}' +haystack = "Ꝋ" +matches = [[0, 3]] + + +# Tests for Unicode emoji properties. +[[test]] +name = "class-emoji1" +regex = '\p{Emoji}' +haystack = "\u23E9" +matches = [[0, 3]] + +[[test]] +name = "class-emoji2" +regex = '\p{emoji}' +haystack = "\U0001F21A" +matches = [[0, 4]] + +[[test]] +name = "class-emoji3" +regex = '\p{extendedpictographic}' +haystack = "\U0001FA6E" +matches = [[0, 4]] + +[[test]] +name = "class-emoji4" +regex = '\p{extendedpictographic}' +haystack = "\U0001FFFD" +matches = [[0, 4]] + + +# Tests for Unicode grapheme cluster properties. +[[test]] +name = "class-gcb1" +regex = '\p{grapheme_cluster_break=prepend}' +haystack = "\U00011D46" +matches = [[0, 4]] + +[[test]] +name = "class-gcb2" +regex = '\p{gcb=regional_indicator}' +haystack = "\U0001F1E6" +matches = [[0, 4]] + +[[test]] +name = "class-gcb3" +regex = '\p{gcb=ri}' +haystack = "\U0001F1E7" +matches = [[0, 4]] + +[[test]] +name = "class-gcb4" +regex = '\p{regionalindicator}' +haystack = "\U0001F1FF" +matches = [[0, 4]] + +[[test]] +name = "class-gcb5" +regex = '\p{gcb=lvt}' +haystack = "\uC989" +matches = [[0, 3]] + +[[test]] +name = "class-gcb6" +regex = '\p{gcb=zwj}' +haystack = "\u200D" +matches = [[0, 3]] + +# Tests for Unicode word boundary properties. +[[test]] +name = "class-word-break1" +regex = '\p{word_break=Hebrew_Letter}' +haystack = "\uFB46" +matches = [[0, 3]] + +[[test]] +name = "class-word-break2" +regex = '\p{wb=hebrewletter}' +haystack = "\uFB46" +matches = [[0, 3]] + +[[test]] +name = "class-word-break3" +regex = '\p{wb=ExtendNumLet}' +haystack = "\uFF3F" +matches = [[0, 3]] + +[[test]] +name = "class-word-break4" +regex = '\p{wb=WSegSpace}' +haystack = "\u3000" +matches = [[0, 3]] + +[[test]] +name = "class-word-break5" +regex = '\p{wb=numeric}' +haystack = "\U0001E950" +matches = [[0, 4]] + +# Tests for Unicode sentence boundary properties. +[[test]] +name = "class-sentence-break1" +regex = '\p{sentence_break=Lower}' +haystack = "\u0469" +matches = [[0, 2]] + +[[test]] +name = "class-sentence-break2" +regex = '\p{sb=lower}' +haystack = "\u0469" +matches = [[0, 2]] + +[[test]] +name = "class-sentence-break3" +regex = '\p{sb=Close}' +haystack = "\uFF60" +matches = [[0, 3]] + +[[test]] +name = "class-sentence-break4" +regex = '\p{sb=Close}' +haystack = "\U0001F677" +matches = [[0, 4]] + +[[test]] +name = "class-sentence-break5" +regex = '\p{sb=SContinue}' +haystack = "\uFF64" +matches = [[0, 3]] diff --git a/third_party/crates/regex-1.13.1/testdata/utf8.toml b/third_party/crates/regex-1.13.1/testdata/utf8.toml new file mode 100644 index 0000000..39e284b --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/utf8.toml @@ -0,0 +1,399 @@ +# These test the UTF-8 modes expose by regex-automata. Namely, when utf8 is +# true, then we promise that the haystack is valid UTF-8. (Otherwise behavior +# is unspecified.) This also corresponds to building the regex engine with the +# following two guarantees: +# +# 1) For any non-empty match reported, its span is guaranteed to correspond to +# valid UTF-8. +# 2) All empty or zero-width matches reported must never split a UTF-8 +# encoded codepoint. If the haystack has invalid UTF-8, then this results in +# unspecified behavior. +# +# The (2) is in particular what we focus our testing on since (1) is generally +# guaranteed by regex-syntax's AST-to-HIR translator and is well tested there. +# The thing with (2) is that it can't be described in the HIR, so the regex +# engines have to handle that case. Thus, we test it here. +# +# Note that it is possible to build a regex that has property (1) but not +# (2), and vice versa. This is done by building the HIR with 'utf8=true' but +# building the Thompson NFA with 'utf8=false'. We don't test that here because +# the harness doesn't expose a way to enable or disable UTF-8 mode with that +# granularity. Instead, those combinations are lightly tested via doc examples. +# That's not to say that (1) without (2) is uncommon. Indeed, ripgrep uses it +# because it cannot guarantee that its haystack is valid UTF-8. + +# This tests that an empty regex doesn't split a codepoint. +[[test]] +name = "empty-utf8yes" +regex = '' +haystack = '☃' +matches = [[0, 0], [3, 3]] +unicode = true +utf8 = true + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8yes-overlapping" +regex = '' +haystack = '☃' +matches = [[0, 0], [3, 3]] +unicode = true +utf8 = true +match-kind = "all" +search-kind = "overlapping" + +# This tests that an empty regex DOES split a codepoint when utf=false. +[[test]] +name = "empty-utf8no" +regex = '' +haystack = '☃' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] +unicode = true +utf8 = false + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8no-overlapping" +regex = '' +haystack = '☃' +matches = [[0, 0], [1, 1], [2, 2], [3, 3]] +unicode = true +utf8 = false +match-kind = "all" +search-kind = "overlapping" + +# This tests that an empty regex doesn't split a codepoint, even if we give +# it bounds entirely within the codepoint. +# +# This is one of the trickier cases and is what motivated the current UTF-8 +# mode design. In particular, at one point, this test failed the 'is_match' +# variant of the test but not 'find'. This is because the 'is_match' code path +# is specifically optimized for "was a match found" rather than "where is the +# match." In the former case, you don't really care about the empty-vs-non-empty +# matches, and thus, the codepoint splitting filtering logic wasn't getting +# applied. (In multiple ways across multiple regex engines.) In this way, you +# can wind up with a situation where 'is_match' says "yes," but 'find' says, +# "I didn't find anything." Which is... not great. +# +# I could have decided to say that providing boundaries that themselves split +# a codepoint would have unspecified behavior. But I couldn't quite convince +# myself that such boundaries were the only way to get an inconsistency between +# 'is_match' and 'find'. +# +# Note that I also tried to come up with a test like this that fails without +# using `bounds`. Specifically, a test where 'is_match' and 'find' disagree. +# But I couldn't do it, and I'm tempted to conclude it is impossible. The +# fundamental problem is that you need to simultaneously produce an empty match +# that splits a codepoint while *not* matching before or after the codepoint. +[[test]] +name = "empty-utf8yes-bounds" +regex = '' +haystack = '𝛃' +bounds = [1, 3] +matches = [] +unicode = true +utf8 = true + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8yes-bounds-overlapping" +regex = '' +haystack = '𝛃' +bounds = [1, 3] +matches = [] +unicode = true +utf8 = true +match-kind = "all" +search-kind = "overlapping" + +# This tests that an empty regex splits a codepoint when the bounds are +# entirely within the codepoint. +[[test]] +name = "empty-utf8no-bounds" +regex = '' +haystack = '𝛃' +bounds = [1, 3] +matches = [[1, 1], [2, 2], [3, 3]] +unicode = true +utf8 = false + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8no-bounds-overlapping" +regex = '' +haystack = '𝛃' +bounds = [1, 3] +matches = [[1, 1], [2, 2], [3, 3]] +unicode = true +utf8 = false +match-kind = "all" +search-kind = "overlapping" + +# In this test, we anchor the search. Since the start position is also a UTF-8 +# boundary, we get a match. +[[test]] +name = "empty-utf8yes-anchored" +regex = '' +haystack = '𝛃' +matches = [[0, 0]] +anchored = true +unicode = true +utf8 = true + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8yes-anchored-overlapping" +regex = '' +haystack = '𝛃' +matches = [[0, 0]] +anchored = true +unicode = true +utf8 = true +match-kind = "all" +search-kind = "overlapping" + +# Same as above, except with UTF-8 mode disabled. It almost doesn't change the +# result, except for the fact that since this is an anchored search and we +# always find all matches, the test harness will keep reporting matches until +# none are found. Because it's anchored, matches will be reported so long as +# they are directly adjacent. Since with UTF-8 mode the next anchored search +# after the match at [0, 0] fails, iteration stops (and doesn't find the last +# match at [4, 4]). +[[test]] +name = "empty-utf8no-anchored" +regex = '' +haystack = '𝛃' +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] +anchored = true +unicode = true +utf8 = false + +# Tests the overlapping case of the above. +# +# Note that overlapping anchored searches are a little weird, and it's not +# totally clear what their semantics ought to be. For now, we just test the +# current behavior of our test shim that implements overlapping search. (This +# is one of the reasons why we don't really expose regex-level overlapping +# searches.) +[[test]] +name = "empty-utf8no-anchored-overlapping" +regex = '' +haystack = '𝛃' +matches = [[0, 0]] +anchored = true +unicode = true +utf8 = false +match-kind = "all" +search-kind = "overlapping" + +# In this test, we anchor the search, but also set bounds. The bounds start the +# search in the middle of a codepoint, so there should never be a match. +[[test]] +name = "empty-utf8yes-anchored-bounds" +regex = '' +haystack = '𝛃' +matches = [] +bounds = [1, 3] +anchored = true +unicode = true +utf8 = true + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8yes-anchored-bounds-overlapping" +regex = '' +haystack = '𝛃' +matches = [] +bounds = [1, 3] +anchored = true +unicode = true +utf8 = true +match-kind = "all" +search-kind = "overlapping" + +# Same as above, except with UTF-8 mode disabled. Without UTF-8 mode enabled, +# matching within a codepoint is allowed. And remember, as in the anchored test +# above with UTF-8 mode disabled, iteration will report all adjacent matches. +# The matches at [0, 0] and [4, 4] are not included because of the bounds of +# the search. +[[test]] +name = "empty-utf8no-anchored-bounds" +regex = '' +haystack = '𝛃' +bounds = [1, 3] +matches = [[1, 1], [2, 2], [3, 3]] +anchored = true +unicode = true +utf8 = false + +# Tests the overlapping case of the above. +# +# Note that overlapping anchored searches are a little weird, and it's not +# totally clear what their semantics ought to be. For now, we just test the +# current behavior of our test shim that implements overlapping search. (This +# is one of the reasons why we don't really expose regex-level overlapping +# searches.) +[[test]] +name = "empty-utf8no-anchored-bounds-overlapping" +regex = '' +haystack = '𝛃' +bounds = [1, 3] +matches = [[1, 1]] +anchored = true +unicode = true +utf8 = false +match-kind = "all" +search-kind = "overlapping" + +# This tests that we find the match at the end of the string when the bounds +# exclude the first match. +[[test]] +name = "empty-utf8yes-startbound" +regex = '' +haystack = '𝛃' +bounds = [1, 4] +matches = [[4, 4]] +unicode = true +utf8 = true + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8yes-startbound-overlapping" +regex = '' +haystack = '𝛃' +bounds = [1, 4] +matches = [[4, 4]] +unicode = true +utf8 = true +match-kind = "all" +search-kind = "overlapping" + +# Same as above, except since UTF-8 mode is disabled, we also find the matches +# inbetween that split the codepoint. +[[test]] +name = "empty-utf8no-startbound" +regex = '' +haystack = '𝛃' +bounds = [1, 4] +matches = [[1, 1], [2, 2], [3, 3], [4, 4]] +unicode = true +utf8 = false + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8no-startbound-overlapping" +regex = '' +haystack = '𝛃' +bounds = [1, 4] +matches = [[1, 1], [2, 2], [3, 3], [4, 4]] +unicode = true +utf8 = false +match-kind = "all" +search-kind = "overlapping" + +# This tests that we don't find any matches in an anchored search, even when +# the bounds include a match (at the end). +[[test]] +name = "empty-utf8yes-anchored-startbound" +regex = '' +haystack = '𝛃' +bounds = [1, 4] +matches = [] +anchored = true +unicode = true +utf8 = true + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8yes-anchored-startbound-overlapping" +regex = '' +haystack = '𝛃' +bounds = [1, 4] +matches = [] +anchored = true +unicode = true +utf8 = true +match-kind = "all" +search-kind = "overlapping" + +# Same as above, except since UTF-8 mode is disabled, we also find the matches +# inbetween that split the codepoint. Even though this is an anchored search, +# since the matches are adjacent, we find all of them. +[[test]] +name = "empty-utf8no-anchored-startbound" +regex = '' +haystack = '𝛃' +bounds = [1, 4] +matches = [[1, 1], [2, 2], [3, 3], [4, 4]] +anchored = true +unicode = true +utf8 = false + +# Tests the overlapping case of the above. +# +# Note that overlapping anchored searches are a little weird, and it's not +# totally clear what their semantics ought to be. For now, we just test the +# current behavior of our test shim that implements overlapping search. (This +# is one of the reasons why we don't really expose regex-level overlapping +# searches.) +[[test]] +name = "empty-utf8no-anchored-startbound-overlapping" +regex = '' +haystack = '𝛃' +bounds = [1, 4] +matches = [[1, 1]] +anchored = true +unicode = true +utf8 = false +match-kind = "all" +search-kind = "overlapping" + +# This tests that we find the match at the end of the haystack in UTF-8 mode +# when our bounds only include the empty string at the end of the haystack. +[[test]] +name = "empty-utf8yes-anchored-endbound" +regex = '' +haystack = '𝛃' +bounds = [4, 4] +matches = [[4, 4]] +anchored = true +unicode = true +utf8 = true + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8yes-anchored-endbound-overlapping" +regex = '' +haystack = '𝛃' +bounds = [4, 4] +matches = [[4, 4]] +anchored = true +unicode = true +utf8 = true +match-kind = "all" +search-kind = "overlapping" + +# Same as above, but with UTF-8 mode disabled. Results remain the same since +# the only possible match does not split a codepoint. +[[test]] +name = "empty-utf8no-anchored-endbound" +regex = '' +haystack = '𝛃' +bounds = [4, 4] +matches = [[4, 4]] +anchored = true +unicode = true +utf8 = false + +# Tests the overlapping case of the above. +[[test]] +name = "empty-utf8no-anchored-endbound-overlapping" +regex = '' +haystack = '𝛃' +bounds = [4, 4] +matches = [[4, 4]] +anchored = true +unicode = true +utf8 = false +match-kind = "all" +search-kind = "overlapping" diff --git a/third_party/crates/regex-1.13.1/testdata/word-boundary-special.toml b/third_party/crates/regex-1.13.1/testdata/word-boundary-special.toml new file mode 100644 index 0000000..2b5a2a0 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/word-boundary-special.toml @@ -0,0 +1,687 @@ +# These tests are for the "special" word boundary assertions. That is, +# \b{start}, \b{end}, \b{start-half}, \b{end-half}. These are specialty +# assertions for more niche use cases, but hitting those cases without these +# assertions is difficult. For example, \b{start-half} and \b{end-half} are +# used to implement the -w/--word-regexp flag in a grep program. + +# Tests for (?-u:\b{start}) + +[[test]] +name = "word-start-ascii-010" +regex = '\b{start}' +haystack = "a" +matches = [[0, 0]] +unicode = false + +[[test]] +name = "word-start-ascii-020" +regex = '\b{start}' +haystack = "a " +matches = [[0, 0]] +unicode = false + +[[test]] +name = "word-start-ascii-030" +regex = '\b{start}' +haystack = " a " +matches = [[1, 1]] +unicode = false + +[[test]] +name = "word-start-ascii-040" +regex = '\b{start}' +haystack = "" +matches = [] +unicode = false + +[[test]] +name = "word-start-ascii-050" +regex = '\b{start}' +haystack = "ab" +matches = [[0, 0]] +unicode = false + +[[test]] +name = "word-start-ascii-060" +regex = '\b{start}' +haystack = "𝛃" +matches = [] +unicode = false + +[[test]] +name = "word-start-ascii-060-bounds" +regex = '\b{start}' +haystack = "𝛃" +bounds = [2, 3] +matches = [] +unicode = false + +[[test]] +name = "word-start-ascii-070" +regex = '\b{start}' +haystack = " 𝛃 " +matches = [] +unicode = false + +[[test]] +name = "word-start-ascii-080" +regex = '\b{start}' +haystack = "𝛃𐆀" +matches = [] +unicode = false + +[[test]] +name = "word-start-ascii-090" +regex = '\b{start}' +haystack = "𝛃b" +matches = [[4, 4]] +unicode = false + +[[test]] +name = "word-start-ascii-110" +regex = '\b{start}' +haystack = "b𝛃" +matches = [[0, 0]] +unicode = false + +# Tests for (?-u:\b{end}) + +[[test]] +name = "word-end-ascii-010" +regex = '\b{end}' +haystack = "a" +matches = [[1, 1]] +unicode = false + +[[test]] +name = "word-end-ascii-020" +regex = '\b{end}' +haystack = "a " +matches = [[1, 1]] +unicode = false + +[[test]] +name = "word-end-ascii-030" +regex = '\b{end}' +haystack = " a " +matches = [[2, 2]] +unicode = false + +[[test]] +name = "word-end-ascii-040" +regex = '\b{end}' +haystack = "" +matches = [] +unicode = false + +[[test]] +name = "word-end-ascii-050" +regex = '\b{end}' +haystack = "ab" +matches = [[2, 2]] +unicode = false + +[[test]] +name = "word-end-ascii-060" +regex = '\b{end}' +haystack = "𝛃" +matches = [] +unicode = false + +[[test]] +name = "word-end-ascii-060-bounds" +regex = '\b{end}' +haystack = "𝛃" +bounds = [2, 3] +matches = [] +unicode = false + +[[test]] +name = "word-end-ascii-070" +regex = '\b{end}' +haystack = " 𝛃 " +matches = [] +unicode = false + +[[test]] +name = "word-end-ascii-080" +regex = '\b{end}' +haystack = "𝛃𐆀" +matches = [] +unicode = false + +[[test]] +name = "word-end-ascii-090" +regex = '\b{end}' +haystack = "𝛃b" +matches = [[5, 5]] +unicode = false + +[[test]] +name = "word-end-ascii-110" +regex = '\b{end}' +haystack = "b𝛃" +matches = [[1, 1]] +unicode = false + +# Tests for \b{start} + +[[test]] +name = "word-start-unicode-010" +regex = '\b{start}' +haystack = "a" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-unicode-020" +regex = '\b{start}' +haystack = "a " +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-unicode-030" +regex = '\b{start}' +haystack = " a " +matches = [[1, 1]] +unicode = true + +[[test]] +name = "word-start-unicode-040" +regex = '\b{start}' +haystack = "" +matches = [] +unicode = true + +[[test]] +name = "word-start-unicode-050" +regex = '\b{start}' +haystack = "ab" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-unicode-060" +regex = '\b{start}' +haystack = "𝛃" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-unicode-060-bounds" +regex = '\b{start}' +haystack = "𝛃" +bounds = [2, 3] +matches = [] +unicode = true + +[[test]] +name = "word-start-unicode-070" +regex = '\b{start}' +haystack = " 𝛃 " +matches = [[1, 1]] +unicode = true + +[[test]] +name = "word-start-unicode-080" +regex = '\b{start}' +haystack = "𝛃𐆀" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-unicode-090" +regex = '\b{start}' +haystack = "𝛃b" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-unicode-110" +regex = '\b{start}' +haystack = "b𝛃" +matches = [[0, 0]] +unicode = true + +# Tests for \b{end} + +[[test]] +name = "word-end-unicode-010" +regex = '\b{end}' +haystack = "a" +matches = [[1, 1]] +unicode = true + +[[test]] +name = "word-end-unicode-020" +regex = '\b{end}' +haystack = "a " +matches = [[1, 1]] +unicode = true + +[[test]] +name = "word-end-unicode-030" +regex = '\b{end}' +haystack = " a " +matches = [[2, 2]] +unicode = true + +[[test]] +name = "word-end-unicode-040" +regex = '\b{end}' +haystack = "" +matches = [] +unicode = true + +[[test]] +name = "word-end-unicode-050" +regex = '\b{end}' +haystack = "ab" +matches = [[2, 2]] +unicode = true + +[[test]] +name = "word-end-unicode-060" +regex = '\b{end}' +haystack = "𝛃" +matches = [[4, 4]] +unicode = true + +[[test]] +name = "word-end-unicode-060-bounds" +regex = '\b{end}' +haystack = "𝛃" +bounds = [2, 3] +matches = [] +unicode = true + +[[test]] +name = "word-end-unicode-070" +regex = '\b{end}' +haystack = " 𝛃 " +matches = [[5, 5]] +unicode = true + +[[test]] +name = "word-end-unicode-080" +regex = '\b{end}' +haystack = "𝛃𐆀" +matches = [[4, 4]] +unicode = true + +[[test]] +name = "word-end-unicode-090" +regex = '\b{end}' +haystack = "𝛃b" +matches = [[5, 5]] +unicode = true + +[[test]] +name = "word-end-unicode-110" +regex = '\b{end}' +haystack = "b𝛃" +matches = [[5, 5]] +unicode = true + +# Tests for (?-u:\b{start-half}) + +[[test]] +name = "word-start-half-ascii-010" +regex = '\b{start-half}' +haystack = "a" +matches = [[0, 0]] +unicode = false + +[[test]] +name = "word-start-half-ascii-020" +regex = '\b{start-half}' +haystack = "a " +matches = [[0, 0], [2, 2]] +unicode = false + +[[test]] +name = "word-start-half-ascii-030" +regex = '\b{start-half}' +haystack = " a " +matches = [[0, 0], [1, 1], [3, 3]] +unicode = false + +[[test]] +name = "word-start-half-ascii-040" +regex = '\b{start-half}' +haystack = "" +matches = [[0, 0]] +unicode = false + +[[test]] +name = "word-start-half-ascii-050" +regex = '\b{start-half}' +haystack = "ab" +matches = [[0, 0]] +unicode = false + +[[test]] +name = "word-start-half-ascii-060" +regex = '\b{start-half}' +haystack = "𝛃" +matches = [[0, 0], [4, 4]] +unicode = false + +[[test]] +name = "word-start-half-ascii-060-noutf8" +regex = '\b{start-half}' +haystack = "𝛃" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]] +unicode = false +utf8 = false + +[[test]] +name = "word-start-half-ascii-060-bounds" +regex = '\b{start-half}' +haystack = "𝛃" +bounds = [2, 3] +matches = [] +unicode = false + +[[test]] +name = "word-start-half-ascii-070" +regex = '\b{start-half}' +haystack = " 𝛃 " +matches = [[0, 0], [1, 1], [5, 5], [6, 6]] +unicode = false + +[[test]] +name = "word-start-half-ascii-080" +regex = '\b{start-half}' +haystack = "𝛃𐆀" +matches = [[0, 0], [4, 4], [8, 8]] +unicode = false + +[[test]] +name = "word-start-half-ascii-090" +regex = '\b{start-half}' +haystack = "𝛃b" +matches = [[0, 0], [4, 4]] +unicode = false + +[[test]] +name = "word-start-half-ascii-110" +regex = '\b{start-half}' +haystack = "b𝛃" +matches = [[0, 0], [5, 5]] +unicode = false + +# Tests for (?-u:\b{end-half}) + +[[test]] +name = "word-end-half-ascii-010" +regex = '\b{end-half}' +haystack = "a" +matches = [[1, 1]] +unicode = false + +[[test]] +name = "word-end-half-ascii-020" +regex = '\b{end-half}' +haystack = "a " +matches = [[1, 1], [2, 2]] +unicode = false + +[[test]] +name = "word-end-half-ascii-030" +regex = '\b{end-half}' +haystack = " a " +matches = [[0, 0], [2, 2], [3, 3]] +unicode = false + +[[test]] +name = "word-end-half-ascii-040" +regex = '\b{end-half}' +haystack = "" +matches = [[0, 0]] +unicode = false + +[[test]] +name = "word-end-half-ascii-050" +regex = '\b{end-half}' +haystack = "ab" +matches = [[2, 2]] +unicode = false + +[[test]] +name = "word-end-half-ascii-060" +regex = '\b{end-half}' +haystack = "𝛃" +matches = [[0, 0], [4, 4]] +unicode = false + +[[test]] +name = "word-end-half-ascii-060-bounds" +regex = '\b{end-half}' +haystack = "𝛃" +bounds = [2, 3] +matches = [] +unicode = false + +[[test]] +name = "word-end-half-ascii-070" +regex = '\b{end-half}' +haystack = " 𝛃 " +matches = [[0, 0], [1, 1], [5, 5], [6, 6]] +unicode = false + +[[test]] +name = "word-end-half-ascii-080" +regex = '\b{end-half}' +haystack = "𝛃𐆀" +matches = [[0, 0], [4, 4], [8, 8]] +unicode = false + +[[test]] +name = "word-end-half-ascii-090" +regex = '\b{end-half}' +haystack = "𝛃b" +matches = [[0, 0], [5, 5]] +unicode = false + +[[test]] +name = "word-end-half-ascii-110" +regex = '\b{end-half}' +haystack = "b𝛃" +matches = [[1, 1], [5, 5]] +unicode = false + +# Tests for \b{start-half} + +[[test]] +name = "word-start-half-unicode-010" +regex = '\b{start-half}' +haystack = "a" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-half-unicode-020" +regex = '\b{start-half}' +haystack = "a " +matches = [[0, 0], [2, 2]] +unicode = true + +[[test]] +name = "word-start-half-unicode-030" +regex = '\b{start-half}' +haystack = " a " +matches = [[0, 0], [1, 1], [3, 3]] +unicode = true + +[[test]] +name = "word-start-half-unicode-040" +regex = '\b{start-half}' +haystack = "" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-half-unicode-050" +regex = '\b{start-half}' +haystack = "ab" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-half-unicode-060" +regex = '\b{start-half}' +haystack = "𝛃" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-half-unicode-060-bounds" +regex = '\b{start-half}' +haystack = "𝛃" +bounds = [2, 3] +matches = [] +unicode = true + +[[test]] +name = "word-start-half-unicode-070" +regex = '\b{start-half}' +haystack = " 𝛃 " +matches = [[0, 0], [1, 1], [6, 6]] +unicode = true + +[[test]] +name = "word-start-half-unicode-080" +regex = '\b{start-half}' +haystack = "𝛃𐆀" +matches = [[0, 0], [8, 8]] +unicode = true + +[[test]] +name = "word-start-half-unicode-090" +regex = '\b{start-half}' +haystack = "𝛃b" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-start-half-unicode-110" +regex = '\b{start-half}' +haystack = "b𝛃" +matches = [[0, 0]] +unicode = true + +# Tests for \b{end-half} + +[[test]] +name = "word-end-half-unicode-010" +regex = '\b{end-half}' +haystack = "a" +matches = [[1, 1]] +unicode = true + +[[test]] +name = "word-end-half-unicode-020" +regex = '\b{end-half}' +haystack = "a " +matches = [[1, 1], [2, 2]] +unicode = true + +[[test]] +name = "word-end-half-unicode-030" +regex = '\b{end-half}' +haystack = " a " +matches = [[0, 0], [2, 2], [3, 3]] +unicode = true + +[[test]] +name = "word-end-half-unicode-040" +regex = '\b{end-half}' +haystack = "" +matches = [[0, 0]] +unicode = true + +[[test]] +name = "word-end-half-unicode-050" +regex = '\b{end-half}' +haystack = "ab" +matches = [[2, 2]] +unicode = true + +[[test]] +name = "word-end-half-unicode-060" +regex = '\b{end-half}' +haystack = "𝛃" +matches = [[4, 4]] +unicode = true + +[[test]] +name = "word-end-half-unicode-060-bounds" +regex = '\b{end-half}' +haystack = "𝛃" +bounds = [2, 3] +matches = [] +unicode = true + +[[test]] +name = "word-end-half-unicode-070" +regex = '\b{end-half}' +haystack = " 𝛃 " +matches = [[0, 0], [5, 5], [6, 6]] +unicode = true + +[[test]] +name = "word-end-half-unicode-080" +regex = '\b{end-half}' +haystack = "𝛃𐆀" +matches = [[4, 4], [8, 8]] +unicode = true + +[[test]] +name = "word-end-half-unicode-090" +regex = '\b{end-half}' +haystack = "𝛃b" +matches = [[5, 5]] +unicode = true + +[[test]] +name = "word-end-half-unicode-110" +regex = '\b{end-half}' +haystack = "b𝛃" +matches = [[5, 5]] +unicode = true + +# Specialty tests. + +# Since \r is special cased in the start state computation (to deal with CRLF +# mode), this test ensures that the correct start state is computed when the +# pattern starts with a half word boundary assertion. +[[test]] +name = "word-start-half-ascii-carriage" +regex = '\b{start-half}[a-z]+' +haystack = 'ABC\rabc' +matches = [[4, 7]] +bounds = [4, 7] +unescape = true + +# Since \n is also special cased in the start state computation, this test +# ensures that the correct start state is computed when the pattern starts with +# a half word boundary assertion. +[[test]] +name = "word-start-half-ascii-linefeed" +regex = '\b{start-half}[a-z]+' +haystack = 'ABC\nabc' +matches = [[4, 7]] +bounds = [4, 7] +unescape = true + +# Like the carriage return test above, but with a custom line terminator. +[[test]] +name = "word-start-half-ascii-customlineterm" +regex = '\b{start-half}[a-z]+' +haystack = 'ABC!abc' +matches = [[4, 7]] +bounds = [4, 7] +unescape = true +line-terminator = '!' diff --git a/third_party/crates/regex-1.13.1/testdata/word-boundary.toml b/third_party/crates/regex-1.13.1/testdata/word-boundary.toml new file mode 100644 index 0000000..1d86fc9 --- /dev/null +++ b/third_party/crates/regex-1.13.1/testdata/word-boundary.toml @@ -0,0 +1,781 @@ +# Some of these are cribbed from RE2's test suite. + +# These test \b. Below are tests for \B. +[[test]] +name = "wb1" +regex = '\b' +haystack = "" +matches = [] +unicode = false + +[[test]] +name = "wb2" +regex = '\b' +haystack = "a" +matches = [[0, 0], [1, 1]] +unicode = false + +[[test]] +name = "wb3" +regex = '\b' +haystack = "ab" +matches = [[0, 0], [2, 2]] +unicode = false + +[[test]] +name = "wb4" +regex = '^\b' +haystack = "ab" +matches = [[0, 0]] +unicode = false + +[[test]] +name = "wb5" +regex = '\b$' +haystack = "ab" +matches = [[2, 2]] +unicode = false + +[[test]] +name = "wb6" +regex = '^\b$' +haystack = "ab" +matches = [] +unicode = false + +[[test]] +name = "wb7" +regex = '\bbar\b' +haystack = "nobar bar foo bar" +matches = [[6, 9], [14, 17]] +unicode = false + +[[test]] +name = "wb8" +regex = 'a\b' +haystack = "faoa x" +matches = [[3, 4]] +unicode = false + +[[test]] +name = "wb9" +regex = '\bbar' +haystack = "bar x" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb10" +regex = '\bbar' +haystack = "foo\nbar x" +matches = [[4, 7]] +unicode = false + +[[test]] +name = "wb11" +regex = 'bar\b' +haystack = "foobar" +matches = [[3, 6]] +unicode = false + +[[test]] +name = "wb12" +regex = 'bar\b' +haystack = "foobar\nxxx" +matches = [[3, 6]] +unicode = false + +[[test]] +name = "wb13" +regex = '(?:foo|bar|[A-Z])\b' +haystack = "foo" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb14" +regex = '(?:foo|bar|[A-Z])\b' +haystack = "foo\n" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb15" +regex = '\b(?:foo|bar|[A-Z])' +haystack = "foo" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb16" +regex = '\b(?:foo|bar|[A-Z])\b' +haystack = "X" +matches = [[0, 1]] +unicode = false + +[[test]] +name = "wb17" +regex = '\b(?:foo|bar|[A-Z])\b' +haystack = "XY" +matches = [] +unicode = false + +[[test]] +name = "wb18" +regex = '\b(?:foo|bar|[A-Z])\b' +haystack = "bar" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb19" +regex = '\b(?:foo|bar|[A-Z])\b' +haystack = "foo" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb20" +regex = '\b(?:foo|bar|[A-Z])\b' +haystack = "foo\n" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb21" +regex = '\b(?:foo|bar|[A-Z])\b' +haystack = "ffoo bbar N x" +matches = [[10, 11]] +unicode = false + +[[test]] +name = "wb22" +regex = '\b(?:fo|foo)\b' +haystack = "fo" +matches = [[0, 2]] +unicode = false + +[[test]] +name = "wb23" +regex = '\b(?:fo|foo)\b' +haystack = "foo" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb24" +regex = '\b\b' +haystack = "" +matches = [] +unicode = false + +[[test]] +name = "wb25" +regex = '\b\b' +haystack = "a" +matches = [[0, 0], [1, 1]] +unicode = false + +[[test]] +name = "wb26" +regex = '\b$' +haystack = "" +matches = [] +unicode = false + +[[test]] +name = "wb27" +regex = '\b$' +haystack = "x" +matches = [[1, 1]] +unicode = false + +[[test]] +name = "wb28" +regex = '\b$' +haystack = "y x" +matches = [[3, 3]] +unicode = false + +[[test]] +name = "wb29" +regex = '(?-u:\b).$' +haystack = "x" +matches = [[0, 1]] + +[[test]] +name = "wb30" +regex = '^\b(?:fo|foo)\b' +haystack = "fo" +matches = [[0, 2]] +unicode = false + +[[test]] +name = "wb31" +regex = '^\b(?:fo|foo)\b' +haystack = "foo" +matches = [[0, 3]] +unicode = false + +[[test]] +name = "wb32" +regex = '^\b$' +haystack = "" +matches = [] +unicode = false + +[[test]] +name = "wb33" +regex = '^\b$' +haystack = "x" +matches = [] +unicode = false + +[[test]] +name = "wb34" +regex = '^(?-u:\b).$' +haystack = "x" +matches = [[0, 1]] + +[[test]] +name = "wb35" +regex = '^(?-u:\b).(?-u:\b)$' +haystack = "x" +matches = [[0, 1]] + +[[test]] +name = "wb36" +regex = '^^^^^\b$$$$$' +haystack = "" +matches = [] +unicode = false + +[[test]] +name = "wb37" +regex = '^^^^^(?-u:\b).$$$$$' +haystack = "x" +matches = [[0, 1]] + +[[test]] +name = "wb38" +regex = '^^^^^\b$$$$$' +haystack = "x" +matches = [] +unicode = false + +[[test]] +name = "wb39" +regex = '^^^^^(?-u:\b\b\b).(?-u:\b\b\b)$$$$$' +haystack = "x" +matches = [[0, 1]] + +[[test]] +name = "wb40" +regex = '(?-u:\b).+(?-u:\b)' +haystack = "$$abc$$" +matches = [[2, 5]] + +[[test]] +name = "wb41" +regex = '\b' +haystack = "a b c" +matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]] +unicode = false + +[[test]] +name = "wb42" +regex = '\bfoo\b' +haystack = "zzz foo zzz" +matches = [[4, 7]] +unicode = false + +[[test]] +name = "wb43" +regex = '\b^' +haystack = "ab" +matches = [[0, 0]] +unicode = false + +[[test]] +name = "wb44" +regex = '$\b' +haystack = "ab" +matches = [[2, 2]] +unicode = false + + +# Tests for \B. Note that \B is not allowed if UTF-8 mode is enabled, so we +# have to disable it for most of these tests. This is because \B can match at +# non-UTF-8 boundaries. +[[test]] +name = "nb1" +regex = '\Bfoo\B' +haystack = "n foo xfoox that" +matches = [[7, 10]] +unicode = false +utf8 = false + +[[test]] +name = "nb2" +regex = 'a\B' +haystack = "faoa x" +matches = [[1, 2]] +unicode = false +utf8 = false + +[[test]] +name = "nb3" +regex = '\Bbar' +haystack = "bar x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb4" +regex = '\Bbar' +haystack = "foo\nbar x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb5" +regex = 'bar\B' +haystack = "foobar" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb6" +regex = 'bar\B' +haystack = "foobar\nxxx" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb7" +regex = '(?:foo|bar|[A-Z])\B' +haystack = "foox" +matches = [[0, 3]] +unicode = false +utf8 = false + +[[test]] +name = "nb8" +regex = '(?:foo|bar|[A-Z])\B' +haystack = "foo\n" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb9" +regex = '\B' +haystack = "" +matches = [[0, 0]] +unicode = false +utf8 = false + +[[test]] +name = "nb10" +regex = '\B' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb11" +regex = '\B(?:foo|bar|[A-Z])' +haystack = "foo" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb12" +regex = '\B(?:foo|bar|[A-Z])\B' +haystack = "xXy" +matches = [[1, 2]] +unicode = false +utf8 = false + +[[test]] +name = "nb13" +regex = '\B(?:foo|bar|[A-Z])\B' +haystack = "XY" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb14" +regex = '\B(?:foo|bar|[A-Z])\B' +haystack = "XYZ" +matches = [[1, 2]] +unicode = false +utf8 = false + +[[test]] +name = "nb15" +regex = '\B(?:foo|bar|[A-Z])\B' +haystack = "abara" +matches = [[1, 4]] +unicode = false +utf8 = false + +[[test]] +name = "nb16" +regex = '\B(?:foo|bar|[A-Z])\B' +haystack = "xfoo_" +matches = [[1, 4]] +unicode = false +utf8 = false + +[[test]] +name = "nb17" +regex = '\B(?:foo|bar|[A-Z])\B' +haystack = "xfoo\n" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb18" +regex = '\B(?:foo|bar|[A-Z])\B' +haystack = "foo bar vNX" +matches = [[9, 10]] +unicode = false +utf8 = false + +[[test]] +name = "nb19" +regex = '\B(?:fo|foo)\B' +haystack = "xfoo" +matches = [[1, 3]] +unicode = false +utf8 = false + +[[test]] +name = "nb20" +regex = '\B(?:foo|fo)\B' +haystack = "xfooo" +matches = [[1, 4]] +unicode = false +utf8 = false + +[[test]] +name = "nb21" +regex = '\B\B' +haystack = "" +matches = [[0, 0]] +unicode = false +utf8 = false + +[[test]] +name = "nb22" +regex = '\B\B' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb23" +regex = '\B$' +haystack = "" +matches = [[0, 0]] +unicode = false +utf8 = false + +[[test]] +name = "nb24" +regex = '\B$' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb25" +regex = '\B$' +haystack = "y x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb26" +regex = '\B.$' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb27" +regex = '^\B(?:fo|foo)\B' +haystack = "fo" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb28" +regex = '^\B(?:fo|foo)\B' +haystack = "fo" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb29" +regex = '^\B' +haystack = "" +matches = [[0, 0]] +unicode = false +utf8 = false + +[[test]] +name = "nb30" +regex = '^\B' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb31" +regex = '^\B\B' +haystack = "" +matches = [[0, 0]] +unicode = false +utf8 = false + +[[test]] +name = "nb32" +regex = '^\B\B' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb33" +regex = '^\B$' +haystack = "" +matches = [[0, 0]] +unicode = false +utf8 = false + +[[test]] +name = "nb34" +regex = '^\B$' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb35" +regex = '^\B.$' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb36" +regex = '^\B.\B$' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb37" +regex = '^^^^^\B$$$$$' +haystack = "" +matches = [[0, 0]] +unicode = false +utf8 = false + +[[test]] +name = "nb38" +regex = '^^^^^\B.$$$$$' +haystack = "x" +matches = [] +unicode = false +utf8 = false + +[[test]] +name = "nb39" +regex = '^^^^^\B$$$$$' +haystack = "x" +matches = [] +unicode = false +utf8 = false + + +# unicode1* and unicode2* work for both Unicode and ASCII because all matches +# are reported as byte offsets, and « and » do not correspond to word +# boundaries at either the character or byte level. +[[test]] +name = "unicode1" +regex = '\bx\b' +haystack = "«x" +matches = [[2, 3]] + +[[test]] +name = "unicode1-only-ascii" +regex = '\bx\b' +haystack = "«x" +matches = [[2, 3]] +unicode = false + +[[test]] +name = "unicode2" +regex = '\bx\b' +haystack = "x»" +matches = [[0, 1]] + +[[test]] +name = "unicode2-only-ascii" +regex = '\bx\b' +haystack = "x»" +matches = [[0, 1]] +unicode = false + +# ASCII word boundaries are completely oblivious to Unicode characters, so +# even though β is a character, an ASCII \b treats it as a word boundary +# when it is adjacent to another ASCII character. (The ASCII \b only looks +# at the leading byte of β.) For Unicode \b, the tests are precisely inverted. +[[test]] +name = "unicode3" +regex = '\bx\b' +haystack = 'áxβ' +matches = [] + +[[test]] +name = "unicode3-only-ascii" +regex = '\bx\b' +haystack = 'áxβ' +matches = [[2, 3]] +unicode = false + +[[test]] +name = "unicode4" +regex = '\Bx\B' +haystack = 'áxβ' +matches = [[2, 3]] + +[[test]] +name = "unicode4-only-ascii" +regex = '\Bx\B' +haystack = 'áxβ' +matches = [] +unicode = false +utf8 = false + +# The same as above, but with \b instead of \B as a sanity check. +[[test]] +name = "unicode5" +regex = '\b' +haystack = "0\U0007EF5E" +matches = [[0, 0], [1, 1]] + +[[test]] +name = "unicode5-only-ascii" +regex = '\b' +haystack = "0\U0007EF5E" +matches = [[0, 0], [1, 1]] +unicode = false +utf8 = false + +[[test]] +name = "unicode5-noutf8" +regex = '\b' +haystack = '0\xFF\xFF\xFF\xFF' +matches = [[0, 0], [1, 1]] +unescape = true +utf8 = false + +[[test]] +name = "unicode5-noutf8-only-ascii" +regex = '\b' +haystack = '0\xFF\xFF\xFF\xFF' +matches = [[0, 0], [1, 1]] +unescape = true +unicode = false +utf8 = false + +# Weird special case to ensure that ASCII \B treats each individual code unit +# as a non-word byte. (The specific codepoint is irrelevant. It's an arbitrary +# codepoint that uses 4 bytes in its UTF-8 encoding and is not a member of the +# \w character class.) +[[test]] +name = "unicode5-not" +regex = '\B' +haystack = "0\U0007EF5E" +matches = [[5, 5]] + +[[test]] +name = "unicode5-not-only-ascii" +regex = '\B' +haystack = "0\U0007EF5E" +matches = [[2, 2], [3, 3], [4, 4], [5, 5]] +unicode = false +utf8 = false + +# This gets no matches since \B only matches in the presence of valid UTF-8 +# when Unicode is enabled, even when UTF-8 mode is disabled. +[[test]] +name = "unicode5-not-noutf8" +regex = '\B' +haystack = '0\xFF\xFF\xFF\xFF' +matches = [] +unescape = true +utf8 = false + +# But this DOES get matches since \B in ASCII mode only looks at individual +# bytes. +[[test]] +name = "unicode5-not-noutf8-only-ascii" +regex = '\B' +haystack = '0\xFF\xFF\xFF\xFF' +matches = [[2, 2], [3, 3], [4, 4], [5, 5]] +unescape = true +unicode = false +utf8 = false + +# Some tests of no particular significance. +[[test]] +name = "unicode6" +regex = '\b[0-9]+\b' +haystack = "foo 123 bar 456 quux 789" +matches = [[4, 7], [12, 15], [21, 24]] + +[[test]] +name = "unicode7" +regex = '\b[0-9]+\b' +haystack = "foo 123 bar a456 quux 789" +matches = [[4, 7], [22, 25]] + +[[test]] +name = "unicode8" +regex = '\b[0-9]+\b' +haystack = "foo 123 bar 456a quux 789" +matches = [[4, 7], [22, 25]] + +# A variant of the problem described here: +# https://github.com/google/re2/blob/89567f5de5b23bb5ad0c26cbafc10bdc7389d1fa/re2/dfa.cc#L658-L667 +[[test]] +name = "alt-with-assertion-repetition" +regex = '(?:\b|%)+' +haystack = "z%" +bounds = [1, 2] +anchored = true +matches = [[1, 1]] diff --git a/third_party/crates/regex-1.13.1/tests/lib.rs b/third_party/crates/regex-1.13.1/tests/lib.rs new file mode 100644 index 0000000..b3f6942 --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/lib.rs @@ -0,0 +1,58 @@ +#![cfg_attr(feature = "pattern", feature(pattern))] + +mod fuzz; +mod misc; +mod regression; +mod regression_fuzz; +mod replace; +#[cfg(feature = "pattern")] +mod searcher; +mod suite_bytes; +mod suite_bytes_set; +mod suite_string; +mod suite_string_set; + +const BLACKLIST: &[&str] = &[ + // Nothing to blacklist yet! +]; + +fn suite() -> anyhow::Result<regex_test::RegexTests> { + let _ = env_logger::try_init(); + + let mut tests = regex_test::RegexTests::new(); + macro_rules! load { + ($name:expr) => {{ + const DATA: &[u8] = + include_bytes!(concat!("../testdata/", $name, ".toml")); + tests.load_slice($name, DATA)?; + }}; + } + + load!("anchored"); + load!("bytes"); + load!("crazy"); + load!("crlf"); + load!("earliest"); + load!("empty"); + load!("expensive"); + load!("flags"); + load!("iter"); + load!("leftmost-all"); + load!("line-terminator"); + load!("misc"); + load!("multiline"); + load!("no-unicode"); + load!("overlapping"); + load!("regression"); + load!("set"); + load!("substring"); + load!("unicode"); + load!("utf8"); + load!("word-boundary"); + load!("word-boundary-special"); + load!("fowler/basic"); + load!("fowler/nullsubexpr"); + load!("fowler/repetition"); + + Ok(tests) +} diff --git a/third_party/crates/regex-1.13.1/tests/misc.rs b/third_party/crates/regex-1.13.1/tests/misc.rs new file mode 100644 index 0000000..3e646ce --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/misc.rs @@ -0,0 +1,161 @@ +use regex::{bytes, regex, Regex}; + +#[test] +fn unclosed_group_error() { + let err = Regex::new(r"(").unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("unclosed group"), "error message: {msg:?}"); +} + +#[test] +fn regex_string() { + assert_eq!(r"[a-zA-Z0-9]+", regex!(r"[a-zA-Z0-9]+").as_str()); + assert_eq!(r"[a-zA-Z0-9]+", &format!("{}", regex!(r"[a-zA-Z0-9]+"))); + assert_eq!( + r#"Regex("[a-zA-Z0-9]+")"#, + &format!("{:?}", regex!(r"[a-zA-Z0-9]+")) + ); +} + +#[test] +fn capture_names() { + let re = regex!(r"(.)(?P<a>.)"); + assert_eq!(3, re.captures_len()); + assert_eq!((3, Some(3)), re.capture_names().size_hint()); + assert_eq!( + vec![None, None, Some("a")], + re.capture_names().collect::<Vec<_>>() + ); +} + +#[test] +fn capture_index() { + let re = regex!(r"^(?P<name>.+)$"); + let cap = re.captures("abc").unwrap(); + assert_eq!(&cap[0], "abc"); + assert_eq!(&cap[1], "abc"); + assert_eq!(&cap["name"], "abc"); +} + +#[test] +#[should_panic] +fn capture_index_panic_usize() { + let re = regex!(r"^(?P<name>.+)$"); + let cap = re.captures("abc").unwrap(); + let _ = cap[2]; +} + +#[test] +#[should_panic] +fn capture_index_panic_name() { + let re = regex!(r"^(?P<name>.+)$"); + let cap = re.captures("abc").unwrap(); + let _ = cap["bad name"]; +} + +#[test] +fn capture_index_lifetime() { + // This is a test of whether the types on `caps["..."]` are general + // enough. If not, this will fail to typecheck. + fn inner(s: &str) -> usize { + let re = regex!(r"(?P<number>[0-9]+)"); + let caps = re.captures(s).unwrap(); + caps["number"].len() + } + assert_eq!(3, inner("123")); +} + +#[test] +fn capture_misc() { + let re = regex!(r"(.)(?P<a>a)?(.)(?P<b>.)"); + let cap = re.captures("abc").unwrap(); + + assert_eq!(5, cap.len()); + + assert_eq!((0, 3), { + let m = cap.get(0).unwrap(); + (m.start(), m.end()) + }); + assert_eq!(None, cap.get(2)); + assert_eq!((2, 3), { + let m = cap.get(4).unwrap(); + (m.start(), m.end()) + }); + + assert_eq!("abc", cap.get(0).unwrap().as_str()); + assert_eq!(None, cap.get(2)); + assert_eq!("c", cap.get(4).unwrap().as_str()); + + assert_eq!(None, cap.name("a")); + assert_eq!("c", cap.name("b").unwrap().as_str()); +} + +#[test] +fn sub_capture_matches() { + let re = regex!(r"([a-z])(([a-z])|([0-9]))"); + let cap = re.captures("a5").unwrap(); + let subs: Vec<_> = cap.iter().collect(); + + assert_eq!(5, subs.len()); + assert!(subs[0].is_some()); + assert!(subs[1].is_some()); + assert!(subs[2].is_some()); + assert!(subs[3].is_none()); + assert!(subs[4].is_some()); + + assert_eq!("a5", subs[0].unwrap().as_str()); + assert_eq!("a", subs[1].unwrap().as_str()); + assert_eq!("5", subs[2].unwrap().as_str()); + assert_eq!("5", subs[4].unwrap().as_str()); +} + +// Test that the DFA can handle pathological cases. (This should result in the +// DFA's cache being flushed too frequently, which should cause it to quit and +// fall back to the NFA algorithm.) +#[test] +fn dfa_handles_pathological_case() { + fn ones_and_zeroes(count: usize) -> String { + let mut s = String::new(); + for i in 0..count { + if i % 3 == 0 { + s.push('1'); + } else { + s.push('0'); + } + } + s + } + + let re = regex!(r"[01]*1[01]{20}$"); + let text = { + let mut pieces = ones_and_zeroes(100_000); + pieces.push('1'); + pieces.push_str(&ones_and_zeroes(20)); + pieces + }; + assert!(re.is_match(&text)); +} + +#[test] +fn re_macro() { + // >1 regex! should work in a scope since the static is in a block + assert!(regex!("foo").is_match("foo")); + assert!(regex!("bar").is_match("bar")); + + let re: &Regex = regex!("foo"); + ensure_static(re); + + fn ensure_static(_re: &'static Regex) {} +} + +#[test] +fn re_bytes_macro() { + // >1 regex! should work in a scope since the static is in a block + assert!(bytes::regex!("foo").is_match(b"foo")); + assert!(bytes::regex!("bar").is_match(b"bar")); + + let re: &bytes::Regex = bytes::regex!("foo"); + ensure_static(re); + + fn ensure_static(_re: &'static bytes::Regex) {} +} diff --git a/third_party/crates/regex-1.13.1/tests/regression.rs b/third_party/crates/regex-1.13.1/tests/regression.rs new file mode 100644 index 0000000..a586701 --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/regression.rs @@ -0,0 +1,94 @@ +use regex::Regex; + +macro_rules! regex { + ($pattern:expr) => { + regex::Regex::new($pattern).unwrap() + }; +} + +// See: https://github.com/rust-lang/regex/issues/48 +#[test] +fn invalid_regexes_no_crash() { + assert!(Regex::new("(*)").is_err()); + assert!(Regex::new("(?:?)").is_err()); + assert!(Regex::new("(?)").is_err()); + assert!(Regex::new("*").is_err()); +} + +// See: https://github.com/rust-lang/regex/issues/98 +#[test] +fn regression_many_repeat_stack_overflow() { + let re = regex!("^.{1,2500}"); + assert_eq!( + vec![0..1], + re.find_iter("a").map(|m| m.range()).collect::<Vec<_>>() + ); +} + +// See: https://github.com/rust-lang/regex/issues/555 +#[test] +fn regression_invalid_repetition_expr() { + assert!(Regex::new("(?m){1,1}").is_err()); +} + +// See: https://github.com/rust-lang/regex/issues/527 +#[test] +fn regression_invalid_flags_expression() { + assert!(Regex::new("(((?x)))").is_ok()); +} + +// See: https://github.com/rust-lang/regex/issues/129 +#[test] +fn regression_captures_rep() { + let re = regex!(r"([a-f]){2}(?P<foo>[x-z])"); + let caps = re.captures("abx").unwrap(); + assert_eq!(&caps["foo"], "x"); +} + +// See: https://github.com/BurntSushi/ripgrep/issues/1247 +#[cfg(feature = "unicode-perl")] +#[test] +fn regression_nfa_stops1() { + let re = regex::bytes::Regex::new(r"\bs(?:[ab])").unwrap(); + assert_eq!(0, re.find_iter(b"s\xE4").count()); +} + +// See: https://github.com/rust-lang/regex/issues/981 +#[cfg(feature = "unicode")] +#[test] +fn regression_bad_word_boundary() { + let re = regex!(r#"(?i:(?:\b|_)win(?:32|64|dows)?(?:\b|_))"#); + let hay = "ubi-Darwin-x86_64.tar.gz"; + assert!(!re.is_match(hay)); + let hay = "ubi-Windows-x86_64.zip"; + assert!(re.is_match(hay)); +} + +// See: https://github.com/rust-lang/regex/issues/982 +#[cfg(feature = "unicode-perl")] +#[test] +fn regression_unicode_perl_not_enabled() { + let pat = r"(\d+\s?(years|year|y))?\s?(\d+\s?(months|month|m))?\s?(\d+\s?(weeks|week|w))?\s?(\d+\s?(days|day|d))?\s?(\d+\s?(hours|hour|h))?"; + assert!(Regex::new(pat).is_ok()); +} + +// See: https://github.com/rust-lang/regex/issues/995 +#[test] +fn regression_big_regex_overflow() { + let pat = r" {2147483516}{2147483416}{5}"; + assert!(Regex::new(pat).is_err()); +} + +// See: https://github.com/rust-lang/regex/issues/999 +#[test] +fn regression_complete_literals_suffix_incorrect() { + let needles = vec![ + "aA", "bA", "cA", "dA", "eA", "fA", "gA", "hA", "iA", "jA", "kA", + "lA", "mA", "nA", "oA", "pA", "qA", "rA", "sA", "tA", "uA", "vA", + "wA", "xA", "yA", "zA", + ]; + let pattern = needles.join("|"); + let re = regex!(&pattern); + let hay = "FUBAR"; + assert_eq!(0, re.find_iter(hay).count()); +} diff --git a/third_party/crates/regex-1.13.1/tests/regression_fuzz.rs b/third_party/crates/regex-1.13.1/tests/regression_fuzz.rs new file mode 100644 index 0000000..f90ad4c --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/regression_fuzz.rs @@ -0,0 +1,61 @@ +// These tests are only run for the "default" test target because some of them +// can take quite a long time. Some of them take long enough that it's not +// practical to run them in debug mode. :-/ + +use regex::Regex; + +macro_rules! regex { + ($pattern:expr) => { + regex::Regex::new($pattern).unwrap() + }; +} + +// See: https://oss-fuzz.com/testcase-detail/5673225499181056 +// +// Ignored by default since it takes too long in debug mode (almost a minute). +#[test] +#[ignore] +fn fuzz1() { + regex!(r"1}{55}{0}*{1}{55}{55}{5}*{1}{55}+{56}|;**"); +} + +// See: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=26505 +// See: https://github.com/rust-lang/regex/issues/722 +#[test] +#[cfg(feature = "unicode")] +fn empty_any_errors_no_panic() { + assert!(Regex::new(r"\P{any}").is_ok()); +} + +// This tests that a very large regex errors during compilation instead of +// using gratuitous amounts of memory. The specific problem is that the +// compiler wasn't accounting for the memory used by Unicode character classes +// correctly. +// +// See: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=33579 +#[test] +fn big_regex_fails_to_compile() { + let pat = "[\u{0}\u{e}\u{2}\\w~~>[l\t\u{0}]p?<]{971158}"; + assert!(Regex::new(pat).is_err()); +} + +// This was caught while on master but before a release went out(!). +// +// See: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=58173 +#[test] +fn todo() { + let pat = "(?:z|xx)@|xx"; + assert!(Regex::new(pat).is_ok()); +} + +// This was caused by the fuzzer, and then minimized by hand. +// +// This was caused by a bug in DFA determinization that mishandled NFA fail +// states. +#[test] +fn fail_branch_prevents_match() { + let pat = r".*[a&&b]A|B"; + let hay = "B"; + let re = Regex::new(pat).unwrap(); + assert!(re.is_match(hay)); +} diff --git a/third_party/crates/regex-1.13.1/tests/replace.rs b/third_party/crates/regex-1.13.1/tests/replace.rs new file mode 100644 index 0000000..f26ae46 --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/replace.rs @@ -0,0 +1,183 @@ +macro_rules! replace( + ($name:ident, $which:ident, $re:expr, + $search:expr, $replace:expr, $result:expr) => ( + #[test] + fn $name() { + let re = regex::Regex::new($re).unwrap(); + assert_eq!(re.$which($search, $replace), $result); + } + ); +); + +replace!(first, replace, r"[0-9]", "age: 26", "Z", "age: Z6"); +replace!(plus, replace, r"[0-9]+", "age: 26", "Z", "age: Z"); +replace!(all, replace_all, r"[0-9]", "age: 26", "Z", "age: ZZ"); +replace!(groups, replace, r"([^ ]+)[ ]+([^ ]+)", "w1 w2", "$2 $1", "w2 w1"); +replace!( + double_dollar, + replace, + r"([^ ]+)[ ]+([^ ]+)", + "w1 w2", + "$2 $$1", + "w2 $1" +); +// replace!(adjacent_index, replace, +// r"([^aeiouy])ies$", "skies", "$1y", "sky"); +replace!( + named, + replace_all, + r"(?P<first>[^ ]+)[ ]+(?P<last>[^ ]+)(?P<space>[ ]*)", + "w1 w2 w3 w4", + "$last $first$space", + "w2 w1 w4 w3" +); +replace!( + trim, + replace_all, + "^[ \t]+|[ \t]+$", + " \t trim me\t \t", + "", + "trim me" +); +replace!(number_hyphen, replace, r"(.)(.)", "ab", "$1-$2", "a-b"); +// replace!(number_underscore, replace, r"(.)(.)", "ab", "$1_$2", "a_b"); +replace!( + simple_expand, + replace_all, + r"([a-z]) ([a-z])", + "a b", + "$2 $1", + "b a" +); +replace!( + literal_dollar1, + replace_all, + r"([a-z]+) ([a-z]+)", + "a b", + "$$1", + "$1" +); +replace!( + literal_dollar2, + replace_all, + r"([a-z]+) ([a-z]+)", + "a b", + "$2 $$c $1", + "b $c a" +); +replace!( + no_expand1, + replace, + r"([^ ]+)[ ]+([^ ]+)", + "w1 w2", + regex::NoExpand("$2 $1"), + "$2 $1" +); +replace!( + no_expand2, + replace, + r"([^ ]+)[ ]+([^ ]+)", + "w1 w2", + regex::NoExpand("$$1"), + "$$1" +); +replace!( + closure_returning_reference, + replace, + r"([0-9]+)", + "age: 26", + |captures: ®ex::Captures<'_>| { captures[1][0..1].to_owned() }, + "age: 2" +); +replace!( + closure_returning_value, + replace, + r"[0-9]+", + "age: 26", + |_captures: ®ex::Captures<'_>| "Z".to_owned(), + "age: Z" +); + +// See https://github.com/rust-lang/regex/issues/314 +replace!( + match_at_start_replace_with_empty, + replace_all, + r"foo", + "foobar", + "", + "bar" +); + +// See https://github.com/rust-lang/regex/issues/393 +replace!(single_empty_match, replace, r"^", "bar", "foo", "foobar"); + +// See https://github.com/rust-lang/regex/issues/399 +replace!( + capture_longest_possible_name, + replace_all, + r"(.)", + "b", + "${1}a $1a", + "ba " +); + +replace!( + impl_string, + replace, + r"[0-9]", + "age: 26", + "Z".to_string(), + "age: Z6" +); +replace!( + impl_string_ref, + replace, + r"[0-9]", + "age: 26", + &"Z".to_string(), + "age: Z6" +); +replace!( + impl_cow_str_borrowed, + replace, + r"[0-9]", + "age: 26", + std::borrow::Cow::<'_, str>::Borrowed("Z"), + "age: Z6" +); +replace!( + impl_cow_str_borrowed_ref, + replace, + r"[0-9]", + "age: 26", + &std::borrow::Cow::<'_, str>::Borrowed("Z"), + "age: Z6" +); +replace!( + impl_cow_str_owned, + replace, + r"[0-9]", + "age: 26", + std::borrow::Cow::<'_, str>::Owned("Z".to_string()), + "age: Z6" +); +replace!( + impl_cow_str_owned_ref, + replace, + r"[0-9]", + "age: 26", + &std::borrow::Cow::<'_, str>::Owned("Z".to_string()), + "age: Z6" +); + +#[test] +fn replacen_no_captures() { + let re = regex::Regex::new(r"[0-9]").unwrap(); + assert_eq!(re.replacen("age: 1234", 2, "Z"), "age: ZZ34"); +} + +#[test] +fn replacen_with_captures() { + let re = regex::Regex::new(r"([0-9])").unwrap(); + assert_eq!(re.replacen("age: 1234", 2, "${1}Z"), "age: 1Z2Z34"); +} diff --git a/third_party/crates/regex-1.13.1/tests/searcher.rs b/third_party/crates/regex-1.13.1/tests/searcher.rs new file mode 100644 index 0000000..f6dae13 --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/searcher.rs @@ -0,0 +1,93 @@ +macro_rules! searcher { + ($name:ident, $re:expr, $haystack:expr) => ( + searcher!($name, $re, $haystack, vec vec![]); + ); + ($name:ident, $re:expr, $haystack:expr, $($steps:expr,)*) => ( + searcher!($name, $re, $haystack, vec vec![$($steps),*]); + ); + ($name:ident, $re:expr, $haystack:expr, $($steps:expr),*) => ( + searcher!($name, $re, $haystack, vec vec![$($steps),*]); + ); + ($name:ident, $re:expr, $haystack:expr, vec $expect_steps:expr) => ( + #[test] + #[allow(unused_imports)] + fn $name() { + use std::str::pattern::{Pattern, Searcher}; + use std::str::pattern::SearchStep::{Match, Reject, Done}; + let re = regex::Regex::new($re).unwrap(); + let mut se = re.into_searcher($haystack); + let mut got_steps = vec![]; + loop { + match se.next() { + Done => break, + step => { got_steps.push(step); } + } + } + assert_eq!(got_steps, $expect_steps); + } + ); +} + +searcher!(searcher_empty_regex_empty_haystack, r"", "", Match(0, 0)); +searcher!( + searcher_empty_regex, + r"", + "ab", + Match(0, 0), + Reject(0, 1), + Match(1, 1), + Reject(1, 2), + Match(2, 2) +); +searcher!(searcher_empty_haystack, r"\d", ""); +searcher!(searcher_one_match, r"\d", "5", Match(0, 1)); +searcher!(searcher_no_match, r"\d", "a", Reject(0, 1)); +searcher!( + searcher_two_adjacent_matches, + r"\d", + "56", + Match(0, 1), + Match(1, 2) +); +searcher!( + searcher_two_non_adjacent_matches, + r"\d", + "5a6", + Match(0, 1), + Reject(1, 2), + Match(2, 3) +); +searcher!(searcher_reject_first, r"\d", "a6", Reject(0, 1), Match(1, 2)); +searcher!( + searcher_one_zero_length_matches, + r"\d*", + "a1b2", + Match(0, 0), // ^ + Reject(0, 1), // a + Match(1, 2), // a1 + Reject(2, 3), // a1b + Match(3, 4), // a1b2 +); +searcher!( + searcher_many_zero_length_matches, + r"\d*", + "a1bbb2", + Match(0, 0), // ^ + Reject(0, 1), // a + Match(1, 2), // a1 + Reject(2, 3), // a1b + Match(3, 3), // a1bb + Reject(3, 4), // a1bb + Match(4, 4), // a1bbb + Reject(4, 5), // a1bbb + Match(5, 6), // a1bbba +); +searcher!( + searcher_unicode, + r".+?", + "Ⅰ1Ⅱ2", + Match(0, 3), + Match(3, 4), + Match(4, 7), + Match(7, 8) +); diff --git a/third_party/crates/regex-1.13.1/tests/suite_bytes.rs b/third_party/crates/regex-1.13.1/tests/suite_bytes.rs new file mode 100644 index 0000000..784b1a4 --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/suite_bytes.rs @@ -0,0 +1,108 @@ +use { + anyhow::Result, + regex::bytes::{Regex, RegexBuilder}, + regex_test::{ + CompiledRegex, Match, RegexTest, Span, TestResult, TestRunner, + }, +}; + +/// Tests the default configuration of the hybrid NFA/DFA. +#[test] +fn default() -> Result<()> { + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "find", "captures"], |test| test.compiles()) + .blacklist_iter(super::BLACKLIST) + .test_iter(crate::suite()?.iter(), compiler) + .assert(); + Ok(()) +} + +fn run_test(re: &Regex, test: &RegexTest) -> TestResult { + match test.additional_name() { + "is_match" => TestResult::matched(re.is_match(test.haystack())), + "find" => TestResult::matches( + re.find_iter(test.haystack()) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: 0, + span: Span { start: m.start(), end: m.end() }, + }), + ), + "captures" => { + let it = re + .captures_iter(test.haystack()) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|caps| testify_captures(&caps)); + TestResult::captures(it) + } + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Converts the given regex test to a closure that searches with a +/// `bytes::Regex`. If the test configuration is unsupported, then a +/// `CompiledRegex` that skips the test is returned. +fn compiler( + test: &RegexTest, + _patterns: &[String], +) -> anyhow::Result<CompiledRegex> { + let skip = Ok(CompiledRegex::skip()); + + // We're only testing bytes::Regex here, which supports one pattern only. + let pattern = match test.regexes().len() { + 1 => &test.regexes()[0], + _ => return skip, + }; + // We only test is_match, find_iter and captures_iter. All of those are + // leftmost searches. + if !matches!(test.search_kind(), regex_test::SearchKind::Leftmost) { + return skip; + } + // The top-level single-pattern regex API always uses leftmost-first. + if !matches!(test.match_kind(), regex_test::MatchKind::LeftmostFirst) { + return skip; + } + // The top-level regex API always runs unanchored searches. ... But we can + // handle tests that are anchored but have only one match. + if test.anchored() && test.match_limit() != Some(1) { + return skip; + } + // We don't support tests with explicit search bounds. We could probably + // support this by using the 'find_at' (and such) APIs. + let bounds = test.bounds(); + if !(bounds.start == 0 && bounds.end == test.haystack().len()) { + return skip; + } + // The bytes::Regex API specifically does not support enabling UTF-8 mode. + // It could I suppose, but currently it does not. That is, it permits + // matches to have offsets that split codepoints. + if test.utf8() { + return skip; + } + // If the test requires Unicode but the Unicode feature isn't enabled, + // skip it. This is a little aggressive, but the test suite doesn't + // have any easy way of communicating which Unicode features are needed. + if test.unicode() && !cfg!(feature = "unicode") { + return skip; + } + let re = RegexBuilder::new(pattern) + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .line_terminator(test.line_terminator()) + .build()?; + Ok(CompiledRegex::compiled(move |test| run_test(&re, test))) +} + +/// Convert `Captures` into the test suite's capture values. +fn testify_captures( + caps: ®ex::bytes::Captures<'_>, +) -> regex_test::Captures { + let spans = caps.iter().map(|group| { + group.map(|m| regex_test::Span { start: m.start(), end: m.end() }) + }); + // This unwrap is OK because we assume our 'caps' represents a match, and + // a match always gives a non-zero number of groups with the first group + // being non-None. + regex_test::Captures::new(0, spans).unwrap() +} diff --git a/third_party/crates/regex-1.13.1/tests/suite_bytes_set.rs b/third_party/crates/regex-1.13.1/tests/suite_bytes_set.rs new file mode 100644 index 0000000..9b75f8d --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/suite_bytes_set.rs @@ -0,0 +1,71 @@ +use { + anyhow::Result, + regex::bytes::{RegexSet, RegexSetBuilder}, + regex_test::{CompiledRegex, RegexTest, TestResult, TestRunner}, +}; + +/// Tests the default configuration of the hybrid NFA/DFA. +#[test] +fn default() -> Result<()> { + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "which"], |test| test.compiles()) + .blacklist_iter(super::BLACKLIST) + .test_iter(crate::suite()?.iter(), compiler) + .assert(); + Ok(()) +} + +fn run_test(re: &RegexSet, test: &RegexTest) -> TestResult { + match test.additional_name() { + "is_match" => TestResult::matched(re.is_match(test.haystack())), + "which" => TestResult::which(re.matches(test.haystack()).iter()), + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Converts the given regex test to a closure that searches with a +/// `bytes::Regex`. If the test configuration is unsupported, then a +/// `CompiledRegex` that skips the test is returned. +fn compiler( + test: &RegexTest, + _patterns: &[String], +) -> anyhow::Result<CompiledRegex> { + let skip = Ok(CompiledRegex::skip()); + + // The top-level RegexSet API only supports "overlapping" semantics. + if !matches!(test.search_kind(), regex_test::SearchKind::Overlapping) { + return skip; + } + // The top-level RegexSet API only supports "all" semantics. + if !matches!(test.match_kind(), regex_test::MatchKind::All) { + return skip; + } + // The top-level RegexSet API always runs unanchored searches. + if test.anchored() { + return skip; + } + // We don't support tests with explicit search bounds. + let bounds = test.bounds(); + if !(bounds.start == 0 && bounds.end == test.haystack().len()) { + return skip; + } + // The bytes::Regex API specifically does not support enabling UTF-8 mode. + // It could I suppose, but currently it does not. That is, it permits + // matches to have offsets that split codepoints. + if test.utf8() { + return skip; + } + // If the test requires Unicode but the Unicode feature isn't enabled, + // skip it. This is a little aggressive, but the test suite doesn't + // have any easy way of communicating which Unicode features are needed. + if test.unicode() && !cfg!(feature = "unicode") { + return skip; + } + let re = RegexSetBuilder::new(test.regexes()) + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .line_terminator(test.line_terminator()) + .build()?; + Ok(CompiledRegex::compiled(move |test| run_test(&re, test))) +} diff --git a/third_party/crates/regex-1.13.1/tests/suite_string.rs b/third_party/crates/regex-1.13.1/tests/suite_string.rs new file mode 100644 index 0000000..2a6d770 --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/suite_string.rs @@ -0,0 +1,113 @@ +use { + anyhow::Result, + regex::{Regex, RegexBuilder}, + regex_test::{ + CompiledRegex, Match, RegexTest, Span, TestResult, TestRunner, + }, +}; + +/// Tests the default configuration of the hybrid NFA/DFA. +#[test] +fn default() -> Result<()> { + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "find", "captures"], |test| test.compiles()) + .blacklist_iter(super::BLACKLIST) + .test_iter(crate::suite()?.iter(), compiler) + .assert(); + Ok(()) +} + +fn run_test(re: &Regex, test: &RegexTest) -> TestResult { + let hay = match std::str::from_utf8(test.haystack()) { + Ok(hay) => hay, + Err(err) => { + return TestResult::fail(&format!( + "haystack is not valid UTF-8: {err}", + )); + } + }; + match test.additional_name() { + "is_match" => TestResult::matched(re.is_match(hay)), + "find" => TestResult::matches( + re.find_iter(hay) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: 0, + span: Span { start: m.start(), end: m.end() }, + }), + ), + "captures" => { + let it = re + .captures_iter(hay) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|caps| testify_captures(&caps)); + TestResult::captures(it) + } + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Converts the given regex test to a closure that searches with a +/// `bytes::Regex`. If the test configuration is unsupported, then a +/// `CompiledRegex` that skips the test is returned. +fn compiler( + test: &RegexTest, + _patterns: &[String], +) -> anyhow::Result<CompiledRegex> { + let skip = Ok(CompiledRegex::skip()); + + // We're only testing bytes::Regex here, which supports one pattern only. + let pattern = match test.regexes().len() { + 1 => &test.regexes()[0], + _ => return skip, + }; + // We only test is_match, find_iter and captures_iter. All of those are + // leftmost searches. + if !matches!(test.search_kind(), regex_test::SearchKind::Leftmost) { + return skip; + } + // The top-level single-pattern regex API always uses leftmost-first. + if !matches!(test.match_kind(), regex_test::MatchKind::LeftmostFirst) { + return skip; + } + // The top-level regex API always runs unanchored searches. ... But we can + // handle tests that are anchored but have only one match. + if test.anchored() && test.match_limit() != Some(1) { + return skip; + } + // We don't support tests with explicit search bounds. We could probably + // support this by using the 'find_at' (and such) APIs. + let bounds = test.bounds(); + if !(bounds.start == 0 && bounds.end == test.haystack().len()) { + return skip; + } + // The Regex API specifically does not support disabling UTF-8 mode because + // it can only search &str which is always valid UTF-8. + if !test.utf8() { + return skip; + } + // If the test requires Unicode but the Unicode feature isn't enabled, + // skip it. This is a little aggressive, but the test suite doesn't + // have any easy way of communicating which Unicode features are needed. + if test.unicode() && !cfg!(feature = "unicode") { + return skip; + } + let re = RegexBuilder::new(pattern) + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .line_terminator(test.line_terminator()) + .build()?; + Ok(CompiledRegex::compiled(move |test| run_test(&re, test))) +} + +/// Convert `Captures` into the test suite's capture values. +fn testify_captures(caps: ®ex::Captures<'_>) -> regex_test::Captures { + let spans = caps.iter().map(|group| { + group.map(|m| regex_test::Span { start: m.start(), end: m.end() }) + }); + // This unwrap is OK because we assume our 'caps' represents a match, and + // a match always gives a non-zero number of groups with the first group + // being non-None. + regex_test::Captures::new(0, spans).unwrap() +} diff --git a/third_party/crates/regex-1.13.1/tests/suite_string_set.rs b/third_party/crates/regex-1.13.1/tests/suite_string_set.rs new file mode 100644 index 0000000..122e39c --- /dev/null +++ b/third_party/crates/regex-1.13.1/tests/suite_string_set.rs @@ -0,0 +1,78 @@ +use { + anyhow::Result, + regex::{RegexSet, RegexSetBuilder}, + regex_test::{CompiledRegex, RegexTest, TestResult, TestRunner}, +}; + +/// Tests the default configuration of the hybrid NFA/DFA. +#[test] +fn default() -> Result<()> { + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "which"], |test| test.compiles()) + .blacklist_iter(super::BLACKLIST) + .test_iter(crate::suite()?.iter(), compiler) + .assert(); + Ok(()) +} + +fn run_test(re: &RegexSet, test: &RegexTest) -> TestResult { + let hay = match std::str::from_utf8(test.haystack()) { + Ok(hay) => hay, + Err(err) => { + return TestResult::fail(&format!( + "haystack is not valid UTF-8: {err}", + )); + } + }; + match test.additional_name() { + "is_match" => TestResult::matched(re.is_match(hay)), + "which" => TestResult::which(re.matches(hay).iter()), + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Converts the given regex test to a closure that searches with a +/// `bytes::Regex`. If the test configuration is unsupported, then a +/// `CompiledRegex` that skips the test is returned. +fn compiler( + test: &RegexTest, + _patterns: &[String], +) -> anyhow::Result<CompiledRegex> { + let skip = Ok(CompiledRegex::skip()); + + // The top-level RegexSet API only supports "overlapping" semantics. + if !matches!(test.search_kind(), regex_test::SearchKind::Overlapping) { + return skip; + } + // The top-level RegexSet API only supports "all" semantics. + if !matches!(test.match_kind(), regex_test::MatchKind::All) { + return skip; + } + // The top-level RegexSet API always runs unanchored searches. + if test.anchored() { + return skip; + } + // We don't support tests with explicit search bounds. + let bounds = test.bounds(); + if !(bounds.start == 0 && bounds.end == test.haystack().len()) { + return skip; + } + // The Regex API specifically does not support disabling UTF-8 mode because + // it can only search &str which is always valid UTF-8. + if !test.utf8() { + return skip; + } + // If the test requires Unicode but the Unicode feature isn't enabled, + // skip it. This is a little aggressive, but the test suite doesn't + // have any easy way of communicating which Unicode features are needed. + if test.unicode() && !cfg!(feature = "unicode") { + return skip; + } + let re = RegexSetBuilder::new(test.regexes()) + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .line_terminator(test.line_terminator()) + .build()?; + Ok(CompiledRegex::compiled(move |test| run_test(&re, test))) +} diff --git a/third_party/crates/regex-automata-0.4.18/.cargo-checksum.json b/third_party/crates/regex-automata-0.4.18/.cargo-checksum.json new file mode 100644 index 0000000..3ea2077 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"73e6fb721d8d4db7ed1a9a9c97aa91728d16a246721660aa6a89853c1c051f9a","Cargo.lock":"950dff7078a38c3e41562992454789a553d7c9ee082e0035e24ca681c72bd7b0","Cargo.toml":"25d528e0bd6dd67c78462d3c7a426698b5d6e702052b25daf7599c0a211315b6","Cargo.toml.orig":"024ca34ebb2bd31f2df219ad258e10abb14b6285970fb6629d583e724dc9630d","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb","README.md":"a7cfb89cd6d4de6b6e8b18e872227e5f1e47d91345e377aa1a75affc18c53aa1","src/dfa/accel.rs":"73f8e0c492a6c669fafbb872020091d6bfa5236503c9c0922aa94fd939fb2b1e","src/dfa/automaton.rs":"a2af61cdfb7f16a8419a25ccb3ae250afe736ff397c7a3101c8a77781d096a9b","src/dfa/dense.rs":"57896339ba77188b6a506c54b1d19fe58fbca380fde6b816d4eade0feddccfee","src/dfa/determinize.rs":"d72dc41a7e93b9289370d2a4e7d8524612be1870283504c703f9c08f9f3b316c","src/dfa/minimize.rs":"b5cadb462b9f24cd4aa7a665e75fb813cd06858a92b8986c9c5ae7fd9a60dfab","src/dfa/mod.rs":"530a1025d516a6df949eee46009acc5ef58c9e6788ec3d76702811734d76212f","src/dfa/onepass.rs":"c7de14acbe402ed8bc249db8c3dfa0000b085054ff980a5415fe68b2457d2615","src/dfa/regex.rs":"567c7a59ca194117986f1818c092b31f825e860fb1b2c55c7de87de97eebb787","src/dfa/remapper.rs":"ca096abc0f8e45c43a2adf3a7743b8857714ae7411a623edea41cc3ce906a169","src/dfa/search.rs":"61dac118b0ab7a1817b605fc4c712b5b6f061f5b917722895ab57cd30a72b847","src/dfa/sparse.rs":"7bceedc5fc41bd981458bd6502422d9ec2fd8d1af3c3647081302581476742eb","src/dfa/special.rs":"1b939ad4addf2efb87fcd1ae67d7818b72540c017d895846ab7968cec267aee1","src/dfa/start.rs":"46b1dbaf8e4518ddddda6bbe596621aae36f8ba694390483a22355d9d799be8e","src/hybrid/dfa.rs":"cd0e71ad86161c9a49c6023d9dde8e07895ad03a9586723c3fb1f1c14bdb7faa","src/hybrid/error.rs":"2bca7eb9ff3859d2b2f5afcf00e618f5671c43e32f5ce8e7ab9de44b906a9422","src/hybrid/id.rs":"a529d45c5a7dd5ed64a471d3fab5a8c6a7aa2bd64bb3a81d5d6f1fcca424d41d","src/hybrid/mod.rs":"ca21e89062bdb5a0998d5cd1bc78609af1f6b795533e5982be969c383ac0463a","src/hybrid/regex.rs":"47815d025526330291f4cd749b4dd79b1122ef208fe6f0a49715c70fc1ea47c8","src/hybrid/search.rs":"5b2b3f2017a4b71164c7b2c42110e5503675bc741661ba2c42761921b0302938","src/lib.rs":"47d562a98f5f50f7cbbffb1f103e277871c7419da05dbb2b0db78dee6e7b4c2e","src/macros.rs":"3e4b39252bfa471fad384160a43f113ebfec7bec46a85d16f006622881dd2081","src/meta/error.rs":"729ec5e2474ed2449fb47f3e0eeb65586ceeed0a6a67f00678f09eb5a46da931","src/meta/limited.rs":"69f399e0e39a0addfe623562dad5e980cd73f9b2b6a5fcb6c71933c9ed7cf27b","src/meta/literal.rs":"2a4e71c5ffdd7b31f7f624a6a8bba3be0cddac1883ddbba6a01a48034a077978","src/meta/mod.rs":"dbd2c56eaf54e901f3f9af6844e7bb1e97b7b024087d887358cf9a7e42c20c23","src/meta/prefix.rs":"2624ca41a929178a9b03c6ff9fd966e5fe67bc44296acad4439930d5fd703bfe","src/meta/regex.rs":"da85acff388ff352f4ddc2ce282c9b63c5581f25f3aa2ca9af6ee43811f9d864","src/meta/reverse_inner.rs":"cae924f308268d58d11139a49da6d478fbf1ac67be5383965bd5d2d992bc1470","src/meta/reverse_suffix.rs":"e8d0bf7688494e98e8965917cb1767f040c739d3b4c5743231e150019404fbfd","src/meta/stopat.rs":"acb6122e17d10a9b1b5e72d6030e6d95748227975bad0ff5cbbcc2587edfa6df","src/meta/strategy.rs":"ae5f22a6435c0e0d99f22a506e578fc95275e98fd0f7fe424ee6474730a09a5f","src/meta/wrappers.rs":"79b1d63d40e4e4a1a59318f06849c97776687be0ec0de924a3dac5c986004f61","src/nfa/mod.rs":"32a0ed46f4a0a9b4b81b53abf4aa7138e2fd456435897495fce74820b980d4d2","src/nfa/thompson/backtrack.rs":"041015ea153c1e485e9cf39ec60d1e51c7ab9e400ecd77cad2078af45775339b","src/nfa/thompson/builder.rs":"7adf6aba69171f6acd47fea0fec85ba589154fead83f2042a1c6fe9486aa4dbd","src/nfa/thompson/compiler.rs":"cc1fbc44f0106049f6c0020ee8beb879415bc2951bb53c9efdf76c8b6c2af816","src/nfa/thompson/error.rs":"12208c44486575f3ac505754e6559e0b93cac09351a9720ff63cd6fd548ba63d","src/nfa/thompson/literal_trie.rs":"3b5cf36842a31f8b50d820835e3959f878c0dedce0f17932bca449e1b6198651","src/nfa/thompson/map.rs":"fcd17ce7359b5179ef2e809fc9152dfa0b6c61d3d849d8c502497e1d0d8b0fa9","src/nfa/thompson/mod.rs":"dddbbd6f0e7076f369dd12a21aea4eb7e81e9c037d115201871e278cceed0a76","src/nfa/thompson/nfa.rs":"3ab46b912ece5218ba95e29e0c169b23a869796c7bcb138385c562a57c62a2a3","src/nfa/thompson/pikevm.rs":"230f879f05d7d9f868344407064d1e7a05131a13c0c90ab59d5a20f15af0ec56","src/nfa/thompson/range_trie.rs":"2304cab5cd580ca10961fbb14c75c163c2b7fcd29040622190d36f7935b446d3","src/util/alphabet.rs":"4f94d317459b43c7748e7a963935d61632fb70bd7c09dd9e536e354586d21df6","src/util/captures.rs":"81e48d060fc9bea41f98157676a0b262da54f6b1358be41e413c9e3e960f0155","src/util/determinize/mod.rs":"82f34f4e408aaf06f8e04a38a6c9ce0abdcc20b61581e29283ef099f84bc67a1","src/util/determinize/state.rs":"a850af545b7d0bd706f0bf72fdba504b6efdeea181763657109e10fef53aa88d","src/util/empty.rs":"13ec7d6cbd1520db5b4c1dae294f4419fa88d39d2bfc16f4ef258473d609f91c","src/util/escape.rs":"2c8275c56b75018a0d8f8363b5827eb620f2cb52e2e919d8dace2846e7e0cf3c","src/util/int.rs":"b863a62f8ba1edf24858416fc01f15b38bee7af2494ebeb037e1acbf0319415e","src/util/interpolate.rs":"1d716d26ed80beb0ba6526e1fb75fdb009b95122bf0907045237c7e9e4bfbe88","src/util/iter.rs":"d61335dc6b99b134d75c1b75e01f88e2dfe1174d48a36ac5a5e5efbc4c6114e2","src/util/lazy.rs":"116ff2eed0bb6d2aa574c812c74f732fb06c91beb1667e0e5d2a3210023d7db5","src/util/look.rs":"49026d8bcb6f9229b2d462e363eb6c6dc9d1e17de6acb190105b2d71ca5a9490","src/util/memchr.rs":"573109ce4983907083ae0b29a084a324b9b53da369b4d96f7f3a21fd5c8eb5c9","src/util/mod.rs":"6c828a493f0f88c8b515aee4f8faf91ba653eb07e8fc3c23c0524553410803f9","src/util/pool.rs":"6aaebecd38d4b4ea9d70468b45f67dffa479737f60fb1947bfe9dd8a79c787f3","src/util/prefilter/aho_corasick.rs":"0c1efb3d69138e5200ba78b78f4627173403f8419c1a88dd2833cb01dec63b41","src/util/prefilter/byteset.rs":"a0c737b19e316eff2f183e27866bf62395964a2c62252b6d81ba6d003e754d97","src/util/prefilter/memchr.rs":"145e443a5ee305c736da4bb36c8b0b37877903fd5f1ee3b9014bba0dba9ffa2c","src/util/prefilter/memmem.rs":"c10ad4f1a48919c06378a8babd6669d697bb84d3ea1a5d939278bf31f8eb20b4","src/util/prefilter/mod.rs":"bcdbca12a3f2879605cf1d3216338994040fa812a00c9ad140a1bc522c60fb20","src/util/prefilter/teddy.rs":"4e4c446e9ba349c9042e11ddf40c999af19465764131bd7f5a65f5ec9346a7cc","src/util/primitives.rs":"8a9cc19ef2e1ab183943cdc2d2f095b02252476e32b7e9fff4a06a251749b068","src/util/search.rs":"74157bd77eb6f80e8ab5952ef21e8fb2d3452c1817ec671a7a3574fbddae1ae7","src/util/sparse_set.rs":"acbe7197f5e5fc95b45f54ba1e4e24f21226714af44a391e507d52b7b23cbaf6","src/util/start.rs":"1ab2dec7c452ae943118cd1c3b6becc84afba1fbb8b6894d81ef7d65141d95ab","src/util/syntax.rs":"cff4712c95fc5f94063e6e11a51b42d7678d5f5b82b492f11fcbb3928c3d6c8d","src/util/unicode_data/mod.rs":"54c3e10bbc393e9881bfac3295815b160f59e69e2056bc29ee7cf0addd8e3cf7","src/util/unicode_data/perl_word.rs":"30f073baae28ea34c373c7778c00f20c1621c3e644404eff031f7d1cc8e9c9e2","src/util/utf8.rs":"a02a559f0ec4013aa8bbc1a2d8717cfa0c82ed81577a4366fbbd8ef8660264fc","src/util/wire.rs":"f4eb8517d6d7ff165b39e829ce7ec7075ee40550878334456887abde486c5885","tests/dfa/api.rs":"c6ddbca1177c377a42bac1e19e79dc8c840a7a0af2042e6c3c08e46ba1a288fe","tests/dfa/mod.rs":"924d8fff500b9b7b140082623023e78007058a87323151cd8e361462945e4f16","tests/dfa/onepass/mod.rs":"33ca171775a61cf66407fb6a5fb2b5b940fdbc55378478bc7be71289d8d55ff5","tests/dfa/onepass/regression.rs":"7678ef4d065103cf822d548c24e323a605de502abb9c81b22ebf0e9f9e8eeea4","tests/dfa/onepass/suite.rs":"f6a9cba40773db81fcd82ab605ba18ca415908f9857845e7621d47888cb67c91","tests/dfa/regression.rs":"8ade68fc4c75385883a4318592caa2b10ef868b9df0de53693812101c91abd8b","tests/dfa/suite.rs":"26cfc5a89a2ceda338d15e9cde0aeb6a050ec4f751fb29b017eac54a9d9a0074","tests/fuzz/dense.rs":"3e1099a0cce61e85abc0ad81bc592e85f497f159ef0e5d1d32bac1936aa6f20c","tests/fuzz/sparse.rs":"ba61db4927ab28953037a4b20317399c86d01b4d774e46c020ade19029215e25","tests/gen/dense/mod.rs":"5ae1cfb46212a674118ada2f66f37b25188e84643d406b95eb4665d722344262","tests/gen/dense/multi_pattern_v2.rs":"29b1e9a799adecbdbe7cd05e9748f664c2b915b10b1d2f5d36cfb6453826d1d2","tests/gen/mod.rs":"043773510e02f51def43ee0c2b8b867c53ecc8638c8a9233b2ac098de9c3ac1e","tests/gen/sparse/mod.rs":"5ae1cfb46212a674118ada2f66f37b25188e84643d406b95eb4665d722344262","tests/gen/sparse/multi_pattern_v2.rs":"e00fb2a510a215460aab84573196b1f51bb65884ff494c2382534c04f6fdbfe9","tests/hybrid/api.rs":"2b9216aca4b17cb8f56781ff72059c903c362f45b69b5043809e73c87eade047","tests/hybrid/mod.rs":"4856a49a4d9b5e9e079c2719a5e75c32408b37e9b76cbdea057b388a3537af6d","tests/hybrid/suite.rs":"d49081a07b13e923c9d31c211942439c015b970b2b9d2f38fd49935803e22bb1","tests/lib.rs":"9775b3c62fb338ea5c1bd3513a6589eff4b5c8d35c599439d9363dbf98c6f8d4","tests/meta/mod.rs":"d08f4ecb8ec243be584944c9602af1ed3a48a8732dd11cd573b0d1d182171303","tests/meta/suite.rs":"7cafd709c61481f2267de671768f880f8bbd4740f4cb523a449481abc80aa08a","tests/nfa/mod.rs":"49055c358e38d97e42acb1602c671f97dddf24cafe089490f0e79ed208d74d9b","tests/nfa/thompson/backtrack/mod.rs":"33ca171775a61cf66407fb6a5fb2b5b940fdbc55378478bc7be71289d8d55ff5","tests/nfa/thompson/backtrack/regression.rs":"8529d6652716b8e1b56a39d6e55e243373f23a93c267b1d6b0dd79a4e987ad2c","tests/nfa/thompson/backtrack/suite.rs":"c14b12ad3292b103d7f5be69c297c737adbeea65379cee12a75596601312c430","tests/nfa/thompson/mod.rs":"de9f5bcea1a8d1f03c85c55ad8c0747877d69e344fcd6c6886b0a402f0661291","tests/nfa/thompson/pikevm/mod.rs":"33ca171775a61cf66407fb6a5fb2b5b940fdbc55378478bc7be71289d8d55ff5","tests/nfa/thompson/pikevm/regression.rs":"23f352d2e1a2485c88c13cacfa0c5b86f35dab336f26abc7a50fa465f7ecf4bb","tests/nfa/thompson/pikevm/suite.rs":"cf21a58532f3dc8fd76df715093d1a9333b0c4072261b63c48ac8c86ca31fe25"},"package":"ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"} \ No newline at end of file diff --git a/third_party/crates/regex-automata-0.4.18/.cargo_vcs_info.json b/third_party/crates/regex-automata-0.4.18/.cargo_vcs_info.json new file mode 100644 index 0000000..f8672c3 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "e6ad938198e68339a593f85d228f2d44cb54718f" + }, + "path_in_vcs": "regex-automata" +} \ No newline at end of file diff --git a/third_party/crates/regex-automata-0.4.18/BUILD.bazel b/third_party/crates/regex-automata-0.4.18/BUILD.bazel new file mode 100644 index 0000000..d9aa82c --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/BUILD.bazel @@ -0,0 +1,139 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_library( + name = "regex_automata", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "alloc", + "dfa-onepass", + "hybrid", + "meta", + "nfa-backtrack", + "nfa-pikevm", + "nfa-thompson", + "perf-inline", + "perf-literal", + "perf-literal-multisubstring", + "perf-literal-substring", + "std", + "syntax", + "unicode", + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", + "unicode-word-boundary", + ], + crate_root = "src/lib.rs", + edition = "2021", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=regex-automata", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.4.18", + deps = [ + "//third_party/crates/aho-corasick-1.1.5:aho_corasick", + "//third_party/crates/memchr-2.8.3:memchr", + "//third_party/crates/regex-syntax-0.8.11:regex_syntax", + ], +) diff --git a/third_party/crates/regex-automata-0.4.18/Cargo.lock b/third_party/crates/regex-automata-0.4.18/Cargo.lock new file mode 100644 index 0000000..707f9cf --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/Cargo.lock @@ -0,0 +1,371 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "log", + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bstr" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" +dependencies = [ + "memchr", + "serde_core", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "doc-comment" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "termcolor", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +dependencies = [ + "log", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quickcheck" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b" +dependencies = [ + "rand", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "regex-automata" +version = "0.4.18" +dependencies = [ + "aho-corasick", + "anyhow", + "bstr", + "doc-comment", + "env_logger", + "log", + "memchr", + "quickcheck", + "regex-syntax", + "regex-test", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "regex-test" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da40f0939bc4c598b4326abdbb363a8987aa43d0526e5624aefcf3ed90344e62" +dependencies = [ + "anyhow", + "bstr", + "serde", + "toml", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] diff --git a/third_party/crates/regex-automata-0.4.18/Cargo.toml b/third_party/crates/regex-automata-0.4.18/Cargo.toml new file mode 100644 index 0000000..3af345b --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/Cargo.toml @@ -0,0 +1,216 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.65" +name = "regex-automata" +version = "0.4.18" +authors = [ + "The Rust Project Developers", + "Andrew Gallant <jamslam@gmail.com>", +] +build = false +include = [ + "/CHANGELOG.md", + "/Cargo.toml", + "/LICENSE-MIT", + "/LICENSE-APACHE", + "/README.md", + "/UNICODE.md", + "bench/README.md", + "src/**/*.rs", + "testdata/**.toml", + "tests/**/*.rs", + "bench/**/*.rs", + "benches/**/*.rs", + "LICENSE-UNICODE", + "!/tests/fuzz/mod.rs", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Automata construction and matching using regular expressions." +homepage = "https://github.com/rust-lang/regex/tree/master/regex-automata" +documentation = "https://docs.rs/regex-automata" +readme = "README.md" +keywords = [ + "regex", + "dfa", + "automata", + "automaton", + "nfa", +] +categories = ["text-processing"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/regex" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs_regex", +] + +[features] +alloc = [] +default = [ + "std", + "syntax", + "perf", + "unicode", + "meta", + "nfa", + "dfa", + "hybrid", +] +dfa = [ + "dfa-build", + "dfa-search", + "dfa-onepass", +] +dfa-build = [ + "nfa-thompson", + "dfa-search", +] +dfa-onepass = ["nfa-thompson"] +dfa-search = [] +hybrid = [ + "alloc", + "nfa-thompson", +] +internal-instrument = ["internal-instrument-pikevm"] +internal-instrument-pikevm = [ + "logging", + "std", +] +logging = [ + "dep:log", + "aho-corasick?/logging", + "memchr?/logging", +] +meta = [ + "syntax", + "nfa-pikevm", +] +nfa = [ + "nfa-thompson", + "nfa-pikevm", + "nfa-backtrack", +] +nfa-backtrack = ["nfa-thompson"] +nfa-pikevm = ["nfa-thompson"] +nfa-thompson = ["alloc"] +perf = [ + "perf-inline", + "perf-literal", +] +perf-inline = [] +perf-literal = [ + "perf-literal-substring", + "perf-literal-multisubstring", +] +perf-literal-multisubstring = ["dep:aho-corasick"] +perf-literal-substring = [ + "aho-corasick?/perf-literal", + "dep:memchr", +] +std = [ + "regex-syntax?/std", + "memchr?/std", + "aho-corasick?/std", + "alloc", +] +syntax = [ + "dep:regex-syntax", + "alloc", +] +unicode = [ + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", + "unicode-word-boundary", + "regex-syntax?/unicode", +] +unicode-age = ["regex-syntax?/unicode-age"] +unicode-bool = ["regex-syntax?/unicode-bool"] +unicode-case = ["regex-syntax?/unicode-case"] +unicode-gencat = ["regex-syntax?/unicode-gencat"] +unicode-perl = ["regex-syntax?/unicode-perl"] +unicode-script = ["regex-syntax?/unicode-script"] +unicode-segment = ["regex-syntax?/unicode-segment"] +unicode-word-boundary = [] + +[lib] +name = "regex_automata" +path = "src/lib.rs" +bench = false + +[[test]] +name = "integration" +path = "tests/lib.rs" + +[dependencies.aho-corasick] +version = "1.0.0" +optional = true +default-features = false + +[dependencies.log] +version = "0.4.14" +optional = true + +[dependencies.memchr] +version = "2.6.0" +optional = true +default-features = false + +[dependencies.regex-syntax] +version = "0.8.5" +optional = true +default-features = false + +[dev-dependencies.anyhow] +version = "1.0.69" + +[dev-dependencies.bstr] +version = "1.3.0" +features = ["std"] +default-features = false + +[dev-dependencies.doc-comment] +version = "0.3.3" + +[dev-dependencies.env_logger] +version = "0.9.3" +features = [ + "atty", + "humantime", + "termcolor", +] +default-features = false + +[dev-dependencies.quickcheck] +version = "1.0.3" +default-features = false + +[dev-dependencies.regex-test] +version = "0.1.0" + +[lints.rust.unexpected_cfgs] +level = "allow" +priority = 0 +check-cfg = ["cfg(docsrs_regex)"] diff --git a/third_party/crates/regex-automata-0.4.18/Cargo.toml.orig b/third_party/crates/regex-automata-0.4.18/Cargo.toml.orig new file mode 100644 index 0000000..afba6f0 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/Cargo.toml.orig @@ -0,0 +1,135 @@ +[package] +name = "regex-automata" +version = "0.4.18" #:version +authors = ["The Rust Project Developers", "Andrew Gallant <jamslam@gmail.com>"] +description = "Automata construction and matching using regular expressions." +documentation = "https://docs.rs/regex-automata" +homepage = "https://github.com/rust-lang/regex/tree/master/regex-automata" +repository = "https://github.com/rust-lang/regex" +readme = "README.md" +keywords = ["regex", "dfa", "automata", "automaton", "nfa"] +license = "MIT OR Apache-2.0" +categories = ["text-processing"] +edition = "2021" +autoexamples = false +rust-version = "1.65" +include.workspace = true + +[lib] +bench = false + +# This crate has many many many features. See the crate docs for a description +# of each and when you might want to use them. +[features] +default = ["std", "syntax", "perf", "unicode", "meta", "nfa", "dfa", "hybrid"] +std = ["regex-syntax?/std", "memchr?/std", "aho-corasick?/std", "alloc"] +alloc = [] +logging = ["dep:log", "aho-corasick?/logging", "memchr?/logging"] + +syntax = ["dep:regex-syntax", "alloc"] + +meta = ["syntax", "nfa-pikevm"] + +nfa = ["nfa-thompson", "nfa-pikevm", "nfa-backtrack"] +nfa-thompson = ["alloc"] +nfa-pikevm = ["nfa-thompson"] +nfa-backtrack = ["nfa-thompson"] + +dfa = ["dfa-build", "dfa-search", "dfa-onepass"] +dfa-build = ["nfa-thompson", "dfa-search"] +dfa-search = [] +dfa-onepass = ["nfa-thompson"] + +hybrid = ["alloc", "nfa-thompson"] + +perf = ["perf-inline", "perf-literal"] +perf-inline = [] +perf-literal = ["perf-literal-substring", "perf-literal-multisubstring"] +perf-literal-substring = ["aho-corasick?/perf-literal", "dep:memchr"] +perf-literal-multisubstring = ["dep:aho-corasick"] + +# Enables all Unicode features. This expands if new Unicode features are added. +unicode = [ + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", + "unicode-word-boundary", + "regex-syntax?/unicode", +] +# Enables use of the `Age` property, e.g., `\p{Age:3.0}`. +unicode-age = ["regex-syntax?/unicode-age"] +# Enables use of a smattering of boolean properties, e.g., `\p{Emoji}`. +unicode-bool = ["regex-syntax?/unicode-bool"] +# Enables Unicode-aware case insensitive matching, e.g., `(?i)β`. +unicode-case = ["regex-syntax?/unicode-case"] +# Enables Unicode general categories, e.g., `\p{Letter}` or `\pL`. +unicode-gencat = ["regex-syntax?/unicode-gencat"] +# Enables Unicode-aware Perl classes corresponding to `\w`, `\s` and `\d`. +unicode-perl = ["regex-syntax?/unicode-perl"] +# Enables Unicode scripts and script extensions, e.g., `\p{Greek}`. +unicode-script = ["regex-syntax?/unicode-script"] +# Enables Unicode segmentation properties, e.g., `\p{gcb=Extend}`. +unicode-segment = ["regex-syntax?/unicode-segment"] +# Enables Unicode word boundary support. If this is enabled with unicode-perl, +# then data tables from regex-syntax are used. Otherwise, a new data table +# inside regex-automata will be included. +unicode-word-boundary = [] + +# These are strictly internal features that may be removed or changed in +# non-compatible ways. +internal-instrument = ["internal-instrument-pikevm"] +internal-instrument-pikevm = ["logging", "std"] + +[dependencies] +aho-corasick = { version = "1.0.0", optional = true, default-features = false } +log = { version = "0.4.14", optional = true } +memchr = { version = "2.6.0", optional = true, default-features = false } +regex-syntax = { path = "../regex-syntax", version = "0.8.5", optional = true, default-features = false } + +[dev-dependencies] +anyhow = "1.0.69" +bstr = { version = "1.3.0", default-features = false, features = ["std"] } +doc-comment = "0.3.3" +quickcheck = { version = "1.0.3", default-features = false } +regex-test = { path = "../regex-test", version = "0.1.0" } + +[dev-dependencies.env_logger] +version = "0.9.3" +default-features = false +features = ["atty", "humantime", "termcolor"] + +# We put these tests here because they are written primarily against the +# regex-automata API, and in particular use regex-automata features for +# conditional compilation. If we moved these up as tests on 'regex' proper, +# then we'd need to duplicate regex-automata's complex features on 'regex' too, +# which I really do not want to do. +[[test]] +path = "tests/lib.rs" +name = "integration" + +[package.metadata.docs.rs] +# We want to document all features. +all-features = true +# Since this crate's feature setup is pretty complicated, it is worth opting +# into a nightly unstable option to show the features that need to be enabled +# for public API items. To do that, we set 'docsrs_regex', and when that's +# enabled, we enable the 'doc_cfg' feature. +# +# To test this locally, run: +# +# RUSTDOCFLAGS="--cfg docsrs_regex" cargo +nightly doc --all-features +# +# Note that we use `docsrs_regex` instead of the more standard `docsrs` because +# other crates use that same `cfg` knob. And since we are enabling a nightly +# feature, they sometimes break. By using our "own" `cfg` knob, we are closer +# to being masters of our own destiny. +rustdoc-args = ["--cfg", "docsrs_regex"] + +# This squashes the (AFAIK) erroneous warning that `docsrs_regex` is not a +# valid `cfg` knob. +[lints.rust] +unexpected_cfgs = { level = "allow", check-cfg = ['cfg(docsrs_regex)'] } diff --git a/third_party/crates/regex-automata-0.4.18/LICENSE-APACHE b/third_party/crates/regex-automata-0.4.18/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/crates/regex-automata-0.4.18/LICENSE-MIT b/third_party/crates/regex-automata-0.4.18/LICENSE-MIT new file mode 100644 index 0000000..39d4bdb --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2014 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third_party/crates/regex-automata-0.4.18/README.md b/third_party/crates/regex-automata-0.4.18/README.md new file mode 100644 index 0000000..cb6e86c --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/README.md @@ -0,0 +1,117 @@ +regex-automata +============== +This crate exposes a variety of regex engines used by the `regex` crate. +It provides a vast, sprawling and "expert" level API to each regex engine. +The regex engines provided by this crate focus heavily on finite automata +implementations and specifically guarantee worst case `O(m * n)` time +complexity for all searches. (Where `m ~ len(regex)` and `n ~ len(haystack)`.) + +[![Build status](https://github.com/rust-lang/regex/workflows/ci/badge.svg)](https://github.com/rust-lang/regex/actions) +[![Crates.io](https://img.shields.io/crates/v/regex-automata.svg)](https://crates.io/crates/regex-automata) + + +### Documentation + +https://docs.rs/regex-automata + + +### Example + +This example shows how to search for matches of multiple regexes, where each +regex uses the same capture group names to parse different key-value formats. + +```rust +use regex_automata::{meta::Regex, PatternID}; + +let re = Regex::new_many(&[ + r#"(?m)^(?<key>[[:word:]]+)=(?<val>[[:word:]]+)$"#, + r#"(?m)^(?<key>[[:word:]]+)="(?<val>[^"]+)"$"#, + r#"(?m)^(?<key>[[:word:]]+)='(?<val>[^']+)'$"#, + r#"(?m)^(?<key>[[:word:]]+):\s*(?<val>[[:word:]]+)$"#, +]).unwrap(); +let hay = r#" +best_album="Blow Your Face Out" +best_quote='"then as it was, then again it will be"' +best_year=1973 +best_simpsons_episode: HOMR +"#; +let mut kvs = vec![]; +for caps in re.captures_iter(hay) { + // N.B. One could use capture indices '1' and '2' here + // as well. Capture indices are local to each pattern. + // (Just like names are.) + let key = &hay[caps.get_group_by_name("key").unwrap()]; + let val = &hay[caps.get_group_by_name("val").unwrap()]; + kvs.push((key, val)); +} +assert_eq!(kvs, vec![ + ("best_album", "Blow Your Face Out"), + ("best_quote", "\"then as it was, then again it will be\""), + ("best_year", "1973"), + ("best_simpsons_episode", "HOMR"), +]); +``` + + +### Safety + +**I welcome audits of `unsafe` code.** + +This crate tries to be extremely conservative in its use of `unsafe`, but does +use it in a few spots. In general, I am very open to removing uses of `unsafe` +if it doesn't result in measurable performance regressions and doesn't result +in significantly more complex code. + +Below is an outline of how `unsafe` is used in this crate. + +* `util::pool::Pool` makes use of `unsafe` to implement a fast path for +accessing an element of the pool. The fast path applies to the first thread +that uses the pool. In effect, the fast path is fast because it avoids a mutex +lock. `unsafe` is also used in the no-std version of `Pool` to implement a spin +lock for synchronization. +* `util::lazy::Lazy` uses `unsafe` to implement a variant of +`once_cell::sync::Lazy` that works in no-std environments. A no-std no-alloc +implementation is also provided that requires use of `unsafe`. +* The `dfa` module makes extensive use of `unsafe` to support zero-copy +deserialization of DFAs. The high level problem is that you need to get from +`&[u8]` to the internal representation of a DFA without doing any copies. +This is required for support in no-std no-alloc environments. It also makes +deserialization extremely cheap. +* The `dfa` and `hybrid` modules use `unsafe` to explicitly elide bounds checks +in the core search loops. This makes the codegen tighter and typically leads to +consistent 5-10% performance improvements on some workloads. + +In general, the above reflect the only uses of `unsafe` throughout the entire +`regex` crate. At present, there are no plans to meaningfully expand the use +of `unsafe`. With that said, one thing folks have been asking for is cheap +deserialization of a `regex::Regex`. My sense is that this feature will require +a lot more `unsafe` in places to support zero-copy deserialization. It is +unclear at this point whether this will be pursued. + + +### Motivation + +I started out building this crate because I wanted to re-work the `regex` +crate internals to make it more amenable to optimizations. It turns out that +there are a lot of different ways to build regex engines and even more ways to +compose them. Moreover, heuristic literal optimizations are often tricky to +get correct, but the fruit they bear is attractive. All of these things were +difficult to expand upon without risking the introduction of more bugs. So I +decided to tear things down and start fresh. + +In the course of doing so, I ended up designing strong boundaries between each +component so that each component could be reasoned and tested independently. +This also made it somewhat natural to expose the components as a library unto +itself. Namely, folks have been asking for more capabilities in the regex +crate for a long time, but these capabilities usually come with additional API +complexity that I didn't want to introduce in the `regex` crate proper. But +exposing them in an "expert" level crate like `regex-automata` seemed quite +fine. + +In the end, I do still somewhat consider this crate an experiment. It is +unclear whether the strong boundaries between components will be an impediment +to ongoing development or not. De-coupling tends to lead to slower development +in my experience, and when you mix in the added cost of not introducing +breaking changes all the time, things can get quite complicated. But, I +don't think anyone has ever release the internals of a regex engine as a +library before. So it will be interesting to see how it plays out! diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/accel.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/accel.rs new file mode 100644 index 0000000..47c8460 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/accel.rs @@ -0,0 +1,517 @@ +// This module defines some core types for dealing with accelerated DFA states. +// Briefly, a DFA state can be "accelerated" if all of its transitions except +// for a few loop back to itself. This directly implies that the only way out +// of such a state is if a byte corresponding to one of those non-loopback +// transitions is found. Such states are often found in simple repetitions in +// non-Unicode regexes. For example, consider '(?-u)[^a]+a'. We can look at its +// DFA with regex-cli: +// +// $ regex-cli debug dense dfa -p '(?-u)[^a]+a' -BbC --no-table +// D 000000: +// Q 000001: +// *000002: +// A 000003: \x00-` => 3, a => 8, b-\xFF => 3 +// A 000004: \x00-` => 4, a => 7, b-\xFF => 4 +// 000005: \x00-` => 4, b-\xFF => 4 +// 000006: \x00-` => 3, a => 6, b-\xFF => 3 +// 000007: \x00-\xFF => 2, EOI => 2 +// 000008: \x00-\xFF => 2, EOI => 2 +// +// In particular, state 3 is accelerated (shown via the 'A' indicator) since +// the only way to leave that state once entered is to see an 'a' byte. If +// there is a long run of non-'a' bytes, then using something like 'memchr' +// to find the next 'a' byte can be significantly faster than just using the +// standard byte-at-a-time state machine. +// +// Unfortunately, this optimization rarely applies when Unicode is enabled. +// For example, patterns like '[^a]' don't actually match any byte that isn't +// 'a', but rather, any UTF-8 encoding of a Unicode scalar value that isn't +// 'a'. This makes the state machine much more complex---far beyond a single +// state---and removes the ability to easily accelerate it. (Because if the +// machine sees a non-UTF-8 sequence, then the machine won't match through it.) +// +// In practice, we only consider accelerating states that have 3 or fewer +// non-loop transitions. At a certain point, you get diminishing returns, but +// also because that's what the memchr crate supports. The structures below +// hard-code this assumption and provide (de)serialization APIs for use inside +// a DFA. +// +// And finally, note that there is some trickery involved in making it very +// fast to not only check whether a state is accelerated at search time, but +// also to access the bytes to search for to implement the acceleration itself. +// dfa/special.rs provides more detail, but the short story is that all +// accelerated states appear contiguously in a DFA. This means we can represent +// the ID space of all accelerated DFA states with a single range. So given +// a state ID, we can determine whether it's accelerated via +// +// min_accel_id <= id <= max_accel_id +// +// And find its corresponding accelerator with: +// +// accels.get((id - min_accel_id) / dfa_stride) + +#[cfg(feature = "dfa-build")] +use alloc::{vec, vec::Vec}; + +use crate::util::{ + int::Pointer, + memchr, + wire::{self, DeserializeError, Endian, SerializeError}, +}; + +/// The base type used to represent a collection of accelerators. +/// +/// While an `Accel` is represented as a fixed size array of bytes, a +/// *collection* of `Accel`s (called `Accels`) is represented internally as a +/// slice of u32. While it's a bit unnatural to do this and costs us a bit of +/// fairly low-risk not-safe code, it lets us remove the need for a second type +/// parameter in the definition of dense::DFA. (Which really wants everything +/// to be a slice of u32.) +type AccelTy = u32; + +/// The size of the unit of representation for accelerators. +/// +/// ACCEL_CAP *must* be a multiple of this size. +const ACCEL_TY_SIZE: usize = core::mem::size_of::<AccelTy>(); + +/// The maximum length in bytes that a single Accel can be. This is distinct +/// from the capacity of an accelerator in that the length represents only the +/// bytes that should be read. +const ACCEL_LEN: usize = 4; + +/// The capacity of each accelerator, in bytes. We set this to 8 since it's a +/// multiple of 4 (our ID size) and because it gives us a little wiggle room +/// if we want to support more accel bytes in the future without a breaking +/// change. +/// +/// This MUST be a multiple of ACCEL_TY_SIZE. +const ACCEL_CAP: usize = 8; + +/// Search for between 1 and 3 needle bytes in the given haystack, starting the +/// search at the given position. If `needles` has a length other than 1-3, +/// then this panics. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn find_fwd( + needles: &[u8], + haystack: &[u8], + at: usize, +) -> Option<usize> { + let bs = needles; + let i = match needles.len() { + 1 => memchr::memchr(bs[0], &haystack[at..])?, + 2 => memchr::memchr2(bs[0], bs[1], &haystack[at..])?, + 3 => memchr::memchr3(bs[0], bs[1], bs[2], &haystack[at..])?, + 0 => panic!("cannot find with empty needles"), + n => panic!("invalid needles length: {n}"), + }; + Some(at + i) +} + +/// Search for between 1 and 3 needle bytes in the given haystack in reverse, +/// starting the search at the given position. If `needles` has a length other +/// than 1-3, then this panics. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn find_rev( + needles: &[u8], + haystack: &[u8], + at: usize, +) -> Option<usize> { + let bs = needles; + match needles.len() { + 1 => memchr::memrchr(bs[0], &haystack[..at]), + 2 => memchr::memrchr2(bs[0], bs[1], &haystack[..at]), + 3 => memchr::memrchr3(bs[0], bs[1], bs[2], &haystack[..at]), + 0 => panic!("cannot find with empty needles"), + n => panic!("invalid needles length: {n}"), + } +} + +/// Represents the accelerators for all accelerated states in a dense DFA. +/// +/// The `A` type parameter represents the type of the underlying bytes. +/// Generally, this is either `&[AccelTy]` or `Vec<AccelTy>`. +#[derive(Clone)] +pub(crate) struct Accels<A> { + /// A length prefixed slice of contiguous accelerators. See the top comment + /// in this module for more details on how we can jump from a DFA's state + /// ID to an accelerator in this list. + /// + /// The first 4 bytes always correspond to the number of accelerators + /// that follow. + accels: A, +} + +#[cfg(feature = "dfa-build")] +impl Accels<Vec<AccelTy>> { + /// Create an empty sequence of accelerators for a DFA. + pub fn empty() -> Accels<Vec<AccelTy>> { + Accels { accels: vec![0] } + } + + /// Add an accelerator to this sequence. + /// + /// This adds to the accelerator to the end of the sequence and therefore + /// should be done in correspondence with its state in the DFA. + /// + /// This panics if this results in more accelerators than AccelTy::MAX. + pub fn add(&mut self, accel: Accel) { + self.accels.extend_from_slice(&accel.as_accel_tys()); + let len = self.len(); + self.set_len(len + 1); + } + + /// Set the number of accelerators in this sequence, which is encoded in + /// the first 4 bytes of the underlying bytes. + fn set_len(&mut self, new_len: usize) { + // The only way an accelerator gets added is if a state exists for + // it, and if a state exists, then its index is guaranteed to be + // representable by a AccelTy by virtue of the guarantees provided by + // StateID. + let new_len = AccelTy::try_from(new_len).unwrap(); + self.accels[0] = new_len; + } +} + +impl<'a> Accels<&'a [AccelTy]> { + /// Deserialize a sequence of accelerators from the given bytes. If there + /// was a problem deserializing, then an error is returned. + /// + /// This is guaranteed to run in constant time. This does not guarantee + /// that every accelerator in the returned collection is valid. Thus, + /// accessing one may panic, or not-safe code that relies on accelerators + /// being correct my result in UB. + /// + /// Callers may check the validity of every accelerator with the `validate` + /// method. + pub fn from_bytes_unchecked( + mut slice: &'a [u8], + ) -> Result<(Accels<&'a [AccelTy]>, usize), DeserializeError> { + let slice_start = slice.as_ptr().as_usize(); + + let (accel_len, _) = + wire::try_read_u32_as_usize(slice, "accelerators length")?; + // The accelerator length is part of the accel_tys slice that + // we deserialize. This is perhaps a bit idiosyncratic. It would + // probably be better to split out the length into a real field. + + let accel_tys_len = wire::add( + wire::mul(accel_len, 2, "total number of accelerator accel_tys")?, + 1, + "total number of accel_tys", + )?; + let accel_tys_bytes_len = wire::mul( + ACCEL_TY_SIZE, + accel_tys_len, + "total number of bytes in accelerators", + )?; + wire::check_slice_len(slice, accel_tys_bytes_len, "accelerators")?; + wire::check_alignment::<AccelTy>(slice)?; + let accel_tys = &slice[..accel_tys_bytes_len]; + slice = &slice[accel_tys_bytes_len..]; + // SAFETY: We've checked the length and alignment above, and since + // slice is just bytes and AccelTy is just a u32, we can safely cast to + // a slice of &[AccelTy]. + let accels = unsafe { + core::slice::from_raw_parts( + accel_tys.as_ptr().cast::<AccelTy>(), + accel_tys_len, + ) + }; + Ok((Accels { accels }, slice.as_ptr().as_usize() - slice_start)) + } +} + +impl<A: AsRef<[AccelTy]>> Accels<A> { + /// Return an owned version of the accelerators. + #[cfg(feature = "alloc")] + pub fn to_owned(&self) -> Accels<alloc::vec::Vec<AccelTy>> { + Accels { accels: self.accels.as_ref().to_vec() } + } + + /// Return a borrowed version of the accelerators. + pub fn as_ref(&self) -> Accels<&[AccelTy]> { + Accels { accels: self.accels.as_ref() } + } + + /// Return the bytes representing the serialization of the accelerators. + pub fn as_bytes(&self) -> &[u8] { + let accels = self.accels.as_ref(); + // SAFETY: This is safe because accels is a just a slice of AccelTy, + // and u8 always has a smaller alignment. + unsafe { + core::slice::from_raw_parts( + accels.as_ptr().cast::<u8>(), + accels.len() * ACCEL_TY_SIZE, + ) + } + } + + /// Returns the memory usage, in bytes, of these accelerators. + /// + /// The memory usage is computed based on the number of bytes used to + /// represent all of the accelerators. + /// + /// This does **not** include the stack size used by this value. + pub fn memory_usage(&self) -> usize { + self.as_bytes().len() + } + + /// Return the bytes to search for corresponding to the accelerator in this + /// sequence at index `i`. If no such accelerator exists, then this panics. + /// + /// The significance of the index is that it should be in correspondence + /// with the index of the corresponding DFA. That is, accelerated DFA + /// states are stored contiguously in the DFA and have an ordering implied + /// by their respective state IDs. The state's index in that sequence + /// corresponds to the index of its corresponding accelerator. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub fn needles(&self, i: usize) -> &[u8] { + if i >= self.len() { + panic!("invalid accelerator index {i}"); + } + let bytes = self.as_bytes(); + let offset = ACCEL_TY_SIZE + i * ACCEL_CAP; + let len = usize::from(bytes[offset]); + &bytes[offset + 1..offset + 1 + len] + } + + /// Return the total number of accelerators in this sequence. + pub fn len(&self) -> usize { + // This should never panic since deserialization checks that the + // length can fit into a usize. + usize::try_from(self.accels.as_ref()[0]).unwrap() + } + + /// Return the accelerator in this sequence at index `i`. If no such + /// accelerator exists, then this returns None. + /// + /// See the docs for `needles` on the significance of the index. + fn get(&self, i: usize) -> Option<Accel> { + if i >= self.len() { + return None; + } + let offset = ACCEL_TY_SIZE + i * ACCEL_CAP; + let accel = Accel::from_slice(&self.as_bytes()[offset..]) + .expect("Accels must contain valid accelerators"); + Some(accel) + } + + /// Returns an iterator of accelerators in this sequence. + fn iter(&self) -> IterAccels<'_, A> { + IterAccels { accels: self, i: 0 } + } + + /// Writes these accelerators to the given byte buffer using the indicated + /// endianness. If the given buffer is too small, then an error is + /// returned. Upon success, the total number of bytes written is returned. + /// The number of bytes written is guaranteed to be a multiple of 8. + pub fn write_to<E: Endian>( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + assert_eq!( + nwrite % ACCEL_TY_SIZE, + 0, + "expected accelerator bytes written to be a multiple \ + of {ACCEL_TY_SIZE}", + ); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("accelerators")); + } + + // The number of accelerators can never exceed AccelTy::MAX. + E::write_u32(AccelTy::try_from(self.len()).unwrap(), dst); + // The actual accelerators are just raw bytes and thus their endianness + // is irrelevant. So we can copy them as bytes. + dst[ACCEL_TY_SIZE..nwrite] + .copy_from_slice(&self.as_bytes()[ACCEL_TY_SIZE..nwrite]); + Ok(nwrite) + } + + /// Validates that every accelerator in this collection can be successfully + /// deserialized as a valid accelerator. + pub fn validate(&self) -> Result<(), DeserializeError> { + for chunk in self.as_bytes()[ACCEL_TY_SIZE..].chunks(ACCEL_CAP) { + let _ = Accel::from_slice(chunk)?; + } + Ok(()) + } + + /// Returns the total number of bytes written by `write_to`. + pub fn write_to_len(&self) -> usize { + self.as_bytes().len() + } +} + +impl<A: AsRef<[AccelTy]>> core::fmt::Debug for Accels<A> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "Accels(")?; + let mut list = f.debug_list(); + for a in self.iter() { + list.entry(&a); + } + list.finish()?; + write!(f, ")") + } +} + +#[derive(Debug)] +struct IterAccels<'a, A: AsRef<[AccelTy]>> { + accels: &'a Accels<A>, + i: usize, +} + +impl<'a, A: AsRef<[AccelTy]>> Iterator for IterAccels<'a, A> { + type Item = Accel; + + fn next(&mut self) -> Option<Accel> { + let accel = self.accels.get(self.i)?; + self.i += 1; + Some(accel) + } +} + +/// Accel represents a structure for determining how to "accelerate" a DFA +/// state. +/// +/// Namely, it contains zero or more bytes that must be seen in order for the +/// DFA to leave the state it is associated with. In practice, the actual range +/// is 1 to 3 bytes. +/// +/// The purpose of acceleration is to identify states whose vast majority +/// of transitions are just loops back to the same state. For example, +/// in the regex `(?-u)^[^a]+b`, the corresponding DFA will have a state +/// (corresponding to `[^a]+`) where all transitions *except* for `a` and +/// `b` loop back to itself. Thus, this state can be "accelerated" by simply +/// looking for the next occurrence of either `a` or `b` instead of explicitly +/// following transitions. (In this case, `b` transitions to the next state +/// where as `a` would transition to the dead state.) +#[derive(Clone)] +pub(crate) struct Accel { + /// The first byte is the length. Subsequent bytes are the accelerated + /// bytes. + /// + /// Note that we make every accelerator 8 bytes as a slightly wasteful + /// way of making sure alignment is always correct for state ID sizes of + /// 1, 2, 4 and 8. This should be okay since accelerated states aren't + /// particularly common, especially when Unicode is enabled. + bytes: [u8; ACCEL_CAP], +} + +impl Accel { + /// Returns an empty accel, where no bytes are accelerated. + #[cfg(feature = "dfa-build")] + pub fn new() -> Accel { + Accel { bytes: [0; ACCEL_CAP] } + } + + /// Returns a verified accelerator derived from the beginning of the given + /// slice. + /// + /// If the slice is not long enough or contains invalid bytes for an + /// accelerator, then this returns an error. + pub fn from_slice(mut slice: &[u8]) -> Result<Accel, DeserializeError> { + slice = &slice[..core::cmp::min(ACCEL_LEN, slice.len())]; + let bytes = slice + .try_into() + .map_err(|_| DeserializeError::buffer_too_small("accelerator"))?; + Accel::from_bytes(bytes) + } + + /// Returns a verified accelerator derived from raw bytes. + /// + /// If the given bytes are invalid, then this returns an error. + fn from_bytes(bytes: [u8; 4]) -> Result<Accel, DeserializeError> { + if usize::from(bytes[0]) >= ACCEL_LEN { + return Err(DeserializeError::generic( + "accelerator bytes cannot have length more than 3", + )); + } + Ok(Accel::from_bytes_unchecked(bytes)) + } + + /// Returns an accelerator derived from raw bytes. + /// + /// This does not check whether the given bytes are valid. Invalid bytes + /// cannot sacrifice memory safety, but may result in panics or silent + /// logic bugs. + fn from_bytes_unchecked(bytes: [u8; 4]) -> Accel { + Accel { bytes: [bytes[0], bytes[1], bytes[2], bytes[3], 0, 0, 0, 0] } + } + + /// Attempts to add the given byte to this accelerator. If the accelerator + /// is already full or thinks the byte is a poor accelerator, then this + /// returns false. Otherwise, returns true. + /// + /// If the given byte is already in this accelerator, then it panics. + #[cfg(feature = "dfa-build")] + pub fn add(&mut self, byte: u8) -> bool { + if self.len() >= 3 { + return false; + } + // As a special case, we totally reject trying to accelerate a state + // with an ASCII space. In most cases, it occurs very frequently, and + // tends to result in worse overall performance. + if byte == b' ' { + return false; + } + assert!( + !self.contains(byte), + "accelerator already contains {:?}", + crate::util::escape::DebugByte(byte) + ); + self.bytes[self.len() + 1] = byte; + self.bytes[0] += 1; + true + } + + /// Return the number of bytes in this accelerator. + pub fn len(&self) -> usize { + usize::from(self.bytes[0]) + } + + /// Returns true if and only if there are no bytes in this accelerator. + #[cfg(feature = "dfa-build")] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the slice of bytes to accelerate. + /// + /// If this accelerator is empty, then this returns an empty slice. + fn needles(&self) -> &[u8] { + &self.bytes[1..1 + self.len()] + } + + /// Returns true if and only if this accelerator will accelerate the given + /// byte. + #[cfg(feature = "dfa-build")] + fn contains(&self, byte: u8) -> bool { + self.needles().iter().position(|&b| b == byte).is_some() + } + + /// Returns the accelerator bytes as an array of AccelTys. + #[cfg(feature = "dfa-build")] + fn as_accel_tys(&self) -> [AccelTy; 2] { + assert_eq!(ACCEL_CAP, 8); + // These unwraps are OK since ACCEL_CAP is set to 8. + let first = + AccelTy::from_ne_bytes(self.bytes[0..4].try_into().unwrap()); + let second = + AccelTy::from_ne_bytes(self.bytes[4..8].try_into().unwrap()); + [first, second] + } +} + +impl core::fmt::Debug for Accel { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "Accel(")?; + let mut set = f.debug_set(); + for &b in self.needles() { + set.entry(&crate::util::escape::DebugByte(b)); + } + set.finish()?; + write!(f, ")") + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/automaton.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/automaton.rs new file mode 100644 index 0000000..189700d --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/automaton.rs @@ -0,0 +1,2260 @@ +#[cfg(feature = "alloc")] +use crate::util::search::PatternSet; +use crate::{ + dfa::search, + util::{ + empty, + prefilter::Prefilter, + primitives::{PatternID, StateID}, + search::{Anchored, HalfMatch, Input, MatchError}, + start, + }, +}; + +/// A trait describing the interface of a deterministic finite automaton (DFA). +/// +/// The complexity of this trait probably means that it's unlikely for others +/// to implement it. The primary purpose of the trait is to provide for a way +/// of abstracting over different types of DFAs. In this crate, that means +/// dense DFAs and sparse DFAs. (Dense DFAs are fast but memory hungry, where +/// as sparse DFAs are slower but come with a smaller memory footprint. But +/// they otherwise provide exactly equivalent expressive power.) For example, a +/// [`dfa::regex::Regex`](crate::dfa::regex::Regex) is generic over this trait. +/// +/// Normally, a DFA's execution model is very simple. You might have a single +/// start state, zero or more final or "match" states and a function that +/// transitions from one state to the next given the next byte of input. +/// Unfortunately, the interface described by this trait is significantly +/// more complicated than this. The complexity has a number of different +/// reasons, mostly motivated by performance, functionality or space savings: +/// +/// * A DFA can search for multiple patterns simultaneously. This +/// means extra information is returned when a match occurs. Namely, +/// a match is not just an offset, but an offset plus a pattern ID. +/// [`Automaton::pattern_len`] returns the number of patterns compiled into +/// the DFA, [`Automaton::match_len`] returns the total number of patterns +/// that match in a particular state and [`Automaton::match_pattern`] permits +/// iterating over the patterns that match in a particular state. +/// * A DFA can have multiple start states, and the choice of which start +/// state to use depends on the content of the string being searched and +/// position of the search, as well as whether the search is an anchored +/// search for a specific pattern in the DFA. Moreover, computing the start +/// state also depends on whether you're doing a forward or a reverse search. +/// [`Automaton::start_state_forward`] and [`Automaton::start_state_reverse`] +/// are used to compute the start state for forward and reverse searches, +/// respectively. +/// * All matches are delayed by one byte to support things like `$` and `\b` +/// at the end of a pattern. Therefore, every use of a DFA is required to use +/// [`Automaton::next_eoi_state`] +/// at the end of the search to compute the final transition. +/// * For optimization reasons, some states are treated specially. Every +/// state is either special or not, which can be determined via the +/// [`Automaton::is_special_state`] method. If it's special, then the state +/// must be at least one of a few possible types of states. (Note that some +/// types can overlap, for example, a match state can also be an accel state. +/// But some types can't. If a state is a dead state, then it can never be any +/// other type of state.) Those types are: +/// * A dead state. A dead state means the DFA will never enter a match +/// state. This can be queried via the [`Automaton::is_dead_state`] method. +/// * A quit state. A quit state occurs if the DFA had to stop the search +/// prematurely for some reason. This can be queried via the +/// [`Automaton::is_quit_state`] method. +/// * A match state. A match state occurs when a match is found. When a DFA +/// enters a match state, the search may stop immediately (when looking +/// for the earliest match), or it may continue to find the leftmost-first +/// match. This can be queried via the [`Automaton::is_match_state`] +/// method. +/// * A start state. A start state is where a search begins. For every +/// search, there is exactly one start state that is used, however, a +/// DFA may contain many start states. When the search is in a start +/// state, it may use a prefilter to quickly skip to candidate matches +/// without executing the DFA on every byte. This can be queried via the +/// [`Automaton::is_start_state`] method. +/// * An accel state. An accel state is a state that is accelerated. +/// That is, it is a state where _most_ of its transitions loop back to +/// itself and only a small number of transitions lead to other states. +/// This kind of state is said to be accelerated because a search routine +/// can quickly look for the bytes leading out of the state instead of +/// continuing to execute the DFA on each byte. This can be queried via the +/// [`Automaton::is_accel_state`] method. And the bytes that lead out of +/// the state can be queried via the [`Automaton::accelerator`] method. +/// +/// There are a number of provided methods on this trait that implement +/// efficient searching (for forwards and backwards) with a DFA using +/// all of the above features of this trait. In particular, given the +/// complexity of all these features, implementing a search routine in +/// this trait can be a little subtle. With that said, it is possible to +/// somewhat simplify the search routine. For example, handling accelerated +/// states is strictly optional, since it is always correct to assume that +/// `Automaton::is_accel_state` returns false. However, one complex part of +/// writing a search routine using this trait is handling the 1-byte delay of a +/// match. That is not optional. +/// +/// # Safety +/// +/// This trait is not safe to implement so that code may rely on the +/// correctness of implementations of this trait to avoid undefined behavior. +/// The primary correctness guarantees are: +/// +/// * `Automaton::start_state` always returns a valid state ID or an error or +/// panics. +/// * `Automaton::next_state`, when given a valid state ID, always returns +/// a valid state ID for all values of `anchored` and `byte`, or otherwise +/// panics. +/// +/// In general, the rest of the methods on `Automaton` need to uphold their +/// contracts as well. For example, `Automaton::is_dead` should only returns +/// true if the given state ID is actually a dead state. +pub unsafe trait Automaton { + /// Transitions from the current state to the next state, given the next + /// byte of input. + /// + /// Implementations must guarantee that the returned ID is always a valid + /// ID when `current` refers to a valid ID. Moreover, the transition + /// function must be defined for all possible values of `input`. + /// + /// # Panics + /// + /// If the given ID does not refer to a valid state, then this routine + /// may panic but it also may not panic and instead return an invalid ID. + /// However, if the caller provides an invalid ID then this must never + /// sacrifice memory safety. + /// + /// # Example + /// + /// This shows a simplistic example for walking a DFA for a given haystack + /// by using the `next_state` method. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, Input}; + /// + /// let dfa = dense::DFA::new(r"[a-z]+r")?; + /// let haystack = "bar".as_bytes(); + /// + /// // The start state is determined by inspecting the position and the + /// // initial bytes of the haystack. + /// let mut state = dfa.start_state_forward(&Input::new(haystack))?; + /// // Walk all the bytes in the haystack. + /// for &b in haystack { + /// state = dfa.next_state(state, b); + /// } + /// // Matches are always delayed by 1 byte, so we must explicitly walk the + /// // special "EOI" transition at the end of the search. + /// state = dfa.next_eoi_state(state); + /// assert!(dfa.is_match_state(state)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn next_state(&self, current: StateID, input: u8) -> StateID; + + /// Transitions from the current state to the next state, given the next + /// byte of input. + /// + /// Unlike [`Automaton::next_state`], implementations may implement this + /// more efficiently by assuming that the `current` state ID is valid. + /// Typically, this manifests by eliding bounds checks. + /// + /// # Safety + /// + /// Callers of this method must guarantee that `current` refers to a valid + /// state ID. If `current` is not a valid state ID for this automaton, then + /// calling this routine may result in undefined behavior. + /// + /// If `current` is valid, then implementations must guarantee that the ID + /// returned is valid for all possible values of `input`. + unsafe fn next_state_unchecked( + &self, + current: StateID, + input: u8, + ) -> StateID; + + /// Transitions from the current state to the next state for the special + /// EOI symbol. + /// + /// Implementations must guarantee that the returned ID is always a valid + /// ID when `current` refers to a valid ID. + /// + /// This routine must be called at the end of every search in a correct + /// implementation of search. Namely, DFAs in this crate delay matches + /// by one byte in order to support look-around operators. Thus, after + /// reaching the end of a haystack, a search implementation must follow one + /// last EOI transition. + /// + /// It is best to think of EOI as an additional symbol in the alphabet of + /// a DFA that is distinct from every other symbol. That is, the alphabet + /// of DFAs in this crate has a logical size of 257 instead of 256, where + /// 256 corresponds to every possible inhabitant of `u8`. (In practice, the + /// physical alphabet size may be smaller because of alphabet compression + /// via equivalence classes, but EOI is always represented somehow in the + /// alphabet.) + /// + /// # Panics + /// + /// If the given ID does not refer to a valid state, then this routine + /// may panic but it also may not panic and instead return an invalid ID. + /// However, if the caller provides an invalid ID then this must never + /// sacrifice memory safety. + /// + /// # Example + /// + /// This shows a simplistic example for walking a DFA for a given haystack, + /// and then finishing the search with the final EOI transition. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, Input}; + /// + /// let dfa = dense::DFA::new(r"[a-z]+r")?; + /// let haystack = "bar".as_bytes(); + /// + /// // The start state is determined by inspecting the position and the + /// // initial bytes of the haystack. + /// // + /// // The unwrap is OK because we aren't requesting a start state for a + /// // specific pattern. + /// let mut state = dfa.start_state_forward(&Input::new(haystack))?; + /// // Walk all the bytes in the haystack. + /// for &b in haystack { + /// state = dfa.next_state(state, b); + /// } + /// // Matches are always delayed by 1 byte, so we must explicitly walk + /// // the special "EOI" transition at the end of the search. Without this + /// // final transition, the assert below will fail since the DFA will not + /// // have entered a match state yet! + /// state = dfa.next_eoi_state(state); + /// assert!(dfa.is_match_state(state)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn next_eoi_state(&self, current: StateID) -> StateID; + + /// Return the ID of the start state for this DFA for the given starting + /// configuration. + /// + /// Unlike typical DFA implementations, the start state for DFAs in this + /// crate is dependent on a few different factors: + /// + /// * The [`Anchored`] mode of the search. Unanchored, anchored and + /// anchored searches for a specific [`PatternID`] all use different start + /// states. + /// * Whether a "look-behind" byte exists. For example, the `^` anchor + /// matches if and only if there is no look-behind byte. + /// * The specific value of that look-behind byte. For example, a `(?m:^)` + /// assertion only matches when there is either no look-behind byte, or + /// when the look-behind byte is a line terminator. + /// + /// The [starting configuration](start::Config) provides the above + /// information. + /// + /// This routine can be used for either forward or reverse searches. + /// Although, as a convenience, if you have an [`Input`], then it may + /// be more succinct to use [`Automaton::start_state_forward`] or + /// [`Automaton::start_state_reverse`]. Note, for example, that the + /// convenience routines return a [`MatchError`] on failure where as this + /// routine returns a [`StartError`]. + /// + /// # Errors + /// + /// This may return a [`StartError`] if the search needs to give up when + /// determining the start state (for example, if it sees a "quit" byte). + /// This can also return an error if the given configuration contains an + /// unsupported [`Anchored`] configuration. + fn start_state( + &self, + config: &start::Config, + ) -> Result<StateID, StartError>; + + /// Return the ID of the start state for this DFA when executing a forward + /// search. + /// + /// This is a convenience routine for calling [`Automaton::start_state`] + /// that converts the given [`Input`] to a [start + /// configuration](start::Config). Additionally, if an error occurs, it is + /// converted from a [`StartError`] to a [`MatchError`] using the offset + /// information in the given [`Input`]. + /// + /// # Errors + /// + /// This may return a [`MatchError`] if the search needs to give up + /// when determining the start state (for example, if it sees a "quit" + /// byte). This can also return an error if the given `Input` contains an + /// unsupported [`Anchored`] configuration. + fn start_state_forward( + &self, + input: &Input<'_>, + ) -> Result<StateID, MatchError> { + let config = start::Config::from_input_forward(input); + self.start_state(&config).map_err(|err| match err { + StartError::Quit { byte } => { + let offset = input + .start() + .checked_sub(1) + .expect("no quit in start without look-behind"); + MatchError::quit(byte, offset) + } + StartError::UnsupportedAnchored { mode } => { + MatchError::unsupported_anchored(mode) + } + }) + } + + /// Return the ID of the start state for this DFA when executing a reverse + /// search. + /// + /// This is a convenience routine for calling [`Automaton::start_state`] + /// that converts the given [`Input`] to a [start + /// configuration](start::Config). Additionally, if an error occurs, it is + /// converted from a [`StartError`] to a [`MatchError`] using the offset + /// information in the given [`Input`]. + /// + /// # Errors + /// + /// This may return a [`MatchError`] if the search needs to give up + /// when determining the start state (for example, if it sees a "quit" + /// byte). This can also return an error if the given `Input` contains an + /// unsupported [`Anchored`] configuration. + fn start_state_reverse( + &self, + input: &Input<'_>, + ) -> Result<StateID, MatchError> { + let config = start::Config::from_input_reverse(input); + self.start_state(&config).map_err(|err| match err { + StartError::Quit { byte } => { + let offset = input.end(); + MatchError::quit(byte, offset) + } + StartError::UnsupportedAnchored { mode } => { + MatchError::unsupported_anchored(mode) + } + }) + } + + /// If this DFA has a universal starting state for the given anchor mode + /// and the DFA supports universal starting states, then this returns that + /// state's identifier. + /// + /// A DFA is said to have a universal starting state when the starting + /// state is invariant with respect to the haystack. Usually, the starting + /// state is chosen depending on the bytes immediately surrounding the + /// starting position of a search. However, the starting state only differs + /// when one or more of the patterns in the DFA have look-around assertions + /// in its prefix. + /// + /// Stated differently, if none of the patterns in a DFA have look-around + /// assertions in their prefix, then the DFA has a universal starting state + /// and _may_ be returned by this method. + /// + /// It is always correct for implementations to return `None`, and indeed, + /// this is what the default implementation does. When this returns `None`, + /// callers must use either `start_state_forward` or `start_state_reverse` + /// to get the starting state. + /// + /// # Use case + /// + /// There are a few reasons why one might want to use this: + /// + /// * If you know your regex patterns have no look-around assertions in + /// their prefix, then calling this routine is likely cheaper and perhaps + /// more semantically meaningful. + /// * When implementing prefilter support in a DFA regex implementation, + /// it is necessary to re-compute the start state after a candidate + /// is returned from the prefilter. However, this is only needed when + /// there isn't a universal start state. When one exists, one can avoid + /// re-computing the start state. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// dfa::{Automaton, dense::DFA}, + /// Anchored, + /// }; + /// + /// // There are no look-around assertions in the prefixes of any of the + /// // patterns, so we get a universal start state. + /// let dfa = DFA::new_many(&["[0-9]+", "[a-z]+$", "[A-Z]+"])?; + /// assert!(dfa.universal_start_state(Anchored::No).is_some()); + /// assert!(dfa.universal_start_state(Anchored::Yes).is_some()); + /// + /// // One of the patterns has a look-around assertion in its prefix, + /// // so this means there is no longer a universal start state. + /// let dfa = DFA::new_many(&["[0-9]+", "^[a-z]+$", "[A-Z]+"])?; + /// assert!(!dfa.universal_start_state(Anchored::No).is_some()); + /// assert!(!dfa.universal_start_state(Anchored::Yes).is_some()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + fn universal_start_state(&self, _mode: Anchored) -> Option<StateID> { + None + } + + /// Returns true if and only if the given identifier corresponds to a + /// "special" state. A special state is one or more of the following: + /// a dead state, a quit state, a match state, a start state or an + /// accelerated state. + /// + /// A correct implementation _may_ always return false for states that + /// are either start states or accelerated states, since that information + /// is only intended to be used for optimization purposes. Correct + /// implementations must return true if the state is a dead, quit or match + /// state. This is because search routines using this trait must be able + /// to rely on `is_special_state` as an indicator that a state may need + /// special treatment. (For example, when a search routine sees a dead + /// state, it must terminate.) + /// + /// This routine permits search implementations to use a single branch to + /// check whether a state needs special attention before executing the next + /// transition. The example below shows how to do this. + /// + /// # Example + /// + /// This example shows how `is_special_state` can be used to implement a + /// correct search routine with minimal branching. In particular, this + /// search routine implements "leftmost" matching, which means that it + /// doesn't immediately stop once a match is found. Instead, it continues + /// until it reaches a dead state. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{Automaton, dense}, + /// HalfMatch, MatchError, Input, + /// }; + /// + /// fn find<A: Automaton>( + /// dfa: &A, + /// haystack: &[u8], + /// ) -> Result<Option<HalfMatch>, MatchError> { + /// // The start state is determined by inspecting the position and the + /// // initial bytes of the haystack. Note that start states can never + /// // be match states (since DFAs in this crate delay matches by 1 + /// // byte), so we don't need to check if the start state is a match. + /// let mut state = dfa.start_state_forward(&Input::new(haystack))?; + /// let mut last_match = None; + /// // Walk all the bytes in the haystack. We can quit early if we see + /// // a dead or a quit state. The former means the automaton will + /// // never transition to any other state. The latter means that the + /// // automaton entered a condition in which its search failed. + /// for (i, &b) in haystack.iter().enumerate() { + /// state = dfa.next_state(state, b); + /// if dfa.is_special_state(state) { + /// if dfa.is_match_state(state) { + /// last_match = Some(HalfMatch::new( + /// dfa.match_pattern(state, 0), + /// i, + /// )); + /// } else if dfa.is_dead_state(state) { + /// return Ok(last_match); + /// } else if dfa.is_quit_state(state) { + /// // It is possible to enter into a quit state after + /// // observing a match has occurred. In that case, we + /// // should return the match instead of an error. + /// if last_match.is_some() { + /// return Ok(last_match); + /// } + /// return Err(MatchError::quit(b, i)); + /// } + /// // Implementors may also want to check for start or accel + /// // states and handle them differently for performance + /// // reasons. But it is not necessary for correctness. + /// } + /// } + /// // Matches are always delayed by 1 byte, so we must explicitly walk + /// // the special "EOI" transition at the end of the search. + /// state = dfa.next_eoi_state(state); + /// if dfa.is_match_state(state) { + /// last_match = Some(HalfMatch::new( + /// dfa.match_pattern(state, 0), + /// haystack.len(), + /// )); + /// } + /// Ok(last_match) + /// } + /// + /// // We use a greedy '+' operator to show how the search doesn't just + /// // stop once a match is detected. It continues extending the match. + /// // Using '[a-z]+?' would also work as expected and stop the search + /// // early. Greediness is built into the automaton. + /// let dfa = dense::DFA::new(r"[a-z]+")?; + /// let haystack = "123 foobar 4567".as_bytes(); + /// let mat = find(&dfa, haystack)?.unwrap(); + /// assert_eq!(mat.pattern().as_usize(), 0); + /// assert_eq!(mat.offset(), 10); + /// + /// // Here's another example that tests our handling of the special EOI + /// // transition. This will fail to find a match if we don't call + /// // 'next_eoi_state' at the end of the search since the match isn't + /// // found until the final byte in the haystack. + /// let dfa = dense::DFA::new(r"[0-9]{4}")?; + /// let haystack = "123 foobar 4567".as_bytes(); + /// let mat = find(&dfa, haystack)?.unwrap(); + /// assert_eq!(mat.pattern().as_usize(), 0); + /// assert_eq!(mat.offset(), 15); + /// + /// // And note that our search implementation above automatically works + /// // with multi-DFAs. Namely, `dfa.match_pattern(match_state, 0)` selects + /// // the appropriate pattern ID for us. + /// let dfa = dense::DFA::new_many(&[r"[a-z]+", r"[0-9]+"])?; + /// let haystack = "123 foobar 4567".as_bytes(); + /// let mat = find(&dfa, haystack)?.unwrap(); + /// assert_eq!(mat.pattern().as_usize(), 1); + /// assert_eq!(mat.offset(), 3); + /// let mat = find(&dfa, &haystack[3..])?.unwrap(); + /// assert_eq!(mat.pattern().as_usize(), 0); + /// assert_eq!(mat.offset(), 7); + /// let mat = find(&dfa, &haystack[10..])?.unwrap(); + /// assert_eq!(mat.pattern().as_usize(), 1); + /// assert_eq!(mat.offset(), 5); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn is_special_state(&self, id: StateID) -> bool; + + /// Returns true if and only if the given identifier corresponds to a dead + /// state. When a DFA enters a dead state, it is impossible to leave. That + /// is, every transition on a dead state by definition leads back to the + /// same dead state. + /// + /// In practice, the dead state always corresponds to the identifier `0`. + /// Moreover, in practice, there is only one dead state. + /// + /// The existence of a dead state is not strictly required in the classical + /// model of finite state machines, where one generally only cares about + /// the question of whether an input sequence matches or not. Dead states + /// are not needed to answer that question, since one can immediately quit + /// as soon as one enters a final or "match" state. However, we don't just + /// care about matches but also care about the location of matches, and + /// more specifically, care about semantics like "greedy" matching. + /// + /// For example, given the pattern `a+` and the input `aaaz`, the dead + /// state won't be entered until the state machine reaches `z` in the + /// input, at which point, the search routine can quit. But without the + /// dead state, the search routine wouldn't know when to quit. In a + /// classical representation, the search routine would stop after seeing + /// the first `a` (which is when the search would enter a match state). But + /// this wouldn't implement "greedy" matching where `a+` matches as many + /// `a`'s as possible. + /// + /// # Example + /// + /// See the example for [`Automaton::is_special_state`] for how to use this + /// method correctly. + fn is_dead_state(&self, id: StateID) -> bool; + + /// Returns true if and only if the given identifier corresponds to a quit + /// state. A quit state is like a dead state (it has no transitions other + /// than to itself), except it indicates that the DFA failed to complete + /// the search. When this occurs, callers can neither accept or reject that + /// a match occurred. + /// + /// In practice, the quit state always corresponds to the state immediately + /// following the dead state. (Which is not usually represented by `1`, + /// since state identifiers are pre-multiplied by the state machine's + /// alphabet stride, and the alphabet stride varies between DFAs.) + /// + /// The typical way in which a quit state can occur is when heuristic + /// support for Unicode word boundaries is enabled via the + /// [`dense::Config::unicode_word_boundary`](crate::dfa::dense::Config::unicode_word_boundary) + /// option. But other options, like the lower level + /// [`dense::Config::quit`](crate::dfa::dense::Config::quit) + /// configuration, can also result in a quit state being entered. The + /// purpose of the quit state is to provide a way to execute a fast DFA + /// in common cases while delegating to slower routines when the DFA quits. + /// + /// The default search implementations provided by this crate will return a + /// [`MatchError::quit`] error when a quit state is entered. + /// + /// # Example + /// + /// See the example for [`Automaton::is_special_state`] for how to use this + /// method correctly. + fn is_quit_state(&self, id: StateID) -> bool; + + /// Returns true if and only if the given identifier corresponds to a + /// match state. A match state is also referred to as a "final" state and + /// indicates that a match has been found. + /// + /// If all you care about is whether a particular pattern matches in the + /// input sequence, then a search routine can quit early as soon as the + /// machine enters a match state. However, if you're looking for the + /// standard "leftmost-first" match location, then search _must_ continue + /// until either the end of the input or until the machine enters a dead + /// state. (Since either condition implies that no other useful work can + /// be done.) Namely, when looking for the location of a match, then + /// search implementations should record the most recent location in + /// which a match state was entered, but otherwise continue executing the + /// search as normal. (The search may even leave the match state.) Once + /// the termination condition is reached, the most recently recorded match + /// location should be returned. + /// + /// Finally, one additional power given to match states in this crate + /// is that they are always associated with a specific pattern in order + /// to support multi-DFAs. See [`Automaton::match_pattern`] for more + /// details and an example for how to query the pattern associated with a + /// particular match state. + /// + /// # Example + /// + /// See the example for [`Automaton::is_special_state`] for how to use this + /// method correctly. + fn is_match_state(&self, id: StateID) -> bool; + + /// Returns true only if the given identifier corresponds to a start + /// state + /// + /// A start state is a state in which a DFA begins a search. + /// All searches begin in a start state. Moreover, since all matches are + /// delayed by one byte, a start state can never be a match state. + /// + /// The main role of a start state is, as mentioned, to be a starting + /// point for a DFA. This starting point is determined via one of + /// [`Automaton::start_state_forward`] or + /// [`Automaton::start_state_reverse`], depending on whether one is doing + /// a forward or a reverse search, respectively. + /// + /// A secondary use of start states is for prefix acceleration. Namely, + /// while executing a search, if one detects that you're in a start state, + /// then it may be faster to look for the next match of a prefix of the + /// pattern, if one exists. If a prefix exists and since all matches must + /// begin with that prefix, then skipping ahead to occurrences of that + /// prefix may be much faster than executing the DFA. + /// + /// As mentioned in the documentation for + /// [`is_special_state`](Automaton::is_special_state) implementations + /// _may_ always return false, even if the given identifier is a start + /// state. This is because knowing whether a state is a start state or not + /// is not necessary for correctness and is only treated as a potential + /// performance optimization. (For example, the implementations of this + /// trait in this crate will only return true when the given identifier + /// corresponds to a start state and when [specialization of start + /// states](crate::dfa::dense::Config::specialize_start_states) was enabled + /// during DFA construction. If start state specialization is disabled + /// (which is the default), then this method will always return false.) + /// + /// # Example + /// + /// This example shows how to implement your own search routine that does + /// a prefix search whenever the search enters a start state. + /// + /// Note that you do not need to implement your own search routine + /// to make use of prefilters like this. The search routines + /// provided by this crate already implement prefilter support via + /// the [`Prefilter`](crate::util::prefilter::Prefilter) trait. + /// A prefilter can be added to your search configuration with + /// [`dense::Config::prefilter`](crate::dfa::dense::Config::prefilter) for + /// dense and sparse DFAs in this crate. + /// + /// This example is meant to show how you might deal with prefilters in a + /// simplified case if you are implementing your own search routine. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{Automaton, dense}, + /// HalfMatch, MatchError, Input, + /// }; + /// + /// fn find_byte(slice: &[u8], at: usize, byte: u8) -> Option<usize> { + /// // Would be faster to use the memchr crate, but this is still + /// // faster than running through the DFA. + /// slice[at..].iter().position(|&b| b == byte).map(|i| at + i) + /// } + /// + /// fn find<A: Automaton>( + /// dfa: &A, + /// haystack: &[u8], + /// prefix_byte: Option<u8>, + /// ) -> Result<Option<HalfMatch>, MatchError> { + /// // See the Automaton::is_special_state example for similar code + /// // with more comments. + /// + /// let mut state = dfa.start_state_forward(&Input::new(haystack))?; + /// let mut last_match = None; + /// let mut pos = 0; + /// while pos < haystack.len() { + /// let b = haystack[pos]; + /// state = dfa.next_state(state, b); + /// pos += 1; + /// if dfa.is_special_state(state) { + /// if dfa.is_match_state(state) { + /// last_match = Some(HalfMatch::new( + /// dfa.match_pattern(state, 0), + /// pos - 1, + /// )); + /// } else if dfa.is_dead_state(state) { + /// return Ok(last_match); + /// } else if dfa.is_quit_state(state) { + /// // It is possible to enter into a quit state after + /// // observing a match has occurred. In that case, we + /// // should return the match instead of an error. + /// if last_match.is_some() { + /// return Ok(last_match); + /// } + /// return Err(MatchError::quit(b, pos - 1)); + /// } else if dfa.is_start_state(state) { + /// // If we're in a start state and know all matches begin + /// // with a particular byte, then we can quickly skip to + /// // candidate matches without running the DFA through + /// // every byte inbetween. + /// if let Some(prefix_byte) = prefix_byte { + /// pos = match find_byte(haystack, pos, prefix_byte) { + /// Some(pos) => pos, + /// None => break, + /// }; + /// } + /// } + /// } + /// } + /// // Matches are always delayed by 1 byte, so we must explicitly walk + /// // the special "EOI" transition at the end of the search. + /// state = dfa.next_eoi_state(state); + /// if dfa.is_match_state(state) { + /// last_match = Some(HalfMatch::new( + /// dfa.match_pattern(state, 0), + /// haystack.len(), + /// )); + /// } + /// Ok(last_match) + /// } + /// + /// // In this example, it's obvious that all occurrences of our pattern + /// // begin with 'Z', so we pass in 'Z'. Note also that we need to + /// // enable start state specialization, or else it won't be possible to + /// // detect start states during a search. ('is_start_state' would always + /// // return false.) + /// let dfa = dense::DFA::builder() + /// .configure(dense::DFA::config().specialize_start_states(true)) + /// .build(r"Z[a-z]+")?; + /// let haystack = "123 foobar Zbaz quux".as_bytes(); + /// let mat = find(&dfa, haystack, Some(b'Z'))?.unwrap(); + /// assert_eq!(mat.pattern().as_usize(), 0); + /// assert_eq!(mat.offset(), 15); + /// + /// // But note that we don't need to pass in a prefix byte. If we don't, + /// // then the search routine does no acceleration. + /// let mat = find(&dfa, haystack, None)?.unwrap(); + /// assert_eq!(mat.pattern().as_usize(), 0); + /// assert_eq!(mat.offset(), 15); + /// + /// // However, if we pass an incorrect byte, then the prefix search will + /// // result in incorrect results. + /// assert_eq!(find(&dfa, haystack, Some(b'X'))?, None); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn is_start_state(&self, id: StateID) -> bool; + + /// Returns true if and only if the given identifier corresponds to an + /// accelerated state. + /// + /// An accelerated state is a special optimization + /// trick implemented by this crate. Namely, if + /// [`dense::Config::accelerate`](crate::dfa::dense::Config::accelerate) is + /// enabled (and it is by default), then DFAs generated by this crate will + /// tag states meeting certain characteristics as accelerated. States meet + /// this criteria whenever most of their transitions are self-transitions. + /// That is, transitions that loop back to the same state. When a small + /// number of transitions aren't self-transitions, then it follows that + /// there are only a small number of bytes that can cause the DFA to leave + /// that state. Thus, there is an opportunity to look for those bytes + /// using more optimized routines rather than continuing to run through + /// the DFA. This trick is similar to the prefilter idea described in + /// the documentation of [`Automaton::is_start_state`] with two main + /// differences: + /// + /// 1. It is more limited since acceleration only applies to single bytes. + /// This means states are rarely accelerated when Unicode mode is enabled + /// (which is enabled by default). + /// 2. It can occur anywhere in the DFA, which increases optimization + /// opportunities. + /// + /// Like the prefilter idea, the main downside (and a possible reason to + /// disable it) is that it can lead to worse performance in some cases. + /// Namely, if a state is accelerated for very common bytes, then the + /// overhead of checking for acceleration and using the more optimized + /// routines to look for those bytes can cause overall performance to be + /// worse than if acceleration wasn't enabled at all. + /// + /// A simple example of a regex that has an accelerated state is + /// `(?-u)[^a]+a`. Namely, the `[^a]+` sub-expression gets compiled down + /// into a single state where all transitions except for `a` loop back to + /// itself, and where `a` is the only transition (other than the special + /// EOI transition) that goes to some other state. Thus, this state can + /// be accelerated and implemented more efficiently by calling an + /// optimized routine like `memchr` with `a` as the needle. Notice that + /// the `(?-u)` to disable Unicode is necessary here, as without it, + /// `[^a]` will match any UTF-8 encoding of any Unicode scalar value other + /// than `a`. This more complicated expression compiles down to many DFA + /// states and the simple acceleration optimization is no longer available. + /// + /// Typically, this routine is used to guard calls to + /// [`Automaton::accelerator`], which returns the accelerated bytes for + /// the specified state. + fn is_accel_state(&self, id: StateID) -> bool; + + /// Returns the total number of patterns compiled into this DFA. + /// + /// In the case of a DFA that contains no patterns, this must return `0`. + /// + /// # Example + /// + /// This example shows the pattern length for a DFA that never matches: + /// + /// ``` + /// use regex_automata::dfa::{Automaton, dense::DFA}; + /// + /// let dfa: DFA<Vec<u32>> = DFA::never_match()?; + /// assert_eq!(dfa.pattern_len(), 0); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And another example for a DFA that matches at every position: + /// + /// ``` + /// use regex_automata::dfa::{Automaton, dense::DFA}; + /// + /// let dfa: DFA<Vec<u32>> = DFA::always_match()?; + /// assert_eq!(dfa.pattern_len(), 1); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And finally, a DFA that was constructed from multiple patterns: + /// + /// ``` + /// use regex_automata::dfa::{Automaton, dense::DFA}; + /// + /// let dfa = DFA::new_many(&["[0-9]+", "[a-z]+", "[A-Z]+"])?; + /// assert_eq!(dfa.pattern_len(), 3); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn pattern_len(&self) -> usize; + + /// Returns the total number of patterns that match in this state. + /// + /// If the given state is not a match state, then implementations may + /// panic. + /// + /// If the DFA was compiled with one pattern, then this must necessarily + /// always return `1` for all match states. + /// + /// Implementations must guarantee that [`Automaton::match_pattern`] can be + /// called with indices up to (but not including) the length returned by + /// this routine without panicking. + /// + /// # Panics + /// + /// Implementations are permitted to panic if the provided state ID does + /// not correspond to a match state. + /// + /// # Example + /// + /// This example shows a simple instance of implementing overlapping + /// matches. In particular, it shows not only how to determine how many + /// patterns have matched in a particular state, but also how to access + /// which specific patterns have matched. + /// + /// Notice that we must use + /// [`MatchKind::All`](crate::MatchKind::All) + /// when building the DFA. If we used + /// [`MatchKind::LeftmostFirst`](crate::MatchKind::LeftmostFirst) + /// instead, then the DFA would not be constructed in a way that + /// supports overlapping matches. (It would only report a single pattern + /// that matches at any particular point in time.) + /// + /// Another thing to take note of is the patterns used and the order in + /// which the pattern IDs are reported. In the example below, pattern `3` + /// is yielded first. Why? Because it corresponds to the match that + /// appears first. Namely, the `@` symbol is part of `\S+` but not part + /// of any of the other patterns. Since the `\S+` pattern has a match that + /// starts to the left of any other pattern, its ID is returned before any + /// other. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::{Automaton, dense}, Input, MatchKind}; + /// + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new().match_kind(MatchKind::All)) + /// .build_many(&[ + /// r"[[:word:]]+", r"[a-z]+", r"[A-Z]+", r"[[:^space:]]+", + /// ])?; + /// let haystack = "@bar".as_bytes(); + /// + /// // The start state is determined by inspecting the position and the + /// // initial bytes of the haystack. + /// let mut state = dfa.start_state_forward(&Input::new(haystack))?; + /// // Walk all the bytes in the haystack. + /// for &b in haystack { + /// state = dfa.next_state(state, b); + /// } + /// state = dfa.next_eoi_state(state); + /// + /// assert!(dfa.is_match_state(state)); + /// assert_eq!(dfa.match_len(state), 3); + /// // The following calls are guaranteed to not panic since `match_len` + /// // returned `3` above. + /// assert_eq!(dfa.match_pattern(state, 0).as_usize(), 3); + /// assert_eq!(dfa.match_pattern(state, 1).as_usize(), 0); + /// assert_eq!(dfa.match_pattern(state, 2).as_usize(), 1); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn match_len(&self, id: StateID) -> usize; + + /// Returns the pattern ID corresponding to the given match index in the + /// given state. + /// + /// See [`Automaton::match_len`] for an example of how to use this + /// method correctly. Note that if you know your DFA is compiled with a + /// single pattern, then this routine is never necessary since it will + /// always return a pattern ID of `0` for an index of `0` when `id` + /// corresponds to a match state. + /// + /// Typically, this routine is used when implementing an overlapping + /// search, as the example for `Automaton::match_len` does. + /// + /// # Panics + /// + /// If the state ID is not a match state or if the match index is out + /// of bounds for the given state, then this routine may either panic + /// or produce an incorrect result. If the state ID is correct and the + /// match index is correct, then this routine must always produce a valid + /// `PatternID`. + fn match_pattern(&self, id: StateID, index: usize) -> PatternID; + + /// Returns true if and only if this automaton can match the empty string. + /// When it returns false, all possible matches are guaranteed to have a + /// non-zero length. + /// + /// This is useful as cheap way to know whether code needs to handle the + /// case of a zero length match. This is particularly important when UTF-8 + /// modes are enabled, as when UTF-8 mode is enabled, empty matches that + /// split a codepoint must never be reported. This extra handling can + /// sometimes be costly, and since regexes matching an empty string are + /// somewhat rare, it can be beneficial to treat such regexes specially. + /// + /// # Example + /// + /// This example shows a few different DFAs and whether they match the + /// empty string or not. Notice the empty string isn't merely a matter + /// of a string of length literally `0`, but rather, whether a match can + /// occur between specific pairs of bytes. + /// + /// ``` + /// use regex_automata::{dfa::{dense::DFA, Automaton}, util::syntax}; + /// + /// // The empty regex matches the empty string. + /// let dfa = DFA::new("")?; + /// assert!(dfa.has_empty(), "empty matches empty"); + /// // The '+' repetition operator requires at least one match, and so + /// // does not match the empty string. + /// let dfa = DFA::new("a+")?; + /// assert!(!dfa.has_empty(), "+ does not match empty"); + /// // But the '*' repetition operator does. + /// let dfa = DFA::new("a*")?; + /// assert!(dfa.has_empty(), "* does match empty"); + /// // And wrapping '+' in an operator that can match an empty string also + /// // causes it to match the empty string too. + /// let dfa = DFA::new("(a+)*")?; + /// assert!(dfa.has_empty(), "+ inside of * matches empty"); + /// + /// // If a regex is just made of a look-around assertion, even if the + /// // assertion requires some kind of non-empty string around it (such as + /// // \b), then it is still treated as if it matches the empty string. + /// // Namely, if a match occurs of just a look-around assertion, then the + /// // match returned is empty. + /// let dfa = DFA::builder() + /// .configure(DFA::config().unicode_word_boundary(true)) + /// .syntax(syntax::Config::new().utf8(false)) + /// .build(r"^$\A\z\b\B(?-u:\b\B)")?; + /// assert!(dfa.has_empty(), "assertions match empty"); + /// // Even when an assertion is wrapped in a '+', it still matches the + /// // empty string. + /// let dfa = DFA::new(r"^+")?; + /// assert!(dfa.has_empty(), "+ of an assertion matches empty"); + /// + /// // An alternation with even one branch that can match the empty string + /// // is also said to match the empty string overall. + /// let dfa = DFA::new("foo|(bar)?|quux")?; + /// assert!(dfa.has_empty(), "alternations can match empty"); + /// + /// // An NFA that matches nothing does not match the empty string. + /// let dfa = DFA::new("[a&&b]")?; + /// assert!(!dfa.has_empty(), "never matching means not matching empty"); + /// // But if it's wrapped in something that doesn't require a match at + /// // all, then it can match the empty string! + /// let dfa = DFA::new("[a&&b]*")?; + /// assert!(dfa.has_empty(), "* on never-match still matches empty"); + /// // Since a '+' requires a match, using it on something that can never + /// // match will itself produce a regex that can never match anything, + /// // and thus does not match the empty string. + /// let dfa = DFA::new("[a&&b]+")?; + /// assert!(!dfa.has_empty(), "+ on never-match still matches nothing"); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn has_empty(&self) -> bool; + + /// Whether UTF-8 mode is enabled for this DFA or not. + /// + /// When UTF-8 mode is enabled, all matches reported by a DFA are + /// guaranteed to correspond to spans of valid UTF-8. This includes + /// zero-width matches. For example, the DFA must guarantee that the empty + /// regex will not match at the positions between code units in the UTF-8 + /// encoding of a single codepoint. + /// + /// See [`thompson::Config::utf8`](crate::nfa::thompson::Config::utf8) for + /// more information. + /// + /// # Example + /// + /// This example shows how UTF-8 mode can impact the match spans that may + /// be reported in certain cases. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense::DFA, Automaton}, + /// nfa::thompson, + /// HalfMatch, Input, + /// }; + /// + /// // UTF-8 mode is enabled by default. + /// let re = DFA::new("")?; + /// assert!(re.is_utf8()); + /// let mut input = Input::new("☃"); + /// let got = re.try_search_fwd(&input)?; + /// assert_eq!(Some(HalfMatch::must(0, 0)), got); + /// + /// // Even though an empty regex matches at 1..1, our next match is + /// // 3..3 because 1..1 and 2..2 split the snowman codepoint (which is + /// // three bytes long). + /// input.set_start(1); + /// let got = re.try_search_fwd(&input)?; + /// assert_eq!(Some(HalfMatch::must(0, 3)), got); + /// + /// // But if we disable UTF-8, then we'll get matches at 1..1 and 2..2: + /// let re = DFA::builder() + /// .thompson(thompson::Config::new().utf8(false)) + /// .build("")?; + /// assert!(!re.is_utf8()); + /// let got = re.try_search_fwd(&input)?; + /// assert_eq!(Some(HalfMatch::must(0, 1)), got); + /// + /// input.set_start(2); + /// let got = re.try_search_fwd(&input)?; + /// assert_eq!(Some(HalfMatch::must(0, 2)), got); + /// + /// input.set_start(3); + /// let got = re.try_search_fwd(&input)?; + /// assert_eq!(Some(HalfMatch::must(0, 3)), got); + /// + /// input.set_start(4); + /// let got = re.try_search_fwd(&input)?; + /// assert_eq!(None, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn is_utf8(&self) -> bool; + + /// Returns true if and only if this DFA is limited to returning matches + /// whose start position is `0`. + /// + /// Note that if you're using DFAs provided by + /// this crate, then this is _orthogonal_ to + /// [`Config::start_kind`](crate::dfa::dense::Config::start_kind). + /// + /// This is useful in some cases because if a DFA is limited to producing + /// matches that start at offset `0`, then a reverse search is never + /// required for finding the start of a match. + /// + /// # Example + /// + /// ``` + /// use regex_automata::dfa::{dense::DFA, Automaton}; + /// + /// // The empty regex matches anywhere + /// let dfa = DFA::new("")?; + /// assert!(!dfa.is_always_start_anchored(), "empty matches anywhere"); + /// // 'a' matches anywhere. + /// let dfa = DFA::new("a")?; + /// assert!(!dfa.is_always_start_anchored(), "'a' matches anywhere"); + /// // '^' only matches at offset 0! + /// let dfa = DFA::new("^a")?; + /// assert!(dfa.is_always_start_anchored(), "'^a' matches only at 0"); + /// // But '(?m:^)' matches at 0 but at other offsets too. + /// let dfa = DFA::new("(?m:^)a")?; + /// assert!(!dfa.is_always_start_anchored(), "'(?m:^)a' matches anywhere"); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + fn is_always_start_anchored(&self) -> bool; + + /// Return a slice of bytes to accelerate for the given state, if possible. + /// + /// If the given state has no accelerator, then an empty slice must be + /// returned. If `Automaton::is_accel_state` returns true for the given ID, + /// then this routine _must_ return a non-empty slice. But note that it is + /// not required for an implementation of this trait to ever return `true` + /// for `is_accel_state`, even if the state _could_ be accelerated. That + /// is, acceleration is an optional optimization. But the return values of + /// `is_accel_state` and `accelerator` must be in sync. + /// + /// If the given ID is not a valid state ID for this automaton, then + /// implementations may panic or produce incorrect results. + /// + /// See [`Automaton::is_accel_state`] for more details on state + /// acceleration. + /// + /// By default, this method will always return an empty slice. + /// + /// # Example + /// + /// This example shows a contrived case in which we build a regex that we + /// know is accelerated and extract the accelerator from a state. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{Automaton, dense}, + /// util::{primitives::StateID, syntax}, + /// }; + /// + /// let dfa = dense::Builder::new() + /// // We disable Unicode everywhere and permit the regex to match + /// // invalid UTF-8. e.g., [^abc] matches \xFF, which is not valid + /// // UTF-8. If we left Unicode enabled, [^abc] would match any UTF-8 + /// // encoding of any Unicode scalar value except for 'a', 'b' or 'c'. + /// // That translates to a much more complicated DFA, and also + /// // inhibits the 'accelerator' optimization that we are trying to + /// // demonstrate in this example. + /// .syntax(syntax::Config::new().unicode(false).utf8(false)) + /// .build("[^abc]+a")?; + /// + /// // Here we just pluck out the state that we know is accelerated. + /// // While the stride calculations are something that can be relied + /// // on by callers, the specific position of the accelerated state is + /// // implementation defined. + /// // + /// // N.B. We get '3' by inspecting the state machine using 'regex-cli'. + /// // e.g., try `regex-cli debug dense dfa -p '[^abc]+a' -BbUC`. + /// let id = StateID::new(3 * dfa.stride()).unwrap(); + /// let accelerator = dfa.accelerator(id); + /// // The `[^abc]+` sub-expression permits [a, b, c] to be accelerated. + /// assert_eq!(accelerator, &[b'a', b'b', b'c']); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + fn accelerator(&self, _id: StateID) -> &[u8] { + &[] + } + + /// Returns the prefilter associated with a DFA, if one exists. + /// + /// The default implementation of this trait always returns `None`. And + /// indeed, it is always correct to return `None`. + /// + /// For DFAs in this crate, a prefilter can be attached to a DFA via + /// [`dense::Config::prefilter`](crate::dfa::dense::Config::prefilter). + /// + /// Do note that prefilters are not serialized by DFAs in this crate. + /// So if you deserialize a DFA that had a prefilter attached to it + /// at serialization time, then it will not have a prefilter after + /// deserialization. + #[inline] + fn get_prefilter(&self) -> Option<&Prefilter> { + None + } + + /// Executes a forward search and returns the end position of the leftmost + /// match that is found. If no match exists, then `None` is returned. + /// + /// In particular, this method continues searching even after it enters + /// a match state. The search only terminates once it has reached the + /// end of the input or when it has entered a dead or quit state. Upon + /// termination, the position of the last byte seen while still in a match + /// state is returned. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the DFA quitting. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Notes for implementors + /// + /// Implementors of this trait are not required to implement any particular + /// match semantics (such as leftmost-first), which are instead manifest in + /// the DFA's transitions. But this search routine should behave as a + /// general "leftmost" search. + /// + /// In particular, this method must continue searching even after it enters + /// a match state. The search should only terminate once it has reached + /// the end of the input or when it has entered a dead or quit state. Upon + /// termination, the position of the last byte seen while still in a match + /// state is returned. + /// + /// Since this trait provides an implementation for this method by default, + /// it's unlikely that one will need to implement this. + /// + /// # Example + /// + /// This example shows how to use this method with a + /// [`dense::DFA`](crate::dfa::dense::DFA). + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, HalfMatch, Input}; + /// + /// let dfa = dense::DFA::new("foo[0-9]+")?; + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new(b"foo12345"))?); + /// + /// // Even though a match is found after reading the first byte (`a`), + /// // the leftmost first match semantics demand that we find the earliest + /// // match that prefers earlier parts of the pattern over latter parts. + /// let dfa = dense::DFA::new("abc|a")?; + /// let expected = Some(HalfMatch::must(0, 3)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new(b"abc"))?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specific pattern search + /// + /// This example shows how to build a multi-DFA that permits searching for + /// specific patterns. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// dfa::{Automaton, dense}, + /// Anchored, HalfMatch, PatternID, Input, + /// }; + /// + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new().starts_for_each_pattern(true)) + /// .build_many(&["[a-z0-9]{6}", "[a-z][a-z0-9]{5}"])?; + /// let haystack = "foo123".as_bytes(); + /// + /// // Since we are using the default leftmost-first match and both + /// // patterns match at the same starting position, only the first pattern + /// // will be returned in this case when doing a search for any of the + /// // patterns. + /// let expected = Some(HalfMatch::must(0, 6)); + /// let got = dfa.try_search_fwd(&Input::new(haystack))?; + /// assert_eq!(expected, got); + /// + /// // But if we want to check whether some other pattern matches, then we + /// // can provide its pattern ID. + /// let input = Input::new(haystack) + /// .anchored(Anchored::Pattern(PatternID::must(1))); + /// let expected = Some(HalfMatch::must(1, 6)); + /// let got = dfa.try_search_fwd(&input)?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specifying the bounds of a search + /// + /// This example shows how providing the bounds of a search can produce + /// different results than simply sub-slicing the haystack. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, HalfMatch, Input}; + /// + /// // N.B. We disable Unicode here so that we use a simple ASCII word + /// // boundary. Alternatively, we could enable heuristic support for + /// // Unicode word boundaries. + /// let dfa = dense::DFA::new(r"(?-u)\b[0-9]{3}\b")?; + /// let haystack = "foo123bar".as_bytes(); + /// + /// // Since we sub-slice the haystack, the search doesn't know about the + /// // larger context and assumes that `123` is surrounded by word + /// // boundaries. And of course, the match position is reported relative + /// // to the sub-slice as well, which means we get `3` instead of `6`. + /// let input = Input::new(&haystack[3..6]); + /// let expected = Some(HalfMatch::must(0, 3)); + /// let got = dfa.try_search_fwd(&input)?; + /// assert_eq!(expected, got); + /// + /// // But if we provide the bounds of the search within the context of the + /// // entire haystack, then the search can take the surrounding context + /// // into account. (And if we did find a match, it would be reported + /// // as a valid offset into `haystack` instead of its sub-slice.) + /// let input = Input::new(haystack).range(3..6); + /// let expected = None; + /// let got = dfa.try_search_fwd(&input)?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + fn try_search_fwd( + &self, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, MatchError> { + let utf8empty = self.has_empty() && self.is_utf8(); + let hm = match search::find_fwd(&self, input)? { + None => return Ok(None), + Some(hm) if !utf8empty => return Ok(Some(hm)), + Some(hm) => hm, + }; + // We get to this point when we know our DFA can match the empty string + // AND when UTF-8 mode is enabled. In this case, we skip any matches + // whose offset splits a codepoint. Such a match is necessarily a + // zero-width match, because UTF-8 mode requires the underlying NFA + // to be built such that all non-empty matches span valid UTF-8. + // Therefore, any match that ends in the middle of a codepoint cannot + // be part of a span of valid UTF-8 and thus must be an empty match. + // In such cases, we skip it, so as not to report matches that split a + // codepoint. + // + // Note that this is not a checked assumption. Callers *can* provide an + // NFA with UTF-8 mode enabled but produces non-empty matches that span + // invalid UTF-8. But doing so is documented to result in unspecified + // behavior. + empty::skip_splits_fwd(input, hm, hm.offset(), |input| { + let got = search::find_fwd(&self, input)?; + Ok(got.map(|hm| (hm, hm.offset()))) + }) + } + + /// Executes a reverse search and returns the start of the position of the + /// leftmost match that is found. If no match exists, then `None` is + /// returned. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the DFA quitting. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This example shows how to use this method with a + /// [`dense::DFA`](crate::dfa::dense::DFA). In particular, this + /// routine is principally useful when used in conjunction with the + /// [`nfa::thompson::Config::reverse`](crate::nfa::thompson::Config::reverse) + /// configuration. In general, it's unlikely to be correct to use + /// both `try_search_fwd` and `try_search_rev` with the same DFA since + /// any particular DFA will only support searching in one direction with + /// respect to the pattern. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson, + /// dfa::{Automaton, dense}, + /// HalfMatch, Input, + /// }; + /// + /// let dfa = dense::Builder::new() + /// .thompson(thompson::Config::new().reverse(true)) + /// .build("foo[0-9]+")?; + /// let expected = Some(HalfMatch::must(0, 0)); + /// assert_eq!(expected, dfa.try_search_rev(&Input::new(b"foo12345"))?); + /// + /// // Even though a match is found after reading the last byte (`c`), + /// // the leftmost first match semantics demand that we find the earliest + /// // match that prefers earlier parts of the pattern over latter parts. + /// let dfa = dense::Builder::new() + /// .thompson(thompson::Config::new().reverse(true)) + /// .build("abc|c")?; + /// let expected = Some(HalfMatch::must(0, 0)); + /// assert_eq!(expected, dfa.try_search_rev(&Input::new(b"abc"))?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: UTF-8 mode + /// + /// This examples demonstrates that UTF-8 mode applies to reverse + /// DFAs. When UTF-8 mode is enabled in the underlying NFA, then all + /// matches reported must correspond to valid UTF-8 spans. This includes + /// prohibiting zero-width matches that split a codepoint. + /// + /// UTF-8 mode is enabled by default. Notice below how the only zero-width + /// matches reported are those at UTF-8 boundaries: + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense::DFA, Automaton}, + /// nfa::thompson, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .thompson(thompson::Config::new().reverse(true)) + /// .build(r"")?; + /// + /// // Run the reverse DFA to collect all matches. + /// let mut input = Input::new("☃"); + /// let mut matches = vec![]; + /// loop { + /// match dfa.try_search_rev(&input)? { + /// None => break, + /// Some(hm) => { + /// matches.push(hm); + /// if hm.offset() == 0 || input.end() == 0 { + /// break; + /// } else if hm.offset() < input.end() { + /// input.set_end(hm.offset()); + /// } else { + /// // This is only necessary to handle zero-width + /// // matches, which of course occur in this example. + /// // Without this, the search would never advance + /// // backwards beyond the initial match. + /// input.set_end(input.end() - 1); + /// } + /// } + /// } + /// } + /// + /// // No matches split a codepoint. + /// let expected = vec![ + /// HalfMatch::must(0, 3), + /// HalfMatch::must(0, 0), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Now let's look at the same example, but with UTF-8 mode on the + /// original NFA disabled (which results in disabling UTF-8 mode on the + /// DFA): + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense::DFA, Automaton}, + /// nfa::thompson, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .thompson(thompson::Config::new().reverse(true).utf8(false)) + /// .build(r"")?; + /// + /// // Run the reverse DFA to collect all matches. + /// let mut input = Input::new("☃"); + /// let mut matches = vec![]; + /// loop { + /// match dfa.try_search_rev(&input)? { + /// None => break, + /// Some(hm) => { + /// matches.push(hm); + /// if hm.offset() == 0 || input.end() == 0 { + /// break; + /// } else if hm.offset() < input.end() { + /// input.set_end(hm.offset()); + /// } else { + /// // This is only necessary to handle zero-width + /// // matches, which of course occur in this example. + /// // Without this, the search would never advance + /// // backwards beyond the initial match. + /// input.set_end(input.end() - 1); + /// } + /// } + /// } + /// } + /// + /// // No matches split a codepoint. + /// let expected = vec![ + /// HalfMatch::must(0, 3), + /// HalfMatch::must(0, 2), + /// HalfMatch::must(0, 1), + /// HalfMatch::must(0, 0), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + fn try_search_rev( + &self, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, MatchError> { + let utf8empty = self.has_empty() && self.is_utf8(); + let hm = match search::find_rev(self, input)? { + None => return Ok(None), + Some(hm) if !utf8empty => return Ok(Some(hm)), + Some(hm) => hm, + }; + empty::skip_splits_rev(input, hm, hm.offset(), |input| { + let got = search::find_rev(self, input)?; + Ok(got.map(|hm| (hm, hm.offset()))) + }) + } + + /// Executes an overlapping forward search. Matches, if one exists, can be + /// obtained via the [`OverlappingState::get_match`] method. + /// + /// This routine is principally only useful when searching for multiple + /// patterns on inputs where multiple patterns may match the same regions + /// of text. In particular, callers must preserve the automaton's search + /// state from prior calls so that the implementation knows where the last + /// match occurred. + /// + /// When using this routine to implement an iterator of overlapping + /// matches, the `start` of the search should always be set to the end + /// of the last match. If more patterns match at the previous location, + /// then they will be immediately returned. (This is tracked by the given + /// overlapping state.) Otherwise, the search continues at the starting + /// position given. + /// + /// If for some reason you want the search to forget about its previous + /// state and restart the search at a particular position, then setting the + /// state to [`OverlappingState::start`] will accomplish that. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the DFA quitting. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This example shows how to run a basic overlapping search with a + /// [`dense::DFA`](crate::dfa::dense::DFA). Notice that we build the + /// automaton with a `MatchKind::All` configuration. Overlapping searches + /// are unlikely to work as one would expect when using the default + /// `MatchKind::LeftmostFirst` match semantics, since leftmost-first + /// matching is fundamentally incompatible with overlapping searches. + /// Namely, overlapping searches need to report matches as they are seen, + /// where as leftmost-first searches will continue searching even after a + /// match has been observed in order to find the conventional end position + /// of the match. More concretely, leftmost-first searches use dead states + /// to terminate a search after a specific match can no longer be extended. + /// Overlapping searches instead do the opposite by continuing the search + /// to find totally new matches (potentially of other patterns). + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// dfa::{Automaton, OverlappingState, dense}, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new().match_kind(MatchKind::All)) + /// .build_many(&[r"[[:word:]]+$", r"[[:^space:]]+$"])?; + /// let haystack = "@foo"; + /// let mut state = OverlappingState::start(); + /// + /// let expected = Some(HalfMatch::must(1, 4)); + /// dfa.try_search_overlapping_fwd(&Input::new(haystack), &mut state)?; + /// assert_eq!(expected, state.get_match()); + /// + /// // The first pattern also matches at the same position, so re-running + /// // the search will yield another match. Notice also that the first + /// // pattern is returned after the second. This is because the second + /// // pattern begins its match before the first, is therefore an earlier + /// // match and is thus reported first. + /// let expected = Some(HalfMatch::must(0, 4)); + /// dfa.try_search_overlapping_fwd(&Input::new(haystack), &mut state)?; + /// assert_eq!(expected, state.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + fn try_search_overlapping_fwd( + &self, + input: &Input<'_>, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + let utf8empty = self.has_empty() && self.is_utf8(); + search::find_overlapping_fwd(self, input, state)?; + match state.get_match() { + None => Ok(()), + Some(_) if !utf8empty => Ok(()), + Some(_) => skip_empty_utf8_splits_overlapping( + input, + state, + |input, state| { + search::find_overlapping_fwd(self, input, state) + }, + ), + } + } + + /// Executes a reverse overlapping forward search. Matches, if one exists, + /// can be obtained via the [`OverlappingState::get_match`] method. + /// + /// When using this routine to implement an iterator of overlapping + /// matches, the `start` of the search should remain invariant throughout + /// iteration. The `OverlappingState` given to the search will keep track + /// of the current position of the search. (This is because multiple + /// matches may be reported at the same position, so only the search + /// implementation itself knows when to advance the position.) + /// + /// If for some reason you want the search to forget about its previous + /// state and restart the search at a particular position, then setting the + /// state to [`OverlappingState::start`] will accomplish that. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the DFA quitting. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example: UTF-8 mode + /// + /// This examples demonstrates that UTF-8 mode applies to reverse + /// DFAs. When UTF-8 mode is enabled in the underlying NFA, then all + /// matches reported must correspond to valid UTF-8 spans. This includes + /// prohibiting zero-width matches that split a codepoint. + /// + /// UTF-8 mode is enabled by default. Notice below how the only zero-width + /// matches reported are those at UTF-8 boundaries: + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense::DFA, Automaton, OverlappingState}, + /// nfa::thompson, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .thompson(thompson::Config::new().reverse(true)) + /// .build_many(&[r"", r"☃"])?; + /// + /// // Run the reverse DFA to collect all matches. + /// let input = Input::new("☃"); + /// let mut state = OverlappingState::start(); + /// let mut matches = vec![]; + /// loop { + /// dfa.try_search_overlapping_rev(&input, &mut state)?; + /// match state.get_match() { + /// None => break, + /// Some(hm) => matches.push(hm), + /// } + /// } + /// + /// // No matches split a codepoint. + /// let expected = vec![ + /// HalfMatch::must(0, 3), + /// HalfMatch::must(1, 0), + /// HalfMatch::must(0, 0), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Now let's look at the same example, but with UTF-8 mode on the + /// original NFA disabled (which results in disabling UTF-8 mode on the + /// DFA): + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense::DFA, Automaton, OverlappingState}, + /// nfa::thompson, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .thompson(thompson::Config::new().reverse(true).utf8(false)) + /// .build_many(&[r"", r"☃"])?; + /// + /// // Run the reverse DFA to collect all matches. + /// let input = Input::new("☃"); + /// let mut state = OverlappingState::start(); + /// let mut matches = vec![]; + /// loop { + /// dfa.try_search_overlapping_rev(&input, &mut state)?; + /// match state.get_match() { + /// None => break, + /// Some(hm) => matches.push(hm), + /// } + /// } + /// + /// // Now *all* positions match, even within a codepoint, + /// // because we lifted the requirement that matches + /// // correspond to valid UTF-8 spans. + /// let expected = vec![ + /// HalfMatch::must(0, 3), + /// HalfMatch::must(0, 2), + /// HalfMatch::must(0, 1), + /// HalfMatch::must(1, 0), + /// HalfMatch::must(0, 0), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + fn try_search_overlapping_rev( + &self, + input: &Input<'_>, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + let utf8empty = self.has_empty() && self.is_utf8(); + search::find_overlapping_rev(self, input, state)?; + match state.get_match() { + None => Ok(()), + Some(_) if !utf8empty => Ok(()), + Some(_) => skip_empty_utf8_splits_overlapping( + input, + state, + |input, state| { + search::find_overlapping_rev(self, input, state) + }, + ), + } + } + + /// Writes the set of patterns that match anywhere in the given search + /// configuration to `patset`. If multiple patterns match at the same + /// position and the underlying DFA supports overlapping matches, then all + /// matching patterns are written to the given set. + /// + /// Unless all of the patterns in this DFA are anchored, then generally + /// speaking, this will visit every byte in the haystack. + /// + /// This search routine *does not* clear the pattern set. This gives some + /// flexibility to the caller (e.g., running multiple searches with the + /// same pattern set), but does make the API bug-prone if you're reusing + /// the same pattern set for multiple searches but intended them to be + /// independent. + /// + /// If a pattern ID matched but the given `PatternSet` does not have + /// sufficient capacity to store it, then it is not inserted and silently + /// dropped. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the DFA quitting. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This example shows how to find all matching patterns in a haystack, + /// even when some patterns match at the same position as other patterns. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// dfa::{Automaton, dense::DFA}, + /// Input, MatchKind, PatternSet, + /// }; + /// + /// let patterns = &[ + /// r"[[:word:]]+", + /// r"[0-9]+", + /// r"[[:alpha:]]+", + /// r"foo", + /// r"bar", + /// r"barfoo", + /// r"foobar", + /// ]; + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .build_many(patterns)?; + /// + /// let input = Input::new("foobar"); + /// let mut patset = PatternSet::new(dfa.pattern_len()); + /// dfa.try_which_overlapping_matches(&input, &mut patset)?; + /// let expected = vec![0, 2, 3, 4, 6]; + /// let got: Vec<usize> = patset.iter().map(|p| p.as_usize()).collect(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "alloc")] + #[inline] + fn try_which_overlapping_matches( + &self, + input: &Input<'_>, + patset: &mut PatternSet, + ) -> Result<(), MatchError> { + let mut state = OverlappingState::start(); + while let Some(m) = { + self.try_search_overlapping_fwd(input, &mut state)?; + state.get_match() + } { + let _ = patset.insert(m.pattern()); + // There's nothing left to find, so we can stop. Or the caller + // asked us to. + if patset.is_full() || input.get_earliest() { + break; + } + } + Ok(()) + } +} + +unsafe impl<'a, A: Automaton + ?Sized> Automaton for &'a A { + #[inline] + fn next_state(&self, current: StateID, input: u8) -> StateID { + (**self).next_state(current, input) + } + + #[inline] + unsafe fn next_state_unchecked( + &self, + current: StateID, + input: u8, + ) -> StateID { + (**self).next_state_unchecked(current, input) + } + + #[inline] + fn next_eoi_state(&self, current: StateID) -> StateID { + (**self).next_eoi_state(current) + } + + #[inline] + fn start_state( + &self, + config: &start::Config, + ) -> Result<StateID, StartError> { + (**self).start_state(config) + } + + #[inline] + fn start_state_forward( + &self, + input: &Input<'_>, + ) -> Result<StateID, MatchError> { + (**self).start_state_forward(input) + } + + #[inline] + fn start_state_reverse( + &self, + input: &Input<'_>, + ) -> Result<StateID, MatchError> { + (**self).start_state_reverse(input) + } + + #[inline] + fn universal_start_state(&self, mode: Anchored) -> Option<StateID> { + (**self).universal_start_state(mode) + } + + #[inline] + fn is_special_state(&self, id: StateID) -> bool { + (**self).is_special_state(id) + } + + #[inline] + fn is_dead_state(&self, id: StateID) -> bool { + (**self).is_dead_state(id) + } + + #[inline] + fn is_quit_state(&self, id: StateID) -> bool { + (**self).is_quit_state(id) + } + + #[inline] + fn is_match_state(&self, id: StateID) -> bool { + (**self).is_match_state(id) + } + + #[inline] + fn is_start_state(&self, id: StateID) -> bool { + (**self).is_start_state(id) + } + + #[inline] + fn is_accel_state(&self, id: StateID) -> bool { + (**self).is_accel_state(id) + } + + #[inline] + fn pattern_len(&self) -> usize { + (**self).pattern_len() + } + + #[inline] + fn match_len(&self, id: StateID) -> usize { + (**self).match_len(id) + } + + #[inline] + fn match_pattern(&self, id: StateID, index: usize) -> PatternID { + (**self).match_pattern(id, index) + } + + #[inline] + fn has_empty(&self) -> bool { + (**self).has_empty() + } + + #[inline] + fn is_utf8(&self) -> bool { + (**self).is_utf8() + } + + #[inline] + fn is_always_start_anchored(&self) -> bool { + (**self).is_always_start_anchored() + } + + #[inline] + fn accelerator(&self, id: StateID) -> &[u8] { + (**self).accelerator(id) + } + + #[inline] + fn get_prefilter(&self) -> Option<&Prefilter> { + (**self).get_prefilter() + } + + #[inline] + fn try_search_fwd( + &self, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, MatchError> { + (**self).try_search_fwd(input) + } + + #[inline] + fn try_search_rev( + &self, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, MatchError> { + (**self).try_search_rev(input) + } + + #[inline] + fn try_search_overlapping_fwd( + &self, + input: &Input<'_>, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + (**self).try_search_overlapping_fwd(input, state) + } + + #[inline] + fn try_search_overlapping_rev( + &self, + input: &Input<'_>, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + (**self).try_search_overlapping_rev(input, state) + } + + #[cfg(feature = "alloc")] + #[inline] + fn try_which_overlapping_matches( + &self, + input: &Input<'_>, + patset: &mut PatternSet, + ) -> Result<(), MatchError> { + (**self).try_which_overlapping_matches(input, patset) + } +} + +/// Represents the current state of an overlapping search. +/// +/// This is used for overlapping searches since they need to know something +/// about the previous search. For example, when multiple patterns match at the +/// same position, this state tracks the last reported pattern so that the next +/// search knows whether to report another matching pattern or continue with +/// the search at the next position. Additionally, it also tracks which state +/// the last search call terminated in. +/// +/// This type provides little introspection capabilities. The only thing a +/// caller can do is construct it and pass it around to permit search routines +/// to use it to track state, and also ask whether a match has been found. +/// +/// Callers should always provide a fresh state constructed via +/// [`OverlappingState::start`] when starting a new search. Reusing state from +/// a previous search may result in incorrect results. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OverlappingState { + /// The match reported by the most recent overlapping search to use this + /// state. + /// + /// If a search does not find any matches, then it is expected to clear + /// this value. + pub(crate) mat: Option<HalfMatch>, + /// The state ID of the state at which the search was in when the call + /// terminated. When this is a match state, `last_match` must be set to a + /// non-None value. + /// + /// A `None` value indicates the start state of the corresponding + /// automaton. We cannot use the actual ID, since any one automaton may + /// have many start states, and which one is in use depends on several + /// search-time factors. + pub(crate) id: Option<StateID>, + /// The position of the search. + /// + /// When `id` is None (i.e., we are starting a search), this is set to + /// the beginning of the search as given by the caller regardless of its + /// current value. Subsequent calls to an overlapping search pick up at + /// this offset. + pub(crate) at: usize, + /// The index into the matching patterns of the next match to report if the + /// current state is a match state. Note that this may be 1 greater than + /// the total number of matches to report for the current match state. (In + /// which case, no more matches should be reported at the current position + /// and the search should advance to the next position.) + pub(crate) next_match_index: Option<usize>, + /// This is set to true when a reverse overlapping search has entered its + /// EOI transitions. + /// + /// This isn't used in a forward search because it knows to stop once the + /// position exceeds the end of the search range. In a reverse search, + /// since we use unsigned offsets, we don't "know" once we've gone past + /// `0`. So the only way to detect it is with this extra flag. The reverse + /// overlapping search knows to terminate specifically after it has + /// reported all matches after following the EOI transition. + pub(crate) rev_eoi: bool, +} + +impl OverlappingState { + /// Create a new overlapping state that begins at the start state of any + /// automaton. + pub fn start() -> OverlappingState { + OverlappingState { + mat: None, + id: None, + at: 0, + next_match_index: None, + rev_eoi: false, + } + } + + /// Return the match result of the most recent search to execute with this + /// state. + /// + /// A searches will clear this result automatically, such that if no + /// match is found, this will correctly report `None`. + pub fn get_match(&self) -> Option<HalfMatch> { + self.mat + } +} + +/// An error that can occur when computing the start state for a search. +/// +/// Computing a start state can fail for a few reasons, either based on +/// incorrect configuration or even based on whether the look-behind byte +/// triggers a quit state. Typically one does not need to handle this error +/// if you're using [`Automaton::start_state_forward`] (or its reverse +/// counterpart), as that routine automatically converts `StartError` to a +/// [`MatchError`] for you. +/// +/// This error may be returned by the [`Automaton::start_state`] routine. +/// +/// This error implements the `std::error::Error` trait when the `std` feature +/// is enabled. +/// +/// This error is marked as non-exhaustive. New variants may be added in a +/// semver compatible release. +#[non_exhaustive] +#[derive(Clone, Debug)] +pub enum StartError { + /// An error that occurs when a starting configuration's look-behind byte + /// is in this DFA's quit set. + Quit { + /// The quit byte that was found. + byte: u8, + }, + /// An error that occurs when the caller requests an anchored mode that + /// isn't supported by the DFA. + UnsupportedAnchored { + /// The anchored mode given that is unsupported. + mode: Anchored, + }, +} + +impl StartError { + pub(crate) fn quit(byte: u8) -> StartError { + StartError::Quit { byte } + } + + pub(crate) fn unsupported_anchored(mode: Anchored) -> StartError { + StartError::UnsupportedAnchored { mode } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for StartError {} + +impl core::fmt::Display for StartError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match *self { + StartError::Quit { byte } => write!( + f, + "error computing start state because the look-behind byte \ + {:?} triggered a quit state", + crate::util::escape::DebugByte(byte), + ), + StartError::UnsupportedAnchored { mode: Anchored::Yes } => { + write!( + f, + "error computing start state because \ + anchored searches are not supported or enabled" + ) + } + StartError::UnsupportedAnchored { mode: Anchored::No } => { + write!( + f, + "error computing start state because \ + unanchored searches are not supported or enabled" + ) + } + StartError::UnsupportedAnchored { + mode: Anchored::Pattern(pid), + } => { + write!( + f, + "error computing start state because \ + anchored searches for a specific pattern ({}) \ + are not supported or enabled", + pid.as_usize(), + ) + } + } + } +} + +/// Runs the given overlapping `search` function (forwards or backwards) until +/// a match is found whose offset does not split a codepoint. +/// +/// This is *not* always correct to call. It should only be called when the DFA +/// has UTF-8 mode enabled *and* it can produce zero-width matches. Calling +/// this when both of those things aren't true might result in legitimate +/// matches getting skipped. +#[cold] +#[inline(never)] +fn skip_empty_utf8_splits_overlapping<F>( + input: &Input<'_>, + state: &mut OverlappingState, + mut search: F, +) -> Result<(), MatchError> +where + F: FnMut(&Input<'_>, &mut OverlappingState) -> Result<(), MatchError>, +{ + // Note that this routine works for forwards and reverse searches + // even though there's no code here to handle those cases. That's + // because overlapping searches drive themselves to completion via + // `OverlappingState`. So all we have to do is push it until no matches are + // found. + + let mut hm = match state.get_match() { + None => return Ok(()), + Some(hm) => hm, + }; + if input.get_anchored().is_anchored() { + if !input.is_char_boundary(hm.offset()) { + state.mat = None; + } + return Ok(()); + } + while !input.is_char_boundary(hm.offset()) { + search(input, state)?; + hm = match state.get_match() { + None => return Ok(()), + Some(hm) => hm, + }; + } + Ok(()) +} + +/// Write a prefix "state" indicator for fmt::Debug impls. +/// +/// Specifically, this tries to succinctly distinguish the different types of +/// states: dead states, quit states, accelerated states, start states and +/// match states. It even accounts for the possible overlapping of different +/// state types. +pub(crate) fn fmt_state_indicator<A: Automaton>( + f: &mut core::fmt::Formatter<'_>, + dfa: A, + id: StateID, +) -> core::fmt::Result { + if dfa.is_dead_state(id) { + write!(f, "D")?; + if dfa.is_start_state(id) { + write!(f, ">")?; + } else { + write!(f, " ")?; + } + } else if dfa.is_quit_state(id) { + write!(f, "Q ")?; + } else if dfa.is_start_state(id) { + if dfa.is_accel_state(id) { + write!(f, "A>")?; + } else { + write!(f, " >")?; + } + } else if dfa.is_match_state(id) { + if dfa.is_accel_state(id) { + write!(f, "A*")?; + } else { + write!(f, " *")?; + } + } else if dfa.is_accel_state(id) { + write!(f, "A ")?; + } else { + write!(f, " ")?; + } + Ok(()) +} + +#[cfg(all(test, feature = "syntax", feature = "dfa-build"))] +mod tests { + // A basic test ensuring that our Automaton trait is object safe. (This is + // the main reason why we don't define the search routines as generic over + // Into<Input>.) + #[test] + fn object_safe() { + use crate::{ + dfa::{dense, Automaton}, + HalfMatch, Input, + }; + + let dfa = dense::DFA::new("abc").unwrap(); + let dfa: &dyn Automaton = &dfa; + assert_eq!( + Ok(Some(HalfMatch::must(0, 6))), + dfa.try_search_fwd(&Input::new(b"xyzabcxyz")), + ); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/dense.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/dense.rs new file mode 100644 index 0000000..e878375 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/dense.rs @@ -0,0 +1,5282 @@ +/*! +Types and routines specific to dense DFAs. + +This module is the home of [`dense::DFA`](DFA). + +This module also contains a [`dense::Builder`](Builder) and a +[`dense::Config`](Config) for building and configuring a dense DFA. +*/ + +#[cfg(feature = "dfa-build")] +use core::cmp; +use core::{fmt, iter, mem::size_of, slice}; + +#[cfg(feature = "dfa-build")] +use alloc::{ + collections::{BTreeMap, BTreeSet}, + vec, + vec::Vec, +}; + +#[cfg(feature = "dfa-build")] +use crate::{ + dfa::{ + accel::Accel, determinize, minimize::Minimizer, remapper::Remapper, + sparse, + }, + nfa::thompson, + util::{look::LookMatcher, search::MatchKind}, +}; +use crate::{ + dfa::{ + accel::Accels, + automaton::{fmt_state_indicator, Automaton, StartError}, + special::Special, + start::StartKind, + DEAD, + }, + util::{ + alphabet::{self, ByteClasses, ByteSet}, + int::{Pointer, Usize}, + prefilter::Prefilter, + primitives::{PatternID, StateID}, + search::Anchored, + start::{self, Start, StartByteMap}, + wire::{self, DeserializeError, Endian, SerializeError}, + }, +}; + +/// The label that is pre-pended to a serialized DFA. +const LABEL: &str = "rust-regex-automata-dfa-dense"; + +/// The format version of dense regexes. This version gets incremented when a +/// change occurs. A change may not necessarily be a breaking change, but the +/// version does permit good error messages in the case where a breaking change +/// is made. +const VERSION: u32 = 2; + +/// The configuration used for compiling a dense DFA. +/// +/// As a convenience, [`DFA::config`] is an alias for [`Config::new`]. The +/// advantage of the former is that it often lets you avoid importing the +/// `Config` type directly. +/// +/// A dense DFA configuration is a simple data object that is typically used +/// with [`dense::Builder::configure`](self::Builder::configure). +/// +/// The default configuration guarantees that a search will never return +/// a "quit" error, although it is possible for a search to fail if +/// [`Config::starts_for_each_pattern`] wasn't enabled (which it is +/// not by default) and an [`Anchored::Pattern`] mode is requested via +/// [`Input`](crate::Input). +#[cfg(feature = "dfa-build")] +#[derive(Clone, Debug, Default)] +pub struct Config { + // As with other configuration types in this crate, we put all our knobs + // in options so that we can distinguish between "default" and "not set." + // This makes it possible to easily combine multiple configurations + // without default values overwriting explicitly specified values. See the + // 'overwrite' method. + // + // For docs on the fields below, see the corresponding method setters. + accelerate: Option<bool>, + pre: Option<Option<Prefilter>>, + minimize: Option<bool>, + match_kind: Option<MatchKind>, + start_kind: Option<StartKind>, + starts_for_each_pattern: Option<bool>, + byte_classes: Option<bool>, + unicode_word_boundary: Option<bool>, + quitset: Option<ByteSet>, + specialize_start_states: Option<bool>, + dfa_size_limit: Option<Option<usize>>, + determinize_size_limit: Option<Option<usize>>, +} + +#[cfg(feature = "dfa-build")] +impl Config { + /// Return a new default dense DFA compiler configuration. + pub fn new() -> Config { + Config::default() + } + + /// Enable state acceleration. + /// + /// When enabled, DFA construction will analyze each state to determine + /// whether it is eligible for simple acceleration. Acceleration typically + /// occurs when most of a state's transitions loop back to itself, leaving + /// only a select few bytes that will exit the state. When this occurs, + /// other routines like `memchr` can be used to look for those bytes which + /// may be much faster than traversing the DFA. + /// + /// Callers may elect to disable this if consistent performance is more + /// desirable than variable performance. Namely, acceleration can sometimes + /// make searching slower than it otherwise would be if the transitions + /// that leave accelerated states are traversed frequently. + /// + /// See [`Automaton::accelerator`] for an example. + /// + /// This is enabled by default. + pub fn accelerate(mut self, yes: bool) -> Config { + self.accelerate = Some(yes); + self + } + + /// Set a prefilter to be used whenever a start state is entered. + /// + /// A [`Prefilter`] in this context is meant to accelerate searches by + /// looking for literal prefixes that every match for the corresponding + /// pattern (or patterns) must start with. Once a prefilter produces a + /// match, the underlying search routine continues on to try and confirm + /// the match. + /// + /// Be warned that setting a prefilter does not guarantee that the search + /// will be faster. While it's usually a good bet, if the prefilter + /// produces a lot of false positive candidates (i.e., positions matched + /// by the prefilter but not by the regex), then the overall result can + /// be slower than if you had just executed the regex engine without any + /// prefilters. + /// + /// Note that unless [`Config::specialize_start_states`] has been + /// explicitly set, then setting this will also enable (when `pre` is + /// `Some`) or disable (when `pre` is `None`) start state specialization. + /// This occurs because without start state specialization, a prefilter + /// is likely to be less effective. And without a prefilter, start state + /// specialization is usually pointless. + /// + /// **WARNING:** Note that prefilters are not preserved as part of + /// serialization. Serializing a DFA will drop its prefilter. + /// + /// By default no prefilter is set. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense::DFA, Automaton}, + /// util::prefilter::Prefilter, + /// Input, HalfMatch, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["foo", "bar"]); + /// let re = DFA::builder() + /// .configure(DFA::config().prefilter(pre)) + /// .build(r"(foo|bar)[a-z]+")?; + /// let input = Input::new("foo1 barfox bar"); + /// assert_eq!( + /// Some(HalfMatch::must(0, 11)), + /// re.try_search_fwd(&input)?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Be warned though that an incorrect prefilter can lead to incorrect + /// results! + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense::DFA, Automaton}, + /// util::prefilter::Prefilter, + /// Input, HalfMatch, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["foo", "car"]); + /// let re = DFA::builder() + /// .configure(DFA::config().prefilter(pre)) + /// .build(r"(foo|bar)[a-z]+")?; + /// let input = Input::new("foo1 barfox bar"); + /// assert_eq!( + /// // No match reported even though there clearly is one! + /// None, + /// re.try_search_fwd(&input)?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn prefilter(mut self, pre: Option<Prefilter>) -> Config { + self.pre = Some(pre); + if self.specialize_start_states.is_none() { + self.specialize_start_states = + Some(self.get_prefilter().is_some()); + } + self + } + + /// Minimize the DFA. + /// + /// When enabled, the DFA built will be minimized such that it is as small + /// as possible. + /// + /// Whether one enables minimization or not depends on the types of costs + /// you're willing to pay and how much you care about its benefits. In + /// particular, minimization has worst case `O(n*k*logn)` time and `O(k*n)` + /// space, where `n` is the number of DFA states and `k` is the alphabet + /// size. In practice, minimization can be quite costly in terms of both + /// space and time, so it should only be done if you're willing to wait + /// longer to produce a DFA. In general, you might want a minimal DFA in + /// the following circumstances: + /// + /// 1. You would like to optimize for the size of the automaton. This can + /// manifest in one of two ways. Firstly, if you're converting the + /// DFA into Rust code (or a table embedded in the code), then a minimal + /// DFA will translate into a corresponding reduction in code size, and + /// thus, also the final compiled binary size. Secondly, if you are + /// building many DFAs and putting them on the heap, you'll be able to + /// fit more if they are smaller. Note though that building a minimal + /// DFA itself requires additional space; you only realize the space + /// savings once the minimal DFA is constructed (at which point, the + /// space used for minimization is freed). + /// 2. You've observed that a smaller DFA results in faster match + /// performance. Naively, this isn't guaranteed since there is no + /// inherent difference between matching with a bigger-than-minimal + /// DFA and a minimal DFA. However, a smaller DFA may make use of your + /// CPU's cache more efficiently. + /// 3. You are trying to establish an equivalence between regular + /// languages. The standard method for this is to build a minimal DFA + /// for each language and then compare them. If the DFAs are equivalent + /// (up to state renaming), then the languages are equivalent. + /// + /// Typically, minimization only makes sense as an offline process. That + /// is, one might minimize a DFA before serializing it to persistent + /// storage. In practical terms, minimization can take around an order of + /// magnitude more time than compiling the initial DFA via determinization. + /// + /// This option is disabled by default. + pub fn minimize(mut self, yes: bool) -> Config { + self.minimize = Some(yes); + self + } + + /// Set the desired match semantics. + /// + /// The default is [`MatchKind::LeftmostFirst`], which corresponds to the + /// match semantics of Perl-like regex engines. That is, when multiple + /// patterns would match at the same leftmost position, the pattern that + /// appears first in the concrete syntax is chosen. + /// + /// Currently, the only other kind of match semantics supported is + /// [`MatchKind::All`]. This corresponds to classical DFA construction + /// where all possible matches are added to the DFA. + /// + /// Typically, `All` is used when one wants to execute an overlapping + /// search and `LeftmostFirst` otherwise. In particular, it rarely makes + /// sense to use `All` with the various "leftmost" find routines, since the + /// leftmost routines depend on the `LeftmostFirst` automata construction + /// strategy. Specifically, `LeftmostFirst` adds dead states to the DFA + /// as a way to terminate the search and report a match. `LeftmostFirst` + /// also supports non-greedy matches using this strategy where as `All` + /// does not. + /// + /// # Example: overlapping search + /// + /// This example shows the typical use of `MatchKind::All`, which is to + /// report overlapping matches. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// dfa::{Automaton, OverlappingState, dense}, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new().match_kind(MatchKind::All)) + /// .build_many(&[r"\w+$", r"\S+$"])?; + /// let input = Input::new("@foo"); + /// let mut state = OverlappingState::start(); + /// + /// let expected = Some(HalfMatch::must(1, 4)); + /// dfa.try_search_overlapping_fwd(&input, &mut state)?; + /// assert_eq!(expected, state.get_match()); + /// + /// // The first pattern also matches at the same position, so re-running + /// // the search will yield another match. Notice also that the first + /// // pattern is returned after the second. This is because the second + /// // pattern begins its match before the first, is therefore an earlier + /// // match and is thus reported first. + /// let expected = Some(HalfMatch::must(0, 4)); + /// dfa.try_search_overlapping_fwd(&input, &mut state)?; + /// assert_eq!(expected, state.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: reverse automaton to find start of match + /// + /// Another example for using `MatchKind::All` is for constructing a + /// reverse automaton to find the start of a match. `All` semantics are + /// used for this in order to find the longest possible match, which + /// corresponds to the leftmost starting position. + /// + /// Note that if you need the starting position then + /// [`dfa::regex::Regex`](crate::dfa::regex::Regex) will handle this for + /// you, so it's usually not necessary to do this yourself. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense, Automaton, StartKind}, + /// nfa::thompson::NFA, + /// Anchored, HalfMatch, Input, MatchKind, + /// }; + /// + /// let haystack = "123foobar456".as_bytes(); + /// let pattern = r"[a-z]+r"; + /// + /// let dfa_fwd = dense::DFA::new(pattern)?; + /// let dfa_rev = dense::Builder::new() + /// .thompson(NFA::config().reverse(true)) + /// .configure(dense::Config::new() + /// // This isn't strictly necessary since both anchored and + /// // unanchored searches are supported by default. But since + /// // finding the start-of-match only requires anchored searches, + /// // we can get rid of the unanchored configuration and possibly + /// // slim down our DFA considerably. + /// .start_kind(StartKind::Anchored) + /// .match_kind(MatchKind::All) + /// ) + /// .build(pattern)?; + /// let expected_fwd = HalfMatch::must(0, 9); + /// let expected_rev = HalfMatch::must(0, 3); + /// let got_fwd = dfa_fwd.try_search_fwd(&Input::new(haystack))?.unwrap(); + /// // Here we don't specify the pattern to search for since there's only + /// // one pattern and we're doing a leftmost search. But if this were an + /// // overlapping search, you'd need to specify the pattern that matched + /// // in the forward direction. (Otherwise, you might wind up finding the + /// // starting position of a match of some other pattern.) That in turn + /// // requires building the reverse automaton with starts_for_each_pattern + /// // enabled. Indeed, this is what Regex does internally. + /// let input = Input::new(haystack) + /// .range(..got_fwd.offset()) + /// .anchored(Anchored::Yes); + /// let got_rev = dfa_rev.try_search_rev(&input)?.unwrap(); + /// assert_eq!(expected_fwd, got_fwd); + /// assert_eq!(expected_rev, got_rev); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn match_kind(mut self, kind: MatchKind) -> Config { + self.match_kind = Some(kind); + self + } + + /// The type of starting state configuration to use for a DFA. + /// + /// By default, the starting state configuration is [`StartKind::Both`]. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense::DFA, Automaton, StartKind}, + /// Anchored, HalfMatch, Input, + /// }; + /// + /// let haystack = "quux foo123"; + /// let expected = HalfMatch::must(0, 11); + /// + /// // By default, DFAs support both anchored and unanchored searches. + /// let dfa = DFA::new(r"[0-9]+")?; + /// let input = Input::new(haystack); + /// assert_eq!(Some(expected), dfa.try_search_fwd(&input)?); + /// + /// // But if we only need anchored searches, then we can build a DFA + /// // that only supports anchored searches. This leads to a smaller DFA + /// // (potentially significantly smaller in some cases), but a DFA that + /// // will panic if you try to use it with an unanchored search. + /// let dfa = DFA::builder() + /// .configure(DFA::config().start_kind(StartKind::Anchored)) + /// .build(r"[0-9]+")?; + /// let input = Input::new(haystack) + /// .range(8..) + /// .anchored(Anchored::Yes); + /// assert_eq!(Some(expected), dfa.try_search_fwd(&input)?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn start_kind(mut self, kind: StartKind) -> Config { + self.start_kind = Some(kind); + self + } + + /// Whether to compile a separate start state for each pattern in the + /// automaton. + /// + /// When enabled, a separate **anchored** start state is added for each + /// pattern in the DFA. When this start state is used, then the DFA will + /// only search for matches for the pattern specified, even if there are + /// other patterns in the DFA. + /// + /// The main downside of this option is that it can potentially increase + /// the size of the DFA and/or increase the time it takes to build the DFA. + /// + /// There are a few reasons one might want to enable this (it's disabled + /// by default): + /// + /// 1. When looking for the start of an overlapping match (using a + /// reverse DFA), doing it correctly requires starting the reverse search + /// using the starting state of the pattern that matched in the forward + /// direction. Indeed, when building a [`Regex`](crate::dfa::regex::Regex), + /// it will automatically enable this option when building the reverse DFA + /// internally. + /// 2. When you want to use a DFA with multiple patterns to both search + /// for matches of any pattern or to search for anchored matches of one + /// particular pattern while using the same DFA. (Otherwise, you would need + /// to compile a new DFA for each pattern.) + /// 3. Since the start states added for each pattern are anchored, if you + /// compile an unanchored DFA with one pattern while also enabling this + /// option, then you can use the same DFA to perform anchored or unanchored + /// searches. The latter you get with the standard search APIs. The former + /// you get from the various `_at` search methods that allow you specify a + /// pattern ID to search for. + /// + /// By default this is disabled. + /// + /// # Example + /// + /// This example shows how to use this option to permit the same DFA to + /// run both anchored and unanchored searches for a single pattern. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{dense, Automaton}, + /// Anchored, HalfMatch, PatternID, Input, + /// }; + /// + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new().starts_for_each_pattern(true)) + /// .build(r"foo[0-9]+")?; + /// let haystack = "quux foo123"; + /// + /// // Here's a normal unanchored search. Notice that we use 'None' for the + /// // pattern ID. Since the DFA was built as an unanchored machine, it + /// // use its default unanchored starting state. + /// let expected = HalfMatch::must(0, 11); + /// let input = Input::new(haystack); + /// assert_eq!(Some(expected), dfa.try_search_fwd(&input)?); + /// // But now if we explicitly specify the pattern to search ('0' being + /// // the only pattern in the DFA), then it will use the starting state + /// // for that specific pattern which is always anchored. Since the + /// // pattern doesn't have a match at the beginning of the haystack, we + /// // find nothing. + /// let input = Input::new(haystack) + /// .anchored(Anchored::Pattern(PatternID::must(0))); + /// assert_eq!(None, dfa.try_search_fwd(&input)?); + /// // And finally, an anchored search is not the same as putting a '^' at + /// // beginning of the pattern. An anchored search can only match at the + /// // beginning of the *search*, which we can change: + /// let input = Input::new(haystack) + /// .anchored(Anchored::Pattern(PatternID::must(0))) + /// .range(5..); + /// assert_eq!(Some(expected), dfa.try_search_fwd(&input)?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn starts_for_each_pattern(mut self, yes: bool) -> Config { + self.starts_for_each_pattern = Some(yes); + self + } + + /// Whether to attempt to shrink the size of the DFA's alphabet or not. + /// + /// This option is enabled by default and should never be disabled unless + /// one is debugging a generated DFA. + /// + /// When enabled, the DFA will use a map from all possible bytes to their + /// corresponding equivalence class. Each equivalence class represents a + /// set of bytes that does not discriminate between a match and a non-match + /// in the DFA. For example, the pattern `[ab]+` has at least two + /// equivalence classes: a set containing `a` and `b` and a set containing + /// every byte except for `a` and `b`. `a` and `b` are in the same + /// equivalence class because they never discriminate between a match and a + /// non-match. + /// + /// The advantage of this map is that the size of the transition table + /// can be reduced drastically from `#states * 256 * sizeof(StateID)` to + /// `#states * k * sizeof(StateID)` where `k` is the number of equivalence + /// classes (rounded up to the nearest power of 2). As a result, total + /// space usage can decrease substantially. Moreover, since a smaller + /// alphabet is used, DFA compilation becomes faster as well. + /// + /// **WARNING:** This is only useful for debugging DFAs. Disabling this + /// does not yield any speed advantages. Namely, even when this is + /// disabled, a byte class map is still used while searching. The only + /// difference is that every byte will be forced into its own distinct + /// equivalence class. This is useful for debugging the actual generated + /// transitions because it lets one see the transitions defined on actual + /// bytes instead of the equivalence classes. + pub fn byte_classes(mut self, yes: bool) -> Config { + self.byte_classes = Some(yes); + self + } + + /// Heuristically enable Unicode word boundaries. + /// + /// When set, this will attempt to implement Unicode word boundaries as if + /// they were ASCII word boundaries. This only works when the search input + /// is ASCII only. If a non-ASCII byte is observed while searching, then a + /// [`MatchError::quit`](crate::MatchError::quit) error is returned. + /// + /// A possible alternative to enabling this option is to simply use an + /// ASCII word boundary, e.g., via `(?-u:\b)`. The main reason to use this + /// option is if you absolutely need Unicode support. This option lets one + /// use a fast search implementation (a DFA) for some potentially very + /// common cases, while providing the option to fall back to some other + /// regex engine to handle the general case when an error is returned. + /// + /// If the pattern provided has no Unicode word boundary in it, then this + /// option has no effect. (That is, quitting on a non-ASCII byte only + /// occurs when this option is enabled _and_ a Unicode word boundary is + /// present in the pattern.) + /// + /// This is almost equivalent to setting all non-ASCII bytes to be quit + /// bytes. The only difference is that this will cause non-ASCII bytes to + /// be quit bytes _only_ when a Unicode word boundary is present in the + /// pattern. + /// + /// When enabling this option, callers _must_ be prepared to handle + /// a [`MatchError`](crate::MatchError) error during search. + /// When using a [`Regex`](crate::dfa::regex::Regex), this corresponds + /// to using the `try_` suite of methods. Alternatively, if + /// callers can guarantee that their input is ASCII only, then a + /// [`MatchError::quit`](crate::MatchError::quit) error will never be + /// returned while searching. + /// + /// This is disabled by default. + /// + /// # Example + /// + /// This example shows how to heuristically enable Unicode word boundaries + /// in a pattern. It also shows what happens when a search comes across a + /// non-ASCII byte. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{Automaton, dense}, + /// HalfMatch, Input, MatchError, + /// }; + /// + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new().unicode_word_boundary(true)) + /// .build(r"\b[0-9]+\b")?; + /// + /// // The match occurs before the search ever observes the snowman + /// // character, so no error occurs. + /// let haystack = "foo 123 ☃".as_bytes(); + /// let expected = Some(HalfMatch::must(0, 7)); + /// let got = dfa.try_search_fwd(&Input::new(haystack))?; + /// assert_eq!(expected, got); + /// + /// // Notice that this search fails, even though the snowman character + /// // occurs after the ending match offset. This is because search + /// // routines read one byte past the end of the search to account for + /// // look-around, and indeed, this is required here to determine whether + /// // the trailing \b matches. + /// let haystack = "foo 123 ☃".as_bytes(); + /// let expected = MatchError::quit(0xE2, 8); + /// let got = dfa.try_search_fwd(&Input::new(haystack)); + /// assert_eq!(Err(expected), got); + /// + /// // Another example is executing a search where the span of the haystack + /// // we specify is all ASCII, but there is non-ASCII just before it. This + /// // correctly also reports an error. + /// let input = Input::new("β123").range(2..); + /// let expected = MatchError::quit(0xB2, 1); + /// let got = dfa.try_search_fwd(&input); + /// assert_eq!(Err(expected), got); + /// + /// // And similarly for the trailing word boundary. + /// let input = Input::new("123β").range(..3); + /// let expected = MatchError::quit(0xCE, 3); + /// let got = dfa.try_search_fwd(&input); + /// assert_eq!(Err(expected), got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn unicode_word_boundary(mut self, yes: bool) -> Config { + // We have a separate option for this instead of just setting the + // appropriate quit bytes here because we don't want to set quit bytes + // for every regex. We only want to set them when the regex contains a + // Unicode word boundary. + self.unicode_word_boundary = Some(yes); + self + } + + /// Add a "quit" byte to the DFA. + /// + /// When a quit byte is seen during search time, then search will return + /// a [`MatchError::quit`](crate::MatchError::quit) error indicating the + /// offset at which the search stopped. + /// + /// A quit byte will always overrule any other aspects of a regex. For + /// example, if the `x` byte is added as a quit byte and the regex `\w` is + /// used, then observing `x` will cause the search to quit immediately + /// despite the fact that `x` is in the `\w` class. + /// + /// This mechanism is primarily useful for heuristically enabling certain + /// features like Unicode word boundaries in a DFA. Namely, if the input + /// to search is ASCII, then a Unicode word boundary can be implemented + /// via an ASCII word boundary with no change in semantics. Thus, a DFA + /// can attempt to match a Unicode word boundary but give up as soon as it + /// observes a non-ASCII byte. Indeed, if callers set all non-ASCII bytes + /// to be quit bytes, then Unicode word boundaries will be permitted when + /// building DFAs. Of course, callers should enable + /// [`Config::unicode_word_boundary`] if they want this behavior instead. + /// (The advantage being that non-ASCII quit bytes will only be added if a + /// Unicode word boundary is in the pattern.) + /// + /// When enabling this option, callers _must_ be prepared to handle a + /// [`MatchError`](crate::MatchError) error during search. When using a + /// [`Regex`](crate::dfa::regex::Regex), this corresponds to using the + /// `try_` suite of methods. + /// + /// By default, there are no quit bytes set. + /// + /// # Panics + /// + /// This panics if heuristic Unicode word boundaries are enabled and any + /// non-ASCII byte is removed from the set of quit bytes. Namely, enabling + /// Unicode word boundaries requires setting every non-ASCII byte to a quit + /// byte. So if the caller attempts to undo any of that, then this will + /// panic. + /// + /// # Example + /// + /// This example shows how to cause a search to terminate if it sees a + /// `\n` byte. This could be useful if, for example, you wanted to prevent + /// a user supplied pattern from matching across a line boundary. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::{Automaton, dense}, Input, MatchError}; + /// + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new().quit(b'\n', true)) + /// .build(r"foo\p{any}+bar")?; + /// + /// let haystack = "foo\nbar".as_bytes(); + /// // Normally this would produce a match, since \p{any} contains '\n'. + /// // But since we instructed the automaton to enter a quit state if a + /// // '\n' is observed, this produces a match error instead. + /// let expected = MatchError::quit(b'\n', 3); + /// let got = dfa.try_search_fwd(&Input::new(haystack)).unwrap_err(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn quit(mut self, byte: u8, yes: bool) -> Config { + if self.get_unicode_word_boundary() && !byte.is_ascii() && !yes { + panic!( + "cannot set non-ASCII byte to be non-quit when \ + Unicode word boundaries are enabled" + ); + } + if self.quitset.is_none() { + self.quitset = Some(ByteSet::empty()); + } + if yes { + self.quitset.as_mut().unwrap().add(byte); + } else { + self.quitset.as_mut().unwrap().remove(byte); + } + self + } + + /// Enable specializing start states in the DFA. + /// + /// When start states are specialized, an implementor of a search routine + /// using a lazy DFA can tell when the search has entered a starting state. + /// When start states aren't specialized, then it is impossible to know + /// whether the search has entered a start state. + /// + /// Ideally, this option wouldn't need to exist and we could always + /// specialize start states. The problem is that start states can be quite + /// active. This in turn means that an efficient search routine is likely + /// to ping-pong between a heavily optimized hot loop that handles most + /// states and to a less optimized specialized handling of start states. + /// This causes branches to get heavily mispredicted and overall can + /// materially decrease throughput. Therefore, specializing start states + /// should only be enabled when it is needed. + /// + /// Knowing whether a search is in a start state is typically useful when a + /// prefilter is active for the search. A prefilter is typically only run + /// when in a start state and a prefilter can greatly accelerate a search. + /// Therefore, the possible cost of specializing start states is worth it + /// in this case. Otherwise, if you have no prefilter, there is likely no + /// reason to specialize start states. + /// + /// This is disabled by default, but note that it is automatically + /// enabled (or disabled) if [`Config::prefilter`] is set. Namely, unless + /// `specialize_start_states` has already been set, [`Config::prefilter`] + /// will automatically enable or disable it based on whether a prefilter + /// is present or not, respectively. This is done because a prefilter's + /// effectiveness is rooted in being executed whenever the DFA is in a + /// start state, and that's only possible to do when they are specialized. + /// + /// Note that it is plausibly reasonable to _disable_ this option + /// explicitly while _enabling_ a prefilter. In that case, a prefilter + /// will still be run at the beginning of a search, but never again. This + /// in theory could strike a good balance if you're in a situation where a + /// prefilter is likely to produce many false positive candidates. + /// + /// # Example + /// + /// This example shows how to enable start state specialization and then + /// shows how to check whether a state is a start state or not. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, Input}; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().specialize_start_states(true)) + /// .build(r"[a-z]+")?; + /// + /// let haystack = "123 foobar 4567".as_bytes(); + /// let sid = dfa.start_state_forward(&Input::new(haystack))?; + /// // The ID returned by 'start_state_forward' will always be tagged as + /// // a start state when start state specialization is enabled. + /// assert!(dfa.is_special_state(sid)); + /// assert!(dfa.is_start_state(sid)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Compare the above with the default DFA configuration where start states + /// are _not_ specialized. In this case, the start state is not tagged at + /// all: + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, Input}; + /// + /// let dfa = DFA::new(r"[a-z]+")?; + /// + /// let haystack = "123 foobar 4567"; + /// let sid = dfa.start_state_forward(&Input::new(haystack))?; + /// // Start states are not special in the default configuration! + /// assert!(!dfa.is_special_state(sid)); + /// assert!(!dfa.is_start_state(sid)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn specialize_start_states(mut self, yes: bool) -> Config { + self.specialize_start_states = Some(yes); + self + } + + /// Set a size limit on the total heap used by a DFA. + /// + /// This size limit is expressed in bytes and is applied during + /// determinization of an NFA into a DFA. If the DFA's heap usage, and only + /// the DFA, exceeds this configured limit, then determinization is stopped + /// and an error is returned. + /// + /// This limit does not apply to auxiliary storage used during + /// determinization that isn't part of the generated DFA. + /// + /// This limit is only applied during determinization. Currently, there is + /// no way to post-pone this check to after minimization if minimization + /// was enabled. + /// + /// The total limit on heap used during determinization is the sum of the + /// DFA and determinization size limits. + /// + /// The default is no limit. + /// + /// # Example + /// + /// This example shows a DFA that fails to build because of a configured + /// size limit. This particular example also serves as a cautionary tale + /// demonstrating just how big DFAs with large Unicode character classes + /// can get. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::{dense, Automaton}, Input}; + /// + /// // 6MB isn't enough! + /// dense::Builder::new() + /// .configure(dense::Config::new().dfa_size_limit(Some(6_000_000))) + /// .build(r"\w{20}") + /// .unwrap_err(); + /// + /// // ... but 7MB probably is! + /// // (Note that DFA sizes aren't necessarily stable between releases.) + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new().dfa_size_limit(Some(7_000_000))) + /// .build(r"\w{20}")?; + /// let haystack = "A".repeat(20).into_bytes(); + /// assert!(dfa.try_search_fwd(&Input::new(&haystack))?.is_some()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// While one needs a little more than 6MB to represent `\w{20}`, it + /// turns out that you only need a little more than 6KB to represent + /// `(?-u:\w{20})`. So only use Unicode if you need it! + /// + /// As with [`Config::determinize_size_limit`], the size of a DFA is + /// influenced by other factors, such as what start state configurations + /// to support. For example, if you only need unanchored searches and not + /// anchored searches, then configuring the DFA to only support unanchored + /// searches can reduce its size. By default, DFAs support both unanchored + /// and anchored searches. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::{dense, Automaton, StartKind}, Input}; + /// + /// // 3MB isn't enough! + /// dense::Builder::new() + /// .configure(dense::Config::new() + /// .dfa_size_limit(Some(3_000_000)) + /// .start_kind(StartKind::Unanchored) + /// ) + /// .build(r"\w{20}") + /// .unwrap_err(); + /// + /// // ... but 4MB probably is! + /// // (Note that DFA sizes aren't necessarily stable between releases.) + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new() + /// .dfa_size_limit(Some(4_000_000)) + /// .start_kind(StartKind::Unanchored) + /// ) + /// .build(r"\w{20}")?; + /// let haystack = "A".repeat(20).into_bytes(); + /// assert!(dfa.try_search_fwd(&Input::new(&haystack))?.is_some()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn dfa_size_limit(mut self, bytes: Option<usize>) -> Config { + self.dfa_size_limit = Some(bytes); + self + } + + /// Set a size limit on the total heap used by determinization. + /// + /// This size limit is expressed in bytes and is applied during + /// determinization of an NFA into a DFA. If the heap used for auxiliary + /// storage during determinization (memory that is not in the DFA but + /// necessary for building the DFA) exceeds this configured limit, then + /// determinization is stopped and an error is returned. + /// + /// This limit does not apply to heap used by the DFA itself. + /// + /// The total limit on heap used during determinization is the sum of the + /// DFA and determinization size limits. + /// + /// The default is no limit. + /// + /// # Example + /// + /// This example shows a DFA that fails to build because of a + /// configured size limit on the amount of heap space used by + /// determinization. This particular example complements the example for + /// [`Config::dfa_size_limit`] by demonstrating that not only does Unicode + /// potentially make DFAs themselves big, but it also results in more + /// auxiliary storage during determinization. (Although, auxiliary storage + /// is still not as much as the DFA itself.) + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// # if !cfg!(target_pointer_width = "64") { return Ok(()); } // see #1039 + /// use regex_automata::{dfa::{dense, Automaton}, Input}; + /// + /// // 700KB isn't enough! + /// dense::Builder::new() + /// .configure(dense::Config::new() + /// .determinize_size_limit(Some(700_000)) + /// ) + /// .build(r"\w{20}") + /// .unwrap_err(); + /// + /// // ... but 800KB probably is! + /// // (Note that auxiliary storage sizes aren't necessarily stable between + /// // releases.) + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new() + /// .determinize_size_limit(Some(800_000)) + /// ) + /// .build(r"\w{20}")?; + /// let haystack = "A".repeat(20).into_bytes(); + /// assert!(dfa.try_search_fwd(&Input::new(&haystack))?.is_some()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Note that some parts of the configuration on a DFA can have a + /// big impact on how big the DFA is, and thus, how much memory is + /// used. For example, the default setting for [`Config::start_kind`] is + /// [`StartKind::Both`]. But if you only need an anchored search, for + /// example, then it can be much cheaper to build a DFA that only supports + /// anchored searches. (Running an unanchored search with it would panic.) + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// # if !cfg!(target_pointer_width = "64") { return Ok(()); } // see #1039 + /// use regex_automata::{ + /// dfa::{dense, Automaton, StartKind}, + /// Anchored, Input, + /// }; + /// + /// // 200KB isn't enough! + /// dense::Builder::new() + /// .configure(dense::Config::new() + /// .determinize_size_limit(Some(200_000)) + /// .start_kind(StartKind::Anchored) + /// ) + /// .build(r"\w{20}") + /// .unwrap_err(); + /// + /// // ... but 300KB probably is! + /// // (Note that auxiliary storage sizes aren't necessarily stable between + /// // releases.) + /// let dfa = dense::Builder::new() + /// .configure(dense::Config::new() + /// .determinize_size_limit(Some(300_000)) + /// .start_kind(StartKind::Anchored) + /// ) + /// .build(r"\w{20}")?; + /// let haystack = "A".repeat(20).into_bytes(); + /// let input = Input::new(&haystack).anchored(Anchored::Yes); + /// assert!(dfa.try_search_fwd(&input)?.is_some()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn determinize_size_limit(mut self, bytes: Option<usize>) -> Config { + self.determinize_size_limit = Some(bytes); + self + } + + /// Returns whether this configuration has enabled simple state + /// acceleration. + pub fn get_accelerate(&self) -> bool { + self.accelerate.unwrap_or(true) + } + + /// Returns the prefilter attached to this configuration, if any. + pub fn get_prefilter(&self) -> Option<&Prefilter> { + self.pre.as_ref().unwrap_or(&None).as_ref() + } + + /// Returns whether this configuration has enabled the expensive process + /// of minimizing a DFA. + pub fn get_minimize(&self) -> bool { + self.minimize.unwrap_or(false) + } + + /// Returns the match semantics set in this configuration. + pub fn get_match_kind(&self) -> MatchKind { + self.match_kind.unwrap_or(MatchKind::LeftmostFirst) + } + + /// Returns the starting state configuration for a DFA. + pub fn get_starts(&self) -> StartKind { + self.start_kind.unwrap_or(StartKind::Both) + } + + /// Returns whether this configuration has enabled anchored starting states + /// for every pattern in the DFA. + pub fn get_starts_for_each_pattern(&self) -> bool { + self.starts_for_each_pattern.unwrap_or(false) + } + + /// Returns whether this configuration has enabled byte classes or not. + /// This is typically a debugging oriented option, as disabling it confers + /// no speed benefit. + pub fn get_byte_classes(&self) -> bool { + self.byte_classes.unwrap_or(true) + } + + /// Returns whether this configuration has enabled heuristic Unicode word + /// boundary support. When enabled, it is possible for a search to return + /// an error. + pub fn get_unicode_word_boundary(&self) -> bool { + self.unicode_word_boundary.unwrap_or(false) + } + + /// Returns whether this configuration will instruct the DFA to enter a + /// quit state whenever the given byte is seen during a search. When at + /// least one byte has this enabled, it is possible for a search to return + /// an error. + pub fn get_quit(&self, byte: u8) -> bool { + self.quitset.map_or(false, |q| q.contains(byte)) + } + + /// Returns whether this configuration will instruct the DFA to + /// "specialize" start states. When enabled, the DFA will mark start states + /// as "special" so that search routines using the DFA can detect when + /// it's in a start state and do some kind of optimization (like run a + /// prefilter). + pub fn get_specialize_start_states(&self) -> bool { + self.specialize_start_states.unwrap_or(false) + } + + /// Returns the DFA size limit of this configuration if one was set. + /// The size limit is total number of bytes on the heap that a DFA is + /// permitted to use. If the DFA exceeds this limit during construction, + /// then construction is stopped and an error is returned. + pub fn get_dfa_size_limit(&self) -> Option<usize> { + self.dfa_size_limit.unwrap_or(None) + } + + /// Returns the determinization size limit of this configuration if one + /// was set. The size limit is total number of bytes on the heap that + /// determinization is permitted to use. If determinization exceeds this + /// limit during construction, then construction is stopped and an error is + /// returned. + /// + /// This is different from the DFA size limit in that this only applies to + /// the auxiliary storage used during determinization. Once determinization + /// is complete, this memory is freed. + /// + /// The limit on the total heap memory used is the sum of the DFA and + /// determinization size limits. + pub fn get_determinize_size_limit(&self) -> Option<usize> { + self.determinize_size_limit.unwrap_or(None) + } + + /// Overwrite the default configuration such that the options in `o` are + /// always used. If an option in `o` is not set, then the corresponding + /// option in `self` is used. If it's not set in `self` either, then it + /// remains not set. + pub(crate) fn overwrite(&self, o: Config) -> Config { + Config { + accelerate: o.accelerate.or(self.accelerate), + pre: o.pre.or_else(|| self.pre.clone()), + minimize: o.minimize.or(self.minimize), + match_kind: o.match_kind.or(self.match_kind), + start_kind: o.start_kind.or(self.start_kind), + starts_for_each_pattern: o + .starts_for_each_pattern + .or(self.starts_for_each_pattern), + byte_classes: o.byte_classes.or(self.byte_classes), + unicode_word_boundary: o + .unicode_word_boundary + .or(self.unicode_word_boundary), + quitset: o.quitset.or(self.quitset), + specialize_start_states: o + .specialize_start_states + .or(self.specialize_start_states), + dfa_size_limit: o.dfa_size_limit.or(self.dfa_size_limit), + determinize_size_limit: o + .determinize_size_limit + .or(self.determinize_size_limit), + } + } +} + +/// A builder for constructing a deterministic finite automaton from regular +/// expressions. +/// +/// This builder provides two main things: +/// +/// 1. It provides a few different `build` routines for actually constructing +/// a DFA from different kinds of inputs. The most convenient is +/// [`Builder::build`], which builds a DFA directly from a pattern string. The +/// most flexible is [`Builder::build_from_nfa`], which builds a DFA straight +/// from an NFA. +/// 2. The builder permits configuring a number of things. +/// [`Builder::configure`] is used with [`Config`] to configure aspects of +/// the DFA and the construction process itself. [`Builder::syntax`] and +/// [`Builder::thompson`] permit configuring the regex parser and Thompson NFA +/// construction, respectively. The syntax and thompson configurations only +/// apply when building from a pattern string. +/// +/// This builder always constructs a *single* DFA. As such, this builder +/// can only be used to construct regexes that either detect the presence +/// of a match or find the end location of a match. A single DFA cannot +/// produce both the start and end of a match. For that information, use a +/// [`Regex`](crate::dfa::regex::Regex), which can be similarly configured +/// using [`regex::Builder`](crate::dfa::regex::Builder). The main reason to +/// use a DFA directly is if the end location of a match is enough for your use +/// case. Namely, a `Regex` will construct two DFAs instead of one, since a +/// second reverse DFA is needed to find the start of a match. +/// +/// Note that if one wants to build a sparse DFA, you must first build a dense +/// DFA and convert that to a sparse DFA. There is no way to build a sparse +/// DFA without first building a dense DFA. +/// +/// # Example +/// +/// This example shows how to build a minimized DFA that completely disables +/// Unicode. That is: +/// +/// * Things such as `\w`, `.` and `\b` are no longer Unicode-aware. `\w` +/// and `\b` are ASCII-only while `.` matches any byte except for `\n` +/// (instead of any UTF-8 encoding of a Unicode scalar value except for +/// `\n`). Things that are Unicode only, such as `\pL`, are not allowed. +/// * The pattern itself is permitted to match invalid UTF-8. For example, +/// things like `[^a]` that match any byte except for `a` are permitted. +/// +/// ``` +/// use regex_automata::{ +/// dfa::{Automaton, dense}, +/// util::syntax, +/// HalfMatch, Input, +/// }; +/// +/// let dfa = dense::Builder::new() +/// .configure(dense::Config::new().minimize(false)) +/// .syntax(syntax::Config::new().unicode(false).utf8(false)) +/// .build(r"foo[^b]ar.*")?; +/// +/// let haystack = b"\xFEfoo\xFFar\xE2\x98\xFF\n"; +/// let expected = Some(HalfMatch::must(0, 10)); +/// let got = dfa.try_search_fwd(&Input::new(haystack))?; +/// assert_eq!(expected, got); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[cfg(feature = "dfa-build")] +#[derive(Clone, Debug)] +pub struct Builder { + config: Config, + #[cfg(feature = "syntax")] + thompson: thompson::Compiler, +} + +#[cfg(feature = "dfa-build")] +impl Builder { + /// Create a new dense DFA builder with the default configuration. + pub fn new() -> Builder { + Builder { + config: Config::default(), + #[cfg(feature = "syntax")] + thompson: thompson::Compiler::new(), + } + } + + /// Build a DFA from the given pattern. + /// + /// If there was a problem parsing or compiling the pattern, then an error + /// is returned. + #[cfg(feature = "syntax")] + pub fn build(&self, pattern: &str) -> Result<OwnedDFA, BuildError> { + self.build_many(&[pattern]) + } + + /// Build a DFA from the given patterns. + /// + /// When matches are returned, the pattern ID corresponds to the index of + /// the pattern in the slice given. + #[cfg(feature = "syntax")] + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<OwnedDFA, BuildError> { + let nfa = self + .thompson + .clone() + // We can always forcefully disable captures because DFAs do not + // support them. + .configure( + thompson::Config::new() + .which_captures(thompson::WhichCaptures::None), + ) + .build_many(patterns) + .map_err(BuildError::nfa)?; + self.build_from_nfa(&nfa) + } + + /// Build a DFA from the given NFA. + /// + /// # Example + /// + /// This example shows how to build a DFA if you already have an NFA in + /// hand. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{Automaton, dense}, + /// nfa::thompson::NFA, + /// HalfMatch, Input, + /// }; + /// + /// let haystack = "foo123bar".as_bytes(); + /// + /// // This shows how to set non-default options for building an NFA. + /// let nfa = NFA::compiler() + /// .configure(NFA::config().shrink(true)) + /// .build(r"[0-9]+")?; + /// let dfa = dense::Builder::new().build_from_nfa(&nfa)?; + /// let expected = Some(HalfMatch::must(0, 6)); + /// let got = dfa.try_search_fwd(&Input::new(haystack))?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_from_nfa( + &self, + nfa: &thompson::NFA, + ) -> Result<OwnedDFA, BuildError> { + let mut quitset = self.config.quitset.unwrap_or(ByteSet::empty()); + if self.config.get_unicode_word_boundary() + && nfa.look_set_any().contains_word_unicode() + { + for b in 0x80..=0xFF { + quitset.add(b); + } + } + let classes = if !self.config.get_byte_classes() { + // DFAs will always use the equivalence class map, but enabling + // this option is useful for debugging. Namely, this will cause all + // transitions to be defined over their actual bytes instead of an + // opaque equivalence class identifier. The former is much easier + // to grok as a human. + ByteClasses::singletons() + } else { + let mut set = nfa.byte_class_set().clone(); + // It is important to distinguish any "quit" bytes from all other + // bytes. Otherwise, a non-quit byte may end up in the same + // class as a quit byte, and thus cause the DFA to stop when it + // shouldn't. + // + // Test case: + // + // regex-cli find match dense --unicode-word-boundary \ + // -p '^#' -p '\b10\.55\.182\.100\b' -y @conn.json.1000x.log + if !quitset.is_empty() { + set.add_set(&quitset); + } + set.byte_classes() + }; + + let mut dfa = DFA::initial( + classes, + nfa.pattern_len(), + self.config.get_starts(), + nfa.look_matcher(), + self.config.get_starts_for_each_pattern(), + self.config.get_prefilter().map(|p| p.clone()), + quitset, + Flags::from_nfa(&nfa), + )?; + determinize::Config::new() + .match_kind(self.config.get_match_kind()) + .quit(quitset) + .dfa_size_limit(self.config.get_dfa_size_limit()) + .determinize_size_limit(self.config.get_determinize_size_limit()) + .run(nfa, &mut dfa)?; + if self.config.get_minimize() { + dfa.minimize(); + } + if self.config.get_accelerate() { + dfa.accelerate(); + } + // The state shuffling done before this point always assumes that start + // states should be marked as "special," even though it isn't the + // default configuration. State shuffling is complex enough as it is, + // so it's simpler to just "fix" our special state ID ranges to not + // include starting states after-the-fact. + if !self.config.get_specialize_start_states() { + dfa.special.set_no_special_start_states(); + } + // Look for and set the universal starting states. + dfa.set_universal_starts(); + dfa.tt.table.shrink_to_fit(); + dfa.st.table.shrink_to_fit(); + dfa.ms.slices.shrink_to_fit(); + dfa.ms.pattern_ids.shrink_to_fit(); + Ok(dfa) + } + + /// Apply the given dense DFA configuration options to this builder. + pub fn configure(&mut self, config: Config) -> &mut Builder { + self.config = self.config.overwrite(config); + self + } + + /// Set the syntax configuration for this builder using + /// [`syntax::Config`](crate::util::syntax::Config). + /// + /// This permits setting things like case insensitivity, Unicode and multi + /// line mode. + /// + /// These settings only apply when constructing a DFA directly from a + /// pattern. + #[cfg(feature = "syntax")] + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Builder { + self.thompson.syntax(config); + self + } + + /// Set the Thompson NFA configuration for this builder using + /// [`nfa::thompson::Config`](crate::nfa::thompson::Config). + /// + /// This permits setting things like whether the DFA should match the regex + /// in reverse or if additional time should be spent shrinking the size of + /// the NFA. + /// + /// These settings only apply when constructing a DFA directly from a + /// pattern. + #[cfg(feature = "syntax")] + pub fn thompson(&mut self, config: thompson::Config) -> &mut Builder { + self.thompson.configure(config); + self + } +} + +#[cfg(feature = "dfa-build")] +impl Default for Builder { + fn default() -> Builder { + Builder::new() + } +} + +/// A convenience alias for an owned DFA. We use this particular instantiation +/// a lot in this crate, so it's worth giving it a name. This instantiation +/// is commonly used for mutable APIs on the DFA while building it. The main +/// reason for making DFAs generic is no_std support, and more generally, +/// making it possible to load a DFA from an arbitrary slice of bytes. +#[cfg(feature = "alloc")] +pub(crate) type OwnedDFA = DFA<alloc::vec::Vec<u32>>; + +/// A dense table-based deterministic finite automaton (DFA). +/// +/// All dense DFAs have one or more start states, zero or more match states +/// and a transition table that maps the current state and the current byte +/// of input to the next state. A DFA can use this information to implement +/// fast searching. In particular, the use of a dense DFA generally makes the +/// trade off that match speed is the most valuable characteristic, even if +/// building the DFA may take significant time *and* space. (More concretely, +/// building a DFA takes time and space that is exponential in the size of the +/// pattern in the worst case.) As such, the processing of every byte of input +/// is done with a small constant number of operations that does not vary with +/// the pattern, its size or the size of the alphabet. If your needs don't line +/// up with this trade off, then a dense DFA may not be an adequate solution to +/// your problem. +/// +/// In contrast, a [`sparse::DFA`] makes the opposite +/// trade off: it uses less space but will execute a variable number of +/// instructions per byte at match time, which makes it slower for matching. +/// (Note that space usage is still exponential in the size of the pattern in +/// the worst case.) +/// +/// A DFA can be built using the default configuration via the +/// [`DFA::new`] constructor. Otherwise, one can +/// configure various aspects via [`dense::Builder`](Builder). +/// +/// A single DFA fundamentally supports the following operations: +/// +/// 1. Detection of a match. +/// 2. Location of the end of a match. +/// 3. In the case of a DFA with multiple patterns, which pattern matched is +/// reported as well. +/// +/// A notable absence from the above list of capabilities is the location of +/// the *start* of a match. In order to provide both the start and end of +/// a match, *two* DFAs are required. This functionality is provided by a +/// [`Regex`](crate::dfa::regex::Regex). +/// +/// # Type parameters +/// +/// A `DFA` has one type parameter, `T`, which is used to represent state IDs, +/// pattern IDs and accelerators. `T` is typically a `Vec<u32>` or a `&[u32]`. +/// +/// # The `Automaton` trait +/// +/// This type implements the [`Automaton`] trait, which means it can be used +/// for searching. For example: +/// +/// ``` +/// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; +/// +/// let dfa = DFA::new("foo[0-9]+")?; +/// let expected = HalfMatch::must(0, 8); +/// assert_eq!(Some(expected), dfa.try_search_fwd(&Input::new("foo12345"))?); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone)] +pub struct DFA<T> { + /// The transition table for this DFA. This includes the transitions + /// themselves, along with the stride, number of states and the equivalence + /// class mapping. + tt: TransitionTable<T>, + /// The set of starting state identifiers for this DFA. The starting state + /// IDs act as pointers into the transition table. The specific starting + /// state chosen for each search is dependent on the context at which the + /// search begins. + st: StartTable<T>, + /// The set of match states and the patterns that match for each + /// corresponding match state. + /// + /// This structure is technically only needed because of support for + /// multi-regexes. Namely, multi-regexes require answering not just whether + /// a match exists, but _which_ patterns match. So we need to store the + /// matching pattern IDs for each match state. We do this even when there + /// is only one pattern for the sake of simplicity. In practice, this uses + /// up very little space for the case of one pattern. + ms: MatchStates<T>, + /// Information about which states are "special." Special states are states + /// that are dead, quit, matching, starting or accelerated. For more info, + /// see the docs for `Special`. + special: Special, + /// The accelerators for this DFA. + /// + /// If a state is accelerated, then there exist only a small number of + /// bytes that can cause the DFA to leave the state. This permits searching + /// to use optimized routines to find those specific bytes instead of using + /// the transition table. + /// + /// All accelerated states exist in a contiguous range in the DFA's + /// transition table. See dfa/special.rs for more details on how states are + /// arranged. + accels: Accels<T>, + /// Any prefilter attached to this DFA. + /// + /// Note that currently prefilters are not serialized. When deserializing + /// a DFA from bytes, this is always set to `None`. + pre: Option<Prefilter>, + /// The set of "quit" bytes for this DFA. + /// + /// This is only used when computing the start state for a particular + /// position in a haystack. Namely, in the case where there is a quit + /// byte immediately before the start of the search, this set needs to be + /// explicitly consulted. In all other cases, quit bytes are detected by + /// the DFA itself, by transitioning all quit bytes to a special "quit + /// state." + quitset: ByteSet, + /// Various flags describing the behavior of this DFA. + flags: Flags, +} + +#[cfg(feature = "dfa-build")] +impl OwnedDFA { + /// Parse the given regular expression using a default configuration and + /// return the corresponding DFA. + /// + /// If you want a non-default configuration, then use the + /// [`dense::Builder`](Builder) to set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, HalfMatch, Input}; + /// + /// let dfa = dense::DFA::new("foo[0-9]+bar")?; + /// let expected = Some(HalfMatch::must(0, 11)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345bar"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new(pattern: &str) -> Result<OwnedDFA, BuildError> { + Builder::new().build(pattern) + } + + /// Parse the given regular expressions using a default configuration and + /// return the corresponding multi-DFA. + /// + /// If you want a non-default configuration, then use the + /// [`dense::Builder`](Builder) to set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, HalfMatch, Input}; + /// + /// let dfa = dense::DFA::new_many(&["[0-9]+", "[a-z]+"])?; + /// let expected = Some(HalfMatch::must(1, 3)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345bar"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new_many<P: AsRef<str>>( + patterns: &[P], + ) -> Result<OwnedDFA, BuildError> { + Builder::new().build_many(patterns) + } +} + +#[cfg(feature = "dfa-build")] +impl OwnedDFA { + /// Create a new DFA that matches every input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, HalfMatch, Input}; + /// + /// let dfa = dense::DFA::always_match()?; + /// + /// let expected = Some(HalfMatch::must(0, 0)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new(""))?); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn always_match() -> Result<OwnedDFA, BuildError> { + let nfa = thompson::NFA::always_match(); + Builder::new().build_from_nfa(&nfa) + } + + /// Create a new DFA that never matches any input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, Input}; + /// + /// let dfa = dense::DFA::never_match()?; + /// assert_eq!(None, dfa.try_search_fwd(&Input::new(""))?); + /// assert_eq!(None, dfa.try_search_fwd(&Input::new("foo"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn never_match() -> Result<OwnedDFA, BuildError> { + let nfa = thompson::NFA::never_match(); + Builder::new().build_from_nfa(&nfa) + } + + /// Create an initial DFA with the given equivalence classes, pattern + /// length and whether anchored starting states are enabled for each + /// pattern. An initial DFA can be further mutated via determinization. + fn initial( + classes: ByteClasses, + pattern_len: usize, + starts: StartKind, + lookm: &LookMatcher, + starts_for_each_pattern: bool, + pre: Option<Prefilter>, + quitset: ByteSet, + flags: Flags, + ) -> Result<OwnedDFA, BuildError> { + let start_pattern_len = + if starts_for_each_pattern { Some(pattern_len) } else { None }; + Ok(DFA { + tt: TransitionTable::minimal(classes), + st: StartTable::dead(starts, lookm, start_pattern_len)?, + ms: MatchStates::empty(pattern_len), + special: Special::new(), + accels: Accels::empty(), + pre, + quitset, + flags, + }) + } +} + +#[cfg(feature = "dfa-build")] +impl DFA<&[u32]> { + /// Return a new default dense DFA compiler configuration. + /// + /// This is a convenience routine to avoid needing to import the [`Config`] + /// type when customizing the construction of a dense DFA. + pub fn config() -> Config { + Config::new() + } + + /// Create a new dense DFA builder with the default configuration. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Builder`] type in common cases. + pub fn builder() -> Builder { + Builder::new() + } +} + +impl<T: AsRef<[u32]>> DFA<T> { + /// Cheaply return a borrowed version of this dense DFA. Specifically, + /// the DFA returned always uses `&[u32]` for its transition table. + pub fn as_ref(&self) -> DFA<&'_ [u32]> { + DFA { + tt: self.tt.as_ref(), + st: self.st.as_ref(), + ms: self.ms.as_ref(), + special: self.special, + accels: self.accels(), + pre: self.pre.clone(), + quitset: self.quitset, + flags: self.flags, + } + } + + /// Return an owned version of this sparse DFA. Specifically, the DFA + /// returned always uses `Vec<u32>` for its transition table. + /// + /// Effectively, this returns a dense DFA whose transition table lives on + /// the heap. + #[cfg(feature = "alloc")] + pub fn to_owned(&self) -> OwnedDFA { + DFA { + tt: self.tt.to_owned(), + st: self.st.to_owned(), + ms: self.ms.to_owned(), + special: self.special, + accels: self.accels().to_owned(), + pre: self.pre.clone(), + quitset: self.quitset, + flags: self.flags, + } + } + + /// Returns the starting state configuration for this DFA. + /// + /// The default is [`StartKind::Both`], which means the DFA supports both + /// unanchored and anchored searches. However, this can generally lead to + /// bigger DFAs. Therefore, a DFA might be compiled with support for just + /// unanchored or anchored searches. In that case, running a search with + /// an unsupported configuration will panic. + pub fn start_kind(&self) -> StartKind { + self.st.kind + } + + /// Returns the start byte map used for computing the `Start` configuration + /// at the beginning of a search. + pub(crate) fn start_map(&self) -> &StartByteMap { + &self.st.start_map + } + + /// Returns true only if this DFA has starting states for each pattern. + /// + /// When a DFA has starting states for each pattern, then a search with the + /// DFA can be configured to only look for anchored matches of a specific + /// pattern. Specifically, APIs like [`Automaton::try_search_fwd`] can + /// accept a non-None `pattern_id` if and only if this method returns true. + /// Otherwise, calling `try_search_fwd` will panic. + /// + /// Note that if the DFA has no patterns, this always returns false. + pub fn starts_for_each_pattern(&self) -> bool { + self.st.pattern_len.is_some() + } + + /// Returns the equivalence classes that make up the alphabet for this DFA. + /// + /// Unless [`Config::byte_classes`] was disabled, it is possible that + /// multiple distinct bytes are grouped into the same equivalence class + /// if it is impossible for them to discriminate between a match and a + /// non-match. This has the effect of reducing the overall alphabet size + /// and in turn potentially substantially reducing the size of the DFA's + /// transition table. + /// + /// The downside of using equivalence classes like this is that every state + /// transition will automatically use this map to convert an arbitrary + /// byte to its corresponding equivalence class. In practice this has a + /// negligible impact on performance. + pub fn byte_classes(&self) -> &ByteClasses { + &self.tt.classes + } + + /// Returns the total number of elements in the alphabet for this DFA. + /// + /// That is, this returns the total number of transitions that each state + /// in this DFA must have. Typically, a normal byte oriented DFA would + /// always have an alphabet size of 256, corresponding to the number of + /// unique values in a single byte. However, this implementation has two + /// peculiarities that impact the alphabet length: + /// + /// * Every state has a special "EOI" transition that is only followed + /// after the end of some haystack is reached. This EOI transition is + /// necessary to account for one byte of look-ahead when implementing + /// things like `\b` and `$`. + /// * Bytes are grouped into equivalence classes such that no two bytes in + /// the same class can distinguish a match from a non-match. For example, + /// in the regex `^[a-z]+$`, the ASCII bytes `a-z` could all be in the + /// same equivalence class. This leads to a massive space savings. + /// + /// Note though that the alphabet length does _not_ necessarily equal the + /// total stride space taken up by a single DFA state in the transition + /// table. Namely, for performance reasons, the stride is always the + /// smallest power of two that is greater than or equal to the alphabet + /// length. For this reason, [`DFA::stride`] or [`DFA::stride2`] are + /// often more useful. The alphabet length is typically useful only for + /// informational purposes. + pub fn alphabet_len(&self) -> usize { + self.tt.alphabet_len() + } + + /// Returns the total stride for every state in this DFA, expressed as the + /// exponent of a power of 2. The stride is the amount of space each state + /// takes up in the transition table, expressed as a number of transitions. + /// (Unused transitions map to dead states.) + /// + /// The stride of a DFA is always equivalent to the smallest power of 2 + /// that is greater than or equal to the DFA's alphabet length. This + /// definition uses extra space, but permits faster translation between + /// premultiplied state identifiers and contiguous indices (by using shifts + /// instead of relying on integer division). + /// + /// For example, if the DFA's stride is 16 transitions, then its `stride2` + /// is `4` since `2^4 = 16`. + /// + /// The minimum `stride2` value is `1` (corresponding to a stride of `2`) + /// while the maximum `stride2` value is `9` (corresponding to a stride of + /// `512`). The maximum is not `8` since the maximum alphabet size is `257` + /// when accounting for the special EOI transition. However, an alphabet + /// length of that size is exceptionally rare since the alphabet is shrunk + /// into equivalence classes. + pub fn stride2(&self) -> usize { + self.tt.stride2 + } + + /// Returns the total stride for every state in this DFA. This corresponds + /// to the total number of transitions used by each state in this DFA's + /// transition table. + /// + /// Please see [`DFA::stride2`] for more information. In particular, this + /// returns the stride as the number of transitions, where as `stride2` + /// returns it as the exponent of a power of 2. + pub fn stride(&self) -> usize { + self.tt.stride() + } + + /// Returns the memory usage, in bytes, of this DFA. + /// + /// The memory usage is computed based on the number of bytes used to + /// represent this DFA. + /// + /// This does **not** include the stack size used up by this DFA. To + /// compute that, use `std::mem::size_of::<dense::DFA>()`. + pub fn memory_usage(&self) -> usize { + self.tt.memory_usage() + + self.st.memory_usage() + + self.ms.memory_usage() + + self.accels.memory_usage() + } +} + +/// Routines for converting a dense DFA to other representations, such as +/// sparse DFAs or raw bytes suitable for persistent storage. +impl<T: AsRef<[u32]>> DFA<T> { + /// Convert this dense DFA to a sparse DFA. + /// + /// If a `StateID` is too small to represent all states in the sparse + /// DFA, then this returns an error. In most cases, if a dense DFA is + /// constructable with `StateID` then a sparse DFA will be as well. + /// However, it is not guaranteed. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense}, HalfMatch, Input}; + /// + /// let dense = dense::DFA::new("foo[0-9]+")?; + /// let sparse = dense.to_sparse()?; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, sparse.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "dfa-build")] + pub fn to_sparse(&self) -> Result<sparse::DFA<Vec<u8>>, BuildError> { + sparse::DFA::from_dense(self) + } + + /// Serialize this DFA as raw bytes to a `Vec<u8>` in little endian + /// format. Upon success, the `Vec<u8>` and the initial padding length are + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// The padding returned is non-zero if the returned `Vec<u8>` starts at + /// an address that does not have the same alignment as `u32`. The padding + /// corresponds to the number of leading bytes written to the returned + /// `Vec<u8>`. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA: + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // N.B. We use native endianness here to make the example work, but + /// // using to_bytes_little_endian would work on a little endian target. + /// let (buf, _) = original_dfa.to_bytes_native_endian(); + /// // Even if buf has initial padding, DFA::from_bytes will automatically + /// // ignore it. + /// let dfa: DFA<&[u32]> = DFA::from_bytes(&buf)?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "dfa-build")] + pub fn to_bytes_little_endian(&self) -> (Vec<u8>, usize) { + self.to_bytes::<wire::LE>() + } + + /// Serialize this DFA as raw bytes to a `Vec<u8>` in big endian + /// format. Upon success, the `Vec<u8>` and the initial padding length are + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// The padding returned is non-zero if the returned `Vec<u8>` starts at + /// an address that does not have the same alignment as `u32`. The padding + /// corresponds to the number of leading bytes written to the returned + /// `Vec<u8>`. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA: + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // N.B. We use native endianness here to make the example work, but + /// // using to_bytes_big_endian would work on a big endian target. + /// let (buf, _) = original_dfa.to_bytes_native_endian(); + /// // Even if buf has initial padding, DFA::from_bytes will automatically + /// // ignore it. + /// let dfa: DFA<&[u32]> = DFA::from_bytes(&buf)?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "dfa-build")] + pub fn to_bytes_big_endian(&self) -> (Vec<u8>, usize) { + self.to_bytes::<wire::BE>() + } + + /// Serialize this DFA as raw bytes to a `Vec<u8>` in native endian + /// format. Upon success, the `Vec<u8>` and the initial padding length are + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// The padding returned is non-zero if the returned `Vec<u8>` starts at + /// an address that does not have the same alignment as `u32`. The padding + /// corresponds to the number of leading bytes written to the returned + /// `Vec<u8>`. + /// + /// Generally speaking, native endian format should only be used when + /// you know that the target you're compiling the DFA for matches the + /// endianness of the target on which you're compiling DFA. For example, + /// if serialization and deserialization happen in the same process or on + /// the same machine. Otherwise, when serializing a DFA for use in a + /// portable environment, you'll almost certainly want to serialize _both_ + /// a little endian and a big endian version and then load the correct one + /// based on the target's configuration. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA: + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// let (buf, _) = original_dfa.to_bytes_native_endian(); + /// // Even if buf has initial padding, DFA::from_bytes will automatically + /// // ignore it. + /// let dfa: DFA<&[u32]> = DFA::from_bytes(&buf)?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "dfa-build")] + pub fn to_bytes_native_endian(&self) -> (Vec<u8>, usize) { + self.to_bytes::<wire::NE>() + } + + /// The implementation of the public `to_bytes` serialization methods, + /// which is generic over endianness. + #[cfg(feature = "dfa-build")] + fn to_bytes<E: Endian>(&self) -> (Vec<u8>, usize) { + let len = self.write_to_len(); + let (mut buf, padding) = wire::alloc_aligned_buffer::<u32>(len); + // This should always succeed since the only possible serialization + // error is providing a buffer that's too small, but we've ensured that + // `buf` is big enough here. + self.as_ref().write_to::<E>(&mut buf[padding..]).unwrap(); + (buf, padding) + } + + /// Serialize this DFA as raw bytes to the given slice, in little endian + /// format. Upon success, the total number of bytes written to `dst` is + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// Note that unlike the various `to_byte_*` routines, this does not write + /// any padding. Callers are responsible for handling alignment correctly. + /// + /// # Errors + /// + /// This returns an error if the given destination slice is not big enough + /// to contain the full serialized DFA. If an error occurs, then nothing + /// is written to `dst`. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA without + /// dynamic memory allocation. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // Create a 4KB buffer on the stack to store our serialized DFA. We + /// // need to use a special type to force the alignment of our [u8; N] + /// // array to be aligned to a 4 byte boundary. Otherwise, deserializing + /// // the DFA may fail because of an alignment mismatch. + /// #[repr(C)] + /// struct Aligned<B: ?Sized> { + /// _align: [u32; 0], + /// bytes: B, + /// } + /// let mut buf = Aligned { _align: [], bytes: [0u8; 4 * (1<<10)] }; + /// // N.B. We use native endianness here to make the example work, but + /// // using write_to_little_endian would work on a little endian target. + /// let written = original_dfa.write_to_native_endian(&mut buf.bytes)?; + /// let dfa: DFA<&[u32]> = DFA::from_bytes(&buf.bytes[..written])?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn write_to_little_endian( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + self.as_ref().write_to::<wire::LE>(dst) + } + + /// Serialize this DFA as raw bytes to the given slice, in big endian + /// format. Upon success, the total number of bytes written to `dst` is + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// Note that unlike the various `to_byte_*` routines, this does not write + /// any padding. Callers are responsible for handling alignment correctly. + /// + /// # Errors + /// + /// This returns an error if the given destination slice is not big enough + /// to contain the full serialized DFA. If an error occurs, then nothing + /// is written to `dst`. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA without + /// dynamic memory allocation. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // Create a 4KB buffer on the stack to store our serialized DFA. We + /// // need to use a special type to force the alignment of our [u8; N] + /// // array to be aligned to a 4 byte boundary. Otherwise, deserializing + /// // the DFA may fail because of an alignment mismatch. + /// #[repr(C)] + /// struct Aligned<B: ?Sized> { + /// _align: [u32; 0], + /// bytes: B, + /// } + /// let mut buf = Aligned { _align: [], bytes: [0u8; 4 * (1<<10)] }; + /// // N.B. We use native endianness here to make the example work, but + /// // using write_to_big_endian would work on a big endian target. + /// let written = original_dfa.write_to_native_endian(&mut buf.bytes)?; + /// let dfa: DFA<&[u32]> = DFA::from_bytes(&buf.bytes[..written])?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn write_to_big_endian( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + self.as_ref().write_to::<wire::BE>(dst) + } + + /// Serialize this DFA as raw bytes to the given slice, in native endian + /// format. Upon success, the total number of bytes written to `dst` is + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// Generally speaking, native endian format should only be used when + /// you know that the target you're compiling the DFA for matches the + /// endianness of the target on which you're compiling DFA. For example, + /// if serialization and deserialization happen in the same process or on + /// the same machine. Otherwise, when serializing a DFA for use in a + /// portable environment, you'll almost certainly want to serialize _both_ + /// a little endian and a big endian version and then load the correct one + /// based on the target's configuration. + /// + /// Note that unlike the various `to_byte_*` routines, this does not write + /// any padding. Callers are responsible for handling alignment correctly. + /// + /// # Errors + /// + /// This returns an error if the given destination slice is not big enough + /// to contain the full serialized DFA. If an error occurs, then nothing + /// is written to `dst`. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA without + /// dynamic memory allocation. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // Create a 4KB buffer on the stack to store our serialized DFA. We + /// // need to use a special type to force the alignment of our [u8; N] + /// // array to be aligned to a 4 byte boundary. Otherwise, deserializing + /// // the DFA may fail because of an alignment mismatch. + /// #[repr(C)] + /// struct Aligned<B: ?Sized> { + /// _align: [u32; 0], + /// bytes: B, + /// } + /// let mut buf = Aligned { _align: [], bytes: [0u8; 4 * (1<<10)] }; + /// let written = original_dfa.write_to_native_endian(&mut buf.bytes)?; + /// let dfa: DFA<&[u32]> = DFA::from_bytes(&buf.bytes[..written])?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn write_to_native_endian( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + self.as_ref().write_to::<wire::NE>(dst) + } + + /// Return the total number of bytes required to serialize this DFA. + /// + /// This is useful for determining the size of the buffer required to pass + /// to one of the serialization routines: + /// + /// * [`DFA::write_to_little_endian`] + /// * [`DFA::write_to_big_endian`] + /// * [`DFA::write_to_native_endian`] + /// + /// Passing a buffer smaller than the size returned by this method will + /// result in a serialization error. Serialization routines are guaranteed + /// to succeed when the buffer is big enough. + /// + /// # Example + /// + /// This example shows how to dynamically allocate enough room to serialize + /// a DFA. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// let mut buf = vec![0; original_dfa.write_to_len()]; + /// // This is guaranteed to succeed, because the only serialization error + /// // that can occur is when the provided buffer is too small. But + /// // write_to_len guarantees a correct size. + /// let written = original_dfa.write_to_native_endian(&mut buf).unwrap(); + /// // But this is not guaranteed to succeed! In particular, + /// // deserialization requires proper alignment for &[u32], but our buffer + /// // was allocated as a &[u8] whose required alignment is smaller than + /// // &[u32]. However, it's likely to work in practice because of how most + /// // allocators work. So if you write code like this, make sure to either + /// // handle the error correctly and/or run it under Miri since Miri will + /// // likely provoke the error by returning Vec<u8> buffers with alignment + /// // less than &[u32]. + /// let dfa: DFA<&[u32]> = match DFA::from_bytes(&buf[..written]) { + /// // As mentioned above, it is legal for an error to be returned + /// // here. It is quite difficult to get a Vec<u8> with a guaranteed + /// // alignment equivalent to Vec<u32>. + /// Err(_) => return Ok(()), + /// Ok((dfa, _)) => dfa, + /// }; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Note that this example isn't actually guaranteed to work! In + /// particular, if `buf` is not aligned to a 4-byte boundary, then the + /// `DFA::from_bytes` call will fail. If you need this to work, then you + /// either need to deal with adding some initial padding yourself, or use + /// one of the `to_bytes` methods, which will do it for you. + pub fn write_to_len(&self) -> usize { + wire::write_label_len(LABEL) + + wire::write_endianness_check_len() + + wire::write_version_len() + + size_of::<u32>() // unused, intended for future flexibility + + self.flags.write_to_len() + + self.tt.write_to_len() + + self.st.write_to_len() + + self.ms.write_to_len() + + self.special.write_to_len() + + self.accels.write_to_len() + + self.quitset.write_to_len() + } +} + +impl<'a> DFA<&'a [u32]> { + /// Safely deserialize a DFA with a specific state identifier + /// representation. Upon success, this returns both the deserialized DFA + /// and the number of bytes read from the given slice. Namely, the contents + /// of the slice beyond the DFA are not read. + /// + /// Deserializing a DFA using this routine will never allocate heap memory. + /// For safety purposes, the DFA's transition table will be verified such + /// that every transition points to a valid state. If this verification is + /// too costly, then a [`DFA::from_bytes_unchecked`] API is provided, which + /// will always execute in constant time. + /// + /// The bytes given must be generated by one of the serialization APIs + /// of a `DFA` using a semver compatible release of this crate. Those + /// include: + /// + /// * [`DFA::to_bytes_little_endian`] + /// * [`DFA::to_bytes_big_endian`] + /// * [`DFA::to_bytes_native_endian`] + /// * [`DFA::write_to_little_endian`] + /// * [`DFA::write_to_big_endian`] + /// * [`DFA::write_to_native_endian`] + /// + /// The `to_bytes` methods allocate and return a `Vec<u8>` for you, along + /// with handling alignment correctly. The `write_to` methods do not + /// allocate and write to an existing slice (which may be on the stack). + /// Since deserialization always uses the native endianness of the target + /// platform, the serialization API you use should match the endianness of + /// the target platform. (It's often a good idea to generate serialized + /// DFAs for both forms of endianness and then load the correct one based + /// on endianness.) + /// + /// # Errors + /// + /// Generally speaking, it's easier to state the conditions in which an + /// error is _not_ returned. All of the following must be true: + /// + /// * The bytes given must be produced by one of the serialization APIs + /// on this DFA, as mentioned above. + /// * The endianness of the target platform matches the endianness used to + /// serialized the provided DFA. + /// * The slice given must have the same alignment as `u32`. + /// + /// If any of the above are not true, then an error will be returned. + /// + /// # Panics + /// + /// This routine will never panic for any input. + /// + /// # Example + /// + /// This example shows how to serialize a DFA to raw bytes, deserialize it + /// and then use it for searching. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// let initial = DFA::new("foo[0-9]+")?; + /// let (bytes, _) = initial.to_bytes_native_endian(); + /// let dfa: DFA<&[u32]> = DFA::from_bytes(&bytes)?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: dealing with alignment and padding + /// + /// In the above example, we used the `to_bytes_native_endian` method to + /// serialize a DFA, but we ignored part of its return value corresponding + /// to padding added to the beginning of the serialized DFA. This is OK + /// because deserialization will skip this initial padding. What matters + /// is that the address immediately following the padding has an alignment + /// that matches `u32`. That is, the following is an equivalent but + /// alternative way to write the above example: + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// let initial = DFA::new("foo[0-9]+")?; + /// // Serialization returns the number of leading padding bytes added to + /// // the returned Vec<u8>. + /// let (bytes, pad) = initial.to_bytes_native_endian(); + /// let dfa: DFA<&[u32]> = DFA::from_bytes(&bytes[pad..])?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// This padding is necessary because Rust's standard library does + /// not expose any safe and robust way of creating a `Vec<u8>` with a + /// guaranteed alignment other than 1. Now, in practice, the underlying + /// allocator is likely to provide a `Vec<u8>` that meets our alignment + /// requirements, which means `pad` is zero in practice most of the time. + /// + /// The purpose of exposing the padding like this is flexibility for the + /// caller. For example, if one wants to embed a serialized DFA into a + /// compiled program, then it's important to guarantee that it starts at a + /// `u32`-aligned address. The simplest way to do this is to discard the + /// padding bytes and set it up so that the serialized DFA itself begins at + /// a properly aligned address. We can show this in two parts. The first + /// part is serializing the DFA to a file: + /// + /// ```no_run + /// use regex_automata::dfa::dense::DFA; + /// + /// let dfa = DFA::new("foo[0-9]+")?; + /// + /// let (bytes, pad) = dfa.to_bytes_big_endian(); + /// // Write the contents of the DFA *without* the initial padding. + /// std::fs::write("foo.bigendian.dfa", &bytes[pad..])?; + /// + /// // Do it again, but this time for little endian. + /// let (bytes, pad) = dfa.to_bytes_little_endian(); + /// std::fs::write("foo.littleendian.dfa", &bytes[pad..])?; + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And now the second part is embedding the DFA into the compiled program + /// and deserializing it at runtime on first use. We use conditional + /// compilation to choose the correct endianness. + /// + /// ```no_run + /// use regex_automata::{ + /// dfa::{Automaton, dense::DFA}, + /// util::{lazy::Lazy, wire::AlignAs}, + /// HalfMatch, Input, + /// }; + /// + /// // This crate provides its own "lazy" type, kind of like + /// // lazy_static! or once_cell::sync::Lazy. But it works in no-alloc + /// // no-std environments and let's us write this using completely + /// // safe code. + /// static RE: Lazy<DFA<&'static [u32]>> = Lazy::new(|| { + /// # const _: &str = stringify! { + /// // This assignment is made possible (implicitly) via the + /// // CoerceUnsized trait. This is what guarantees that our + /// // bytes are stored in memory on a 4 byte boundary. You + /// // *must* do this or something equivalent for correct + /// // deserialization. + /// static ALIGNED: &AlignAs<[u8], u32> = &AlignAs { + /// _align: [], + /// #[cfg(target_endian = "big")] + /// bytes: *include_bytes!("foo.bigendian.dfa"), + /// #[cfg(target_endian = "little")] + /// bytes: *include_bytes!("foo.littleendian.dfa"), + /// }; + /// # }; + /// # static ALIGNED: &AlignAs<[u8], u32> = &AlignAs { + /// # _align: [], + /// # bytes: [], + /// # }; + /// + /// let (dfa, _) = DFA::from_bytes(&ALIGNED.bytes) + /// .expect("serialized DFA should be valid"); + /// dfa + /// }); + /// + /// let expected = Ok(Some(HalfMatch::must(0, 8))); + /// assert_eq!(expected, RE.try_search_fwd(&Input::new("foo12345"))); + /// ``` + /// + /// An alternative to [`util::lazy::Lazy`](crate::util::lazy::Lazy) + /// is [`lazy_static`](https://crates.io/crates/lazy_static) or + /// [`once_cell`](https://crates.io/crates/once_cell), which provide + /// stronger guarantees (like the initialization function only being + /// executed once). And `once_cell` in particular provides a more + /// expressive API. But a `Lazy` value from this crate is likely just fine + /// in most circumstances. + /// + /// Note that regardless of which initialization method you use, you + /// will still need to use the [`AlignAs`](crate::util::wire::AlignAs) + /// trick above to force correct alignment, but this is safe to do and + /// `from_bytes` will return an error if you get it wrong. + pub fn from_bytes( + slice: &'a [u8], + ) -> Result<(DFA<&'a [u32]>, usize), DeserializeError> { + // SAFETY: This is safe because we validate the transition table, start + // table, match states and accelerators below. If any validation fails, + // then we return an error. + let (dfa, nread) = unsafe { DFA::from_bytes_unchecked(slice)? }; + // Note that validation order is important here: + // + // * `MatchState::validate` can be called with an untrusted DFA. + // * `TransistionTable::validate` uses `dfa.ms` through `match_len`. + // * `StartTable::validate` needs a valid transition table. + // + // So... validate the match states first. + dfa.accels.validate()?; + dfa.ms.validate(&dfa)?; + dfa.tt.validate(&dfa)?; + dfa.st.validate(&dfa)?; + // N.B. dfa.special doesn't have a way to do unchecked deserialization, + // so it has already been validated. + for state in dfa.states() { + // If the state is an accel state, then it must have a non-empty + // accelerator. + if dfa.is_accel_state(state.id()) { + let index = dfa.accelerator_index(state.id()); + if index >= dfa.accels.len() { + return Err(DeserializeError::generic( + "found DFA state with invalid accelerator index", + )); + } + let needles = dfa.accels.needles(index); + if !(1 <= needles.len() && needles.len() <= 3) { + return Err(DeserializeError::generic( + "accelerator needles has invalid length", + )); + } + } + } + Ok((dfa, nread)) + } + + /// Deserialize a DFA with a specific state identifier representation in + /// constant time by omitting the verification of the validity of the + /// transition table and other data inside the DFA. + /// + /// This is just like [`DFA::from_bytes`], except it can potentially return + /// a DFA that exhibits undefined behavior if its transition table contains + /// invalid state identifiers. + /// + /// This routine is useful if you need to deserialize a DFA cheaply + /// and cannot afford the transition table validation performed by + /// `from_bytes`. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, dense::DFA}, HalfMatch, Input}; + /// + /// let initial = DFA::new("foo[0-9]+")?; + /// let (bytes, _) = initial.to_bytes_native_endian(); + /// // SAFETY: This is guaranteed to be safe since the bytes given come + /// // directly from a compatible serialization routine. + /// let dfa: DFA<&[u32]> = unsafe { DFA::from_bytes_unchecked(&bytes)?.0 }; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub unsafe fn from_bytes_unchecked( + slice: &'a [u8], + ) -> Result<(DFA<&'a [u32]>, usize), DeserializeError> { + let mut nr = 0; + + nr += wire::skip_initial_padding(slice); + wire::check_alignment::<StateID>(&slice[nr..])?; + nr += wire::read_label(&slice[nr..], LABEL)?; + nr += wire::read_endianness_check(&slice[nr..])?; + nr += wire::read_version(&slice[nr..], VERSION)?; + + let _unused = wire::try_read_u32(&slice[nr..], "unused space")?; + nr += size_of::<u32>(); + + let (flags, nread) = Flags::from_bytes(&slice[nr..])?; + nr += nread; + + let (tt, nread) = TransitionTable::from_bytes_unchecked(&slice[nr..])?; + nr += nread; + + let (st, nread) = StartTable::from_bytes_unchecked(&slice[nr..])?; + nr += nread; + + let (ms, nread) = MatchStates::from_bytes_unchecked(&slice[nr..])?; + nr += nread; + + let (special, nread) = Special::from_bytes(&slice[nr..])?; + nr += nread; + special.validate_state_len(tt.len(), tt.stride2)?; + + let (accels, nread) = Accels::from_bytes_unchecked(&slice[nr..])?; + nr += nread; + + let (quitset, nread) = ByteSet::from_bytes(&slice[nr..])?; + nr += nread; + + // Prefilters don't support serialization, so they're always absent. + let pre = None; + Ok((DFA { tt, st, ms, special, accels, pre, quitset, flags }, nr)) + } + + /// The implementation of the public `write_to` serialization methods, + /// which is generic over endianness. + /// + /// This is defined only for &[u32] to reduce binary size/compilation time. + fn write_to<E: Endian>( + &self, + mut dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("dense DFA")); + } + dst = &mut dst[..nwrite]; + + let mut nw = 0; + nw += wire::write_label(LABEL, &mut dst[nw..])?; + nw += wire::write_endianness_check::<E>(&mut dst[nw..])?; + nw += wire::write_version::<E>(VERSION, &mut dst[nw..])?; + nw += { + // Currently unused, intended for future flexibility + E::write_u32(0, &mut dst[nw..]); + size_of::<u32>() + }; + nw += self.flags.write_to::<E>(&mut dst[nw..])?; + nw += self.tt.write_to::<E>(&mut dst[nw..])?; + nw += self.st.write_to::<E>(&mut dst[nw..])?; + nw += self.ms.write_to::<E>(&mut dst[nw..])?; + nw += self.special.write_to::<E>(&mut dst[nw..])?; + nw += self.accels.write_to::<E>(&mut dst[nw..])?; + nw += self.quitset.write_to::<E>(&mut dst[nw..])?; + Ok(nw) + } +} + +/// Other routines that work for all `T`. +impl<T> DFA<T> { + /// Set or unset the prefilter attached to this DFA. + /// + /// This is useful when one has deserialized a DFA from `&[u8]`. + /// Deserialization does not currently include prefilters, so if you + /// want prefilter acceleration, you'll need to rebuild it and attach + /// it here. + pub fn set_prefilter(&mut self, prefilter: Option<Prefilter>) { + self.pre = prefilter + } +} + +// The following methods implement mutable routines on the internal +// representation of a DFA. As such, we must fix the first type parameter to a +// `Vec<u32>` since a generic `T: AsRef<[u32]>` does not permit mutation. We +// can get away with this because these methods are internal to the crate and +// are exclusively used during construction of the DFA. +#[cfg(feature = "dfa-build")] +impl OwnedDFA { + /// Add a start state of this DFA. + pub(crate) fn set_start_state( + &mut self, + anchored: Anchored, + start: Start, + id: StateID, + ) { + assert!(self.tt.is_valid(id), "invalid start state"); + self.st.set_start(anchored, start, id); + } + + /// Set the given transition to this DFA. Both the `from` and `to` states + /// must already exist. + pub(crate) fn set_transition( + &mut self, + from: StateID, + byte: alphabet::Unit, + to: StateID, + ) { + self.tt.set(from, byte, to); + } + + /// An empty state (a state where all transitions lead to a dead state) + /// and return its identifier. The identifier returned is guaranteed to + /// not point to any other existing state. + /// + /// If adding a state would exceed `StateID::LIMIT`, then this returns an + /// error. + pub(crate) fn add_empty_state(&mut self) -> Result<StateID, BuildError> { + self.tt.add_empty_state() + } + + /// Swap the two states given in the transition table. + /// + /// This routine does not do anything to check the correctness of this + /// swap. Callers must ensure that other states pointing to id1 and id2 are + /// updated appropriately. + pub(crate) fn swap_states(&mut self, id1: StateID, id2: StateID) { + self.tt.swap(id1, id2); + } + + /// Remap all of the state identifiers in this DFA according to the map + /// function given. This includes all transitions and all starting state + /// identifiers. + pub(crate) fn remap(&mut self, map: impl Fn(StateID) -> StateID) { + // We could loop over each state ID and call 'remap_state' here, but + // this is more direct: just map every transition directly. This + // technically might do a little extra work since the alphabet length + // is likely less than the stride, but if that is indeed an issue we + // should benchmark it and fix it. + for sid in self.tt.table_mut().iter_mut() { + *sid = map(*sid); + } + for sid in self.st.table_mut().iter_mut() { + *sid = map(*sid); + } + } + + /// Remap the transitions for the state given according to the function + /// given. This applies the given map function to every transition in the + /// given state and changes the transition in place to the result of the + /// map function for that transition. + pub(crate) fn remap_state( + &mut self, + id: StateID, + map: impl Fn(StateID) -> StateID, + ) { + self.tt.remap(id, map); + } + + /// Truncate the states in this DFA to the given length. + /// + /// This routine does not do anything to check the correctness of this + /// truncation. Callers must ensure that other states pointing to truncated + /// states are updated appropriately. + pub(crate) fn truncate_states(&mut self, len: usize) { + self.tt.truncate(len); + } + + /// Minimize this DFA in place using Hopcroft's algorithm. + pub(crate) fn minimize(&mut self) { + Minimizer::new(self).run(); + } + + /// Updates the match state pattern ID map to use the one provided. + /// + /// This is useful when it's convenient to manipulate matching states + /// (and their corresponding pattern IDs) as a map. In particular, the + /// representation used by a DFA for this map is not amenable to mutation, + /// so if things need to be changed (like when shuffling states), it's + /// often easier to work with the map form. + pub(crate) fn set_pattern_map( + &mut self, + map: &BTreeMap<StateID, Vec<PatternID>>, + ) -> Result<(), BuildError> { + self.ms = self.ms.new_with_map(map)?; + Ok(()) + } + + /// Find states that have a small number of non-loop transitions and mark + /// them as candidates for acceleration during search. + pub(crate) fn accelerate(&mut self) { + // dead and quit states can never be accelerated. + if self.state_len() <= 2 { + return; + } + + // Go through every state and record their accelerator, if possible. + let mut accels = BTreeMap::new(); + // Count the number of accelerated match, start and non-match/start + // states. + let (mut cmatch, mut cstart, mut cnormal) = (0, 0, 0); + for state in self.states() { + if let Some(accel) = state.accelerate(self.byte_classes()) { + debug!( + "accelerating full DFA state {}: {:?}", + state.id().as_usize(), + accel, + ); + accels.insert(state.id(), accel); + if self.is_match_state(state.id()) { + cmatch += 1; + } else if self.is_start_state(state.id()) { + cstart += 1; + } else { + assert!(!self.is_dead_state(state.id())); + assert!(!self.is_quit_state(state.id())); + cnormal += 1; + } + } + } + // If no states were able to be accelerated, then we're done. + if accels.is_empty() { + return; + } + let original_accels_len = accels.len(); + + // A remapper keeps track of state ID changes. Once we're done + // shuffling, the remapper is used to rewrite all transitions in the + // DFA based on the new positions of states. + let mut remapper = Remapper::new(self); + + // As we swap states, if they are match states, we need to swap their + // pattern ID lists too (for multi-regexes). We do this by converting + // the lists to an easily swappable map, and then convert back to + // MatchStates once we're done. + let mut new_matches = self.ms.to_map(self); + + // There is at least one state that gets accelerated, so these are + // guaranteed to get set to sensible values below. + self.special.min_accel = StateID::MAX; + self.special.max_accel = StateID::ZERO; + let update_special_accel = + |special: &mut Special, accel_id: StateID| { + special.min_accel = cmp::min(special.min_accel, accel_id); + special.max_accel = cmp::max(special.max_accel, accel_id); + }; + + // Start by shuffling match states. Any match states that are + // accelerated get moved to the end of the match state range. + if cmatch > 0 && self.special.matches() { + // N.B. special.{min,max}_match do not need updating, since the + // range/number of match states does not change. Only the ordering + // of match states may change. + let mut next_id = self.special.max_match; + let mut cur_id = next_id; + while cur_id >= self.special.min_match { + if let Some(accel) = accels.remove(&cur_id) { + accels.insert(next_id, accel); + update_special_accel(&mut self.special, next_id); + + // No need to do any actual swapping for equivalent IDs. + if cur_id != next_id { + remapper.swap(self, cur_id, next_id); + + // Swap pattern IDs for match states. + let cur_pids = new_matches.remove(&cur_id).unwrap(); + let next_pids = new_matches.remove(&next_id).unwrap(); + new_matches.insert(cur_id, next_pids); + new_matches.insert(next_id, cur_pids); + } + next_id = self.tt.prev_state_id(next_id); + } + cur_id = self.tt.prev_state_id(cur_id); + } + } + + // This is where it gets tricky. Without acceleration, start states + // normally come right after match states. But we want accelerated + // states to be a single contiguous range (to make it very fast + // to determine whether a state *is* accelerated), while also keeping + // match and starting states as contiguous ranges for the same reason. + // So what we do here is shuffle states such that it looks like this: + // + // DQMMMMAAAAASSSSSSNNNNNNN + // | | + // |---------| + // accelerated states + // + // Where: + // D - dead state + // Q - quit state + // M - match state (may be accelerated) + // A - normal state that is accelerated + // S - start state (may be accelerated) + // N - normal state that is NOT accelerated + // + // We implement this by shuffling states, which is done by a sequence + // of pairwise swaps. We start by looking at all normal states to be + // accelerated. When we find one, we swap it with the earliest starting + // state, and then swap that with the earliest normal state. This + // preserves the contiguous property. + // + // Once we're done looking for accelerated normal states, now we look + // for accelerated starting states by moving them to the beginning + // of the starting state range (just like we moved accelerated match + // states to the end of the matching state range). + // + // For a more detailed/different perspective on this, see the docs + // in dfa/special.rs. + if cnormal > 0 { + // our next available starting and normal states for swapping. + let mut next_start_id = self.special.min_start; + let mut cur_id = self.to_state_id(self.state_len() - 1); + // This is guaranteed to exist since cnormal > 0. + let mut next_norm_id = + self.tt.next_state_id(self.special.max_start); + while cur_id >= next_norm_id { + if let Some(accel) = accels.remove(&cur_id) { + remapper.swap(self, next_start_id, cur_id); + remapper.swap(self, next_norm_id, cur_id); + // Keep our accelerator map updated with new IDs if the + // states we swapped were also accelerated. + if let Some(accel2) = accels.remove(&next_norm_id) { + accels.insert(cur_id, accel2); + } + if let Some(accel2) = accels.remove(&next_start_id) { + accels.insert(next_norm_id, accel2); + } + accels.insert(next_start_id, accel); + update_special_accel(&mut self.special, next_start_id); + // Our start range shifts one to the right now. + self.special.min_start = + self.tt.next_state_id(self.special.min_start); + self.special.max_start = + self.tt.next_state_id(self.special.max_start); + next_start_id = self.tt.next_state_id(next_start_id); + next_norm_id = self.tt.next_state_id(next_norm_id); + } + // This is pretty tricky, but if our 'next_norm_id' state also + // happened to be accelerated, then the result is that it is + // now in the position of cur_id, so we need to consider it + // again. This loop is still guaranteed to terminate though, + // because when accels contains cur_id, we're guaranteed to + // increment next_norm_id even if cur_id remains unchanged. + if !accels.contains_key(&cur_id) { + cur_id = self.tt.prev_state_id(cur_id); + } + } + } + // Just like we did for match states, but we want to move accelerated + // start states to the beginning of the range instead of the end. + if cstart > 0 { + // N.B. special.{min,max}_start do not need updating, since the + // range/number of start states does not change at this point. Only + // the ordering of start states may change. + let mut next_id = self.special.min_start; + let mut cur_id = next_id; + while cur_id <= self.special.max_start { + if let Some(accel) = accels.remove(&cur_id) { + remapper.swap(self, cur_id, next_id); + accels.insert(next_id, accel); + update_special_accel(&mut self.special, next_id); + next_id = self.tt.next_state_id(next_id); + } + cur_id = self.tt.next_state_id(cur_id); + } + } + + // Remap all transitions in our DFA and assert some things. + remapper.remap(self); + // This unwrap is OK because acceleration never changes the number of + // match states or patterns in those match states. Since acceleration + // runs after the pattern map has been set at least once, we know that + // our match states cannot error. + self.set_pattern_map(&new_matches).unwrap(); + self.special.set_max(); + self.special.validate().expect("special state ranges should validate"); + self.special + .validate_state_len(self.state_len(), self.stride2()) + .expect( + "special state ranges should be consistent with state length", + ); + assert_eq!( + self.special.accel_len(self.stride()), + // We record the number of accelerated states initially detected + // since the accels map is itself mutated in the process above. + // If mutated incorrectly, its size may change, and thus can't be + // trusted as a source of truth of how many accelerated states we + // expected there to be. + original_accels_len, + "mismatch with expected number of accelerated states", + ); + + // And finally record our accelerators. We kept our accels map updated + // as we shuffled states above, so the accelerators should now + // correspond to a contiguous range in the state ID space. (Which we + // assert.) + let mut prev: Option<StateID> = None; + for (id, accel) in accels { + assert!(prev.map_or(true, |p| self.tt.next_state_id(p) == id)); + prev = Some(id); + self.accels.add(accel); + } + } + + /// Shuffle the states in this DFA so that starting states, match + /// states and accelerated states are all contiguous. + /// + /// See dfa/special.rs for more details. + pub(crate) fn shuffle( + &mut self, + mut matches: BTreeMap<StateID, Vec<PatternID>>, + ) -> Result<(), BuildError> { + // The determinizer always adds a quit state and it is always second. + self.special.quit_id = self.to_state_id(1); + // If all we have are the dead and quit states, then we're done and + // the DFA will never produce a match. + if self.state_len() <= 2 { + self.special.set_max(); + return Ok(()); + } + + // Collect all our non-DEAD start states into a convenient set and + // confirm there is no overlap with match states. In the classical DFA + // construction, start states can be match states. But because of + // look-around, we delay all matches by a byte, which prevents start + // states from being match states. + let mut is_start: BTreeSet<StateID> = BTreeSet::new(); + for (start_id, _, _) in self.starts() { + // If a starting configuration points to a DEAD state, then we + // don't want to shuffle it. The DEAD state is always the first + // state with ID=0. So we can just leave it be. + if start_id == DEAD { + continue; + } + assert!( + !matches.contains_key(&start_id), + "{start_id:?} is both a start and a match state, \ + which is not allowed", + ); + is_start.insert(start_id); + } + + // We implement shuffling by a sequence of pairwise swaps of states. + // Since we have a number of things referencing states via their + // IDs and swapping them changes their IDs, we need to record every + // swap we make so that we can remap IDs. The remapper handles this + // book-keeping for us. + let mut remapper = Remapper::new(self); + + // Shuffle matching states. + if matches.is_empty() { + self.special.min_match = DEAD; + self.special.max_match = DEAD; + } else { + // The determinizer guarantees that the first two states are the + // dead and quit states, respectively. We want our match states to + // come right after quit. + let mut next_id = self.to_state_id(2); + let mut new_matches = BTreeMap::new(); + self.special.min_match = next_id; + for (id, pids) in matches { + remapper.swap(self, next_id, id); + new_matches.insert(next_id, pids); + // If we swapped a start state, then update our set. + if is_start.contains(&next_id) { + is_start.remove(&next_id); + is_start.insert(id); + } + next_id = self.tt.next_state_id(next_id); + } + matches = new_matches; + self.special.max_match = cmp::max( + self.special.min_match, + self.tt.prev_state_id(next_id), + ); + } + + // Shuffle starting states. + { + let mut next_id = self.to_state_id(2); + if self.special.matches() { + next_id = self.tt.next_state_id(self.special.max_match); + } + self.special.min_start = next_id; + for id in is_start { + remapper.swap(self, next_id, id); + next_id = self.tt.next_state_id(next_id); + } + self.special.max_start = cmp::max( + self.special.min_start, + self.tt.prev_state_id(next_id), + ); + } + + // Finally remap all transitions in our DFA. + remapper.remap(self); + self.set_pattern_map(&matches)?; + self.special.set_max(); + self.special.validate().expect("special state ranges should validate"); + self.special + .validate_state_len(self.state_len(), self.stride2()) + .expect( + "special state ranges should be consistent with state length", + ); + Ok(()) + } + + /// Checks whether there are universal start states (both anchored and + /// unanchored), and if so, sets the relevant fields to the start state + /// IDs. + /// + /// Universal start states occur precisely when the all patterns in the + /// DFA have no look-around assertions in their prefix. + fn set_universal_starts(&mut self) { + assert_eq!(6, Start::len(), "expected 6 start configurations"); + + let start_id = |dfa: &mut OwnedDFA, + anchored: Anchored, + start: Start| { + // This OK because we only call 'start' under conditions + // in which we know it will succeed. + dfa.st.start(anchored, start).expect("valid Input configuration") + }; + if self.start_kind().has_unanchored() { + let anchor = Anchored::No; + let sid = start_id(self, anchor, Start::NonWordByte); + if sid == start_id(self, anchor, Start::WordByte) + && sid == start_id(self, anchor, Start::Text) + && sid == start_id(self, anchor, Start::LineLF) + && sid == start_id(self, anchor, Start::LineCR) + && sid == start_id(self, anchor, Start::CustomLineTerminator) + { + self.st.universal_start_unanchored = Some(sid); + } + } + if self.start_kind().has_anchored() { + let anchor = Anchored::Yes; + let sid = start_id(self, anchor, Start::NonWordByte); + if sid == start_id(self, anchor, Start::WordByte) + && sid == start_id(self, anchor, Start::Text) + && sid == start_id(self, anchor, Start::LineLF) + && sid == start_id(self, anchor, Start::LineCR) + && sid == start_id(self, anchor, Start::CustomLineTerminator) + { + self.st.universal_start_anchored = Some(sid); + } + } + } +} + +// A variety of generic internal methods for accessing DFA internals. +impl<T: AsRef<[u32]>> DFA<T> { + /// Return the info about special states. + pub(crate) fn special(&self) -> &Special { + &self.special + } + + /// Return the info about special states as a mutable borrow. + #[cfg(feature = "dfa-build")] + pub(crate) fn special_mut(&mut self) -> &mut Special { + &mut self.special + } + + /// Returns the quit set (may be empty) used by this DFA. + pub(crate) fn quitset(&self) -> &ByteSet { + &self.quitset + } + + /// Returns the flags for this DFA. + pub(crate) fn flags(&self) -> &Flags { + &self.flags + } + + /// Returns an iterator over all states in this DFA. + /// + /// This iterator yields a tuple for each state. The first element of the + /// tuple corresponds to a state's identifier, and the second element + /// corresponds to the state itself (comprised of its transitions). + pub(crate) fn states(&self) -> StateIter<'_, T> { + self.tt.states() + } + + /// Return the total number of states in this DFA. Every DFA has at least + /// 1 state, even the empty DFA. + pub(crate) fn state_len(&self) -> usize { + self.tt.len() + } + + /// Return an iterator over all pattern IDs for the given match state. + /// + /// If the given state is not a match state, then this panics. + #[cfg(feature = "dfa-build")] + pub(crate) fn pattern_id_slice(&self, id: StateID) -> &[PatternID] { + assert!(self.is_match_state(id)); + self.ms.pattern_id_slice(self.match_state_index(id)) + } + + /// Return the total number of pattern IDs for the given match state. + /// + /// If the given state is not a match state, then this panics. + pub(crate) fn match_pattern_len(&self, id: StateID) -> usize { + assert!(self.is_match_state(id)); + self.ms.pattern_len(self.match_state_index(id)) + } + + /// Returns the total number of patterns matched by this DFA. + pub(crate) fn pattern_len(&self) -> usize { + self.ms.pattern_len + } + + /// Returns a map from match state ID to a list of pattern IDs that match + /// in that state. + #[cfg(feature = "dfa-build")] + pub(crate) fn pattern_map(&self) -> BTreeMap<StateID, Vec<PatternID>> { + self.ms.to_map(self) + } + + /// Returns the ID of the quit state for this DFA. + #[cfg(feature = "dfa-build")] + pub(crate) fn quit_id(&self) -> StateID { + self.to_state_id(1) + } + + /// Convert the given state identifier to the state's index. The state's + /// index corresponds to the position in which it appears in the transition + /// table. When a DFA is NOT premultiplied, then a state's identifier is + /// also its index. When a DFA is premultiplied, then a state's identifier + /// is equal to `index * alphabet_len`. This routine reverses that. + pub(crate) fn to_index(&self, id: StateID) -> usize { + self.tt.to_index(id) + } + + /// Convert an index to a state (in the range 0..self.state_len()) to an + /// actual state identifier. + /// + /// This is useful when using a `Vec<T>` as an efficient map keyed by state + /// to some other information (such as a remapped state ID). + #[cfg(feature = "dfa-build")] + pub(crate) fn to_state_id(&self, index: usize) -> StateID { + self.tt.to_state_id(index) + } + + /// Return the table of state IDs for this DFA's start states. + pub(crate) fn starts(&self) -> StartStateIter<'_> { + self.st.iter() + } + + /// Returns the index of the match state for the given ID. If the + /// given ID does not correspond to a match state, then this may + /// panic or produce an incorrect result. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn match_state_index(&self, id: StateID) -> usize { + debug_assert!(self.is_match_state(id)); + // This is one of the places where we rely on the fact that match + // states are contiguous in the transition table. Namely, that the + // first match state ID always corresponds to dfa.special.min_match. + // From there, since we know the stride, we can compute the overall + // index of any match state given the match state's ID. + let min = self.special().min_match.as_usize(); + // CORRECTNESS: We're allowed to produce an incorrect result or panic, + // so both the subtraction and the unchecked StateID construction is + // OK. + self.to_index(StateID::new_unchecked(id.as_usize() - min)) + } + + /// Returns the index of the accelerator state for the given ID. If the + /// given ID does not correspond to an accelerator state, then this may + /// panic or produce an incorrect result. + fn accelerator_index(&self, id: StateID) -> usize { + let min = self.special().min_accel.as_usize(); + // CORRECTNESS: We're allowed to produce an incorrect result or panic, + // so both the subtraction and the unchecked StateID construction is + // OK. + self.to_index(StateID::new_unchecked(id.as_usize() - min)) + } + + /// Return the accelerators for this DFA. + fn accels(&self) -> Accels<&[u32]> { + self.accels.as_ref() + } + + /// Return this DFA's transition table as a slice. + fn trans(&self) -> &[StateID] { + self.tt.table() + } +} + +impl<T: AsRef<[u32]>> fmt::Debug for DFA<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "dense::DFA(")?; + for state in self.states() { + fmt_state_indicator(f, self, state.id())?; + let id = if f.alternate() { + state.id().as_usize() + } else { + self.to_index(state.id()) + }; + write!(f, "{id:06?}: ")?; + state.fmt(f)?; + write!(f, "\n")?; + } + writeln!(f, "")?; + for (i, (start_id, anchored, sty)) in self.starts().enumerate() { + let id = if f.alternate() { + start_id.as_usize() + } else { + self.to_index(start_id) + }; + if i % self.st.stride == 0 { + match anchored { + Anchored::No => writeln!(f, "START-GROUP(unanchored)")?, + Anchored::Yes => writeln!(f, "START-GROUP(anchored)")?, + Anchored::Pattern(pid) => { + writeln!(f, "START_GROUP(pattern: {pid:?})")? + } + } + } + writeln!(f, " {sty:?} => {id:06?}")?; + } + if self.pattern_len() > 1 { + writeln!(f, "")?; + for i in 0..self.ms.len() { + let id = self.ms.match_state_id(self, i); + let id = if f.alternate() { + id.as_usize() + } else { + self.to_index(id) + }; + write!(f, "MATCH({id:06?}): ")?; + for (i, &pid) in self.ms.pattern_id_slice(i).iter().enumerate() + { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{pid:?}")?; + } + writeln!(f, "")?; + } + } + writeln!(f, "state length: {:?}", self.state_len())?; + writeln!(f, "pattern length: {:?}", self.pattern_len())?; + writeln!(f, "flags: {:?}", self.flags)?; + writeln!(f, ")")?; + Ok(()) + } +} + +// SAFETY: We assert that our implementation of each method is correct. +unsafe impl<T: AsRef<[u32]>> Automaton for DFA<T> { + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_special_state(&self, id: StateID) -> bool { + self.special.is_special_state(id) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_dead_state(&self, id: StateID) -> bool { + self.special.is_dead_state(id) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_quit_state(&self, id: StateID) -> bool { + self.special.is_quit_state(id) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_match_state(&self, id: StateID) -> bool { + self.special.is_match_state(id) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_start_state(&self, id: StateID) -> bool { + self.special.is_start_state(id) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_accel_state(&self, id: StateID) -> bool { + self.special.is_accel_state(id) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn next_state(&self, current: StateID, input: u8) -> StateID { + let input = self.byte_classes().get(input); + let o = current.as_usize() + usize::from(input); + self.trans()[o] + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + unsafe fn next_state_unchecked( + &self, + current: StateID, + byte: u8, + ) -> StateID { + // We don't (or shouldn't) need an unchecked variant for the byte + // class mapping, since bound checks should be omitted automatically + // by virtue of its representation. If this ends up not being true as + // confirmed by codegen, please file an issue. ---AG + let class = self.byte_classes().get(byte); + let o = current.as_usize() + usize::from(class); + let next = *self.trans().get_unchecked(o); + next + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn next_eoi_state(&self, current: StateID) -> StateID { + let eoi = self.byte_classes().eoi().as_usize(); + let o = current.as_usize() + eoi; + self.trans()[o] + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn pattern_len(&self) -> usize { + self.ms.pattern_len + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn match_len(&self, id: StateID) -> usize { + self.match_pattern_len(id) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn match_pattern(&self, id: StateID, match_index: usize) -> PatternID { + // This is an optimization for the very common case of a DFA with a + // single pattern. This conditional avoids a somewhat more costly path + // that finds the pattern ID from the state machine, which requires + // a bit of slicing/pointer-chasing. This optimization tends to only + // matter when matches are frequent. + if self.ms.pattern_len == 1 { + return PatternID::ZERO; + } + let state_index = self.match_state_index(id); + self.ms.pattern_id(state_index, match_index) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn has_empty(&self) -> bool { + self.flags.has_empty + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_utf8(&self) -> bool { + self.flags.is_utf8 + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_always_start_anchored(&self) -> bool { + self.flags.is_always_start_anchored + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn start_state( + &self, + config: &start::Config, + ) -> Result<StateID, StartError> { + let anchored = config.get_anchored(); + let start = match config.get_look_behind() { + None => Start::Text, + Some(byte) => { + if !self.quitset.is_empty() && self.quitset.contains(byte) { + return Err(StartError::quit(byte)); + } + self.st.start_map.get(byte) + } + }; + self.st.start(anchored, start) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn universal_start_state(&self, mode: Anchored) -> Option<StateID> { + match mode { + Anchored::No => self.st.universal_start_unanchored, + Anchored::Yes => self.st.universal_start_anchored, + Anchored::Pattern(_) => None, + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn accelerator(&self, id: StateID) -> &[u8] { + if !self.is_accel_state(id) { + return &[]; + } + self.accels.needles(self.accelerator_index(id)) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn get_prefilter(&self) -> Option<&Prefilter> { + self.pre.as_ref() + } +} + +/// The transition table portion of a dense DFA. +/// +/// The transition table is the core part of the DFA in that it describes how +/// to move from one state to another based on the input sequence observed. +#[derive(Clone)] +pub(crate) struct TransitionTable<T> { + /// A contiguous region of memory representing the transition table in + /// row-major order. The representation is dense. That is, every state + /// has precisely the same number of transitions. The maximum number of + /// transitions per state is 257 (256 for each possible byte value, plus 1 + /// for the special EOI transition). If a DFA has been instructed to use + /// byte classes (the default), then the number of transitions is usually + /// substantially fewer. + /// + /// In practice, T is either `Vec<u32>` or `&[u32]`. + table: T, + /// A set of equivalence classes, where a single equivalence class + /// represents a set of bytes that never discriminate between a match + /// and a non-match in the DFA. Each equivalence class corresponds to a + /// single character in this DFA's alphabet, where the maximum number of + /// characters is 257 (each possible value of a byte plus the special + /// EOI transition). Consequently, the number of equivalence classes + /// corresponds to the number of transitions for each DFA state. Note + /// though that the *space* used by each DFA state in the transition table + /// may be larger. The total space used by each DFA state is known as the + /// stride. + /// + /// The only time the number of equivalence classes is fewer than 257 is if + /// the DFA's kind uses byte classes (which is the default). Equivalence + /// classes should generally only be disabled when debugging, so that + /// the transitions themselves aren't obscured. Disabling them has no + /// other benefit, since the equivalence class map is always used while + /// searching. In the vast majority of cases, the number of equivalence + /// classes is substantially smaller than 257, particularly when large + /// Unicode classes aren't used. + classes: ByteClasses, + /// The stride of each DFA state, expressed as a power-of-two exponent. + /// + /// The stride of a DFA corresponds to the total amount of space used by + /// each DFA state in the transition table. This may be bigger than the + /// size of a DFA's alphabet, since the stride is always the smallest + /// power of two greater than or equal to the alphabet size. + /// + /// While this wastes space, this avoids the need for integer division + /// to convert between premultiplied state IDs and their corresponding + /// indices. Instead, we can use simple bit-shifts. + /// + /// See the docs for the `stride2` method for more details. + /// + /// The minimum `stride2` value is `1` (corresponding to a stride of `2`) + /// while the maximum `stride2` value is `9` (corresponding to a stride of + /// `512`). The maximum is not `8` since the maximum alphabet size is `257` + /// when accounting for the special EOI transition. However, an alphabet + /// length of that size is exceptionally rare since the alphabet is shrunk + /// into equivalence classes. + stride2: usize, +} + +impl<'a> TransitionTable<&'a [u32]> { + /// Deserialize a transition table starting at the beginning of `slice`. + /// Upon success, return the total number of bytes read along with the + /// transition table. + /// + /// If there was a problem deserializing any part of the transition table, + /// then this returns an error. Notably, if the given slice does not have + /// the same alignment as `StateID`, then this will return an error (among + /// other possible errors). + /// + /// This is guaranteed to execute in constant time. + /// + /// # Safety + /// + /// This routine is not safe because it does not check the validity of the + /// transition table itself. In particular, the transition table can be + /// quite large, so checking its validity can be somewhat expensive. An + /// invalid transition table is not safe because other code may rely on the + /// transition table being correct (such as explicit bounds check elision). + /// Therefore, an invalid transition table can lead to undefined behavior. + /// + /// Callers that use this function must either pass on the safety invariant + /// or guarantee that the bytes given contain a valid transition table. + /// This guarantee is upheld by the bytes written by `write_to`. + unsafe fn from_bytes_unchecked( + mut slice: &'a [u8], + ) -> Result<(TransitionTable<&'a [u32]>, usize), DeserializeError> { + let slice_start = slice.as_ptr().as_usize(); + + let (state_len, nr) = + wire::try_read_u32_as_usize(slice, "state length")?; + slice = &slice[nr..]; + + let (stride2, nr) = wire::try_read_u32_as_usize(slice, "stride2")?; + slice = &slice[nr..]; + + let (classes, nr) = ByteClasses::from_bytes(slice)?; + slice = &slice[nr..]; + + // The alphabet length (determined by the byte class map) cannot be + // bigger than the stride (total space used by each DFA state). + if stride2 > 9 { + return Err(DeserializeError::generic( + "dense DFA has invalid stride2 (too big)", + )); + } + // It also cannot be zero, since even a DFA that never matches anything + // has a non-zero number of states with at least two equivalence + // classes: one for all 256 byte values and another for the EOI + // sentinel. + if stride2 < 1 { + return Err(DeserializeError::generic( + "dense DFA has invalid stride2 (too small)", + )); + } + // This is OK since 1 <= stride2 <= 9. + let stride = + 1usize.checked_shl(u32::try_from(stride2).unwrap()).unwrap(); + if classes.alphabet_len() > stride { + return Err(DeserializeError::generic( + "alphabet size cannot be bigger than transition table stride", + )); + } + + let trans_len = + wire::shl(state_len, stride2, "dense table transition length")?; + let table_bytes_len = wire::mul( + trans_len, + StateID::SIZE, + "dense table state byte length", + )?; + wire::check_slice_len(slice, table_bytes_len, "transition table")?; + wire::check_alignment::<StateID>(slice)?; + let table_bytes = &slice[..table_bytes_len]; + slice = &slice[table_bytes_len..]; + // SAFETY: Since StateID is always representable as a u32, all we need + // to do is ensure that we have the proper length and alignment. We've + // checked both above, so the cast below is safe. + // + // N.B. This is the only not-safe code in this function. + let table = core::slice::from_raw_parts( + table_bytes.as_ptr().cast::<u32>(), + trans_len, + ); + let tt = TransitionTable { table, classes, stride2 }; + Ok((tt, slice.as_ptr().as_usize() - slice_start)) + } +} + +#[cfg(feature = "dfa-build")] +impl TransitionTable<Vec<u32>> { + /// Create a minimal transition table with just two states: a dead state + /// and a quit state. The alphabet length and stride of the transition + /// table is determined by the given set of equivalence classes. + fn minimal(classes: ByteClasses) -> TransitionTable<Vec<u32>> { + let mut tt = TransitionTable { + table: vec![], + classes, + stride2: classes.stride2(), + }; + // Two states, regardless of alphabet size, can always fit into u32. + tt.add_empty_state().unwrap(); // dead state + tt.add_empty_state().unwrap(); // quit state + tt + } + + /// Set a transition in this table. Both the `from` and `to` states must + /// already exist, otherwise this panics. `unit` should correspond to the + /// transition out of `from` to set to `to`. + fn set(&mut self, from: StateID, unit: alphabet::Unit, to: StateID) { + assert!(self.is_valid(from), "invalid 'from' state"); + assert!(self.is_valid(to), "invalid 'to' state"); + self.table[from.as_usize() + self.classes.get_by_unit(unit)] = + to.as_u32(); + } + + /// Add an empty state (a state where all transitions lead to a dead state) + /// and return its identifier. The identifier returned is guaranteed to + /// not point to any other existing state. + /// + /// If adding a state would exhaust the state identifier space, then this + /// returns an error. + fn add_empty_state(&mut self) -> Result<StateID, BuildError> { + // Normally, to get a fresh state identifier, we would just + // take the index of the next state added to the transition + // table. However, we actually perform an optimization here + // that pre-multiplies state IDs by the stride, such that they + // point immediately at the beginning of their transitions in + // the transition table. This avoids an extra multiplication + // instruction for state lookup at search time. + // + // Premultiplied identifiers means that instead of your matching + // loop looking something like this: + // + // state = dfa.start + // for byte in haystack: + // next = dfa.transitions[state * stride + byte] + // if dfa.is_match(next): + // return true + // return false + // + // it can instead look like this: + // + // state = dfa.start + // for byte in haystack: + // next = dfa.transitions[state + byte] + // if dfa.is_match(next): + // return true + // return false + // + // In other words, we save a multiplication instruction in the + // critical path. This turns out to be a decent performance win. + // The cost of using premultiplied state ids is that they can + // require a bigger state id representation. (And they also make + // the code a bit more complex, especially during minimization and + // when reshuffling states, as one needs to convert back and forth + // between state IDs and state indices.) + // + // To do this, we simply take the index of the state into the + // entire transition table, rather than the index of the state + // itself. e.g., If the stride is 64, then the ID of the 3rd state + // is 192, not 2. + let next = self.table.len(); + let id = + StateID::new(next).map_err(|_| BuildError::too_many_states())?; + self.table.extend(iter::repeat(0).take(self.stride())); + Ok(id) + } + + /// Swap the two states given in this transition table. + /// + /// This routine does not do anything to check the correctness of this + /// swap. Callers must ensure that other states pointing to id1 and id2 are + /// updated appropriately. + /// + /// Both id1 and id2 must point to valid states, otherwise this panics. + fn swap(&mut self, id1: StateID, id2: StateID) { + assert!(self.is_valid(id1), "invalid 'id1' state: {id1:?}"); + assert!(self.is_valid(id2), "invalid 'id2' state: {id2:?}"); + // We only need to swap the parts of the state that are used. So if the + // stride is 64, but the alphabet length is only 33, then we save a lot + // of work. + for b in 0..self.classes.alphabet_len() { + self.table.swap(id1.as_usize() + b, id2.as_usize() + b); + } + } + + /// Remap the transitions for the state given according to the function + /// given. This applies the given map function to every transition in the + /// given state and changes the transition in place to the result of the + /// map function for that transition. + fn remap(&mut self, id: StateID, map: impl Fn(StateID) -> StateID) { + for byte in 0..self.alphabet_len() { + let i = id.as_usize() + byte; + let next = self.table()[i]; + self.table_mut()[id.as_usize() + byte] = map(next); + } + } + + /// Truncate the states in this transition table to the given length. + /// + /// This routine does not do anything to check the correctness of this + /// truncation. Callers must ensure that other states pointing to truncated + /// states are updated appropriately. + fn truncate(&mut self, len: usize) { + self.table.truncate(len << self.stride2); + } +} + +impl<T: AsRef<[u32]>> TransitionTable<T> { + /// Writes a serialized form of this transition table to the buffer given. + /// If the buffer is too small, then an error is returned. To determine + /// how big the buffer must be, use `write_to_len`. + fn write_to<E: Endian>( + &self, + mut dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("transition table")); + } + dst = &mut dst[..nwrite]; + + // write state length + // Unwrap is OK since number of states is guaranteed to fit in a u32. + E::write_u32(u32::try_from(self.len()).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + + // write state stride (as power of 2) + // Unwrap is OK since stride2 is guaranteed to be <= 9. + E::write_u32(u32::try_from(self.stride2).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + + // write byte class map + let n = self.classes.write_to(dst)?; + dst = &mut dst[n..]; + + // write actual transitions + for &sid in self.table() { + let n = wire::write_state_id::<E>(sid, &mut dst); + dst = &mut dst[n..]; + } + Ok(nwrite) + } + + /// Returns the number of bytes the serialized form of this transition + /// table will use. + fn write_to_len(&self) -> usize { + size_of::<u32>() // state length + + size_of::<u32>() // stride2 + + self.classes.write_to_len() + + (self.table().len() * StateID::SIZE) + } + + /// Validates that every state ID in this transition table is valid. + /// + /// That is, every state ID can be used to correctly index a state in this + /// table. + fn validate(&self, dfa: &DFA<T>) -> Result<(), DeserializeError> { + let sp = &dfa.special; + for state in self.states() { + // We check that the ID itself is well formed. That is, if it's + // a special state then it must actually be a quit, dead, accel, + // match or start state. + if sp.is_special_state(state.id()) { + let is_actually_special = sp.is_dead_state(state.id()) + || sp.is_quit_state(state.id()) + || sp.is_match_state(state.id()) + || sp.is_start_state(state.id()) + || sp.is_accel_state(state.id()); + if !is_actually_special { + // This is kind of a cryptic error message... + return Err(DeserializeError::generic( + "found dense state tagged as special but \ + wasn't actually special", + )); + } + if sp.is_match_state(state.id()) + && dfa.match_len(state.id()) == 0 + { + return Err(DeserializeError::generic( + "found match state with zero pattern IDs", + )); + } + } + for (_, to) in state.transitions() { + if !self.is_valid(to) { + return Err(DeserializeError::generic( + "found invalid state ID in transition table", + )); + } + } + } + Ok(()) + } + + /// Converts this transition table to a borrowed value. + fn as_ref(&self) -> TransitionTable<&'_ [u32]> { + TransitionTable { + table: self.table.as_ref(), + classes: self.classes.clone(), + stride2: self.stride2, + } + } + + /// Converts this transition table to an owned value. + #[cfg(feature = "alloc")] + fn to_owned(&self) -> TransitionTable<alloc::vec::Vec<u32>> { + TransitionTable { + table: self.table.as_ref().to_vec(), + classes: self.classes.clone(), + stride2: self.stride2, + } + } + + /// Return the state for the given ID. If the given ID is not valid, then + /// this panics. + fn state(&self, id: StateID) -> State<'_> { + assert!(self.is_valid(id)); + + let i = id.as_usize(); + State { + id, + stride2: self.stride2, + transitions: &self.table()[i..i + self.alphabet_len()], + } + } + + /// Returns an iterator over all states in this transition table. + /// + /// This iterator yields a tuple for each state. The first element of the + /// tuple corresponds to a state's identifier, and the second element + /// corresponds to the state itself (comprised of its transitions). + fn states(&self) -> StateIter<'_, T> { + StateIter { + tt: self, + it: self.table().chunks(self.stride()).enumerate(), + } + } + + /// Convert a state identifier to an index to a state (in the range + /// 0..self.len()). + /// + /// This is useful when using a `Vec<T>` as an efficient map keyed by state + /// to some other information (such as a remapped state ID). + /// + /// If the given ID is not valid, then this may panic or produce an + /// incorrect index. + fn to_index(&self, id: StateID) -> usize { + id.as_usize() >> self.stride2 + } + + /// Convert an index to a state (in the range 0..self.len()) to an actual + /// state identifier. + /// + /// This is useful when using a `Vec<T>` as an efficient map keyed by state + /// to some other information (such as a remapped state ID). + /// + /// If the given index is not in the specified range, then this may panic + /// or produce an incorrect state ID. + fn to_state_id(&self, index: usize) -> StateID { + // CORRECTNESS: If the given index is not valid, then it is not + // required for this to panic or return a valid state ID. + StateID::new_unchecked(index << self.stride2) + } + + /// Returns the state ID for the state immediately following the one given. + /// + /// This does not check whether the state ID returned is invalid. In fact, + /// if the state ID given is the last state in this DFA, then the state ID + /// returned is guaranteed to be invalid. + #[cfg(feature = "dfa-build")] + fn next_state_id(&self, id: StateID) -> StateID { + self.to_state_id(self.to_index(id).checked_add(1).unwrap()) + } + + /// Returns the state ID for the state immediately preceding the one given. + /// + /// If the dead ID given (which is zero), then this panics. + #[cfg(feature = "dfa-build")] + fn prev_state_id(&self, id: StateID) -> StateID { + self.to_state_id(self.to_index(id).checked_sub(1).unwrap()) + } + + /// Returns the table as a slice of state IDs. + fn table(&self) -> &[StateID] { + wire::u32s_to_state_ids(self.table.as_ref()) + } + + /// Returns the total number of states in this transition table. + /// + /// Note that a DFA always has at least two states: the dead and quit + /// states. In particular, the dead state always has ID 0 and is + /// correspondingly always the first state. The dead state is never a match + /// state. + fn len(&self) -> usize { + self.table().len() >> self.stride2 + } + + /// Returns the total stride for every state in this DFA. This corresponds + /// to the total number of transitions used by each state in this DFA's + /// transition table. + fn stride(&self) -> usize { + 1 << self.stride2 + } + + /// Returns the total number of elements in the alphabet for this + /// transition table. This is always less than or equal to `self.stride()`. + /// It is only equal when the alphabet length is a power of 2. Otherwise, + /// it is always strictly less. + fn alphabet_len(&self) -> usize { + self.classes.alphabet_len() + } + + /// Returns true if and only if the given state ID is valid for this + /// transition table. Validity in this context means that the given ID can + /// be used as a valid offset with `self.stride()` to index this transition + /// table. + fn is_valid(&self, id: StateID) -> bool { + let id = id.as_usize(); + id < self.table().len() && id % self.stride() == 0 + } + + /// Return the memory usage, in bytes, of this transition table. + /// + /// This does not include the size of a `TransitionTable` value itself. + fn memory_usage(&self) -> usize { + self.table().len() * StateID::SIZE + } +} + +#[cfg(feature = "dfa-build")] +impl<T: AsMut<[u32]>> TransitionTable<T> { + /// Returns the table as a slice of state IDs. + fn table_mut(&mut self) -> &mut [StateID] { + wire::u32s_to_state_ids_mut(self.table.as_mut()) + } +} + +/// The set of all possible starting states in a DFA. +/// +/// The set of starting states corresponds to the possible choices one can make +/// in terms of starting a DFA. That is, before following the first transition, +/// you first need to select the state that you start in. +/// +/// Normally, a DFA converted from an NFA that has a single starting state +/// would itself just have one starting state. However, our support for look +/// around generally requires more starting states. The correct starting state +/// is chosen based on certain properties of the position at which we begin +/// our search. +/// +/// Before listing those properties, we first must define two terms: +/// +/// * `haystack` - The bytes to execute the search. The search always starts +/// at the beginning of `haystack` and ends before or at the end of +/// `haystack`. +/// * `context` - The (possibly empty) bytes surrounding `haystack`. `haystack` +/// must be contained within `context` such that `context` is at least as big +/// as `haystack`. +/// +/// This split is crucial for dealing with look-around. For example, consider +/// the context `foobarbaz`, the haystack `bar` and the regex `^bar$`. This +/// regex should _not_ match the haystack since `bar` does not appear at the +/// beginning of the input. Similarly, the regex `\Bbar\B` should match the +/// haystack because `bar` is not surrounded by word boundaries. But a search +/// that does not take context into account would not permit `\B` to match +/// since the beginning of any string matches a word boundary. Similarly, a +/// search that does not take context into account when searching `^bar$` in +/// the haystack `bar` would produce a match when it shouldn't. +/// +/// Thus, it follows that the starting state is chosen based on the following +/// criteria, derived from the position at which the search starts in the +/// `context` (corresponding to the start of `haystack`): +/// +/// 1. If the search starts at the beginning of `context`, then the `Text` +/// start state is used. (Since `^` corresponds to +/// `hir::Anchor::Start`.) +/// 2. If the search starts at a position immediately following a line +/// terminator, then the `Line` start state is used. (Since `(?m:^)` +/// corresponds to `hir::Anchor::StartLF`.) +/// 3. If the search starts at a position immediately following a byte +/// classified as a "word" character (`[_0-9a-zA-Z]`), then the `WordByte` +/// start state is used. (Since `(?-u:\b)` corresponds to a word boundary.) +/// 4. Otherwise, if the search starts at a position immediately following +/// a byte that is not classified as a "word" character (`[^_0-9a-zA-Z]`), +/// then the `NonWordByte` start state is used. (Since `(?-u:\B)` +/// corresponds to a not-word-boundary.) +/// +/// (N.B. Unicode word boundaries are not supported by the DFA because they +/// require multi-byte look-around and this is difficult to support in a DFA.) +/// +/// To further complicate things, we also support constructing individual +/// anchored start states for each pattern in the DFA. (Which is required to +/// implement overlapping regexes correctly, but is also generally useful.) +/// Thus, when individual start states for each pattern are enabled, then the +/// total number of start states represented is `4 + (4 * #patterns)`, where +/// the 4 comes from each of the 4 possibilities above. The first 4 represents +/// the starting states for the entire DFA, which support searching for +/// multiple patterns simultaneously (possibly unanchored). +/// +/// If individual start states are disabled, then this will only store 4 +/// start states. Typically, individual start states are only enabled when +/// constructing the reverse DFA for regex matching. But they are also useful +/// for building DFAs that can search for a specific pattern or even to support +/// both anchored and unanchored searches with the same DFA. +/// +/// Note though that while the start table always has either `4` or +/// `4 + (4 * #patterns)` starting state *ids*, the total number of states +/// might be considerably smaller. That is, many of the IDs may be duplicative. +/// (For example, if a regex doesn't have a `\b` sub-pattern, then there's no +/// reason to generate a unique starting state for handling word boundaries. +/// Similarly for start/end anchors.) +#[derive(Clone)] +pub(crate) struct StartTable<T> { + /// The initial start state IDs. + /// + /// In practice, T is either `Vec<u32>` or `&[u32]`. + /// + /// The first `2 * stride` (currently always 8) entries always correspond + /// to the starts states for the entire DFA, with the first 4 entries being + /// for unanchored searches and the second 4 entries being for anchored + /// searches. To keep things simple, we always use 8 entries even if the + /// `StartKind` is not both. + /// + /// After that, there are `stride * patterns` state IDs, where `patterns` + /// may be zero in the case of a DFA with no patterns or in the case where + /// the DFA was built without enabling starting states for each pattern. + table: T, + /// The starting state configuration supported. When 'both', both + /// unanchored and anchored searches work. When 'unanchored', anchored + /// searches panic. When 'anchored', unanchored searches panic. + kind: StartKind, + /// The start state configuration for every possible byte. + start_map: StartByteMap, + /// The number of starting state IDs per pattern. + stride: usize, + /// The total number of patterns for which starting states are encoded. + /// This is `None` for DFAs that were built without start states for each + /// pattern. Thus, one cannot use this field to say how many patterns + /// are in the DFA in all cases. It is specific to how many patterns are + /// represented in this start table. + pattern_len: Option<usize>, + /// The universal starting state for unanchored searches. This is only + /// present when the DFA supports unanchored searches and when all starting + /// state IDs for an unanchored search are equivalent. + universal_start_unanchored: Option<StateID>, + /// The universal starting state for anchored searches. This is only + /// present when the DFA supports anchored searches and when all starting + /// state IDs for an anchored search are equivalent. + universal_start_anchored: Option<StateID>, +} + +#[cfg(feature = "dfa-build")] +impl StartTable<Vec<u32>> { + /// Create a valid set of start states all pointing to the dead state. + /// + /// When the corresponding DFA is constructed with start states for each + /// pattern, then `patterns` should be the number of patterns. Otherwise, + /// it should be zero. + /// + /// If the total table size could exceed the allocatable limit, then this + /// returns an error. In practice, this is unlikely to be able to occur, + /// since it's likely that allocation would have failed long before it got + /// to this point. + fn dead( + kind: StartKind, + lookm: &LookMatcher, + pattern_len: Option<usize>, + ) -> Result<StartTable<Vec<u32>>, BuildError> { + if let Some(len) = pattern_len { + assert!(len <= PatternID::LIMIT); + } + let stride = Start::len(); + // OK because 2*4 is never going to overflow anything. + let starts_len = stride.checked_mul(2).unwrap(); + let pattern_starts_len = + match stride.checked_mul(pattern_len.unwrap_or(0)) { + Some(x) => x, + None => return Err(BuildError::too_many_start_states()), + }; + let table_len = match starts_len.checked_add(pattern_starts_len) { + Some(x) => x, + None => return Err(BuildError::too_many_start_states()), + }; + if let Err(_) = isize::try_from(table_len) { + return Err(BuildError::too_many_start_states()); + } + let table = vec![DEAD.as_u32(); table_len]; + let start_map = StartByteMap::new(lookm); + Ok(StartTable { + table, + kind, + start_map, + stride, + pattern_len, + universal_start_unanchored: None, + universal_start_anchored: None, + }) + } +} + +impl<'a> StartTable<&'a [u32]> { + /// Deserialize a table of start state IDs starting at the beginning of + /// `slice`. Upon success, return the total number of bytes read along with + /// the table of starting state IDs. + /// + /// If there was a problem deserializing any part of the starting IDs, + /// then this returns an error. Notably, if the given slice does not have + /// the same alignment as `StateID`, then this will return an error (among + /// other possible errors). + /// + /// This is guaranteed to execute in constant time. + /// + /// # Safety + /// + /// This routine is not safe because it does not check the validity of the + /// starting state IDs themselves. In particular, the number of starting + /// IDs can be of variable length, so it's possible that checking their + /// validity cannot be done in constant time. An invalid starting state + /// ID is not safe because other code may rely on the starting IDs being + /// correct (such as explicit bounds check elision). Therefore, an invalid + /// start ID can lead to undefined behavior. + /// + /// Callers that use this function must either pass on the safety invariant + /// or guarantee that the bytes given contain valid starting state IDs. + /// This guarantee is upheld by the bytes written by `write_to`. + unsafe fn from_bytes_unchecked( + mut slice: &'a [u8], + ) -> Result<(StartTable<&'a [u32]>, usize), DeserializeError> { + let slice_start = slice.as_ptr().as_usize(); + + let (kind, nr) = StartKind::from_bytes(slice)?; + slice = &slice[nr..]; + + let (start_map, nr) = StartByteMap::from_bytes(slice)?; + slice = &slice[nr..]; + + let (stride, nr) = + wire::try_read_u32_as_usize(slice, "start table stride")?; + slice = &slice[nr..]; + if stride != Start::len() { + return Err(DeserializeError::generic( + "invalid starting table stride", + )); + } + + let (maybe_pattern_len, nr) = + wire::try_read_u32_as_usize(slice, "start table patterns")?; + slice = &slice[nr..]; + let pattern_len = if maybe_pattern_len.as_u32() == u32::MAX { + None + } else { + Some(maybe_pattern_len) + }; + if pattern_len.map_or(false, |len| len > PatternID::LIMIT) { + return Err(DeserializeError::generic( + "invalid number of patterns", + )); + } + + let (universal_unanchored, nr) = + wire::try_read_u32(slice, "universal unanchored start")?; + slice = &slice[nr..]; + let universal_start_unanchored = if universal_unanchored == u32::MAX { + None + } else { + Some(StateID::try_from(universal_unanchored).map_err(|e| { + DeserializeError::state_id_error( + e, + "universal unanchored start", + ) + })?) + }; + + let (universal_anchored, nr) = + wire::try_read_u32(slice, "universal anchored start")?; + slice = &slice[nr..]; + let universal_start_anchored = if universal_anchored == u32::MAX { + None + } else { + Some(StateID::try_from(universal_anchored).map_err(|e| { + DeserializeError::state_id_error(e, "universal anchored start") + })?) + }; + + let pattern_table_size = wire::mul( + stride, + pattern_len.unwrap_or(0), + "invalid pattern length", + )?; + // Our start states always start with a two stride of start states for + // the entire automaton. The first stride is for unanchored starting + // states and the second stride is for anchored starting states. What + // follows it are an optional set of start states for each pattern. + let start_state_len = wire::add( + wire::mul(2, stride, "start state stride too big")?, + pattern_table_size, + "invalid 'any' pattern starts size", + )?; + let table_bytes_len = wire::mul( + start_state_len, + StateID::SIZE, + "pattern table bytes length", + )?; + wire::check_slice_len(slice, table_bytes_len, "start ID table")?; + wire::check_alignment::<StateID>(slice)?; + let table_bytes = &slice[..table_bytes_len]; + slice = &slice[table_bytes_len..]; + // SAFETY: Since StateID is always representable as a u32, all we need + // to do is ensure that we have the proper length and alignment. We've + // checked both above, so the cast below is safe. + // + // N.B. This is the only not-safe code in this function. + let table = core::slice::from_raw_parts( + table_bytes.as_ptr().cast::<u32>(), + start_state_len, + ); + let st = StartTable { + table, + kind, + start_map, + stride, + pattern_len, + universal_start_unanchored, + universal_start_anchored, + }; + Ok((st, slice.as_ptr().as_usize() - slice_start)) + } +} + +impl<T: AsRef<[u32]>> StartTable<T> { + /// Writes a serialized form of this start table to the buffer given. If + /// the buffer is too small, then an error is returned. To determine how + /// big the buffer must be, use `write_to_len`. + fn write_to<E: Endian>( + &self, + mut dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small( + "starting table ids", + )); + } + dst = &mut dst[..nwrite]; + + // write start kind + let nw = self.kind.write_to::<E>(dst)?; + dst = &mut dst[nw..]; + // write start byte map + let nw = self.start_map.write_to(dst)?; + dst = &mut dst[nw..]; + // write stride + // Unwrap is OK since the stride is always 4 (currently). + E::write_u32(u32::try_from(self.stride).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + // write pattern length + // Unwrap is OK since number of patterns is guaranteed to fit in a u32. + E::write_u32( + u32::try_from(self.pattern_len.unwrap_or(0xFFFF_FFFF)).unwrap(), + dst, + ); + dst = &mut dst[size_of::<u32>()..]; + // write universal start unanchored state id, u32::MAX if absent + E::write_u32( + self.universal_start_unanchored + .map_or(u32::MAX, |sid| sid.as_u32()), + dst, + ); + dst = &mut dst[size_of::<u32>()..]; + // write universal start anchored state id, u32::MAX if absent + E::write_u32( + self.universal_start_anchored.map_or(u32::MAX, |sid| sid.as_u32()), + dst, + ); + dst = &mut dst[size_of::<u32>()..]; + // write start IDs + for &sid in self.table() { + let n = wire::write_state_id::<E>(sid, &mut dst); + dst = &mut dst[n..]; + } + Ok(nwrite) + } + + /// Returns the number of bytes the serialized form of this start ID table + /// will use. + fn write_to_len(&self) -> usize { + self.kind.write_to_len() + + self.start_map.write_to_len() + + size_of::<u32>() // stride + + size_of::<u32>() // # patterns + + size_of::<u32>() // universal unanchored start + + size_of::<u32>() // universal anchored start + + (self.table().len() * StateID::SIZE) + } + + /// Validates that every state ID in this start table is valid by checking + /// it against the given transition table (which must be for the same DFA). + /// + /// That is, every state ID can be used to correctly index a state. + fn validate(&self, dfa: &DFA<T>) -> Result<(), DeserializeError> { + let tt = &dfa.tt; + if !self.universal_start_unanchored.map_or(true, |s| tt.is_valid(s)) { + return Err(DeserializeError::generic( + "found invalid universal unanchored starting state ID", + )); + } + if !self.universal_start_anchored.map_or(true, |s| tt.is_valid(s)) { + return Err(DeserializeError::generic( + "found invalid universal anchored starting state ID", + )); + } + for &id in self.table() { + if !tt.is_valid(id) { + return Err(DeserializeError::generic( + "found invalid starting state ID", + )); + } + if dfa.special.is_match_state(id) { + return Err(DeserializeError::generic( + "start states cannot be match states", + )); + } + } + Ok(()) + } + + /// Converts this start list to a borrowed value. + fn as_ref(&self) -> StartTable<&'_ [u32]> { + StartTable { + table: self.table.as_ref(), + kind: self.kind, + start_map: self.start_map.clone(), + stride: self.stride, + pattern_len: self.pattern_len, + universal_start_unanchored: self.universal_start_unanchored, + universal_start_anchored: self.universal_start_anchored, + } + } + + /// Converts this start list to an owned value. + #[cfg(feature = "alloc")] + fn to_owned(&self) -> StartTable<alloc::vec::Vec<u32>> { + StartTable { + table: self.table.as_ref().to_vec(), + kind: self.kind, + start_map: self.start_map.clone(), + stride: self.stride, + pattern_len: self.pattern_len, + universal_start_unanchored: self.universal_start_unanchored, + universal_start_anchored: self.universal_start_anchored, + } + } + + /// Return the start state for the given input and starting configuration. + /// This returns an error if the input configuration is not supported by + /// this DFA. For example, requesting an unanchored search when the DFA was + /// not built with unanchored starting states. Or asking for an anchored + /// pattern search with an invalid pattern ID or on a DFA that was not + /// built with start states for each pattern. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn start( + &self, + anchored: Anchored, + start: Start, + ) -> Result<StateID, StartError> { + let start_index = start.as_usize(); + let index = match anchored { + Anchored::No => { + if !self.kind.has_unanchored() { + return Err(StartError::unsupported_anchored(anchored)); + } + start_index + } + Anchored::Yes => { + if !self.kind.has_anchored() { + return Err(StartError::unsupported_anchored(anchored)); + } + self.stride + start_index + } + Anchored::Pattern(pid) => { + let len = match self.pattern_len { + None => { + return Err(StartError::unsupported_anchored(anchored)) + } + Some(len) => len, + }; + if pid.as_usize() >= len { + return Ok(DEAD); + } + (2 * self.stride) + + (self.stride * pid.as_usize()) + + start_index + } + }; + Ok(self.table()[index]) + } + + /// Returns an iterator over all start state IDs in this table. + /// + /// Each item is a triple of: start state ID, the start state type and the + /// pattern ID (if any). + fn iter(&self) -> StartStateIter<'_> { + StartStateIter { st: self.as_ref(), i: 0 } + } + + /// Returns the table as a slice of state IDs. + fn table(&self) -> &[StateID] { + wire::u32s_to_state_ids(self.table.as_ref()) + } + + /// Return the memory usage, in bytes, of this start list. + /// + /// This does not include the size of a `StartList` value itself. + fn memory_usage(&self) -> usize { + self.table().len() * StateID::SIZE + } +} + +#[cfg(feature = "dfa-build")] +impl<T: AsMut<[u32]>> StartTable<T> { + /// Set the start state for the given index and pattern. + /// + /// If the pattern ID or state ID are not valid, then this will panic. + fn set_start(&mut self, anchored: Anchored, start: Start, id: StateID) { + let start_index = start.as_usize(); + let index = match anchored { + Anchored::No => start_index, + Anchored::Yes => self.stride + start_index, + Anchored::Pattern(pid) => { + let pid = pid.as_usize(); + let len = self + .pattern_len + .expect("start states for each pattern enabled"); + assert!(pid < len, "invalid pattern ID {pid:?}"); + self.stride + .checked_mul(pid) + .unwrap() + .checked_add(self.stride.checked_mul(2).unwrap()) + .unwrap() + .checked_add(start_index) + .unwrap() + } + }; + self.table_mut()[index] = id; + } + + /// Returns the table as a mutable slice of state IDs. + fn table_mut(&mut self) -> &mut [StateID] { + wire::u32s_to_state_ids_mut(self.table.as_mut()) + } +} + +/// An iterator over start state IDs. +/// +/// This iterator yields a triple of start state ID, the anchored mode and the +/// start state type. If a pattern ID is relevant, then the anchored mode will +/// contain it. Start states with an anchored mode containing a pattern ID will +/// only occur when the DFA was compiled with start states for each pattern +/// (which is disabled by default). +pub(crate) struct StartStateIter<'a> { + st: StartTable<&'a [u32]>, + i: usize, +} + +impl<'a> Iterator for StartStateIter<'a> { + type Item = (StateID, Anchored, Start); + + fn next(&mut self) -> Option<(StateID, Anchored, Start)> { + let i = self.i; + let table = self.st.table(); + if i >= table.len() { + return None; + } + self.i += 1; + + // This unwrap is okay since the stride of the starting state table + // must always match the number of start state types. + let start_type = Start::from_usize(i % self.st.stride).unwrap(); + let anchored = if i < self.st.stride { + Anchored::No + } else if i < (2 * self.st.stride) { + Anchored::Yes + } else { + let pid = (i - (2 * self.st.stride)) / self.st.stride; + Anchored::Pattern(PatternID::new(pid).unwrap()) + }; + Some((table[i], anchored, start_type)) + } +} + +/// This type represents that patterns that should be reported whenever a DFA +/// enters a match state. This structure exists to support DFAs that search for +/// matches for multiple regexes. +/// +/// This structure relies on the fact that all match states in a DFA occur +/// contiguously in the DFA's transition table. (See dfa/special.rs for a more +/// detailed breakdown of the representation.) Namely, when a match occurs, we +/// know its state ID. Since we know the start and end of the contiguous region +/// of match states, we can use that to compute the position at which the match +/// state occurs. That in turn is used as an offset into this structure. +#[derive(Clone, Debug)] +struct MatchStates<T> { + /// Slices is a flattened sequence of pairs, where each pair points to a + /// sub-slice of pattern_ids. The first element of the pair is an offset + /// into pattern_ids and the second element of the pair is the number + /// of 32-bit pattern IDs starting at that position. That is, each pair + /// corresponds to a single DFA match state and its corresponding match + /// IDs. The number of pairs always corresponds to the number of distinct + /// DFA match states. + /// + /// In practice, T is either Vec<u32> or &[u32]. + slices: T, + /// A flattened sequence of pattern IDs for each DFA match state. The only + /// way to correctly read this sequence is indirectly via `slices`. + /// + /// In practice, T is either Vec<u32> or &[u32]. + pattern_ids: T, + /// The total number of unique patterns represented by these match states. + pattern_len: usize, +} + +impl<'a> MatchStates<&'a [u32]> { + unsafe fn from_bytes_unchecked( + mut slice: &'a [u8], + ) -> Result<(MatchStates<&'a [u32]>, usize), DeserializeError> { + let slice_start = slice.as_ptr().as_usize(); + + // Read the total number of match states. + let (state_len, nr) = + wire::try_read_u32_as_usize(slice, "match state length")?; + slice = &slice[nr..]; + + // Read the slice start/length pairs. + let pair_len = wire::mul(2, state_len, "match state offset pairs")?; + let slices_bytes_len = wire::mul( + pair_len, + PatternID::SIZE, + "match state slice offset byte length", + )?; + wire::check_slice_len(slice, slices_bytes_len, "match state slices")?; + wire::check_alignment::<PatternID>(slice)?; + let slices_bytes = &slice[..slices_bytes_len]; + slice = &slice[slices_bytes_len..]; + // SAFETY: Since PatternID is always representable as a u32, all we + // need to do is ensure that we have the proper length and alignment. + // We've checked both above, so the cast below is safe. + // + // N.B. This is one of the few not-safe snippets in this function, + // so we mark it explicitly to call it out. + let slices = core::slice::from_raw_parts( + slices_bytes.as_ptr().cast::<u32>(), + pair_len, + ); + + // Read the total number of unique pattern IDs (which is always 1 more + // than the maximum pattern ID in this automaton, since pattern IDs are + // handed out contiguously starting at 0). + let (pattern_len, nr) = + wire::try_read_u32_as_usize(slice, "pattern length")?; + slice = &slice[nr..]; + + // Now read the pattern ID length. We don't need to store this + // explicitly, but we need it to know how many pattern IDs to read. + let (idlen, nr) = + wire::try_read_u32_as_usize(slice, "pattern ID length")?; + slice = &slice[nr..]; + + // Read the actual pattern IDs. + let pattern_ids_len = + wire::mul(idlen, PatternID::SIZE, "pattern ID byte length")?; + wire::check_slice_len(slice, pattern_ids_len, "match pattern IDs")?; + wire::check_alignment::<PatternID>(slice)?; + let pattern_ids_bytes = &slice[..pattern_ids_len]; + slice = &slice[pattern_ids_len..]; + // SAFETY: Since PatternID is always representable as a u32, all we + // need to do is ensure that we have the proper length and alignment. + // We've checked both above, so the cast below is safe. + // + // N.B. This is one of the few not-safe snippets in this function, + // so we mark it explicitly to call it out. + let pattern_ids = core::slice::from_raw_parts( + pattern_ids_bytes.as_ptr().cast::<u32>(), + idlen, + ); + + let ms = MatchStates { slices, pattern_ids, pattern_len }; + Ok((ms, slice.as_ptr().as_usize() - slice_start)) + } +} + +#[cfg(feature = "dfa-build")] +impl MatchStates<Vec<u32>> { + fn empty(pattern_len: usize) -> MatchStates<Vec<u32>> { + assert!(pattern_len <= PatternID::LIMIT); + MatchStates { slices: vec![], pattern_ids: vec![], pattern_len } + } + + fn new( + matches: &BTreeMap<StateID, Vec<PatternID>>, + pattern_len: usize, + ) -> Result<MatchStates<Vec<u32>>, BuildError> { + let mut m = MatchStates::empty(pattern_len); + for (_, pids) in matches.iter() { + let start = PatternID::new(m.pattern_ids.len()) + .map_err(|_| BuildError::too_many_match_pattern_ids())?; + m.slices.push(start.as_u32()); + // This is always correct since the number of patterns in a single + // match state can never exceed maximum number of allowable + // patterns. Why? Because a pattern can only appear once in a + // particular match state, by construction. (And since our pattern + // ID limit is one less than u32::MAX, we're guaranteed that the + // length fits in a u32.) + m.slices.push(u32::try_from(pids.len()).unwrap()); + for &pid in pids { + m.pattern_ids.push(pid.as_u32()); + } + } + m.pattern_len = pattern_len; + Ok(m) + } + + fn new_with_map( + &self, + matches: &BTreeMap<StateID, Vec<PatternID>>, + ) -> Result<MatchStates<Vec<u32>>, BuildError> { + MatchStates::new(matches, self.pattern_len) + } +} + +impl<T: AsRef<[u32]>> MatchStates<T> { + /// Writes a serialized form of these match states to the buffer given. If + /// the buffer is too small, then an error is returned. To determine how + /// big the buffer must be, use `write_to_len`. + fn write_to<E: Endian>( + &self, + mut dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("match states")); + } + dst = &mut dst[..nwrite]; + + // write state ID length + // Unwrap is OK since number of states is guaranteed to fit in a u32. + E::write_u32(u32::try_from(self.len()).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + + // write slice offset pairs + for &pid in self.slices() { + let n = wire::write_pattern_id::<E>(pid, &mut dst); + dst = &mut dst[n..]; + } + + // write unique pattern ID length + // Unwrap is OK since number of patterns is guaranteed to fit in a u32. + E::write_u32(u32::try_from(self.pattern_len).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + + // write pattern ID length + // Unwrap is OK since we check at construction (and deserialization) + // that the number of patterns is representable as a u32. + E::write_u32(u32::try_from(self.pattern_ids().len()).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + + // write pattern IDs + for &pid in self.pattern_ids() { + let n = wire::write_pattern_id::<E>(pid, &mut dst); + dst = &mut dst[n..]; + } + + Ok(nwrite) + } + + /// Returns the number of bytes the serialized form of these match states + /// will use. + fn write_to_len(&self) -> usize { + size_of::<u32>() // match state length + + (self.slices().len() * PatternID::SIZE) + + size_of::<u32>() // unique pattern ID length + + size_of::<u32>() // pattern ID length + + (self.pattern_ids().len() * PatternID::SIZE) + } + + /// Validates that the match state info is itself internally consistent and + /// consistent with the recorded match state region in the given DFA. + fn validate(&self, dfa: &DFA<T>) -> Result<(), DeserializeError> { + if self.len() != dfa.special.match_len(dfa.stride()) { + return Err(DeserializeError::generic( + "match state length mismatch", + )); + } + for si in 0..self.len() { + let start = self.slices()[si * 2].as_usize(); + let len = self.slices()[si * 2 + 1].as_usize(); + if start >= self.pattern_ids().len() { + return Err(DeserializeError::generic( + "invalid pattern ID start offset", + )); + } + if start + len > self.pattern_ids().len() { + return Err(DeserializeError::generic( + "invalid pattern ID length", + )); + } + for mi in 0..len { + let pid = self.pattern_id(si, mi); + if pid.as_usize() >= self.pattern_len { + return Err(DeserializeError::generic( + "invalid pattern ID", + )); + } + } + } + Ok(()) + } + + /// Converts these match states back into their map form. This is useful + /// when shuffling states, as the normal MatchStates representation is not + /// amenable to easy state swapping. But with this map, to swap id1 and + /// id2, all you need to do is: + /// + /// if let Some(pids) = map.remove(&id1) { + /// map.insert(id2, pids); + /// } + /// + /// Once shuffling is done, use MatchStates::new to convert back. + #[cfg(feature = "dfa-build")] + fn to_map(&self, dfa: &DFA<T>) -> BTreeMap<StateID, Vec<PatternID>> { + let mut map = BTreeMap::new(); + for i in 0..self.len() { + let mut pids = vec![]; + for j in 0..self.pattern_len(i) { + pids.push(self.pattern_id(i, j)); + } + map.insert(self.match_state_id(dfa, i), pids); + } + map + } + + /// Converts these match states to a borrowed value. + fn as_ref(&self) -> MatchStates<&'_ [u32]> { + MatchStates { + slices: self.slices.as_ref(), + pattern_ids: self.pattern_ids.as_ref(), + pattern_len: self.pattern_len, + } + } + + /// Converts these match states to an owned value. + #[cfg(feature = "alloc")] + fn to_owned(&self) -> MatchStates<alloc::vec::Vec<u32>> { + MatchStates { + slices: self.slices.as_ref().to_vec(), + pattern_ids: self.pattern_ids.as_ref().to_vec(), + pattern_len: self.pattern_len, + } + } + + /// Returns the match state ID given the match state index. (Where the + /// first match state corresponds to index 0.) + /// + /// This panics if there is no match state at the given index. + fn match_state_id(&self, dfa: &DFA<T>, index: usize) -> StateID { + assert!(dfa.special.matches(), "no match states to index"); + // This is one of the places where we rely on the fact that match + // states are contiguous in the transition table. Namely, that the + // first match state ID always corresponds to dfa.special.min_start. + // From there, since we know the stride, we can compute the ID of any + // match state given its index. + let stride2 = u32::try_from(dfa.stride2()).unwrap(); + let offset = index.checked_shl(stride2).unwrap(); + let id = dfa.special.min_match.as_usize().checked_add(offset).unwrap(); + let sid = StateID::new(id).unwrap(); + assert!(dfa.is_match_state(sid)); + sid + } + + /// Returns the pattern ID at the given match index for the given match + /// state. + /// + /// The match state index is the state index minus the state index of the + /// first match state in the DFA. + /// + /// The match index is the index of the pattern ID for the given state. + /// The index must be less than `self.pattern_len(state_index)`. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn pattern_id(&self, state_index: usize, match_index: usize) -> PatternID { + self.pattern_id_slice(state_index)[match_index] + } + + /// Returns the number of patterns in the given match state. + /// + /// The match state index is the state index minus the state index of the + /// first match state in the DFA. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn pattern_len(&self, state_index: usize) -> usize { + self.slices()[state_index * 2 + 1].as_usize() + } + + /// Returns all of the pattern IDs for the given match state index. + /// + /// The match state index is the state index minus the state index of the + /// first match state in the DFA. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn pattern_id_slice(&self, state_index: usize) -> &[PatternID] { + let start = self.slices()[state_index * 2].as_usize(); + let len = self.pattern_len(state_index); + &self.pattern_ids()[start..start + len] + } + + /// Returns the pattern ID offset slice of u32 as a slice of PatternID. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn slices(&self) -> &[PatternID] { + wire::u32s_to_pattern_ids(self.slices.as_ref()) + } + + /// Returns the total number of match states. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn len(&self) -> usize { + assert_eq!(0, self.slices().len() % 2); + self.slices().len() / 2 + } + + /// Returns the pattern ID slice of u32 as a slice of PatternID. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn pattern_ids(&self) -> &[PatternID] { + wire::u32s_to_pattern_ids(self.pattern_ids.as_ref()) + } + + /// Return the memory usage, in bytes, of these match pairs. + fn memory_usage(&self) -> usize { + (self.slices().len() + self.pattern_ids().len()) * PatternID::SIZE + } +} + +/// A common set of flags for both dense and sparse DFAs. This primarily +/// centralizes the serialization format of these flags at a bitset. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Flags { + /// Whether the DFA can match the empty string. When this is false, all + /// matches returned by this DFA are guaranteed to have non-zero length. + pub(crate) has_empty: bool, + /// Whether the DFA should only produce matches with spans that correspond + /// to valid UTF-8. This also includes omitting any zero-width matches that + /// split the UTF-8 encoding of a codepoint. + pub(crate) is_utf8: bool, + /// Whether the DFA is always anchored or not, regardless of `Input` + /// configuration. This is useful for avoiding a reverse scan even when + /// executing unanchored searches. + pub(crate) is_always_start_anchored: bool, +} + +impl Flags { + /// Creates a set of flags for a DFA from an NFA. + /// + /// N.B. This constructor was defined at the time of writing because all + /// of the flags are derived directly from the NFA. If this changes in the + /// future, we might be more thoughtful about how the `Flags` value is + /// itself built. + #[cfg(feature = "dfa-build")] + fn from_nfa(nfa: &thompson::NFA) -> Flags { + Flags { + has_empty: nfa.has_empty(), + is_utf8: nfa.is_utf8(), + is_always_start_anchored: nfa.is_always_start_anchored(), + } + } + + /// Deserializes the flags from the given slice. On success, this also + /// returns the number of bytes read from the slice. + pub(crate) fn from_bytes( + slice: &[u8], + ) -> Result<(Flags, usize), DeserializeError> { + let (bits, nread) = wire::try_read_u32(slice, "flag bitset")?; + let flags = Flags { + has_empty: bits & (1 << 0) != 0, + is_utf8: bits & (1 << 1) != 0, + is_always_start_anchored: bits & (1 << 2) != 0, + }; + Ok((flags, nread)) + } + + /// Writes these flags to the given byte slice. If the buffer is too small, + /// then an error is returned. To determine how big the buffer must be, + /// use `write_to_len`. + pub(crate) fn write_to<E: Endian>( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + fn bool_to_int(b: bool) -> u32 { + if b { + 1 + } else { + 0 + } + } + + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("flag bitset")); + } + let bits = (bool_to_int(self.has_empty) << 0) + | (bool_to_int(self.is_utf8) << 1) + | (bool_to_int(self.is_always_start_anchored) << 2); + E::write_u32(bits, dst); + Ok(nwrite) + } + + /// Returns the number of bytes the serialized form of these flags + /// will use. + pub(crate) fn write_to_len(&self) -> usize { + size_of::<u32>() + } +} + +/// An iterator over all states in a DFA. +/// +/// This iterator yields a tuple for each state. The first element of the +/// tuple corresponds to a state's identifier, and the second element +/// corresponds to the state itself (comprised of its transitions). +/// +/// `'a` corresponding to the lifetime of original DFA, `T` corresponds to +/// the type of the transition table itself. +pub(crate) struct StateIter<'a, T> { + tt: &'a TransitionTable<T>, + it: iter::Enumerate<slice::Chunks<'a, StateID>>, +} + +impl<'a, T: AsRef<[u32]>> Iterator for StateIter<'a, T> { + type Item = State<'a>; + + fn next(&mut self) -> Option<State<'a>> { + self.it.next().map(|(index, _)| { + let id = self.tt.to_state_id(index); + self.tt.state(id) + }) + } +} + +/// An immutable representation of a single DFA state. +/// +/// `'a` corresponding to the lifetime of a DFA's transition table. +pub(crate) struct State<'a> { + id: StateID, + stride2: usize, + transitions: &'a [StateID], +} + +impl<'a> State<'a> { + /// Return an iterator over all transitions in this state. This yields + /// a number of transitions equivalent to the alphabet length of the + /// corresponding DFA. + /// + /// Each transition is represented by a tuple. The first element is + /// the input byte for that transition and the second element is the + /// transitions itself. + pub(crate) fn transitions(&self) -> StateTransitionIter<'_> { + StateTransitionIter { + len: self.transitions.len(), + it: self.transitions.iter().enumerate(), + } + } + + /// Return an iterator over a sparse representation of the transitions in + /// this state. Only non-dead transitions are returned. + /// + /// The "sparse" representation in this case corresponds to a sequence of + /// triples. The first two elements of the triple comprise an inclusive + /// byte range while the last element corresponds to the transition taken + /// for all bytes in the range. + /// + /// This is somewhat more condensed than the classical sparse + /// representation (where you have an element for every non-dead + /// transition), but in practice, checking if a byte is in a range is very + /// cheap and using ranges tends to conserve quite a bit more space. + pub(crate) fn sparse_transitions(&self) -> StateSparseTransitionIter<'_> { + StateSparseTransitionIter { dense: self.transitions(), cur: None } + } + + /// Returns the identifier for this state. + pub(crate) fn id(&self) -> StateID { + self.id + } + + /// Analyzes this state to determine whether it can be accelerated. If so, + /// it returns an accelerator that contains at least one byte. + #[cfg(feature = "dfa-build")] + fn accelerate(&self, classes: &ByteClasses) -> Option<Accel> { + // We just try to add bytes to our accelerator. Once adding fails + // (because we've added too many bytes), then give up. + let mut accel = Accel::new(); + for (class, id) in self.transitions() { + if id == self.id() { + continue; + } + for unit in classes.elements(class) { + if let Some(byte) = unit.as_u8() { + if !accel.add(byte) { + return None; + } + } + } + } + if accel.is_empty() { + None + } else { + Some(accel) + } + } +} + +impl<'a> fmt::Debug for State<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for (i, (start, end, sid)) in self.sparse_transitions().enumerate() { + let id = if f.alternate() { + sid.as_usize() + } else { + sid.as_usize() >> self.stride2 + }; + if i > 0 { + write!(f, ", ")?; + } + if start == end { + write!(f, "{start:?} => {id:?}")?; + } else { + write!(f, "{start:?}-{end:?} => {id:?}")?; + } + } + Ok(()) + } +} + +/// An iterator over all transitions in a single DFA state. This yields +/// a number of transitions equivalent to the alphabet length of the +/// corresponding DFA. +/// +/// Each transition is represented by a tuple. The first element is the input +/// byte for that transition and the second element is the transition itself. +#[derive(Debug)] +pub(crate) struct StateTransitionIter<'a> { + len: usize, + it: iter::Enumerate<slice::Iter<'a, StateID>>, +} + +impl<'a> Iterator for StateTransitionIter<'a> { + type Item = (alphabet::Unit, StateID); + + fn next(&mut self) -> Option<(alphabet::Unit, StateID)> { + self.it.next().map(|(i, &id)| { + let unit = if i + 1 == self.len { + alphabet::Unit::eoi(i) + } else { + let b = u8::try_from(i) + .expect("raw byte alphabet is never exceeded"); + alphabet::Unit::u8(b) + }; + (unit, id) + }) + } +} + +/// An iterator over all non-DEAD transitions in a single DFA state using a +/// sparse representation. +/// +/// Each transition is represented by a triple. The first two elements of the +/// triple comprise an inclusive byte range while the last element corresponds +/// to the transition taken for all bytes in the range. +/// +/// As a convenience, this always returns `alphabet::Unit` values of the same +/// type. That is, you'll never get a (byte, EOI) or a (EOI, byte). Only (byte, +/// byte) and (EOI, EOI) values are yielded. +#[derive(Debug)] +pub(crate) struct StateSparseTransitionIter<'a> { + dense: StateTransitionIter<'a>, + cur: Option<(alphabet::Unit, alphabet::Unit, StateID)>, +} + +impl<'a> Iterator for StateSparseTransitionIter<'a> { + type Item = (alphabet::Unit, alphabet::Unit, StateID); + + fn next(&mut self) -> Option<(alphabet::Unit, alphabet::Unit, StateID)> { + while let Some((unit, next)) = self.dense.next() { + let (prev_start, prev_end, prev_next) = match self.cur { + Some(t) => t, + None => { + self.cur = Some((unit, unit, next)); + continue; + } + }; + if prev_next == next && !unit.is_eoi() { + self.cur = Some((prev_start, unit, prev_next)); + } else { + self.cur = Some((unit, unit, next)); + if prev_next != DEAD { + return Some((prev_start, prev_end, prev_next)); + } + } + } + if let Some((start, end, next)) = self.cur.take() { + if next != DEAD { + return Some((start, end, next)); + } + } + None + } +} + +/// An error that occurred during the construction of a DFA. +/// +/// This error does not provide many introspection capabilities. There are +/// generally only two things you can do with it: +/// +/// * Obtain a human readable message via its `std::fmt::Display` impl. +/// * Access an underlying [`nfa::thompson::BuildError`](thompson::BuildError) +/// type from its `source` method via the `std::error::Error` trait. This error +/// only occurs when using convenience routines for building a DFA directly +/// from a pattern string. +/// +/// When the `std` feature is enabled, this implements the `std::error::Error` +/// trait. +#[cfg(feature = "dfa-build")] +#[derive(Clone, Debug)] +pub struct BuildError { + kind: BuildErrorKind, +} + +#[cfg(feature = "dfa-build")] +impl BuildError { + /// Returns true if and only if this error corresponds to an error with DFA + /// construction that occurred because of exceeding a size limit. + /// + /// While this can occur when size limits like [`Config::dfa_size_limit`] + /// or [`Config::determinize_size_limit`] are exceeded, this can also occur + /// when the number of states or patterns exceeds a hard-coded maximum. + /// (Where these maximums are derived based on the values representable by + /// [`StateID`] and [`PatternID`].) + /// + /// This predicate is useful in contexts where you want to distinguish + /// between errors related to something provided by an end user (for + /// example, an invalid regex pattern) and errors related to configured + /// heuristics. For example, building a DFA might be an optimization that + /// you want to skip if construction fails because of an exceeded size + /// limit, but where you want to bubble up an error if it fails for some + /// other reason. + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// # if !cfg!(target_pointer_width = "64") { return Ok(()); } // see #1039 + /// use regex_automata::{dfa::{dense, Automaton}, Input}; + /// + /// let err = dense::Builder::new() + /// .configure(dense::Config::new() + /// .determinize_size_limit(Some(100_000)) + /// ) + /// .build(r"\w{20}") + /// .unwrap_err(); + /// // This error occurs because a size limit was exceeded. + /// // But things are otherwise valid. + /// assert!(err.is_size_limit_exceeded()); + /// + /// let err = dense::Builder::new() + /// .build(r"\bxyz\b") + /// .unwrap_err(); + /// // This error occurs because a Unicode word boundary + /// // was used without enabling heuristic support for it. + /// // So... not related to size limits. + /// assert!(!err.is_size_limit_exceeded()); + /// + /// let err = dense::Builder::new() + /// .build(r"(xyz") + /// .unwrap_err(); + /// // This error occurs because the pattern is invalid. + /// // So... not related to size limits. + /// assert!(!err.is_size_limit_exceeded()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_size_limit_exceeded(&self) -> bool { + use self::BuildErrorKind::*; + + match self.kind { + NFA(_) | Unsupported(_) => false, + TooManyStates + | TooManyStartStates + | TooManyMatchPatternIDs + | DFAExceededSizeLimit { .. } + | DeterminizeExceededSizeLimit { .. } => true, + } + } +} + +/// The kind of error that occurred during the construction of a DFA. +/// +/// Note that this error is non-exhaustive. Adding new variants is not +/// considered a breaking change. +#[cfg(feature = "dfa-build")] +#[derive(Clone, Debug)] +enum BuildErrorKind { + /// An error that occurred while constructing an NFA as a precursor step + /// before a DFA is compiled. + NFA(thompson::BuildError), + /// An error that occurred because an unsupported regex feature was used. + /// The message string describes which unsupported feature was used. + /// + /// The primary regex feature that is unsupported by DFAs is the Unicode + /// word boundary look-around assertion (`\b`). This can be worked around + /// by either using an ASCII word boundary (`(?-u:\b)`) or by enabling + /// Unicode word boundaries when building a DFA. + Unsupported(&'static str), + /// An error that occurs if too many states are produced while building a + /// DFA. + TooManyStates, + /// An error that occurs if too many start states are needed while building + /// a DFA. + /// + /// This is a kind of oddball error that occurs when building a DFA with + /// start states enabled for each pattern and enough patterns to cause + /// the table of start states to overflow `usize`. + TooManyStartStates, + /// This is another oddball error that can occur if there are too many + /// patterns spread out across too many match states. + TooManyMatchPatternIDs, + /// An error that occurs if the DFA got too big during determinization. + DFAExceededSizeLimit { limit: usize }, + /// An error that occurs if auxiliary storage (not the DFA) used during + /// determinization got too big. + DeterminizeExceededSizeLimit { limit: usize }, +} + +#[cfg(feature = "dfa-build")] +impl BuildError { + /// Return the kind of this error. + fn kind(&self) -> &BuildErrorKind { + &self.kind + } + + pub(crate) fn nfa(err: thompson::BuildError) -> BuildError { + BuildError { kind: BuildErrorKind::NFA(err) } + } + + pub(crate) fn unsupported_dfa_word_boundary_unicode() -> BuildError { + let msg = "cannot build DFAs for regexes with Unicode word \ + boundaries; switch to ASCII word boundaries, or \ + heuristically enable Unicode word boundaries or use a \ + different regex engine"; + BuildError { kind: BuildErrorKind::Unsupported(msg) } + } + + pub(crate) fn too_many_states() -> BuildError { + BuildError { kind: BuildErrorKind::TooManyStates } + } + + pub(crate) fn too_many_start_states() -> BuildError { + BuildError { kind: BuildErrorKind::TooManyStartStates } + } + + pub(crate) fn too_many_match_pattern_ids() -> BuildError { + BuildError { kind: BuildErrorKind::TooManyMatchPatternIDs } + } + + pub(crate) fn dfa_exceeded_size_limit(limit: usize) -> BuildError { + BuildError { kind: BuildErrorKind::DFAExceededSizeLimit { limit } } + } + + pub(crate) fn determinize_exceeded_size_limit(limit: usize) -> BuildError { + BuildError { + kind: BuildErrorKind::DeterminizeExceededSizeLimit { limit }, + } + } +} + +#[cfg(all(feature = "std", feature = "dfa-build"))] +impl std::error::Error for BuildError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self.kind() { + BuildErrorKind::NFA(ref err) => Some(err), + _ => None, + } + } +} + +#[cfg(feature = "dfa-build")] +impl core::fmt::Display for BuildError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.kind() { + BuildErrorKind::NFA(_) => write!(f, "error building NFA"), + BuildErrorKind::Unsupported(ref msg) => { + write!(f, "unsupported regex feature for DFAs: {msg}") + } + BuildErrorKind::TooManyStates => write!( + f, + "number of DFA states exceeds limit of {}", + StateID::LIMIT, + ), + BuildErrorKind::TooManyStartStates => { + let stride = Start::len(); + // The start table has `stride` entries for starting states for + // the entire DFA, and then `stride` entries for each pattern + // if start states for each pattern are enabled (which is the + // only way this error can occur). Thus, the total number of + // patterns that can fit in the table is `stride` less than + // what we can allocate. + let max = usize::try_from(core::isize::MAX).unwrap(); + let limit = (max - stride) / stride; + write!( + f, + "compiling DFA with start states exceeds pattern \ + pattern limit of {}", + limit, + ) + } + BuildErrorKind::TooManyMatchPatternIDs => write!( + f, + "compiling DFA with total patterns in all match states \ + exceeds limit of {}", + PatternID::LIMIT, + ), + BuildErrorKind::DFAExceededSizeLimit { limit } => write!( + f, + "DFA exceeded size limit of {limit:?} during determinization", + ), + BuildErrorKind::DeterminizeExceededSizeLimit { limit } => { + write!(f, "determinization exceeded size limit of {limit:?}") + } + } + } +} + +#[cfg(all(test, feature = "syntax", feature = "dfa-build"))] +mod tests { + use crate::{Input, MatchError}; + + use super::*; + + #[test] + fn errors_with_unicode_word_boundary() { + let pattern = r"\b"; + assert!(Builder::new().build(pattern).is_err()); + } + + #[test] + fn roundtrip_never_match() { + let dfa = DFA::never_match().unwrap(); + let (buf, _) = dfa.to_bytes_native_endian(); + let dfa: DFA<&[u32]> = DFA::from_bytes(&buf).unwrap().0; + + assert_eq!(None, dfa.try_search_fwd(&Input::new("foo12345")).unwrap()); + } + + #[test] + fn roundtrip_always_match() { + use crate::HalfMatch; + + let dfa = DFA::always_match().unwrap(); + let (buf, _) = dfa.to_bytes_native_endian(); + let dfa: DFA<&[u32]> = DFA::from_bytes(&buf).unwrap().0; + + assert_eq!( + Some(HalfMatch::must(0, 0)), + dfa.try_search_fwd(&Input::new("foo12345")).unwrap() + ); + } + + // See the analogous test in src/hybrid/dfa.rs. + #[test] + fn heuristic_unicode_reverse() { + let dfa = DFA::builder() + .configure(DFA::config().unicode_word_boundary(true)) + .thompson(thompson::Config::new().reverse(true)) + .build(r"\b[0-9]+\b") + .unwrap(); + + let input = Input::new("β123").range(2..); + let expected = MatchError::quit(0xB2, 1); + let got = dfa.try_search_rev(&input); + assert_eq!(Err(expected), got); + + let input = Input::new("123β").range(..3); + let expected = MatchError::quit(0xCE, 3); + let got = dfa.try_search_rev(&input); + assert_eq!(Err(expected), got); + } + + // This panics in `TransitionTable::validate` if the match states are not + // validated first. + // + // See: https://github.com/rust-lang/regex/pull/1295 + #[test] + fn regression_validation_order() { + let mut dfa = DFA::new("abc").unwrap(); + dfa.ms = MatchStates { + slices: vec![], + pattern_ids: vec![], + pattern_len: 1, + }; + let (buf, _) = dfa.to_bytes_native_endian(); + DFA::from_bytes(&buf).unwrap_err(); + } + + // A starting state can never be a match state, since all matches are + // delayed by one byte. The search routines rely on this and assert it, + // so `from_bytes` must reject a serialized DFA whose start table points + // at a match state. The sparse DFA already rejected this; the dense DFA + // did not, so searching with such a DFA tripped the assertion in + // `dfa::search::init_fwd`. + #[test] + fn regression_start_state_not_match() { + let mut dfa = DFA::new("abc").unwrap(); + let min_match = dfa.special.min_match; + for id in dfa.st.table_mut() { + *id = min_match; + } + let (buf, _) = dfa.to_bytes_native_endian(); + DFA::from_bytes(&buf).unwrap_err(); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/determinize.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/determinize.rs new file mode 100644 index 0000000..d53815c --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/determinize.rs @@ -0,0 +1,599 @@ +use alloc::{collections::BTreeMap, vec::Vec}; + +use crate::{ + dfa::{ + dense::{self, BuildError}, + DEAD, + }, + nfa::thompson, + util::{ + self, + alphabet::{self, ByteSet}, + determinize::{State, StateBuilderEmpty, StateBuilderNFA}, + primitives::{PatternID, StateID}, + search::{Anchored, MatchKind}, + sparse_set::SparseSets, + start::Start, + }, +}; + +/// A builder for configuring and running a DFA determinizer. +#[derive(Clone, Debug)] +pub(crate) struct Config { + match_kind: MatchKind, + quit: ByteSet, + dfa_size_limit: Option<usize>, + determinize_size_limit: Option<usize>, +} + +impl Config { + /// Create a new default config for a determinizer. The determinizer may be + /// configured before calling `run`. + pub fn new() -> Config { + Config { + match_kind: MatchKind::LeftmostFirst, + quit: ByteSet::empty(), + dfa_size_limit: None, + determinize_size_limit: None, + } + } + + /// Run determinization on the given NFA and write the resulting DFA into + /// the one given. The DFA given should be initialized but otherwise empty. + /// "Initialized" means that it is setup to handle the NFA's byte classes, + /// number of patterns and whether to build start states for each pattern. + pub fn run( + &self, + nfa: &thompson::NFA, + dfa: &mut dense::OwnedDFA, + ) -> Result<(), BuildError> { + let dead = State::dead(); + let quit = State::dead(); + let mut cache = StateMap::default(); + // We only insert the dead state here since its representation is + // identical to the quit state. And we never want anything pointing + // to the quit state other than specific transitions derived from the + // determinizer's configured "quit" bytes. + // + // We do put the quit state into 'builder_states' below. This ensures + // that a proper DFA state ID is allocated for it, and that no other + // DFA state uses the "location after the DEAD state." That is, it + // is assumed that the quit state is always the state immediately + // following the DEAD state. + cache.insert(dead.clone(), DEAD); + + let runner = Runner { + config: self.clone(), + nfa, + dfa, + builder_states: alloc::vec![dead, quit], + cache, + memory_usage_state: 0, + sparses: SparseSets::new(nfa.states().len()), + stack: alloc::vec![], + scratch_state_builder: StateBuilderEmpty::new(), + }; + runner.run() + } + + /// The match semantics to use for determinization. + /// + /// MatchKind::All corresponds to the standard textbook construction. + /// All possible match states are represented in the DFA. + /// MatchKind::LeftmostFirst permits greediness and otherwise tries to + /// simulate the match semantics of backtracking regex engines. Namely, + /// only a subset of match states are built, and dead states are used to + /// stop searches with an unanchored prefix. + /// + /// The default is MatchKind::LeftmostFirst. + pub fn match_kind(&mut self, kind: MatchKind) -> &mut Config { + self.match_kind = kind; + self + } + + /// The set of bytes to use that will cause the DFA to enter a quit state, + /// stop searching and return an error. By default, this is empty. + pub fn quit(&mut self, set: ByteSet) -> &mut Config { + self.quit = set; + self + } + + /// The limit, in bytes of the heap, that the DFA is permitted to use. This + /// does not include the auxiliary heap storage used by determinization. + pub fn dfa_size_limit(&mut self, bytes: Option<usize>) -> &mut Config { + self.dfa_size_limit = bytes; + self + } + + /// The limit, in bytes of the heap, that determinization itself is allowed + /// to use. This does not include the size of the DFA being built. + pub fn determinize_size_limit( + &mut self, + bytes: Option<usize>, + ) -> &mut Config { + self.determinize_size_limit = bytes; + self + } +} + +/// The actual implementation of determinization that converts an NFA to a DFA +/// through powerset construction. +/// +/// This determinizer roughly follows the typical powerset construction, where +/// each DFA state is comprised of one or more NFA states. In the worst case, +/// there is one DFA state for every possible combination of NFA states. In +/// practice, this only happens in certain conditions, typically when there are +/// bounded repetitions. +/// +/// The main differences between this implementation and typical deteminization +/// are that this implementation delays matches by one state and hackily makes +/// look-around work. Comments below attempt to explain this. +/// +/// The lifetime variable `'a` refers to the lifetime of the NFA or DFA, +/// whichever is shorter. +#[derive(Debug)] +struct Runner<'a> { + /// The configuration used to initialize determinization. + config: Config, + /// The NFA we're converting into a DFA. + nfa: &'a thompson::NFA, + /// The DFA we're building. + dfa: &'a mut dense::OwnedDFA, + /// Each DFA state being built is defined as an *ordered* set of NFA + /// states, along with some meta facts about the ordered set of NFA states. + /// + /// This is never empty. The first state is always a dummy state such that + /// a state id == 0 corresponds to a dead state. The second state is always + /// the quit state. + /// + /// Why do we have states in both a `Vec` and in a cache map below? + /// Well, they serve two different roles based on access patterns. + /// `builder_states` is the canonical home of each state, and provides + /// constant random access by a DFA state's ID. The cache map below, on + /// the other hand, provides a quick way of searching for identical DFA + /// states by using the DFA state as a key in the map. Of course, we use + /// reference counting to avoid actually duplicating the state's data + /// itself. (Although this has never been benchmarked.) Note that the cache + /// map does not give us full minimization; it just lets us avoid some very + /// obvious redundant states. + /// + /// Note that the index into this Vec isn't quite the DFA's state ID. + /// Rather, it's just an index. To get the state ID, you have to multiply + /// it by the DFA's stride. That's done by self.dfa.from_index. And the + /// inverse is self.dfa.to_index. + /// + /// Moreover, DFA states don't usually retain the IDs assigned to them + /// by their position in this Vec. After determinization completes, + /// states are shuffled around to support other optimizations. See the + /// sibling 'special' module for more details on that. (The reason for + /// mentioning this is that if you print out the DFA for debugging during + /// determinization, and then print out the final DFA after it is fully + /// built, then the state IDs likely won't match up.) + builder_states: Vec<State>, + /// A cache of DFA states that already exist and can be easily looked up + /// via ordered sets of NFA states. + /// + /// See `builder_states` docs for why we store states in two different + /// ways. + cache: StateMap, + /// The memory usage, in bytes, used by builder_states and cache. We track + /// this as new states are added since states use a variable amount of + /// heap. Tracking this as we add states makes it possible to compute the + /// total amount of memory used by the determinizer in constant time. + memory_usage_state: usize, + /// A pair of sparse sets for tracking ordered sets of NFA state IDs. + /// These are reused throughout determinization. A bounded sparse set + /// gives us constant time insertion, membership testing and clearing. + sparses: SparseSets, + /// Scratch space for a stack of NFA states to visit, for depth first + /// visiting without recursion. + stack: Vec<StateID>, + /// Scratch space for storing an ordered sequence of NFA states, for + /// amortizing allocation. This is principally useful for when we avoid + /// adding a new DFA state since it already exists. In order to detect this + /// case though, we still need an ordered set of NFA state IDs. So we use + /// this space to stage that ordered set before we know whether we need to + /// create a new DFA state or not. + scratch_state_builder: StateBuilderEmpty, +} + +/// A map from states to state identifiers. When using std, we use a standard +/// hashmap, since it's a bit faster for this use case. (Other maps, like +/// one's based on FNV, have not yet been benchmarked.) +/// +/// The main purpose of this map is to reuse states where possible. This won't +/// fully minimize the DFA, but it works well in a lot of cases. +#[cfg(feature = "std")] +type StateMap = std::collections::HashMap<State, StateID>; +#[cfg(not(feature = "std"))] +type StateMap = BTreeMap<State, StateID>; + +impl<'a> Runner<'a> { + /// Build the DFA. If there was a problem constructing the DFA (e.g., if + /// the chosen state identifier representation is too small), then an error + /// is returned. + fn run(mut self) -> Result<(), BuildError> { + if self.nfa.look_set_any().contains_word_unicode() + && !self.config.quit.contains_range(0x80, 0xFF) + { + return Err(BuildError::unsupported_dfa_word_boundary_unicode()); + } + + // A sequence of "representative" bytes drawn from each equivalence + // class. These representative bytes are fed to the NFA to compute + // state transitions. This allows us to avoid re-computing state + // transitions for bytes that are guaranteed to produce identical + // results. Since computing the representatives needs to do a little + // work, we do it once here because we'll be iterating over them a lot. + let representatives: Vec<alphabet::Unit> = + self.dfa.byte_classes().representatives(..).collect(); + // The set of all DFA state IDs that still need to have their + // transitions set. We start by seeding this with all starting states. + let mut uncompiled = alloc::vec![]; + self.add_all_starts(&mut uncompiled)?; + while let Some(dfa_id) = uncompiled.pop() { + for &unit in &representatives { + if unit.as_u8().map_or(false, |b| self.config.quit.contains(b)) + { + continue; + } + // In many cases, the state we transition to has already been + // computed. 'cached_state' will do the minimal amount of work + // to check this, and if it exists, immediately return an + // already existing state ID. + let (next_dfa_id, is_new) = self.cached_state(dfa_id, unit)?; + self.dfa.set_transition(dfa_id, unit, next_dfa_id); + // If the state ID we got back is newly created, then we need + // to compile it, so add it to our uncompiled frontier. + if is_new { + uncompiled.push(next_dfa_id); + } + } + } + debug!( + "determinization complete, memory usage: {}, \ + dense DFA size: {}, \ + is reverse? {}", + self.memory_usage(), + self.dfa.memory_usage(), + self.nfa.is_reverse(), + ); + + // A map from DFA state ID to one or more NFA match IDs. Each NFA match + // ID corresponds to a distinct regex pattern that matches in the state + // corresponding to the key. + let mut matches: BTreeMap<StateID, Vec<PatternID>> = BTreeMap::new(); + self.cache.clear(); + #[cfg(feature = "logging")] + let mut total_pat_len = 0; + for (i, state) in self.builder_states.into_iter().enumerate() { + if let Some(pat_ids) = state.match_pattern_ids() { + let id = self.dfa.to_state_id(i); + log! { + total_pat_len += pat_ids.len(); + } + matches.insert(id, pat_ids); + } + } + log! { + use core::mem::size_of; + let per_elem = size_of::<StateID>() + size_of::<Vec<PatternID>>(); + let pats = total_pat_len * size_of::<PatternID>(); + let mem = (matches.len() * per_elem) + pats; + log::debug!("matches map built, memory usage: {mem}"); + } + // At this point, we shuffle the "special" states in the final DFA. + // This permits a DFA's match loop to detect a match condition (among + // other things) by merely inspecting the current state's identifier, + // and avoids the need for any additional auxiliary storage. + self.dfa.shuffle(matches)?; + Ok(()) + } + + /// Return the identifier for the next DFA state given an existing DFA + /// state and an input byte. If the next DFA state already exists, then + /// return its identifier from the cache. Otherwise, build the state, cache + /// it and return its identifier. + /// + /// This routine returns a boolean indicating whether a new state was + /// built. If a new state is built, then the caller needs to add it to its + /// frontier of uncompiled DFA states to compute transitions for. + fn cached_state( + &mut self, + dfa_id: StateID, + unit: alphabet::Unit, + ) -> Result<(StateID, bool), BuildError> { + // Compute the set of all reachable NFA states, including epsilons. + let empty_builder = self.get_state_builder(); + let builder = util::determinize::next( + self.nfa, + self.config.match_kind, + &mut self.sparses, + &mut self.stack, + &self.builder_states[self.dfa.to_index(dfa_id)], + unit, + empty_builder, + ); + self.maybe_add_state(builder) + } + + /// Compute the set of DFA start states and add their identifiers in + /// 'dfa_state_ids' (no duplicates are added). + fn add_all_starts( + &mut self, + dfa_state_ids: &mut Vec<StateID>, + ) -> Result<(), BuildError> { + // These should be the first states added. + assert!(dfa_state_ids.is_empty()); + // We only want to add (un)anchored starting states that is consistent + // with our DFA's configuration. Unconditionally adding both (although + // it is the default) can make DFAs quite a bit bigger. + if self.dfa.start_kind().has_unanchored() { + self.add_start_group(Anchored::No, dfa_state_ids)?; + } + if self.dfa.start_kind().has_anchored() { + self.add_start_group(Anchored::Yes, dfa_state_ids)?; + } + // I previously has an 'assert' here checking that either + // 'dfa_state_ids' was non-empty, or the NFA had zero patterns. But it + // turns out this isn't always true. For example, the NFA might have + // one or more patterns but where all such patterns are just 'fail' + // states. These will ultimately just compile down to DFA dead states, + // and since the dead state was added earlier, no new DFA states are + // added. And thus, it is valid and okay for 'dfa_state_ids' to be + // empty even if there are a non-zero number of patterns in the NFA. + + // We only need to compute anchored start states for each pattern if it + // was requested to do so. + if self.dfa.starts_for_each_pattern() { + for pid in self.nfa.patterns() { + self.add_start_group(Anchored::Pattern(pid), dfa_state_ids)?; + } + } + Ok(()) + } + + /// Add a group of start states for the given match pattern ID. Any new + /// DFA states added are pushed on to 'dfa_state_ids'. (No duplicates are + /// pushed.) + /// + /// When pattern_id is None, then this will compile a group of unanchored + /// start states (if the DFA is unanchored). When the pattern_id is + /// present, then this will compile a group of anchored start states that + /// only match the given pattern. + /// + /// This panics if `anchored` corresponds to an invalid pattern ID. + fn add_start_group( + &mut self, + anchored: Anchored, + dfa_state_ids: &mut Vec<StateID>, + ) -> Result<(), BuildError> { + let nfa_start = match anchored { + Anchored::No => self.nfa.start_unanchored(), + Anchored::Yes => self.nfa.start_anchored(), + Anchored::Pattern(pid) => { + self.nfa.start_pattern(pid).expect("valid pattern ID") + } + }; + + // When compiling start states, we're careful not to build additional + // states that aren't necessary. For example, if the NFA has no word + // boundary assertion, then there's no reason to have distinct start + // states for 'NonWordByte' and 'WordByte' starting configurations. + // Instead, the 'WordByte' starting configuration can just point + // directly to the start state for the 'NonWordByte' config. + // + // Note though that we only need to care about assertions in the prefix + // of an NFA since this only concerns the starting states. (Actually, + // the most precisely thing we could do it is look at the prefix + // assertions of each pattern when 'anchored == Anchored::Pattern', + // and then only compile extra states if the prefix is non-empty.) But + // we settle for simplicity here instead of absolute minimalism. It is + // somewhat rare, after all, for multiple patterns in the same regex to + // have different prefix look-arounds. + + let (id, is_new) = + self.add_one_start(nfa_start, Start::NonWordByte)?; + self.dfa.set_start_state(anchored, Start::NonWordByte, id); + if is_new { + dfa_state_ids.push(id); + } + + if !self.nfa.look_set_prefix_any().contains_word() { + self.dfa.set_start_state(anchored, Start::WordByte, id); + } else { + let (id, is_new) = + self.add_one_start(nfa_start, Start::WordByte)?; + self.dfa.set_start_state(anchored, Start::WordByte, id); + if is_new { + dfa_state_ids.push(id); + } + } + if !self.nfa.look_set_prefix_any().contains_anchor() { + self.dfa.set_start_state(anchored, Start::Text, id); + self.dfa.set_start_state(anchored, Start::LineLF, id); + self.dfa.set_start_state(anchored, Start::LineCR, id); + self.dfa.set_start_state( + anchored, + Start::CustomLineTerminator, + id, + ); + } else { + let (id, is_new) = self.add_one_start(nfa_start, Start::Text)?; + self.dfa.set_start_state(anchored, Start::Text, id); + if is_new { + dfa_state_ids.push(id); + } + + let (id, is_new) = self.add_one_start(nfa_start, Start::LineLF)?; + self.dfa.set_start_state(anchored, Start::LineLF, id); + if is_new { + dfa_state_ids.push(id); + } + + let (id, is_new) = self.add_one_start(nfa_start, Start::LineCR)?; + self.dfa.set_start_state(anchored, Start::LineCR, id); + if is_new { + dfa_state_ids.push(id); + } + + let (id, is_new) = + self.add_one_start(nfa_start, Start::CustomLineTerminator)?; + self.dfa.set_start_state( + anchored, + Start::CustomLineTerminator, + id, + ); + if is_new { + dfa_state_ids.push(id); + } + } + + Ok(()) + } + + /// Add a new DFA start state corresponding to the given starting NFA + /// state, and the starting search configuration. (The starting search + /// configuration essentially tells us which look-behind assertions are + /// true for this particular state.) + /// + /// The boolean returned indicates whether the state ID returned is a newly + /// created state, or a previously cached state. + fn add_one_start( + &mut self, + nfa_start: StateID, + start: Start, + ) -> Result<(StateID, bool), BuildError> { + // Compute the look-behind assertions that are true in this starting + // configuration, and the determine the epsilon closure. While + // computing the epsilon closure, we only follow conditional epsilon + // transitions that satisfy the look-behind assertions in 'look_have'. + let mut builder_matches = self.get_state_builder().into_matches(); + util::determinize::set_lookbehind_from_start( + self.nfa, + &start, + &mut builder_matches, + ); + self.sparses.set1.clear(); + util::determinize::epsilon_closure( + self.nfa, + nfa_start, + builder_matches.look_have(), + &mut self.stack, + &mut self.sparses.set1, + ); + let mut builder = builder_matches.into_nfa(); + util::determinize::add_nfa_states( + &self.nfa, + &self.sparses.set1, + &mut builder, + ); + self.maybe_add_state(builder) + } + + /// Adds the given state to the DFA being built depending on whether it + /// already exists in this determinizer's cache. + /// + /// If it does exist, then the memory used by 'state' is put back into the + /// determinizer and the previously created state's ID is returned. (Along + /// with 'false', indicating that no new state was added.) + /// + /// If it does not exist, then the state is added to the DFA being built + /// and a fresh ID is allocated (if ID allocation fails, then an error is + /// returned) and returned. (Along with 'true', indicating that a new state + /// was added.) + fn maybe_add_state( + &mut self, + builder: StateBuilderNFA, + ) -> Result<(StateID, bool), BuildError> { + if let Some(&cached_id) = self.cache.get(builder.as_bytes()) { + // Since we have a cached state, put the constructed state's + // memory back into our scratch space, so that it can be reused. + self.put_state_builder(builder); + return Ok((cached_id, false)); + } + self.add_state(builder).map(|sid| (sid, true)) + } + + /// Add the given state to the DFA and make it available in the cache. + /// + /// The state initially has no transitions. That is, it transitions to the + /// dead state for all possible inputs, and transitions to the quit state + /// for all quit bytes. + /// + /// If adding the state would exceed the maximum value for StateID, then an + /// error is returned. + fn add_state( + &mut self, + builder: StateBuilderNFA, + ) -> Result<StateID, BuildError> { + let id = self.dfa.add_empty_state()?; + if !self.config.quit.is_empty() { + for b in self.config.quit.iter() { + self.dfa.set_transition( + id, + alphabet::Unit::u8(b), + self.dfa.quit_id(), + ); + } + } + let state = builder.to_state(); + // States use reference counting internally, so we only need to count + // their memory usage once. + self.memory_usage_state += state.memory_usage(); + self.builder_states.push(state.clone()); + self.cache.insert(state, id); + self.put_state_builder(builder); + if let Some(limit) = self.config.dfa_size_limit { + if self.dfa.memory_usage() > limit { + return Err(BuildError::dfa_exceeded_size_limit(limit)); + } + } + if let Some(limit) = self.config.determinize_size_limit { + if self.memory_usage() > limit { + return Err(BuildError::determinize_exceeded_size_limit( + limit, + )); + } + } + Ok(id) + } + + /// Returns a state builder from this determinizer that might have existing + /// capacity. This helps avoid allocs in cases where a state is built that + /// turns out to already be cached. + /// + /// Callers must put the state builder back with 'put_state_builder', + /// otherwise the allocation reuse won't work. + fn get_state_builder(&mut self) -> StateBuilderEmpty { + core::mem::replace( + &mut self.scratch_state_builder, + StateBuilderEmpty::new(), + ) + } + + /// Puts the given state builder back into this determinizer for reuse. + /// + /// Note that building a 'State' from a builder always creates a new + /// alloc, so callers should always put the builder back. + fn put_state_builder(&mut self, builder: StateBuilderNFA) { + let _ = core::mem::replace( + &mut self.scratch_state_builder, + builder.clear(), + ); + } + + /// Return the memory usage, in bytes, of this determinizer at the current + /// point in time. This does not include memory used by the NFA or the + /// dense DFA itself. + fn memory_usage(&self) -> usize { + use core::mem::size_of; + + self.builder_states.len() * size_of::<State>() + // Maps likely use more memory than this, but it's probably close. + + self.cache.len() * (size_of::<State>() + size_of::<StateID>()) + + self.memory_usage_state + + self.stack.capacity() * size_of::<StateID>() + + self.scratch_state_builder.capacity() + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/minimize.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/minimize.rs new file mode 100644 index 0000000..fea925b --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/minimize.rs @@ -0,0 +1,463 @@ +use core::{cell::RefCell, fmt, mem}; + +use alloc::{collections::BTreeMap, rc::Rc, vec, vec::Vec}; + +use crate::{ + dfa::{automaton::Automaton, dense, DEAD}, + util::{ + alphabet, + primitives::{PatternID, StateID}, + }, +}; + +/// An implementation of Hopcroft's algorithm for minimizing DFAs. +/// +/// The algorithm implemented here is mostly taken from Wikipedia: +/// https://en.wikipedia.org/wiki/DFA_minimization#Hopcroft's_algorithm +/// +/// This code has had some light optimization attention paid to it, +/// particularly in the form of reducing allocation as much as possible. +/// However, it is still generally slow. Future optimization work should +/// probably focus on the bigger picture rather than micro-optimizations. For +/// example: +/// +/// 1. Figure out how to more intelligently create initial partitions. That is, +/// Hopcroft's algorithm starts by creating two partitions of DFA states +/// that are known to NOT be equivalent: match states and non-match states. +/// The algorithm proceeds by progressively refining these partitions into +/// smaller partitions. If we could start with more partitions, then we +/// could reduce the amount of work that Hopcroft's algorithm needs to do. +/// 2. For every partition that we visit, we find all incoming transitions to +/// every state in the partition for *every* element in the alphabet. (This +/// is why using byte classes can significantly decrease minimization times, +/// since byte classes shrink the alphabet.) This is quite costly and there +/// is perhaps some redundant work being performed depending on the specific +/// states in the set. For example, we might be able to only visit some +/// elements of the alphabet based on the transitions. +/// 3. Move parts of minimization into determinization. If minimization has +/// fewer states to deal with, then it should run faster. A prime example +/// of this might be large Unicode classes, which are generated in way that +/// can create a lot of redundant states. (Some work has been done on this +/// point during NFA compilation via the algorithm described in the +/// "Incremental Construction of MinimalAcyclic Finite-State Automata" +/// paper.) +pub(crate) struct Minimizer<'a> { + dfa: &'a mut dense::OwnedDFA, + in_transitions: Vec<Vec<Vec<StateID>>>, + partitions: Vec<StateSet>, + waiting: Vec<StateSet>, +} + +impl<'a> fmt::Debug for Minimizer<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Minimizer") + .field("dfa", &self.dfa) + .field("in_transitions", &self.in_transitions) + .field("partitions", &self.partitions) + .field("waiting", &self.waiting) + .finish() + } +} + +/// A set of states. A state set makes up a single partition in Hopcroft's +/// algorithm. +/// +/// It is represented by an ordered set of state identifiers. We use shared +/// ownership so that a single state set can be in both the set of partitions +/// and in the set of waiting sets simultaneously without an additional +/// allocation. Generally, once a state set is built, it becomes immutable. +/// +/// We use this representation because it avoids the overhead of more +/// traditional set data structures (HashSet/BTreeSet), and also because +/// computing intersection/subtraction on this representation is especially +/// fast. +#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord)] +struct StateSet { + ids: Rc<RefCell<Vec<StateID>>>, +} + +impl<'a> Minimizer<'a> { + pub fn new(dfa: &'a mut dense::OwnedDFA) -> Minimizer<'a> { + let in_transitions = Minimizer::incoming_transitions(dfa); + let partitions = Minimizer::initial_partitions(dfa); + let waiting = partitions.clone(); + Minimizer { dfa, in_transitions, partitions, waiting } + } + + pub fn run(mut self) { + let stride2 = self.dfa.stride2(); + let as_state_id = |index: usize| -> StateID { + StateID::new(index << stride2).unwrap() + }; + let as_index = |id: StateID| -> usize { id.as_usize() >> stride2 }; + + let mut incoming = StateSet::empty(); + let mut scratch1 = StateSet::empty(); + let mut scratch2 = StateSet::empty(); + let mut newparts = vec![]; + + // This loop is basically Hopcroft's algorithm. Everything else is just + // shuffling data around to fit our representation. + while let Some(set) = self.waiting.pop() { + for b in self.dfa.byte_classes().iter() { + self.find_incoming_to(b, &set, &mut incoming); + // If incoming is empty, then the intersection with any other + // set must also be empty. So 'newparts' just ends up being + // 'self.partitions'. So there's no need to go through the loop + // below. + // + // This actually turns out to be rather large optimization. On + // the order of making minimization 4-5x faster. It's likely + // that the vast majority of all states have very few incoming + // transitions. + if incoming.is_empty() { + continue; + } + + for p in 0..self.partitions.len() { + self.partitions[p].intersection(&incoming, &mut scratch1); + if scratch1.is_empty() { + newparts.push(self.partitions[p].clone()); + continue; + } + + self.partitions[p].subtract(&incoming, &mut scratch2); + if scratch2.is_empty() { + newparts.push(self.partitions[p].clone()); + continue; + } + + let (x, y) = + (scratch1.deep_clone(), scratch2.deep_clone()); + newparts.push(x.clone()); + newparts.push(y.clone()); + match self.find_waiting(&self.partitions[p]) { + Some(i) => { + self.waiting[i] = x; + self.waiting.push(y); + } + None => { + if x.len() <= y.len() { + self.waiting.push(x); + } else { + self.waiting.push(y); + } + } + } + } + newparts = mem::replace(&mut self.partitions, newparts); + newparts.clear(); + } + } + + // At this point, we now have a minimal partitioning of states, where + // each partition is an equivalence class of DFA states. Now we need to + // use this partitioning to update the DFA to only contain one state for + // each partition. + + // Create a map from DFA state ID to the representative ID of the + // equivalence class to which it belongs. The representative ID of an + // equivalence class of states is the minimum ID in that class. + let mut state_to_part = vec![DEAD; self.dfa.state_len()]; + for p in &self.partitions { + p.iter(|id| state_to_part[as_index(id)] = p.min()); + } + + // Generate a new contiguous sequence of IDs for minimal states, and + // create a map from equivalence IDs to the new IDs. Thus, the new + // minimal ID of *any* state in the unminimized DFA can be obtained + // with minimals_ids[state_to_part[old_id]]. + let mut minimal_ids = vec![DEAD; self.dfa.state_len()]; + let mut new_index = 0; + for state in self.dfa.states() { + if state_to_part[as_index(state.id())] == state.id() { + minimal_ids[as_index(state.id())] = as_state_id(new_index); + new_index += 1; + } + } + // The total number of states in the minimal DFA. + let minimal_count = new_index; + // Convenience function for remapping state IDs. This takes an old ID, + // looks up its Hopcroft partition and then maps that to the new ID + // range. + let remap = |old| minimal_ids[as_index(state_to_part[as_index(old)])]; + + // Re-map this DFA in place such that the only states remaining + // correspond to the representative states of every equivalence class. + for id in (0..self.dfa.state_len()).map(as_state_id) { + // If this state isn't a representative for an equivalence class, + // then we skip it since it won't appear in the minimal DFA. + if state_to_part[as_index(id)] != id { + continue; + } + self.dfa.remap_state(id, remap); + self.dfa.swap_states(id, minimal_ids[as_index(id)]); + } + // Trim off all unused states from the pre-minimized DFA. This + // represents all states that were merged into a non-singleton + // equivalence class of states, and appeared after the first state + // in each such class. (Because the state with the smallest ID in each + // equivalence class is its representative ID.) + self.dfa.truncate_states(minimal_count); + + // Update the new start states, which is now just the minimal ID of + // whatever state the old start state was collapsed into. Also, we + // collect everything before-hand to work around the borrow checker. + // We're already allocating so much that this is probably fine. If this + // turns out to be costly, then I guess add a `starts_mut` iterator. + let starts: Vec<_> = self.dfa.starts().collect(); + for (old_start_id, anchored, start_type) in starts { + self.dfa.set_start_state( + anchored, + start_type, + remap(old_start_id), + ); + } + + // Update the match state pattern ID list for multi-regexes. All we + // need to do is remap the match state IDs. The pattern ID lists are + // always the same as they were since match states with distinct + // pattern ID lists are always considered distinct states. + let mut pmap = BTreeMap::new(); + for (match_id, pattern_ids) in self.dfa.pattern_map() { + let new_id = remap(match_id); + pmap.insert(new_id, pattern_ids); + } + // This unwrap is OK because minimization never increases the number of + // match states or patterns in those match states. Since minimization + // runs after the pattern map has already been set at least once, we + // know that our match states cannot error. + self.dfa.set_pattern_map(&pmap).unwrap(); + + // In order to update the ID of the maximum match state, we need to + // find the maximum ID among all of the match states in the minimized + // DFA. This is not necessarily the new ID of the unminimized maximum + // match state, since that could have been collapsed with a much + // earlier match state. Therefore, to find the new max match state, + // we iterate over all previous match states, find their corresponding + // new minimal ID, and take the maximum of those. + let old = self.dfa.special().clone(); + let new = self.dfa.special_mut(); + // ... but only remap if we had match states. + if old.matches() { + new.min_match = StateID::MAX; + new.max_match = StateID::ZERO; + for i in as_index(old.min_match)..=as_index(old.max_match) { + let new_id = remap(as_state_id(i)); + if new_id < new.min_match { + new.min_match = new_id; + } + if new_id > new.max_match { + new.max_match = new_id; + } + } + } + // ... same, but for start states. + if old.starts() { + new.min_start = StateID::MAX; + new.max_start = StateID::ZERO; + for i in as_index(old.min_start)..=as_index(old.max_start) { + let new_id = remap(as_state_id(i)); + if new_id == DEAD { + continue; + } + if new_id < new.min_start { + new.min_start = new_id; + } + if new_id > new.max_start { + new.max_start = new_id; + } + } + if new.max_start == DEAD { + new.min_start = DEAD; + } + } + new.quit_id = remap(new.quit_id); + new.set_max(); + } + + fn find_waiting(&self, set: &StateSet) -> Option<usize> { + self.waiting.iter().position(|s| s == set) + } + + fn find_incoming_to( + &self, + b: alphabet::Unit, + set: &StateSet, + incoming: &mut StateSet, + ) { + incoming.clear(); + set.iter(|id| { + for &inid in + &self.in_transitions[self.dfa.to_index(id)][b.as_usize()] + { + incoming.add(inid); + } + }); + incoming.canonicalize(); + } + + fn initial_partitions(dfa: &dense::OwnedDFA) -> Vec<StateSet> { + // For match states, we know that two match states with different + // pattern ID lists will *always* be distinct, so we can partition them + // initially based on that. + let mut matching: BTreeMap<Vec<PatternID>, StateSet> = BTreeMap::new(); + let mut is_quit = StateSet::empty(); + let mut no_match = StateSet::empty(); + for state in dfa.states() { + if dfa.is_match_state(state.id()) { + let mut pids = vec![]; + for i in 0..dfa.match_len(state.id()) { + pids.push(dfa.match_pattern(state.id(), i)); + } + matching + .entry(pids) + .or_insert(StateSet::empty()) + .add(state.id()); + } else if dfa.is_quit_state(state.id()) { + is_quit.add(state.id()); + } else { + no_match.add(state.id()); + } + } + + let mut sets: Vec<StateSet> = + matching.into_iter().map(|(_, set)| set).collect(); + sets.push(no_match); + sets.push(is_quit); + sets + } + + fn incoming_transitions(dfa: &dense::OwnedDFA) -> Vec<Vec<Vec<StateID>>> { + let mut incoming = vec![]; + for _ in dfa.states() { + incoming.push(vec![vec![]; dfa.alphabet_len()]); + } + for state in dfa.states() { + for (b, next) in state.transitions() { + incoming[dfa.to_index(next)][b.as_usize()].push(state.id()); + } + } + incoming + } +} + +impl StateSet { + fn empty() -> StateSet { + StateSet { ids: Rc::new(RefCell::new(vec![])) } + } + + fn add(&mut self, id: StateID) { + self.ids.borrow_mut().push(id); + } + + fn min(&self) -> StateID { + self.ids.borrow()[0] + } + + fn canonicalize(&mut self) { + self.ids.borrow_mut().sort(); + self.ids.borrow_mut().dedup(); + } + + fn clear(&mut self) { + self.ids.borrow_mut().clear(); + } + + fn len(&self) -> usize { + self.ids.borrow().len() + } + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn deep_clone(&self) -> StateSet { + let ids = self.ids.borrow().iter().cloned().collect(); + StateSet { ids: Rc::new(RefCell::new(ids)) } + } + + fn iter<F: FnMut(StateID)>(&self, mut f: F) { + for &id in self.ids.borrow().iter() { + f(id); + } + } + + fn intersection(&self, other: &StateSet, dest: &mut StateSet) { + dest.clear(); + if self.is_empty() || other.is_empty() { + return; + } + + let (seta, setb) = (self.ids.borrow(), other.ids.borrow()); + let (mut ita, mut itb) = (seta.iter().cloned(), setb.iter().cloned()); + let (mut a, mut b) = (ita.next().unwrap(), itb.next().unwrap()); + loop { + if a == b { + dest.add(a); + a = match ita.next() { + None => break, + Some(a) => a, + }; + b = match itb.next() { + None => break, + Some(b) => b, + }; + } else if a < b { + a = match ita.next() { + None => break, + Some(a) => a, + }; + } else { + b = match itb.next() { + None => break, + Some(b) => b, + }; + } + } + } + + fn subtract(&self, other: &StateSet, dest: &mut StateSet) { + dest.clear(); + if self.is_empty() || other.is_empty() { + self.iter(|s| dest.add(s)); + return; + } + + let (seta, setb) = (self.ids.borrow(), other.ids.borrow()); + let (mut ita, mut itb) = (seta.iter().cloned(), setb.iter().cloned()); + let (mut a, mut b) = (ita.next().unwrap(), itb.next().unwrap()); + loop { + if a == b { + a = match ita.next() { + None => break, + Some(a) => a, + }; + b = match itb.next() { + None => { + dest.add(a); + break; + } + Some(b) => b, + }; + } else if a < b { + dest.add(a); + a = match ita.next() { + None => break, + Some(a) => a, + }; + } else { + b = match itb.next() { + None => { + dest.add(a); + break; + } + Some(b) => b, + }; + } + } + for a in ita { + dest.add(a); + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/mod.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/mod.rs new file mode 100644 index 0000000..ff718cc --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/mod.rs @@ -0,0 +1,360 @@ +/*! +A module for building and searching with deterministic finite automata (DFAs). + +Like other modules in this crate, DFAs support a rich regex syntax with Unicode +features. DFAs also have extensive options for configuring the best space vs +time trade off for your use case and provides support for cheap deserialization +of automata for use in `no_std` environments. + +If you're looking for lazy DFAs that build themselves incrementally during +search, then please see the top-level [`hybrid` module](crate::hybrid). + +# Overview + +This section gives a brief overview of the primary types in this module: + +* A [`regex::Regex`] provides a way to search for matches of a regular +expression using DFAs. This includes iterating over matches with both the start +and end positions of each match. +* A [`dense::DFA`] provides low level access to a DFA that uses a dense +representation (uses lots of space, but fast searching). +* A [`sparse::DFA`] provides the same API as a `dense::DFA`, but uses a sparse +representation (uses less space, but slower searching). +* An [`Automaton`] trait that defines an interface that both dense and sparse +DFAs implement. (A `regex::Regex` is generic over this trait.) +* Both dense DFAs and sparse DFAs support serialization to raw bytes (e.g., +[`dense::DFA::to_bytes_little_endian`]) and cheap deserialization (e.g., +[`dense::DFA::from_bytes`]). + +There is also a [`onepass`] module that provides a [one-pass +DFA](onepass::DFA). The unique advantage of this DFA is that, for the class +of regexes it can be built with, it supports reporting the spans of matching +capturing groups. It is the only DFA in this crate capable of such a thing. + +# Example: basic regex searching + +This example shows how to compile a regex using the default configuration +and then use it to find matches in a byte string: + +``` +use regex_automata::{Match, dfa::regex::Regex}; + +let re = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")?; +let text = b"2018-12-24 2016-10-08"; +let matches: Vec<Match> = re.find_iter(text).collect(); +assert_eq!(matches, vec![ + Match::must(0, 0..10), + Match::must(0, 11..21), +]); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +# Example: searching with regex sets + +The DFAs in this module all fully support searching with multiple regexes +simultaneously. You can use this support with standard leftmost-first style +searching to find non-overlapping matches: + +``` +# if cfg!(miri) { return Ok(()); } // miri takes too long +use regex_automata::{Match, dfa::regex::Regex}; + +let re = Regex::new_many(&[r"\w+", r"\S+"])?; +let text = b"@foo bar"; +let matches: Vec<Match> = re.find_iter(text).collect(); +assert_eq!(matches, vec![ + Match::must(1, 0..4), + Match::must(0, 5..8), +]); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +# Example: use sparse DFAs + +By default, compiling a regex will use dense DFAs internally. This uses more +memory, but executes searches more quickly. If you can abide slower searches +(somewhere around 3-5x), then sparse DFAs might make more sense since they can +use significantly less space. + +Using sparse DFAs is as easy as using `Regex::new_sparse` instead of +`Regex::new`: + +``` +use regex_automata::{Match, dfa::regex::Regex}; + +let re = Regex::new_sparse(r"[0-9]{4}-[0-9]{2}-[0-9]{2}").unwrap(); +let text = b"2018-12-24 2016-10-08"; +let matches: Vec<Match> = re.find_iter(text).collect(); +assert_eq!(matches, vec![ + Match::must(0, 0..10), + Match::must(0, 11..21), +]); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +If you already have dense DFAs for some reason, they can be converted to sparse +DFAs and used to build a new `Regex`. For example: + +``` +use regex_automata::{Match, dfa::regex::Regex}; + +let dense_re = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}").unwrap(); +let sparse_re = Regex::builder().build_from_dfas( + dense_re.forward().to_sparse()?, + dense_re.reverse().to_sparse()?, +); +let text = b"2018-12-24 2016-10-08"; +let matches: Vec<Match> = sparse_re.find_iter(text).collect(); +assert_eq!(matches, vec![ + Match::must(0, 0..10), + Match::must(0, 11..21), +]); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +# Example: deserialize a DFA + +This shows how to first serialize a DFA into raw bytes, and then deserialize +those raw bytes back into a DFA. While this particular example is a +bit contrived, this same technique can be used in your program to +deserialize a DFA at start up time or by memory mapping a file. + +``` +use regex_automata::{Match, dfa::{dense, regex::Regex}}; + +let re1 = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}").unwrap(); +// serialize both the forward and reverse DFAs, see note below +let (fwd_bytes, fwd_pad) = re1.forward().to_bytes_native_endian(); +let (rev_bytes, rev_pad) = re1.reverse().to_bytes_native_endian(); +// now deserialize both---we need to specify the correct type! +let fwd: dense::DFA<&[u32]> = dense::DFA::from_bytes(&fwd_bytes[fwd_pad..])?.0; +let rev: dense::DFA<&[u32]> = dense::DFA::from_bytes(&rev_bytes[rev_pad..])?.0; +// finally, reconstruct our regex +let re2 = Regex::builder().build_from_dfas(fwd, rev); + +// we can use it like normal +let text = b"2018-12-24 2016-10-08"; +let matches: Vec<Match> = re2.find_iter(text).collect(); +assert_eq!(matches, vec![ + Match::must(0, 0..10), + Match::must(0, 11..21), +]); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +There are a few points worth noting here: + +* We need to extract the raw DFAs used by the regex and serialize those. You +can build the DFAs manually yourself using [`dense::Builder`], but using +the DFAs from a `Regex` guarantees that the DFAs are built correctly. (In +particular, a `Regex` constructs a reverse DFA for finding the starting +location of matches.) +* To convert the DFA to raw bytes, we use the `to_bytes_native_endian` method. +In practice, you'll want to use either [`dense::DFA::to_bytes_little_endian`] +or [`dense::DFA::to_bytes_big_endian`], depending on which platform you're +deserializing your DFA from. If you intend to deserialize on either platform, +then you'll need to serialize both and deserialize the right one depending on +your target's endianness. +* Safely deserializing a DFA requires verifying the raw bytes, particularly if +they are untrusted, since an invalid DFA could cause logical errors, panics +or even undefined behavior. This verification step requires visiting all of +the transitions in the DFA, which can be costly. If cheaper verification is +desired, then [`dense::DFA::from_bytes_unchecked`] is available that only does +verification that can be performed in constant time. However, one can only use +this routine if the caller can guarantee that the bytes provided encoded a +valid DFA. + +The same process can be achieved with sparse DFAs as well: + +``` +use regex_automata::{Match, dfa::{sparse, regex::Regex}}; + +let re1 = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}").unwrap(); +// serialize both +let fwd_bytes = re1.forward().to_sparse()?.to_bytes_native_endian(); +let rev_bytes = re1.reverse().to_sparse()?.to_bytes_native_endian(); +// now deserialize both---we need to specify the correct type! +let fwd: sparse::DFA<&[u8]> = sparse::DFA::from_bytes(&fwd_bytes)?.0; +let rev: sparse::DFA<&[u8]> = sparse::DFA::from_bytes(&rev_bytes)?.0; +// finally, reconstruct our regex +let re2 = Regex::builder().build_from_dfas(fwd, rev); + +// we can use it like normal +let text = b"2018-12-24 2016-10-08"; +let matches: Vec<Match> = re2.find_iter(text).collect(); +assert_eq!(matches, vec![ + Match::must(0, 0..10), + Match::must(0, 11..21), +]); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +Note that unlike dense DFAs, sparse DFAs have no alignment requirements. +Conversely, dense DFAs must be aligned to the same alignment as a +[`StateID`](crate::util::primitives::StateID). + +# Support for `no_std` and `alloc`-only + +This crate comes with `alloc` and `std` features that are enabled by default. +When the `alloc` or `std` features are enabled, the API of this module will +include the facilities necessary for compiling, serializing, deserializing +and searching with DFAs. When only the `alloc` feature is enabled, then +implementations of the `std::error::Error` trait are dropped, but everything +else generally remains the same. When both the `alloc` and `std` features are +disabled, the API of this module will shrink such that it only includes the +facilities necessary for deserializing and searching with DFAs. + +The intended workflow for `no_std` environments is thus as follows: + +* Write a program with the `alloc` or `std` features that compiles and +serializes a regular expression. You may need to serialize both little and big +endian versions of each DFA. (So that's 4 DFAs in total for each regex.) +* In your `no_std` environment, follow the examples above for deserializing +your previously serialized DFAs into regexes. You can then search with them as +you would any regex. + +Deserialization can happen anywhere. For example, with bytes embedded into a +binary or with a file memory mapped at runtime. + +The `regex-cli` command (found in the same repository as this crate) can be +used to serialize DFAs to files and generate Rust code to read them. + +# Syntax + +This module supports the same syntax as the `regex` crate, since they share the +same parser. You can find an exhaustive list of supported syntax in the +[documentation for the `regex` crate](https://docs.rs/regex/1/regex/#syntax). + +There are two things that are not supported by the DFAs in this module: + +* Capturing groups. The DFAs (and [`Regex`](regex::Regex)es built on top +of them) can only find the offsets of an entire match, but cannot resolve +the offsets of each capturing group. This is because DFAs do not have the +expressive power necessary. +* Unicode word boundaries. These present particularly difficult challenges for +DFA construction and would result in an explosion in the number of states. +One can enable [`dense::Config::unicode_word_boundary`] though, which provides +heuristic support for Unicode word boundaries that only works on ASCII text. +Otherwise, one can use `(?-u:\b)` for an ASCII word boundary, which will work +on any input. + +There are no plans to lift either of these limitations. + +Note that these restrictions are identical to the restrictions on lazy DFAs. + +# Differences with general purpose regexes + +The main goal of the [`regex`](https://docs.rs/regex) crate is to serve as a +general purpose regular expression engine. It aims to automatically balance low +compile times, fast search times and low memory usage, while also providing +a convenient API for users. In contrast, this module provides a lower level +regular expression interface based exclusively on DFAs that is a bit less +convenient while providing more explicit control over memory usage and search +times. + +Here are some specific negative differences: + +* **Compilation can take an exponential amount of time and space** in the size +of the regex pattern. While most patterns do not exhibit worst case exponential +time, such patterns do exist. For example, `[01]*1[01]{N}` will build a DFA +with approximately `2^(N+2)` states. For this reason, untrusted patterns should +not be compiled with this module. (In the future, the API may expose an option +to return an error if the DFA gets too big.) +* This module does not support sub-match extraction via capturing groups, which +can be achieved with the regex crate's "captures" API. +* While the regex crate doesn't necessarily sport fast compilation times, +the regexes in this module are almost universally slow to compile, especially +when they contain large Unicode character classes. For example, on my system, +compiling `\w{50}` takes about 1 second and almost 15MB of memory! (Compiling +a sparse regex takes about the same time but only uses about 1.2MB of +memory.) Conversely, compiling the same regex without Unicode support, e.g., +`(?-u)\w{50}`, takes under 1 millisecond and about 15KB of memory. For this +reason, you should only use Unicode character classes if you absolutely need +them! (They are enabled by default though.) +* This module does not support Unicode word boundaries. ASCII word boundaries +may be used though by disabling Unicode or selectively doing so in the syntax, +e.g., `(?-u:\b)`. There is also an option to +[heuristically enable Unicode word boundaries](crate::dfa::dense::Config::unicode_word_boundary), +where the corresponding DFA will give up if any non-ASCII byte is seen. +* As a lower level API, this module does not do literal optimizations +automatically. Although it does provide hooks in its API to make use of the +[`Prefilter`](crate::util::prefilter::Prefilter) trait. Missing literal +optimizations means that searches may run much slower than what you're +accustomed to, although, it does provide more predictable and consistent +performance. +* There is no `&str` API like in the regex crate. In this module, all APIs +operate on `&[u8]`. By default, match indices are +guaranteed to fall on UTF-8 boundaries, unless either of +[`syntax::Config::utf8`](crate::util::syntax::Config::utf8) or +[`thompson::Config::utf8`](crate::nfa::thompson::Config::utf8) are disabled. + +With some of the downsides out of the way, here are some positive differences: + +* Both dense and sparse DFAs can be serialized to raw bytes, and then cheaply +deserialized. Deserialization can be done in constant time with the unchecked +APIs, since searching can be performed directly on the raw serialized bytes of +a DFA. +* This module was specifically designed so that the searching phase of a +DFA has minimal runtime requirements, and can therefore be used in `no_std` +environments. While `no_std` environments cannot compile regexes, they can +deserialize pre-compiled regexes. +* Since this module builds DFAs ahead of time, it will generally out-perform +the `regex` crate on equivalent tasks. The performance difference is likely +not large. However, because of a complex set of optimizations in the regex +crate (like literal optimizations), an accurate performance comparison may be +difficult to do. +* Sparse DFAs provide a way to build a DFA ahead of time that sacrifices search +performance a small amount, but uses much less storage space. Potentially even +less than what the regex crate uses. +* This module exposes DFAs directly, such as [`dense::DFA`] and +[`sparse::DFA`], which enables one to do less work in some cases. For example, +if you only need the end of a match and not the start of a match, then you can +use a DFA directly without building a `Regex`, which always requires a second +DFA to find the start of a match. +* This module provides more control over memory usage. Aside from choosing +between dense and sparse DFAs, one can also choose a smaller state identifier +representation to use less space. Also, one can enable DFA minimization +via [`dense::Config::minimize`], but it can increase compilation times +dramatically. +*/ + +#[cfg(feature = "dfa-search")] +pub use crate::dfa::{ + automaton::{Automaton, OverlappingState, StartError}, + start::StartKind, +}; + +/// This is an alias for a state ID of zero. It has special significance +/// because it always corresponds to the first state in a DFA, and the first +/// state in a DFA is always "dead." That is, the dead state always has all +/// of its transitions set to itself. Moreover, the dead state is used as a +/// sentinel for various things. e.g., In search, reaching a dead state means +/// that the search must stop. +const DEAD: crate::util::primitives::StateID = + crate::util::primitives::StateID::ZERO; + +#[cfg(feature = "dfa-search")] +pub mod dense; +#[cfg(feature = "dfa-onepass")] +pub mod onepass; +#[cfg(feature = "dfa-search")] +pub mod regex; +#[cfg(feature = "dfa-search")] +pub mod sparse; + +#[cfg(feature = "dfa-search")] +pub(crate) mod accel; +#[cfg(feature = "dfa-search")] +mod automaton; +#[cfg(feature = "dfa-build")] +mod determinize; +#[cfg(feature = "dfa-build")] +mod minimize; +#[cfg(any(feature = "dfa-build", feature = "dfa-onepass"))] +mod remapper; +#[cfg(feature = "dfa-search")] +mod search; +#[cfg(feature = "dfa-search")] +mod special; +#[cfg(feature = "dfa-search")] +mod start; diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/onepass.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/onepass.rs new file mode 100644 index 0000000..5b6a0bd --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/onepass.rs @@ -0,0 +1,3208 @@ +/*! +A DFA that can return spans for matching capturing groups. + +This module is the home of a [one-pass DFA](DFA). + +This module also contains a [`Builder`] and a [`Config`] for building and +configuring a one-pass DFA. +*/ + +// A note on naming and credit: +// +// As far as I know, Russ Cox came up with the practical vision and +// implementation of a "one-pass regex engine." He mentions and describes it +// briefly in the third article of his regexp article series: +// https://swtch.com/~rsc/regexp/regexp3.html +// +// Cox's implementation is in RE2, and the implementation below is most +// heavily inspired by RE2's. The key thing they have in common is that +// their transitions are defined over an alphabet of bytes. In contrast, +// Go's regex engine also has a one-pass engine, but its transitions are +// more firmly rooted on Unicode codepoints. The ideas are the same, but the +// implementations are different. +// +// RE2 tends to call this a "one-pass NFA." Here, we call it a "one-pass DFA." +// They're both true in their own ways: +// +// * The "one-pass" criterion is generally a property of the NFA itself. In +// particular, it is said that an NFA is one-pass if, after each byte of input +// during a search, there is at most one "VM thread" remaining to take for the +// next byte of input. That is, there is never any ambiguity as to the path to +// take through the NFA during a search. +// +// * On the other hand, once a one-pass NFA has its representation converted +// to something where a constant number of instructions is used for each byte +// of input, the implementation looks a lot more like a DFA. It's technically +// more powerful than a DFA since it has side effects (storing offsets inside +// of slots activated by a transition), but it is far closer to a DFA than an +// NFA simulation. +// +// Thus, in this crate, we call it a one-pass DFA. + +use alloc::{vec, vec::Vec}; + +use crate::{ + dfa::{remapper::Remapper, DEAD}, + nfa::thompson::{self, NFA}, + util::{ + alphabet::ByteClasses, + captures::Captures, + escape::DebugByte, + int::{Usize, U32, U64, U8}, + look::{Look, LookSet, UnicodeWordBoundaryError}, + primitives::{NonMaxUsize, PatternID, StateID}, + search::{Anchored, Input, Match, MatchError, MatchKind, Span}, + sparse_set::SparseSet, + }, +}; + +/// The configuration used for building a [one-pass DFA](DFA). +/// +/// A one-pass DFA configuration is a simple data object that is typically used +/// with [`Builder::configure`]. It can be cheaply cloned. +/// +/// A default configuration can be created either with `Config::new`, or +/// perhaps more conveniently, with [`DFA::config`]. +#[derive(Clone, Debug, Default)] +pub struct Config { + match_kind: Option<MatchKind>, + starts_for_each_pattern: Option<bool>, + byte_classes: Option<bool>, + size_limit: Option<Option<usize>>, +} + +impl Config { + /// Return a new default one-pass DFA configuration. + pub fn new() -> Config { + Config::default() + } + + /// Set the desired match semantics. + /// + /// The default is [`MatchKind::LeftmostFirst`], which corresponds to the + /// match semantics of Perl-like regex engines. That is, when multiple + /// patterns would match at the same leftmost position, the pattern that + /// appears first in the concrete syntax is chosen. + /// + /// Currently, the only other kind of match semantics supported is + /// [`MatchKind::All`]. This corresponds to "classical DFA" construction + /// where all possible matches are visited. + /// + /// When it comes to the one-pass DFA, it is rarer for preference order and + /// "longest match" to actually disagree. Since if they did disagree, then + /// the regex typically isn't one-pass. For example, searching `Samwise` + /// for `Sam|Samwise` will report `Sam` for leftmost-first matching and + /// `Samwise` for "longest match" or "all" matching. However, this regex is + /// not one-pass if taken literally. The equivalent regex, `Sam(?:|wise)` + /// is one-pass and `Sam|Samwise` may be optimized to it. + /// + /// The other main difference is that "all" match semantics don't support + /// non-greedy matches. "All" match semantics always try to match as much + /// as possible. + pub fn match_kind(mut self, kind: MatchKind) -> Config { + self.match_kind = Some(kind); + self + } + + /// Whether to compile a separate start state for each pattern in the + /// one-pass DFA. + /// + /// When enabled, a separate **anchored** start state is added for each + /// pattern in the DFA. When this start state is used, then the DFA will + /// only search for matches for the pattern specified, even if there are + /// other patterns in the DFA. + /// + /// The main downside of this option is that it can potentially increase + /// the size of the DFA and/or increase the time it takes to build the DFA. + /// + /// You might want to enable this option when you want to both search for + /// anchored matches of any pattern or to search for anchored matches of + /// one particular pattern while using the same DFA. (Otherwise, you would + /// need to compile a new DFA for each pattern.) + /// + /// By default this is disabled. + /// + /// # Example + /// + /// This example shows how to build a multi-regex and then search for + /// matches for a any of the patterns or matches for a specific pattern. + /// + /// ``` + /// use regex_automata::{ + /// dfa::onepass::DFA, Anchored, Input, Match, PatternID, + /// }; + /// + /// let re = DFA::builder() + /// .configure(DFA::config().starts_for_each_pattern(true)) + /// .build_many(&["[a-z]+", "[0-9]+"])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "123abc"; + /// let input = Input::new(haystack).anchored(Anchored::Yes); + /// + /// // A normal multi-pattern search will show pattern 1 matches. + /// re.try_search(&mut cache, &input, &mut caps)?; + /// assert_eq!(Some(Match::must(1, 0..3)), caps.get_match()); + /// + /// // If we only want to report pattern 0 matches, then we'll get no + /// // match here. + /// let input = input.anchored(Anchored::Pattern(PatternID::must(0))); + /// re.try_search(&mut cache, &input, &mut caps)?; + /// assert_eq!(None, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn starts_for_each_pattern(mut self, yes: bool) -> Config { + self.starts_for_each_pattern = Some(yes); + self + } + + /// Whether to attempt to shrink the size of the DFA's alphabet or not. + /// + /// This option is enabled by default and should never be disabled unless + /// one is debugging a one-pass DFA. + /// + /// When enabled, the DFA will use a map from all possible bytes to their + /// corresponding equivalence class. Each equivalence class represents a + /// set of bytes that does not discriminate between a match and a non-match + /// in the DFA. For example, the pattern `[ab]+` has at least two + /// equivalence classes: a set containing `a` and `b` and a set containing + /// every byte except for `a` and `b`. `a` and `b` are in the same + /// equivalence class because they never discriminate between a match and a + /// non-match. + /// + /// The advantage of this map is that the size of the transition table + /// can be reduced drastically from (approximately) `#states * 256 * + /// sizeof(StateID)` to `#states * k * sizeof(StateID)` where `k` is the + /// number of equivalence classes (rounded up to the nearest power of 2). + /// As a result, total space usage can decrease substantially. Moreover, + /// since a smaller alphabet is used, DFA compilation becomes faster as + /// well. + /// + /// **WARNING:** This is only useful for debugging DFAs. Disabling this + /// does not yield any speed advantages. Namely, even when this is + /// disabled, a byte class map is still used while searching. The only + /// difference is that every byte will be forced into its own distinct + /// equivalence class. This is useful for debugging the actual generated + /// transitions because it lets one see the transitions defined on actual + /// bytes instead of the equivalence classes. + pub fn byte_classes(mut self, yes: bool) -> Config { + self.byte_classes = Some(yes); + self + } + + /// Set a size limit on the total heap used by a one-pass DFA. + /// + /// This size limit is expressed in bytes and is applied during + /// construction of a one-pass DFA. If the DFA's heap usage exceeds + /// this configured limit, then construction is stopped and an error is + /// returned. + /// + /// The default is no limit. + /// + /// # Example + /// + /// This example shows a one-pass DFA that fails to build because of + /// a configured size limit. This particular example also serves as a + /// cautionary tale demonstrating just how big DFAs with large Unicode + /// character classes can get. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::onepass::DFA, Match}; + /// + /// // 6MB isn't enough! + /// DFA::builder() + /// .configure(DFA::config().size_limit(Some(6_000_000))) + /// .build(r"\w{20}") + /// .unwrap_err(); + /// + /// // ... but 7MB probably is! + /// // (Note that DFA sizes aren't necessarily stable between releases.) + /// let re = DFA::builder() + /// .configure(DFA::config().size_limit(Some(7_000_000))) + /// .build(r"\w{20}")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "A".repeat(20); + /// re.captures(&mut cache, &haystack, &mut caps); + /// assert_eq!(Some(Match::must(0, 0..20)), caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// While one needs a little more than 3MB to represent `\w{20}`, it + /// turns out that you only need a little more than 4KB to represent + /// `(?-u:\w{20})`. So only use Unicode if you need it! + pub fn size_limit(mut self, limit: Option<usize>) -> Config { + self.size_limit = Some(limit); + self + } + + /// Returns the match semantics set in this configuration. + pub fn get_match_kind(&self) -> MatchKind { + self.match_kind.unwrap_or(MatchKind::LeftmostFirst) + } + + /// Returns whether this configuration has enabled anchored starting states + /// for every pattern in the DFA. + pub fn get_starts_for_each_pattern(&self) -> bool { + self.starts_for_each_pattern.unwrap_or(false) + } + + /// Returns whether this configuration has enabled byte classes or not. + /// This is typically a debugging oriented option, as disabling it confers + /// no speed benefit. + pub fn get_byte_classes(&self) -> bool { + self.byte_classes.unwrap_or(true) + } + + /// Returns the DFA size limit of this configuration if one was set. + /// The size limit is total number of bytes on the heap that a DFA is + /// permitted to use. If the DFA exceeds this limit during construction, + /// then construction is stopped and an error is returned. + pub fn get_size_limit(&self) -> Option<usize> { + self.size_limit.unwrap_or(None) + } + + /// Overwrite the default configuration such that the options in `o` are + /// always used. If an option in `o` is not set, then the corresponding + /// option in `self` is used. If it's not set in `self` either, then it + /// remains not set. + pub(crate) fn overwrite(&self, o: Config) -> Config { + Config { + match_kind: o.match_kind.or(self.match_kind), + starts_for_each_pattern: o + .starts_for_each_pattern + .or(self.starts_for_each_pattern), + byte_classes: o.byte_classes.or(self.byte_classes), + size_limit: o.size_limit.or(self.size_limit), + } + } +} + +/// A builder for a [one-pass DFA](DFA). +/// +/// This builder permits configuring options for the syntax of a pattern, the +/// NFA construction and the DFA construction. This builder is different from a +/// general purpose regex builder in that it permits fine grain configuration +/// of the construction process. The trade off for this is complexity, and +/// the possibility of setting a configuration that might not make sense. For +/// example, there are two different UTF-8 modes: +/// +/// * [`syntax::Config::utf8`](crate::util::syntax::Config::utf8) controls +/// whether the pattern itself can contain sub-expressions that match invalid +/// UTF-8. +/// * [`thompson::Config::utf8`] controls whether empty matches that split a +/// Unicode codepoint are reported or not. +/// +/// Generally speaking, callers will want to either enable all of these or +/// disable all of these. +/// +/// # Example +/// +/// This example shows how to disable UTF-8 mode in the syntax and the NFA. +/// This is generally what you want for matching on arbitrary bytes. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{ +/// dfa::onepass::DFA, +/// nfa::thompson, +/// util::syntax, +/// Match, +/// }; +/// +/// let re = DFA::builder() +/// .syntax(syntax::Config::new().utf8(false)) +/// .thompson(thompson::Config::new().utf8(false)) +/// .build(r"foo(?-u:[^b])ar.*")?; +/// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); +/// +/// let haystack = b"foo\xFFarzz\xE2\x98\xFF\n"; +/// re.captures(&mut cache, haystack, &mut caps); +/// // Notice that `(?-u:[^b])` matches invalid UTF-8, +/// // but the subsequent `.*` does not! Disabling UTF-8 +/// // on the syntax permits this. +/// // +/// // N.B. This example does not show the impact of +/// // disabling UTF-8 mode on a one-pass DFA Config, +/// // since that only impacts regexes that can +/// // produce matches of length 0. +/// assert_eq!(Some(Match::must(0, 0..8)), caps.get_match()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Builder { + config: Config, + #[cfg(feature = "syntax")] + thompson: thompson::Compiler, +} + +impl Builder { + /// Create a new one-pass DFA builder with the default configuration. + pub fn new() -> Builder { + Builder { + config: Config::default(), + #[cfg(feature = "syntax")] + thompson: thompson::Compiler::new(), + } + } + + /// Build a one-pass DFA from the given pattern. + /// + /// If there was a problem parsing or compiling the pattern, then an error + /// is returned. + #[cfg(feature = "syntax")] + pub fn build(&self, pattern: &str) -> Result<DFA, BuildError> { + self.build_many(&[pattern]) + } + + /// Build a one-pass DFA from the given patterns. + /// + /// When matches are returned, the pattern ID corresponds to the index of + /// the pattern in the slice given. + #[cfg(feature = "syntax")] + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<DFA, BuildError> { + let nfa = + self.thompson.build_many(patterns).map_err(BuildError::nfa)?; + self.build_from_nfa(nfa) + } + + /// Build a DFA from the given NFA. + /// + /// # Example + /// + /// This example shows how to build a DFA if you already have an NFA in + /// hand. + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, nfa::thompson::NFA, Match}; + /// + /// // This shows how to set non-default options for building an NFA. + /// let nfa = NFA::compiler() + /// .configure(NFA::config().shrink(true)) + /// .build(r"[a-z0-9]+")?; + /// let re = DFA::builder().build_from_nfa(nfa)?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// re.captures(&mut cache, "foo123bar", &mut caps); + /// assert_eq!(Some(Match::must(0, 0..9)), caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_from_nfa(&self, nfa: NFA) -> Result<DFA, BuildError> { + // Why take ownership if we're just going to pass a reference to the + // NFA to our internal builder? Well, the first thing to note is that + // an NFA uses reference counting internally, so either choice is going + // to be cheap. So there isn't much cost either way. + // + // The real reason is that a one-pass DFA, semantically, shares + // ownership of an NFA. This is unlike other DFAs that don't share + // ownership of an NFA at all, primarily because they want to be + // self-contained in order to support cheap (de)serialization. + // + // But then why pass a '&nfa' below if we want to share ownership? + // Well, it turns out that using a '&NFA' in our internal builder + // separates its lifetime from the DFA we're building, and this turns + // out to make code a bit more composable. e.g., We can iterate over + // things inside the NFA while borrowing the builder as mutable because + // we know the NFA cannot be mutated. So TL;DR --- this weirdness is + // "because borrow checker." + InternalBuilder::new(self.config.clone(), &nfa).build() + } + + /// Apply the given one-pass DFA configuration options to this builder. + pub fn configure(&mut self, config: Config) -> &mut Builder { + self.config = self.config.overwrite(config); + self + } + + /// Set the syntax configuration for this builder using + /// [`syntax::Config`](crate::util::syntax::Config). + /// + /// This permits setting things like case insensitivity, Unicode and multi + /// line mode. + /// + /// These settings only apply when constructing a one-pass DFA directly + /// from a pattern. + #[cfg(feature = "syntax")] + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Builder { + self.thompson.syntax(config); + self + } + + /// Set the Thompson NFA configuration for this builder using + /// [`nfa::thompson::Config`](crate::nfa::thompson::Config). + /// + /// This permits setting things like whether additional time should be + /// spent shrinking the size of the NFA. + /// + /// These settings only apply when constructing a DFA directly from a + /// pattern. + #[cfg(feature = "syntax")] + pub fn thompson(&mut self, config: thompson::Config) -> &mut Builder { + self.thompson.configure(config); + self + } +} + +/// An internal builder for encapsulating the state necessary to build a +/// one-pass DFA. Typical use is just `InternalBuilder::new(..).build()`. +/// +/// There is no separate pass for determining whether the NFA is one-pass or +/// not. We just try to build the DFA. If during construction we discover that +/// it is not one-pass, we bail out. This is likely to lead to some undesirable +/// expense in some cases, so it might make sense to try an identify common +/// patterns in the NFA that make it definitively not one-pass. That way, we +/// can avoid ever trying to build a one-pass DFA in the first place. For +/// example, '\w*\s' is not one-pass, and since '\w' is Unicode-aware by +/// default, it's probably not a trivial cost to try and build a one-pass DFA +/// for it and then fail. +/// +/// Note that some (immutable) fields are duplicated here. For example, the +/// 'nfa' and 'classes' fields are both in the 'DFA'. They are the same thing, +/// but we duplicate them because it makes composition easier below. Otherwise, +/// since the borrow checker can't see through method calls, the mutable borrow +/// we use to mutate the DFA winds up preventing borrowing from any other part +/// of the DFA, even though we aren't mutating those parts. We only do this +/// because the duplication is cheap. +#[derive(Debug)] +struct InternalBuilder<'a> { + /// The DFA we're building. + dfa: DFA, + /// An unordered collection of NFA state IDs that we haven't yet tried to + /// build into a DFA state yet. + /// + /// This collection does not ultimately wind up including every NFA state + /// ID. Instead, each ID represents a "start" state for a sub-graph of the + /// NFA. The set of NFA states we then use to build a DFA state consists + /// of that "start" state and all states reachable from it via epsilon + /// transitions. + uncompiled_nfa_ids: Vec<StateID>, + /// A map from NFA state ID to DFA state ID. This is useful for easily + /// determining whether an NFA state has been used as a "starting" point + /// to build a DFA state yet. If it hasn't, then it is mapped to DEAD, + /// and since DEAD is specially added and never corresponds to any NFA + /// state, it follows that a mapping to DEAD implies the NFA state has + /// no corresponding DFA state yet. + nfa_to_dfa_id: Vec<StateID>, + /// A stack used to traverse the NFA states that make up a single DFA + /// state. Traversal occurs until the stack is empty, and we only push to + /// the stack when the state ID isn't in 'seen'. Actually, even more than + /// that, if we try to push something on to this stack that is already in + /// 'seen', then we bail out on construction completely, since it implies + /// that the NFA is not one-pass. + stack: Vec<(StateID, Epsilons)>, + /// The set of NFA states that we've visited via 'stack'. + seen: SparseSet, + /// Whether a match NFA state has been observed while constructing a + /// one-pass DFA state. Once a match state is seen, assuming we are using + /// leftmost-first match semantics, then we don't add any more transitions + /// to the DFA state we're building. + matched: bool, + /// The config passed to the builder. + /// + /// This is duplicated in dfa.config. + config: Config, + /// The NFA we're building a one-pass DFA from. + /// + /// This is duplicated in dfa.nfa. + nfa: &'a NFA, + /// The equivalence classes that make up the alphabet for this DFA> + /// + /// This is duplicated in dfa.classes. + classes: ByteClasses, +} + +impl<'a> InternalBuilder<'a> { + /// Create a new builder with an initial empty DFA. + fn new(config: Config, nfa: &'a NFA) -> InternalBuilder<'a> { + let classes = if !config.get_byte_classes() { + // A one-pass DFA will always use the equivalence class map, but + // enabling this option is useful for debugging. Namely, this will + // cause all transitions to be defined over their actual bytes + // instead of an opaque equivalence class identifier. The former is + // much easier to grok as a human. + ByteClasses::singletons() + } else { + nfa.byte_classes().clone() + }; + // Normally a DFA alphabet includes the EOI symbol, but we don't need + // that in the one-pass DFA since we handle look-around explicitly + // without encoding it into the DFA. Thus, we don't need to delay + // matches by 1 byte. However, we reuse the space that *would* be used + // by the EOI transition by putting match information there (like which + // pattern matches and which look-around assertions need to hold). So + // this means our real alphabet length is 1 fewer than what the byte + // classes report, since we don't use EOI. + let alphabet_len = classes.alphabet_len().checked_sub(1).unwrap(); + let stride2 = classes.stride2(); + let dfa = DFA { + config: config.clone(), + nfa: nfa.clone(), + table: vec![], + starts: vec![], + // Since one-pass DFAs have a smaller state ID max than + // StateID::MAX, it follows that StateID::MAX is a valid initial + // value for min_match_id since no state ID can ever be greater + // than it. In the case of a one-pass DFA with no match states, the + // min_match_id will keep this sentinel value. + min_match_id: StateID::MAX, + classes: classes.clone(), + alphabet_len, + stride2, + pateps_offset: alphabet_len, + // OK because PatternID::MAX*2 is guaranteed not to overflow. + explicit_slot_start: nfa.pattern_len().checked_mul(2).unwrap(), + }; + InternalBuilder { + dfa, + uncompiled_nfa_ids: vec![], + nfa_to_dfa_id: vec![DEAD; nfa.states().len()], + stack: vec![], + seen: SparseSet::new(nfa.states().len()), + matched: false, + config, + nfa, + classes, + } + } + + /// Build the DFA from the NFA given to this builder. If the NFA is not + /// one-pass, then return an error. An error may also be returned if a + /// particular limit is exceeded. (Some limits, like the total heap memory + /// used, are configurable. Others, like the total patterns or slots, are + /// hard-coded based on representational limitations.) + fn build(mut self) -> Result<DFA, BuildError> { + self.nfa.look_set_any().available().map_err(BuildError::word)?; + for look in self.nfa.look_set_any().iter() { + // This is a future incompatibility check where if we add any + // more look-around assertions, then the one-pass DFA either + // needs to reject them (what we do here) or it needs to have its + // Transition representation modified to be capable of storing the + // new assertions. + if look.as_repr() > Look::WordUnicodeNegate.as_repr() { + return Err(BuildError::unsupported_look(look)); + } + } + if self.nfa.pattern_len().as_u64() > PatternEpsilons::PATTERN_ID_LIMIT + { + return Err(BuildError::too_many_patterns( + PatternEpsilons::PATTERN_ID_LIMIT, + )); + } + if self.nfa.group_info().explicit_slot_len() > Slots::LIMIT { + return Err(BuildError::not_one_pass( + "too many explicit capturing groups (max is 16)", + )); + } + assert_eq!(DEAD, self.add_empty_state()?); + + // This is where the explicit slots start. We care about this because + // we only need to track explicit slots. The implicit slots---two for + // each pattern---are tracked as part of the search routine itself. + let explicit_slot_start = self.nfa.pattern_len() * 2; + self.add_start_state(None, self.nfa.start_anchored())?; + if self.config.get_starts_for_each_pattern() { + for pid in self.nfa.patterns() { + self.add_start_state( + Some(pid), + self.nfa.start_pattern(pid).unwrap(), + )?; + } + } + // NOTE: One wonders what the effects of treating 'uncompiled_nfa_ids' + // as a stack are. It is really an unordered *set* of NFA state IDs. + // If it, for example, in practice led to discovering whether a regex + // was or wasn't one-pass later than if we processed NFA state IDs in + // ascending order, then that would make this routine more costly in + // the somewhat common case of a regex that isn't one-pass. + while let Some(nfa_id) = self.uncompiled_nfa_ids.pop() { + let dfa_id = self.nfa_to_dfa_id[nfa_id]; + // Once we see a match, we keep going, but don't add any new + // transitions. Normally we'd just stop, but we have to keep + // going in order to verify that our regex is actually one-pass. + self.matched = false; + // The NFA states we've already explored for this DFA state. + self.seen.clear(); + // The NFA states to explore via epsilon transitions. If we ever + // try to push an NFA state that we've already seen, then the NFA + // is not one-pass because it implies there are multiple epsilon + // transition paths that lead to the same NFA state. In other + // words, there is ambiguity. + self.stack_push(nfa_id, Epsilons::empty())?; + while let Some((id, epsilons)) = self.stack.pop() { + match *self.nfa.state(id) { + thompson::State::ByteRange { ref trans } => { + self.compile_transition(dfa_id, trans, epsilons)?; + } + thompson::State::Sparse(ref sparse) => { + for trans in sparse.transitions.iter() { + self.compile_transition(dfa_id, trans, epsilons)?; + } + } + thompson::State::Dense(ref dense) => { + for trans in dense.iter() { + self.compile_transition(dfa_id, &trans, epsilons)?; + } + } + thompson::State::Look { look, next } => { + let looks = epsilons.looks().insert(look); + self.stack_push(next, epsilons.set_looks(looks))?; + } + thompson::State::Union { ref alternates } => { + for &sid in alternates.iter().rev() { + self.stack_push(sid, epsilons)?; + } + } + thompson::State::BinaryUnion { alt1, alt2 } => { + self.stack_push(alt2, epsilons)?; + self.stack_push(alt1, epsilons)?; + } + thompson::State::Capture { next, slot, .. } => { + let slot = slot.as_usize(); + let epsilons = if slot < explicit_slot_start { + // If this is an implicit slot, we don't care + // about it, since we handle implicit slots in + // the search routine. We can get away with that + // because there are 2 implicit slots for every + // pattern. + epsilons + } else { + // Offset our explicit slots so that they start + // at index 0. + let offset = slot - explicit_slot_start; + epsilons.set_slots(epsilons.slots().insert(offset)) + }; + self.stack_push(next, epsilons)?; + } + thompson::State::Fail => { + continue; + } + thompson::State::Match { pattern_id } => { + // If we found two different paths to a match state + // for the same DFA state, then we have ambiguity. + // Thus, it's not one-pass. + if self.matched { + return Err(BuildError::not_one_pass( + "multiple epsilon transitions to match state", + )); + } + self.matched = true; + // Shove the matching pattern ID and the 'epsilons' + // into the current DFA state's pattern epsilons. The + // 'epsilons' includes the slots we need to capture + // before reporting the match and also the conditional + // epsilon transitions we need to check before we can + // report a match. + self.dfa.set_pattern_epsilons( + dfa_id, + PatternEpsilons::empty() + .set_pattern_id(pattern_id) + .set_epsilons(epsilons), + ); + // N.B. It is tempting to just bail out here when + // compiling a leftmost-first DFA, since we will never + // compile any more transitions in that case. But we + // actually need to keep going in order to verify that + // we actually have a one-pass regex. e.g., We might + // see more Match states (e.g., for other patterns) + // that imply that we don't have a one-pass regex. + // So instead, we mark that we've found a match and + // continue on. When we go to compile a new DFA state, + // we just skip that part. But otherwise check that the + // one-pass property is upheld. + } + } + } + } + self.shuffle_states(); + self.dfa.starts.shrink_to_fit(); + self.dfa.table.shrink_to_fit(); + Ok(self.dfa) + } + + /// Shuffle all match states to the end of the transition table and set + /// 'min_match_id' to the ID of the first such match state. + /// + /// The point of this is to make it extremely cheap to determine whether + /// a state is a match state or not. We need to check on this on every + /// transition during a search, so it being cheap is important. This + /// permits us to check it by simply comparing two state identifiers, as + /// opposed to looking for the pattern ID in the state's `PatternEpsilons`. + /// (Which requires a memory load and some light arithmetic.) + fn shuffle_states(&mut self) { + let mut remapper = Remapper::new(&self.dfa); + let mut next_dest = self.dfa.last_state_id(); + for i in (0..self.dfa.state_len()).rev() { + let id = StateID::must(i); + let is_match = + self.dfa.pattern_epsilons(id).pattern_id().is_some(); + if !is_match { + continue; + } + remapper.swap(&mut self.dfa, next_dest, id); + self.dfa.min_match_id = next_dest; + next_dest = self.dfa.prev_state_id(next_dest).expect( + "match states should be a proper subset of all states", + ); + } + remapper.remap(&mut self.dfa); + } + + /// Compile the given NFA transition into the DFA state given. + /// + /// 'Epsilons' corresponds to any conditional epsilon transitions that need + /// to be satisfied to follow this transition, and any slots that need to + /// be saved if the transition is followed. + /// + /// If this transition indicates that the NFA is not one-pass, then + /// this returns an error. (This occurs, for example, if the DFA state + /// already has a transition defined for the same input symbols as the + /// given transition, *and* the result of the old and new transitions is + /// different.) + fn compile_transition( + &mut self, + dfa_id: StateID, + trans: &thompson::Transition, + epsilons: Epsilons, + ) -> Result<(), BuildError> { + let next_dfa_id = self.add_dfa_state_for_nfa_state(trans.next)?; + for byte in self + .classes + .representatives(trans.start..=trans.end) + .filter_map(|r| r.as_u8()) + { + let oldtrans = self.dfa.transition(dfa_id, byte); + let newtrans = + Transition::new(self.matched, next_dfa_id, epsilons); + // If the old transition points to the DEAD state, then we know + // 'byte' has not been mapped to any transition for this DFA state + // yet. So set it unconditionally. Otherwise, we require that the + // old and new transitions are equivalent. Otherwise, there is + // ambiguity and thus the regex is not one-pass. + if oldtrans.state_id() == DEAD { + self.dfa.set_transition(dfa_id, byte, newtrans); + } else if oldtrans != newtrans { + return Err(BuildError::not_one_pass( + "conflicting transition", + )); + } + } + Ok(()) + } + + /// Add a start state to the DFA corresponding to the given NFA starting + /// state ID. + /// + /// If adding a state would blow any limits (configured or hard-coded), + /// then an error is returned. + /// + /// If the starting state is an anchored state for a particular pattern, + /// then callers must provide the pattern ID for that starting state. + /// Callers must also ensure that the first starting state added is the + /// start state for all patterns, and then each anchored starting state for + /// each pattern (if necessary) added in order. Otherwise, this panics. + fn add_start_state( + &mut self, + pid: Option<PatternID>, + nfa_id: StateID, + ) -> Result<StateID, BuildError> { + match pid { + // With no pid, this should be the start state for all patterns + // and thus be the first one. + None => assert!(self.dfa.starts.is_empty()), + // With a pid, we want it to be at self.dfa.starts[pid+1]. + Some(pid) => assert!(self.dfa.starts.len() == pid.one_more()), + } + let dfa_id = self.add_dfa_state_for_nfa_state(nfa_id)?; + self.dfa.starts.push(dfa_id); + Ok(dfa_id) + } + + /// Add a new DFA state corresponding to the given NFA state. If adding a + /// state would blow any limits (configured or hard-coded), then an error + /// is returned. If a DFA state already exists for the given NFA state, + /// then that DFA state's ID is returned and no new states are added. + /// + /// It is not expected that this routine is called for every NFA state. + /// Instead, an NFA state ID will usually correspond to the "start" state + /// for a sub-graph of the NFA, where all states in the sub-graph are + /// reachable via epsilon transitions (conditional or unconditional). That + /// sub-graph of NFA states is ultimately what produces a single DFA state. + fn add_dfa_state_for_nfa_state( + &mut self, + nfa_id: StateID, + ) -> Result<StateID, BuildError> { + // If we've already built a DFA state for the given NFA state, then + // just return that. We definitely do not want to have more than one + // DFA state in existence for the same NFA state, since all but one of + // them will likely become unreachable. And at least some of them are + // likely to wind up being incomplete. + let existing_dfa_id = self.nfa_to_dfa_id[nfa_id]; + if existing_dfa_id != DEAD { + return Ok(existing_dfa_id); + } + // If we don't have any DFA state yet, add it and then add the given + // NFA state to the list of states to explore. + let dfa_id = self.add_empty_state()?; + self.nfa_to_dfa_id[nfa_id] = dfa_id; + self.uncompiled_nfa_ids.push(nfa_id); + Ok(dfa_id) + } + + /// Unconditionally add a new empty DFA state. If adding it would exceed + /// any limits (configured or hard-coded), then an error is returned. The + /// ID of the new state is returned on success. + /// + /// The added state is *not* a match state. + fn add_empty_state(&mut self) -> Result<StateID, BuildError> { + let state_limit = Transition::STATE_ID_LIMIT; + // Note that unlike dense and lazy DFAs, we specifically do NOT + // premultiply our state IDs here. The reason is that we want to pack + // our state IDs into 64-bit transitions with other info, so the fewer + // the bits we use for state IDs the better. If we premultiply, then + // our state ID space shrinks. We justify this by the assumption that + // a one-pass DFA is just already doing a fair bit more work than a + // normal DFA anyway, so an extra multiplication to compute a state + // transition doesn't seem like a huge deal. + let next_id = self.dfa.table.len() >> self.dfa.stride2(); + let id = StateID::new(next_id) + .map_err(|_| BuildError::too_many_states(state_limit))?; + if id.as_u64() > Transition::STATE_ID_LIMIT { + return Err(BuildError::too_many_states(state_limit)); + } + self.dfa + .table + .extend(core::iter::repeat(Transition(0)).take(self.dfa.stride())); + // The default empty value for 'PatternEpsilons' is sadly not all + // zeroes. Instead, a special sentinel is used to indicate that there + // is no pattern. So we need to explicitly set the pattern epsilons to + // the correct "empty" PatternEpsilons. + self.dfa.set_pattern_epsilons(id, PatternEpsilons::empty()); + if let Some(size_limit) = self.config.get_size_limit() { + if self.dfa.memory_usage() > size_limit { + return Err(BuildError::exceeded_size_limit(size_limit)); + } + } + Ok(id) + } + + /// Push the given NFA state ID and its corresponding epsilons (slots and + /// conditional epsilon transitions) on to a stack for use in a depth first + /// traversal of a sub-graph of the NFA. + /// + /// If the given NFA state ID has already been pushed on to the stack, then + /// it indicates the regex is not one-pass and this correspondingly returns + /// an error. + fn stack_push( + &mut self, + nfa_id: StateID, + epsilons: Epsilons, + ) -> Result<(), BuildError> { + // If we already have seen a match and we are compiling a leftmost + // first DFA, then we shouldn't add any more states to look at. This is + // effectively how preference order and non-greediness is implemented. + // if !self.config.get_match_kind().continue_past_first_match() + // && self.matched + // { + // return Ok(()); + // } + if !self.seen.insert(nfa_id) { + return Err(BuildError::not_one_pass( + "multiple epsilon transitions to same state", + )); + } + self.stack.push((nfa_id, epsilons)); + Ok(()) + } +} + +/// A one-pass DFA for executing a subset of anchored regex searches while +/// resolving capturing groups. +/// +/// A one-pass DFA can be built from an NFA that is one-pass. An NFA is +/// one-pass when there is never any ambiguity about how to continue a search. +/// For example, `a*a` is not one-pass because during a search, it's not +/// possible to know whether to continue matching the `a*` or to move on to +/// the single `a`. However, `a*b` is one-pass, because for every byte in the +/// input, it's always clear when to move on from `a*` to `b`. +/// +/// # Only anchored searches are supported +/// +/// In this crate, especially for DFAs, unanchored searches are implemented by +/// treating the pattern as if it had a `(?s-u:.)*?` prefix. While the prefix +/// is one-pass on its own, adding anything after it, e.g., `(?s-u:.)*?a` will +/// make the overall pattern not one-pass. Why? Because the `(?s-u:.)` matches +/// any byte, and there is therefore ambiguity as to when the prefix should +/// stop matching and something else should start matching. +/// +/// Therefore, one-pass DFAs do not support unanchored searches. In addition +/// to many regexes simply not being one-pass, it implies that one-pass DFAs +/// have limited utility. With that said, when a one-pass DFA can be used, it +/// can potentially provide a dramatic speed up over alternatives like the +/// [`BoundedBacktracker`](crate::nfa::thompson::backtrack::BoundedBacktracker) +/// and the [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM). In particular, +/// a one-pass DFA is the only DFA capable of reporting the spans of matching +/// capturing groups. +/// +/// To clarify, when we say that unanchored searches are not supported, what +/// that actually means is: +/// +/// * The high level routines, [`DFA::is_match`] and [`DFA::captures`], always +/// do anchored searches. +/// * Since iterators are most useful in the context of unanchored searches, +/// there is no `DFA::captures_iter` method. +/// * For lower level routines like [`DFA::try_search`], an error will be +/// returned if the given [`Input`] is configured to do an unanchored search or +/// search for an invalid pattern ID. (Note that an [`Input`] is configured to +/// do an unanchored search by default, so just giving a `Input::new` is +/// guaranteed to return an error.) +/// +/// # Other limitations +/// +/// In addition to the [configurable heap limit](Config::size_limit) and +/// the requirement that a regex pattern be one-pass, there are some other +/// limitations: +/// +/// * There is an internal limit on the total number of explicit capturing +/// groups that appear across all patterns. It is somewhat small and there is +/// no way to configure it. If your pattern(s) exceed this limit, then building +/// a one-pass DFA will fail. +/// * If the number of patterns exceeds an internal unconfigurable limit, then +/// building a one-pass DFA will fail. This limit is quite large and you're +/// unlikely to hit it. +/// * If the total number of states exceeds an internal unconfigurable limit, +/// then building a one-pass DFA will fail. This limit is quite large and +/// you're unlikely to hit it. +/// +/// # Other examples of regexes that aren't one-pass +/// +/// One particularly unfortunate example is that enabling Unicode can cause +/// regexes that were one-pass to no longer be one-pass. Consider the regex +/// `(?-u)\w*\s` for example. It is one-pass because there is exactly no +/// overlap between the ASCII definitions of `\w` and `\s`. But `\w*\s` +/// (i.e., with Unicode enabled) is *not* one-pass because `\w` and `\s` get +/// translated to UTF-8 automatons. And while the *codepoints* in `\w` and `\s` +/// do not overlap, the underlying UTF-8 encodings do. Indeed, because of the +/// overlap between UTF-8 automata, the use of Unicode character classes will +/// tend to vastly increase the likelihood of a regex not being one-pass. +/// +/// # How does one know if a regex is one-pass or not? +/// +/// At the time of writing, the only way to know is to try and build a one-pass +/// DFA. The one-pass property is checked while constructing the DFA. +/// +/// This does mean that you might potentially waste some CPU cycles and memory +/// by optimistically trying to build a one-pass DFA. But this is currently the +/// only way. In the future, building a one-pass DFA might be able to use some +/// heuristics to detect common violations of the one-pass property and bail +/// more quickly. +/// +/// # Resource usage +/// +/// Unlike a general DFA, a one-pass DFA has stricter bounds on its resource +/// usage. Namely, construction of a one-pass DFA has a time and space +/// complexity of `O(n)`, where `n ~ nfa.states().len()`. (A general DFA's time +/// and space complexity is `O(2^n)`.) This smaller time bound is achieved +/// because there is at most one DFA state created for each NFA state. If +/// additional DFA states would be required, then the pattern is not one-pass +/// and construction will fail. +/// +/// Note though that currently, this DFA uses a fully dense representation. +/// This means that while its space complexity is no worse than an NFA, it may +/// in practice use more memory because of higher constant factors. The reason +/// for this trade off is two-fold. Firstly, a dense representation makes the +/// search faster. Secondly, the bigger an NFA, the more unlikely it is to be +/// one-pass. Therefore, most one-pass DFAs are usually pretty small. +/// +/// # Example +/// +/// This example shows that the one-pass DFA implements Unicode word boundaries +/// correctly while simultaneously reporting spans for capturing groups that +/// participate in a match. (This is the only DFA that implements full support +/// for Unicode word boundaries.) +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{dfa::onepass::DFA, Match, Span}; +/// +/// let re = DFA::new(r"\b(?P<first>\w+)[[:space:]]+(?P<last>\w+)\b")?; +/// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); +/// +/// re.captures(&mut cache, "Шерлок Холмс", &mut caps); +/// assert_eq!(Some(Match::must(0, 0..23)), caps.get_match()); +/// assert_eq!(Some(Span::from(0..12)), caps.get_group_by_name("first")); +/// assert_eq!(Some(Span::from(13..23)), caps.get_group_by_name("last")); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: iteration +/// +/// Unlike other regex engines in this crate, this one does not provide +/// iterator search functions. This is because a one-pass DFA only supports +/// anchored searches, and so iterator functions are generally not applicable. +/// +/// However, if you know that all of your matches are +/// directly adjacent, then an iterator can be used. The +/// [`util::iter::Searcher`](crate::util::iter::Searcher) type can be used for +/// this purpose: +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{ +/// dfa::onepass::DFA, +/// util::iter::Searcher, +/// Anchored, Input, Span, +/// }; +/// +/// let re = DFA::new(r"\w(\d)\w")?; +/// let (mut cache, caps) = (re.create_cache(), re.create_captures()); +/// let input = Input::new("a1zb2yc3x").anchored(Anchored::Yes); +/// +/// let mut it = Searcher::new(input).into_captures_iter(caps, |input, caps| { +/// Ok(re.try_search(&mut cache, input, caps)?) +/// }).infallible(); +/// let caps0 = it.next().unwrap(); +/// assert_eq!(Some(Span::from(1..2)), caps0.get_group(1)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone)] +pub struct DFA { + /// The configuration provided by the caller. + config: Config, + /// The NFA used to build this DFA. + /// + /// NOTE: We probably don't need to store the NFA here, but we use enough + /// bits from it that it's convenient to do so. And there really isn't much + /// cost to doing so either, since an NFA is reference counted internally. + nfa: NFA, + /// The transition table. Given a state ID 's' and a byte of haystack 'b', + /// the next state is `table[sid + classes[byte]]`. + /// + /// The stride of this table (i.e., the number of columns) is always + /// a power of 2, even if the alphabet length is smaller. This makes + /// converting between state IDs and state indices very cheap. + /// + /// Note that the stride always includes room for one extra "transition" + /// that isn't actually a transition. It is a 'PatternEpsilons' that is + /// used for match states only. Because of this, the maximum number of + /// active columns in the transition table is 257, which means the maximum + /// stride is 512 (the next power of 2 greater than or equal to 257). + table: Vec<Transition>, + /// The DFA state IDs of the starting states. + /// + /// `starts[0]` is always present and corresponds to the starting state + /// when searching for matches of any pattern in the DFA. + /// + /// `starts[i]` where i>0 corresponds to the starting state for the pattern + /// ID 'i-1'. These starting states are optional. + starts: Vec<StateID>, + /// Every state ID >= this value corresponds to a match state. + /// + /// This is what a search uses to detect whether a state is a match state + /// or not. It requires only a simple comparison instead of bit-unpacking + /// the PatternEpsilons from every state. + min_match_id: StateID, + /// The alphabet of this DFA, split into equivalence classes. Bytes in the + /// same equivalence class can never discriminate between a match and a + /// non-match. + classes: ByteClasses, + /// The number of elements in each state in the transition table. This may + /// be less than the stride, since the stride is always a power of 2 and + /// the alphabet length can be anything up to and including 256. + alphabet_len: usize, + /// The number of columns in the transition table, expressed as a power of + /// 2. + stride2: usize, + /// The offset at which the PatternEpsilons for a match state is stored in + /// the transition table. + /// + /// PERF: One wonders whether it would be better to put this in a separate + /// allocation, since only match states have a non-empty PatternEpsilons + /// and the number of match states tends be dwarfed by the number of + /// non-match states. So this would save '8*len(non_match_states)' for each + /// DFA. The question is whether moving this to a different allocation will + /// lead to a perf hit during searches. You might think dealing with match + /// states is rare, but some regexes spend a lot of time in match states + /// gobbling up input. But... match state handling is already somewhat + /// expensive, so maybe this wouldn't do much? Either way, it's worth + /// experimenting. + pateps_offset: usize, + /// The first explicit slot index. This refers to the first slot appearing + /// immediately after the last implicit slot. It is always 'patterns.len() + /// * 2'. + /// + /// We record this because we only store the explicit slots in our DFA + /// transition table that need to be saved. Implicit slots are handled + /// automatically as part of the search. + explicit_slot_start: usize, +} + +impl DFA { + /// Parse the given regular expression using the default configuration and + /// return the corresponding one-pass DFA. + /// + /// If you want a non-default configuration, then use the [`Builder`] to + /// set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, Match}; + /// + /// let re = DFA::new("foo[0-9]+bar")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "foo12345barzzz", &mut caps); + /// assert_eq!(Some(Match::must(0, 0..11)), caps.get_match()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + #[inline] + pub fn new(pattern: &str) -> Result<DFA, BuildError> { + DFA::builder().build(pattern) + } + + /// Like `new`, but parses multiple patterns into a single "multi regex." + /// This similarly uses the default regex configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, Match}; + /// + /// let re = DFA::new_many(&["[a-z]+", "[0-9]+"])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "abc123", &mut caps); + /// assert_eq!(Some(Match::must(0, 0..3)), caps.get_match()); + /// + /// re.captures(&mut cache, "123abc", &mut caps); + /// assert_eq!(Some(Match::must(1, 0..3)), caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + #[inline] + pub fn new_many<P: AsRef<str>>(patterns: &[P]) -> Result<DFA, BuildError> { + DFA::builder().build_many(patterns) + } + + /// Like `new`, but builds a one-pass DFA directly from an NFA. This is + /// useful if you already have an NFA, or even if you hand-assembled the + /// NFA. + /// + /// # Example + /// + /// This shows how to hand assemble a regular expression via its HIR, + /// compile an NFA from it and build a one-pass DFA from the NFA. + /// + /// ``` + /// use regex_automata::{ + /// dfa::onepass::DFA, + /// nfa::thompson::NFA, + /// Match, + /// }; + /// use regex_syntax::hir::{Hir, Class, ClassBytes, ClassBytesRange}; + /// + /// let hir = Hir::class(Class::Bytes(ClassBytes::new(vec![ + /// ClassBytesRange::new(b'0', b'9'), + /// ClassBytesRange::new(b'A', b'Z'), + /// ClassBytesRange::new(b'_', b'_'), + /// ClassBytesRange::new(b'a', b'z'), + /// ]))); + /// + /// let config = NFA::config().nfa_size_limit(Some(1_000)); + /// let nfa = NFA::compiler().configure(config).build_from_hir(&hir)?; + /// + /// let re = DFA::new_from_nfa(nfa)?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let expected = Some(Match::must(0, 0..1)); + /// re.captures(&mut cache, "A", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new_from_nfa(nfa: NFA) -> Result<DFA, BuildError> { + DFA::builder().build_from_nfa(nfa) + } + + /// Create a new one-pass DFA that matches every input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, Match}; + /// + /// let dfa = DFA::always_match()?; + /// let mut cache = dfa.create_cache(); + /// let mut caps = dfa.create_captures(); + /// + /// let expected = Match::must(0, 0..0); + /// dfa.captures(&mut cache, "", &mut caps); + /// assert_eq!(Some(expected), caps.get_match()); + /// dfa.captures(&mut cache, "foo", &mut caps); + /// assert_eq!(Some(expected), caps.get_match()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn always_match() -> Result<DFA, BuildError> { + let nfa = thompson::NFA::always_match(); + Builder::new().build_from_nfa(nfa) + } + + /// Create a new one-pass DFA that never matches any input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::dfa::onepass::DFA; + /// + /// let dfa = DFA::never_match()?; + /// let mut cache = dfa.create_cache(); + /// let mut caps = dfa.create_captures(); + /// + /// dfa.captures(&mut cache, "", &mut caps); + /// assert_eq!(None, caps.get_match()); + /// dfa.captures(&mut cache, "foo", &mut caps); + /// assert_eq!(None, caps.get_match()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn never_match() -> Result<DFA, BuildError> { + let nfa = thompson::NFA::never_match(); + Builder::new().build_from_nfa(nfa) + } + + /// Return a default configuration for a DFA. + /// + /// This is a convenience routine to avoid needing to import the `Config` + /// type when customizing the construction of a DFA. + /// + /// # Example + /// + /// This example shows how to change the match semantics of this DFA from + /// its default "leftmost first" to "all." When using "all," non-greediness + /// doesn't apply and neither does preference order matching. Instead, the + /// longest match possible is always returned. (Although, by construction, + /// it's impossible for a one-pass DFA to have a different answer for + /// "preference order" vs "longest match.") + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, Match, MatchKind}; + /// + /// let re = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .build(r"(abc)+?")?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// re.captures(&mut cache, "abcabc", &mut caps); + /// // Normally, the non-greedy repetition would give us a 0..3 match. + /// assert_eq!(Some(Match::must(0, 0..6)), caps.get_match()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn config() -> Config { + Config::new() + } + + /// Return a builder for configuring the construction of a DFA. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Builder`] type in common cases. + /// + /// # Example + /// + /// This example shows how to use the builder to disable UTF-8 mode. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// dfa::onepass::DFA, + /// nfa::thompson, + /// util::syntax, + /// Match, + /// }; + /// + /// let re = DFA::builder() + /// .syntax(syntax::Config::new().utf8(false)) + /// .thompson(thompson::Config::new().utf8(false)) + /// .build(r"foo(?-u:[^b])ar.*")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// let haystack = b"foo\xFFarzz\xE2\x98\xFF\n"; + /// let expected = Some(Match::must(0, 0..8)); + /// re.captures(&mut cache, haystack, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn builder() -> Builder { + Builder::new() + } + + /// Create a new empty set of capturing groups that is guaranteed to be + /// valid for the search APIs on this DFA. + /// + /// A `Captures` value created for a specific DFA cannot be used with any + /// other DFA. + /// + /// This is a convenience function for [`Captures::all`]. See the + /// [`Captures`] documentation for an explanation of its alternative + /// constructors that permit the DFA to do less work during a search, and + /// thus might make it faster. + #[inline] + pub fn create_captures(&self) -> Captures { + Captures::all(self.nfa.group_info().clone()) + } + + /// Create a new cache for this DFA. + /// + /// The cache returned should only be used for searches for this + /// DFA. If you want to reuse the cache for another DFA, then you + /// must call [`Cache::reset`] with that DFA (or, equivalently, + /// [`DFA::reset_cache`]). + #[inline] + pub fn create_cache(&self) -> Cache { + Cache::new(self) + } + + /// Reset the given cache such that it can be used for searching with the + /// this DFA (and only this DFA). + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different DFA. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different DFA. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::onepass::DFA, Match}; + /// + /// let re1 = DFA::new(r"\w")?; + /// let re2 = DFA::new(r"\W")?; + /// let mut caps1 = re1.create_captures(); + /// let mut caps2 = re2.create_captures(); + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 0..2)), + /// { re1.captures(&mut cache, "Δ", &mut caps1); caps1.get_match() }, + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the one-pass DFA we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// re2.reset_cache(&mut cache); + /// assert_eq!( + /// Some(Match::must(0, 0..3)), + /// { re2.captures(&mut cache, "☃", &mut caps2); caps2.get_match() }, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn reset_cache(&self, cache: &mut Cache) { + cache.reset(self); + } + + /// Return the config for this one-pass DFA. + #[inline] + pub fn get_config(&self) -> &Config { + &self.config + } + + /// Returns a reference to the underlying NFA. + #[inline] + pub fn get_nfa(&self) -> &NFA { + &self.nfa + } + + /// Returns the total number of patterns compiled into this DFA. + /// + /// In the case of a DFA that contains no patterns, this returns `0`. + #[inline] + pub fn pattern_len(&self) -> usize { + self.get_nfa().pattern_len() + } + + /// Returns the total number of states in this one-pass DFA. + /// + /// Note that unlike dense or sparse DFAs, a one-pass DFA does not expose + /// a low level DFA API. Therefore, this routine has little use other than + /// being informational. + #[inline] + pub fn state_len(&self) -> usize { + self.table.len() >> self.stride2() + } + + /// Returns the total number of elements in the alphabet for this DFA. + /// + /// That is, this returns the total number of transitions that each + /// state in this DFA must have. The maximum alphabet size is 256, which + /// corresponds to each possible byte value. + /// + /// The alphabet size may be less than 256 though, and unless + /// [`Config::byte_classes`] is disabled, it is typically must less than + /// 256. Namely, bytes are grouped into equivalence classes such that no + /// two bytes in the same class can distinguish a match from a non-match. + /// For example, in the regex `^[a-z]+$`, the ASCII bytes `a-z` could + /// all be in the same equivalence class. This leads to a massive space + /// savings. + /// + /// Note though that the alphabet length does _not_ necessarily equal the + /// total stride space taken up by a single DFA state in the transition + /// table. Namely, for performance reasons, the stride is always the + /// smallest power of two that is greater than or equal to the alphabet + /// length. For this reason, [`DFA::stride`] or [`DFA::stride2`] are + /// often more useful. The alphabet length is typically useful only for + /// informational purposes. + /// + /// Note also that unlike dense or sparse DFAs, a one-pass DFA does + /// not have a special end-of-input (EOI) transition. This is because + /// a one-pass DFA handles look-around assertions explicitly (like the + /// [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM)) and does not build + /// them into the transitions of the DFA. + #[inline] + pub fn alphabet_len(&self) -> usize { + self.alphabet_len + } + + /// Returns the total stride for every state in this DFA, expressed as the + /// exponent of a power of 2. The stride is the amount of space each state + /// takes up in the transition table, expressed as a number of transitions. + /// (Unused transitions map to dead states.) + /// + /// The stride of a DFA is always equivalent to the smallest power of + /// 2 that is greater than or equal to the DFA's alphabet length. This + /// definition uses extra space, but possibly permits faster translation + /// between state identifiers and their corresponding offsets in this DFA's + /// transition table. + /// + /// For example, if the DFA's stride is 16 transitions, then its `stride2` + /// is `4` since `2^4 = 16`. + /// + /// The minimum `stride2` value is `1` (corresponding to a stride of `2`) + /// while the maximum `stride2` value is `9` (corresponding to a stride + /// of `512`). The maximum in theory should be `8`, but because of some + /// implementation quirks that may be relaxed in the future, it is one more + /// than `8`. (Do note that a maximal stride is incredibly rare, as it + /// would imply that there is almost no redundant in the regex pattern.) + /// + /// Note that unlike dense or sparse DFAs, a one-pass DFA does not expose + /// a low level DFA API. Therefore, this routine has little use other than + /// being informational. + #[inline] + pub fn stride2(&self) -> usize { + self.stride2 + } + + /// Returns the total stride for every state in this DFA. This corresponds + /// to the total number of transitions used by each state in this DFA's + /// transition table. + /// + /// Please see [`DFA::stride2`] for more information. In particular, this + /// returns the stride as the number of transitions, where as `stride2` + /// returns it as the exponent of a power of 2. + /// + /// Note that unlike dense or sparse DFAs, a one-pass DFA does not expose + /// a low level DFA API. Therefore, this routine has little use other than + /// being informational. + #[inline] + pub fn stride(&self) -> usize { + 1 << self.stride2() + } + + /// Returns the memory usage, in bytes, of this DFA. + /// + /// The memory usage is computed based on the number of bytes used to + /// represent this DFA. + /// + /// This does **not** include the stack size used up by this DFA. To + /// compute that, use `std::mem::size_of::<onepass::DFA>()`. + #[inline] + pub fn memory_usage(&self) -> usize { + use core::mem::size_of; + + self.table.len() * size_of::<Transition>() + + self.starts.len() * size_of::<StateID>() + } +} + +impl DFA { + /// Executes an anchored leftmost forward search, and returns true if and + /// only if this one-pass DFA matches the given haystack. + /// + /// This routine may short circuit if it knows that scanning future + /// input will never lead to a different result. In particular, if the + /// underlying DFA enters a match state, then this routine will return + /// `true` immediately without inspecting any future input. (Consider how + /// this might make a difference given the regex `a+` on the haystack + /// `aaaaaaaaaaaaaaa`. This routine can stop after it sees the first `a`, + /// but routines like `find` need to continue searching because `+` is + /// greedy by default.) + /// + /// The given `Input` is forcefully set to use [`Anchored::Yes`] if the + /// given configuration was [`Anchored::No`] (which is the default). + /// + /// # Panics + /// + /// This routine panics if the search could not complete. This can occur + /// in the following circumstances: + /// + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. Concretely, + /// this occurs when using [`Anchored::Pattern`] without enabling + /// [`Config::starts_for_each_pattern`]. + /// + /// When a search panics, callers cannot know whether a match exists or + /// not. + /// + /// Use [`DFA::try_search`] if you want to handle these panics as error + /// values instead. + /// + /// # Example + /// + /// This shows basic usage: + /// + /// ``` + /// use regex_automata::dfa::onepass::DFA; + /// + /// let re = DFA::new("foo[0-9]+bar")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.is_match(&mut cache, "foo12345bar")); + /// assert!(!re.is_match(&mut cache, "foobar")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: consistency with search APIs + /// + /// `is_match` is guaranteed to return `true` whenever `captures` returns + /// a match. This includes searches that are executed entirely within a + /// codepoint: + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, Input}; + /// + /// let re = DFA::new("a*")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(!re.is_match(&mut cache, Input::new("☃").span(1..2))); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Notice that when UTF-8 mode is disabled, then the above reports a + /// match because the restriction against zero-width matches that split a + /// codepoint has been lifted: + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, nfa::thompson::NFA, Input}; + /// + /// let re = DFA::builder() + /// .thompson(NFA::config().utf8(false)) + /// .build("a*")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.is_match(&mut cache, Input::new("☃").span(1..2))); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_match<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + ) -> bool { + let mut input = input.into().earliest(true); + if matches!(input.get_anchored(), Anchored::No) { + input.set_anchored(Anchored::Yes); + } + self.try_search_slots(cache, &input, &mut []).unwrap().is_some() + } + + /// Executes an anchored leftmost forward search, and returns a `Match` if + /// and only if this one-pass DFA matches the given haystack. + /// + /// This routine only includes the overall match span. To get access to the + /// individual spans of each capturing group, use [`DFA::captures`]. + /// + /// The given `Input` is forcefully set to use [`Anchored::Yes`] if the + /// given configuration was [`Anchored::No`] (which is the default). + /// + /// # Panics + /// + /// This routine panics if the search could not complete. This can occur + /// in the following circumstances: + /// + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. Concretely, + /// this occurs when using [`Anchored::Pattern`] without enabling + /// [`Config::starts_for_each_pattern`]. + /// + /// When a search panics, callers cannot know whether a match exists or + /// not. + /// + /// Use [`DFA::try_search`] if you want to handle these panics as error + /// values instead. + /// + /// # Example + /// + /// Leftmost first match semantics corresponds to the match with the + /// smallest starting offset, but where the end offset is determined by + /// preferring earlier branches in the original regular expression. For + /// example, `Sam|Samwise` will match `Sam` in `Samwise`, but `Samwise|Sam` + /// will match `Samwise` in `Samwise`. + /// + /// Generally speaking, the "leftmost first" match is how most backtracking + /// regular expressions tend to work. This is in contrast to POSIX-style + /// regular expressions that yield "leftmost longest" matches. Namely, + /// both `Sam|Samwise` and `Samwise|Sam` match `Samwise` when using + /// leftmost longest semantics. (This crate does not currently support + /// leftmost longest semantics.) + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, Match}; + /// + /// let re = DFA::new("foo[0-9]+")?; + /// let mut cache = re.create_cache(); + /// let expected = Match::must(0, 0..8); + /// assert_eq!(Some(expected), re.find(&mut cache, "foo12345")); + /// + /// // Even though a match is found after reading the first byte (`a`), + /// // the leftmost first match semantics demand that we find the earliest + /// // match that prefers earlier parts of the pattern over later parts. + /// let re = DFA::new("abc|a")?; + /// let mut cache = re.create_cache(); + /// let expected = Match::must(0, 0..3); + /// assert_eq!(Some(expected), re.find(&mut cache, "abc")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + ) -> Option<Match> { + let mut input = input.into(); + if matches!(input.get_anchored(), Anchored::No) { + input.set_anchored(Anchored::Yes); + } + if self.get_nfa().pattern_len() == 1 { + let mut slots = [None, None]; + let pid = + self.try_search_slots(cache, &input, &mut slots).unwrap()?; + let start = slots[0].unwrap().get(); + let end = slots[1].unwrap().get(); + return Some(Match::new(pid, Span { start, end })); + } + let ginfo = self.get_nfa().group_info(); + let slots_len = ginfo.implicit_slot_len(); + let mut slots = vec![None; slots_len]; + let pid = self.try_search_slots(cache, &input, &mut slots).unwrap()?; + let start = slots[pid.as_usize() * 2].unwrap().get(); + let end = slots[pid.as_usize() * 2 + 1].unwrap().get(); + Some(Match::new(pid, Span { start, end })) + } + + /// Executes an anchored leftmost forward search and writes the spans + /// of capturing groups that participated in a match into the provided + /// [`Captures`] value. If no match was found, then [`Captures::is_match`] + /// is guaranteed to return `false`. + /// + /// The given `Input` is forcefully set to use [`Anchored::Yes`] if the + /// given configuration was [`Anchored::No`] (which is the default). + /// + /// # Panics + /// + /// This routine panics if the search could not complete. This can occur + /// in the following circumstances: + /// + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. Concretely, + /// this occurs when using [`Anchored::Pattern`] without enabling + /// [`Config::starts_for_each_pattern`]. + /// + /// When a search panics, callers cannot know whether a match exists or + /// not. + /// + /// Use [`DFA::try_search`] if you want to handle these panics as error + /// values instead. + /// + /// # Example + /// + /// This shows a simple example of a one-pass regex that extracts + /// capturing group spans. + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, Match, Span}; + /// + /// let re = DFA::new( + /// // Notice that we use ASCII here. The corresponding Unicode regex + /// // is sadly not one-pass. + /// "(?P<first>[[:alpha:]]+)[[:space:]]+(?P<last>[[:alpha:]]+)", + /// )?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "Bruce Springsteen", &mut caps); + /// assert_eq!(Some(Match::must(0, 0..17)), caps.get_match()); + /// assert_eq!(Some(Span::from(0..5)), caps.get_group(1)); + /// assert_eq!(Some(Span::from(6..17)), caps.get_group_by_name("last")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn captures<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + caps: &mut Captures, + ) { + let mut input = input.into(); + if matches!(input.get_anchored(), Anchored::No) { + input.set_anchored(Anchored::Yes); + } + self.try_search(cache, &input, caps).unwrap(); + } + + /// Executes an anchored leftmost forward search and writes the spans + /// of capturing groups that participated in a match into the provided + /// [`Captures`] value. If no match was found, then [`Captures::is_match`] + /// is guaranteed to return `false`. + /// + /// The differences with [`DFA::captures`] are: + /// + /// 1. This returns an error instead of panicking if the search fails. + /// 2. Accepts an `&Input` instead of a `Into<Input>`. This permits reusing + /// the same input for multiple searches, which _may_ be important for + /// latency. + /// 3. This does not automatically change the [`Anchored`] mode from `No` + /// to `Yes`. Instead, if [`Input::anchored`] is `Anchored::No`, then an + /// error is returned. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in the following circumstances: + /// + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. Concretely, + /// this occurs when using [`Anchored::Pattern`] without enabling + /// [`Config::starts_for_each_pattern`]. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example: specific pattern search + /// + /// This example shows how to build a multi-regex that permits searching + /// for specific patterns. Note that this is somewhat less useful than + /// in other regex engines, since a one-pass DFA by definition has no + /// ambiguity about which pattern can match at a position. That is, if it + /// were possible for two different patterns to match at the same starting + /// position, then the multi-regex would not be one-pass and construction + /// would have failed. + /// + /// Nevertheless, this can still be useful if you only care about matches + /// for a specific pattern, and want the DFA to report "no match" even if + /// some other pattern would have matched. + /// + /// Note that in order to make use of this functionality, + /// [`Config::starts_for_each_pattern`] must be enabled. It is disabled + /// by default since it may result in higher memory usage. + /// + /// ``` + /// use regex_automata::{ + /// dfa::onepass::DFA, Anchored, Input, Match, PatternID, + /// }; + /// + /// let re = DFA::builder() + /// .configure(DFA::config().starts_for_each_pattern(true)) + /// .build_many(&["[a-z]+", "[0-9]+"])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "123abc"; + /// let input = Input::new(haystack).anchored(Anchored::Yes); + /// + /// // A normal multi-pattern search will show pattern 1 matches. + /// re.try_search(&mut cache, &input, &mut caps)?; + /// assert_eq!(Some(Match::must(1, 0..3)), caps.get_match()); + /// + /// // If we only want to report pattern 0 matches, then we'll get no + /// // match here. + /// let input = input.anchored(Anchored::Pattern(PatternID::must(0))); + /// re.try_search(&mut cache, &input, &mut caps)?; + /// assert_eq!(None, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specifying the bounds of a search + /// + /// This example shows how providing the bounds of a search can produce + /// different results than simply sub-slicing the haystack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::onepass::DFA, Anchored, Input, Match}; + /// + /// // one-pass DFAs fully support Unicode word boundaries! + /// // A sad joke is that a Unicode aware regex like \w+\s is not one-pass. + /// // :-( + /// let re = DFA::new(r"\b[0-9]{3}\b")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "foo123bar"; + /// + /// // Since we sub-slice the haystack, the search doesn't know about + /// // the larger context and assumes that `123` is surrounded by word + /// // boundaries. And of course, the match position is reported relative + /// // to the sub-slice as well, which means we get `0..3` instead of + /// // `3..6`. + /// let expected = Some(Match::must(0, 0..3)); + /// let input = Input::new(&haystack[3..6]).anchored(Anchored::Yes); + /// re.try_search(&mut cache, &input, &mut caps)?; + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we provide the bounds of the search within the context of the + /// // entire haystack, then the search can take the surrounding context + /// // into account. (And if we did find a match, it would be reported + /// // as a valid offset into `haystack` instead of its sub-slice.) + /// let expected = None; + /// let input = Input::new(haystack).range(3..6).anchored(Anchored::Yes); + /// re.try_search(&mut cache, &input, &mut caps)?; + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_search( + &self, + cache: &mut Cache, + input: &Input<'_>, + caps: &mut Captures, + ) -> Result<(), MatchError> { + let pid = self.try_search_slots(cache, input, caps.slots_mut())?; + caps.set_pattern(pid); + Ok(()) + } + + /// Executes an anchored leftmost forward search and writes the spans + /// of capturing groups that participated in a match into the provided + /// `slots`, and returns the matching pattern ID. The contents of the + /// slots for patterns other than the matching pattern are unspecified. If + /// no match was found, then `None` is returned and the contents of all + /// `slots` is unspecified. + /// + /// This is like [`DFA::try_search`], but it accepts a raw slots slice + /// instead of a `Captures` value. This is useful in contexts where you + /// don't want or need to allocate a `Captures`. + /// + /// It is legal to pass _any_ number of slots to this routine. If the regex + /// engine would otherwise write a slot offset that doesn't fit in the + /// provided slice, then it is simply skipped. In general though, there are + /// usually three slice lengths you might want to use: + /// + /// * An empty slice, if you only care about which pattern matched. + /// * A slice with + /// [`pattern_len() * 2`](crate::dfa::onepass::DFA::pattern_len) + /// slots, if you only care about the overall match spans for each matching + /// pattern. + /// * A slice with + /// [`slot_len()`](crate::util::captures::GroupInfo::slot_len) slots, which + /// permits recording match offsets for every capturing group in every + /// pattern. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in the following circumstances: + /// + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. Concretely, + /// this occurs when using [`Anchored::Pattern`] without enabling + /// [`Config::starts_for_each_pattern`]. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This example shows how to find the overall match offsets in a + /// multi-pattern search without allocating a `Captures` value. Indeed, we + /// can put our slots right on the stack. + /// + /// ``` + /// use regex_automata::{dfa::onepass::DFA, Anchored, Input, PatternID}; + /// + /// let re = DFA::new_many(&[ + /// r"[a-zA-Z]+", + /// r"[0-9]+", + /// ])?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("123").anchored(Anchored::Yes); + /// + /// // We only care about the overall match offsets here, so we just + /// // allocate two slots for each pattern. Each slot records the start + /// // and end of the match. + /// let mut slots = [None; 4]; + /// let pid = re.try_search_slots(&mut cache, &input, &mut slots)?; + /// assert_eq!(Some(PatternID::must(1)), pid); + /// + /// // The overall match offsets are always at 'pid * 2' and 'pid * 2 + 1'. + /// // See 'GroupInfo' for more details on the mapping between groups and + /// // slot indices. + /// let slot_start = pid.unwrap().as_usize() * 2; + /// let slot_end = slot_start + 1; + /// assert_eq!(Some(0), slots[slot_start].map(|s| s.get())); + /// assert_eq!(Some(3), slots[slot_end].map(|s| s.get())); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Result<Option<PatternID>, MatchError> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + if !utf8empty { + return self.try_search_slots_imp(cache, input, slots); + } + // See PikeVM::try_search_slots for why we do this. + let min = self.get_nfa().group_info().implicit_slot_len(); + if slots.len() >= min { + return self.try_search_slots_imp(cache, input, slots); + } + if self.get_nfa().pattern_len() == 1 { + let mut enough = [None, None]; + let got = self.try_search_slots_imp(cache, input, &mut enough)?; + // This is OK because we know `enough_slots` is strictly bigger + // than `slots`, otherwise this special case isn't reached. + slots.copy_from_slice(&enough[..slots.len()]); + return Ok(got); + } + let mut enough = vec![None; min]; + let got = self.try_search_slots_imp(cache, input, &mut enough)?; + // This is OK because we know `enough_slots` is strictly bigger than + // `slots`, otherwise this special case isn't reached. + slots.copy_from_slice(&enough[..slots.len()]); + Ok(got) + } + + #[inline(never)] + fn try_search_slots_imp( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Result<Option<PatternID>, MatchError> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + match self.search_imp(cache, input, slots)? { + None => return Ok(None), + Some(pid) if !utf8empty => return Ok(Some(pid)), + Some(pid) => { + // These slot indices are always correct because we know our + // 'pid' is valid and thus we know that the slot indices for it + // are valid. + let slot_start = pid.as_usize().wrapping_mul(2); + let slot_end = slot_start.wrapping_add(1); + // OK because we know we have a match and we know our caller + // provided slots are big enough (which we make true above if + // the caller didn't). Namely, we're only here when 'utf8empty' + // is true, and when that's true, we require slots for every + // pattern. + let start = slots[slot_start].unwrap().get(); + let end = slots[slot_end].unwrap().get(); + // If our match splits a codepoint, then we cannot report is + // as a match. And since one-pass DFAs only support anchored + // searches, we don't try to skip ahead to find the next match. + // We can just quit with nothing. + if start == end && !input.is_char_boundary(start) { + return Ok(None); + } + Ok(Some(pid)) + } + } + } +} + +impl DFA { + fn search_imp( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Result<Option<PatternID>, MatchError> { + // PERF: Some ideas. I ran out of steam after my initial impl to try + // many of these. + // + // 1) Try doing more state shuffling. Right now, all we do is push + // match states to the end of the transition table so that we can do + // 'if sid >= self.min_match_id' to know whether we're in a match + // state or not. But what about doing something like dense DFAs and + // pushing dead, match and states with captures/looks all toward the + // beginning of the transition table. Then we could do 'if sid <= + // self.max_special_id', in which case, we need to do some special + // handling of some sort. Otherwise, we get the happy path, just + // like in a DFA search. The main argument against this is that the + // one-pass DFA is likely to be used most often with capturing groups + // and if capturing groups are common, then this might wind up being a + // pessimization. + // + // 2) Consider moving 'PatternEpsilons' out of the transition table. + // It is only needed for match states and usually a small minority of + // states are match states. Therefore, we're using an extra 'u64' for + // most states. + // + // 3) I played around with the match state handling and it seems like + // there is probably a lot left on the table for improvement. The + // key tension is that the 'find_match' routine is a giant mess, but + // splitting it out into a non-inlineable function is a non-starter + // because the match state might consume input, so 'find_match' COULD + // be called quite a lot, and a function call at that point would trash + // perf. In theory, we could detect whether a match state consumes + // input and then specialize our search routine based on that. In that + // case, maybe an extra function call is OK, but even then, it might be + // too much of a latency hit. Another idea is to just try and figure + // out how to reduce the code size of 'find_match'. RE2 has a trick + // here where the match handling isn't done if we know the next byte of + // input yields a match too. Maybe we adopt that? + // + // This just might be a tricky DFA to optimize. + + if input.is_done() { + return Ok(None); + } + // We unfortunately have a bit of book-keeping to do to set things + // up. We do have to setup our cache and clear all of our slots. In + // particular, clearing the slots is necessary for the case where we + // report a match, but one of the capturing groups didn't participate + // in the match but had a span set from a previous search. That would + // be bad. In theory, we could avoid all this slot clearing if we knew + // that every slot was always activated for every match. Then we would + // know they would always be overwritten when a match is found. + // + // NOTE: We have to be careful here to avoid setting a length that + // exceeds the number of slots in our cache. Otherwise copying into + // the cache later will fail. This can happen when the number of + // caller provided slots is bigger than the number of slots in the + // compiled regex. (It's a bit of a weird case, but for simplicity and + // flexibility reasons, it is an API guarantee that the caller can + // provide any number of slots that they want.) + let explicit_slots_len = core::cmp::min( + Slots::LIMIT, + core::cmp::min( + slots.len().saturating_sub(self.explicit_slot_start), + cache.explicit_slots.len(), + ), + ); + cache.setup_search(explicit_slots_len); + for slot in cache.explicit_slots() { + *slot = None; + } + for slot in slots.iter_mut() { + *slot = None; + } + // We set the starting slots for every pattern up front. This does + // increase our latency somewhat, but it avoids having to do it every + // time we see a match state (which could be many times in a single + // search if the match state consumes input). + for pid in self.nfa.patterns() { + let i = pid.as_usize() * 2; + if i >= slots.len() { + break; + } + slots[i] = NonMaxUsize::new(input.start()); + } + let mut pid = None; + let mut next_sid = match input.get_anchored() { + Anchored::Yes => self.start(), + Anchored::Pattern(pid) => self.start_pattern(pid)?, + Anchored::No => { + // If the regex is itself always anchored, then we're fine, + // even if the search is configured to be unanchored. + if !self.nfa.is_always_start_anchored() { + return Err(MatchError::unsupported_anchored( + Anchored::No, + )); + } + self.start() + } + }; + let leftmost_first = + matches!(self.config.get_match_kind(), MatchKind::LeftmostFirst); + for at in input.start()..input.end() { + let sid = next_sid; + let trans = self.transition(sid, input.haystack()[at]); + next_sid = trans.state_id(); + let epsilons = trans.epsilons(); + if sid >= self.min_match_id { + if self.find_match(cache, input, at, sid, slots, &mut pid) { + if input.get_earliest() + || (leftmost_first && trans.match_wins()) + { + return Ok(pid); + } + } + } + if sid == DEAD + || (!epsilons.looks().is_empty() + && !self.nfa.look_matcher().matches_set_inline( + epsilons.looks(), + input.haystack(), + at, + )) + { + return Ok(pid); + } + epsilons.slots().apply(at, cache.explicit_slots()); + } + if next_sid >= self.min_match_id { + self.find_match( + cache, + input, + input.end(), + next_sid, + slots, + &mut pid, + ); + } + Ok(pid) + } + + /// Assumes 'sid' is a match state and looks for whether a match can + /// be reported. If so, appropriate offsets are written to 'slots' and + /// 'matched_pid' is set to the matching pattern ID. + /// + /// Even when 'sid' is a match state, it's possible that a match won't + /// be reported. For example, when the conditional epsilon transitions + /// leading to the match state aren't satisfied at the given position in + /// the haystack. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn find_match( + &self, + cache: &mut Cache, + input: &Input<'_>, + at: usize, + sid: StateID, + slots: &mut [Option<NonMaxUsize>], + matched_pid: &mut Option<PatternID>, + ) -> bool { + debug_assert!(sid >= self.min_match_id); + let pateps = self.pattern_epsilons(sid); + let epsilons = pateps.epsilons(); + if !epsilons.looks().is_empty() + && !self.nfa.look_matcher().matches_set_inline( + epsilons.looks(), + input.haystack(), + at, + ) + { + return false; + } + let pid = pateps.pattern_id_unchecked(); + // This calculation is always correct because we know our 'pid' is + // valid and thus we know that the slot indices for it are valid. + let slot_end = pid.as_usize().wrapping_mul(2).wrapping_add(1); + // Set the implicit 'end' slot for the matching pattern. (The 'start' + // slot was set at the beginning of the search.) + if slot_end < slots.len() { + slots[slot_end] = NonMaxUsize::new(at); + } + // If the caller provided enough room, copy the previously recorded + // explicit slots from our scratch space to the caller provided slots. + // We *also* need to set any explicit slots that are active as part of + // the path to the match state. + if self.explicit_slot_start < slots.len() { + // NOTE: The 'cache.explicit_slots()' slice is setup at the + // beginning of every search such that it is guaranteed + // to return a slice that is at most equal in length to + // 'slots[explicit_slot_start..]'. It may be smaller in + // cases where the caller provided more slots than there + // are in the compiled regex. In which case, we limit the + // length of `slots` to what we actually have. + let cache_slots = cache.explicit_slots(); + slots[self.explicit_slot_start..][..cache_slots.len()] + .copy_from_slice(cache_slots); + epsilons.slots().apply(at, &mut slots[self.explicit_slot_start..]); + } + *matched_pid = Some(pid); + true + } +} + +impl DFA { + /// Returns the anchored start state for matching any pattern in this DFA. + fn start(&self) -> StateID { + self.starts[0] + } + + /// Returns the anchored start state for matching the given pattern. If + /// 'starts_for_each_pattern' + /// was not enabled, then this returns an error. If the given pattern is + /// not in this DFA, then `Ok(None)` is returned. + fn start_pattern(&self, pid: PatternID) -> Result<StateID, MatchError> { + if !self.config.get_starts_for_each_pattern() { + return Err(MatchError::unsupported_anchored(Anchored::Pattern( + pid, + ))); + } + // 'starts' always has non-zero length. The first entry is always the + // anchored starting state for all patterns, and the following entries + // are optional and correspond to the anchored starting states for + // patterns at pid+1. Thus, starts.len()-1 corresponds to the total + // number of patterns that one can explicitly search for. (And it may + // be zero.) + Ok(self.starts.get(pid.one_more()).copied().unwrap_or(DEAD)) + } + + /// Returns the transition from the given state ID and byte of input. The + /// transition includes the next state ID, the slots that should be saved + /// and any conditional epsilon transitions that must be satisfied in order + /// to take this transition. + fn transition(&self, sid: StateID, byte: u8) -> Transition { + let offset = sid.as_usize() << self.stride2(); + let class = self.classes.get(byte).as_usize(); + self.table[offset + class] + } + + /// Set the transition from the given state ID and byte of input to the + /// transition given. + fn set_transition(&mut self, sid: StateID, byte: u8, to: Transition) { + let offset = sid.as_usize() << self.stride2(); + let class = self.classes.get(byte).as_usize(); + self.table[offset + class] = to; + } + + /// Return an iterator of "sparse" transitions for the given state ID. + /// "sparse" in this context means that consecutive transitions that are + /// equivalent are returned as one group, and transitions to the DEAD state + /// are ignored. + /// + /// This winds up being useful for debug printing, since it's much terser + /// to display runs of equivalent transitions than the transition for every + /// possible byte value. Indeed, in practice, it's very common for runs + /// of equivalent transitions to appear. + fn sparse_transitions(&self, sid: StateID) -> SparseTransitionIter<'_> { + let start = sid.as_usize() << self.stride2(); + let end = start + self.alphabet_len(); + SparseTransitionIter { + it: self.table[start..end].iter().enumerate(), + cur: None, + } + } + + /// Return the pattern epsilons for the given state ID. + /// + /// If the given state ID does not correspond to a match state ID, then the + /// pattern epsilons returned is empty. + fn pattern_epsilons(&self, sid: StateID) -> PatternEpsilons { + let offset = sid.as_usize() << self.stride2(); + PatternEpsilons(self.table[offset + self.pateps_offset].0) + } + + /// Set the pattern epsilons for the given state ID. + fn set_pattern_epsilons(&mut self, sid: StateID, pateps: PatternEpsilons) { + let offset = sid.as_usize() << self.stride2(); + self.table[offset + self.pateps_offset] = Transition(pateps.0); + } + + /// Returns the state ID prior to the one given. This returns None if the + /// given ID is the first DFA state. + fn prev_state_id(&self, id: StateID) -> Option<StateID> { + if id == DEAD { + None + } else { + // CORRECTNESS: Since 'id' is not the first state, subtracting 1 + // is always valid. + Some(StateID::new_unchecked(id.as_usize().checked_sub(1).unwrap())) + } + } + + /// Returns the state ID of the last state in this DFA's transition table. + /// "last" in this context means the last state to appear in memory, i.e., + /// the one with the greatest ID. + fn last_state_id(&self) -> StateID { + // CORRECTNESS: A DFA table is always non-empty since it always at + // least contains a DEAD state. Since every state has the same stride, + // we can just compute what the "next" state ID would have been and + // then subtract 1 from it. + StateID::new_unchecked( + (self.table.len() >> self.stride2()).checked_sub(1).unwrap(), + ) + } + + /// Move the transitions from 'id1' to 'id2' and vice versa. + /// + /// WARNING: This does not update the rest of the transition table to have + /// transitions to 'id1' changed to 'id2' and vice versa. This merely moves + /// the states in memory. + pub(super) fn swap_states(&mut self, id1: StateID, id2: StateID) { + let o1 = id1.as_usize() << self.stride2(); + let o2 = id2.as_usize() << self.stride2(); + for b in 0..self.stride() { + self.table.swap(o1 + b, o2 + b); + } + } + + /// Map all state IDs in this DFA (transition table + start states) + /// according to the closure given. + pub(super) fn remap(&mut self, map: impl Fn(StateID) -> StateID) { + for i in 0..self.state_len() { + let offset = i << self.stride2(); + for b in 0..self.alphabet_len() { + let next = self.table[offset + b].state_id(); + self.table[offset + b].set_state_id(map(next)); + } + } + for i in 0..self.starts.len() { + self.starts[i] = map(self.starts[i]); + } + } +} + +impl core::fmt::Debug for DFA { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + fn debug_state_transitions( + f: &mut core::fmt::Formatter, + dfa: &DFA, + sid: StateID, + ) -> core::fmt::Result { + for (i, (start, end, trans)) in + dfa.sparse_transitions(sid).enumerate() + { + let next = trans.state_id(); + if i > 0 { + write!(f, ", ")?; + } + if start == end { + write!( + f, + "{:?} => {:?}", + DebugByte(start), + next.as_usize(), + )?; + } else { + write!( + f, + "{:?}-{:?} => {:?}", + DebugByte(start), + DebugByte(end), + next.as_usize(), + )?; + } + if trans.match_wins() { + write!(f, " (MW)")?; + } + if !trans.epsilons().is_empty() { + write!(f, " ({:?})", trans.epsilons())?; + } + } + Ok(()) + } + + writeln!(f, "onepass::DFA(")?; + for index in 0..self.state_len() { + let sid = StateID::must(index); + let pateps = self.pattern_epsilons(sid); + if sid == DEAD { + write!(f, "D ")?; + } else if pateps.pattern_id().is_some() { + write!(f, "* ")?; + } else { + write!(f, " ")?; + } + write!(f, "{:06?}", sid.as_usize())?; + if !pateps.is_empty() { + write!(f, " ({pateps:?})")?; + } + write!(f, ": ")?; + debug_state_transitions(f, self, sid)?; + write!(f, "\n")?; + } + writeln!(f, "")?; + for (i, &sid) in self.starts.iter().enumerate() { + if i == 0 { + writeln!(f, "START(ALL): {:?}", sid.as_usize())?; + } else { + writeln!( + f, + "START(pattern: {:?}): {:?}", + i - 1, + sid.as_usize(), + )?; + } + } + writeln!(f, "state length: {:?}", self.state_len())?; + writeln!(f, "pattern length: {:?}", self.pattern_len())?; + writeln!(f, ")")?; + Ok(()) + } +} + +/// An iterator over groups of consecutive equivalent transitions in a single +/// state. +#[derive(Debug)] +struct SparseTransitionIter<'a> { + it: core::iter::Enumerate<core::slice::Iter<'a, Transition>>, + cur: Option<(u8, u8, Transition)>, +} + +impl<'a> Iterator for SparseTransitionIter<'a> { + type Item = (u8, u8, Transition); + + fn next(&mut self) -> Option<(u8, u8, Transition)> { + while let Some((b, &trans)) = self.it.next() { + // Fine because we'll never have more than u8::MAX transitions in + // one state. + let b = b.as_u8(); + let (prev_start, prev_end, prev_trans) = match self.cur { + Some(t) => t, + None => { + self.cur = Some((b, b, trans)); + continue; + } + }; + if prev_trans == trans { + self.cur = Some((prev_start, b, prev_trans)); + } else { + self.cur = Some((b, b, trans)); + if prev_trans.state_id() != DEAD { + return Some((prev_start, prev_end, prev_trans)); + } + } + } + if let Some((start, end, trans)) = self.cur.take() { + if trans.state_id() != DEAD { + return Some((start, end, trans)); + } + } + None + } +} + +/// A cache represents mutable state that a one-pass [`DFA`] requires during a +/// search. +/// +/// For a given one-pass DFA, its corresponding cache may be created either via +/// [`DFA::create_cache`], or via [`Cache::new`]. They are equivalent in every +/// way, except the former does not require explicitly importing `Cache`. +/// +/// A particular `Cache` is coupled with the one-pass DFA from which it was +/// created. It may only be used with that one-pass DFA. A cache and its +/// allocations may be re-purposed via [`Cache::reset`], in which case, it can +/// only be used with the new one-pass DFA (and not the old one). +#[derive(Clone, Debug)] +pub struct Cache { + /// Scratch space used to store slots during a search. Basically, we use + /// the caller provided slots to store slots known when a match occurs. + /// But after a match occurs, we might continue a search but ultimately + /// fail to extend the match. When continuing the search, we need some + /// place to store candidate capture offsets without overwriting the slot + /// offsets recorded for the most recently seen match. + explicit_slots: Vec<Option<NonMaxUsize>>, + /// The number of slots in the caller-provided 'Captures' value for the + /// current search. This is always at most 'explicit_slots.len()', but + /// might be less than it, if the caller provided fewer slots to fill. + explicit_slot_len: usize, +} + +impl Cache { + /// Create a new [`onepass::DFA`](DFA) cache. + /// + /// A potentially more convenient routine to create a cache is + /// [`DFA::create_cache`], as it does not require also importing the + /// `Cache` type. + /// + /// If you want to reuse the returned `Cache` with some other one-pass DFA, + /// then you must call [`Cache::reset`] with the desired one-pass DFA. + pub fn new(re: &DFA) -> Cache { + let mut cache = Cache { explicit_slots: vec![], explicit_slot_len: 0 }; + cache.reset(re); + cache + } + + /// Reset this cache such that it can be used for searching with a + /// different [`onepass::DFA`](DFA). + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different one-pass DFA. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different one-pass + /// DFA. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::onepass::DFA, Match}; + /// + /// let re1 = DFA::new(r"\w")?; + /// let re2 = DFA::new(r"\W")?; + /// let mut caps1 = re1.create_captures(); + /// let mut caps2 = re2.create_captures(); + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 0..2)), + /// { re1.captures(&mut cache, "Δ", &mut caps1); caps1.get_match() }, + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the one-pass DFA we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// re2.reset_cache(&mut cache); + /// assert_eq!( + /// Some(Match::must(0, 0..3)), + /// { re2.captures(&mut cache, "☃", &mut caps2); caps2.get_match() }, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset(&mut self, re: &DFA) { + let explicit_slot_len = re.get_nfa().group_info().explicit_slot_len(); + self.explicit_slots.resize(explicit_slot_len, None); + self.explicit_slot_len = explicit_slot_len; + } + + /// Returns the heap memory usage, in bytes, of this cache. + /// + /// This does **not** include the stack size used up by this cache. To + /// compute that, use `std::mem::size_of::<Cache>()`. + pub fn memory_usage(&self) -> usize { + self.explicit_slots.len() * core::mem::size_of::<Option<NonMaxUsize>>() + } + + fn explicit_slots(&mut self) -> &mut [Option<NonMaxUsize>] { + &mut self.explicit_slots[..self.explicit_slot_len] + } + + fn setup_search(&mut self, explicit_slot_len: usize) { + self.explicit_slot_len = explicit_slot_len; + } +} + +/// Represents a single transition in a one-pass DFA. +/// +/// The high 21 bits corresponds to the state ID. The bit following corresponds +/// to the special "match wins" flag. The remaining low 42 bits corresponds to +/// the transition epsilons, which contains the slots that should be saved when +/// this transition is followed and the conditional epsilon transitions that +/// must be satisfied in order to follow this transition. +#[derive(Clone, Copy, Eq, PartialEq)] +struct Transition(u64); + +impl Transition { + const STATE_ID_BITS: u64 = 21; + const STATE_ID_SHIFT: u64 = 64 - Transition::STATE_ID_BITS; + const STATE_ID_LIMIT: u64 = 1 << Transition::STATE_ID_BITS; + const MATCH_WINS_SHIFT: u64 = 64 - (Transition::STATE_ID_BITS + 1); + const INFO_MASK: u64 = 0x000003FF_FFFFFFFF; + + /// Return a new transition to the given state ID with the given epsilons. + fn new(match_wins: bool, sid: StateID, epsilons: Epsilons) -> Transition { + let match_wins = + if match_wins { 1 << Transition::MATCH_WINS_SHIFT } else { 0 }; + let sid = sid.as_u64() << Transition::STATE_ID_SHIFT; + Transition(sid | match_wins | epsilons.0) + } + + /// Returns true if and only if this transition points to the DEAD state. + fn is_dead(self) -> bool { + self.state_id() == DEAD + } + + /// Return whether this transition has a "match wins" property. + /// + /// When a transition has this property, it means that if a match has been + /// found and the search uses leftmost-first semantics, then that match + /// should be returned immediately instead of continuing on. + /// + /// The "match wins" name comes from RE2, which uses a pretty much + /// identical mechanism for implementing leftmost-first semantics. + fn match_wins(&self) -> bool { + (self.0 >> Transition::MATCH_WINS_SHIFT & 1) == 1 + } + + /// Return the "next" state ID that this transition points to. + fn state_id(&self) -> StateID { + // OK because a Transition has a valid StateID in its upper bits by + // construction. The cast to usize is also correct, even on 16-bit + // targets because, again, we know the upper bits is a valid StateID, + // which can never overflow usize on any supported target. + StateID::new_unchecked( + (self.0 >> Transition::STATE_ID_SHIFT).as_usize(), + ) + } + + /// Set the "next" state ID in this transition. + fn set_state_id(&mut self, sid: StateID) { + *self = Transition::new(self.match_wins(), sid, self.epsilons()); + } + + /// Return the epsilons embedded in this transition. + fn epsilons(&self) -> Epsilons { + Epsilons(self.0 & Transition::INFO_MASK) + } +} + +impl core::fmt::Debug for Transition { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + if self.is_dead() { + return write!(f, "0"); + } + write!(f, "{}", self.state_id().as_usize())?; + if self.match_wins() { + write!(f, "-MW")?; + } + if !self.epsilons().is_empty() { + write!(f, "-{:?}", self.epsilons())?; + } + Ok(()) + } +} + +/// A representation of a match state's pattern ID along with the epsilons for +/// when a match occurs. +/// +/// A match state in a one-pass DFA, unlike in a more general DFA, has exactly +/// one pattern ID. If it had more, then the original NFA would not have been +/// one-pass. +/// +/// The "epsilons" part of this corresponds to what was found in the epsilon +/// transitions between the transition taken in the last byte of input and the +/// ultimate match state. This might include saving slots and/or conditional +/// epsilon transitions that must be satisfied before one can report the match. +/// +/// Technically, every state has room for a 'PatternEpsilons', but it is only +/// ever non-empty for match states. +#[derive(Clone, Copy)] +struct PatternEpsilons(u64); + +impl PatternEpsilons { + const PATTERN_ID_BITS: u64 = 22; + const PATTERN_ID_SHIFT: u64 = 64 - PatternEpsilons::PATTERN_ID_BITS; + // A sentinel value indicating that this is not a match state. We don't + // use 0 since 0 is a valid pattern ID. + const PATTERN_ID_NONE: u64 = 0x00000000_003FFFFF; + const PATTERN_ID_LIMIT: u64 = PatternEpsilons::PATTERN_ID_NONE; + const PATTERN_ID_MASK: u64 = 0xFFFFFC00_00000000; + const EPSILONS_MASK: u64 = 0x000003FF_FFFFFFFF; + + /// Return a new empty pattern epsilons that has no pattern ID and has no + /// epsilons. This is suitable for non-match states. + fn empty() -> PatternEpsilons { + PatternEpsilons( + PatternEpsilons::PATTERN_ID_NONE + << PatternEpsilons::PATTERN_ID_SHIFT, + ) + } + + /// Whether this pattern epsilons is empty or not. It's empty when it has + /// no pattern ID and an empty epsilons. + fn is_empty(self) -> bool { + self.pattern_id().is_none() && self.epsilons().is_empty() + } + + /// Return the pattern ID in this pattern epsilons if one exists. + fn pattern_id(self) -> Option<PatternID> { + let pid = self.0 >> PatternEpsilons::PATTERN_ID_SHIFT; + if pid == PatternEpsilons::PATTERN_ID_LIMIT { + None + } else { + Some(PatternID::new_unchecked(pid.as_usize())) + } + } + + /// Returns the pattern ID without checking whether it's valid. If this is + /// called and there is no pattern ID in this `PatternEpsilons`, then this + /// will likely produce an incorrect result or possibly even a panic or + /// an overflow. But safety will not be violated. + /// + /// This is useful when you know a particular state is a match state. If + /// it's a match state, then it must have a pattern ID. + fn pattern_id_unchecked(self) -> PatternID { + let pid = self.0 >> PatternEpsilons::PATTERN_ID_SHIFT; + PatternID::new_unchecked(pid.as_usize()) + } + + /// Return a new pattern epsilons with the given pattern ID, but the same + /// epsilons. + fn set_pattern_id(self, pid: PatternID) -> PatternEpsilons { + PatternEpsilons( + (pid.as_u64() << PatternEpsilons::PATTERN_ID_SHIFT) + | (self.0 & PatternEpsilons::EPSILONS_MASK), + ) + } + + /// Return the epsilons part of this pattern epsilons. + fn epsilons(self) -> Epsilons { + Epsilons(self.0 & PatternEpsilons::EPSILONS_MASK) + } + + /// Return a new pattern epsilons with the given epsilons, but the same + /// pattern ID. + fn set_epsilons(self, epsilons: Epsilons) -> PatternEpsilons { + PatternEpsilons( + (self.0 & PatternEpsilons::PATTERN_ID_MASK) + | (u64::from(epsilons.0) & PatternEpsilons::EPSILONS_MASK), + ) + } +} + +impl core::fmt::Debug for PatternEpsilons { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + if self.is_empty() { + return write!(f, "N/A"); + } + if let Some(pid) = self.pattern_id() { + write!(f, "{}", pid.as_usize())?; + } + if !self.epsilons().is_empty() { + if self.pattern_id().is_some() { + write!(f, "/")?; + } + write!(f, "{:?}", self.epsilons())?; + } + Ok(()) + } +} + +/// Epsilons represents all of the NFA epsilons transitions that went into a +/// single transition in a single DFA state. In this case, it only represents +/// the epsilon transitions that have some kind of non-consuming side effect: +/// either the transition requires storing the current position of the search +/// into a slot, or the transition is conditional and requires the current +/// position in the input to satisfy an assertion before the transition may be +/// taken. +/// +/// This folds the cumulative effect of a group of NFA states (all connected +/// by epsilon transitions) down into a single set of bits. While these bits +/// can represent all possible conditional epsilon transitions, it only permits +/// storing up to a somewhat small number of slots. +/// +/// Epsilons is represented as a 42-bit integer. For example, it is packed into +/// the lower 42 bits of a `Transition`. (Where the high 22 bits contains a +/// `StateID` and a special "match wins" property.) +#[derive(Clone, Copy)] +struct Epsilons(u64); + +impl Epsilons { + const SLOT_MASK: u64 = 0x000003FF_FFFFFC00; + const SLOT_SHIFT: u64 = 10; + const LOOK_MASK: u64 = 0x00000000_000003FF; + + /// Create a new empty epsilons. It has no slots and no assertions that + /// need to be satisfied. + fn empty() -> Epsilons { + Epsilons(0) + } + + /// Returns true if this epsilons contains no slots and no assertions. + fn is_empty(self) -> bool { + self.0 == 0 + } + + /// Returns the slot epsilon transitions. + fn slots(self) -> Slots { + Slots((self.0 >> Epsilons::SLOT_SHIFT).low_u32()) + } + + /// Set the slot epsilon transitions. + fn set_slots(self, slots: Slots) -> Epsilons { + Epsilons( + (u64::from(slots.0) << Epsilons::SLOT_SHIFT) + | (self.0 & Epsilons::LOOK_MASK), + ) + } + + /// Return the set of look-around assertions in these epsilon transitions. + fn looks(self) -> LookSet { + LookSet { bits: (self.0 & Epsilons::LOOK_MASK).low_u32() } + } + + /// Set the look-around assertions on these epsilon transitions. + fn set_looks(self, look_set: LookSet) -> Epsilons { + Epsilons( + (self.0 & Epsilons::SLOT_MASK) + | (u64::from(look_set.bits) & Epsilons::LOOK_MASK), + ) + } +} + +impl core::fmt::Debug for Epsilons { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut wrote = false; + if !self.slots().is_empty() { + write!(f, "{:?}", self.slots())?; + wrote = true; + } + if !self.looks().is_empty() { + if wrote { + write!(f, "/")?; + } + write!(f, "{:?}", self.looks())?; + wrote = true; + } + if !wrote { + write!(f, "N/A")?; + } + Ok(()) + } +} + +/// The set of epsilon transitions indicating that the current position in a +/// search should be saved to a slot. +/// +/// This *only* represents explicit slots. So for example, the pattern +/// `[a-z]+([0-9]+)([a-z]+)` has: +/// +/// * 3 capturing groups, thus 6 slots. +/// * 1 implicit capturing group, thus 2 implicit slots. +/// * 2 explicit capturing groups, thus 4 explicit slots. +/// +/// While implicit slots are represented by epsilon transitions in an NFA, we +/// do not explicitly represent them here. Instead, implicit slots are assumed +/// to be present and handled automatically in the search code. Therefore, +/// that means we only need to represent explicit slots in our epsilon +/// transitions. +/// +/// Its representation is a bit set. The bit 'i' is set if and only if there +/// exists an explicit slot at index 'c', where 'c = (#patterns * 2) + i'. That +/// is, the bit 'i' corresponds to the first explicit slot and the first +/// explicit slot appears immediately following the last implicit slot. (If +/// this is confusing, see `GroupInfo` for more details on how slots works.) +/// +/// A single `Slots` represents all the active slots in a sub-graph of an NFA, +/// where all the states are connected by epsilon transitions. In effect, when +/// traversing the one-pass DFA during a search, all slots set in a particular +/// transition must be captured by recording the current search position. +/// +/// The API of `Slots` requires the caller to handle the explicit slot offset. +/// That is, a `Slots` doesn't know where the explicit slots start for a +/// particular NFA. Thus, if the callers see's the bit 'i' is set, then they +/// need to do the arithmetic above to find 'c', which is the real actual slot +/// index in the corresponding NFA. +#[derive(Clone, Copy)] +struct Slots(u32); + +impl Slots { + const LIMIT: usize = 32; + + /// Insert the slot at the given bit index. + fn insert(self, slot: usize) -> Slots { + debug_assert!(slot < Slots::LIMIT); + Slots(self.0 | (1 << slot.as_u32())) + } + + /// Remove the slot at the given bit index. + fn remove(self, slot: usize) -> Slots { + debug_assert!(slot < Slots::LIMIT); + Slots(self.0 & !(1 << slot.as_u32())) + } + + /// Returns true if and only if this set contains no slots. + fn is_empty(self) -> bool { + self.0 == 0 + } + + /// Returns an iterator over all of the set bits in this set. + fn iter(self) -> SlotsIter { + SlotsIter { slots: self } + } + + /// For the position `at` in the current haystack, copy it to + /// `caller_explicit_slots` for all slots that are in this set. + /// + /// Callers may pass a slice of any length. Slots in this set bigger than + /// the length of the given explicit slots are simply skipped. + /// + /// The slice *must* correspond only to the explicit slots and the first + /// element of the slice must always correspond to the first explicit slot + /// in the corresponding NFA. + fn apply( + self, + at: usize, + caller_explicit_slots: &mut [Option<NonMaxUsize>], + ) { + if self.is_empty() { + return; + } + let at = NonMaxUsize::new(at); + for slot in self.iter() { + if slot >= caller_explicit_slots.len() { + break; + } + caller_explicit_slots[slot] = at; + } + } +} + +impl core::fmt::Debug for Slots { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "S")?; + for slot in self.iter() { + write!(f, "-{slot:?}")?; + } + Ok(()) + } +} + +/// An iterator over all of the bits set in a slot set. +/// +/// This returns the bit index that is set, so callers may need to offset it +/// to get the actual NFA slot index. +#[derive(Debug)] +struct SlotsIter { + slots: Slots, +} + +impl Iterator for SlotsIter { + type Item = usize; + + fn next(&mut self) -> Option<usize> { + // Number of zeroes here is always <= u8::MAX, and so fits in a usize. + let slot = self.slots.0.trailing_zeros().as_usize(); + if slot >= Slots::LIMIT { + return None; + } + self.slots = self.slots.remove(slot); + Some(slot) + } +} + +/// An error that occurred during the construction of a one-pass DFA. +/// +/// This error does not provide many introspection capabilities. There are +/// generally only two things you can do with it: +/// +/// * Obtain a human readable message via its `std::fmt::Display` impl. +/// * Access an underlying [`thompson::BuildError`] type from its `source` +/// method via the `std::error::Error` trait. This error only occurs when using +/// convenience routines for building a one-pass DFA directly from a pattern +/// string. +/// +/// When the `std` feature is enabled, this implements the `std::error::Error` +/// trait. +#[derive(Clone, Debug)] +pub struct BuildError { + kind: BuildErrorKind, +} + +/// The kind of error that occurred during the construction of a one-pass DFA. +#[derive(Clone, Debug)] +enum BuildErrorKind { + NFA(crate::nfa::thompson::BuildError), + Word(UnicodeWordBoundaryError), + TooManyStates { limit: u64 }, + TooManyPatterns { limit: u64 }, + UnsupportedLook { look: Look }, + ExceededSizeLimit { limit: usize }, + NotOnePass { msg: &'static str }, +} + +impl BuildError { + fn nfa(err: crate::nfa::thompson::BuildError) -> BuildError { + BuildError { kind: BuildErrorKind::NFA(err) } + } + + fn word(err: UnicodeWordBoundaryError) -> BuildError { + BuildError { kind: BuildErrorKind::Word(err) } + } + + fn too_many_states(limit: u64) -> BuildError { + BuildError { kind: BuildErrorKind::TooManyStates { limit } } + } + + fn too_many_patterns(limit: u64) -> BuildError { + BuildError { kind: BuildErrorKind::TooManyPatterns { limit } } + } + + fn unsupported_look(look: Look) -> BuildError { + BuildError { kind: BuildErrorKind::UnsupportedLook { look } } + } + + fn exceeded_size_limit(limit: usize) -> BuildError { + BuildError { kind: BuildErrorKind::ExceededSizeLimit { limit } } + } + + fn not_one_pass(msg: &'static str) -> BuildError { + BuildError { kind: BuildErrorKind::NotOnePass { msg } } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for BuildError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + use self::BuildErrorKind::*; + + match self.kind { + NFA(ref err) => Some(err), + Word(ref err) => Some(err), + _ => None, + } + } +} + +impl core::fmt::Display for BuildError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + use self::BuildErrorKind::*; + + match self.kind { + NFA(_) => write!(f, "error building NFA"), + Word(_) => write!(f, "NFA contains Unicode word boundary"), + TooManyStates { limit } => write!( + f, + "one-pass DFA exceeded a limit of {limit:?} \ + for number of states", + ), + TooManyPatterns { limit } => write!( + f, + "one-pass DFA exceeded a limit of {limit:?} \ + for number of patterns", + ), + UnsupportedLook { look } => write!( + f, + "one-pass DFA does not support the {look:?} assertion", + ), + ExceededSizeLimit { limit } => write!( + f, + "one-pass DFA exceeded size limit of {limit:?} during building", + ), + NotOnePass { msg } => write!( + f, + "one-pass DFA could not be built because \ + pattern is not one-pass: {}", + msg, + ), + } + } +} + +#[cfg(all(test, feature = "syntax"))] +mod tests { + use alloc::string::ToString; + + use super::*; + + #[test] + fn fail_conflicting_transition() { + let predicate = |err: &str| err.contains("conflicting transition"); + + let err = DFA::new(r"a*[ab]").unwrap_err().to_string(); + assert!(predicate(&err), "{err}"); + } + + #[test] + fn fail_multiple_epsilon() { + let predicate = |err: &str| { + err.contains("multiple epsilon transitions to same state") + }; + + let err = DFA::new(r"(^|$)a").unwrap_err().to_string(); + assert!(predicate(&err), "{err}"); + } + + #[test] + fn fail_multiple_match() { + let predicate = |err: &str| { + err.contains("multiple epsilon transitions to match state") + }; + + let err = DFA::new_many(&[r"^", r"$"]).unwrap_err().to_string(); + assert!(predicate(&err), "{err}"); + } + + // This test is meant to build a one-pass regex with the maximum number of + // possible slots. + // + // NOTE: Remember that the slot limit only applies to explicit capturing + // groups. Any number of implicit capturing groups is supported (up to the + // maximum number of supported patterns), since implicit groups are handled + // by the search loop itself. + #[test] + fn max_slots() { + // One too many... + let pat = r"(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)(m)(n)(o)(p)(q)"; + assert!(DFA::new(pat).is_err()); + // Just right. + let pat = r"(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)(m)(n)(o)(p)"; + assert!(DFA::new(pat).is_ok()); + } + + // This test ensures that the one-pass DFA works with all look-around + // assertions that we expect it to work with. + // + // The utility of this test is that each one-pass transition has a small + // amount of space to store look-around assertions. Currently, there is + // logic in the one-pass constructor to ensure there aren't more than ten + // possible assertions. And indeed, there are only ten possible assertions + // (at time of writing), so this is okay. But conceivably, more assertions + // could be added. So we check that things at least work with what we + // expect them to work with. + #[test] + fn assertions() { + // haystack anchors + assert!(DFA::new(r"^").is_ok()); + assert!(DFA::new(r"$").is_ok()); + + // line anchors + assert!(DFA::new(r"(?m)^").is_ok()); + assert!(DFA::new(r"(?m)$").is_ok()); + assert!(DFA::new(r"(?Rm)^").is_ok()); + assert!(DFA::new(r"(?Rm)$").is_ok()); + + // word boundaries + if cfg!(feature = "unicode-word-boundary") { + assert!(DFA::new(r"\b").is_ok()); + assert!(DFA::new(r"\B").is_ok()); + } + assert!(DFA::new(r"(?-u)\b").is_ok()); + assert!(DFA::new(r"(?-u)\B").is_ok()); + } + + #[cfg(not(miri))] // takes too long on miri + #[test] + fn is_one_pass() { + use crate::util::syntax; + + assert!(DFA::new(r"a*b").is_ok()); + if cfg!(feature = "unicode-perl") { + assert!(DFA::new(r"\w").is_ok()); + } + assert!(DFA::new(r"(?-u)\w*\s").is_ok()); + assert!(DFA::new(r"(?s:.)*?").is_ok()); + assert!(DFA::builder() + .syntax(syntax::Config::new().utf8(false)) + .build(r"(?s-u:.)*?") + .is_ok()); + } + + #[test] + fn is_not_one_pass() { + assert!(DFA::new(r"a*a").is_err()); + assert!(DFA::new(r"(?s-u:.)*?").is_err()); + assert!(DFA::new(r"(?s:.)*?a").is_err()); + } + + #[cfg(not(miri))] + #[test] + fn is_not_one_pass_bigger() { + assert!(DFA::new(r"\w*\s").is_err()); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/regex.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/regex.rs new file mode 100644 index 0000000..892c442 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/regex.rs @@ -0,0 +1,870 @@ +/*! +A DFA-backed `Regex`. + +This module provides [`Regex`], which is defined generically over the +[`Automaton`] trait. A `Regex` implements convenience routines you might have +come to expect, such as finding the start/end of a match and iterating over +all non-overlapping matches. This `Regex` type is limited in its capabilities +to what a DFA can provide. Therefore, APIs involving capturing groups, for +example, are not provided. + +Internally, a `Regex` is composed of two DFAs. One is a "forward" DFA that +finds the end offset of a match, where as the other is a "reverse" DFA that +find the start offset of a match. + +See the [parent module](crate::dfa) for examples. +*/ + +#[cfg(feature = "alloc")] +use alloc::vec::Vec; + +#[cfg(feature = "dfa-build")] +use crate::dfa::dense::BuildError; +use crate::{ + dfa::{automaton::Automaton, dense}, + util::{iter, search::Input}, + Anchored, Match, MatchError, +}; +#[cfg(feature = "alloc")] +use crate::{ + dfa::{sparse, StartKind}, + util::search::MatchKind, +}; + +// When the alloc feature is enabled, the regex type sets its A type parameter +// to default to an owned dense DFA. But without alloc, we set no default. This +// makes things a lot more convenient in the common case, since writing out the +// DFA types is pretty annoying. +// +// Since we have two different definitions but only want to write one doc +// string, we use a macro to capture the doc and other attributes once and then +// repeat them for each definition. +macro_rules! define_regex_type { + ($(#[$doc:meta])*) => { + #[cfg(feature = "alloc")] + $(#[$doc])* + pub struct Regex<A = dense::OwnedDFA> { + forward: A, + reverse: A, + } + + #[cfg(not(feature = "alloc"))] + $(#[$doc])* + pub struct Regex<A> { + forward: A, + reverse: A, + } + }; +} + +define_regex_type!( + /// A regular expression that uses deterministic finite automata for fast + /// searching. + /// + /// A regular expression is comprised of two DFAs, a "forward" DFA and a + /// "reverse" DFA. The forward DFA is responsible for detecting the end of + /// a match while the reverse DFA is responsible for detecting the start + /// of a match. Thus, in order to find the bounds of any given match, a + /// forward search must first be run followed by a reverse search. A match + /// found by the forward DFA guarantees that the reverse DFA will also find + /// a match. + /// + /// The type of the DFA used by a `Regex` corresponds to the `A` type + /// parameter, which must satisfy the [`Automaton`] trait. Typically, `A` + /// is either a [`dense::DFA`] or a [`sparse::DFA`], where dense DFAs use + /// more memory but search faster, while sparse DFAs use less memory but + /// search more slowly. + /// + /// # Crate features + /// + /// Note that despite what the documentation auto-generates, the _only_ + /// crate feature needed to use this type is `dfa-search`. You do _not_ + /// need to enable the `alloc` feature. + /// + /// By default, a regex's automaton type parameter is set to + /// `dense::DFA<Vec<u32>>` when the `alloc` feature is enabled. For most + /// in-memory work loads, this is the most convenient type that gives the + /// best search performance. When the `alloc` feature is disabled, no + /// default type is used. + /// + /// # When should I use this? + /// + /// Generally speaking, if you can afford the overhead of building a full + /// DFA for your regex, and you don't need things like capturing groups, + /// then this is a good choice if you're looking to optimize for matching + /// speed. Note however that its speed may be worse than a general purpose + /// regex engine if you don't provide a [`dense::Config::prefilter`] to the + /// underlying DFA. + /// + /// # Sparse DFAs + /// + /// Since a `Regex` is generic over the [`Automaton`] trait, it can be + /// used with any kind of DFA. While this crate constructs dense DFAs by + /// default, it is easy enough to build corresponding sparse DFAs, and then + /// build a regex from them: + /// + /// ``` + /// use regex_automata::dfa::regex::Regex; + /// + /// // First, build a regex that uses dense DFAs. + /// let dense_re = Regex::new("foo[0-9]+")?; + /// + /// // Second, build sparse DFAs from the forward and reverse dense DFAs. + /// let fwd = dense_re.forward().to_sparse()?; + /// let rev = dense_re.reverse().to_sparse()?; + /// + /// // Third, build a new regex from the constituent sparse DFAs. + /// let sparse_re = Regex::builder().build_from_dfas(fwd, rev); + /// + /// // A regex that uses sparse DFAs can be used just like with dense DFAs. + /// assert_eq!(true, sparse_re.is_match(b"foo123")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Alternatively, one can use a [`Builder`] to construct a sparse DFA + /// more succinctly. (Note though that dense DFAs are still constructed + /// first internally, and then converted to sparse DFAs, as in the example + /// above.) + /// + /// ``` + /// use regex_automata::dfa::regex::Regex; + /// + /// let sparse_re = Regex::builder().build_sparse(r"foo[0-9]+")?; + /// // A regex that uses sparse DFAs can be used just like with dense DFAs. + /// assert!(sparse_re.is_match(b"foo123")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Fallibility + /// + /// Most of the search routines defined on this type will _panic_ when the + /// underlying search fails. This might be because the DFA gave up because + /// it saw a quit byte, whether configured explicitly or via heuristic + /// Unicode word boundary support, although neither are enabled by default. + /// Or it might fail because an invalid `Input` configuration is given, + /// for example, with an unsupported [`Anchored`] mode. + /// + /// If you need to handle these error cases instead of allowing them to + /// trigger a panic, then the lower level [`Regex::try_search`] provides + /// a fallible API that never panics. + /// + /// # Example + /// + /// This example shows how to cause a search to terminate if it sees a + /// `\n` byte, and handle the error returned. This could be useful if, for + /// example, you wanted to prevent a user supplied pattern from matching + /// across a line boundary. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{dfa::{self, regex::Regex}, Input, MatchError}; + /// + /// let re = Regex::builder() + /// .dense(dfa::dense::Config::new().quit(b'\n', true)) + /// .build(r"foo\p{any}+bar")?; + /// + /// let input = Input::new("foo\nbar"); + /// // Normally this would produce a match, since \p{any} contains '\n'. + /// // But since we instructed the automaton to enter a quit state if a + /// // '\n' is observed, this produces a match error instead. + /// let expected = MatchError::quit(b'\n', 3); + /// let got = re.try_search(&input).unwrap_err(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[derive(Clone, Debug)] +); + +#[cfg(all(feature = "syntax", feature = "dfa-build"))] +impl Regex { + /// Parse the given regular expression using the default configuration and + /// return the corresponding regex. + /// + /// If you want a non-default configuration, then use the [`Builder`] to + /// set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Match, dfa::regex::Regex}; + /// + /// let re = Regex::new("foo[0-9]+bar")?; + /// assert_eq!( + /// Some(Match::must(0, 3..14)), + /// re.find(b"zzzfoo12345barzzz"), + /// ); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new(pattern: &str) -> Result<Regex, BuildError> { + Builder::new().build(pattern) + } + + /// Like `new`, but parses multiple patterns into a single "regex set." + /// This similarly uses the default regex configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Match, dfa::regex::Regex}; + /// + /// let re = Regex::new_many(&["[a-z]+", "[0-9]+"])?; + /// + /// let mut it = re.find_iter(b"abc 1 foo 4567 0 quux"); + /// assert_eq!(Some(Match::must(0, 0..3)), it.next()); + /// assert_eq!(Some(Match::must(1, 4..5)), it.next()); + /// assert_eq!(Some(Match::must(0, 6..9)), it.next()); + /// assert_eq!(Some(Match::must(1, 10..14)), it.next()); + /// assert_eq!(Some(Match::must(1, 15..16)), it.next()); + /// assert_eq!(Some(Match::must(0, 17..21)), it.next()); + /// assert_eq!(None, it.next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new_many<P: AsRef<str>>( + patterns: &[P], + ) -> Result<Regex, BuildError> { + Builder::new().build_many(patterns) + } +} + +#[cfg(all(feature = "syntax", feature = "dfa-build"))] +impl Regex<sparse::DFA<Vec<u8>>> { + /// Parse the given regular expression using the default configuration, + /// except using sparse DFAs, and return the corresponding regex. + /// + /// If you want a non-default configuration, then use the [`Builder`] to + /// set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Match, dfa::regex::Regex}; + /// + /// let re = Regex::new_sparse("foo[0-9]+bar")?; + /// assert_eq!( + /// Some(Match::must(0, 3..14)), + /// re.find(b"zzzfoo12345barzzz"), + /// ); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new_sparse( + pattern: &str, + ) -> Result<Regex<sparse::DFA<Vec<u8>>>, BuildError> { + Builder::new().build_sparse(pattern) + } + + /// Like `new`, but parses multiple patterns into a single "regex set" + /// using sparse DFAs. This otherwise similarly uses the default regex + /// configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Match, dfa::regex::Regex}; + /// + /// let re = Regex::new_many_sparse(&["[a-z]+", "[0-9]+"])?; + /// + /// let mut it = re.find_iter(b"abc 1 foo 4567 0 quux"); + /// assert_eq!(Some(Match::must(0, 0..3)), it.next()); + /// assert_eq!(Some(Match::must(1, 4..5)), it.next()); + /// assert_eq!(Some(Match::must(0, 6..9)), it.next()); + /// assert_eq!(Some(Match::must(1, 10..14)), it.next()); + /// assert_eq!(Some(Match::must(1, 15..16)), it.next()); + /// assert_eq!(Some(Match::must(0, 17..21)), it.next()); + /// assert_eq!(None, it.next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new_many_sparse<P: AsRef<str>>( + patterns: &[P], + ) -> Result<Regex<sparse::DFA<Vec<u8>>>, BuildError> { + Builder::new().build_many_sparse(patterns) + } +} + +/// Convenience routines for regex construction. +impl Regex<dense::DFA<&'static [u32]>> { + /// Return a builder for configuring the construction of a `Regex`. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Builder`] type in common cases. + /// + /// # Example + /// + /// This example shows how to use the builder to disable UTF-8 mode + /// everywhere. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// dfa::regex::Regex, nfa::thompson, util::syntax, Match, + /// }; + /// + /// let re = Regex::builder() + /// .syntax(syntax::Config::new().utf8(false)) + /// .thompson(thompson::Config::new().utf8(false)) + /// .build(r"foo(?-u:[^b])ar.*")?; + /// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"; + /// let expected = Some(Match::must(0, 1..9)); + /// let got = re.find(haystack); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn builder() -> Builder { + Builder::new() + } +} + +/// Standard search routines for finding and iterating over matches. +impl<A: Automaton> Regex<A> { + /// Returns true if and only if this regex matches the given haystack. + /// + /// This routine may short circuit if it knows that scanning future input + /// will never lead to a different result. In particular, if the underlying + /// DFA enters a match state or a dead state, then this routine will return + /// `true` or `false`, respectively, without inspecting any future input. + /// + /// # Panics + /// + /// This routine panics if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the DFA quitting. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search panics, callers cannot know whether a match exists or + /// not. + /// + /// Use [`Regex::try_search`] if you want to handle these error conditions. + /// + /// # Example + /// + /// ``` + /// use regex_automata::dfa::regex::Regex; + /// + /// let re = Regex::new("foo[0-9]+bar")?; + /// assert_eq!(true, re.is_match("foo12345bar")); + /// assert_eq!(false, re.is_match("foobar")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_match<'h, I: Into<Input<'h>>>(&self, input: I) -> bool { + // Not only can we do an "earliest" search, but we can avoid doing a + // reverse scan too. + let input = input.into().earliest(true); + self.forward().try_search_fwd(&input).map(|x| x.is_some()).unwrap() + } + + /// Returns the start and end offset of the leftmost match. If no match + /// exists, then `None` is returned. + /// + /// # Panics + /// + /// This routine panics if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the DFA quitting. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search panics, callers cannot know whether a match exists or + /// not. + /// + /// Use [`Regex::try_search`] if you want to handle these error conditions. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Match, dfa::regex::Regex}; + /// + /// // Greediness is applied appropriately. + /// let re = Regex::new("foo[0-9]+")?; + /// assert_eq!(Some(Match::must(0, 3..11)), re.find("zzzfoo12345zzz")); + /// + /// // Even though a match is found after reading the first byte (`a`), + /// // the default leftmost-first match semantics demand that we find the + /// // earliest match that prefers earlier parts of the pattern over latter + /// // parts. + /// let re = Regex::new("abc|a")?; + /// assert_eq!(Some(Match::must(0, 0..3)), re.find("abc")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find<'h, I: Into<Input<'h>>>(&self, input: I) -> Option<Match> { + self.try_search(&input.into()).unwrap() + } + + /// Returns an iterator over all non-overlapping leftmost matches in the + /// given bytes. If no match exists, then the iterator yields no elements. + /// + /// This corresponds to the "standard" regex search iterator. + /// + /// # Panics + /// + /// If the search returns an error during iteration, then iteration + /// panics. See [`Regex::find`] for the panic conditions. + /// + /// Use [`Regex::try_search`] with + /// [`util::iter::Searcher`](crate::util::iter::Searcher) if you want to + /// handle these error conditions. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Match, dfa::regex::Regex}; + /// + /// let re = Regex::new("foo[0-9]+")?; + /// let text = "foo1 foo12 foo123"; + /// let matches: Vec<Match> = re.find_iter(text).collect(); + /// assert_eq!(matches, vec![ + /// Match::must(0, 0..4), + /// Match::must(0, 5..10), + /// Match::must(0, 11..17), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find_iter<'r, 'h, I: Into<Input<'h>>>( + &'r self, + input: I, + ) -> FindMatches<'r, 'h, A> { + let it = iter::Searcher::new(input.into()); + FindMatches { re: self, it } + } +} + +/// Lower level fallible search routines that permit controlling where the +/// search starts and ends in a particular sequence. +impl<A: Automaton> Regex<A> { + /// Returns the start and end offset of the leftmost match. If no match + /// exists, then `None` is returned. + /// + /// This is like [`Regex::find`] but with two differences: + /// + /// 1. It is not generic over `Into<Input>` and instead accepts a + /// `&Input`. This permits reusing the same `Input` for multiple searches + /// without needing to create a new one. This _may_ help with latency. + /// 2. It returns an error if the search could not complete where as + /// [`Regex::find`] will panic. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in the following circumstances: + /// + /// * The configuration of the DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the DFA quitting. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + #[inline] + pub fn try_search( + &self, + input: &Input<'_>, + ) -> Result<Option<Match>, MatchError> { + let (fwd, rev) = (self.forward(), self.reverse()); + let end = match fwd.try_search_fwd(input)? { + None => return Ok(None), + Some(end) => end, + }; + // This special cases an empty match at the beginning of the search. If + // our end matches our start, then since a reverse DFA can't match past + // the start, it must follow that our starting position is also our end + // position. So short circuit and skip the reverse search. + if input.start() == end.offset() { + return Ok(Some(Match::new( + end.pattern(), + end.offset()..end.offset(), + ))); + } + // We can also skip the reverse search if we know our search was + // anchored. This occurs either when the input config is anchored or + // when we know the regex itself is anchored. In this case, we know the + // start of the match, if one is found, must be the start of the + // search. + if self.is_anchored(input) { + return Ok(Some(Match::new( + end.pattern(), + input.start()..end.offset(), + ))); + } + // N.B. I have tentatively convinced myself that it isn't necessary + // to specify the specific pattern for the reverse search since the + // reverse search will always find the same pattern to match as the + // forward search. But I lack a rigorous proof. Why not just provide + // the pattern anyway? Well, if it is needed, then leaving it out + // gives us a chance to find a witness. (Also, if we don't need to + // specify the pattern, then we don't need to build the reverse DFA + // with 'starts_for_each_pattern' enabled.) + // + // We also need to be careful to disable 'earliest' for the reverse + // search, since it could be enabled for the forward search. In the + // reverse case, to satisfy "leftmost" criteria, we need to match + // as much as we can. We also need to be careful to make the search + // anchored. We don't want the reverse search to report any matches + // other than the one beginning at the end of our forward search. + let revsearch = input + .clone() + .span(input.start()..end.offset()) + .anchored(Anchored::Yes) + .earliest(false); + let start = rev + .try_search_rev(&revsearch)? + .expect("reverse search must match if forward search does"); + assert_eq!( + start.pattern(), + end.pattern(), + "forward and reverse search must match same pattern", + ); + assert!(start.offset() <= end.offset()); + Ok(Some(Match::new(end.pattern(), start.offset()..end.offset()))) + } + + /// Returns true if either the given input specifies an anchored search + /// or if the underlying DFA is always anchored. + fn is_anchored(&self, input: &Input<'_>) -> bool { + match input.get_anchored() { + Anchored::No => self.forward().is_always_start_anchored(), + Anchored::Yes | Anchored::Pattern(_) => true, + } + } +} + +/// Non-search APIs for querying information about the regex and setting a +/// prefilter. +impl<A: Automaton> Regex<A> { + /// Return the underlying DFA responsible for forward matching. + /// + /// This is useful for accessing the underlying DFA and converting it to + /// some other format or size. See the [`Builder::build_from_dfas`] docs + /// for an example of where this might be useful. + pub fn forward(&self) -> &A { + &self.forward + } + + /// Return the underlying DFA responsible for reverse matching. + /// + /// This is useful for accessing the underlying DFA and converting it to + /// some other format or size. See the [`Builder::build_from_dfas`] docs + /// for an example of where this might be useful. + pub fn reverse(&self) -> &A { + &self.reverse + } + + /// Returns the total number of patterns matched by this regex. + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::dfa::regex::Regex; + /// + /// let re = Regex::new_many(&[r"[a-z]+", r"[0-9]+", r"\w+"])?; + /// assert_eq!(3, re.pattern_len()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn pattern_len(&self) -> usize { + assert_eq!(self.forward().pattern_len(), self.reverse().pattern_len()); + self.forward().pattern_len() + } +} + +/// An iterator over all non-overlapping matches for an infallible search. +/// +/// The iterator yields a [`Match`] value until no more matches could be found. +/// If the underlying regex engine returns an error, then a panic occurs. +/// +/// The type parameters are as follows: +/// +/// * `A` represents the type of the underlying DFA that implements the +/// [`Automaton`] trait. +/// +/// The lifetime parameters are as follows: +/// +/// * `'h` represents the lifetime of the haystack being searched. +/// * `'r` represents the lifetime of the regex object itself. +/// +/// This iterator can be created with the [`Regex::find_iter`] method. +#[derive(Debug)] +pub struct FindMatches<'r, 'h, A> { + re: &'r Regex<A>, + it: iter::Searcher<'h>, +} + +impl<'r, 'h, A: Automaton> Iterator for FindMatches<'r, 'h, A> { + type Item = Match; + + #[inline] + fn next(&mut self) -> Option<Match> { + let FindMatches { re, ref mut it } = *self; + it.advance(|input| re.try_search(input)) + } +} + +/// A builder for a regex based on deterministic finite automatons. +/// +/// This builder permits configuring options for the syntax of a pattern, the +/// NFA construction, the DFA construction and finally the regex searching +/// itself. This builder is different from a general purpose regex builder in +/// that it permits fine grain configuration of the construction process. The +/// trade off for this is complexity, and the possibility of setting a +/// configuration that might not make sense. For example, there are two +/// different UTF-8 modes: +/// +/// * [`syntax::Config::utf8`](crate::util::syntax::Config::utf8) controls +/// whether the pattern itself can contain sub-expressions that match invalid +/// UTF-8. +/// * [`thompson::Config::utf8`](crate::nfa::thompson::Config::utf8) controls +/// how the regex iterators themselves advance the starting position of the +/// next search when a match with zero length is found. +/// +/// Generally speaking, callers will want to either enable all of these or +/// disable all of these. +/// +/// Internally, building a regex requires building two DFAs, where one is +/// responsible for finding the end of a match and the other is responsible +/// for finding the start of a match. If you only need to detect whether +/// something matched, or only the end of a match, then you should use a +/// [`dense::Builder`] to construct a single DFA, which is cheaper than +/// building two DFAs. +/// +/// # Build methods +/// +/// This builder has a few "build" methods. In general, it's the result of +/// combining the following parameters: +/// +/// * Building one or many regexes. +/// * Building a regex with dense or sparse DFAs. +/// +/// The simplest "build" method is [`Builder::build`]. It accepts a single +/// pattern and builds a dense DFA using `usize` for the state identifier +/// representation. +/// +/// The most general "build" method is [`Builder::build_many`], which permits +/// building a regex that searches for multiple patterns simultaneously while +/// using a specific state identifier representation. +/// +/// The most flexible "build" method, but hardest to use, is +/// [`Builder::build_from_dfas`]. This exposes the fact that a [`Regex`] is +/// just a pair of DFAs, and this method allows you to specify those DFAs +/// exactly. +/// +/// # Example +/// +/// This example shows how to disable UTF-8 mode in the syntax and the regex +/// itself. This is generally what you want for matching on arbitrary bytes. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{ +/// dfa::regex::Regex, nfa::thompson, util::syntax, Match, +/// }; +/// +/// let re = Regex::builder() +/// .syntax(syntax::Config::new().utf8(false)) +/// .thompson(thompson::Config::new().utf8(false)) +/// .build(r"foo(?-u:[^b])ar.*")?; +/// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"; +/// let expected = Some(Match::must(0, 1..9)); +/// let got = re.find(haystack); +/// assert_eq!(expected, got); +/// // Notice that `(?-u:[^b])` matches invalid UTF-8, +/// // but the subsequent `.*` does not! Disabling UTF-8 +/// // on the syntax permits this. +/// assert_eq!(b"foo\xFFarzz", &haystack[got.unwrap().range()]); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Builder { + #[cfg(feature = "dfa-build")] + dfa: dense::Builder, +} + +impl Builder { + /// Create a new regex builder with the default configuration. + pub fn new() -> Builder { + Builder { + #[cfg(feature = "dfa-build")] + dfa: dense::Builder::new(), + } + } + + /// Build a regex from the given pattern. + /// + /// If there was a problem parsing or compiling the pattern, then an error + /// is returned. + #[cfg(all(feature = "syntax", feature = "dfa-build"))] + pub fn build(&self, pattern: &str) -> Result<Regex, BuildError> { + self.build_many(&[pattern]) + } + + /// Build a regex from the given pattern using sparse DFAs. + /// + /// If there was a problem parsing or compiling the pattern, then an error + /// is returned. + #[cfg(all(feature = "syntax", feature = "dfa-build"))] + pub fn build_sparse( + &self, + pattern: &str, + ) -> Result<Regex<sparse::DFA<Vec<u8>>>, BuildError> { + self.build_many_sparse(&[pattern]) + } + + /// Build a regex from the given patterns. + #[cfg(all(feature = "syntax", feature = "dfa-build"))] + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<Regex, BuildError> { + let forward = self.dfa.build_many(patterns)?; + let reverse = self + .dfa + .clone() + .configure( + dense::Config::new() + .prefilter(None) + .specialize_start_states(false) + .start_kind(StartKind::Anchored) + .match_kind(MatchKind::All), + ) + .thompson(crate::nfa::thompson::Config::new().reverse(true)) + .build_many(patterns)?; + Ok(self.build_from_dfas(forward, reverse)) + } + + /// Build a sparse regex from the given patterns. + #[cfg(all(feature = "syntax", feature = "dfa-build"))] + pub fn build_many_sparse<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<Regex<sparse::DFA<Vec<u8>>>, BuildError> { + let re = self.build_many(patterns)?; + let forward = re.forward().to_sparse()?; + let reverse = re.reverse().to_sparse()?; + Ok(self.build_from_dfas(forward, reverse)) + } + + /// Build a regex from its component forward and reverse DFAs. + /// + /// This is useful when deserializing a regex from some arbitrary + /// memory region. This is also useful for building regexes from other + /// types of DFAs. + /// + /// If you're building the DFAs from scratch instead of building new DFAs + /// from other DFAs, then you'll need to make sure that the reverse DFA is + /// configured correctly to match the intended semantics. Namely: + /// + /// * It should be anchored. + /// * It should use [`MatchKind::All`] semantics. + /// * It should match in reverse. + /// * Otherwise, its configuration should match the forward DFA. + /// + /// If these conditions aren't satisfied, then the behavior of searches is + /// unspecified. + /// + /// Note that when using this constructor, no configuration is applied. + /// Since this routine provides the DFAs to the builder, there is no + /// opportunity to apply other configuration options. + /// + /// # Example + /// + /// This example is a bit a contrived. The usual use of these methods + /// would involve serializing `initial_re` somewhere and then deserializing + /// it later to build a regex. But in this case, we do everything in + /// memory. + /// + /// ``` + /// use regex_automata::dfa::regex::Regex; + /// + /// let initial_re = Regex::new("foo[0-9]+")?; + /// assert_eq!(true, initial_re.is_match(b"foo123")); + /// + /// let (fwd, rev) = (initial_re.forward(), initial_re.reverse()); + /// let re = Regex::builder().build_from_dfas(fwd, rev); + /// assert_eq!(true, re.is_match(b"foo123")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// This example shows how to build a `Regex` that uses sparse DFAs instead + /// of dense DFAs without using one of the convenience `build_sparse` + /// routines: + /// + /// ``` + /// use regex_automata::dfa::regex::Regex; + /// + /// let initial_re = Regex::new("foo[0-9]+")?; + /// assert_eq!(true, initial_re.is_match(b"foo123")); + /// + /// let fwd = initial_re.forward().to_sparse()?; + /// let rev = initial_re.reverse().to_sparse()?; + /// let re = Regex::builder().build_from_dfas(fwd, rev); + /// assert_eq!(true, re.is_match(b"foo123")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_from_dfas<A: Automaton>( + &self, + forward: A, + reverse: A, + ) -> Regex<A> { + Regex { forward, reverse } + } + + /// Set the syntax configuration for this builder using + /// [`syntax::Config`](crate::util::syntax::Config). + /// + /// This permits setting things like case insensitivity, Unicode and multi + /// line mode. + #[cfg(all(feature = "syntax", feature = "dfa-build"))] + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Builder { + self.dfa.syntax(config); + self + } + + /// Set the Thompson NFA configuration for this builder using + /// [`nfa::thompson::Config`](crate::nfa::thompson::Config). + /// + /// This permits setting things like whether additional time should be + /// spent shrinking the size of the NFA. + #[cfg(all(feature = "syntax", feature = "dfa-build"))] + pub fn thompson( + &mut self, + config: crate::nfa::thompson::Config, + ) -> &mut Builder { + self.dfa.thompson(config); + self + } + + /// Set the dense DFA compilation configuration for this builder using + /// [`dense::Config`]. + /// + /// This permits setting things like whether the underlying DFAs should + /// be minimized. + #[cfg(feature = "dfa-build")] + pub fn dense(&mut self, config: dense::Config) -> &mut Builder { + self.dfa.configure(config); + self + } +} + +impl Default for Builder { + fn default() -> Builder { + Builder::new() + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/remapper.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/remapper.rs new file mode 100644 index 0000000..6e49646 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/remapper.rs @@ -0,0 +1,242 @@ +use alloc::vec::Vec; + +use crate::util::primitives::StateID; + +/// Remappable is a tightly coupled abstraction that facilitates remapping +/// state identifiers in DFAs. +/// +/// The main idea behind remapping state IDs is that DFAs often need to check +/// if a certain state is a "special" state of some kind (like a match state) +/// during a search. Since this is extremely perf critical code, we want this +/// check to be as fast as possible. Partitioning state IDs into, for example, +/// into "non-match" and "match" states means one can tell if a state is a +/// match state via a simple comparison of the state ID. +/// +/// The issue is that during the DFA construction process, it's not +/// particularly easy to partition the states. Instead, the simplest thing is +/// to often just do a pass over all of the states and shuffle them into their +/// desired partitionings. To do that, we need a mechanism for swapping states. +/// Hence, this abstraction. +/// +/// Normally, for such little code, I would just duplicate it. But this is a +/// key optimization and the implementation is a bit subtle. So the abstraction +/// is basically a ham-fisted attempt at DRY. The only place we use this is in +/// the dense and one-pass DFAs. +/// +/// See also src/dfa/special.rs for a more detailed explanation of how dense +/// DFAs are partitioned. +pub(super) trait Remappable: core::fmt::Debug { + /// Return the total number of states. + fn state_len(&self) -> usize; + /// Return the power-of-2 exponent that yields the stride. The pertinent + /// laws here are, where N=stride2: 2^N=stride and len(alphabet) <= stride. + fn stride2(&self) -> usize; + /// Swap the states pointed to by the given IDs. The underlying finite + /// state machine should be mutated such that all of the transitions in + /// `id1` are now in the memory region where the transitions for `id2` + /// were, and all of the transitions in `id2` are now in the memory region + /// where the transitions for `id1` were. + /// + /// Essentially, this "moves" `id1` to `id2` and `id2` to `id1`. + /// + /// It is expected that, after calling this, the underlying value will be + /// left in an inconsistent state, since any other transitions pointing to, + /// e.g., `id1` need to be updated to point to `id2`, since that's where + /// `id1` moved to. + /// + /// In order to "fix" the underlying inconsistent state, a `Remapper` + /// should be used to guarantee that `remap` is called at the appropriate + /// time. + fn swap_states(&mut self, id1: StateID, id2: StateID); + /// This must remap every single state ID in the underlying value according + /// to the function given. For example, in a DFA, this should remap every + /// transition and every starting state ID. + fn remap(&mut self, map: impl Fn(StateID) -> StateID); +} + +/// Remapper is an abstraction the manages the remapping of state IDs in a +/// finite state machine. This is useful when one wants to shuffle states into +/// different positions in the machine. +/// +/// One of the key complexities this manages is the ability to correctly move +/// one state multiple times. +/// +/// Once shuffling is complete, `remap` must be called, which will rewrite +/// all pertinent transitions to updated state IDs. Neglecting to call `remap` +/// will almost certainly result in a corrupt machine. +#[derive(Debug)] +pub(super) struct Remapper { + /// A map from the index of a state to its pre-multiplied identifier. + /// + /// When a state is swapped with another, then their corresponding + /// locations in this map are also swapped. Thus, its new position will + /// still point to its old pre-multiplied StateID. + /// + /// While there is a bit more to it, this then allows us to rewrite the + /// state IDs in a DFA's transition table in a single pass. This is done + /// by iterating over every ID in this map, then iterating over each + /// transition for the state at that ID and re-mapping the transition from + /// `old_id` to `map[dfa.to_index(old_id)]`. That is, we find the position + /// in this map where `old_id` *started*, and set it to where it ended up + /// after all swaps have been completed. + map: Vec<StateID>, + /// A mapper from state index to state ID (and back). + idxmap: IndexMapper, +} + +impl Remapper { + /// Create a new remapper from the given remappable implementation. The + /// remapper can then be used to swap states. The remappable value given + /// here must the same one given to `swap` and `remap`. + pub(super) fn new(r: &impl Remappable) -> Remapper { + let idxmap = IndexMapper { stride2: r.stride2() }; + let map = (0..r.state_len()).map(|i| idxmap.to_state_id(i)).collect(); + Remapper { map, idxmap } + } + + /// Swap two states. Once this is called, callers must follow through to + /// call `remap`, or else it's possible for the underlying remappable + /// value to be in a corrupt state. + pub(super) fn swap( + &mut self, + r: &mut impl Remappable, + id1: StateID, + id2: StateID, + ) { + if id1 == id2 { + return; + } + r.swap_states(id1, id2); + self.map.swap(self.idxmap.to_index(id1), self.idxmap.to_index(id2)); + } + + /// Complete the remapping process by rewriting all state IDs in the + /// remappable value according to the swaps performed. + pub(super) fn remap(mut self, r: &mut impl Remappable) { + // Update the map to account for states that have been swapped + // multiple times. For example, if (A, C) and (C, G) are swapped, then + // transitions previously pointing to A should now point to G. But if + // we don't update our map, they will erroneously be set to C. All we + // do is follow the swaps in our map until we see our original state + // ID. + // + // The intuition here is to think about how changes are made to the + // map: only through pairwise swaps. That means that starting at any + // given state, it is always possible to find the loop back to that + // state by following the swaps represented in the map (which might be + // 0 swaps). + // + // We are also careful to clone the map before starting in order to + // freeze it. We use the frozen map to find our loops, since we need to + // update our map as well. Without freezing it, our updates could break + // the loops referenced above and produce incorrect results. + let oldmap = self.map.clone(); + for i in 0..r.state_len() { + let cur_id = self.idxmap.to_state_id(i); + let mut new_id = oldmap[i]; + if cur_id == new_id { + continue; + } + loop { + let id = oldmap[self.idxmap.to_index(new_id)]; + if cur_id == id { + self.map[i] = new_id; + break; + } + new_id = id; + } + } + r.remap(|next| self.map[self.idxmap.to_index(next)]); + } +} + +/// A simple type for mapping between state indices and state IDs. +/// +/// The reason why this exists is because state IDs are "premultiplied." That +/// is, in order to get to the transitions for a particular state, one need +/// only use the state ID as-is, instead of having to multiple it by transition +/// table's stride. +/// +/// The downside of this is that it's inconvenient to map between state IDs +/// using a dense map, e.g., Vec<StateID>. That's because state IDs look like +/// `0`, `0+stride`, `0+2*stride`, `0+3*stride`, etc., instead of `0`, `1`, +/// `2`, `3`, etc. +/// +/// Since our state IDs are premultiplied, we can convert back-and-forth +/// between IDs and indices by simply unmultiplying the IDs and multiplying the +/// indices. +#[derive(Debug)] +struct IndexMapper { + /// The power of 2 corresponding to the stride of the corresponding + /// transition table. 'id >> stride2' de-multiplies an ID while 'index << + /// stride2' pre-multiplies an index to an ID. + stride2: usize, +} + +impl IndexMapper { + /// Convert a state ID to a state index. + fn to_index(&self, id: StateID) -> usize { + id.as_usize() >> self.stride2 + } + + /// Convert a state index to a state ID. + fn to_state_id(&self, index: usize) -> StateID { + // CORRECTNESS: If the given index is not valid, then it is not + // required for this to panic or return a valid state ID. We'll "just" + // wind up with panics or silent logic errors at some other point. + StateID::new_unchecked(index << self.stride2) + } +} + +#[cfg(feature = "dfa-build")] +mod dense { + use crate::{dfa::dense::OwnedDFA, util::primitives::StateID}; + + use super::Remappable; + + impl Remappable for OwnedDFA { + fn state_len(&self) -> usize { + OwnedDFA::state_len(self) + } + + fn stride2(&self) -> usize { + OwnedDFA::stride2(self) + } + + fn swap_states(&mut self, id1: StateID, id2: StateID) { + OwnedDFA::swap_states(self, id1, id2) + } + + fn remap(&mut self, map: impl Fn(StateID) -> StateID) { + OwnedDFA::remap(self, map) + } + } +} + +#[cfg(feature = "dfa-onepass")] +mod onepass { + use crate::{dfa::onepass::DFA, util::primitives::StateID}; + + use super::Remappable; + + impl Remappable for DFA { + fn state_len(&self) -> usize { + DFA::state_len(self) + } + + fn stride2(&self) -> usize { + // We don't do pre-multiplication for the one-pass DFA, so + // returning 0 has the effect of making state IDs and state indices + // equivalent. + 0 + } + + fn swap_states(&mut self, id1: StateID, id2: StateID) { + DFA::swap_states(self, id1, id2) + } + + fn remap(&mut self, map: impl Fn(StateID) -> StateID) { + DFA::remap(self, map) + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/search.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/search.rs new file mode 100644 index 0000000..09e2178 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/search.rs @@ -0,0 +1,652 @@ +use crate::{ + dfa::{ + accel, + automaton::{Automaton, OverlappingState}, + }, + util::{ + prefilter::Prefilter, + primitives::StateID, + search::{Anchored, HalfMatch, Input, Span}, + }, + MatchError, +}; + +#[inline(never)] +pub fn find_fwd<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, +) -> Result<Option<HalfMatch>, MatchError> { + if input.is_done() { + return Ok(None); + } + let pre = if input.get_anchored().is_anchored() { + None + } else { + dfa.get_prefilter() + }; + // Searching with a pattern ID is always anchored, so we should never use + // a prefilter. + if pre.is_some() { + if input.get_earliest() { + find_fwd_imp(dfa, input, pre, true) + } else { + find_fwd_imp(dfa, input, pre, false) + } + } else { + if input.get_earliest() { + find_fwd_imp(dfa, input, None, true) + } else { + find_fwd_imp(dfa, input, None, false) + } + } +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn find_fwd_imp<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, + pre: Option<&'_ Prefilter>, + earliest: bool, +) -> Result<Option<HalfMatch>, MatchError> { + // See 'prefilter_restart' docs for explanation. + let universal_start = dfa.universal_start_state(Anchored::No).is_some(); + let mut mat = None; + let mut sid = init_fwd(dfa, input)?; + let mut at = input.start(); + // This could just be a closure, but then I think it would be unsound + // because it would need to be safe to invoke. This way, the lack of safety + // is clearer in the code below. + macro_rules! next_unchecked { + ($sid:expr, $at:expr) => {{ + let byte = *input.haystack().get_unchecked($at); + dfa.next_state_unchecked($sid, byte) + }}; + } + + if let Some(ref pre) = pre { + let span = Span::from(at..input.end()); + // If a prefilter doesn't report false positives, then we don't need to + // touch the DFA at all. However, since all matches include the pattern + // ID, and the prefilter infrastructure doesn't report pattern IDs, we + // limit this optimization to cases where there is exactly one pattern. + // In that case, any match must be the 0th pattern. + match pre.find(input.haystack(), span) { + None => return Ok(mat), + Some(ref span) => { + at = span.start; + if !universal_start { + sid = prefilter_restart(dfa, &input, at)?; + } + } + } + } + while at < input.end() { + // SAFETY: There are two safety invariants we need to uphold here in + // the loops below: that 'sid' and 'prev_sid' are valid state IDs + // for this DFA, and that 'at' is a valid index into 'haystack'. + // For the former, we rely on the invariant that next_state* and + // start_state_forward always returns a valid state ID (given a valid + // state ID in the former case). For the latter safety invariant, we + // always guard unchecked access with a check that 'at' is less than + // 'end', where 'end <= haystack.len()'. In the unrolled loop below, we + // ensure that 'at' is always in bounds. + // + // PERF: See a similar comment in src/hybrid/search.rs that justifies + // this extra work to make the search loop fast. The same reasoning and + // benchmarks apply here. + let mut prev_sid; + while at < input.end() { + prev_sid = unsafe { next_unchecked!(sid, at) }; + if dfa.is_special_state(prev_sid) || at + 3 >= input.end() { + core::mem::swap(&mut prev_sid, &mut sid); + break; + } + at += 1; + + sid = unsafe { next_unchecked!(prev_sid, at) }; + if dfa.is_special_state(sid) { + break; + } + at += 1; + + prev_sid = unsafe { next_unchecked!(sid, at) }; + if dfa.is_special_state(prev_sid) { + core::mem::swap(&mut prev_sid, &mut sid); + break; + } + at += 1; + + sid = unsafe { next_unchecked!(prev_sid, at) }; + if dfa.is_special_state(sid) { + break; + } + at += 1; + } + if dfa.is_special_state(sid) { + if dfa.is_start_state(sid) { + if let Some(ref pre) = pre { + // A prefilter finds possible starts of new matches. Once + // a match is pending, the DFA may still be resolving its + // priority. Skipping ahead here could discard that match + // and replace it with a later one. + if mat.is_none() { + let span = Span::from(at..input.end()); + match pre.find(input.haystack(), span) { + None => return Ok(mat), + Some(ref span) => { + // We want to skip any update to 'at' below + // at the end of this iteration and just + // jump immediately back to the next state + // transition at the leading position of the + // candidate match. + // + // ... but only if we actually made progress + // with our prefilter, otherwise if the start + // state has a self-loop, we can get stuck. + if span.start > at { + at = span.start; + if !universal_start { + sid = prefilter_restart( + dfa, &input, at, + )?; + } + continue; + } + } + } + } + } else if dfa.is_accel_state(sid) { + let needles = dfa.accelerator(sid); + at = accel::find_fwd(needles, input.haystack(), at + 1) + .unwrap_or(input.end()); + continue; + } + } else if dfa.is_match_state(sid) { + let pattern = dfa.match_pattern(sid, 0); + mat = Some(HalfMatch::new(pattern, at)); + if earliest { + return Ok(mat); + } + if dfa.is_accel_state(sid) { + let needles = dfa.accelerator(sid); + at = accel::find_fwd(needles, input.haystack(), at + 1) + .unwrap_or(input.end()); + continue; + } + } else if dfa.is_accel_state(sid) { + let needs = dfa.accelerator(sid); + at = accel::find_fwd(needs, input.haystack(), at + 1) + .unwrap_or(input.end()); + continue; + } else if dfa.is_dead_state(sid) { + return Ok(mat); + } else { + // It's important that this is a debug_assert, since this can + // actually be tripped even if DFA::from_bytes succeeds and + // returns a supposedly valid DFA. + return Err(MatchError::quit(input.haystack()[at], at)); + } + } + at += 1; + } + eoi_fwd(dfa, input, &mut sid, &mut mat)?; + Ok(mat) +} + +#[inline(never)] +pub fn find_rev<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, +) -> Result<Option<HalfMatch>, MatchError> { + if input.is_done() { + return Ok(None); + } + if input.get_earliest() { + find_rev_imp(dfa, input, true) + } else { + find_rev_imp(dfa, input, false) + } +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn find_rev_imp<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, + earliest: bool, +) -> Result<Option<HalfMatch>, MatchError> { + let mut mat = None; + let mut sid = init_rev(dfa, input)?; + // In reverse search, the loop below can't handle the case of searching an + // empty slice. Ideally we could write something congruent to the forward + // search, i.e., 'while at >= start', but 'start' might be 0. Since we use + // an unsigned offset, 'at >= 0' is trivially always true. We could avoid + // this extra case handling by using a signed offset, but Rust makes it + // annoying to do. So... We just handle the empty case separately. + if input.start() == input.end() { + eoi_rev(dfa, input, &mut sid, &mut mat)?; + return Ok(mat); + } + + let mut at = input.end() - 1; + macro_rules! next_unchecked { + ($sid:expr, $at:expr) => {{ + let byte = *input.haystack().get_unchecked($at); + dfa.next_state_unchecked($sid, byte) + }}; + } + loop { + // SAFETY: See comments in 'find_fwd' for a safety argument. + let mut prev_sid; + while at >= input.start() { + prev_sid = unsafe { next_unchecked!(sid, at) }; + if dfa.is_special_state(prev_sid) + || at <= input.start().saturating_add(3) + { + core::mem::swap(&mut prev_sid, &mut sid); + break; + } + at -= 1; + + sid = unsafe { next_unchecked!(prev_sid, at) }; + if dfa.is_special_state(sid) { + break; + } + at -= 1; + + prev_sid = unsafe { next_unchecked!(sid, at) }; + if dfa.is_special_state(prev_sid) { + core::mem::swap(&mut prev_sid, &mut sid); + break; + } + at -= 1; + + sid = unsafe { next_unchecked!(prev_sid, at) }; + if dfa.is_special_state(sid) { + break; + } + at -= 1; + } + if dfa.is_special_state(sid) { + if dfa.is_start_state(sid) { + if dfa.is_accel_state(sid) { + let needles = dfa.accelerator(sid); + at = accel::find_rev(needles, input.haystack(), at) + .map(|i| i + 1) + .unwrap_or(input.start()); + } + } else if dfa.is_match_state(sid) { + let pattern = dfa.match_pattern(sid, 0); + // Since reverse searches report the beginning of a match + // and the beginning is inclusive (not exclusive like the + // end of a match), we add 1 to make it inclusive. + mat = Some(HalfMatch::new(pattern, at + 1)); + if earliest { + return Ok(mat); + } + if dfa.is_accel_state(sid) { + let needles = dfa.accelerator(sid); + at = accel::find_rev(needles, input.haystack(), at) + .map(|i| i + 1) + .unwrap_or(input.start()); + } + } else if dfa.is_accel_state(sid) { + let needles = dfa.accelerator(sid); + // If the accelerator returns nothing, why don't we quit the + // search? Well, if the accelerator doesn't find anything, that + // doesn't mean we don't have a match. It just means that we + // can't leave the current state given one of the 255 possible + // byte values. However, there might be an EOI transition. So + // we set 'at' to the end of the haystack, which will cause + // this loop to stop and fall down into the EOI transition. + at = accel::find_rev(needles, input.haystack(), at) + .map(|i| i + 1) + .unwrap_or(input.start()); + } else if dfa.is_dead_state(sid) { + return Ok(mat); + } else { + return Err(MatchError::quit(input.haystack()[at], at)); + } + } + if at == input.start() { + break; + } + at -= 1; + } + eoi_rev(dfa, input, &mut sid, &mut mat)?; + Ok(mat) +} + +#[inline(never)] +pub fn find_overlapping_fwd<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, + state: &mut OverlappingState, +) -> Result<(), MatchError> { + state.mat = None; + if input.is_done() { + return Ok(()); + } + let pre = if input.get_anchored().is_anchored() { + None + } else { + dfa.get_prefilter() + }; + if pre.is_some() { + find_overlapping_fwd_imp(dfa, input, pre, state) + } else { + find_overlapping_fwd_imp(dfa, input, None, state) + } +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn find_overlapping_fwd_imp<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, + pre: Option<&'_ Prefilter>, + state: &mut OverlappingState, +) -> Result<(), MatchError> { + // See 'prefilter_restart' docs for explanation. + let universal_start = dfa.universal_start_state(Anchored::No).is_some(); + let mut sid = match state.id { + None => { + state.at = input.start(); + init_fwd(dfa, input)? + } + Some(sid) => { + if let Some(match_index) = state.next_match_index { + let match_len = dfa.match_len(sid); + if match_index < match_len { + state.next_match_index = Some(match_index + 1); + let pattern = dfa.match_pattern(sid, match_index); + state.mat = Some(HalfMatch::new(pattern, state.at)); + return Ok(()); + } + } + // Once we've reported all matches at a given position, we need to + // advance the search to the next position. + state.at += 1; + if state.at > input.end() { + return Ok(()); + } + sid + } + }; + + // NOTE: We don't optimize the crap out of this routine primarily because + // it seems like most find_overlapping searches will have higher match + // counts, and thus, throughput is perhaps not as important. But if you + // have a use case for something faster, feel free to file an issue. + while state.at < input.end() { + sid = dfa.next_state(sid, input.haystack()[state.at]); + if dfa.is_special_state(sid) { + state.id = Some(sid); + if dfa.is_start_state(sid) { + if let Some(ref pre) = pre { + let span = Span::from(state.at..input.end()); + match pre.find(input.haystack(), span) { + None => return Ok(()), + Some(ref span) => { + if span.start > state.at { + state.at = span.start; + if !universal_start { + sid = prefilter_restart( + dfa, &input, state.at, + )?; + } + continue; + } + } + } + } else if dfa.is_accel_state(sid) { + let needles = dfa.accelerator(sid); + state.at = accel::find_fwd( + needles, + input.haystack(), + state.at + 1, + ) + .unwrap_or(input.end()); + continue; + } + } else if dfa.is_match_state(sid) { + state.next_match_index = Some(1); + let pattern = dfa.match_pattern(sid, 0); + state.mat = Some(HalfMatch::new(pattern, state.at)); + return Ok(()); + } else if dfa.is_accel_state(sid) { + let needs = dfa.accelerator(sid); + // If the accelerator returns nothing, why don't we quit the + // search? Well, if the accelerator doesn't find anything, that + // doesn't mean we don't have a match. It just means that we + // can't leave the current state given one of the 255 possible + // byte values. However, there might be an EOI transition. So + // we set 'at' to the end of the haystack, which will cause + // this loop to stop and fall down into the EOI transition. + state.at = + accel::find_fwd(needs, input.haystack(), state.at + 1) + .unwrap_or(input.end()); + continue; + } else if dfa.is_dead_state(sid) { + return Ok(()); + } else { + return Err(MatchError::quit( + input.haystack()[state.at], + state.at, + )); + } + } + state.at += 1; + } + + let result = eoi_fwd(dfa, input, &mut sid, &mut state.mat); + state.id = Some(sid); + if state.mat.is_some() { + // '1' is always correct here since if we get to this point, this + // always corresponds to the first (index '0') match discovered at + // this position. So the next match to report at this position (if + // it exists) is at index '1'. + state.next_match_index = Some(1); + } + result +} + +#[inline(never)] +pub(crate) fn find_overlapping_rev<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, + state: &mut OverlappingState, +) -> Result<(), MatchError> { + state.mat = None; + if input.is_done() { + return Ok(()); + } + let mut sid = match state.id { + None => { + let sid = init_rev(dfa, input)?; + state.id = Some(sid); + if input.start() == input.end() { + state.rev_eoi = true; + } else { + state.at = input.end() - 1; + } + sid + } + Some(sid) => { + if let Some(match_index) = state.next_match_index { + let match_len = dfa.match_len(sid); + if match_index < match_len { + state.next_match_index = Some(match_index + 1); + let pattern = dfa.match_pattern(sid, match_index); + state.mat = Some(HalfMatch::new(pattern, state.at)); + return Ok(()); + } + } + // Once we've reported all matches at a given position, we need + // to advance the search to the next position. However, if we've + // already followed the EOI transition, then we know we're done + // with the search and there cannot be any more matches to report. + if state.rev_eoi { + return Ok(()); + } else if state.at == input.start() { + // At this point, we should follow the EOI transition. This + // will cause us the skip the main loop below and fall through + // to the final 'eoi_rev' transition. + state.rev_eoi = true; + } else { + // We haven't hit the end of the search yet, so move on. + state.at -= 1; + } + sid + } + }; + while !state.rev_eoi { + sid = dfa.next_state(sid, input.haystack()[state.at]); + if dfa.is_special_state(sid) { + state.id = Some(sid); + if dfa.is_start_state(sid) { + if dfa.is_accel_state(sid) { + let needles = dfa.accelerator(sid); + state.at = + accel::find_rev(needles, input.haystack(), state.at) + .map(|i| i + 1) + .unwrap_or(input.start()); + } + } else if dfa.is_match_state(sid) { + state.next_match_index = Some(1); + let pattern = dfa.match_pattern(sid, 0); + state.mat = Some(HalfMatch::new(pattern, state.at + 1)); + return Ok(()); + } else if dfa.is_accel_state(sid) { + let needles = dfa.accelerator(sid); + // If the accelerator returns nothing, why don't we quit the + // search? Well, if the accelerator doesn't find anything, that + // doesn't mean we don't have a match. It just means that we + // can't leave the current state given one of the 255 possible + // byte values. However, there might be an EOI transition. So + // we set 'at' to the end of the haystack, which will cause + // this loop to stop and fall down into the EOI transition. + state.at = + accel::find_rev(needles, input.haystack(), state.at) + .map(|i| i + 1) + .unwrap_or(input.start()); + } else if dfa.is_dead_state(sid) { + return Ok(()); + } else { + return Err(MatchError::quit( + input.haystack()[state.at], + state.at, + )); + } + } + if state.at == input.start() { + break; + } + state.at -= 1; + } + + let result = eoi_rev(dfa, input, &mut sid, &mut state.mat); + state.rev_eoi = true; + state.id = Some(sid); + if state.mat.is_some() { + // '1' is always correct here since if we get to this point, this + // always corresponds to the first (index '0') match discovered at + // this position. So the next match to report at this position (if + // it exists) is at index '1'. + state.next_match_index = Some(1); + } + result +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn init_fwd<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, +) -> Result<StateID, MatchError> { + let sid = dfa.start_state_forward(input)?; + // Start states can never be match states, since all matches are delayed + // by 1 byte. + debug_assert!(!dfa.is_match_state(sid)); + Ok(sid) +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn init_rev<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, +) -> Result<StateID, MatchError> { + let sid = dfa.start_state_reverse(input)?; + // Start states can never be match states, since all matches are delayed + // by 1 byte. + debug_assert!(!dfa.is_match_state(sid)); + Ok(sid) +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn eoi_fwd<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, + sid: &mut StateID, + mat: &mut Option<HalfMatch>, +) -> Result<(), MatchError> { + let sp = input.get_span(); + match input.haystack().get(sp.end) { + Some(&b) => { + *sid = dfa.next_state(*sid, b); + if dfa.is_match_state(*sid) { + let pattern = dfa.match_pattern(*sid, 0); + *mat = Some(HalfMatch::new(pattern, sp.end)); + } else if dfa.is_quit_state(*sid) { + return Err(MatchError::quit(b, sp.end)); + } + } + None => { + *sid = dfa.next_eoi_state(*sid); + if dfa.is_match_state(*sid) { + let pattern = dfa.match_pattern(*sid, 0); + *mat = Some(HalfMatch::new(pattern, input.haystack().len())); + } + } + } + Ok(()) +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn eoi_rev<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, + sid: &mut StateID, + mat: &mut Option<HalfMatch>, +) -> Result<(), MatchError> { + let sp = input.get_span(); + if sp.start > 0 { + let byte = input.haystack()[sp.start - 1]; + *sid = dfa.next_state(*sid, byte); + if dfa.is_match_state(*sid) { + let pattern = dfa.match_pattern(*sid, 0); + *mat = Some(HalfMatch::new(pattern, sp.start)); + } else if dfa.is_quit_state(*sid) { + return Err(MatchError::quit(byte, sp.start - 1)); + } + } else { + *sid = dfa.next_eoi_state(*sid); + if dfa.is_match_state(*sid) { + let pattern = dfa.match_pattern(*sid, 0); + *mat = Some(HalfMatch::new(pattern, 0)); + } + } + Ok(()) +} + +/// Re-compute the starting state that a DFA should be in after finding a +/// prefilter candidate match at the position `at`. +/// +/// The function with the same name has a bit more docs in hybrid/search.rs. +#[cfg_attr(feature = "perf-inline", inline(always))] +fn prefilter_restart<A: Automaton + ?Sized>( + dfa: &A, + input: &Input<'_>, + at: usize, +) -> Result<StateID, MatchError> { + let mut input = input.clone(); + input.set_start(at); + init_fwd(dfa, &input) +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/sparse.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/sparse.rs new file mode 100644 index 0000000..923502a --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/sparse.rs @@ -0,0 +1,2673 @@ +/*! +Types and routines specific to sparse DFAs. + +This module is the home of [`sparse::DFA`](DFA). + +Unlike the [`dense`] module, this module does not contain a builder or +configuration specific for sparse DFAs. Instead, the intended way to build a +sparse DFA is either by using a default configuration with its constructor +[`sparse::DFA::new`](DFA::new), or by first configuring the construction of a +dense DFA with [`dense::Builder`] and then calling [`dense::DFA::to_sparse`]. +For example, this configures a sparse DFA to do an overlapping search: + +``` +use regex_automata::{ + dfa::{Automaton, OverlappingState, dense}, + HalfMatch, Input, MatchKind, +}; + +let dense_re = dense::Builder::new() + .configure(dense::Config::new().match_kind(MatchKind::All)) + .build(r"Samwise|Sam")?; +let sparse_re = dense_re.to_sparse()?; + +// Setup our haystack and initial start state. +let input = Input::new("Samwise"); +let mut state = OverlappingState::start(); + +// First, 'Sam' will match. +sparse_re.try_search_overlapping_fwd(&input, &mut state)?; +assert_eq!(Some(HalfMatch::must(0, 3)), state.get_match()); + +// And now 'Samwise' will match. +sparse_re.try_search_overlapping_fwd(&input, &mut state)?; +assert_eq!(Some(HalfMatch::must(0, 7)), state.get_match()); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` +*/ + +#[cfg(feature = "dfa-build")] +use core::iter; +use core::{fmt, mem::size_of}; + +#[cfg(feature = "dfa-build")] +use alloc::{vec, vec::Vec}; + +#[cfg(feature = "dfa-build")] +use crate::dfa::dense::{self, BuildError}; +use crate::{ + dfa::{ + automaton::{fmt_state_indicator, Automaton, StartError}, + dense::Flags, + special::Special, + StartKind, DEAD, + }, + util::{ + alphabet::{ByteClasses, ByteSet}, + escape::DebugByte, + int::{Pointer, Usize, U16, U32}, + prefilter::Prefilter, + primitives::{PatternID, StateID}, + search::Anchored, + start::{self, Start, StartByteMap}, + wire::{self, DeserializeError, Endian, SerializeError}, + }, +}; + +const LABEL: &str = "rust-regex-automata-dfa-sparse"; +const VERSION: u32 = 2; + +/// A sparse deterministic finite automaton (DFA) with variable sized states. +/// +/// In contrast to a [dense::DFA], a sparse DFA uses a more space efficient +/// representation for its transitions. Consequently, sparse DFAs may use much +/// less memory than dense DFAs, but this comes at a price. In particular, +/// reading the more space efficient transitions takes more work, and +/// consequently, searching using a sparse DFA is typically slower than a dense +/// DFA. +/// +/// A sparse DFA can be built using the default configuration via the +/// [`DFA::new`] constructor. Otherwise, one can configure various aspects of a +/// dense DFA via [`dense::Builder`], and then convert a dense DFA to a sparse +/// DFA using [`dense::DFA::to_sparse`]. +/// +/// In general, a sparse DFA supports all the same search operations as a dense +/// DFA. +/// +/// Making the choice between a dense and sparse DFA depends on your specific +/// work load. If you can sacrifice a bit of search time performance, then a +/// sparse DFA might be the best choice. In particular, while sparse DFAs are +/// probably always slower than dense DFAs, you may find that they are easily +/// fast enough for your purposes! +/// +/// # Type parameters +/// +/// A `DFA` has one type parameter, `T`, which is used to represent the parts +/// of a sparse DFA. `T` is typically a `Vec<u8>` or a `&[u8]`. +/// +/// # The `Automaton` trait +/// +/// This type implements the [`Automaton`] trait, which means it can be used +/// for searching. For example: +/// +/// ``` +/// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; +/// +/// let dfa = DFA::new("foo[0-9]+")?; +/// let expected = Some(HalfMatch::must(0, 8)); +/// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone)] +pub struct DFA<T> { + // When compared to a dense DFA, a sparse DFA *looks* a lot simpler + // representation-wise. In reality, it is perhaps more complicated. Namely, + // in a dense DFA, all information needs to be very cheaply accessible + // using only state IDs. In a sparse DFA however, each state uses a + // variable amount of space because each state encodes more information + // than just its transitions. Each state also includes an accelerator if + // one exists, along with the matching pattern IDs if the state is a match + // state. + // + // That is, a lot of the complexity is pushed down into how each state + // itself is represented. + tt: Transitions<T>, + st: StartTable<T>, + special: Special, + pre: Option<Prefilter>, + quitset: ByteSet, + flags: Flags, +} + +#[cfg(feature = "dfa-build")] +impl DFA<Vec<u8>> { + /// Parse the given regular expression using a default configuration and + /// return the corresponding sparse DFA. + /// + /// If you want a non-default configuration, then use the + /// [`dense::Builder`] to set your own configuration, and then call + /// [`dense::DFA::to_sparse`] to create a sparse DFA. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse}, HalfMatch, Input}; + /// + /// let dfa = sparse::DFA::new("foo[0-9]+bar")?; + /// + /// let expected = Some(HalfMatch::must(0, 11)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345bar"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new(pattern: &str) -> Result<DFA<Vec<u8>>, BuildError> { + dense::Builder::new() + .build(pattern) + .and_then(|dense| dense.to_sparse()) + } + + /// Parse the given regular expressions using a default configuration and + /// return the corresponding multi-DFA. + /// + /// If you want a non-default configuration, then use the + /// [`dense::Builder`] to set your own configuration, and then call + /// [`dense::DFA::to_sparse`] to create a sparse DFA. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse}, HalfMatch, Input}; + /// + /// let dfa = sparse::DFA::new_many(&["[0-9]+", "[a-z]+"])?; + /// let expected = Some(HalfMatch::must(1, 3)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345bar"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new_many<P: AsRef<str>>( + patterns: &[P], + ) -> Result<DFA<Vec<u8>>, BuildError> { + dense::Builder::new() + .build_many(patterns) + .and_then(|dense| dense.to_sparse()) + } +} + +#[cfg(feature = "dfa-build")] +impl DFA<Vec<u8>> { + /// Create a new DFA that matches every input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// dfa::{Automaton, sparse}, + /// HalfMatch, Input, + /// }; + /// + /// let dfa = sparse::DFA::always_match()?; + /// + /// let expected = Some(HalfMatch::must(0, 0)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new(""))?); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn always_match() -> Result<DFA<Vec<u8>>, BuildError> { + dense::DFA::always_match()?.to_sparse() + } + + /// Create a new sparse DFA that never matches any input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse}, Input}; + /// + /// let dfa = sparse::DFA::never_match()?; + /// assert_eq!(None, dfa.try_search_fwd(&Input::new(""))?); + /// assert_eq!(None, dfa.try_search_fwd(&Input::new("foo"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn never_match() -> Result<DFA<Vec<u8>>, BuildError> { + dense::DFA::never_match()?.to_sparse() + } + + /// The implementation for constructing a sparse DFA from a dense DFA. + pub(crate) fn from_dense<T: AsRef<[u32]>>( + dfa: &dense::DFA<T>, + ) -> Result<DFA<Vec<u8>>, BuildError> { + // In order to build the transition table, we need to be able to write + // state identifiers for each of the "next" transitions in each state. + // Our state identifiers correspond to the byte offset in the + // transition table at which the state is encoded. Therefore, we do not + // actually know what the state identifiers are until we've allocated + // exactly as much space as we need for each state. Thus, construction + // of the transition table happens in two passes. + // + // In the first pass, we fill out the shell of each state, which + // includes the transition length, the input byte ranges and + // zero-filled space for the transitions and accelerators, if present. + // In this first pass, we also build up a map from the state identifier + // index of the dense DFA to the state identifier in this sparse DFA. + // + // In the second pass, we fill in the transitions based on the map + // built in the first pass. + + // The capacity given here reflects a minimum. (Well, the true minimum + // is likely even bigger, but hopefully this saves a few reallocs.) + let mut sparse = Vec::with_capacity(StateID::SIZE * dfa.state_len()); + // This maps state indices from the dense DFA to StateIDs in the sparse + // DFA. We build out this map on the first pass, and then use it in the + // second pass to back-fill our transitions. + let mut remap: Vec<StateID> = vec![DEAD; dfa.state_len()]; + for state in dfa.states() { + let pos = sparse.len(); + + remap[dfa.to_index(state.id())] = StateID::new(pos) + .map_err(|_| BuildError::too_many_states())?; + // zero-filled space for the transition length + sparse.push(0); + sparse.push(0); + + let mut transition_len = 0; + for (unit1, unit2, _) in state.sparse_transitions() { + match (unit1.as_u8(), unit2.as_u8()) { + (Some(b1), Some(b2)) => { + transition_len += 1; + sparse.push(b1); + sparse.push(b2); + } + (None, None) => {} + (Some(_), None) | (None, Some(_)) => { + // can never occur because sparse_transitions never + // groups EOI with any other transition. + unreachable!() + } + } + } + // Add dummy EOI transition. This is never actually read while + // searching, but having space equivalent to the total number + // of transitions is convenient. Otherwise, we'd need to track + // a different number of transitions for the byte ranges as for + // the 'next' states. + // + // N.B. The loop above is not guaranteed to yield the EOI + // transition, since it may point to a DEAD state. By putting + // it here, we always write the EOI transition, and thus + // guarantee that our transition length is >0. Why do we always + // need the EOI transition? Because in order to implement + // Automaton::next_eoi_state, this lets us just ask for the last + // transition. There are probably other/better ways to do this. + transition_len += 1; + sparse.push(0); + sparse.push(0); + + // Check some assumptions about transition length. + assert_ne!( + transition_len, 0, + "transition length should be non-zero", + ); + assert!( + transition_len <= 257, + "expected transition length {transition_len} to be <= 257", + ); + + // Fill in the transition length. + // Since transition length is always <= 257, we use the most + // significant bit to indicate whether this is a match state or + // not. + let ntrans = if dfa.is_match_state(state.id()) { + transition_len | (1 << 15) + } else { + transition_len + }; + wire::NE::write_u16(ntrans, &mut sparse[pos..]); + + // zero-fill the actual transitions. + // Unwraps are OK since transition_length <= 257 and our minimum + // support usize size is 16-bits. + let zeros = usize::try_from(transition_len) + .unwrap() + .checked_mul(StateID::SIZE) + .unwrap(); + sparse.extend(iter::repeat(0).take(zeros)); + + // If this is a match state, write the pattern IDs matched by this + // state. + if dfa.is_match_state(state.id()) { + let plen = dfa.match_pattern_len(state.id()); + // Write the actual pattern IDs with a u32 length prefix. + // First, zero-fill space. + let mut pos = sparse.len(); + // Unwraps are OK since it's guaranteed that plen <= + // PatternID::LIMIT, which is in turn guaranteed to fit into a + // u32. + let zeros = size_of::<u32>() + .checked_mul(plen) + .unwrap() + .checked_add(size_of::<u32>()) + .unwrap(); + sparse.extend(iter::repeat(0).take(zeros)); + + // Now write the length prefix. + wire::NE::write_u32( + // Will never fail since u32::MAX is invalid pattern ID. + // Thus, the number of pattern IDs is representable by a + // u32. + plen.try_into().expect("pattern ID length fits in u32"), + &mut sparse[pos..], + ); + pos += size_of::<u32>(); + + // Now write the pattern IDs. + for &pid in dfa.pattern_id_slice(state.id()) { + pos += wire::write_pattern_id::<wire::NE>( + pid, + &mut sparse[pos..], + ); + } + } + + // And now add the accelerator, if one exists. An accelerator is + // at most 4 bytes and at least 1 byte. The first byte is the + // length, N. N bytes follow the length. The set of bytes that + // follow correspond (exhaustively) to the bytes that must be seen + // to leave this state. + let accel = dfa.accelerator(state.id()); + sparse.push(accel.len().try_into().unwrap()); + sparse.extend_from_slice(accel); + } + + let mut new = DFA { + tt: Transitions { + sparse, + classes: dfa.byte_classes().clone(), + state_len: dfa.state_len(), + pattern_len: dfa.pattern_len(), + }, + st: StartTable::from_dense_dfa(dfa, &remap)?, + special: dfa.special().remap(|id| remap[dfa.to_index(id)]), + pre: dfa.get_prefilter().map(|p| p.clone()), + quitset: dfa.quitset().clone(), + flags: dfa.flags().clone(), + }; + // And here's our second pass. Iterate over all of the dense states + // again, and update the transitions in each of the states in the + // sparse DFA. + for old_state in dfa.states() { + let new_id = remap[dfa.to_index(old_state.id())]; + let mut new_state = new.tt.state_mut(new_id); + let sparse = old_state.sparse_transitions(); + for (i, (_, _, next)) in sparse.enumerate() { + let next = remap[dfa.to_index(next)]; + new_state.set_next_at(i, next); + } + } + new.tt.sparse.shrink_to_fit(); + new.st.table.shrink_to_fit(); + debug!( + "created sparse DFA, memory usage: {} (dense memory usage: {})", + new.memory_usage(), + dfa.memory_usage(), + ); + Ok(new) + } +} + +impl<T: AsRef<[u8]>> DFA<T> { + /// Cheaply return a borrowed version of this sparse DFA. Specifically, the + /// DFA returned always uses `&[u8]` for its transitions. + pub fn as_ref<'a>(&'a self) -> DFA<&'a [u8]> { + DFA { + tt: self.tt.as_ref(), + st: self.st.as_ref(), + special: self.special, + pre: self.pre.clone(), + quitset: self.quitset, + flags: self.flags, + } + } + + /// Return an owned version of this sparse DFA. Specifically, the DFA + /// returned always uses `Vec<u8>` for its transitions. + /// + /// Effectively, this returns a sparse DFA whose transitions live on the + /// heap. + #[cfg(feature = "alloc")] + pub fn to_owned(&self) -> DFA<alloc::vec::Vec<u8>> { + DFA { + tt: self.tt.to_owned(), + st: self.st.to_owned(), + special: self.special, + pre: self.pre.clone(), + quitset: self.quitset, + flags: self.flags, + } + } + + /// Returns the starting state configuration for this DFA. + /// + /// The default is [`StartKind::Both`], which means the DFA supports both + /// unanchored and anchored searches. However, this can generally lead to + /// bigger DFAs. Therefore, a DFA might be compiled with support for just + /// unanchored or anchored searches. In that case, running a search with + /// an unsupported configuration will panic. + pub fn start_kind(&self) -> StartKind { + self.st.kind + } + + /// Returns true only if this DFA has starting states for each pattern. + /// + /// When a DFA has starting states for each pattern, then a search with the + /// DFA can be configured to only look for anchored matches of a specific + /// pattern. Specifically, APIs like [`Automaton::try_search_fwd`] can + /// accept a [`Anchored::Pattern`] if and only if this method returns true. + /// Otherwise, an error will be returned. + /// + /// Note that if the DFA is empty, this always returns false. + pub fn starts_for_each_pattern(&self) -> bool { + self.st.pattern_len.is_some() + } + + /// Returns the equivalence classes that make up the alphabet for this DFA. + /// + /// Unless [`dense::Config::byte_classes`] was disabled, it is possible + /// that multiple distinct bytes are grouped into the same equivalence + /// class if it is impossible for them to discriminate between a match and + /// a non-match. This has the effect of reducing the overall alphabet size + /// and in turn potentially substantially reducing the size of the DFA's + /// transition table. + /// + /// The downside of using equivalence classes like this is that every state + /// transition will automatically use this map to convert an arbitrary + /// byte to its corresponding equivalence class. In practice this has a + /// negligible impact on performance. + pub fn byte_classes(&self) -> &ByteClasses { + &self.tt.classes + } + + /// Returns the memory usage, in bytes, of this DFA. + /// + /// The memory usage is computed based on the number of bytes used to + /// represent this DFA. + /// + /// This does **not** include the stack size used up by this DFA. To + /// compute that, use `std::mem::size_of::<sparse::DFA>()`. + pub fn memory_usage(&self) -> usize { + self.tt.memory_usage() + self.st.memory_usage() + } +} + +/// Routines for converting a sparse DFA to other representations, such as raw +/// bytes suitable for persistent storage. +impl<T: AsRef<[u8]>> DFA<T> { + /// Serialize this DFA as raw bytes to a `Vec<u8>` in little endian + /// format. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// Note that unlike a [`dense::DFA`]'s serialization methods, this does + /// not add any initial padding to the returned bytes. Padding isn't + /// required for sparse DFAs since they have no alignment requirements. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA: + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // N.B. We use native endianness here to make the example work, but + /// // using to_bytes_little_endian would work on a little endian target. + /// let buf = original_dfa.to_bytes_native_endian(); + /// // Even if buf has initial padding, DFA::from_bytes will automatically + /// // ignore it. + /// let dfa: DFA<&[u8]> = DFA::from_bytes(&buf)?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "dfa-build")] + pub fn to_bytes_little_endian(&self) -> Vec<u8> { + self.to_bytes::<wire::LE>() + } + + /// Serialize this DFA as raw bytes to a `Vec<u8>` in big endian + /// format. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// Note that unlike a [`dense::DFA`]'s serialization methods, this does + /// not add any initial padding to the returned bytes. Padding isn't + /// required for sparse DFAs since they have no alignment requirements. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA: + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // N.B. We use native endianness here to make the example work, but + /// // using to_bytes_big_endian would work on a big endian target. + /// let buf = original_dfa.to_bytes_native_endian(); + /// // Even if buf has initial padding, DFA::from_bytes will automatically + /// // ignore it. + /// let dfa: DFA<&[u8]> = DFA::from_bytes(&buf)?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "dfa-build")] + pub fn to_bytes_big_endian(&self) -> Vec<u8> { + self.to_bytes::<wire::BE>() + } + + /// Serialize this DFA as raw bytes to a `Vec<u8>` in native endian + /// format. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// Note that unlike a [`dense::DFA`]'s serialization methods, this does + /// not add any initial padding to the returned bytes. Padding isn't + /// required for sparse DFAs since they have no alignment requirements. + /// + /// Generally speaking, native endian format should only be used when + /// you know that the target you're compiling the DFA for matches the + /// endianness of the target on which you're compiling DFA. For example, + /// if serialization and deserialization happen in the same process or on + /// the same machine. Otherwise, when serializing a DFA for use in a + /// portable environment, you'll almost certainly want to serialize _both_ + /// a little endian and a big endian version and then load the correct one + /// based on the target's configuration. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA: + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// let buf = original_dfa.to_bytes_native_endian(); + /// // Even if buf has initial padding, DFA::from_bytes will automatically + /// // ignore it. + /// let dfa: DFA<&[u8]> = DFA::from_bytes(&buf)?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "dfa-build")] + pub fn to_bytes_native_endian(&self) -> Vec<u8> { + self.to_bytes::<wire::NE>() + } + + /// The implementation of the public `to_bytes` serialization methods, + /// which is generic over endianness. + #[cfg(feature = "dfa-build")] + fn to_bytes<E: Endian>(&self) -> Vec<u8> { + let mut buf = vec![0; self.write_to_len()]; + // This should always succeed since the only possible serialization + // error is providing a buffer that's too small, but we've ensured that + // `buf` is big enough here. + self.write_to::<E>(&mut buf).unwrap(); + buf + } + + /// Serialize this DFA as raw bytes to the given slice, in little endian + /// format. Upon success, the total number of bytes written to `dst` is + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// # Errors + /// + /// This returns an error if the given destination slice is not big enough + /// to contain the full serialized DFA. If an error occurs, then nothing + /// is written to `dst`. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA without + /// dynamic memory allocation. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // Create a 4KB buffer on the stack to store our serialized DFA. + /// let mut buf = [0u8; 4 * (1<<10)]; + /// // N.B. We use native endianness here to make the example work, but + /// // using write_to_little_endian would work on a little endian target. + /// let written = original_dfa.write_to_native_endian(&mut buf)?; + /// let dfa: DFA<&[u8]> = DFA::from_bytes(&buf[..written])?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn write_to_little_endian( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + self.write_to::<wire::LE>(dst) + } + + /// Serialize this DFA as raw bytes to the given slice, in big endian + /// format. Upon success, the total number of bytes written to `dst` is + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// # Errors + /// + /// This returns an error if the given destination slice is not big enough + /// to contain the full serialized DFA. If an error occurs, then nothing + /// is written to `dst`. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA without + /// dynamic memory allocation. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // Create a 4KB buffer on the stack to store our serialized DFA. + /// let mut buf = [0u8; 4 * (1<<10)]; + /// // N.B. We use native endianness here to make the example work, but + /// // using write_to_big_endian would work on a big endian target. + /// let written = original_dfa.write_to_native_endian(&mut buf)?; + /// let dfa: DFA<&[u8]> = DFA::from_bytes(&buf[..written])?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn write_to_big_endian( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + self.write_to::<wire::BE>(dst) + } + + /// Serialize this DFA as raw bytes to the given slice, in native endian + /// format. Upon success, the total number of bytes written to `dst` is + /// returned. + /// + /// The written bytes are guaranteed to be deserialized correctly and + /// without errors in a semver compatible release of this crate by a + /// `DFA`'s deserialization APIs (assuming all other criteria for the + /// deserialization APIs has been satisfied): + /// + /// * [`DFA::from_bytes`] + /// * [`DFA::from_bytes_unchecked`] + /// + /// Generally speaking, native endian format should only be used when + /// you know that the target you're compiling the DFA for matches the + /// endianness of the target on which you're compiling DFA. For example, + /// if serialization and deserialization happen in the same process or on + /// the same machine. Otherwise, when serializing a DFA for use in a + /// portable environment, you'll almost certainly want to serialize _both_ + /// a little endian and a big endian version and then load the correct one + /// based on the target's configuration. + /// + /// # Errors + /// + /// This returns an error if the given destination slice is not big enough + /// to contain the full serialized DFA. If an error occurs, then nothing + /// is written to `dst`. + /// + /// # Example + /// + /// This example shows how to serialize and deserialize a DFA without + /// dynamic memory allocation. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// // Create a 4KB buffer on the stack to store our serialized DFA. + /// let mut buf = [0u8; 4 * (1<<10)]; + /// let written = original_dfa.write_to_native_endian(&mut buf)?; + /// let dfa: DFA<&[u8]> = DFA::from_bytes(&buf[..written])?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn write_to_native_endian( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + self.write_to::<wire::NE>(dst) + } + + /// The implementation of the public `write_to` serialization methods, + /// which is generic over endianness. + fn write_to<E: Endian>( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let mut nw = 0; + nw += wire::write_label(LABEL, &mut dst[nw..])?; + nw += wire::write_endianness_check::<E>(&mut dst[nw..])?; + nw += wire::write_version::<E>(VERSION, &mut dst[nw..])?; + nw += { + // Currently unused, intended for future flexibility + E::write_u32(0, &mut dst[nw..]); + size_of::<u32>() + }; + nw += self.flags.write_to::<E>(&mut dst[nw..])?; + nw += self.tt.write_to::<E>(&mut dst[nw..])?; + nw += self.st.write_to::<E>(&mut dst[nw..])?; + nw += self.special.write_to::<E>(&mut dst[nw..])?; + nw += self.quitset.write_to::<E>(&mut dst[nw..])?; + Ok(nw) + } + + /// Return the total number of bytes required to serialize this DFA. + /// + /// This is useful for determining the size of the buffer required to pass + /// to one of the serialization routines: + /// + /// * [`DFA::write_to_little_endian`] + /// * [`DFA::write_to_big_endian`] + /// * [`DFA::write_to_native_endian`] + /// + /// Passing a buffer smaller than the size returned by this method will + /// result in a serialization error. + /// + /// # Example + /// + /// This example shows how to dynamically allocate enough room to serialize + /// a sparse DFA. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// // Compile our original DFA. + /// let original_dfa = DFA::new("foo[0-9]+")?; + /// + /// let mut buf = vec![0; original_dfa.write_to_len()]; + /// let written = original_dfa.write_to_native_endian(&mut buf)?; + /// let dfa: DFA<&[u8]> = DFA::from_bytes(&buf[..written])?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn write_to_len(&self) -> usize { + wire::write_label_len(LABEL) + + wire::write_endianness_check_len() + + wire::write_version_len() + + size_of::<u32>() // unused, intended for future flexibility + + self.flags.write_to_len() + + self.tt.write_to_len() + + self.st.write_to_len() + + self.special.write_to_len() + + self.quitset.write_to_len() + } +} + +impl<'a> DFA<&'a [u8]> { + /// Safely deserialize a sparse DFA with a specific state identifier + /// representation. Upon success, this returns both the deserialized DFA + /// and the number of bytes read from the given slice. Namely, the contents + /// of the slice beyond the DFA are not read. + /// + /// Deserializing a DFA using this routine will never allocate heap memory. + /// For safety purposes, the DFA's transitions will be verified such that + /// every transition points to a valid state. If this verification is too + /// costly, then a [`DFA::from_bytes_unchecked`] API is provided, which + /// will always execute in constant time. + /// + /// The bytes given must be generated by one of the serialization APIs + /// of a `DFA` using a semver compatible release of this crate. Those + /// include: + /// + /// * [`DFA::to_bytes_little_endian`] + /// * [`DFA::to_bytes_big_endian`] + /// * [`DFA::to_bytes_native_endian`] + /// * [`DFA::write_to_little_endian`] + /// * [`DFA::write_to_big_endian`] + /// * [`DFA::write_to_native_endian`] + /// + /// The `to_bytes` methods allocate and return a `Vec<u8>` for you. The + /// `write_to` methods do not allocate and write to an existing slice + /// (which may be on the stack). Since deserialization always uses the + /// native endianness of the target platform, the serialization API you use + /// should match the endianness of the target platform. (It's often a good + /// idea to generate serialized DFAs for both forms of endianness and then + /// load the correct one based on endianness.) + /// + /// # Errors + /// + /// Generally speaking, it's easier to state the conditions in which an + /// error is _not_ returned. All of the following must be true: + /// + /// * The bytes given must be produced by one of the serialization APIs + /// on this DFA, as mentioned above. + /// * The endianness of the target platform matches the endianness used to + /// serialized the provided DFA. + /// + /// If any of the above are not true, then an error will be returned. + /// + /// Note that unlike deserializing a [`dense::DFA`], deserializing a sparse + /// DFA has no alignment requirements. That is, an alignment of `1` is + /// valid. + /// + /// # Panics + /// + /// This routine will never panic for any input. + /// + /// # Example + /// + /// This example shows how to serialize a DFA to raw bytes, deserialize it + /// and then use it for searching. + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// let initial = DFA::new("foo[0-9]+")?; + /// let bytes = initial.to_bytes_native_endian(); + /// let dfa: DFA<&[u8]> = DFA::from_bytes(&bytes)?.0; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: loading a DFA from static memory + /// + /// One use case this library supports is the ability to serialize a + /// DFA to disk and then use `include_bytes!` to store it in a compiled + /// Rust program. Those bytes can then be cheaply deserialized into a + /// `DFA` structure at runtime and used for searching without having to + /// re-compile the DFA (which can be quite costly). + /// + /// We can show this in two parts. The first part is serializing the DFA to + /// a file: + /// + /// ```no_run + /// use regex_automata::dfa::sparse::DFA; + /// + /// let dfa = DFA::new("foo[0-9]+")?; + /// + /// // Write a big endian serialized version of this DFA to a file. + /// let bytes = dfa.to_bytes_big_endian(); + /// std::fs::write("foo.bigendian.dfa", &bytes)?; + /// + /// // Do it again, but this time for little endian. + /// let bytes = dfa.to_bytes_little_endian(); + /// std::fs::write("foo.littleendian.dfa", &bytes)?; + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And now the second part is embedding the DFA into the compiled program + /// and deserializing it at runtime on first use. We use conditional + /// compilation to choose the correct endianness. We do not need to employ + /// any special tricks to ensure a proper alignment, since a sparse DFA has + /// no alignment requirements. + /// + /// ```no_run + /// use regex_automata::{ + /// dfa::{Automaton, sparse::DFA}, + /// util::lazy::Lazy, + /// HalfMatch, Input, + /// }; + /// + /// // This crate provides its own "lazy" type, kind of like + /// // lazy_static! or once_cell::sync::Lazy. But it works in no-alloc + /// // no-std environments and let's us write this using completely + /// // safe code. + /// static RE: Lazy<DFA<&'static [u8]>> = Lazy::new(|| { + /// # const _: &str = stringify! { + /// #[cfg(target_endian = "big")] + /// static BYTES: &[u8] = include_bytes!("foo.bigendian.dfa"); + /// #[cfg(target_endian = "little")] + /// static BYTES: &[u8] = include_bytes!("foo.littleendian.dfa"); + /// # }; + /// # static BYTES: &[u8] = b""; + /// + /// let (dfa, _) = DFA::from_bytes(BYTES) + /// .expect("serialized DFA should be valid"); + /// dfa + /// }); + /// + /// let expected = Ok(Some(HalfMatch::must(0, 8))); + /// assert_eq!(expected, RE.try_search_fwd(&Input::new("foo12345"))); + /// ``` + /// + /// Alternatively, consider using + /// [`lazy_static`](https://crates.io/crates/lazy_static) + /// or + /// [`once_cell`](https://crates.io/crates/once_cell), + /// which will guarantee safety for you. + pub fn from_bytes( + slice: &'a [u8], + ) -> Result<(DFA<&'a [u8]>, usize), DeserializeError> { + // SAFETY: This is safe because we validate both the sparse transitions + // (by trying to decode every state) and start state ID list below. If + // either validation fails, then we return an error. + let (dfa, nread) = unsafe { DFA::from_bytes_unchecked(slice)? }; + let seen = dfa.tt.validate(&dfa.special)?; + dfa.st.validate(&dfa.special, &seen)?; + // N.B. dfa.special doesn't have a way to do unchecked deserialization, + // so it has already been validated. + Ok((dfa, nread)) + } + + /// Deserialize a DFA with a specific state identifier representation in + /// constant time by omitting the verification of the validity of the + /// sparse transitions. + /// + /// This is just like [`DFA::from_bytes`], except it can potentially return + /// a DFA that exhibits undefined behavior if its transitions contains + /// invalid state identifiers. + /// + /// This routine is useful if you need to deserialize a DFA cheaply and + /// cannot afford the transition validation performed by `from_bytes`. + /// + /// # Safety + /// + /// This routine is not safe because it permits callers to provide + /// arbitrary transitions with possibly incorrect state identifiers. While + /// the various serialization routines will never return an incorrect + /// DFA, there is no guarantee that the bytes provided here are correct. + /// While `from_bytes_unchecked` will still do several forms of basic + /// validation, this routine does not check that the transitions themselves + /// are correct. Given an incorrect transition table, it is possible for + /// the search routines to access out-of-bounds memory because of explicit + /// bounds check elision. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{dfa::{Automaton, sparse::DFA}, HalfMatch, Input}; + /// + /// let initial = DFA::new("foo[0-9]+")?; + /// let bytes = initial.to_bytes_native_endian(); + /// // SAFETY: This is guaranteed to be safe since the bytes given come + /// // directly from a compatible serialization routine. + /// let dfa: DFA<&[u8]> = unsafe { DFA::from_bytes_unchecked(&bytes)?.0 }; + /// + /// let expected = Some(HalfMatch::must(0, 8)); + /// assert_eq!(expected, dfa.try_search_fwd(&Input::new("foo12345"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub unsafe fn from_bytes_unchecked( + slice: &'a [u8], + ) -> Result<(DFA<&'a [u8]>, usize), DeserializeError> { + let mut nr = 0; + + nr += wire::read_label(&slice[nr..], LABEL)?; + nr += wire::read_endianness_check(&slice[nr..])?; + nr += wire::read_version(&slice[nr..], VERSION)?; + + let _unused = wire::try_read_u32(&slice[nr..], "unused space")?; + nr += size_of::<u32>(); + + let (flags, nread) = Flags::from_bytes(&slice[nr..])?; + nr += nread; + + let (tt, nread) = Transitions::from_bytes_unchecked(&slice[nr..])?; + nr += nread; + + let (st, nread) = StartTable::from_bytes_unchecked(&slice[nr..])?; + nr += nread; + + let (special, nread) = Special::from_bytes(&slice[nr..])?; + nr += nread; + if special.max.as_usize() >= tt.sparse().len() { + return Err(DeserializeError::generic( + "max should not be greater than or equal to sparse bytes", + )); + } + + let (quitset, nread) = ByteSet::from_bytes(&slice[nr..])?; + nr += nread; + + // Prefilters don't support serialization, so they're always absent. + let pre = None; + Ok((DFA { tt, st, special, pre, quitset, flags }, nr)) + } +} + +/// Other routines that work for all `T`. +impl<T> DFA<T> { + /// Set or unset the prefilter attached to this DFA. + /// + /// This is useful when one has deserialized a DFA from `&[u8]`. + /// Deserialization does not currently include prefilters, so if you + /// want prefilter acceleration, you'll need to rebuild it and attach + /// it here. + pub fn set_prefilter(&mut self, prefilter: Option<Prefilter>) { + self.pre = prefilter + } +} + +impl<T: AsRef<[u8]>> fmt::Debug for DFA<T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "sparse::DFA(")?; + for state in self.tt.states() { + fmt_state_indicator(f, self, state.id())?; + writeln!(f, "{:06?}: {:?}", state.id().as_usize(), state)?; + } + writeln!(f, "")?; + for (i, (start_id, anchored, sty)) in self.st.iter().enumerate() { + if i % self.st.stride == 0 { + match anchored { + Anchored::No => writeln!(f, "START-GROUP(unanchored)")?, + Anchored::Yes => writeln!(f, "START-GROUP(anchored)")?, + Anchored::Pattern(pid) => writeln!( + f, + "START_GROUP(pattern: {:?})", + pid.as_usize() + )?, + } + } + writeln!(f, " {:?} => {:06?}", sty, start_id.as_usize())?; + } + writeln!(f, "state length: {:?}", self.tt.state_len)?; + writeln!(f, "pattern length: {:?}", self.pattern_len())?; + writeln!(f, "flags: {:?}", self.flags)?; + writeln!(f, ")")?; + Ok(()) + } +} + +// SAFETY: We assert that our implementation of each method is correct. +unsafe impl<T: AsRef<[u8]>> Automaton for DFA<T> { + #[inline] + fn is_special_state(&self, id: StateID) -> bool { + self.special.is_special_state(id) + } + + #[inline] + fn is_dead_state(&self, id: StateID) -> bool { + self.special.is_dead_state(id) + } + + #[inline] + fn is_quit_state(&self, id: StateID) -> bool { + self.special.is_quit_state(id) + } + + #[inline] + fn is_match_state(&self, id: StateID) -> bool { + self.special.is_match_state(id) + } + + #[inline] + fn is_start_state(&self, id: StateID) -> bool { + self.special.is_start_state(id) + } + + #[inline] + fn is_accel_state(&self, id: StateID) -> bool { + self.special.is_accel_state(id) + } + + // This is marked as inline to help dramatically boost sparse searching, + // which decodes each state it enters to follow the next transition. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn next_state(&self, current: StateID, input: u8) -> StateID { + let input = self.tt.classes.get(input); + self.tt.state(current).next(input) + } + + #[inline] + unsafe fn next_state_unchecked( + &self, + current: StateID, + input: u8, + ) -> StateID { + self.next_state(current, input) + } + + #[inline] + fn next_eoi_state(&self, current: StateID) -> StateID { + self.tt.state(current).next_eoi() + } + + #[inline] + fn pattern_len(&self) -> usize { + self.tt.pattern_len + } + + #[inline] + fn match_len(&self, id: StateID) -> usize { + self.tt.state(id).pattern_len() + } + + #[inline] + fn match_pattern(&self, id: StateID, match_index: usize) -> PatternID { + // This is an optimization for the very common case of a DFA with a + // single pattern. This conditional avoids a somewhat more costly path + // that finds the pattern ID from the state machine, which requires + // a bit of slicing/pointer-chasing. This optimization tends to only + // matter when matches are frequent. + if self.tt.pattern_len == 1 { + return PatternID::ZERO; + } + self.tt.state(id).pattern_id(match_index) + } + + #[inline] + fn has_empty(&self) -> bool { + self.flags.has_empty + } + + #[inline] + fn is_utf8(&self) -> bool { + self.flags.is_utf8 + } + + #[inline] + fn is_always_start_anchored(&self) -> bool { + self.flags.is_always_start_anchored + } + + #[inline] + fn start_state( + &self, + config: &start::Config, + ) -> Result<StateID, StartError> { + let anchored = config.get_anchored(); + let start = match config.get_look_behind() { + None => Start::Text, + Some(byte) => { + if !self.quitset.is_empty() && self.quitset.contains(byte) { + return Err(StartError::quit(byte)); + } + self.st.start_map.get(byte) + } + }; + self.st.start(anchored, start) + } + + #[inline] + fn universal_start_state(&self, mode: Anchored) -> Option<StateID> { + match mode { + Anchored::No => self.st.universal_start_unanchored, + Anchored::Yes => self.st.universal_start_anchored, + Anchored::Pattern(_) => None, + } + } + + #[inline] + fn accelerator(&self, id: StateID) -> &[u8] { + self.tt.state(id).accelerator() + } + + #[inline] + fn get_prefilter(&self) -> Option<&Prefilter> { + self.pre.as_ref() + } +} + +/// The transition table portion of a sparse DFA. +/// +/// The transition table is the core part of the DFA in that it describes how +/// to move from one state to another based on the input sequence observed. +/// +/// Unlike a typical dense table based DFA, states in a sparse transition +/// table have variable size. That is, states with more transitions use more +/// space than states with fewer transitions. This means that finding the next +/// transition takes more work than with a dense DFA, but also typically uses +/// much less space. +#[derive(Clone)] +struct Transitions<T> { + /// The raw encoding of each state in this DFA. + /// + /// Each state has the following information: + /// + /// * A set of transitions to subsequent states. Transitions to the dead + /// state are omitted. + /// * If the state can be accelerated, then any additional accelerator + /// information. + /// * If the state is a match state, then the state contains all pattern + /// IDs that match when in that state. + /// + /// To decode a state, use Transitions::state. + /// + /// In practice, T is either Vec<u8> or &[u8]. + sparse: T, + /// A set of equivalence classes, where a single equivalence class + /// represents a set of bytes that never discriminate between a match + /// and a non-match in the DFA. Each equivalence class corresponds to a + /// single character in this DFA's alphabet, where the maximum number of + /// characters is 257 (each possible value of a byte plus the special + /// EOI transition). Consequently, the number of equivalence classes + /// corresponds to the number of transitions for each DFA state. Note + /// though that the *space* used by each DFA state in the transition table + /// may be larger. The total space used by each DFA state is known as the + /// stride and is documented above. + /// + /// The only time the number of equivalence classes is fewer than 257 is + /// if the DFA's kind uses byte classes which is the default. Equivalence + /// classes should generally only be disabled when debugging, so that + /// the transitions themselves aren't obscured. Disabling them has no + /// other benefit, since the equivalence class map is always used while + /// searching. In the vast majority of cases, the number of equivalence + /// classes is substantially smaller than 257, particularly when large + /// Unicode classes aren't used. + /// + /// N.B. Equivalence classes aren't particularly useful in a sparse DFA + /// in the current implementation, since equivalence classes generally tend + /// to correspond to continuous ranges of bytes that map to the same + /// transition. So in a sparse DFA, equivalence classes don't really lead + /// to a space savings. In the future, it would be good to try and remove + /// them from sparse DFAs entirely, but requires a bit of work since sparse + /// DFAs are built from dense DFAs, which are in turn built on top of + /// equivalence classes. + classes: ByteClasses, + /// The total number of states in this DFA. Note that a DFA always has at + /// least one state---the dead state---even the empty DFA. In particular, + /// the dead state always has ID 0 and is correspondingly always the first + /// state. The dead state is never a match state. + state_len: usize, + /// The total number of unique patterns represented by these match states. + pattern_len: usize, +} + +impl<'a> Transitions<&'a [u8]> { + unsafe fn from_bytes_unchecked( + mut slice: &'a [u8], + ) -> Result<(Transitions<&'a [u8]>, usize), DeserializeError> { + let slice_start = slice.as_ptr().as_usize(); + + let (state_len, nr) = + wire::try_read_u32_as_usize(&slice, "state length")?; + slice = &slice[nr..]; + + let (pattern_len, nr) = + wire::try_read_u32_as_usize(&slice, "pattern length")?; + slice = &slice[nr..]; + + let (classes, nr) = ByteClasses::from_bytes(&slice)?; + slice = &slice[nr..]; + + let (len, nr) = + wire::try_read_u32_as_usize(&slice, "sparse transitions length")?; + slice = &slice[nr..]; + + wire::check_slice_len(slice, len, "sparse states byte length")?; + let sparse = &slice[..len]; + slice = &slice[len..]; + + let trans = Transitions { sparse, classes, state_len, pattern_len }; + Ok((trans, slice.as_ptr().as_usize() - slice_start)) + } +} + +impl<T: AsRef<[u8]>> Transitions<T> { + /// Writes a serialized form of this transition table to the buffer given. + /// If the buffer is too small, then an error is returned. To determine + /// how big the buffer must be, use `write_to_len`. + fn write_to<E: Endian>( + &self, + mut dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small( + "sparse transition table", + )); + } + dst = &mut dst[..nwrite]; + + // write state length + E::write_u32(u32::try_from(self.state_len).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + + // write pattern length + E::write_u32(u32::try_from(self.pattern_len).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + + // write byte class map + let n = self.classes.write_to(dst)?; + dst = &mut dst[n..]; + + // write number of bytes in sparse transitions + E::write_u32(u32::try_from(self.sparse().len()).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + + // write actual transitions + let mut id = DEAD; + while id.as_usize() < self.sparse().len() { + let state = self.state(id); + let n = state.write_to::<E>(&mut dst)?; + dst = &mut dst[n..]; + // The next ID is the offset immediately following `state`. + id = StateID::new(id.as_usize() + state.write_to_len()).unwrap(); + } + Ok(nwrite) + } + + /// Returns the number of bytes the serialized form of this transition + /// table will use. + fn write_to_len(&self) -> usize { + size_of::<u32>() // state length + + size_of::<u32>() // pattern length + + self.classes.write_to_len() + + size_of::<u32>() // sparse transitions length + + self.sparse().len() + } + + /// Validates that every state ID in this transition table is valid. + /// + /// That is, every state ID can be used to correctly index a state in this + /// table. + fn validate(&self, sp: &Special) -> Result<Seen, DeserializeError> { + let mut verified = Seen::new(); + // We need to make sure that we decode the correct number of states. + // Otherwise, an empty set of transitions would validate even if the + // recorded state length is non-empty. + let mut len = 0; + // We can't use the self.states() iterator because it assumes the state + // encodings are valid. It could panic if they aren't. + let mut id = DEAD; + while id.as_usize() < self.sparse().len() { + // Before we even decode the state, we check that the ID itself + // is well formed. That is, if it's a special state then it must + // actually be a quit, dead, accel, match or start state. + if sp.is_special_state(id) { + let is_actually_special = sp.is_dead_state(id) + || sp.is_quit_state(id) + || sp.is_match_state(id) + || sp.is_start_state(id) + || sp.is_accel_state(id); + if !is_actually_special { + // This is kind of a cryptic error message... + return Err(DeserializeError::generic( + "found sparse state tagged as special but \ + wasn't actually special", + )); + } + } + let state = self.try_state(sp, id)?; + verified.insert(id); + // The next ID should be the offset immediately following `state`. + id = StateID::new(wire::add( + id.as_usize(), + state.write_to_len(), + "next state ID offset", + )?) + .map_err(|err| { + DeserializeError::state_id_error(err, "next state ID offset") + })?; + len += 1; + } + // Now that we've checked that all top-level states are correct and + // importantly, collected a set of valid state IDs, we have all the + // information we need to check that all transitions are correct too. + // + // Note that we can't use `valid_ids` to iterate because it will + // be empty in no-std no-alloc contexts. (And yes, that means our + // verification isn't quite as good.) We can use `self.states()` + // though at least, since we know that all states can at least be + // decoded and traversed correctly. + for state in self.states() { + // Check that all transitions in this state are correct. + for i in 0..state.ntrans { + let to = state.next_at(i); + // For no-alloc, we just check that the state can decode. It is + // technically possible that the state ID could still point to + // a non-existent state even if it decodes (fuzzing proved this + // to be true), but it shouldn't result in any memory unsafety + // or panics in non-debug mode. + #[cfg(not(feature = "alloc"))] + { + let _ = self.try_state(sp, to)?; + } + #[cfg(feature = "alloc")] + { + if !verified.contains(&to) { + return Err(DeserializeError::generic( + "found transition that points to a \ + non-existent state", + )); + } + } + } + } + if len != self.state_len { + return Err(DeserializeError::generic( + "mismatching sparse state length", + )); + } + Ok(verified) + } + + /// Converts these transitions to a borrowed value. + fn as_ref(&self) -> Transitions<&'_ [u8]> { + Transitions { + sparse: self.sparse(), + classes: self.classes.clone(), + state_len: self.state_len, + pattern_len: self.pattern_len, + } + } + + /// Converts these transitions to an owned value. + #[cfg(feature = "alloc")] + fn to_owned(&self) -> Transitions<alloc::vec::Vec<u8>> { + Transitions { + sparse: self.sparse().to_vec(), + classes: self.classes.clone(), + state_len: self.state_len, + pattern_len: self.pattern_len, + } + } + + /// Return a convenient representation of the given state. + /// + /// This panics if the state is invalid. + /// + /// This is marked as inline to help dramatically boost sparse searching, + /// which decodes each state it enters to follow the next transition. Other + /// functions involved are also inlined, which should hopefully eliminate + /// a lot of the extraneous decoding that is never needed just to follow + /// the next transition. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn state(&self, id: StateID) -> State<'_> { + let mut state = &self.sparse()[id.as_usize()..]; + let mut ntrans = wire::read_u16(&state).as_usize(); + let is_match = (1 << 15) & ntrans != 0; + ntrans &= !(1 << 15); + state = &state[2..]; + + let (input_ranges, state) = state.split_at(ntrans * 2); + let (next, state) = state.split_at(ntrans * StateID::SIZE); + let (pattern_ids, state) = if is_match { + let npats = wire::read_u32(&state).as_usize(); + state[4..].split_at(npats * 4) + } else { + (&[][..], state) + }; + + let accel_len = usize::from(state[0]); + let accel = &state[1..accel_len + 1]; + State { id, is_match, ntrans, input_ranges, next, pattern_ids, accel } + } + + /// Like `state`, but will return an error if the state encoding is + /// invalid. This is useful for verifying states after deserialization, + /// which is required for a safe deserialization API. + /// + /// Note that this only verifies that this state is decodable and that + /// all of its data is consistent. It does not verify that its state ID + /// transitions point to valid states themselves, nor does it verify that + /// every pattern ID is valid. + fn try_state( + &self, + sp: &Special, + id: StateID, + ) -> Result<State<'_>, DeserializeError> { + if id.as_usize() > self.sparse().len() { + return Err(DeserializeError::generic( + "invalid caller provided sparse state ID", + )); + } + let mut state = &self.sparse()[id.as_usize()..]; + // Encoding format starts with a u16 that stores the total number of + // transitions in this state. + let (mut ntrans, _) = + wire::try_read_u16_as_usize(state, "state transition length")?; + let is_match = ((1 << 15) & ntrans) != 0; + ntrans &= !(1 << 15); + state = &state[2..]; + if ntrans > 257 || ntrans == 0 { + return Err(DeserializeError::generic( + "invalid transition length", + )); + } + if is_match && !sp.is_match_state(id) { + return Err(DeserializeError::generic( + "state marked as match but not in match ID range", + )); + } else if !is_match && sp.is_match_state(id) { + return Err(DeserializeError::generic( + "state in match ID range but not marked as match state", + )); + } + + // Each transition has two pieces: an inclusive range of bytes on which + // it is defined, and the state ID that those bytes transition to. The + // pairs come first, followed by a corresponding sequence of state IDs. + let input_ranges_len = ntrans.checked_mul(2).unwrap(); + wire::check_slice_len(state, input_ranges_len, "sparse byte pairs")?; + let (input_ranges, state) = state.split_at(input_ranges_len); + // Every range should be of the form A-B, where A<=B. + for pair in input_ranges.chunks(2) { + let (start, end) = (pair[0], pair[1]); + if start > end { + return Err(DeserializeError::generic("invalid input range")); + } + } + + // And now extract the corresponding sequence of state IDs. We leave + // this sequence as a &[u8] instead of a &[S] because sparse DFAs do + // not have any alignment requirements. + let next_len = ntrans + .checked_mul(self.id_len()) + .expect("state size * #trans should always fit in a usize"); + wire::check_slice_len(state, next_len, "sparse trans state IDs")?; + let (next, state) = state.split_at(next_len); + // We can at least verify that every state ID is in bounds. + for idbytes in next.chunks(self.id_len()) { + let (id, _) = + wire::read_state_id(idbytes, "sparse state ID in try_state")?; + wire::check_slice_len( + self.sparse(), + id.as_usize(), + "invalid sparse state ID", + )?; + } + + // If this is a match state, then read the pattern IDs for this state. + // Pattern IDs is a u32-length prefixed sequence of native endian + // encoded 32-bit integers. + let (pattern_ids, state) = if is_match { + let (npats, nr) = + wire::try_read_u32_as_usize(state, "pattern ID length")?; + let state = &state[nr..]; + if npats == 0 { + return Err(DeserializeError::generic( + "state marked as a match, but pattern length is zero", + )); + } + + let pattern_ids_len = + wire::mul(npats, 4, "sparse pattern ID byte length")?; + wire::check_slice_len( + state, + pattern_ids_len, + "sparse pattern IDs", + )?; + let (pattern_ids, state) = state.split_at(pattern_ids_len); + for patbytes in pattern_ids.chunks(PatternID::SIZE) { + wire::read_pattern_id( + patbytes, + "sparse pattern ID in try_state", + )?; + } + (pattern_ids, state) + } else { + (&[][..], state) + }; + if is_match && pattern_ids.is_empty() { + return Err(DeserializeError::generic( + "state marked as a match, but has no pattern IDs", + )); + } + if sp.is_match_state(id) && pattern_ids.is_empty() { + return Err(DeserializeError::generic( + "state marked special as a match, but has no pattern IDs", + )); + } + if sp.is_match_state(id) != is_match { + return Err(DeserializeError::generic( + "whether state is a match or not is inconsistent", + )); + } + + // Now read this state's accelerator info. The first byte is the length + // of the accelerator, which is typically 0 (for no acceleration) but + // is no bigger than 3. The length indicates the number of bytes that + // follow, where each byte corresponds to a transition out of this + // state. + if state.is_empty() { + return Err(DeserializeError::generic("no accelerator length")); + } + let (accel_len, state) = (usize::from(state[0]), &state[1..]); + + if accel_len > 3 { + return Err(DeserializeError::generic( + "sparse invalid accelerator length", + )); + } else if accel_len == 0 && sp.is_accel_state(id) { + return Err(DeserializeError::generic( + "got no accelerators in state, but in accelerator ID range", + )); + } else if accel_len > 0 && !sp.is_accel_state(id) { + return Err(DeserializeError::generic( + "state in accelerator ID range, but has no accelerators", + )); + } + + wire::check_slice_len( + state, + accel_len, + "sparse corrupt accelerator length", + )?; + let (accel, _) = (&state[..accel_len], &state[accel_len..]); + + let state = State { + id, + is_match, + ntrans, + input_ranges, + next, + pattern_ids, + accel, + }; + if sp.is_quit_state(state.next_at(state.ntrans - 1)) { + return Err(DeserializeError::generic( + "state with EOI transition to quit state is illegal", + )); + } + Ok(state) + } + + /// Return an iterator over all of the states in this DFA. + /// + /// The iterator returned yields tuples, where the first element is the + /// state ID and the second element is the state itself. + fn states(&self) -> StateIter<'_, T> { + StateIter { trans: self, id: DEAD.as_usize() } + } + + /// Returns the sparse transitions as raw bytes. + fn sparse(&self) -> &[u8] { + self.sparse.as_ref() + } + + /// Returns the number of bytes represented by a single state ID. + fn id_len(&self) -> usize { + StateID::SIZE + } + + /// Return the memory usage, in bytes, of these transitions. + /// + /// This does not include the size of a `Transitions` value itself. + fn memory_usage(&self) -> usize { + self.sparse().len() + } +} + +#[cfg(feature = "dfa-build")] +impl<T: AsMut<[u8]>> Transitions<T> { + /// Return a convenient mutable representation of the given state. + /// This panics if the state is invalid. + fn state_mut(&mut self, id: StateID) -> StateMut<'_> { + let mut state = &mut self.sparse_mut()[id.as_usize()..]; + let mut ntrans = wire::read_u16(&state).as_usize(); + let is_match = (1 << 15) & ntrans != 0; + ntrans &= !(1 << 15); + state = &mut state[2..]; + + let (input_ranges, state) = state.split_at_mut(ntrans * 2); + let (next, state) = state.split_at_mut(ntrans * StateID::SIZE); + let (pattern_ids, state) = if is_match { + let npats = wire::read_u32(&state).as_usize(); + state[4..].split_at_mut(npats * 4) + } else { + (&mut [][..], state) + }; + + let accel_len = usize::from(state[0]); + let accel = &mut state[1..accel_len + 1]; + StateMut { + id, + is_match, + ntrans, + input_ranges, + next, + pattern_ids, + accel, + } + } + + /// Returns the sparse transitions as raw mutable bytes. + fn sparse_mut(&mut self) -> &mut [u8] { + self.sparse.as_mut() + } +} + +/// The set of all possible starting states in a DFA. +/// +/// See the eponymous type in the `dense` module for more details. This type +/// is very similar to `dense::StartTable`, except that its underlying +/// representation is `&[u8]` instead of `&[S]`. (The latter would require +/// sparse DFAs to be aligned, which is explicitly something we do not require +/// because we don't really need it.) +#[derive(Clone)] +struct StartTable<T> { + /// The initial start state IDs as a contiguous table of native endian + /// encoded integers, represented by `S`. + /// + /// In practice, T is either Vec<u8> or &[u8] and has no alignment + /// requirements. + /// + /// The first `2 * stride` (currently always 8) entries always correspond + /// to the starts states for the entire DFA, with the first 4 entries being + /// for unanchored searches and the second 4 entries being for anchored + /// searches. To keep things simple, we always use 8 entries even if the + /// `StartKind` is not both. + /// + /// After that, there are `stride * patterns` state IDs, where `patterns` + /// may be zero in the case of a DFA with no patterns or in the case where + /// the DFA was built without enabling starting states for each pattern. + table: T, + /// The starting state configuration supported. When 'both', both + /// unanchored and anchored searches work. When 'unanchored', anchored + /// searches panic. When 'anchored', unanchored searches panic. + kind: StartKind, + /// The start state configuration for every possible byte. + start_map: StartByteMap, + /// The number of starting state IDs per pattern. + stride: usize, + /// The total number of patterns for which starting states are encoded. + /// This is `None` for DFAs that were built without start states for each + /// pattern. Thus, one cannot use this field to say how many patterns + /// are in the DFA in all cases. It is specific to how many patterns are + /// represented in this start table. + pattern_len: Option<usize>, + /// The universal starting state for unanchored searches. This is only + /// present when the DFA supports unanchored searches and when all starting + /// state IDs for an unanchored search are equivalent. + universal_start_unanchored: Option<StateID>, + /// The universal starting state for anchored searches. This is only + /// present when the DFA supports anchored searches and when all starting + /// state IDs for an anchored search are equivalent. + universal_start_anchored: Option<StateID>, +} + +#[cfg(feature = "dfa-build")] +impl StartTable<Vec<u8>> { + fn new<T: AsRef<[u32]>>( + dfa: &dense::DFA<T>, + pattern_len: Option<usize>, + ) -> StartTable<Vec<u8>> { + let stride = Start::len(); + // This is OK since the only way we're here is if a dense DFA could be + // constructed successfully, which uses the same space. + let len = stride + .checked_mul(pattern_len.unwrap_or(0)) + .unwrap() + .checked_add(stride.checked_mul(2).unwrap()) + .unwrap() + .checked_mul(StateID::SIZE) + .unwrap(); + StartTable { + table: vec![0; len], + kind: dfa.start_kind(), + start_map: dfa.start_map().clone(), + stride, + pattern_len, + universal_start_unanchored: dfa + .universal_start_state(Anchored::No), + universal_start_anchored: dfa.universal_start_state(Anchored::Yes), + } + } + + fn from_dense_dfa<T: AsRef<[u32]>>( + dfa: &dense::DFA<T>, + remap: &[StateID], + ) -> Result<StartTable<Vec<u8>>, BuildError> { + // Unless the DFA has start states compiled for each pattern, then + // as far as the starting state table is concerned, there are zero + // patterns to account for. It will instead only store starting states + // for the entire DFA. + let start_pattern_len = if dfa.starts_for_each_pattern() { + Some(dfa.pattern_len()) + } else { + None + }; + let mut sl = StartTable::new(dfa, start_pattern_len); + for (old_start_id, anchored, sty) in dfa.starts() { + let new_start_id = remap[dfa.to_index(old_start_id)]; + sl.set_start(anchored, sty, new_start_id); + } + if let Some(ref mut id) = sl.universal_start_anchored { + *id = remap[dfa.to_index(*id)]; + } + if let Some(ref mut id) = sl.universal_start_unanchored { + *id = remap[dfa.to_index(*id)]; + } + Ok(sl) + } +} + +impl<'a> StartTable<&'a [u8]> { + unsafe fn from_bytes_unchecked( + mut slice: &'a [u8], + ) -> Result<(StartTable<&'a [u8]>, usize), DeserializeError> { + let slice_start = slice.as_ptr().as_usize(); + + let (kind, nr) = StartKind::from_bytes(slice)?; + slice = &slice[nr..]; + + let (start_map, nr) = StartByteMap::from_bytes(slice)?; + slice = &slice[nr..]; + + let (stride, nr) = + wire::try_read_u32_as_usize(slice, "sparse start table stride")?; + slice = &slice[nr..]; + if stride != Start::len() { + return Err(DeserializeError::generic( + "invalid sparse starting table stride", + )); + } + + let (maybe_pattern_len, nr) = + wire::try_read_u32_as_usize(slice, "sparse start table patterns")?; + slice = &slice[nr..]; + let pattern_len = if maybe_pattern_len.as_u32() == u32::MAX { + None + } else { + Some(maybe_pattern_len) + }; + if pattern_len.map_or(false, |len| len > PatternID::LIMIT) { + return Err(DeserializeError::generic( + "sparse invalid number of patterns", + )); + } + + let (universal_unanchored, nr) = + wire::try_read_u32(slice, "universal unanchored start")?; + slice = &slice[nr..]; + let universal_start_unanchored = if universal_unanchored == u32::MAX { + None + } else { + Some(StateID::try_from(universal_unanchored).map_err(|e| { + DeserializeError::state_id_error( + e, + "universal unanchored start", + ) + })?) + }; + + let (universal_anchored, nr) = + wire::try_read_u32(slice, "universal anchored start")?; + slice = &slice[nr..]; + let universal_start_anchored = if universal_anchored == u32::MAX { + None + } else { + Some(StateID::try_from(universal_anchored).map_err(|e| { + DeserializeError::state_id_error(e, "universal anchored start") + })?) + }; + + let pattern_table_size = wire::mul( + stride, + pattern_len.unwrap_or(0), + "sparse invalid pattern length", + )?; + // Our start states always start with a single stride of start states + // for the entire automaton which permit it to match any pattern. What + // follows it are an optional set of start states for each pattern. + let start_state_len = wire::add( + wire::mul(2, stride, "start state stride too big")?, + pattern_table_size, + "sparse invalid 'any' pattern starts size", + )?; + let table_bytes_len = wire::mul( + start_state_len, + StateID::SIZE, + "sparse pattern table bytes length", + )?; + wire::check_slice_len( + slice, + table_bytes_len, + "sparse start ID table", + )?; + let table = &slice[..table_bytes_len]; + slice = &slice[table_bytes_len..]; + + let sl = StartTable { + table, + kind, + start_map, + stride, + pattern_len, + universal_start_unanchored, + universal_start_anchored, + }; + Ok((sl, slice.as_ptr().as_usize() - slice_start)) + } +} + +impl<T: AsRef<[u8]>> StartTable<T> { + fn write_to<E: Endian>( + &self, + mut dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small( + "sparse starting table ids", + )); + } + dst = &mut dst[..nwrite]; + + // write start kind + let nw = self.kind.write_to::<E>(dst)?; + dst = &mut dst[nw..]; + // write start byte map + let nw = self.start_map.write_to(dst)?; + dst = &mut dst[nw..]; + // write stride + E::write_u32(u32::try_from(self.stride).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + // write pattern length + E::write_u32( + u32::try_from(self.pattern_len.unwrap_or(0xFFFF_FFFF)).unwrap(), + dst, + ); + dst = &mut dst[size_of::<u32>()..]; + // write universal start unanchored state id, u32::MAX if absent + E::write_u32( + self.universal_start_unanchored + .map_or(u32::MAX, |sid| sid.as_u32()), + dst, + ); + dst = &mut dst[size_of::<u32>()..]; + // write universal start anchored state id, u32::MAX if absent + E::write_u32( + self.universal_start_anchored.map_or(u32::MAX, |sid| sid.as_u32()), + dst, + ); + dst = &mut dst[size_of::<u32>()..]; + // write start IDs + for (sid, _, _) in self.iter() { + E::write_u32(sid.as_u32(), dst); + dst = &mut dst[StateID::SIZE..]; + } + Ok(nwrite) + } + + /// Returns the number of bytes the serialized form of this transition + /// table will use. + fn write_to_len(&self) -> usize { + self.kind.write_to_len() + + self.start_map.write_to_len() + + size_of::<u32>() // stride + + size_of::<u32>() // # patterns + + size_of::<u32>() // universal unanchored start + + size_of::<u32>() // universal anchored start + + self.table().len() + } + + /// Validates that every starting state ID in this table is valid. + /// + /// That is, every starting state ID can be used to correctly decode a + /// state in the DFA's sparse transitions. + fn validate( + &self, + sp: &Special, + seen: &Seen, + ) -> Result<(), DeserializeError> { + for (id, _, _) in self.iter() { + if !seen.contains(&id) { + return Err(DeserializeError::generic( + "found invalid starting state ID", + )); + } + if sp.is_match_state(id) { + return Err(DeserializeError::generic( + "start states cannot be match states", + )); + } + } + Ok(()) + } + + /// Converts this start list to a borrowed value. + fn as_ref(&self) -> StartTable<&'_ [u8]> { + StartTable { + table: self.table(), + kind: self.kind, + start_map: self.start_map.clone(), + stride: self.stride, + pattern_len: self.pattern_len, + universal_start_unanchored: self.universal_start_unanchored, + universal_start_anchored: self.universal_start_anchored, + } + } + + /// Converts this start list to an owned value. + #[cfg(feature = "alloc")] + fn to_owned(&self) -> StartTable<alloc::vec::Vec<u8>> { + StartTable { + table: self.table().to_vec(), + kind: self.kind, + start_map: self.start_map.clone(), + stride: self.stride, + pattern_len: self.pattern_len, + universal_start_unanchored: self.universal_start_unanchored, + universal_start_anchored: self.universal_start_anchored, + } + } + + /// Return the start state for the given index and pattern ID. If the + /// pattern ID is None, then the corresponding start state for the entire + /// DFA is returned. If the pattern ID is not None, then the corresponding + /// starting state for the given pattern is returned. If this start table + /// does not have individual starting states for each pattern, then this + /// panics. + fn start( + &self, + anchored: Anchored, + start: Start, + ) -> Result<StateID, StartError> { + let start_index = start.as_usize(); + let index = match anchored { + Anchored::No => { + if !self.kind.has_unanchored() { + return Err(StartError::unsupported_anchored(anchored)); + } + start_index + } + Anchored::Yes => { + if !self.kind.has_anchored() { + return Err(StartError::unsupported_anchored(anchored)); + } + self.stride + start_index + } + Anchored::Pattern(pid) => { + let len = match self.pattern_len { + None => { + return Err(StartError::unsupported_anchored(anchored)) + } + Some(len) => len, + }; + if pid.as_usize() >= len { + return Ok(DEAD); + } + (2 * self.stride) + + (self.stride * pid.as_usize()) + + start_index + } + }; + let start = index * StateID::SIZE; + // This OK since we're allowed to assume that the start table contains + // valid StateIDs. + Ok(wire::read_state_id_unchecked(&self.table()[start..]).0) + } + + /// Return an iterator over all start IDs in this table. + fn iter(&self) -> StartStateIter<'_, T> { + StartStateIter { st: self, i: 0 } + } + + /// Returns the total number of start state IDs in this table. + fn len(&self) -> usize { + self.table().len() / StateID::SIZE + } + + /// Returns the table as a raw slice of bytes. + fn table(&self) -> &[u8] { + self.table.as_ref() + } + + /// Return the memory usage, in bytes, of this start list. + /// + /// This does not include the size of a `StartTable` value itself. + fn memory_usage(&self) -> usize { + self.table().len() + } +} + +#[cfg(feature = "dfa-build")] +impl<T: AsMut<[u8]>> StartTable<T> { + /// Set the start state for the given index and pattern. + /// + /// If the pattern ID or state ID are not valid, then this will panic. + fn set_start(&mut self, anchored: Anchored, start: Start, id: StateID) { + let start_index = start.as_usize(); + let index = match anchored { + Anchored::No => start_index, + Anchored::Yes => self.stride + start_index, + Anchored::Pattern(pid) => { + let pid = pid.as_usize(); + let len = self + .pattern_len + .expect("start states for each pattern enabled"); + assert!(pid < len, "invalid pattern ID {pid:?}"); + self.stride + .checked_mul(pid) + .unwrap() + .checked_add(self.stride.checked_mul(2).unwrap()) + .unwrap() + .checked_add(start_index) + .unwrap() + } + }; + let start = index * StateID::SIZE; + let end = start + StateID::SIZE; + wire::write_state_id::<wire::NE>( + id, + &mut self.table.as_mut()[start..end], + ); + } +} + +/// An iterator over all state state IDs in a sparse DFA. +struct StartStateIter<'a, T> { + st: &'a StartTable<T>, + i: usize, +} + +impl<'a, T: AsRef<[u8]>> Iterator for StartStateIter<'a, T> { + type Item = (StateID, Anchored, Start); + + fn next(&mut self) -> Option<(StateID, Anchored, Start)> { + let i = self.i; + if i >= self.st.len() { + return None; + } + self.i += 1; + + // This unwrap is okay since the stride of any DFA must always match + // the number of start state types. + let start_type = Start::from_usize(i % self.st.stride).unwrap(); + let anchored = if i < self.st.stride { + Anchored::No + } else if i < (2 * self.st.stride) { + Anchored::Yes + } else { + let pid = (i - (2 * self.st.stride)) / self.st.stride; + Anchored::Pattern(PatternID::new(pid).unwrap()) + }; + let start = i * StateID::SIZE; + let end = start + StateID::SIZE; + let bytes = self.st.table()[start..end].try_into().unwrap(); + // This is OK since we're allowed to assume that any IDs in this start + // table are correct and valid for this DFA. + let id = StateID::from_ne_bytes_unchecked(bytes); + Some((id, anchored, start_type)) + } +} + +impl<'a, T> fmt::Debug for StartStateIter<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("StartStateIter").field("i", &self.i).finish() + } +} + +/// An iterator over all states in a sparse DFA. +/// +/// This iterator yields tuples, where the first element is the state ID and +/// the second element is the state itself. +struct StateIter<'a, T> { + trans: &'a Transitions<T>, + id: usize, +} + +impl<'a, T: AsRef<[u8]>> Iterator for StateIter<'a, T> { + type Item = State<'a>; + + fn next(&mut self) -> Option<State<'a>> { + if self.id >= self.trans.sparse().len() { + return None; + } + let state = self.trans.state(StateID::new_unchecked(self.id)); + self.id = self.id + state.write_to_len(); + Some(state) + } +} + +impl<'a, T> fmt::Debug for StateIter<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("StateIter").field("id", &self.id).finish() + } +} + +/// A representation of a sparse DFA state that can be cheaply materialized +/// from a state identifier. +#[derive(Clone)] +struct State<'a> { + /// The identifier of this state. + id: StateID, + /// Whether this is a match state or not. + is_match: bool, + /// The number of transitions in this state. + ntrans: usize, + /// Pairs of input ranges, where there is one pair for each transition. + /// Each pair specifies an inclusive start and end byte range for the + /// corresponding transition. + input_ranges: &'a [u8], + /// Transitions to the next state. This slice contains native endian + /// encoded state identifiers, with `S` as the representation. Thus, there + /// are `ntrans * size_of::<S>()` bytes in this slice. + next: &'a [u8], + /// If this is a match state, then this contains the pattern IDs that match + /// when the DFA is in this state. + /// + /// This is a contiguous sequence of 32-bit native endian encoded integers. + pattern_ids: &'a [u8], + /// An accelerator for this state, if present. If this state has no + /// accelerator, then this is an empty slice. When non-empty, this slice + /// has length at most 3 and corresponds to the exhaustive set of bytes + /// that must be seen in order to transition out of this state. + accel: &'a [u8], +} + +impl<'a> State<'a> { + /// Searches for the next transition given an input byte. If no such + /// transition could be found, then a dead state is returned. + /// + /// This is marked as inline to help dramatically boost sparse searching, + /// which decodes each state it enters to follow the next transition. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn next(&self, input: u8) -> StateID { + // This straight linear search was observed to be much better than + // binary search on ASCII haystacks, likely because a binary search + // visits the ASCII case last but a linear search sees it first. A + // binary search does do a little better on non-ASCII haystacks, but + // not by much. There might be a better trade off lurking here. + for i in 0..(self.ntrans - 1) { + let (start, end) = self.range(i); + if start <= input && input <= end { + return self.next_at(i); + } + // We could bail early with an extra branch: if input < b1, then + // we know we'll never find a matching transition. Interestingly, + // this extra branch seems to not help performance, or will even + // hurt it. It's likely very dependent on the DFA itself and what + // is being searched. + } + DEAD + } + + /// Returns the next state ID for the special EOI transition. + fn next_eoi(&self) -> StateID { + self.next_at(self.ntrans - 1) + } + + /// Returns the identifier for this state. + fn id(&self) -> StateID { + self.id + } + + /// Returns the inclusive input byte range for the ith transition in this + /// state. + fn range(&self, i: usize) -> (u8, u8) { + (self.input_ranges[i * 2], self.input_ranges[i * 2 + 1]) + } + + /// Returns the next state for the ith transition in this state. + fn next_at(&self, i: usize) -> StateID { + let start = i * StateID::SIZE; + let end = start + StateID::SIZE; + let bytes = self.next[start..end].try_into().unwrap(); + StateID::from_ne_bytes_unchecked(bytes) + } + + /// Returns the pattern ID for the given match index. If the match index + /// is invalid, then this panics. + fn pattern_id(&self, match_index: usize) -> PatternID { + let start = match_index * PatternID::SIZE; + wire::read_pattern_id_unchecked(&self.pattern_ids[start..]).0 + } + + /// Returns the total number of pattern IDs for this state. This is always + /// zero when `is_match` is false. + fn pattern_len(&self) -> usize { + assert_eq!(0, self.pattern_ids.len() % 4); + self.pattern_ids.len() / 4 + } + + /// Return an accelerator for this state. + fn accelerator(&self) -> &'a [u8] { + self.accel + } + + /// Write the raw representation of this state to the given buffer using + /// the given endianness. + fn write_to<E: Endian>( + &self, + mut dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small( + "sparse state transitions", + )); + } + + let ntrans = + if self.is_match { self.ntrans | (1 << 15) } else { self.ntrans }; + E::write_u16(u16::try_from(ntrans).unwrap(), dst); + dst = &mut dst[size_of::<u16>()..]; + + dst[..self.input_ranges.len()].copy_from_slice(self.input_ranges); + dst = &mut dst[self.input_ranges.len()..]; + + for i in 0..self.ntrans { + E::write_u32(self.next_at(i).as_u32(), dst); + dst = &mut dst[StateID::SIZE..]; + } + + if self.is_match { + E::write_u32(u32::try_from(self.pattern_len()).unwrap(), dst); + dst = &mut dst[size_of::<u32>()..]; + for i in 0..self.pattern_len() { + let pid = self.pattern_id(i); + E::write_u32(pid.as_u32(), dst); + dst = &mut dst[PatternID::SIZE..]; + } + } + + dst[0] = u8::try_from(self.accel.len()).unwrap(); + dst[1..][..self.accel.len()].copy_from_slice(self.accel); + + Ok(nwrite) + } + + /// Return the total number of bytes that this state consumes in its + /// encoded form. + fn write_to_len(&self) -> usize { + let mut len = 2 + + (self.ntrans * 2) + + (self.ntrans * StateID::SIZE) + + (1 + self.accel.len()); + if self.is_match { + len += size_of::<u32>() + self.pattern_ids.len(); + } + len + } +} + +impl<'a> fmt::Debug for State<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut printed = false; + for i in 0..(self.ntrans - 1) { + let next = self.next_at(i); + if next == DEAD { + continue; + } + + if printed { + write!(f, ", ")?; + } + let (start, end) = self.range(i); + if start == end { + write!(f, "{:?} => {:?}", DebugByte(start), next.as_usize())?; + } else { + write!( + f, + "{:?}-{:?} => {:?}", + DebugByte(start), + DebugByte(end), + next.as_usize(), + )?; + } + printed = true; + } + let eoi = self.next_at(self.ntrans - 1); + if eoi != DEAD { + if printed { + write!(f, ", ")?; + } + write!(f, "EOI => {:?}", eoi.as_usize())?; + } + Ok(()) + } +} + +/// A representation of a mutable sparse DFA state that can be cheaply +/// materialized from a state identifier. +#[cfg(feature = "dfa-build")] +struct StateMut<'a> { + /// The identifier of this state. + id: StateID, + /// Whether this is a match state or not. + is_match: bool, + /// The number of transitions in this state. + ntrans: usize, + /// Pairs of input ranges, where there is one pair for each transition. + /// Each pair specifies an inclusive start and end byte range for the + /// corresponding transition. + input_ranges: &'a mut [u8], + /// Transitions to the next state. This slice contains native endian + /// encoded state identifiers, with `S` as the representation. Thus, there + /// are `ntrans * size_of::<S>()` bytes in this slice. + next: &'a mut [u8], + /// If this is a match state, then this contains the pattern IDs that match + /// when the DFA is in this state. + /// + /// This is a contiguous sequence of 32-bit native endian encoded integers. + pattern_ids: &'a [u8], + /// An accelerator for this state, if present. If this state has no + /// accelerator, then this is an empty slice. When non-empty, this slice + /// has length at most 3 and corresponds to the exhaustive set of bytes + /// that must be seen in order to transition out of this state. + accel: &'a mut [u8], +} + +#[cfg(feature = "dfa-build")] +impl<'a> StateMut<'a> { + /// Sets the ith transition to the given state. + fn set_next_at(&mut self, i: usize, next: StateID) { + let start = i * StateID::SIZE; + let end = start + StateID::SIZE; + wire::write_state_id::<wire::NE>(next, &mut self.next[start..end]); + } +} + +#[cfg(feature = "dfa-build")] +impl<'a> fmt::Debug for StateMut<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let state = State { + id: self.id, + is_match: self.is_match, + ntrans: self.ntrans, + input_ranges: self.input_ranges, + next: self.next, + pattern_ids: self.pattern_ids, + accel: self.accel, + }; + fmt::Debug::fmt(&state, f) + } +} + +// In order to validate everything, we not only need to make sure we +// can decode every state, but that every transition in every state +// points to a valid state. There are many duplicative transitions, so +// we record state IDs that we've verified so that we don't redo the +// decoding work. +// +// Except, when in no_std mode, we don't have dynamic memory allocation +// available to us, so we skip this optimization. It's not clear +// whether doing something more clever is worth it just yet. If you're +// profiling this code and need it to run faster, please file an issue. +// +// OK, so we also use this to record the set of valid state IDs. Since +// it is possible for a transition to point to an invalid state ID that +// still (somehow) deserializes to a valid state. So we need to make +// sure our transitions are limited to actually correct state IDs. +// The problem is, I'm not sure how to do this verification step in +// no-std no-alloc mode. I think we'd *have* to store the set of valid +// state IDs in the DFA itself. For now, we don't do this verification +// in no-std no-alloc mode. The worst thing that can happen is an +// incorrect result. But no panics or memory safety problems should +// result. Because we still do validate that the state itself is +// "valid" in the sense that everything it points to actually exists. +// +// ---AG +#[derive(Debug)] +struct Seen { + #[cfg(feature = "alloc")] + set: alloc::collections::BTreeSet<StateID>, + #[cfg(not(feature = "alloc"))] + set: core::marker::PhantomData<StateID>, +} + +#[cfg(feature = "alloc")] +impl Seen { + fn new() -> Seen { + Seen { set: alloc::collections::BTreeSet::new() } + } + fn insert(&mut self, id: StateID) { + self.set.insert(id); + } + fn contains(&self, id: &StateID) -> bool { + self.set.contains(id) + } +} + +#[cfg(not(feature = "alloc"))] +impl Seen { + fn new() -> Seen { + Seen { set: core::marker::PhantomData } + } + fn insert(&mut self, _id: StateID) {} + fn contains(&self, _id: &StateID) -> bool { + true + } +} + +/* +/// A binary search routine specialized specifically to a sparse DFA state's +/// transitions. Specifically, the transitions are defined as a set of pairs +/// of input bytes that delineate an inclusive range of bytes. If the input +/// byte is in the range, then the corresponding transition is a match. +/// +/// This binary search accepts a slice of these pairs and returns the position +/// of the matching pair (the ith transition), or None if no matching pair +/// could be found. +/// +/// Note that this routine is not currently used since it was observed to +/// either decrease performance when searching ASCII, or did not provide enough +/// of a boost on non-ASCII haystacks to be worth it. However, we leave it here +/// for posterity in case we can find a way to use it. +/// +/// In theory, we could use the standard library's search routine if we could +/// cast a `&[u8]` to a `&[(u8, u8)]`, but I don't believe this is currently +/// guaranteed to be safe and is thus UB (since I don't think the in-memory +/// representation of `(u8, u8)` has been nailed down). One could define a +/// repr(C) type, but the casting doesn't seem justified. +#[cfg_attr(feature = "perf-inline", inline(always))] +fn binary_search_ranges(ranges: &[u8], needle: u8) -> Option<usize> { + debug_assert!(ranges.len() % 2 == 0, "ranges must have even length"); + debug_assert!(ranges.len() <= 512, "ranges should be short"); + + let (mut left, mut right) = (0, ranges.len() / 2); + while left < right { + let mid = (left + right) / 2; + let (b1, b2) = (ranges[mid * 2], ranges[mid * 2 + 1]); + if needle < b1 { + right = mid; + } else if needle > b2 { + left = mid + 1; + } else { + return Some(mid); + } + } + None +} +*/ + +#[cfg(all(test, feature = "syntax", feature = "dfa-build"))] +mod tests { + use crate::{ + dfa::{dense::DFA, Automaton}, + nfa::thompson, + Input, MatchError, + }; + + // See the analogous test in src/hybrid/dfa.rs and src/dfa/dense.rs. + #[test] + fn heuristic_unicode_forward() { + let dfa = DFA::builder() + .configure(DFA::config().unicode_word_boundary(true)) + .thompson(thompson::Config::new().reverse(true)) + .build(r"\b[0-9]+\b") + .unwrap() + .to_sparse() + .unwrap(); + + let input = Input::new("β123").range(2..); + let expected = MatchError::quit(0xB2, 1); + let got = dfa.try_search_fwd(&input); + assert_eq!(Err(expected), got); + + let input = Input::new("123β").range(..3); + let expected = MatchError::quit(0xCE, 3); + let got = dfa.try_search_fwd(&input); + assert_eq!(Err(expected), got); + } + + // See the analogous test in src/hybrid/dfa.rs and src/dfa/dense.rs. + #[test] + fn heuristic_unicode_reverse() { + let dfa = DFA::builder() + .configure(DFA::config().unicode_word_boundary(true)) + .thompson(thompson::Config::new().reverse(true)) + .build(r"\b[0-9]+\b") + .unwrap() + .to_sparse() + .unwrap(); + + let input = Input::new("β123").range(2..); + let expected = MatchError::quit(0xB2, 1); + let got = dfa.try_search_rev(&input); + assert_eq!(Err(expected), got); + + let input = Input::new("123β").range(..3); + let expected = MatchError::quit(0xCE, 3); + let got = dfa.try_search_rev(&input); + assert_eq!(Err(expected), got); + } + + // A starting state can never be a match state, since all matches are + // delayed by one byte. The sparse DFA already rejects a serialized DFA + // whose start table points at a match state, but only the dense check + // had a regression test. See the test of the same name in + // src/dfa/dense.rs. + #[test] + fn regression_start_state_not_match() { + use crate::util::primitives::StateID; + + let mut dfa = DFA::new("abc").unwrap().to_sparse().unwrap(); + let min_match = dfa.special.min_match.as_u32().to_ne_bytes(); + for entry in dfa.st.table.chunks_mut(StateID::SIZE) { + entry.copy_from_slice(&min_match); + } + let buf = dfa.to_bytes_native_endian(); + crate::dfa::sparse::DFA::from_bytes(&buf).unwrap_err(); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/special.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/special.rs new file mode 100644 index 0000000..1973231 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/special.rs @@ -0,0 +1,494 @@ +use crate::{ + dfa::DEAD, + util::{ + primitives::StateID, + wire::{self, DeserializeError, Endian, SerializeError}, + }, +}; + +macro_rules! err { + ($msg:expr) => { + return Err(DeserializeError::generic($msg)); + }; +} + +// Special represents the identifiers in a DFA that correspond to "special" +// states. If a state is one or more of the following, then it is considered +// special: +// +// * dead - A non-matching state where all outgoing transitions lead back to +// itself. There is only one of these, regardless of whether minimization +// has run. The dead state always has an ID of 0. i.e., It is always the +// first state in a DFA. +// * quit - A state that is entered whenever a byte is seen that should cause +// a DFA to give up and stop searching. This results in a MatchError::quit +// error being returned at search time. The default configuration for a DFA +// has no quit bytes, which means this state is unreachable by default, +// although it is always present for reasons of implementation simplicity. +// This state is only reachable when the caller configures the DFA to quit +// on certain bytes. There is always exactly one of these states and it +// is always the second state. (Its actual ID depends on the size of the +// alphabet in dense DFAs, since state IDs are premultiplied in order to +// allow them to be used directly as indices into the transition table.) +// * match - An accepting state, i.e., indicative of a match. There may be +// zero or more of these states. +// * accelerated - A state where all of its outgoing transitions, except a +// few, loop back to itself. These states are candidates for acceleration +// via memchr during search. There may be zero or more of these states. +// * start - A non-matching state that indicates where the automaton should +// start during a search. There is always at least one starting state and +// all are guaranteed to be non-match states. (A start state cannot be a +// match state because the DFAs in this crate delay all matches by one byte. +// So every search that finds a match must move through one transition to +// some other match state, even when searching an empty string.) +// +// These are not mutually exclusive categories. Namely, the following +// overlapping can occur: +// +// * {dead, start} - If a DFA can never lead to a match and it is minimized, +// then it will typically compile to something where all starting IDs point +// to the DFA's dead state. +// * {match, accelerated} - It is possible for a match state to have the +// majority of its transitions loop back to itself, which means it's +// possible for a match state to be accelerated. +// * {start, accelerated} - Similarly, it is possible for a start state to be +// accelerated. Note that it is possible for an accelerated state to be +// neither a match or a start state. Also note that just because both match +// and start states overlap with accelerated states does not mean that +// match and start states overlap with each other. In fact, they are +// guaranteed not to overlap. +// +// As a special mention, every DFA always has a dead and a quit state, even +// though from the perspective of the DFA, they are equivalent. (Indeed, +// minimization special cases them to ensure they don't get merged.) The +// purpose of keeping them distinct is to use the quit state as a sentinel to +// distinguish between whether a search finished successfully without finding +// anything or whether it gave up before finishing. +// +// So the main problem we want to solve here is the *fast* detection of whether +// a state is special or not. And we also want to do this while storing as +// little extra data as possible. AND we want to be able to quickly determine +// which categories a state falls into above if it is special. +// +// We achieve this by essentially shuffling all special states to the beginning +// of a DFA. That is, all special states appear before every other non-special +// state. By representing special states this way, we can determine whether a +// state is special or not by a single comparison, where special.max is the +// identifier of the last special state in the DFA: +// +// if current_state <= special.max: +// ... do something with special state +// +// The only thing left to do is to determine what kind of special state +// it is. Because what we do next depends on that. Since special states +// are typically rare, we can afford to do a bit more extra work, but we'd +// still like this to be as fast as possible. The trick we employ here is to +// continue shuffling states even within the special state range. Such that +// one contiguous region corresponds to match states, another for start states +// and then an overlapping range for accelerated states. At a high level, our +// special state detection might look like this (for leftmost searching, where +// we continue searching even after seeing a match): +// +// byte = input[offset] +// current_state = next_state(current_state, byte) +// offset += 1 +// if current_state <= special.max: +// if current_state == 0: +// # We can never leave a dead state, so this always marks the +// # end of our search. +// return last_match +// if current_state == special.quit_id: +// # A quit state means we give up. If he DFA has no quit state, +// # then special.quit_id == 0 == dead, which is handled by the +// # conditional above. +// return Err(MatchError::quit { byte, offset: offset - 1 }) +// if special.min_match <= current_state <= special.max_match: +// last_match = Some(offset) +// if special.min_accel <= current_state <= special.max_accel: +// offset = accelerate(input, offset) +// last_match = Some(offset) +// elif special.min_start <= current_state <= special.max_start: +// offset = prefilter.find(input, offset) +// if special.min_accel <= current_state <= special.max_accel: +// offset = accelerate(input, offset) +// elif special.min_accel <= current_state <= special.max_accel: +// offset = accelerate(input, offset) +// +// There are some small details left out of the logic above. For example, +// in order to accelerate a state, we need to know which bytes to search for. +// This in turn implies some extra data we need to store in the DFA. To keep +// things compact, we would ideally only store +// +// N = special.max_accel - special.min_accel + 1 +// +// items. But state IDs are premultiplied, which means they are not contiguous. +// So in order to take a state ID and index an array of accelerated structures, +// we need to do: +// +// i = (state_id - special.min_accel) / stride +// +// (N.B. 'stride' is always a power of 2, so the above can be implemented via +// '(state_id - special.min_accel) >> stride2', where 'stride2' is x in +// 2^x=stride.) +// +// Moreover, some of these specialty categories may be empty. For example, +// DFAs are not required to have any match states or any accelerated states. +// In that case, the lower and upper bounds are both set to 0 (the dead state +// ID) and the first `current_state == 0` check subsumes cases where the +// ranges are empty. +// +// Loop unrolling, if applicable, has also been left out of the logic above. +// +// Graphically, the ranges look like this, where asterisks indicate ranges +// that can be empty. Each 'x' is a state. +// +// quit +// dead| +// || +// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +// | | | | start | | +// | |-------------| |-------| | +// | match* | | | | +// | | | | | +// | |----------| | | +// | accel* | | +// | | | +// | | | +// |----------------------------|------------------------ +// special non-special* +#[derive(Clone, Copy, Debug)] +pub(crate) struct Special { + /// The identifier of the last special state in a DFA. A state is special + /// if and only if its identifier is less than or equal to `max`. + pub(crate) max: StateID, + /// The identifier of the quit state in a DFA. (There is no analogous field + /// for the dead state since the dead state's ID is always zero, regardless + /// of state ID size.) + pub(crate) quit_id: StateID, + /// The identifier of the first match state. + pub(crate) min_match: StateID, + /// The identifier of the last match state. + pub(crate) max_match: StateID, + /// The identifier of the first accelerated state. + pub(crate) min_accel: StateID, + /// The identifier of the last accelerated state. + pub(crate) max_accel: StateID, + /// The identifier of the first start state. + pub(crate) min_start: StateID, + /// The identifier of the last start state. + pub(crate) max_start: StateID, +} + +impl Special { + /// Creates a new set of special ranges for a DFA. All ranges are initially + /// set to only contain the dead state. This is interpreted as an empty + /// range. + #[cfg(feature = "dfa-build")] + pub(crate) fn new() -> Special { + Special { + max: DEAD, + quit_id: DEAD, + min_match: DEAD, + max_match: DEAD, + min_accel: DEAD, + max_accel: DEAD, + min_start: DEAD, + max_start: DEAD, + } + } + + /// Remaps all of the special state identifiers using the function given. + #[cfg(feature = "dfa-build")] + pub(crate) fn remap(&self, map: impl Fn(StateID) -> StateID) -> Special { + Special { + max: map(self.max), + quit_id: map(self.quit_id), + min_match: map(self.min_match), + max_match: map(self.max_match), + min_accel: map(self.min_accel), + max_accel: map(self.max_accel), + min_start: map(self.min_start), + max_start: map(self.max_start), + } + } + + /// Deserialize the given bytes into special state ranges. If the slice + /// given is not big enough, then this returns an error. Similarly, if + /// any of the expected invariants around special state ranges aren't + /// upheld, an error is returned. Note that this does not guarantee that + /// the information returned is correct. + /// + /// Upon success, this returns the number of bytes read in addition to the + /// special state IDs themselves. + pub(crate) fn from_bytes( + mut slice: &[u8], + ) -> Result<(Special, usize), DeserializeError> { + wire::check_slice_len(slice, 8 * StateID::SIZE, "special states")?; + + let mut nread = 0; + let mut read_id = |what| -> Result<StateID, DeserializeError> { + let (id, nr) = wire::try_read_state_id(slice, what)?; + nread += nr; + slice = &slice[StateID::SIZE..]; + Ok(id) + }; + + let max = read_id("special max id")?; + let quit_id = read_id("special quit id")?; + let min_match = read_id("special min match id")?; + let max_match = read_id("special max match id")?; + let min_accel = read_id("special min accel id")?; + let max_accel = read_id("special max accel id")?; + let min_start = read_id("special min start id")?; + let max_start = read_id("special max start id")?; + + let special = Special { + max, + quit_id, + min_match, + max_match, + min_accel, + max_accel, + min_start, + max_start, + }; + special.validate()?; + assert_eq!(nread, special.write_to_len()); + Ok((special, nread)) + } + + /// Validate that the information describing special states satisfies + /// all known invariants. + pub(crate) fn validate(&self) -> Result<(), DeserializeError> { + // Check that both ends of the range are DEAD or neither are. + if self.min_match == DEAD && self.max_match != DEAD { + err!("min_match is DEAD, but max_match is not"); + } + if self.min_match != DEAD && self.max_match == DEAD { + err!("max_match is DEAD, but min_match is not"); + } + if self.min_accel == DEAD && self.max_accel != DEAD { + err!("min_accel is DEAD, but max_accel is not"); + } + if self.min_accel != DEAD && self.max_accel == DEAD { + err!("max_accel is DEAD, but min_accel is not"); + } + if self.min_start == DEAD && self.max_start != DEAD { + err!("min_start is DEAD, but max_start is not"); + } + if self.min_start != DEAD && self.max_start == DEAD { + err!("max_start is DEAD, but min_start is not"); + } + + // Check that ranges are well formed. + if self.min_match > self.max_match { + err!("min_match should not be greater than max_match"); + } + if self.min_accel > self.max_accel { + err!("min_accel should not be greater than max_accel"); + } + if self.min_start > self.max_start { + err!("min_start should not be greater than max_start"); + } + + // Check that ranges are ordered with respect to one another. + if self.matches() && self.quit_id >= self.min_match { + err!("quit_id should not be greater than min_match"); + } + if self.accels() && self.quit_id >= self.min_accel { + err!("quit_id should not be greater than min_accel"); + } + if self.starts() && self.quit_id >= self.min_start { + err!("quit_id should not be greater than min_start"); + } + if self.matches() && self.accels() && self.min_accel < self.min_match { + err!("min_match should not be greater than min_accel"); + } + if self.matches() && self.starts() && self.min_start < self.min_match { + err!("min_match should not be greater than min_start"); + } + if self.accels() && self.starts() && self.min_start < self.min_accel { + err!("min_accel should not be greater than min_start"); + } + + // Check that max is at least as big as everything else. + if self.max < self.quit_id { + err!("quit_id should not be greater than max"); + } + if self.max < self.max_match { + err!("max_match should not be greater than max"); + } + if self.max < self.max_accel { + err!("max_accel should not be greater than max"); + } + if self.max < self.max_start { + err!("max_start should not be greater than max"); + } + + Ok(()) + } + + /// Validate that the special state information is compatible with the + /// given state len. + pub(crate) fn validate_state_len( + &self, + len: usize, + stride2: usize, + ) -> Result<(), DeserializeError> { + // We assume that 'validate' has already passed, so we know that 'max' + // is truly the max. So all we need to check is that the max state ID + // is less than the state ID len. The max legal value here is len-1, + // which occurs when there are no non-special states. + if (self.max.as_usize() >> stride2) >= len { + err!("max should not be greater than or equal to state length"); + } + Ok(()) + } + + /// Write the IDs and ranges for special states to the given byte buffer. + /// The buffer given must have enough room to store all data, otherwise + /// this will return an error. The number of bytes written is returned + /// on success. The number of bytes written is guaranteed to be a multiple + /// of 8. + pub(crate) fn write_to<E: Endian>( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + use crate::util::wire::write_state_id as write; + + if dst.len() < self.write_to_len() { + return Err(SerializeError::buffer_too_small("special state ids")); + } + + let mut nwrite = 0; + nwrite += write::<E>(self.max, &mut dst[nwrite..]); + nwrite += write::<E>(self.quit_id, &mut dst[nwrite..]); + nwrite += write::<E>(self.min_match, &mut dst[nwrite..]); + nwrite += write::<E>(self.max_match, &mut dst[nwrite..]); + nwrite += write::<E>(self.min_accel, &mut dst[nwrite..]); + nwrite += write::<E>(self.max_accel, &mut dst[nwrite..]); + nwrite += write::<E>(self.min_start, &mut dst[nwrite..]); + nwrite += write::<E>(self.max_start, &mut dst[nwrite..]); + + assert_eq!( + self.write_to_len(), + nwrite, + "expected to write certain number of bytes", + ); + assert_eq!( + nwrite % 8, + 0, + "expected to write multiple of 8 bytes for special states", + ); + Ok(nwrite) + } + + /// Returns the total number of bytes written by `write_to`. + pub(crate) fn write_to_len(&self) -> usize { + 8 * StateID::SIZE + } + + /// Sets the maximum special state ID based on the current values. This + /// should be used once all possible state IDs are set. + #[cfg(feature = "dfa-build")] + pub(crate) fn set_max(&mut self) { + use core::cmp::max; + self.max = max( + self.quit_id, + max(self.max_match, max(self.max_accel, self.max_start)), + ); + } + + /// Sets the maximum special state ID such that starting states are not + /// considered "special." This also marks the min/max starting states as + /// DEAD such that 'is_start_state' always returns false, even if the state + /// is actually a starting state. + /// + /// This is useful when there is no prefilter set. It will avoid + /// ping-ponging between the hot path in the DFA search code and the start + /// state handling code, which is typically only useful for executing a + /// prefilter. + #[cfg(feature = "dfa-build")] + pub(crate) fn set_no_special_start_states(&mut self) { + use core::cmp::max; + self.max = max(self.quit_id, max(self.max_match, self.max_accel)); + self.min_start = DEAD; + self.max_start = DEAD; + } + + /// Returns true if and only if the given state ID is a special state. + #[inline] + pub(crate) fn is_special_state(&self, id: StateID) -> bool { + id <= self.max + } + + /// Returns true if and only if the given state ID is a dead state. + #[inline] + pub(crate) fn is_dead_state(&self, id: StateID) -> bool { + id == DEAD + } + + /// Returns true if and only if the given state ID is a quit state. + #[inline] + pub(crate) fn is_quit_state(&self, id: StateID) -> bool { + !self.is_dead_state(id) && self.quit_id == id + } + + /// Returns true if and only if the given state ID is a match state. + #[inline] + pub(crate) fn is_match_state(&self, id: StateID) -> bool { + !self.is_dead_state(id) && self.min_match <= id && id <= self.max_match + } + + /// Returns true if and only if the given state ID is an accel state. + #[inline] + pub(crate) fn is_accel_state(&self, id: StateID) -> bool { + !self.is_dead_state(id) && self.min_accel <= id && id <= self.max_accel + } + + /// Returns true if and only if the given state ID is a start state. + #[inline] + pub(crate) fn is_start_state(&self, id: StateID) -> bool { + !self.is_dead_state(id) && self.min_start <= id && id <= self.max_start + } + + /// Returns the total number of match states for a dense table based DFA. + #[inline] + pub(crate) fn match_len(&self, stride: usize) -> usize { + if self.matches() { + (self.max_match.as_usize() - self.min_match.as_usize() + stride) + / stride + } else { + 0 + } + } + + /// Returns true if and only if there is at least one match state. + #[inline] + pub(crate) fn matches(&self) -> bool { + self.min_match != DEAD + } + + /// Returns the total number of accel states. + #[cfg(feature = "dfa-build")] + pub(crate) fn accel_len(&self, stride: usize) -> usize { + if self.accels() { + (self.max_accel.as_usize() - self.min_accel.as_usize() + stride) + / stride + } else { + 0 + } + } + + /// Returns true if and only if there is at least one accel state. + #[inline] + pub(crate) fn accels(&self) -> bool { + self.min_accel != DEAD + } + + /// Returns true if and only if there is at least one start state. + #[inline] + pub(crate) fn starts(&self) -> bool { + self.min_start != DEAD + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/dfa/start.rs b/third_party/crates/regex-automata-0.4.18/src/dfa/start.rs new file mode 100644 index 0000000..fddc702 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/dfa/start.rs @@ -0,0 +1,74 @@ +use core::mem::size_of; + +use crate::util::wire::{self, DeserializeError, Endian, SerializeError}; + +/// The kind of anchored starting configurations to support in a DFA. +/// +/// Fully compiled DFAs need to be explicitly configured as to which anchored +/// starting configurations to support. The reason for not just supporting +/// everything unconditionally is that it can use more resources (such as +/// memory and build time). The downside of this is that if you try to execute +/// a search using an [`Anchored`](crate::Anchored) mode that is not supported +/// by the DFA, then the search will return an error. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StartKind { + /// Support both anchored and unanchored searches. + Both, + /// Support only unanchored searches. Requesting an anchored search will + /// panic. + /// + /// Note that even if an unanchored search is requested, the pattern itself + /// may still be anchored. For example, `^abc` will only match `abc` at the + /// start of a haystack. This will remain true, even if the regex engine + /// only supported unanchored searches. + Unanchored, + /// Support only anchored searches. Requesting an unanchored search will + /// panic. + Anchored, +} + +impl StartKind { + pub(crate) fn from_bytes( + slice: &[u8], + ) -> Result<(StartKind, usize), DeserializeError> { + wire::check_slice_len(slice, size_of::<u32>(), "start kind bytes")?; + let (n, nr) = wire::try_read_u32(slice, "start kind integer")?; + match n { + 0 => Ok((StartKind::Both, nr)), + 1 => Ok((StartKind::Unanchored, nr)), + 2 => Ok((StartKind::Anchored, nr)), + _ => Err(DeserializeError::generic("unrecognized start kind")), + } + } + + pub(crate) fn write_to<E: Endian>( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("start kind")); + } + let n = match *self { + StartKind::Both => 0, + StartKind::Unanchored => 1, + StartKind::Anchored => 2, + }; + E::write_u32(n, dst); + Ok(nwrite) + } + + pub(crate) fn write_to_len(&self) -> usize { + size_of::<u32>() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn has_unanchored(&self) -> bool { + matches!(*self, StartKind::Both | StartKind::Unanchored) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn has_anchored(&self) -> bool { + matches!(*self, StartKind::Both | StartKind::Anchored) + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/hybrid/dfa.rs b/third_party/crates/regex-automata-0.4.18/src/hybrid/dfa.rs new file mode 100644 index 0000000..22893d7 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/hybrid/dfa.rs @@ -0,0 +1,4434 @@ +/*! +Types and routines specific to lazy DFAs. + +This module is the home of [`hybrid::dfa::DFA`](DFA). + +This module also contains a [`hybrid::dfa::Builder`](Builder) and a +[`hybrid::dfa::Config`](Config) for configuring and building a lazy DFA. +*/ + +use core::{iter, mem::size_of}; + +use alloc::vec::Vec; + +use crate::{ + hybrid::{ + error::{BuildError, CacheError, StartError}, + id::{LazyStateID, LazyStateIDError}, + search, + }, + nfa::thompson, + util::{ + alphabet::{self, ByteClasses, ByteSet}, + determinize::{self, State, StateBuilderEmpty, StateBuilderNFA}, + empty, + prefilter::Prefilter, + primitives::{PatternID, StateID as NFAStateID}, + search::{ + Anchored, HalfMatch, Input, MatchError, MatchKind, PatternSet, + }, + sparse_set::SparseSets, + start::{self, Start, StartByteMap}, + }, +}; + +/// The minimum number of states that a lazy DFA's cache size must support. +/// +/// This is checked at time of construction to ensure that at least some small +/// number of states can fit in the given capacity allotment. If we can't fit +/// at least this number of states, then the thinking is that it's pretty +/// senseless to use the lazy DFA. More to the point, parts of the code do +/// assume that the cache can fit at least some small number of states. +const MIN_STATES: usize = SENTINEL_STATES + 2; + +/// The number of "sentinel" states that get added to every lazy DFA. +/// +/// These are special states indicating status conditions of a search: unknown, +/// dead and quit. These states in particular also use zero NFA states, so +/// their memory usage is quite small. This is relevant for computing the +/// minimum memory needed for a lazy DFA cache. +const SENTINEL_STATES: usize = 3; + +/// A hybrid NFA/DFA (also called a "lazy DFA") for regex searching. +/// +/// A lazy DFA is a DFA that builds itself at search time. It otherwise has +/// very similar characteristics as a [`dense::DFA`](crate::dfa::dense::DFA). +/// Indeed, both support precisely the same regex features with precisely the +/// same semantics. +/// +/// Where as a `dense::DFA` must be completely built to handle any input before +/// it may be used for search, a lazy DFA starts off effectively empty. During +/// a search, a lazy DFA will build itself depending on whether it has already +/// computed the next transition or not. If it has, then it looks a lot like +/// a `dense::DFA` internally: it does a very fast table based access to find +/// the next transition. Otherwise, if the state hasn't been computed, then it +/// does determinization _for that specific transition_ to compute the next DFA +/// state. +/// +/// The main selling point of a lazy DFA is that, in practice, it has +/// the performance profile of a `dense::DFA` without the weakness of it +/// taking worst case exponential time to build. Indeed, for each byte of +/// input, the lazy DFA will construct as most one new DFA state. Thus, a +/// lazy DFA achieves worst case `O(mn)` time for regex search (where `m ~ +/// pattern.len()` and `n ~ haystack.len()`). +/// +/// The main downsides of a lazy DFA are: +/// +/// 1. It requires mutable "cache" space during search. This is where the +/// transition table, among other things, is stored. +/// 2. In pathological cases (e.g., if the cache is too small), it will run +/// out of room and either require a bigger cache capacity or will repeatedly +/// clear the cache and thus repeatedly regenerate DFA states. Overall, this +/// will tend to be slower than a typical NFA simulation. +/// +/// # Capabilities +/// +/// Like a `dense::DFA`, a single lazy DFA fundamentally supports the following +/// operations: +/// +/// 1. Detection of a match. +/// 2. Location of the end of a match. +/// 3. In the case of a lazy DFA with multiple patterns, which pattern matched +/// is reported as well. +/// +/// A notable absence from the above list of capabilities is the location of +/// the *start* of a match. In order to provide both the start and end of +/// a match, *two* lazy DFAs are required. This functionality is provided by a +/// [`Regex`](crate::hybrid::regex::Regex). +/// +/// # Example +/// +/// This shows how to build a lazy DFA with the default configuration and +/// execute a search. Notice how, in contrast to a `dense::DFA`, we must create +/// a cache and pass it to our search routine. +/// +/// ``` +/// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; +/// +/// let dfa = DFA::new("foo[0-9]+")?; +/// let mut cache = dfa.create_cache(); +/// +/// let expected = Some(HalfMatch::must(0, 8)); +/// assert_eq!(expected, dfa.try_search_fwd( +/// &mut cache, &Input::new("foo12345"))?, +/// ); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct DFA { + config: Config, + nfa: thompson::NFA, + stride2: usize, + start_map: StartByteMap, + classes: ByteClasses, + quitset: ByteSet, + cache_capacity: usize, +} + +impl DFA { + /// Parse the given regular expression using a default configuration and + /// return the corresponding lazy DFA. + /// + /// If you want a non-default configuration, then use the [`Builder`] to + /// set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// let dfa = DFA::new("foo[0-9]+bar")?; + /// let mut cache = dfa.create_cache(); + /// + /// let expected = HalfMatch::must(0, 11); + /// assert_eq!( + /// Some(expected), + /// dfa.try_search_fwd(&mut cache, &Input::new("foo12345bar"))?, + /// ); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new(pattern: &str) -> Result<DFA, BuildError> { + DFA::builder().build(pattern) + } + + /// Parse the given regular expressions using a default configuration and + /// return the corresponding lazy multi-DFA. + /// + /// If you want a non-default configuration, then use the [`Builder`] to + /// set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// let dfa = DFA::new_many(&["[0-9]+", "[a-z]+"])?; + /// let mut cache = dfa.create_cache(); + /// + /// let expected = HalfMatch::must(1, 3); + /// assert_eq!( + /// Some(expected), + /// dfa.try_search_fwd(&mut cache, &Input::new("foo12345bar"))?, + /// ); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new_many<P: AsRef<str>>(patterns: &[P]) -> Result<DFA, BuildError> { + DFA::builder().build_many(patterns) + } + + /// Create a new lazy DFA that matches every input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// let dfa = DFA::always_match()?; + /// let mut cache = dfa.create_cache(); + /// + /// let expected = HalfMatch::must(0, 0); + /// assert_eq!(Some(expected), dfa.try_search_fwd( + /// &mut cache, &Input::new(""))?, + /// ); + /// assert_eq!(Some(expected), dfa.try_search_fwd( + /// &mut cache, &Input::new("foo"))?, + /// ); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn always_match() -> Result<DFA, BuildError> { + let nfa = thompson::NFA::always_match(); + Builder::new().build_from_nfa(nfa) + } + + /// Create a new lazy DFA that never matches any input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, Input}; + /// + /// let dfa = DFA::never_match()?; + /// let mut cache = dfa.create_cache(); + /// + /// assert_eq!(None, dfa.try_search_fwd(&mut cache, &Input::new(""))?); + /// assert_eq!(None, dfa.try_search_fwd(&mut cache, &Input::new("foo"))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn never_match() -> Result<DFA, BuildError> { + let nfa = thompson::NFA::never_match(); + Builder::new().build_from_nfa(nfa) + } + + /// Return a default configuration for a `DFA`. + /// + /// This is a convenience routine to avoid needing to import the [`Config`] + /// type when customizing the construction of a lazy DFA. + /// + /// # Example + /// + /// This example shows how to build a lazy DFA that heuristically supports + /// Unicode word boundaries. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, MatchError, Input}; + /// + /// let re = DFA::builder() + /// .configure(DFA::config().unicode_word_boundary(true)) + /// .build(r"\b\w+\b")?; + /// let mut cache = re.create_cache(); + /// + /// // Since our haystack is all ASCII, the DFA search sees then and knows + /// // it is legal to interpret Unicode word boundaries as ASCII word + /// // boundaries. + /// let input = Input::new("!!foo!!"); + /// let expected = HalfMatch::must(0, 5); + /// assert_eq!(Some(expected), re.try_search_fwd(&mut cache, &input)?); + /// + /// // But if our haystack contains non-ASCII, then the search will fail + /// // with an error. + /// let input = Input::new("!!βββ!!"); + /// let expected = MatchError::quit(b'\xCE', 2); + /// assert_eq!(Err(expected), re.try_search_fwd(&mut cache, &input)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn config() -> Config { + Config::new() + } + + /// Return a builder for configuring the construction of a `Regex`. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Builder`] type in common cases. + /// + /// # Example + /// + /// This example shows how to use the builder to disable UTF-8 mode + /// everywhere for lazy DFAs. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, util::syntax, HalfMatch, Input}; + /// + /// let re = DFA::builder() + /// .syntax(syntax::Config::new().utf8(false)) + /// .build(r"foo(?-u:[^b])ar.*")?; + /// let mut cache = re.create_cache(); + /// + /// let input = Input::new(b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"); + /// let expected = Some(HalfMatch::must(0, 9)); + /// let got = re.try_search_fwd(&mut cache, &input)?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn builder() -> Builder { + Builder::new() + } + + /// Create a new cache for this lazy DFA. + /// + /// The cache returned should only be used for searches for this + /// lazy DFA. If you want to reuse the cache for another DFA, then + /// you must call [`Cache::reset`] with that DFA (or, equivalently, + /// [`DFA::reset_cache`]). + pub fn create_cache(&self) -> Cache { + Cache::new(self) + } + + /// Reset the given cache such that it can be used for searching with the + /// this lazy DFA (and only this DFA). + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different lazy DFA. + /// + /// Resetting a cache sets its "clear count" to 0. This is relevant if the + /// lazy DFA has been configured to "give up" after it has cleared the + /// cache a certain number of times. + /// + /// Any lazy state ID generated by the cache prior to resetting it is + /// invalid after the reset. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different DFA. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// let dfa1 = DFA::new(r"\w")?; + /// let dfa2 = DFA::new(r"\W")?; + /// + /// let mut cache = dfa1.create_cache(); + /// assert_eq!( + /// Some(HalfMatch::must(0, 2)), + /// dfa1.try_search_fwd(&mut cache, &Input::new("Δ"))?, + /// ); + /// + /// // Using 'cache' with dfa2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the DFA we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 'dfa1' is also not + /// // allowed. + /// dfa2.reset_cache(&mut cache); + /// assert_eq!( + /// Some(HalfMatch::must(0, 3)), + /// dfa2.try_search_fwd(&mut cache, &Input::new("☃"))?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset_cache(&self, cache: &mut Cache) { + Lazy::new(self, cache).reset_cache() + } + + /// Returns the total number of patterns compiled into this lazy DFA. + /// + /// In the case of a DFA that contains no patterns, this returns `0`. + /// + /// # Example + /// + /// This example shows the pattern length for a DFA that never matches: + /// + /// ``` + /// use regex_automata::hybrid::dfa::DFA; + /// + /// let dfa = DFA::never_match()?; + /// assert_eq!(dfa.pattern_len(), 0); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And another example for a DFA that matches at every position: + /// + /// ``` + /// use regex_automata::hybrid::dfa::DFA; + /// + /// let dfa = DFA::always_match()?; + /// assert_eq!(dfa.pattern_len(), 1); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And finally, a DFA that was constructed from multiple patterns: + /// + /// ``` + /// use regex_automata::hybrid::dfa::DFA; + /// + /// let dfa = DFA::new_many(&["[0-9]+", "[a-z]+", "[A-Z]+"])?; + /// assert_eq!(dfa.pattern_len(), 3); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn pattern_len(&self) -> usize { + self.nfa.pattern_len() + } + + /// Returns the equivalence classes that make up the alphabet for this DFA. + /// + /// Unless [`Config::byte_classes`] was disabled, it is possible that + /// multiple distinct bytes are grouped into the same equivalence class + /// if it is impossible for them to discriminate between a match and a + /// non-match. This has the effect of reducing the overall alphabet size + /// and in turn potentially substantially reducing the size of the DFA's + /// transition table. + /// + /// The downside of using equivalence classes like this is that every state + /// transition will automatically use this map to convert an arbitrary + /// byte to its corresponding equivalence class. In practice this has a + /// negligible impact on performance. + pub fn byte_classes(&self) -> &ByteClasses { + &self.classes + } + + /// Returns this lazy DFA's configuration. + pub fn get_config(&self) -> &Config { + &self.config + } + + /// Returns a reference to the underlying NFA. + pub fn get_nfa(&self) -> &thompson::NFA { + &self.nfa + } + + /// Returns the stride, as a base-2 exponent, required for these + /// equivalence classes. + /// + /// The stride is always the smallest power of 2 that is greater than or + /// equal to the alphabet length. This is done so that converting between + /// state IDs and indices can be done with shifts alone, which is much + /// faster than integer division. + fn stride2(&self) -> usize { + self.stride2 + } + + /// Returns the total stride for every state in this lazy DFA. This + /// corresponds to the total number of transitions used by each state in + /// this DFA's transition table. + fn stride(&self) -> usize { + 1 << self.stride2() + } + + /// Returns the memory usage, in bytes, of this lazy DFA. + /// + /// This does **not** include the stack size used up by this lazy DFA. To + /// compute that, use `std::mem::size_of::<DFA>()`. This also does not + /// include the size of the `Cache` used. + /// + /// This also does not include any heap memory used by the NFA inside of + /// this hybrid NFA/DFA. This is because the NFA's ownership is shared, and + /// thus not owned by this hybrid NFA/DFA. More practically, several regex + /// engines in this crate embed an NFA, and reporting the NFA's memory + /// usage in all of them would likely result in reporting higher heap + /// memory than is actually used. + pub fn memory_usage(&self) -> usize { + // The only thing that uses heap memory in a DFA is the NFA. But the + // NFA has shared ownership, so reporting its memory as part of the + // hybrid DFA is likely to lead to double-counting the NFA memory + // somehow. In particular, this DFA does not really own an NFA, so + // including it in the DFA's memory usage doesn't seem semantically + // correct. + 0 + } +} + +impl DFA { + /// Executes a forward search and returns the end position of the leftmost + /// match that is found. If no match exists, then `None` is returned. + /// + /// In particular, this method continues searching even after it enters + /// a match state. The search only terminates once it has reached the + /// end of the input or when it has entered a dead or quit state. Upon + /// termination, the position of the last byte seen while still in a match + /// state is returned. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This example shows how to run a basic search. + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// let dfa = DFA::new("foo[0-9]+")?; + /// let mut cache = dfa.create_cache(); + /// let expected = HalfMatch::must(0, 8); + /// assert_eq!(Some(expected), dfa.try_search_fwd( + /// &mut cache, &Input::new("foo12345"))?, + /// ); + /// + /// // Even though a match is found after reading the first byte (`a`), + /// // the leftmost first match semantics demand that we find the earliest + /// // match that prefers earlier parts of the pattern over later parts. + /// let dfa = DFA::new("abc|a")?; + /// let mut cache = dfa.create_cache(); + /// let expected = HalfMatch::must(0, 3); + /// assert_eq!(Some(expected), dfa.try_search_fwd( + /// &mut cache, &Input::new("abc"))?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specific pattern search + /// + /// This example shows how to build a lazy multi-DFA that permits searching + /// for specific patterns. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// Anchored, HalfMatch, PatternID, Input, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().starts_for_each_pattern(true)) + /// .build_many(&["[a-z0-9]{6}", "[a-z][a-z0-9]{5}"])?; + /// let mut cache = dfa.create_cache(); + /// let haystack = "foo123"; + /// + /// // Since we are using the default leftmost-first match and both + /// // patterns match at the same starting position, only the first pattern + /// // will be returned in this case when doing a search for any of the + /// // patterns. + /// let expected = Some(HalfMatch::must(0, 6)); + /// let got = dfa.try_search_fwd(&mut cache, &Input::new(haystack))?; + /// assert_eq!(expected, got); + /// + /// // But if we want to check whether some other pattern matches, then we + /// // can provide its pattern ID. + /// let expected = Some(HalfMatch::must(1, 6)); + /// let input = Input::new(haystack) + /// .anchored(Anchored::Pattern(PatternID::must(1))); + /// let got = dfa.try_search_fwd(&mut cache, &input)?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specifying the bounds of a search + /// + /// This example shows how providing the bounds of a search can produce + /// different results than simply sub-slicing the haystack. + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// // N.B. We disable Unicode here so that we use a simple ASCII word + /// // boundary. Alternatively, we could enable heuristic support for + /// // Unicode word boundaries since our haystack is pure ASCII. + /// let dfa = DFA::new(r"(?-u)\b[0-9]{3}\b")?; + /// let mut cache = dfa.create_cache(); + /// let haystack = "foo123bar"; + /// + /// // Since we sub-slice the haystack, the search doesn't know about the + /// // larger context and assumes that `123` is surrounded by word + /// // boundaries. And of course, the match position is reported relative + /// // to the sub-slice as well, which means we get `3` instead of `6`. + /// let expected = Some(HalfMatch::must(0, 3)); + /// let got = dfa.try_search_fwd( + /// &mut cache, + /// &Input::new(&haystack[3..6]), + /// )?; + /// assert_eq!(expected, got); + /// + /// // But if we provide the bounds of the search within the context of the + /// // entire haystack, then the search can take the surrounding context + /// // into account. (And if we did find a match, it would be reported + /// // as a valid offset into `haystack` instead of its sub-slice.) + /// let expected = None; + /// let got = dfa.try_search_fwd( + /// &mut cache, + /// &Input::new(haystack).range(3..6), + /// )?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_search_fwd( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, MatchError> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + let hm = match search::find_fwd(self, cache, input)? { + None => return Ok(None), + Some(hm) if !utf8empty => return Ok(Some(hm)), + Some(hm) => hm, + }; + // We get to this point when we know our DFA can match the empty string + // AND when UTF-8 mode is enabled. In this case, we skip any matches + // whose offset splits a codepoint. Such a match is necessarily a + // zero-width match, because UTF-8 mode requires the underlying NFA + // to be built such that all non-empty matches span valid UTF-8. + // Therefore, any match that ends in the middle of a codepoint cannot + // be part of a span of valid UTF-8 and thus must be an empty match. + // In such cases, we skip it, so as not to report matches that split a + // codepoint. + // + // Note that this is not a checked assumption. Callers *can* provide an + // NFA with UTF-8 mode enabled but produces non-empty matches that span + // invalid UTF-8. But doing so is documented to result in unspecified + // behavior. + empty::skip_splits_fwd(input, hm, hm.offset(), |input| { + let got = search::find_fwd(self, cache, input)?; + Ok(got.map(|hm| (hm, hm.offset()))) + }) + } + + /// Executes a reverse search and returns the start of the position of the + /// leftmost match that is found. If no match exists, then `None` is + /// returned. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This routine is principally useful when used in + /// conjunction with the + /// [`nfa::thompson::Config::reverse`](crate::nfa::thompson::Config::reverse) + /// configuration. In general, it's unlikely to be correct to use both + /// `try_search_fwd` and `try_search_rev` with the same DFA since any + /// particular DFA will only support searching in one direction with + /// respect to the pattern. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson, + /// hybrid::dfa::DFA, + /// HalfMatch, Input, + /// }; + /// + /// let dfa = DFA::builder() + /// .thompson(thompson::Config::new().reverse(true)) + /// .build("foo[0-9]+")?; + /// let mut cache = dfa.create_cache(); + /// let expected = HalfMatch::must(0, 0); + /// assert_eq!( + /// Some(expected), + /// dfa.try_search_rev(&mut cache, &Input::new("foo12345"))?, + /// ); + /// + /// // Even though a match is found after reading the last byte (`c`), + /// // the leftmost first match semantics demand that we find the earliest + /// // match that prefers earlier parts of the pattern over latter parts. + /// let dfa = DFA::builder() + /// .thompson(thompson::Config::new().reverse(true)) + /// .build("abc|c")?; + /// let mut cache = dfa.create_cache(); + /// let expected = HalfMatch::must(0, 0); + /// assert_eq!(Some(expected), dfa.try_search_rev( + /// &mut cache, &Input::new("abc"))?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: UTF-8 mode + /// + /// This examples demonstrates that UTF-8 mode applies to reverse + /// DFAs. When UTF-8 mode is enabled in the underlying NFA, then all + /// matches reported must correspond to valid UTF-8 spans. This includes + /// prohibiting zero-width matches that split a codepoint. + /// + /// UTF-8 mode is enabled by default. Notice below how the only zero-width + /// matches reported are those at UTF-8 boundaries: + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// nfa::thompson, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .thompson(thompson::Config::new().reverse(true)) + /// .build(r"")?; + /// let mut cache = dfa.create_cache(); + /// + /// // Run the reverse DFA to collect all matches. + /// let mut input = Input::new("☃"); + /// let mut matches = vec![]; + /// loop { + /// match dfa.try_search_rev(&mut cache, &input)? { + /// None => break, + /// Some(hm) => { + /// matches.push(hm); + /// if hm.offset() == 0 || input.end() == 0 { + /// break; + /// } else if hm.offset() < input.end() { + /// input.set_end(hm.offset()); + /// } else { + /// // This is only necessary to handle zero-width + /// // matches, which of course occur in this example. + /// // Without this, the search would never advance + /// // backwards beyond the initial match. + /// input.set_end(input.end() - 1); + /// } + /// } + /// } + /// } + /// + /// // No matches split a codepoint. + /// let expected = vec![ + /// HalfMatch::must(0, 3), + /// HalfMatch::must(0, 0), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Now let's look at the same example, but with UTF-8 mode on the + /// underlying NFA disabled: + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// nfa::thompson, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .thompson(thompson::Config::new().reverse(true).utf8(false)) + /// .build(r"")?; + /// let mut cache = dfa.create_cache(); + /// + /// // Run the reverse DFA to collect all matches. + /// let mut input = Input::new("☃"); + /// let mut matches = vec![]; + /// loop { + /// match dfa.try_search_rev(&mut cache, &input)? { + /// None => break, + /// Some(hm) => { + /// matches.push(hm); + /// if hm.offset() == 0 || input.end() == 0 { + /// break; + /// } else if hm.offset() < input.end() { + /// input.set_end(hm.offset()); + /// } else { + /// // This is only necessary to handle zero-width + /// // matches, which of course occur in this example. + /// // Without this, the search would never advance + /// // backwards beyond the initial match. + /// input.set_end(input.end() - 1); + /// } + /// } + /// } + /// } + /// + /// // No matches split a codepoint. + /// let expected = vec![ + /// HalfMatch::must(0, 3), + /// HalfMatch::must(0, 2), + /// HalfMatch::must(0, 1), + /// HalfMatch::must(0, 0), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_search_rev( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, MatchError> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + let hm = match search::find_rev(self, cache, input)? { + None => return Ok(None), + Some(hm) if !utf8empty => return Ok(Some(hm)), + Some(hm) => hm, + }; + empty::skip_splits_rev(input, hm, hm.offset(), |input| { + let got = search::find_rev(self, cache, input)?; + Ok(got.map(|hm| (hm, hm.offset()))) + }) + } + + /// Executes an overlapping forward search and returns the end position of + /// matches as they are found. If no match exists, then `None` is returned. + /// + /// This routine is principally only useful when searching for multiple + /// patterns on inputs where multiple patterns may match the same regions + /// of text. In particular, callers must preserve the automaton's search + /// state from prior calls so that the implementation knows where the last + /// match occurred. + /// + /// When using this routine to implement an iterator of overlapping + /// matches, the `start` of the search should remain invariant throughout + /// iteration. The `OverlappingState` given to the search will keep track + /// of the current position of the search. (This is because multiple + /// matches may be reported at the same position, so only the search + /// implementation itself knows when to advance the position.) + /// + /// If for some reason you want the search to forget about its previous + /// state and restart the search at a particular position, then setting the + /// state to [`OverlappingState::start`] will accomplish that. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This example shows how to run a basic overlapping search. Notice + /// that we build the automaton with a `MatchKind::All` configuration. + /// Overlapping searches are unlikely to work as one would expect when + /// using the default `MatchKind::LeftmostFirst` match semantics, since + /// leftmost-first matching is fundamentally incompatible with overlapping + /// searches. Namely, overlapping searches need to report matches as they + /// are seen, where as leftmost-first searches will continue searching even + /// after a match has been observed in order to find the conventional end + /// position of the match. More concretely, leftmost-first searches use + /// dead states to terminate a search after a specific match can no longer + /// be extended. Overlapping searches instead do the opposite by continuing + /// the search to find totally new matches (potentially of other patterns). + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// hybrid::dfa::{DFA, OverlappingState}, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .build_many(&[r"\w+$", r"\S+$"])?; + /// let mut cache = dfa.create_cache(); + /// + /// let haystack = "@foo"; + /// let mut state = OverlappingState::start(); + /// + /// let expected = Some(HalfMatch::must(1, 4)); + /// dfa.try_search_overlapping_fwd( + /// &mut cache, &Input::new(haystack), &mut state, + /// )?; + /// assert_eq!(expected, state.get_match()); + /// + /// // The first pattern also matches at the same position, so re-running + /// // the search will yield another match. Notice also that the first + /// // pattern is returned after the second. This is because the second + /// // pattern begins its match before the first, is therefore an earlier + /// // match and is thus reported first. + /// let expected = Some(HalfMatch::must(0, 4)); + /// dfa.try_search_overlapping_fwd( + /// &mut cache, &Input::new(haystack), &mut state, + /// )?; + /// assert_eq!(expected, state.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_search_overlapping_fwd( + &self, + cache: &mut Cache, + input: &Input<'_>, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + search::find_overlapping_fwd(self, cache, input, state)?; + match state.get_match() { + None => Ok(()), + Some(_) if !utf8empty => Ok(()), + Some(_) => skip_empty_utf8_splits_overlapping( + input, + state, + |input, state| { + search::find_overlapping_fwd(self, cache, input, state) + }, + ), + } + } + + /// Executes a reverse overlapping search and returns the start of the + /// position of the leftmost match that is found. If no match exists, then + /// `None` is returned. + /// + /// When using this routine to implement an iterator of overlapping + /// matches, the `start` of the search should remain invariant throughout + /// iteration. The `OverlappingState` given to the search will keep track + /// of the current position of the search. (This is because multiple + /// matches may be reported at the same position, so only the search + /// implementation itself knows when to advance the position.) + /// + /// If for some reason you want the search to forget about its previous + /// state and restart the search at a particular position, then setting the + /// state to [`OverlappingState::start`] will accomplish that. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example: UTF-8 mode + /// + /// This examples demonstrates that UTF-8 mode applies to reverse + /// DFAs. When UTF-8 mode is enabled in the underlying NFA, then all + /// matches reported must correspond to valid UTF-8 spans. This includes + /// prohibiting zero-width matches that split a codepoint. + /// + /// UTF-8 mode is enabled by default. Notice below how the only zero-width + /// matches reported are those at UTF-8 boundaries: + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::{DFA, OverlappingState}, + /// nfa::thompson, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .thompson(thompson::Config::new().reverse(true)) + /// .build_many(&[r"", r"☃"])?; + /// let mut cache = dfa.create_cache(); + /// + /// // Run the reverse DFA to collect all matches. + /// let input = Input::new("☃"); + /// let mut state = OverlappingState::start(); + /// let mut matches = vec![]; + /// loop { + /// dfa.try_search_overlapping_rev(&mut cache, &input, &mut state)?; + /// match state.get_match() { + /// None => break, + /// Some(hm) => matches.push(hm), + /// } + /// } + /// + /// // No matches split a codepoint. + /// let expected = vec![ + /// HalfMatch::must(0, 3), + /// HalfMatch::must(1, 0), + /// HalfMatch::must(0, 0), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Now let's look at the same example, but with UTF-8 mode on the + /// underlying NFA disabled: + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::{DFA, OverlappingState}, + /// nfa::thompson, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .thompson(thompson::Config::new().reverse(true).utf8(false)) + /// .build_many(&[r"", r"☃"])?; + /// let mut cache = dfa.create_cache(); + /// + /// // Run the reverse DFA to collect all matches. + /// let input = Input::new("☃"); + /// let mut state = OverlappingState::start(); + /// let mut matches = vec![]; + /// loop { + /// dfa.try_search_overlapping_rev(&mut cache, &input, &mut state)?; + /// match state.get_match() { + /// None => break, + /// Some(hm) => matches.push(hm), + /// } + /// } + /// + /// // Now *all* positions match, even within a codepoint, + /// // because we lifted the requirement that matches + /// // correspond to valid UTF-8 spans. + /// let expected = vec![ + /// HalfMatch::must(0, 3), + /// HalfMatch::must(0, 2), + /// HalfMatch::must(0, 1), + /// HalfMatch::must(1, 0), + /// HalfMatch::must(0, 0), + /// ]; + /// assert_eq!(expected, matches); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_search_overlapping_rev( + &self, + cache: &mut Cache, + input: &Input<'_>, + state: &mut OverlappingState, + ) -> Result<(), MatchError> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + search::find_overlapping_rev(self, cache, input, state)?; + match state.get_match() { + None => Ok(()), + Some(_) if !utf8empty => Ok(()), + Some(_) => skip_empty_utf8_splits_overlapping( + input, + state, + |input, state| { + search::find_overlapping_rev(self, cache, input, state) + }, + ), + } + } + + /// Writes the set of patterns that match anywhere in the given search + /// configuration to `patset`. If multiple patterns match at the same + /// position and the underlying DFA supports overlapping matches, then all + /// matching patterns are written to the given set. + /// + /// Unless all of the patterns in this DFA are anchored, then generally + /// speaking, this will visit every byte in the haystack. + /// + /// This search routine *does not* clear the pattern set. This gives some + /// flexibility to the caller (e.g., running multiple searches with the + /// same pattern set), but does make the API bug-prone if you're reusing + /// the same pattern set for multiple searches but intended them to be + /// independent. + /// + /// If a pattern ID matched but the given `PatternSet` does not have + /// sufficient capacity to store it, then it is not inserted and silently + /// dropped. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This example shows how to find all matching patterns in a haystack, + /// even when some patterns match at the same position as other patterns. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// Input, MatchKind, PatternSet, + /// }; + /// + /// let patterns = &[ + /// r"\w+", r"\d+", r"\pL+", r"foo", r"bar", r"barfoo", r"foobar", + /// ]; + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .build_many(patterns)?; + /// let mut cache = dfa.create_cache(); + /// + /// let input = Input::new("foobar"); + /// let mut patset = PatternSet::new(dfa.pattern_len()); + /// dfa.try_which_overlapping_matches(&mut cache, &input, &mut patset)?; + /// let expected = vec![0, 2, 3, 4, 6]; + /// let got: Vec<usize> = patset.iter().map(|p| p.as_usize()).collect(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) -> Result<(), MatchError> { + let mut state = OverlappingState::start(); + while let Some(m) = { + self.try_search_overlapping_fwd(cache, input, &mut state)?; + state.get_match() + } { + let _ = patset.try_insert(m.pattern()); + // There's nothing left to find, so we can stop. Or the caller + // asked us to. + if patset.is_full() || input.get_earliest() { + break; + } + } + Ok(()) + } +} + +impl DFA { + /// Transitions from the current state to the next state, given the next + /// byte of input. + /// + /// The given cache is used to either reuse pre-computed state + /// transitions, or to store this newly computed transition for future + /// reuse. Thus, this routine guarantees that it will never return a state + /// ID that has an "unknown" tag. + /// + /// # State identifier validity + /// + /// The only valid value for `current` is the lazy state ID returned + /// by the most recent call to `next_state`, `next_state_untagged`, + /// `next_state_untagged_unchecked`, `start_state_forward` or + /// `state_state_reverse` for the given `cache`. Any state ID returned from + /// prior calls to these routines (with the same `cache`) is considered + /// invalid (even if it gives an appearance of working). State IDs returned + /// from _any_ prior call for different `cache` values are also always + /// invalid. + /// + /// The returned ID is always a valid ID when `current` refers to a valid + /// ID. Moreover, this routine is defined for all possible values of + /// `input`. + /// + /// These validity rules are not checked, even in debug mode. Callers are + /// required to uphold these rules themselves. + /// + /// Violating these state ID validity rules will not sacrifice memory + /// safety, but _may_ produce an incorrect result or a panic. + /// + /// # Panics + /// + /// If the given ID does not refer to a valid state, then this routine + /// may panic but it also may not panic and instead return an invalid or + /// incorrect ID. + /// + /// # Example + /// + /// This shows a simplistic example for walking a lazy DFA for a given + /// haystack by using the `next_state` method. + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, Input}; + /// + /// let dfa = DFA::new(r"[a-z]+r")?; + /// let mut cache = dfa.create_cache(); + /// let haystack = "bar".as_bytes(); + /// + /// // The start state is determined by inspecting the position and the + /// // initial bytes of the haystack. + /// let mut sid = dfa.start_state_forward( + /// &mut cache, &Input::new(haystack), + /// )?; + /// // Walk all the bytes in the haystack. + /// for &b in haystack { + /// sid = dfa.next_state(&mut cache, sid, b)?; + /// } + /// // Matches are always delayed by 1 byte, so we must explicitly walk the + /// // special "EOI" transition at the end of the search. + /// sid = dfa.next_eoi_state(&mut cache, sid)?; + /// assert!(sid.is_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn next_state( + &self, + cache: &mut Cache, + current: LazyStateID, + input: u8, + ) -> Result<LazyStateID, CacheError> { + let class = usize::from(self.classes.get(input)); + let offset = current.as_usize_untagged() + class; + let sid = cache.trans[offset]; + if !sid.is_unknown() { + return Ok(sid); + } + let unit = alphabet::Unit::u8(input); + Lazy::new(self, cache).cache_next_state(current, unit) + } + + /// Transitions from the current state to the next state, given the next + /// byte of input and a state ID that is not tagged. + /// + /// The only reason to use this routine is performance. In particular, the + /// `next_state` method needs to do some additional checks, among them is + /// to account for identifiers to states that are not yet computed. In + /// such a case, the transition is computed on the fly. However, if it is + /// known that the `current` state ID is untagged, then these checks can be + /// omitted. + /// + /// Since this routine does not compute states on the fly, it does not + /// modify the cache and thus cannot return an error. Consequently, `cache` + /// does not need to be mutable and it is possible for this routine to + /// return a state ID corresponding to the special "unknown" state. In + /// this case, it is the caller's responsibility to use the prior state + /// ID and `input` with `next_state` in order to force the computation of + /// the unknown transition. Otherwise, trying to use the "unknown" state + /// ID will just result in transitioning back to itself, and thus never + /// terminating. (This is technically a special exemption to the state ID + /// validity rules, but is permissible since this routine is guaranteed to + /// never mutate the given `cache`, and thus the identifier is guaranteed + /// to remain valid.) + /// + /// See [`LazyStateID`] for more details on what it means for a state ID + /// to be tagged. Also, see + /// [`next_state_untagged_unchecked`](DFA::next_state_untagged_unchecked) + /// for this same idea, but with bounds checks forcefully elided. + /// + /// # State identifier validity + /// + /// The only valid value for `current` is an **untagged** lazy + /// state ID returned by the most recent call to `next_state`, + /// `next_state_untagged`, `next_state_untagged_unchecked`, + /// `start_state_forward` or `state_state_reverse` for the given `cache`. + /// Any state ID returned from prior calls to these routines (with the + /// same `cache`) is considered invalid (even if it gives an appearance + /// of working). State IDs returned from _any_ prior call for different + /// `cache` values are also always invalid. + /// + /// The returned ID is always a valid ID when `current` refers to a valid + /// ID, although it may be tagged. Moreover, this routine is defined for + /// all possible values of `input`. + /// + /// Not all validity rules are checked, even in debug mode. Callers are + /// required to uphold these rules themselves. + /// + /// Violating these state ID validity rules will not sacrifice memory + /// safety, but _may_ produce an incorrect result or a panic. + /// + /// # Panics + /// + /// If the given ID does not refer to a valid state, then this routine + /// may panic but it also may not panic and instead return an invalid or + /// incorrect ID. + /// + /// # Example + /// + /// This shows a simplistic example for walking a lazy DFA for a given + /// haystack by using the `next_state_untagged` method where possible. + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, Input}; + /// + /// let dfa = DFA::new(r"[a-z]+r")?; + /// let mut cache = dfa.create_cache(); + /// let haystack = "bar".as_bytes(); + /// + /// // The start state is determined by inspecting the position and the + /// // initial bytes of the haystack. + /// let mut sid = dfa.start_state_forward( + /// &mut cache, &Input::new(haystack), + /// )?; + /// // Walk all the bytes in the haystack. + /// let mut at = 0; + /// while at < haystack.len() { + /// if sid.is_tagged() { + /// sid = dfa.next_state(&mut cache, sid, haystack[at])?; + /// } else { + /// let mut prev_sid = sid; + /// // We attempt to chew through as much as we can while moving + /// // through untagged state IDs. Thus, the transition function + /// // does less work on average per byte. (Unrolling this loop + /// // may help even more.) + /// while at < haystack.len() { + /// prev_sid = sid; + /// sid = dfa.next_state_untagged( + /// &mut cache, sid, haystack[at], + /// ); + /// at += 1; + /// if sid.is_tagged() { + /// break; + /// } + /// } + /// // We must ensure that we never proceed to the next iteration + /// // with an unknown state ID. If we don't account for this + /// // case, then search isn't guaranteed to terminate since all + /// // transitions on unknown states loop back to itself. + /// if sid.is_unknown() { + /// sid = dfa.next_state( + /// &mut cache, prev_sid, haystack[at - 1], + /// )?; + /// } + /// } + /// } + /// // Matches are always delayed by 1 byte, so we must explicitly walk the + /// // special "EOI" transition at the end of the search. + /// sid = dfa.next_eoi_state(&mut cache, sid)?; + /// assert!(sid.is_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn next_state_untagged( + &self, + cache: &Cache, + current: LazyStateID, + input: u8, + ) -> LazyStateID { + debug_assert!(!current.is_tagged()); + let class = usize::from(self.classes.get(input)); + let offset = current.as_usize_unchecked() + class; + cache.trans[offset] + } + + /// Transitions from the current state to the next state, eliding bounds + /// checks, given the next byte of input and a state ID that is not tagged. + /// + /// The only reason to use this routine is performance. In particular, the + /// `next_state` method needs to do some additional checks, among them is + /// to account for identifiers to states that are not yet computed. In + /// such a case, the transition is computed on the fly. However, if it is + /// known that the `current` state ID is untagged, then these checks can be + /// omitted. + /// + /// Since this routine does not compute states on the fly, it does not + /// modify the cache and thus cannot return an error. Consequently, `cache` + /// does not need to be mutable and it is possible for this routine to + /// return a state ID corresponding to the special "unknown" state. In + /// this case, it is the caller's responsibility to use the prior state + /// ID and `input` with `next_state` in order to force the computation of + /// the unknown transition. Otherwise, trying to use the "unknown" state + /// ID will just result in transitioning back to itself, and thus never + /// terminating. (This is technically a special exemption to the state ID + /// validity rules, but is permissible since this routine is guaranteed to + /// never mutate the given `cache`, and thus the identifier is guaranteed + /// to remain valid.) + /// + /// See [`LazyStateID`] for more details on what it means for a state ID + /// to be tagged. Also, see + /// [`next_state_untagged`](DFA::next_state_untagged) + /// for this same idea, but with memory safety guaranteed by retaining + /// bounds checks. + /// + /// # State identifier validity + /// + /// The only valid value for `current` is an **untagged** lazy + /// state ID returned by the most recent call to `next_state`, + /// `next_state_untagged`, `next_state_untagged_unchecked`, + /// `start_state_forward` or `state_state_reverse` for the given `cache`. + /// Any state ID returned from prior calls to these routines (with the + /// same `cache`) is considered invalid (even if it gives an appearance + /// of working). State IDs returned from _any_ prior call for different + /// `cache` values are also always invalid. + /// + /// The returned ID is always a valid ID when `current` refers to a valid + /// ID, although it may be tagged. Moreover, this routine is defined for + /// all possible values of `input`. + /// + /// Not all validity rules are checked, even in debug mode. Callers are + /// required to uphold these rules themselves. + /// + /// Violating these state ID validity rules will not sacrifice memory + /// safety, but _may_ produce an incorrect result or a panic. + /// + /// # Safety + /// + /// Callers of this method must guarantee that `current` refers to a valid + /// state ID according to the rules described above. If `current` is not a + /// valid state ID for this automaton, then calling this routine may result + /// in undefined behavior. + /// + /// If `current` is valid, then the ID returned is valid for all possible + /// values of `input`. + #[inline] + pub unsafe fn next_state_untagged_unchecked( + &self, + cache: &Cache, + current: LazyStateID, + input: u8, + ) -> LazyStateID { + debug_assert!(!current.is_tagged()); + let class = usize::from(self.classes.get(input)); + let offset = current.as_usize_unchecked() + class; + *cache.trans.get_unchecked(offset) + } + + /// Transitions from the current state to the next state for the special + /// EOI symbol. + /// + /// The given cache is used to either reuse pre-computed state + /// transitions, or to store this newly computed transition for future + /// reuse. Thus, this routine guarantees that it will never return a state + /// ID that has an "unknown" tag. + /// + /// This routine must be called at the end of every search in a correct + /// implementation of search. Namely, lazy DFAs in this crate delay matches + /// by one byte in order to support look-around operators. Thus, after + /// reaching the end of a haystack, a search implementation must follow one + /// last EOI transition. + /// + /// It is best to think of EOI as an additional symbol in the alphabet of a + /// DFA that is distinct from every other symbol. That is, the alphabet of + /// lazy DFAs in this crate has a logical size of 257 instead of 256, where + /// 256 corresponds to every possible inhabitant of `u8`. (In practice, the + /// physical alphabet size may be smaller because of alphabet compression + /// via equivalence classes, but EOI is always represented somehow in the + /// alphabet.) + /// + /// # State identifier validity + /// + /// The only valid value for `current` is the lazy state ID returned + /// by the most recent call to `next_state`, `next_state_untagged`, + /// `next_state_untagged_unchecked`, `start_state_forward` or + /// `state_state_reverse` for the given `cache`. Any state ID returned from + /// prior calls to these routines (with the same `cache`) is considered + /// invalid (even if it gives an appearance of working). State IDs returned + /// from _any_ prior call for different `cache` values are also always + /// invalid. + /// + /// The returned ID is always a valid ID when `current` refers to a valid + /// ID. + /// + /// These validity rules are not checked, even in debug mode. Callers are + /// required to uphold these rules themselves. + /// + /// Violating these state ID validity rules will not sacrifice memory + /// safety, but _may_ produce an incorrect result or a panic. + /// + /// # Panics + /// + /// If the given ID does not refer to a valid state, then this routine + /// may panic but it also may not panic and instead return an invalid or + /// incorrect ID. + /// + /// # Example + /// + /// This shows a simplistic example for walking a DFA for a given haystack, + /// and then finishing the search with the final EOI transition. + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, Input}; + /// + /// let dfa = DFA::new(r"[a-z]+r")?; + /// let mut cache = dfa.create_cache(); + /// let haystack = "bar".as_bytes(); + /// + /// // The start state is determined by inspecting the position and the + /// // initial bytes of the haystack. + /// let mut sid = dfa.start_state_forward( + /// &mut cache, &Input::new(haystack), + /// )?; + /// // Walk all the bytes in the haystack. + /// for &b in haystack { + /// sid = dfa.next_state(&mut cache, sid, b)?; + /// } + /// // Matches are always delayed by 1 byte, so we must explicitly walk + /// // the special "EOI" transition at the end of the search. Without this + /// // final transition, the assert below will fail since the DFA will not + /// // have entered a match state yet! + /// sid = dfa.next_eoi_state(&mut cache, sid)?; + /// assert!(sid.is_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn next_eoi_state( + &self, + cache: &mut Cache, + current: LazyStateID, + ) -> Result<LazyStateID, CacheError> { + let eoi = self.classes.eoi().as_usize(); + let offset = current.as_usize_untagged() + eoi; + let sid = cache.trans[offset]; + if !sid.is_unknown() { + return Ok(sid); + } + let unit = self.classes.eoi(); + Lazy::new(self, cache).cache_next_state(current, unit) + } + + /// Return the ID of the start state for this lazy DFA for the given + /// starting configuration. + /// + /// Unlike typical DFA implementations, the start state for DFAs in this + /// crate is dependent on a few different factors: + /// + /// * The [`Anchored`] mode of the search. Unanchored, anchored and + /// anchored searches for a specific [`PatternID`] all use different start + /// states. + /// * Whether a "look-behind" byte exists. For example, the `^` anchor + /// matches if and only if there is no look-behind byte. + /// * The specific value of that look-behind byte. For example, a `(?m:^)` + /// assertion only matches when there is either no look-behind byte, or + /// when the look-behind byte is a line terminator. + /// + /// The [starting configuration](start::Config) provides the above + /// information. + /// + /// This routine can be used for either forward or reverse searches. + /// Although, as a convenience, if you have an [`Input`], then it + /// may be more succinct to use [`DFA::start_state_forward`] or + /// [`DFA::start_state_reverse`]. Note, for example, that the convenience + /// routines return a [`MatchError`] on failure where as this routine + /// returns a [`StartError`]. + /// + /// # Errors + /// + /// This may return a [`StartError`] if the search needs to give up when + /// determining the start state (for example, if it sees a "quit" byte + /// or if the cache has become inefficient). This can also return an + /// error if the given configuration contains an unsupported [`Anchored`] + /// configuration. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub fn start_state( + &self, + cache: &mut Cache, + config: &start::Config, + ) -> Result<LazyStateID, StartError> { + let lazy = LazyRef::new(self, cache); + let anchored = config.get_anchored(); + let start = match config.get_look_behind() { + None => Start::Text, + Some(byte) => { + if !self.quitset.is_empty() && self.quitset.contains(byte) { + return Err(StartError::quit(byte)); + } + self.start_map.get(byte) + } + }; + let start_id = lazy.get_cached_start_id(anchored, start)?; + if !start_id.is_unknown() { + return Ok(start_id); + } + Lazy::new(self, cache).cache_start_group(anchored, start) + } + + /// Return the ID of the start state for this lazy DFA when executing a + /// forward search. + /// + /// This is a convenience routine for calling [`DFA::start_state`] that + /// converts the given [`Input`] to a [start configuration](start::Config). + /// Additionally, if an error occurs, it is converted from a [`StartError`] + /// to a [`MatchError`] using the offset information in the given + /// [`Input`]. + /// + /// # Errors + /// + /// This may return a [`MatchError`] if the search needs to give up when + /// determining the start state (for example, if it sees a "quit" byte or + /// if the cache has become inefficient). This can also return an error if + /// the given `Input` contains an unsupported [`Anchored`] configuration. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub fn start_state_forward( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<LazyStateID, MatchError> { + let config = start::Config::from_input_forward(input); + self.start_state(cache, &config).map_err(|err| match err { + StartError::Cache { .. } => MatchError::gave_up(input.start()), + StartError::Quit { byte } => { + let offset = input + .start() + .checked_sub(1) + .expect("no quit in start without look-behind"); + MatchError::quit(byte, offset) + } + StartError::UnsupportedAnchored { mode } => { + MatchError::unsupported_anchored(mode) + } + }) + } + + /// Return the ID of the start state for this lazy DFA when executing a + /// reverse search. + /// + /// This is a convenience routine for calling [`DFA::start_state`] that + /// converts the given [`Input`] to a [start configuration](start::Config). + /// Additionally, if an error occurs, it is converted from a [`StartError`] + /// to a [`MatchError`] using the offset information in the given + /// [`Input`]. + /// + /// # Errors + /// + /// This may return a [`MatchError`] if the search needs to give up when + /// determining the start state (for example, if it sees a "quit" byte or + /// if the cache has become inefficient). This can also return an error if + /// the given `Input` contains an unsupported [`Anchored`] configuration. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub fn start_state_reverse( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<LazyStateID, MatchError> { + let config = start::Config::from_input_reverse(input); + self.start_state(cache, &config).map_err(|err| match err { + StartError::Cache { .. } => MatchError::gave_up(input.end()), + StartError::Quit { byte } => { + let offset = input.end(); + MatchError::quit(byte, offset) + } + StartError::UnsupportedAnchored { mode } => { + MatchError::unsupported_anchored(mode) + } + }) + } + + /// Returns the total number of patterns that match in this state. + /// + /// If the lazy DFA was compiled with one pattern, then this must + /// necessarily always return `1` for all match states. + /// + /// A lazy DFA guarantees that [`DFA::match_pattern`] can be called with + /// indices up to (but not including) the length returned by this routine + /// without panicking. + /// + /// # Panics + /// + /// If the given state is not a match state, then this may either panic + /// or return an incorrect result. + /// + /// # Example + /// + /// This example shows a simple instance of implementing overlapping + /// matches. In particular, it shows not only how to determine how many + /// patterns have matched in a particular state, but also how to access + /// which specific patterns have matched. + /// + /// Notice that we must use [`MatchKind::All`] when building the DFA. If we + /// used [`MatchKind::LeftmostFirst`] instead, then the DFA would not be + /// constructed in a way that supports overlapping matches. (It would only + /// report a single pattern that matches at any particular point in time.) + /// + /// Another thing to take note of is the patterns used and the order in + /// which the pattern IDs are reported. In the example below, pattern `3` + /// is yielded first. Why? Because it corresponds to the match that + /// appears first. Namely, the `@` symbol is part of `\S+` but not part + /// of any of the other patterns. Since the `\S+` pattern has a match that + /// starts to the left of any other pattern, its ID is returned before any + /// other. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, Input, MatchKind}; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .build_many(&[ + /// r"\w+", r"[a-z]+", r"[A-Z]+", r"\S+", + /// ])?; + /// let mut cache = dfa.create_cache(); + /// let haystack = "@bar".as_bytes(); + /// + /// // The start state is determined by inspecting the position and the + /// // initial bytes of the haystack. + /// let mut sid = dfa.start_state_forward( + /// &mut cache, &Input::new(haystack), + /// )?; + /// // Walk all the bytes in the haystack. + /// for &b in haystack { + /// sid = dfa.next_state(&mut cache, sid, b)?; + /// } + /// sid = dfa.next_eoi_state(&mut cache, sid)?; + /// + /// assert!(sid.is_match()); + /// assert_eq!(dfa.match_len(&mut cache, sid), 3); + /// // The following calls are guaranteed to not panic since `match_len` + /// // returned `3` above. + /// assert_eq!(dfa.match_pattern(&mut cache, sid, 0).as_usize(), 3); + /// assert_eq!(dfa.match_pattern(&mut cache, sid, 1).as_usize(), 0); + /// assert_eq!(dfa.match_pattern(&mut cache, sid, 2).as_usize(), 1); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn match_len(&self, cache: &Cache, id: LazyStateID) -> usize { + assert!(id.is_match()); + LazyRef::new(self, cache).get_cached_state(id).match_len() + } + + /// Returns the pattern ID corresponding to the given match index in the + /// given state. + /// + /// See [`DFA::match_len`] for an example of how to use this method + /// correctly. Note that if you know your lazy DFA is configured with a + /// single pattern, then this routine is never necessary since it will + /// always return a pattern ID of `0` for an index of `0` when `id` + /// corresponds to a match state. + /// + /// Typically, this routine is used when implementing an overlapping + /// search, as the example for `DFA::match_len` does. + /// + /// # Panics + /// + /// If the state ID is not a match state or if the match index is out + /// of bounds for the given state, then this routine may either panic + /// or produce an incorrect result. If the state ID is correct and the + /// match index is correct, then this routine always produces a valid + /// `PatternID`. + #[inline] + pub fn match_pattern( + &self, + cache: &Cache, + id: LazyStateID, + match_index: usize, + ) -> PatternID { + // This is an optimization for the very common case of a DFA with a + // single pattern. This conditional avoids a somewhat more costly path + // that finds the pattern ID from the corresponding `State`, which + // requires a bit of slicing/pointer-chasing. This optimization tends + // to only matter when matches are frequent. + if self.pattern_len() == 1 { + return PatternID::ZERO; + } + LazyRef::new(self, cache) + .get_cached_state(id) + .match_pattern(match_index) + } +} + +/// A cache represents a partially computed DFA. +/// +/// A cache is the key component that differentiates a classical DFA and a +/// hybrid NFA/DFA (also called a "lazy DFA"). Where a classical DFA builds a +/// complete transition table that can handle all possible inputs, a hybrid +/// NFA/DFA starts with an empty transition table and builds only the parts +/// required during search. The parts that are built are stored in a cache. For +/// this reason, a cache is a required parameter for nearly every operation on +/// a [`DFA`]. +/// +/// Caches can be created from their corresponding DFA via +/// [`DFA::create_cache`]. A cache can only be used with either the DFA that +/// created it, or the DFA that was most recently used to reset it with +/// [`Cache::reset`]. Using a cache with any other DFA may result in panics +/// or incorrect results. +#[derive(Clone, Debug)] +pub struct Cache { + // N.B. If you're looking to understand how determinization works, it + // is probably simpler to first grok src/dfa/determinize.rs, since that + // doesn't have the "laziness" component. + /// The transition table. + /// + /// Given a `current` LazyStateID and an `input` byte, the next state can + /// be computed via `trans[untagged(current) + equiv_class(input)]`. Notice + /// that no multiplication is used. That's because state identifiers are + /// "premultiplied." + /// + /// Note that the next state may be the "unknown" state. In this case, the + /// next state is not known and determinization for `current` on `input` + /// must be performed. + trans: Vec<LazyStateID>, + /// The starting states for this DFA. + /// + /// These are computed lazily. Initially, these are all set to "unknown" + /// lazy state IDs. + /// + /// When 'starts_for_each_pattern' is disabled (the default), then the size + /// of this is constrained to the possible starting configurations based + /// on the search parameters. (At time of writing, that's 4.) However, + /// when starting states for each pattern is enabled, then there are N + /// additional groups of starting states, where each group reflects the + /// different possible configurations and N is the number of patterns. + starts: Vec<LazyStateID>, + /// A sequence of NFA/DFA powerset states that have been computed for this + /// lazy DFA. This sequence is indexable by untagged LazyStateIDs. (Every + /// tagged LazyStateID can be used to index this sequence by converting it + /// to its untagged form.) + states: Vec<State>, + /// A map from states to their corresponding IDs. This map may be accessed + /// via the raw byte representation of a state, which means that a `State` + /// does not need to be allocated to determine whether it already exists + /// in this map. Indeed, the existence of such a state is what determines + /// whether we allocate a new `State` or not. + /// + /// The higher level idea here is that we do just enough determinization + /// for a state to check whether we've already computed it. If we have, + /// then we can save a little (albeit not much) work. The real savings is + /// in memory usage. If we never checked for trivially duplicate states, + /// then our memory usage would explode to unreasonable levels. + states_to_id: StateMap, + /// Sparse sets used to track which NFA states have been visited during + /// various traversals. + sparses: SparseSets, + /// Scratch space for traversing the NFA graph. (We use space on the heap + /// instead of the call stack.) + stack: Vec<NFAStateID>, + /// Scratch space for building a NFA/DFA powerset state. This is used to + /// help amortize allocation since not every powerset state generated is + /// added to the cache. In particular, if it already exists in the cache, + /// then there is no need to allocate a new `State` for it. + scratch_state_builder: StateBuilderEmpty, + /// A simple abstraction for handling the saving of at most a single state + /// across a cache clearing. This is required for correctness. Namely, if + /// adding a new state after clearing the cache fails, then the caller + /// must retain the ability to continue using the state ID given. The + /// state corresponding to the state ID is what we preserve across cache + /// clearings. + state_saver: StateSaver, + /// The memory usage, in bytes, used by 'states' and 'states_to_id'. We + /// track this as new states are added since states use a variable amount + /// of heap. Tracking this as we add states makes it possible to compute + /// the total amount of memory used by the determinizer in constant time. + memory_usage_state: usize, + /// The number of times the cache has been cleared. When a minimum cache + /// clear count is set, then the cache will return an error instead of + /// clearing the cache if the count has been exceeded. + clear_count: usize, + /// The total number of bytes searched since the last time this cache was + /// cleared, not including the current search. + /// + /// This can be added to the length of the current search to get the true + /// total number of bytes searched. + /// + /// This is generally only non-zero when the + /// `Cache::search_{start,update,finish}` APIs are used to track search + /// progress. + bytes_searched: usize, + /// The progress of the current search. + /// + /// This is only non-`None` when callers utilize the `Cache::search_start`, + /// `Cache::search_update` and `Cache::search_finish` APIs. + /// + /// The purpose of recording search progress is to be able to make a + /// determination about the efficiency of the cache. Namely, by keeping + /// track of the + progress: Option<SearchProgress>, +} + +impl Cache { + /// Create a new cache for the given lazy DFA. + /// + /// The cache returned should only be used for searches for the given DFA. + /// If you want to reuse the cache for another DFA, then you must call + /// [`Cache::reset`] with that DFA. + pub fn new(dfa: &DFA) -> Cache { + let mut cache = Cache { + trans: alloc::vec![], + starts: alloc::vec![], + states: alloc::vec![], + states_to_id: StateMap::new(), + sparses: SparseSets::new(dfa.get_nfa().states().len()), + stack: alloc::vec![], + scratch_state_builder: StateBuilderEmpty::new(), + state_saver: StateSaver::none(), + memory_usage_state: 0, + clear_count: 0, + bytes_searched: 0, + progress: None, + }; + debug!("pre-init lazy DFA cache size: {}", cache.memory_usage()); + Lazy { dfa, cache: &mut cache }.init_cache(); + debug!("post-init lazy DFA cache size: {}", cache.memory_usage()); + cache + } + + /// Reset this cache such that it can be used for searching with the given + /// lazy DFA (and only that DFA). + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different lazy DFA. + /// + /// Resetting a cache sets its "clear count" to 0. This is relevant if the + /// lazy DFA has been configured to "give up" after it has cleared the + /// cache a certain number of times. + /// + /// Any lazy state ID generated by the cache prior to resetting it is + /// invalid after the reset. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different DFA. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// let dfa1 = DFA::new(r"\w")?; + /// let dfa2 = DFA::new(r"\W")?; + /// + /// let mut cache = dfa1.create_cache(); + /// assert_eq!( + /// Some(HalfMatch::must(0, 2)), + /// dfa1.try_search_fwd(&mut cache, &Input::new("Δ"))?, + /// ); + /// + /// // Using 'cache' with dfa2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the DFA we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 'dfa1' is also not + /// // allowed. + /// cache.reset(&dfa2); + /// assert_eq!( + /// Some(HalfMatch::must(0, 3)), + /// dfa2.try_search_fwd(&mut cache, &Input::new("☃"))?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset(&mut self, dfa: &DFA) { + Lazy::new(dfa, self).reset_cache() + } + + /// Initializes a new search starting at the given position. + /// + /// If a previous search was unfinished, then it is finished automatically + /// and a new search is begun. + /// + /// Note that keeping track of search progress is _not necessary_ + /// for correct implementations of search using a lazy DFA. Keeping + /// track of search progress is only necessary if you want the + /// [`Config::minimum_bytes_per_state`] configuration knob to work. + #[inline] + pub fn search_start(&mut self, at: usize) { + // If a previous search wasn't marked as finished, then finish it + // now automatically. + if let Some(p) = self.progress.take() { + self.bytes_searched += p.len(); + } + self.progress = Some(SearchProgress { start: at, at }); + } + + /// Updates the current search to indicate that it has search to the + /// current position. + /// + /// No special care needs to be taken for reverse searches. Namely, the + /// position given may be _less than_ the starting position of the search. + /// + /// # Panics + /// + /// This panics if no search has been started by [`Cache::search_start`]. + #[inline] + pub fn search_update(&mut self, at: usize) { + let p = + self.progress.as_mut().expect("no in-progress search to update"); + p.at = at; + } + + /// Indicates that a search has finished at the given position. + /// + /// # Panics + /// + /// This panics if no search has been started by [`Cache::search_start`]. + #[inline] + pub fn search_finish(&mut self, at: usize) { + let mut p = + self.progress.take().expect("no in-progress search to finish"); + p.at = at; + self.bytes_searched += p.len(); + } + + /// Returns the total number of bytes that have been searched since this + /// cache was last cleared. + /// + /// This is useful for determining the efficiency of the cache. For + /// example, the lazy DFA uses this value in conjunction with the + /// [`Config::minimum_bytes_per_state`] knob to help determine whether it + /// should quit searching. + /// + /// This always returns `0` if search progress isn't being tracked. Note + /// that the lazy DFA search routines in this crate always track search + /// progress. + pub fn search_total_len(&self) -> usize { + self.bytes_searched + self.progress.as_ref().map_or(0, |p| p.len()) + } + + /// Returns the total number of times this cache has been cleared since it + /// was either created or last reset. + /// + /// This is useful for informational purposes or if you want to change + /// search strategies based on the number of times the cache has been + /// cleared. + pub fn clear_count(&self) -> usize { + self.clear_count + } + + /// Returns the heap memory usage, in bytes, of this cache. + /// + /// This does **not** include the stack size used up by this cache. To + /// compute that, use `std::mem::size_of::<Cache>()`. + pub fn memory_usage(&self) -> usize { + const ID_SIZE: usize = size_of::<LazyStateID>(); + const STATE_SIZE: usize = size_of::<State>(); + + // NOTE: If you make changes to the below, then + // 'minimum_cache_capacity' should be updated correspondingly. + + self.trans.len() * ID_SIZE + + self.starts.len() * ID_SIZE + + self.states.len() * STATE_SIZE + // Maps likely use more memory than this, but it's probably close. + + self.states_to_id.len() * (STATE_SIZE + ID_SIZE) + + self.sparses.memory_usage() + + self.stack.capacity() * ID_SIZE + + self.scratch_state_builder.capacity() + // Heap memory used by 'State' in both 'states' and 'states_to_id'. + + self.memory_usage_state + } +} + +/// Keeps track of the progress of the current search. +/// +/// This is updated via the `Cache::search_{start,update,finish}` APIs to +/// record how many bytes have been searched. This permits computing a +/// heuristic that represents the efficiency of a cache, and thus helps inform +/// whether the lazy DFA should give up or not. +#[derive(Clone, Debug)] +struct SearchProgress { + start: usize, + at: usize, +} + +impl SearchProgress { + /// Returns the length, in bytes, of this search so far. + /// + /// This automatically handles the case of a reverse search, where `at` + /// is likely to be less than `start`. + fn len(&self) -> usize { + if self.start <= self.at { + self.at - self.start + } else { + self.start - self.at + } + } +} + +/// A map from states to state identifiers. When using std, we use a standard +/// hashmap, since it's a bit faster for this use case. (Other maps, like +/// one's based on FNV, have not yet been benchmarked.) +/// +/// The main purpose of this map is to reuse states where possible. This won't +/// fully minimize the DFA, but it works well in a lot of cases. +#[cfg(feature = "std")] +type StateMap = std::collections::HashMap<State, LazyStateID>; +#[cfg(not(feature = "std"))] +type StateMap = alloc::collections::BTreeMap<State, LazyStateID>; + +/// A type that groups methods that require the base NFA/DFA and writable +/// access to the cache. +#[derive(Debug)] +struct Lazy<'i, 'c> { + dfa: &'i DFA, + cache: &'c mut Cache, +} + +impl<'i, 'c> Lazy<'i, 'c> { + /// Creates a new 'Lazy' wrapper for a DFA and its corresponding cache. + fn new(dfa: &'i DFA, cache: &'c mut Cache) -> Lazy<'i, 'c> { + Lazy { dfa, cache } + } + + /// Return an immutable view by downgrading a writable cache to a read-only + /// cache. + fn as_ref<'a>(&'a self) -> LazyRef<'i, 'a> { + LazyRef::new(self.dfa, self.cache) + } + + /// This is marked as 'inline(never)' to avoid bloating methods on 'DFA' + /// like 'next_state' and 'next_eoi_state' that are called in critical + /// areas. The idea is to let the optimizer focus on the other areas of + /// those methods as the hot path. + /// + /// Here's an example that justifies 'inline(never)' + /// + /// ```ignore + /// regex-cli find match hybrid \ + /// --cache-capacity 100000000 \ + /// -p '\pL{100}' + /// all-codepoints-utf8-100x + /// ``` + /// + /// Where 'all-codepoints-utf8-100x' is the UTF-8 encoding of every + /// codepoint, in sequence, repeated 100 times. + /// + /// With 'inline(never)' hyperfine reports 1.1s per run. With + /// 'inline(always)', hyperfine reports 1.23s. So that's a 10% improvement. + #[cold] + #[inline(never)] + fn cache_next_state( + &mut self, + mut current: LazyStateID, + unit: alphabet::Unit, + ) -> Result<LazyStateID, CacheError> { + let stride2 = self.dfa.stride2(); + let empty_builder = self.get_state_builder(); + let builder = determinize::next( + self.dfa.get_nfa(), + self.dfa.get_config().get_match_kind(), + &mut self.cache.sparses, + &mut self.cache.stack, + &self.cache.states[current.as_usize_untagged() >> stride2], + unit, + empty_builder, + ); + // This is subtle, but if we *might* clear the cache, then we should + // try to save the current state so that we can re-map its ID after + // cache clearing. We *might* clear the cache when either the new + // state can't fit in the cache or when the number of transitions has + // reached the maximum. Even if either of these conditions is true, + // the cache might not be cleared if we can reuse an existing state. + // But we don't know that at this point. Moreover, we don't save the + // current state every time because it is costly. + // + // TODO: We should try to find a way to make this less subtle and error + // prone. ---AG + let save_state = !self.as_ref().state_builder_fits_in_cache(&builder) + || self.cache.trans.len() >= LazyStateID::MAX; + if save_state { + self.save_state(current); + } + let next = self.add_builder_state(builder, |sid| sid)?; + if save_state { + current = self.saved_state_id(); + } + // This is the payoff. The next time 'next_state' is called with this + // state and alphabet unit, it will find this transition and avoid + // having to re-determinize this transition. + self.set_transition(current, unit, next); + Ok(next) + } + + /// Compute and cache the starting state for the given pattern ID (if + /// present) and the starting configuration. + /// + /// This panics if a pattern ID is given and the DFA isn't configured to + /// build anchored start states for each pattern. + /// + /// This will never return an unknown lazy state ID. + /// + /// If caching this state would otherwise result in a cache that has been + /// cleared too many times, then an error is returned. + #[cold] + #[inline(never)] + fn cache_start_group( + &mut self, + anchored: Anchored, + start: Start, + ) -> Result<LazyStateID, StartError> { + let nfa_start_id = match anchored { + Anchored::No => self.dfa.get_nfa().start_unanchored(), + Anchored::Yes => self.dfa.get_nfa().start_anchored(), + Anchored::Pattern(pid) => { + if !self.dfa.get_config().get_starts_for_each_pattern() { + return Err(StartError::unsupported_anchored(anchored)); + } + match self.dfa.get_nfa().start_pattern(pid) { + None => return Ok(self.as_ref().dead_id()), + Some(sid) => sid, + } + } + }; + + let id = self + .cache_start_one(nfa_start_id, start) + .map_err(StartError::cache)?; + self.set_start_state(anchored, start, id); + Ok(id) + } + + /// Compute and cache the starting state for the given NFA state ID and the + /// starting configuration. The NFA state ID might be one of the following: + /// + /// 1) An unanchored start state to match any pattern. + /// 2) An anchored start state to match any pattern. + /// 3) An anchored start state for a particular pattern. + /// + /// This will never return an unknown lazy state ID. + /// + /// If caching this state would otherwise result in a cache that has been + /// cleared too many times, then an error is returned. + fn cache_start_one( + &mut self, + nfa_start_id: NFAStateID, + start: Start, + ) -> Result<LazyStateID, CacheError> { + let mut builder_matches = self.get_state_builder().into_matches(); + determinize::set_lookbehind_from_start( + self.dfa.get_nfa(), + &start, + &mut builder_matches, + ); + self.cache.sparses.set1.clear(); + determinize::epsilon_closure( + self.dfa.get_nfa(), + nfa_start_id, + builder_matches.look_have(), + &mut self.cache.stack, + &mut self.cache.sparses.set1, + ); + let mut builder = builder_matches.into_nfa(); + determinize::add_nfa_states( + &self.dfa.get_nfa(), + &self.cache.sparses.set1, + &mut builder, + ); + let tag_starts = self.dfa.get_config().get_specialize_start_states(); + self.add_builder_state(builder, |id| { + if tag_starts { + id.to_start() + } else { + id + } + }) + } + + /// Either add the given builder state to this cache, or return an ID to an + /// equivalent state already in this cache. + /// + /// In the case where no equivalent state exists, the idmap function given + /// may be used to transform the identifier allocated. This is useful if + /// the caller needs to tag the ID with additional information. + /// + /// This will never return an unknown lazy state ID. + /// + /// If caching this state would otherwise result in a cache that has been + /// cleared too many times, then an error is returned. + fn add_builder_state( + &mut self, + builder: StateBuilderNFA, + idmap: impl Fn(LazyStateID) -> LazyStateID, + ) -> Result<LazyStateID, CacheError> { + if let Some(&cached_id) = + self.cache.states_to_id.get(builder.as_bytes()) + { + // Since we have a cached state, put the constructed state's + // memory back into our scratch space, so that it can be reused. + self.put_state_builder(builder); + return Ok(cached_id); + } + let result = self.add_state(builder.to_state(), idmap); + self.put_state_builder(builder); + result + } + + /// Allocate a new state ID and add the given state to this cache. + /// + /// The idmap function given may be used to transform the identifier + /// allocated. This is useful if the caller needs to tag the ID with + /// additional information. + /// + /// This will never return an unknown lazy state ID. + /// + /// If caching this state would otherwise result in a cache that has been + /// cleared too many times, then an error is returned. + fn add_state( + &mut self, + state: State, + idmap: impl Fn(LazyStateID) -> LazyStateID, + ) -> Result<LazyStateID, CacheError> { + if !self.as_ref().state_fits_in_cache(&state) { + self.try_clear_cache()?; + } + // It's important for this to come second, since the above may clear + // the cache. If we clear the cache after ID generation, then the ID + // is likely bunk since it would have been generated based on a larger + // transition table. + let mut id = idmap(self.next_state_id()?); + if state.is_match() { + id = id.to_match(); + } + // Add room in the transition table. Since this is a fresh state, all + // of its transitions are unknown. + self.cache.trans.extend( + iter::repeat(self.as_ref().unknown_id()).take(self.dfa.stride()), + ); + // When we add a sentinel state, we never want to set any quit + // transitions. Technically, this is harmless, since sentinel states + // have all of their transitions set to loop back to themselves. But + // when creating sentinel states before the quit sentinel state, + // this will try to call 'set_transition' on a state ID that doesn't + // actually exist yet, which isn't allowed. So we just skip doing so + // entirely. + if !self.dfa.quitset.is_empty() && !self.as_ref().is_sentinel(id) { + let quit_id = self.as_ref().quit_id(); + for b in self.dfa.quitset.iter() { + self.set_transition(id, alphabet::Unit::u8(b), quit_id); + } + } + self.cache.memory_usage_state += state.memory_usage(); + self.cache.states.push(state.clone()); + self.cache.states_to_id.insert(state, id); + Ok(id) + } + + /// Allocate a new state ID. + /// + /// This will never return an unknown lazy state ID. + /// + /// If caching this state would otherwise result in a cache that has been + /// cleared too many times, then an error is returned. + fn next_state_id(&mut self) -> Result<LazyStateID, CacheError> { + let sid = match LazyStateID::new(self.cache.trans.len()) { + Ok(sid) => sid, + Err(_) => { + self.try_clear_cache()?; + // This has to pass since we check that ID capacity at + // construction time can fit at least MIN_STATES states. + LazyStateID::new(self.cache.trans.len()).unwrap() + } + }; + Ok(sid) + } + + /// Attempt to clear the cache used by this lazy DFA. + /// + /// If clearing the cache exceeds the minimum number of required cache + /// clearings, then this will return a cache error. In this case, + /// callers should bubble this up as the cache can't be used until it is + /// reset. Implementations of search should convert this error into a + /// [`MatchError::gave_up`]. + /// + /// If 'self.state_saver' is set to save a state, then this state is + /// persisted through cache clearing. Otherwise, the cache is returned to + /// its state after initialization with two exceptions: its clear count + /// is incremented and some of its memory likely has additional capacity. + /// That is, clearing a cache does _not_ release memory. + /// + /// Otherwise, any lazy state ID generated by the cache prior to resetting + /// it is invalid after the reset. + fn try_clear_cache(&mut self) -> Result<(), CacheError> { + let c = self.dfa.get_config(); + if let Some(min_count) = c.get_minimum_cache_clear_count() { + if self.cache.clear_count >= min_count { + if let Some(min_bytes_per) = c.get_minimum_bytes_per_state() { + let len = self.cache.search_total_len(); + let min_bytes = + min_bytes_per.saturating_mul(self.cache.states.len()); + // If we've searched 0 bytes then probably something has + // gone wrong and the lazy DFA search implementation isn't + // correctly updating the search progress state. + if len == 0 { + trace!( + "number of bytes searched is 0, but \ + a minimum bytes per state searched ({}) is \ + enabled, maybe Cache::search_update \ + is not being used?", + min_bytes_per, + ); + } + if len < min_bytes { + trace!( + "lazy DFA cache has been cleared {} times, \ + which exceeds the limit of {}, \ + AND its bytes searched per state is less \ + than the configured minimum of {}, \ + therefore lazy DFA is giving up \ + (bytes searched since cache clear = {}, \ + number of states = {})", + self.cache.clear_count, + min_count, + min_bytes_per, + len, + self.cache.states.len(), + ); + return Err(CacheError::bad_efficiency()); + } else { + trace!( + "lazy DFA cache has been cleared {} times, \ + which exceeds the limit of {}, \ + AND its bytes searched per state is greater \ + than the configured minimum of {}, \ + therefore lazy DFA is continuing! \ + (bytes searched since cache clear = {}, \ + number of states = {})", + self.cache.clear_count, + min_count, + min_bytes_per, + len, + self.cache.states.len(), + ); + } + } else { + trace!( + "lazy DFA cache has been cleared {} times, \ + which exceeds the limit of {}, \ + since there is no configured bytes per state \ + minimum, lazy DFA is giving up", + self.cache.clear_count, + min_count, + ); + return Err(CacheError::too_many_cache_clears()); + } + } + } + self.clear_cache(); + Ok(()) + } + + /// Clears _and_ resets the cache. Resetting the cache means that no + /// states are persisted and the clear count is reset to 0. No heap memory + /// is released. + /// + /// Note that the caller may reset a cache with a different DFA than what + /// it was created from. In which case, the cache can now be used with the + /// new DFA (and not the old DFA). + fn reset_cache(&mut self) { + self.cache.state_saver = StateSaver::none(); + self.clear_cache(); + // If a new DFA is used, it might have a different number of NFA + // states, so we need to make sure our sparse sets have the appropriate + // size. + self.cache.sparses.resize(self.dfa.get_nfa().states().len()); + self.cache.clear_count = 0; + self.cache.progress = None; + } + + /// Clear the cache used by this lazy DFA. + /// + /// If 'self.state_saver' is set to save a state, then this state is + /// persisted through cache clearing. Otherwise, the cache is returned to + /// its state after initialization with two exceptions: its clear count + /// is incremented and some of its memory likely has additional capacity. + /// That is, clearing a cache does _not_ release memory. + /// + /// Otherwise, any lazy state ID generated by the cache prior to resetting + /// it is invalid after the reset. + fn clear_cache(&mut self) { + self.cache.trans.clear(); + self.cache.starts.clear(); + self.cache.states.clear(); + self.cache.states_to_id.clear(); + self.cache.memory_usage_state = 0; + self.cache.clear_count += 1; + self.cache.bytes_searched = 0; + if let Some(ref mut progress) = self.cache.progress { + progress.start = progress.at; + } + trace!( + "lazy DFA cache has been cleared (count: {})", + self.cache.clear_count + ); + self.init_cache(); + // If the state we want to save is one of the sentinel + // (unknown/dead/quit) states, then 'init_cache' adds those back, and + // their identifier values remains invariant. So there's no need to add + // it again. (And indeed, doing so would be incorrect!) + if let Some((old_id, state)) = self.cache.state_saver.take_to_save() { + // If the state is one of the special sentinel states, then it is + // automatically added by cache initialization and its ID always + // remains the same. With that said, this should never occur since + // the sentinel states are all loop states back to themselves. So + // we should never be in a position where we're attempting to save + // a sentinel state since we never compute transitions out of a + // sentinel state. + assert!( + !self.as_ref().is_sentinel(old_id), + "cannot save sentinel state" + ); + let new_id = self + .add_state(state, |id| { + if old_id.is_start() { + // We don't need to consult the + // 'specialize_start_states' config knob here, because + // if it's disabled, old_id.is_start() will never + // return true. + id.to_start() + } else { + id + } + }) + // The unwrap here is OK because lazy DFA creation ensures that + // we have room in the cache to add MIN_STATES states. Since + // 'init_cache' above adds 3, this adds a 4th. + .expect("adding one state after cache clear must work"); + self.cache.state_saver = StateSaver::Saved(new_id); + } + } + + /// Initialize this cache from emptiness to a place where it can be used + /// for search. + /// + /// This is called both at cache creation time and after the cache has been + /// cleared. + /// + /// Primarily, this adds the three sentinel states and allocates some + /// initial memory. + fn init_cache(&mut self) { + // Why multiply by 2 here? Because we make room for both the unanchored + // and anchored start states. Unanchored is first and then anchored. + let mut starts_len = Start::len().checked_mul(2).unwrap(); + // ... but if we also want start states for every pattern, we make room + // for that too. + if self.dfa.get_config().get_starts_for_each_pattern() { + starts_len += Start::len() * self.dfa.pattern_len(); + } + self.cache + .starts + .extend(iter::repeat(self.as_ref().unknown_id()).take(starts_len)); + // This is the set of NFA states that corresponds to each of our three + // sentinel states: the empty set. + let dead = State::dead(); + // This sets up some states that we use as sentinels that are present + // in every DFA. While it would be technically possible to implement + // this DFA without explicitly putting these states in the transition + // table, this is convenient to do to make `next_state` correct for all + // valid state IDs without needing explicit conditionals to special + // case these sentinel states. + // + // All three of these states are "dead" states. That is, all of + // them transition only to themselves. So once you enter one of + // these states, it's impossible to leave them. Thus, any correct + // search routine must explicitly check for these state types. (Sans + // `unknown`, since that is only used internally to represent missing + // states.) + let unk_id = + self.add_state(dead.clone(), |id| id.to_unknown()).unwrap(); + let dead_id = self.add_state(dead.clone(), |id| id.to_dead()).unwrap(); + let quit_id = self.add_state(dead.clone(), |id| id.to_quit()).unwrap(); + assert_eq!(unk_id, self.as_ref().unknown_id()); + assert_eq!(dead_id, self.as_ref().dead_id()); + assert_eq!(quit_id, self.as_ref().quit_id()); + // The idea here is that if you start in an unknown/dead/quit state and + // try to transition on them, then you should end up where you started. + self.set_all_transitions(unk_id, unk_id); + self.set_all_transitions(dead_id, dead_id); + self.set_all_transitions(quit_id, quit_id); + // All of these states are technically equivalent from the FSM + // perspective, so putting all three of them in the cache isn't + // possible. (They are distinct merely because we use their + // identifiers as sentinels to mean something, as indicated by the + // names.) Moreover, we wouldn't want to do that. Unknown and quit + // states are special in that they are artificial constructions + // this implementation. But dead states are a natural part of + // determinization. When you reach a point in the NFA where you cannot + // go anywhere else, a dead state will naturally arise and we MUST + // reuse the canonical dead state that we've created here. Why? Because + // it is the state ID that tells the search routine whether a state is + // dead or not, and thus, whether to stop the search. Having a bunch of + // distinct dead states would be quite wasteful! + self.cache.states_to_id.insert(dead, dead_id); + } + + /// Save the state corresponding to the ID given such that the state + /// persists through a cache clearing. + /// + /// While the state may persist, the ID may not. In order to discover the + /// new state ID, one must call 'saved_state_id' after a cache clearing. + fn save_state(&mut self, id: LazyStateID) { + let state = self.as_ref().get_cached_state(id).clone(); + self.cache.state_saver = StateSaver::ToSave { id, state }; + } + + /// Returns the updated lazy state ID for a state that was persisted + /// through a cache clearing. + /// + /// It is only correct to call this routine when both a state has been + /// saved and the cache has just been cleared. Otherwise, this panics. + fn saved_state_id(&mut self) -> LazyStateID { + self.cache + .state_saver + .take_saved() + .expect("state saver does not have saved state ID") + } + + /// Set all transitions on the state 'from' to 'to'. + fn set_all_transitions(&mut self, from: LazyStateID, to: LazyStateID) { + for unit in self.dfa.classes.representatives(..) { + self.set_transition(from, unit, to); + } + } + + /// Set the transition on 'from' for 'unit' to 'to'. + /// + /// This panics if either 'from' or 'to' is invalid. + /// + /// All unit values are OK. + fn set_transition( + &mut self, + from: LazyStateID, + unit: alphabet::Unit, + to: LazyStateID, + ) { + assert!(self.as_ref().is_valid(from), "invalid 'from' id: {from:?}"); + assert!(self.as_ref().is_valid(to), "invalid 'to' id: {to:?}"); + let offset = + from.as_usize_untagged() + self.dfa.classes.get_by_unit(unit); + self.cache.trans[offset] = to; + } + + /// Set the start ID for the given pattern ID (if given) and starting + /// configuration to the ID given. + /// + /// This panics if 'id' is not valid or if a pattern ID is given and + /// 'starts_for_each_pattern' is not enabled. + fn set_start_state( + &mut self, + anchored: Anchored, + start: Start, + id: LazyStateID, + ) { + assert!(self.as_ref().is_valid(id)); + let start_index = start.as_usize(); + let index = match anchored { + Anchored::No => start_index, + Anchored::Yes => Start::len() + start_index, + Anchored::Pattern(pid) => { + assert!( + self.dfa.get_config().get_starts_for_each_pattern(), + "attempted to search for a specific pattern \ + without enabling starts_for_each_pattern", + ); + let pid = pid.as_usize(); + (2 * Start::len()) + (Start::len() * pid) + start_index + } + }; + self.cache.starts[index] = id; + } + + /// Returns a state builder from this DFA that might have existing + /// capacity. This helps avoid allocs in cases where a state is built that + /// turns out to already be cached. + /// + /// Callers must put the state builder back with 'put_state_builder', + /// otherwise the allocation reuse won't work. + fn get_state_builder(&mut self) -> StateBuilderEmpty { + core::mem::replace( + &mut self.cache.scratch_state_builder, + StateBuilderEmpty::new(), + ) + } + + /// Puts the given state builder back into this DFA for reuse. + /// + /// Note that building a 'State' from a builder always creates a new alloc, + /// so callers should always put the builder back. + fn put_state_builder(&mut self, builder: StateBuilderNFA) { + let _ = core::mem::replace( + &mut self.cache.scratch_state_builder, + builder.clear(), + ); + } +} + +/// A type that groups methods that require the base NFA/DFA and read-only +/// access to the cache. +#[derive(Debug)] +struct LazyRef<'i, 'c> { + dfa: &'i DFA, + cache: &'c Cache, +} + +impl<'i, 'c> LazyRef<'i, 'c> { + /// Creates a new 'Lazy' wrapper for a DFA and its corresponding cache. + fn new(dfa: &'i DFA, cache: &'c Cache) -> LazyRef<'i, 'c> { + LazyRef { dfa, cache } + } + + /// Return the ID of the start state for the given configuration. + /// + /// If the start state has not yet been computed, then this returns an + /// unknown lazy state ID. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn get_cached_start_id( + &self, + anchored: Anchored, + start: Start, + ) -> Result<LazyStateID, StartError> { + let start_index = start.as_usize(); + let index = match anchored { + Anchored::No => start_index, + Anchored::Yes => Start::len() + start_index, + Anchored::Pattern(pid) => { + if !self.dfa.get_config().get_starts_for_each_pattern() { + return Err(StartError::unsupported_anchored(anchored)); + } + if pid.as_usize() >= self.dfa.pattern_len() { + return Ok(self.dead_id()); + } + (2 * Start::len()) + + (Start::len() * pid.as_usize()) + + start_index + } + }; + Ok(self.cache.starts[index]) + } + + /// Return the cached NFA/DFA powerset state for the given ID. + /// + /// This panics if the given ID does not address a valid state. + fn get_cached_state(&self, sid: LazyStateID) -> &State { + let index = sid.as_usize_untagged() >> self.dfa.stride2(); + &self.cache.states[index] + } + + /// Returns true if and only if the given ID corresponds to a "sentinel" + /// state. + /// + /// A sentinel state is a state that signifies a special condition of + /// search, and where every transition maps back to itself. See LazyStateID + /// for more details. Note that start and match states are _not_ sentinels + /// since they may otherwise be real states with non-trivial transitions. + /// The purposes of sentinel states is purely to indicate something. Their + /// transitions are not meant to be followed. + fn is_sentinel(&self, id: LazyStateID) -> bool { + id == self.unknown_id() || id == self.dead_id() || id == self.quit_id() + } + + /// Returns the ID of the unknown state for this lazy DFA. + fn unknown_id(&self) -> LazyStateID { + // This unwrap is OK since 0 is always a valid state ID. + LazyStateID::new(0).unwrap().to_unknown() + } + + /// Returns the ID of the dead state for this lazy DFA. + fn dead_id(&self) -> LazyStateID { + // This unwrap is OK since the maximum value here is 1 * 512 = 512, + // which is <= 2047 (the maximum state ID on 16-bit systems). Where + // 512 is the worst case for our equivalence classes (every byte is a + // distinct class). + LazyStateID::new(1 << self.dfa.stride2()).unwrap().to_dead() + } + + /// Returns the ID of the quit state for this lazy DFA. + fn quit_id(&self) -> LazyStateID { + // This unwrap is OK since the maximum value here is 2 * 512 = 1024, + // which is <= 2047 (the maximum state ID on 16-bit systems). Where + // 512 is the worst case for our equivalence classes (every byte is a + // distinct class). + LazyStateID::new(2 << self.dfa.stride2()).unwrap().to_quit() + } + + /// Returns true if and only if the given ID is valid. + /// + /// An ID is valid if it is both a valid index into the transition table + /// and is a multiple of the DFA's stride. + fn is_valid(&self, id: LazyStateID) -> bool { + let id = id.as_usize_untagged(); + id < self.cache.trans.len() && id % self.dfa.stride() == 0 + } + + /// Returns true if adding the state given would fit in this cache. + fn state_fits_in_cache(&self, state: &State) -> bool { + let needed = self.cache.memory_usage() + + self.memory_usage_for_one_more_state(state.memory_usage()); + trace!( + "lazy DFA cache capacity state check: {:?} ?<=? {:?}", + needed, + self.dfa.cache_capacity + ); + needed <= self.dfa.cache_capacity + } + + /// Returns true if adding the state to be built by the given builder would + /// fit in this cache. + fn state_builder_fits_in_cache(&self, state: &StateBuilderNFA) -> bool { + let needed = self.cache.memory_usage() + + self.memory_usage_for_one_more_state(state.as_bytes().len()); + trace!( + "lazy DFA cache capacity state builder check: {:?} ?<=? {:?}", + needed, + self.dfa.cache_capacity + ); + needed <= self.dfa.cache_capacity + } + + /// Returns the additional memory usage, in bytes, required to add one more + /// state to this cache. The given size should be the heap size, in bytes, + /// that would be used by the new state being added. + fn memory_usage_for_one_more_state( + &self, + state_heap_size: usize, + ) -> usize { + const ID_SIZE: usize = size_of::<LazyStateID>(); + const STATE_SIZE: usize = size_of::<State>(); + + self.dfa.stride() * ID_SIZE // additional space needed in trans table + + STATE_SIZE // space in cache.states + + (STATE_SIZE + ID_SIZE) // space in cache.states_to_id + + state_heap_size // heap memory used by state itself + } +} + +/// A simple type that encapsulates the saving of a state ID through a cache +/// clearing. +/// +/// A state ID can be marked for saving with ToSave, while a state ID can be +/// saved itself with Saved. +#[derive(Clone, Debug)] +enum StateSaver { + /// An empty state saver. In this case, no states (other than the special + /// sentinel states) are preserved after clearing the cache. + None, + /// An ID of a state (and the state itself) that should be preserved after + /// the lazy DFA's cache has been cleared. After clearing, the updated ID + /// is stored in 'Saved' since it may have changed. + ToSave { id: LazyStateID, state: State }, + /// An ID that of a state that has been persisted through a lazy DFA + /// cache clearing. The ID recorded here corresponds to an ID that was + /// once marked as ToSave. The IDs are likely not equivalent even though + /// the states they point to are. + Saved(LazyStateID), +} + +impl StateSaver { + /// Create an empty state saver. + fn none() -> StateSaver { + StateSaver::None + } + + /// Replace this state saver with an empty saver, and if this saver is a + /// request to save a state, return that request. + fn take_to_save(&mut self) -> Option<(LazyStateID, State)> { + match core::mem::replace(self, StateSaver::None) { + StateSaver::None | StateSaver::Saved(_) => None, + StateSaver::ToSave { id, state } => Some((id, state)), + } + } + + /// Replace this state saver with an empty saver, and if this saver is a + /// saved state (or a request to save a state), return that state's ID. + /// + /// The idea here is that a request to save a state isn't necessarily + /// honored because it might not be needed. e.g., Some higher level code + /// might request a state to be saved on the off chance that the cache gets + /// cleared when a new state is added at a lower level. But if that new + /// state is never added, then the cache is never cleared and the state and + /// its ID remain unchanged. + fn take_saved(&mut self) -> Option<LazyStateID> { + match core::mem::replace(self, StateSaver::None) { + StateSaver::None => None, + StateSaver::Saved(id) | StateSaver::ToSave { id, .. } => Some(id), + } + } +} + +/// The configuration used for building a lazy DFA. +/// +/// As a convenience, [`DFA::config`] is an alias for [`Config::new`]. The +/// advantage of the former is that it often lets you avoid importing the +/// `Config` type directly. +/// +/// A lazy DFA configuration is a simple data object that is typically used +/// with [`Builder::configure`]. +/// +/// The default configuration guarantees that a search will never return a +/// "gave up" or "quit" error, although it is possible for a search to fail +/// if [`Config::starts_for_each_pattern`] wasn't enabled (which it is not by +/// default) and an [`Anchored::Pattern`] mode is requested via [`Input`]. +#[derive(Clone, Debug, Default)] +pub struct Config { + // As with other configuration types in this crate, we put all our knobs + // in options so that we can distinguish between "default" and "not set." + // This makes it possible to easily combine multiple configurations + // without default values overwriting explicitly specified values. See the + // 'overwrite' method. + // + // For docs on the fields below, see the corresponding method setters. + match_kind: Option<MatchKind>, + pre: Option<Option<Prefilter>>, + starts_for_each_pattern: Option<bool>, + byte_classes: Option<bool>, + unicode_word_boundary: Option<bool>, + quitset: Option<ByteSet>, + specialize_start_states: Option<bool>, + cache_capacity: Option<usize>, + skip_cache_capacity_check: Option<bool>, + minimum_cache_clear_count: Option<Option<usize>>, + minimum_bytes_per_state: Option<Option<usize>>, +} + +impl Config { + /// Return a new default lazy DFA builder configuration. + pub fn new() -> Config { + Config::default() + } + + /// Set the desired match semantics. + /// + /// The default is [`MatchKind::LeftmostFirst`], which corresponds to the + /// match semantics of Perl-like regex engines. That is, when multiple + /// patterns would match at the same leftmost position, the pattern that + /// appears first in the concrete syntax is chosen. + /// + /// Currently, the only other kind of match semantics supported is + /// [`MatchKind::All`]. This corresponds to classical DFA construction + /// where all possible matches are added to the lazy DFA. + /// + /// Typically, `All` is used when one wants to execute an overlapping + /// search and `LeftmostFirst` otherwise. In particular, it rarely makes + /// sense to use `All` with the various "leftmost" find routines, since the + /// leftmost routines depend on the `LeftmostFirst` automata construction + /// strategy. Specifically, `LeftmostFirst` adds dead states to the + /// lazy DFA as a way to terminate the search and report a match. + /// `LeftmostFirst` also supports non-greedy matches using this strategy + /// where as `All` does not. + /// + /// # Example: overlapping search + /// + /// This example shows the typical use of `MatchKind::All`, which is to + /// report overlapping matches. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// hybrid::dfa::{DFA, OverlappingState}, + /// HalfMatch, Input, MatchKind, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .build_many(&[r"\w+$", r"\S+$"])?; + /// let mut cache = dfa.create_cache(); + /// let haystack = "@foo"; + /// let mut state = OverlappingState::start(); + /// + /// let expected = Some(HalfMatch::must(1, 4)); + /// dfa.try_search_overlapping_fwd( + /// &mut cache, &Input::new(haystack), &mut state, + /// )?; + /// assert_eq!(expected, state.get_match()); + /// + /// // The first pattern also matches at the same position, so re-running + /// // the search will yield another match. Notice also that the first + /// // pattern is returned after the second. This is because the second + /// // pattern begins its match before the first, is therefore an earlier + /// // match and is thus reported first. + /// let expected = Some(HalfMatch::must(0, 4)); + /// dfa.try_search_overlapping_fwd( + /// &mut cache, &Input::new(haystack), &mut state, + /// )?; + /// assert_eq!(expected, state.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: reverse automaton to find start of match + /// + /// Another example for using `MatchKind::All` is for constructing a + /// reverse automaton to find the start of a match. `All` semantics are + /// used for this in order to find the longest possible match, which + /// corresponds to the leftmost starting position. + /// + /// Note that if you need the starting position then + /// [`hybrid::regex::Regex`](crate::hybrid::regex::Regex) will handle this + /// for you, so it's usually not necessary to do this yourself. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// nfa::thompson::NFA, + /// Anchored, HalfMatch, Input, MatchKind, + /// }; + /// + /// let input = Input::new("123foobar456"); + /// let pattern = r"[a-z]+r"; + /// + /// let dfa_fwd = DFA::new(pattern)?; + /// let dfa_rev = DFA::builder() + /// .thompson(NFA::config().reverse(true)) + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .build(pattern)?; + /// let mut cache_fwd = dfa_fwd.create_cache(); + /// let mut cache_rev = dfa_rev.create_cache(); + /// + /// let expected_fwd = HalfMatch::must(0, 9); + /// let expected_rev = HalfMatch::must(0, 3); + /// let got_fwd = dfa_fwd.try_search_fwd(&mut cache_fwd, &input)?.unwrap(); + /// // Here we don't specify the pattern to search for since there's only + /// // one pattern and we're doing a leftmost search. But if this were an + /// // overlapping search, you'd need to specify the pattern that matched + /// // in the forward direction. (Otherwise, you might wind up finding the + /// // starting position of a match of some other pattern.) That in turn + /// // requires building the reverse automaton with starts_for_each_pattern + /// // enabled. + /// let input = input + /// .clone() + /// .range(..got_fwd.offset()) + /// .anchored(Anchored::Yes); + /// let got_rev = dfa_rev.try_search_rev(&mut cache_rev, &input)?.unwrap(); + /// assert_eq!(expected_fwd, got_fwd); + /// assert_eq!(expected_rev, got_rev); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn match_kind(mut self, kind: MatchKind) -> Config { + self.match_kind = Some(kind); + self + } + + /// Set a prefilter to be used whenever a start state is entered. + /// + /// A [`Prefilter`] in this context is meant to accelerate searches by + /// looking for literal prefixes that every match for the corresponding + /// pattern (or patterns) must start with. Once a prefilter produces a + /// match, the underlying search routine continues on to try and confirm + /// the match. + /// + /// Be warned that setting a prefilter does not guarantee that the search + /// will be faster. While it's usually a good bet, if the prefilter + /// produces a lot of false positive candidates (i.e., positions matched + /// by the prefilter but not by the regex), then the overall result can + /// be slower than if you had just executed the regex engine without any + /// prefilters. + /// + /// Note that unless [`Config::specialize_start_states`] has been + /// explicitly set, then setting this will also enable (when `pre` is + /// `Some`) or disable (when `pre` is `None`) start state specialization. + /// This occurs because without start state specialization, a prefilter + /// is likely to be less effective. And without a prefilter, start state + /// specialization is usually pointless. + /// + /// By default no prefilter is set. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// util::prefilter::Prefilter, + /// Input, HalfMatch, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["foo", "bar"]); + /// let re = DFA::builder() + /// .configure(DFA::config().prefilter(pre)) + /// .build(r"(foo|bar)[a-z]+")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("foo1 barfox bar"); + /// assert_eq!( + /// Some(HalfMatch::must(0, 11)), + /// re.try_search_fwd(&mut cache, &input)?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Be warned though that an incorrect prefilter can lead to incorrect + /// results! + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// util::prefilter::Prefilter, + /// Input, HalfMatch, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["foo", "car"]); + /// let re = DFA::builder() + /// .configure(DFA::config().prefilter(pre)) + /// .build(r"(foo|bar)[a-z]+")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("foo1 barfox bar"); + /// assert_eq!( + /// // No match reported even though there clearly is one! + /// None, + /// re.try_search_fwd(&mut cache, &input)?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn prefilter(mut self, pre: Option<Prefilter>) -> Config { + self.pre = Some(pre); + if self.specialize_start_states.is_none() { + self.specialize_start_states = + Some(self.get_prefilter().is_some()); + } + self + } + + /// Whether to compile a separate start state for each pattern in the + /// lazy DFA. + /// + /// When enabled, a separate **anchored** start state is added for each + /// pattern in the lazy DFA. When this start state is used, then the DFA + /// will only search for matches for the pattern specified, even if there + /// are other patterns in the DFA. + /// + /// The main downside of this option is that it can potentially increase + /// the size of the DFA and/or increase the time it takes to build the + /// DFA at search time. However, since this is configuration for a lazy + /// DFA, these states aren't actually built unless they're used. Enabling + /// this isn't necessarily free, however, as it may result in higher cache + /// usage. + /// + /// There are a few reasons one might want to enable this (it's disabled + /// by default): + /// + /// 1. When looking for the start of an overlapping match (using a reverse + /// DFA), doing it correctly requires starting the reverse search using the + /// starting state of the pattern that matched in the forward direction. + /// Indeed, when building a [`Regex`](crate::hybrid::regex::Regex), it + /// will automatically enable this option when building the reverse DFA + /// internally. + /// 2. When you want to use a DFA with multiple patterns to both search + /// for matches of any pattern or to search for anchored matches of one + /// particular pattern while using the same DFA. (Otherwise, you would need + /// to compile a new DFA for each pattern.) + /// + /// By default this is disabled. + /// + /// # Example + /// + /// This example shows how to use this option to permit the same lazy DFA + /// to run both general searches for any pattern and anchored searches for + /// a specific pattern. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// Anchored, HalfMatch, Input, PatternID, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().starts_for_each_pattern(true)) + /// .build_many(&[r"[a-z0-9]{6}", r"[a-z][a-z0-9]{5}"])?; + /// let mut cache = dfa.create_cache(); + /// let haystack = "bar foo123"; + /// + /// // Here's a normal unanchored search that looks for any pattern. + /// let expected = HalfMatch::must(0, 10); + /// let input = Input::new(haystack); + /// assert_eq!(Some(expected), dfa.try_search_fwd(&mut cache, &input)?); + /// // We can also do a normal anchored search for any pattern. Since it's + /// // an anchored search, we position the start of the search where we + /// // know the match will begin. + /// let expected = HalfMatch::must(0, 10); + /// let input = Input::new(haystack).range(4..); + /// assert_eq!(Some(expected), dfa.try_search_fwd(&mut cache, &input)?); + /// // Since we compiled anchored start states for each pattern, we can + /// // also look for matches of other patterns explicitly, even if a + /// // different pattern would have normally matched. + /// let expected = HalfMatch::must(1, 10); + /// let input = Input::new(haystack) + /// .range(4..) + /// .anchored(Anchored::Pattern(PatternID::must(1))); + /// assert_eq!(Some(expected), dfa.try_search_fwd(&mut cache, &input)?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn starts_for_each_pattern(mut self, yes: bool) -> Config { + self.starts_for_each_pattern = Some(yes); + self + } + + /// Whether to attempt to shrink the size of the lazy DFA's alphabet or + /// not. + /// + /// This option is enabled by default and should never be disabled unless + /// one is debugging the lazy DFA. + /// + /// When enabled, the lazy DFA will use a map from all possible bytes + /// to their corresponding equivalence class. Each equivalence class + /// represents a set of bytes that does not discriminate between a match + /// and a non-match in the DFA. For example, the pattern `[ab]+` has at + /// least two equivalence classes: a set containing `a` and `b` and a set + /// containing every byte except for `a` and `b`. `a` and `b` are in the + /// same equivalence classes because they never discriminate between a + /// match and a non-match. + /// + /// The advantage of this map is that the size of the transition table + /// can be reduced drastically from `#states * 256 * sizeof(LazyStateID)` + /// to `#states * k * sizeof(LazyStateID)` where `k` is the number of + /// equivalence classes (rounded up to the nearest power of 2). As a + /// result, total space usage can decrease substantially. Moreover, since a + /// smaller alphabet is used, DFA compilation during search becomes faster + /// as well since it will potentially be able to reuse a single transition + /// for multiple bytes. + /// + /// **WARNING:** This is only useful for debugging lazy DFAs. Disabling + /// this does not yield any speed advantages. Namely, even when this is + /// disabled, a byte class map is still used while searching. The only + /// difference is that every byte will be forced into its own distinct + /// equivalence class. This is useful for debugging the actual generated + /// transitions because it lets one see the transitions defined on actual + /// bytes instead of the equivalence classes. + pub fn byte_classes(mut self, yes: bool) -> Config { + self.byte_classes = Some(yes); + self + } + + /// Heuristically enable Unicode word boundaries. + /// + /// When set, this will attempt to implement Unicode word boundaries as if + /// they were ASCII word boundaries. This only works when the search input + /// is ASCII only. If a non-ASCII byte is observed while searching, then a + /// [`MatchError::quit`] error is returned. + /// + /// A possible alternative to enabling this option is to simply use an + /// ASCII word boundary, e.g., via `(?-u:\b)`. The main reason to use this + /// option is if you absolutely need Unicode support. This option lets one + /// use a fast search implementation (a DFA) for some potentially very + /// common cases, while providing the option to fall back to some other + /// regex engine to handle the general case when an error is returned. + /// + /// If the pattern provided has no Unicode word boundary in it, then this + /// option has no effect. (That is, quitting on a non-ASCII byte only + /// occurs when this option is enabled _and_ a Unicode word boundary is + /// present in the pattern.) + /// + /// This is almost equivalent to setting all non-ASCII bytes to be quit + /// bytes. The only difference is that this will cause non-ASCII bytes to + /// be quit bytes _only_ when a Unicode word boundary is present in the + /// pattern. + /// + /// When enabling this option, callers _must_ be prepared to + /// handle a [`MatchError`] error during search. When using a + /// [`Regex`](crate::hybrid::regex::Regex), this corresponds to using the + /// `try_` suite of methods. Alternatively, if callers can guarantee that + /// their input is ASCII only, then a [`MatchError::quit`] error will never + /// be returned while searching. + /// + /// This is disabled by default. + /// + /// # Example + /// + /// This example shows how to heuristically enable Unicode word boundaries + /// in a pattern. It also shows what happens when a search comes across a + /// non-ASCII byte. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// HalfMatch, Input, MatchError, + /// }; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().unicode_word_boundary(true)) + /// .build(r"\b[0-9]+\b")?; + /// let mut cache = dfa.create_cache(); + /// + /// // The match occurs before the search ever observes the snowman + /// // character, so no error occurs. + /// let haystack = "foo 123 ☃"; + /// let expected = Some(HalfMatch::must(0, 7)); + /// let got = dfa.try_search_fwd(&mut cache, &Input::new(haystack))?; + /// assert_eq!(expected, got); + /// + /// // Notice that this search fails, even though the snowman character + /// // occurs after the ending match offset. This is because search + /// // routines read one byte past the end of the search to account for + /// // look-around, and indeed, this is required here to determine whether + /// // the trailing \b matches. + /// let haystack = "foo 123 ☃"; + /// let expected = MatchError::quit(0xE2, 8); + /// let got = dfa.try_search_fwd(&mut cache, &Input::new(haystack)); + /// assert_eq!(Err(expected), got); + /// + /// // Another example is executing a search where the span of the haystack + /// // we specify is all ASCII, but there is non-ASCII just before it. This + /// // correctly also reports an error. + /// let input = Input::new("β123").range(2..); + /// let expected = MatchError::quit(0xB2, 1); + /// let got = dfa.try_search_fwd(&mut cache, &input); + /// assert_eq!(Err(expected), got); + /// + /// // And similarly for the trailing word boundary. + /// let input = Input::new("123β").range(..3); + /// let expected = MatchError::quit(0xCE, 3); + /// let got = dfa.try_search_fwd(&mut cache, &input); + /// assert_eq!(Err(expected), got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn unicode_word_boundary(mut self, yes: bool) -> Config { + // We have a separate option for this instead of just setting the + // appropriate quit bytes here because we don't want to set quit bytes + // for every regex. We only want to set them when the regex contains a + // Unicode word boundary. + self.unicode_word_boundary = Some(yes); + self + } + + /// Add a "quit" byte to the lazy DFA. + /// + /// When a quit byte is seen during search time, then search will return a + /// [`MatchError::quit`] error indicating the offset at which the search + /// stopped. + /// + /// A quit byte will always overrule any other aspects of a regex. For + /// example, if the `x` byte is added as a quit byte and the regex `\w` is + /// used, then observing `x` will cause the search to quit immediately + /// despite the fact that `x` is in the `\w` class. + /// + /// This mechanism is primarily useful for heuristically enabling certain + /// features like Unicode word boundaries in a DFA. Namely, if the input + /// to search is ASCII, then a Unicode word boundary can be implemented + /// via an ASCII word boundary with no change in semantics. Thus, a DFA + /// can attempt to match a Unicode word boundary but give up as soon as it + /// observes a non-ASCII byte. Indeed, if callers set all non-ASCII bytes + /// to be quit bytes, then Unicode word boundaries will be permitted when + /// building lazy DFAs. Of course, callers should enable + /// [`Config::unicode_word_boundary`] if they want this behavior instead. + /// (The advantage being that non-ASCII quit bytes will only be added if a + /// Unicode word boundary is in the pattern.) + /// + /// When enabling this option, callers _must_ be prepared to + /// handle a [`MatchError`] error during search. When using a + /// [`Regex`](crate::hybrid::regex::Regex), this corresponds to using the + /// `try_` suite of methods. + /// + /// By default, there are no quit bytes set. + /// + /// # Panics + /// + /// This panics if heuristic Unicode word boundaries are enabled and any + /// non-ASCII byte is removed from the set of quit bytes. Namely, enabling + /// Unicode word boundaries requires setting every non-ASCII byte to a quit + /// byte. So if the caller attempts to undo any of that, then this will + /// panic. + /// + /// # Example + /// + /// This example shows how to cause a search to terminate if it sees a + /// `\n` byte. This could be useful if, for example, you wanted to prevent + /// a user supplied pattern from matching across a line boundary. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, MatchError, Input}; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().quit(b'\n', true)) + /// .build(r"foo\p{any}+bar")?; + /// let mut cache = dfa.create_cache(); + /// + /// let haystack = "foo\nbar"; + /// // Normally this would produce a match, since \p{any} contains '\n'. + /// // But since we instructed the automaton to enter a quit state if a + /// // '\n' is observed, this produces a match error instead. + /// let expected = MatchError::quit(b'\n', 3); + /// let got = dfa.try_search_fwd( + /// &mut cache, + /// &Input::new(haystack), + /// ).unwrap_err(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn quit(mut self, byte: u8, yes: bool) -> Config { + if self.get_unicode_word_boundary() && !byte.is_ascii() && !yes { + panic!( + "cannot set non-ASCII byte to be non-quit when \ + Unicode word boundaries are enabled" + ); + } + if self.quitset.is_none() { + self.quitset = Some(ByteSet::empty()); + } + if yes { + self.quitset.as_mut().unwrap().add(byte); + } else { + self.quitset.as_mut().unwrap().remove(byte); + } + self + } + + /// Enable specializing start states in the lazy DFA. + /// + /// When start states are specialized, an implementor of a search routine + /// using a lazy DFA can tell when the search has entered a starting state. + /// When start states aren't specialized, then it is impossible to know + /// whether the search has entered a start state. + /// + /// Ideally, this option wouldn't need to exist and we could always + /// specialize start states. The problem is that start states can be quite + /// active. This in turn means that an efficient search routine is likely + /// to ping-pong between a heavily optimized hot loop that handles most + /// states and to a less optimized specialized handling of start states. + /// This causes branches to get heavily mispredicted and overall can + /// materially decrease throughput. Therefore, specializing start states + /// should only be enabled when it is needed. + /// + /// Knowing whether a search is in a start state is typically useful when a + /// prefilter is active for the search. A prefilter is typically only run + /// when in a start state and a prefilter can greatly accelerate a search. + /// Therefore, the possible cost of specializing start states is worth it + /// in this case. Otherwise, if you have no prefilter, there is likely no + /// reason to specialize start states. + /// + /// This is disabled by default, but note that it is automatically + /// enabled (or disabled) if [`Config::prefilter`] is set. Namely, unless + /// `specialize_start_states` has already been set, [`Config::prefilter`] + /// will automatically enable or disable it based on whether a prefilter + /// is present or not, respectively. This is done because a prefilter's + /// effectiveness is rooted in being executed whenever the DFA is in a + /// start state, and that's only possible to do when they are specialized. + /// + /// Note that it is plausibly reasonable to _disable_ this option + /// explicitly while _enabling_ a prefilter. In that case, a prefilter + /// will still be run at the beginning of a search, but never again. This + /// in theory could strike a good balance if you're in a situation where a + /// prefilter is likely to produce many false positive candidates. + /// + /// # Example + /// + /// This example shows how to enable start state specialization and then + /// shows how to check whether a state is a start state or not. + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, MatchError, Input}; + /// + /// let dfa = DFA::builder() + /// .configure(DFA::config().specialize_start_states(true)) + /// .build(r"[a-z]+")?; + /// let mut cache = dfa.create_cache(); + /// + /// let haystack = "123 foobar 4567".as_bytes(); + /// let sid = dfa.start_state_forward(&mut cache, &Input::new(haystack))?; + /// // The ID returned by 'start_state_forward' will always be tagged as + /// // a start state when start state specialization is enabled. + /// assert!(sid.is_tagged()); + /// assert!(sid.is_start()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Compare the above with the default lazy DFA configuration where + /// start states are _not_ specialized. In this case, the start state + /// is not tagged and `sid.is_start()` returns false. + /// + /// ``` + /// use regex_automata::{hybrid::dfa::DFA, MatchError, Input}; + /// + /// let dfa = DFA::new(r"[a-z]+")?; + /// let mut cache = dfa.create_cache(); + /// + /// let haystack = "123 foobar 4567".as_bytes(); + /// let sid = dfa.start_state_forward(&mut cache, &Input::new(haystack))?; + /// // Start states are not tagged in the default configuration! + /// assert!(!sid.is_tagged()); + /// assert!(!sid.is_start()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn specialize_start_states(mut self, yes: bool) -> Config { + self.specialize_start_states = Some(yes); + self + } + + /// Sets the maximum amount of heap memory, in bytes, to allocate to the + /// cache for use during a lazy DFA search. If the lazy DFA would otherwise + /// use more heap memory, then, depending on other configuration knobs, + /// either stop the search and return an error or clear the cache and + /// continue the search. + /// + /// The default cache capacity is some "reasonable" number that will + /// accommodate most regular expressions. You may find that if you need + /// to build a large DFA then it may be necessary to increase the cache + /// capacity. + /// + /// Note that while building a lazy DFA will do a "minimum" check to ensure + /// the capacity is big enough, this is more or less about correctness. + /// If the cache is bigger than the minimum but still "too small," then the + /// lazy DFA could wind up spending a lot of time clearing the cache and + /// recomputing transitions, thus negating the performance benefits of a + /// lazy DFA. Thus, setting the cache capacity is mostly an experimental + /// endeavor. For most common patterns, however, the default should be + /// sufficient. + /// + /// For more details on how the lazy DFA's cache is used, see the + /// documentation for [`Cache`]. + /// + /// # Example + /// + /// This example shows what happens if the configured cache capacity is + /// too small. In such cases, one can override the cache capacity to make + /// it bigger. Alternatively, one might want to use less memory by setting + /// a smaller cache capacity. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// let pattern = r"\p{L}{1000}"; + /// + /// // The default cache capacity is likely too small to deal with regexes + /// // that are very large. Large repetitions of large Unicode character + /// // classes are a common way to make very large regexes. + /// let _ = DFA::new(pattern).unwrap_err(); + /// // Bump up the capacity to something bigger. + /// let dfa = DFA::builder() + /// .configure(DFA::config().cache_capacity(100 * (1<<20))) // 100 MB + /// .build(pattern)?; + /// let mut cache = dfa.create_cache(); + /// + /// let haystack = "ͰͲͶͿΆΈΉΊΌΎΏΑΒΓΔΕΖΗΘΙ".repeat(50); + /// let expected = Some(HalfMatch::must(0, 2000)); + /// let got = dfa.try_search_fwd(&mut cache, &Input::new(&haystack))?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn cache_capacity(mut self, bytes: usize) -> Config { + self.cache_capacity = Some(bytes); + self + } + + /// Configures construction of a lazy DFA to use the minimum cache capacity + /// if the configured capacity is otherwise too small for the provided NFA. + /// + /// This is useful if you never want lazy DFA construction to fail because + /// of a capacity that is too small. + /// + /// In general, this option is typically not a good idea. In particular, + /// while a minimum cache capacity does permit the lazy DFA to function + /// where it otherwise couldn't, it's plausible that it may not function + /// well if it's constantly running out of room. In that case, the speed + /// advantages of the lazy DFA may be negated. On the other hand, the + /// "minimum" cache capacity computed may not be completely accurate and + /// could actually be bigger than what is really necessary. Therefore, it + /// is plausible that using the minimum cache capacity could still result + /// in very good performance. + /// + /// This is disabled by default. + /// + /// # Example + /// + /// This example shows what happens if the configured cache capacity is + /// too small. In such cases, one could override the capacity explicitly. + /// An alternative, demonstrated here, let's us force construction to use + /// the minimum cache capacity if the configured capacity is otherwise + /// too small. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, HalfMatch, Input}; + /// + /// let pattern = r"\p{L}{1000}"; + /// + /// // The default cache capacity is likely too small to deal with regexes + /// // that are very large. Large repetitions of large Unicode character + /// // classes are a common way to make very large regexes. + /// let _ = DFA::new(pattern).unwrap_err(); + /// // Configure construction such it automatically selects the minimum + /// // cache capacity if it would otherwise be too small. + /// let dfa = DFA::builder() + /// .configure(DFA::config().skip_cache_capacity_check(true)) + /// .build(pattern)?; + /// let mut cache = dfa.create_cache(); + /// + /// let haystack = "ͰͲͶͿΆΈΉΊΌΎΏΑΒΓΔΕΖΗΘΙ".repeat(50); + /// let expected = Some(HalfMatch::must(0, 2000)); + /// let got = dfa.try_search_fwd(&mut cache, &Input::new(&haystack))?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn skip_cache_capacity_check(mut self, yes: bool) -> Config { + self.skip_cache_capacity_check = Some(yes); + self + } + + /// Configure a lazy DFA search to quit after a certain number of cache + /// clearings. + /// + /// When a minimum is set, then a lazy DFA search will *possibly* "give + /// up" after the minimum number of cache clearings has occurred. This is + /// typically useful in scenarios where callers want to detect whether the + /// lazy DFA search is "efficient" or not. If the cache is cleared too many + /// times, this is a good indicator that it is not efficient, and thus, the + /// caller may wish to use some other regex engine. + /// + /// Note that the number of times a cache is cleared is a property of + /// the cache itself. Thus, if a cache is used in a subsequent search + /// with a similarly configured lazy DFA, then it could cause the + /// search to "give up" if the cache needed to be cleared, depending + /// on its internal count and configured minimum. The cache clear + /// count can only be reset to `0` via [`DFA::reset_cache`] (or + /// [`Regex::reset_cache`](crate::hybrid::regex::Regex::reset_cache) if + /// you're using the `Regex` API). + /// + /// By default, no minimum is configured. Thus, a lazy DFA search will + /// never give up due to cache clearings. If you do set this option, you + /// might consider also setting [`Config::minimum_bytes_per_state`] in + /// order for the lazy DFA to take efficiency into account before giving + /// up. + /// + /// # Example + /// + /// This example uses a somewhat pathological configuration to demonstrate + /// the _possible_ behavior of cache clearing and how it might result + /// in a search that returns an error. + /// + /// It is important to note that the precise mechanics of how and when + /// a cache gets cleared is an implementation detail. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::dfa::DFA, Input, MatchError, MatchErrorKind}; + /// + /// // This is a carefully chosen regex. The idea is to pick one + /// // that requires some decent number of states (hence the bounded + /// // repetition). But we specifically choose to create a class with an + /// // ASCII letter and a non-ASCII letter so that we can check that no new + /// // states are created once the cache is full. Namely, if we fill up the + /// // cache on a haystack of 'a's, then in order to match one 'β', a new + /// // state will need to be created since a 'β' is encoded with multiple + /// // bytes. Since there's no room for this state, the search should quit + /// // at the very first position. + /// let pattern = r"[aβ]{100}"; + /// let dfa = DFA::builder() + /// .configure( + /// // Configure it so that we have the minimum cache capacity + /// // possible. And that if any clearings occur, the search quits. + /// DFA::config() + /// .skip_cache_capacity_check(true) + /// .cache_capacity(0) + /// .minimum_cache_clear_count(Some(0)), + /// ) + /// .build(pattern)?; + /// let mut cache = dfa.create_cache(); + /// + /// // Our search will give up before reaching the end! + /// let haystack = "a".repeat(101).into_bytes(); + /// let result = dfa.try_search_fwd(&mut cache, &Input::new(&haystack)); + /// assert!(matches!( + /// *result.unwrap_err().kind(), + /// MatchErrorKind::GaveUp { .. }, + /// )); + /// + /// // Now that we know the cache is full, if we search a haystack that we + /// // know will require creating at least one new state, it should not + /// // be able to make much progress. + /// let haystack = "β".repeat(101).into_bytes(); + /// let result = dfa.try_search_fwd(&mut cache, &Input::new(&haystack)); + /// assert!(matches!( + /// *result.unwrap_err().kind(), + /// MatchErrorKind::GaveUp { .. }, + /// )); + /// + /// // If we reset the cache, then we should be able to create more states + /// // and make more progress with searching for betas. + /// cache.reset(&dfa); + /// let haystack = "β".repeat(101).into_bytes(); + /// let result = dfa.try_search_fwd(&mut cache, &Input::new(&haystack)); + /// assert!(matches!( + /// *result.unwrap_err().kind(), + /// MatchErrorKind::GaveUp { .. }, + /// )); + /// + /// // ... switching back to ASCII still makes progress since it just needs + /// // to set transitions on existing states! + /// let haystack = "a".repeat(101).into_bytes(); + /// let result = dfa.try_search_fwd(&mut cache, &Input::new(&haystack)); + /// assert!(matches!( + /// *result.unwrap_err().kind(), + /// MatchErrorKind::GaveUp { .. }, + /// )); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn minimum_cache_clear_count(mut self, min: Option<usize>) -> Config { + self.minimum_cache_clear_count = Some(min); + self + } + + /// Configure a lazy DFA search to quit only when its efficiency drops + /// below the given minimum. + /// + /// The efficiency of the cache is determined by the number of DFA states + /// compiled per byte of haystack searched. For example, if the efficiency + /// is 2, then it means the lazy DFA is creating a new DFA state after + /// searching approximately 2 bytes in a haystack. Generally speaking, 2 + /// is quite bad and it's likely that even a slower regex engine like the + /// [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM) would be faster. + /// + /// This has no effect if [`Config::minimum_cache_clear_count`] is not set. + /// Namely, this option only kicks in when the cache has been cleared more + /// than the minimum number. If no minimum is set, then the cache is simply + /// cleared whenever it fills up and it is impossible for the lazy DFA to + /// quit due to ineffective use of the cache. + /// + /// In general, if one is setting [`Config::minimum_cache_clear_count`], + /// then one should probably also set this knob as well. The reason is + /// that the absolute number of times the cache is cleared is generally + /// not a great predictor of efficiency. For example, if a new DFA state + /// is created for every 1,000 bytes searched, then it wouldn't be hard + /// for the cache to get cleared more than `N` times and then cause the + /// lazy DFA to quit. But a new DFA state every 1,000 bytes is likely quite + /// good from a performance perspective, and it's likely that the lazy + /// DFA should continue searching, even if it requires clearing the cache + /// occasionally. + /// + /// Finally, note that if you're implementing your own lazy DFA search + /// routine and also want this efficiency check to work correctly, then + /// you'll need to use the following routines to record search progress: + /// + /// * Call [`Cache::search_start`] at the beginning of every search. + /// * Call [`Cache::search_update`] whenever [`DFA::next_state`] is + /// called. + /// * Call [`Cache::search_finish`] before completing a search. (It is + /// not strictly necessary to call this when an error is returned, as + /// `Cache::search_start` will automatically finish the previous search + /// for you. But calling it where possible before returning helps improve + /// the accuracy of how many bytes have actually been searched.) + pub fn minimum_bytes_per_state(mut self, min: Option<usize>) -> Config { + self.minimum_bytes_per_state = Some(min); + self + } + + /// Returns the match semantics set in this configuration. + pub fn get_match_kind(&self) -> MatchKind { + self.match_kind.unwrap_or(MatchKind::LeftmostFirst) + } + + /// Returns the prefilter set in this configuration, if one at all. + pub fn get_prefilter(&self) -> Option<&Prefilter> { + self.pre.as_ref().unwrap_or(&None).as_ref() + } + + /// Returns whether this configuration has enabled anchored starting states + /// for every pattern in the DFA. + pub fn get_starts_for_each_pattern(&self) -> bool { + self.starts_for_each_pattern.unwrap_or(false) + } + + /// Returns whether this configuration has enabled byte classes or not. + /// This is typically a debugging oriented option, as disabling it confers + /// no speed benefit. + pub fn get_byte_classes(&self) -> bool { + self.byte_classes.unwrap_or(true) + } + + /// Returns whether this configuration has enabled heuristic Unicode word + /// boundary support. When enabled, it is possible for a search to return + /// an error. + pub fn get_unicode_word_boundary(&self) -> bool { + self.unicode_word_boundary.unwrap_or(false) + } + + /// Returns whether this configuration will instruct the lazy DFA to enter + /// a quit state whenever the given byte is seen during a search. When at + /// least one byte has this enabled, it is possible for a search to return + /// an error. + pub fn get_quit(&self, byte: u8) -> bool { + self.quitset.map_or(false, |q| q.contains(byte)) + } + + /// Returns whether this configuration will instruct the lazy DFA to + /// "specialize" start states. When enabled, the lazy DFA will tag start + /// states so that search routines using the lazy DFA can detect when + /// it's in a start state and do some kind of optimization (like run a + /// prefilter). + pub fn get_specialize_start_states(&self) -> bool { + self.specialize_start_states.unwrap_or(false) + } + + /// Returns the cache capacity set on this configuration. + pub fn get_cache_capacity(&self) -> usize { + self.cache_capacity.unwrap_or(2 * (1 << 20)) + } + + /// Returns whether the cache capacity check should be skipped. + pub fn get_skip_cache_capacity_check(&self) -> bool { + self.skip_cache_capacity_check.unwrap_or(false) + } + + /// Returns, if set, the minimum number of times the cache must be cleared + /// before a lazy DFA search can give up. When no minimum is set, then a + /// search will never quit and will always clear the cache whenever it + /// fills up. + pub fn get_minimum_cache_clear_count(&self) -> Option<usize> { + self.minimum_cache_clear_count.unwrap_or(None) + } + + /// Returns, if set, the minimum number of bytes per state that need to be + /// processed in order for the lazy DFA to keep going. If the minimum falls + /// below this number (and the cache has been cleared a minimum number of + /// times), then the lazy DFA will return a "gave up" error. + pub fn get_minimum_bytes_per_state(&self) -> Option<usize> { + self.minimum_bytes_per_state.unwrap_or(None) + } + + /// Returns the minimum lazy DFA cache capacity required for the given NFA. + /// + /// The cache capacity required for a particular NFA may change without + /// notice. Callers should not rely on it being stable. + /// + /// This is useful for informational purposes, but can also be useful for + /// other reasons. For example, if one wants to check the minimum cache + /// capacity themselves or if one wants to set the capacity based on the + /// minimum. + /// + /// This may return an error if this configuration does not support all of + /// the instructions used in the given NFA. For example, if the NFA has a + /// Unicode word boundary but this configuration does not enable heuristic + /// support for Unicode word boundaries. + pub fn get_minimum_cache_capacity( + &self, + nfa: &thompson::NFA, + ) -> Result<usize, BuildError> { + let quitset = self.quit_set_from_nfa(nfa)?; + let classes = self.byte_classes_from_nfa(nfa, &quitset); + let starts = self.get_starts_for_each_pattern(); + Ok(minimum_cache_capacity(nfa, &classes, starts)) + } + + /// Returns the byte class map used during search from the given NFA. + /// + /// If byte classes are disabled on this configuration, then a map is + /// returned that puts each byte in its own equivalent class. + fn byte_classes_from_nfa( + &self, + nfa: &thompson::NFA, + quit: &ByteSet, + ) -> ByteClasses { + if !self.get_byte_classes() { + // The lazy DFA will always use the equivalence class map, but + // enabling this option is useful for debugging. Namely, this will + // cause all transitions to be defined over their actual bytes + // instead of an opaque equivalence class identifier. The former is + // much easier to grok as a human. + ByteClasses::singletons() + } else { + let mut set = nfa.byte_class_set().clone(); + // It is important to distinguish any "quit" bytes from all other + // bytes. Otherwise, a non-quit byte may end up in the same class + // as a quit byte, and thus cause the DFA stop when it shouldn't. + // + // Test case: + // + // regex-cli find match hybrid --unicode-word-boundary \ + // -p '^#' -p '\b10\.55\.182\.100\b' -y @conn.json.1000x.log + if !quit.is_empty() { + set.add_set(&quit); + } + set.byte_classes() + } + } + + /// Return the quit set for this configuration and the given NFA. + /// + /// This may return an error if the NFA is incompatible with this + /// configuration's quit set. For example, if the NFA has a Unicode word + /// boundary and the quit set doesn't include non-ASCII bytes. + fn quit_set_from_nfa( + &self, + nfa: &thompson::NFA, + ) -> Result<ByteSet, BuildError> { + let mut quit = self.quitset.unwrap_or(ByteSet::empty()); + if nfa.look_set_any().contains_word_unicode() { + if self.get_unicode_word_boundary() { + for b in 0x80..=0xFF { + quit.add(b); + } + } else { + // If heuristic support for Unicode word boundaries wasn't + // enabled, then we can still check if our quit set is correct. + // If the caller set their quit bytes in a way that causes the + // DFA to quit on at least all non-ASCII bytes, then that's all + // we need for heuristic support to work. + if !quit.contains_range(0x80, 0xFF) { + return Err( + BuildError::unsupported_dfa_word_boundary_unicode(), + ); + } + } + } + Ok(quit) + } + + /// Overwrite the default configuration such that the options in `o` are + /// always used. If an option in `o` is not set, then the corresponding + /// option in `self` is used. If it's not set in `self` either, then it + /// remains not set. + fn overwrite(&self, o: Config) -> Config { + Config { + match_kind: o.match_kind.or(self.match_kind), + pre: o.pre.or_else(|| self.pre.clone()), + starts_for_each_pattern: o + .starts_for_each_pattern + .or(self.starts_for_each_pattern), + byte_classes: o.byte_classes.or(self.byte_classes), + unicode_word_boundary: o + .unicode_word_boundary + .or(self.unicode_word_boundary), + quitset: o.quitset.or(self.quitset), + specialize_start_states: o + .specialize_start_states + .or(self.specialize_start_states), + cache_capacity: o.cache_capacity.or(self.cache_capacity), + skip_cache_capacity_check: o + .skip_cache_capacity_check + .or(self.skip_cache_capacity_check), + minimum_cache_clear_count: o + .minimum_cache_clear_count + .or(self.minimum_cache_clear_count), + minimum_bytes_per_state: o + .minimum_bytes_per_state + .or(self.minimum_bytes_per_state), + } + } +} + +/// A builder for constructing a lazy deterministic finite automaton from +/// regular expressions. +/// +/// As a convenience, [`DFA::builder`] is an alias for [`Builder::new`]. The +/// advantage of the former is that it often lets you avoid importing the +/// `Builder` type directly. +/// +/// This builder provides two main things: +/// +/// 1. It provides a few different `build` routines for actually constructing +/// a DFA from different kinds of inputs. The most convenient is +/// [`Builder::build`], which builds a DFA directly from a pattern string. The +/// most flexible is [`Builder::build_from_nfa`], which builds a DFA straight +/// from an NFA. +/// 2. The builder permits configuring a number of things. +/// [`Builder::configure`] is used with [`Config`] to configure aspects of +/// the DFA and the construction process itself. [`Builder::syntax`] and +/// [`Builder::thompson`] permit configuring the regex parser and Thompson NFA +/// construction, respectively. The syntax and thompson configurations only +/// apply when building from a pattern string. +/// +/// This builder always constructs a *single* lazy DFA. As such, this builder +/// can only be used to construct regexes that either detect the presence +/// of a match or find the end location of a match. A single DFA cannot +/// produce both the start and end of a match. For that information, use a +/// [`Regex`](crate::hybrid::regex::Regex), which can be similarly configured +/// using [`regex::Builder`](crate::hybrid::regex::Builder). The main reason +/// to use a DFA directly is if the end location of a match is enough for your +/// use case. Namely, a `Regex` will construct two lazy DFAs instead of one, +/// since a second reverse DFA is needed to find the start of a match. +/// +/// # Example +/// +/// This example shows how to build a lazy DFA that uses a tiny cache capacity +/// and completely disables Unicode. That is: +/// +/// * Things such as `\w`, `.` and `\b` are no longer Unicode-aware. `\w` +/// and `\b` are ASCII-only while `.` matches any byte except for `\n` +/// (instead of any UTF-8 encoding of a Unicode scalar value except for +/// `\n`). Things that are Unicode only, such as `\pL`, are not allowed. +/// * The pattern itself is permitted to match invalid UTF-8. For example, +/// things like `[^a]` that match any byte except for `a` are permitted. +/// +/// ``` +/// use regex_automata::{ +/// hybrid::dfa::DFA, +/// nfa::thompson, +/// util::syntax, +/// HalfMatch, Input, +/// }; +/// +/// let dfa = DFA::builder() +/// .configure(DFA::config().cache_capacity(5_000)) +/// .thompson(thompson::Config::new().utf8(false)) +/// .syntax(syntax::Config::new().unicode(false).utf8(false)) +/// .build(r"foo[^b]ar.*")?; +/// let mut cache = dfa.create_cache(); +/// +/// let haystack = b"\xFEfoo\xFFar\xE2\x98\xFF\n"; +/// let expected = Some(HalfMatch::must(0, 10)); +/// let got = dfa.try_search_fwd(&mut cache, &Input::new(haystack))?; +/// assert_eq!(expected, got); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Builder { + config: Config, + #[cfg(feature = "syntax")] + thompson: thompson::Compiler, +} + +impl Builder { + /// Create a new lazy DFA builder with the default configuration. + pub fn new() -> Builder { + Builder { + config: Config::default(), + #[cfg(feature = "syntax")] + thompson: thompson::Compiler::new(), + } + } + + /// Build a lazy DFA from the given pattern. + /// + /// If there was a problem parsing or compiling the pattern, then an error + /// is returned. + #[cfg(feature = "syntax")] + pub fn build(&self, pattern: &str) -> Result<DFA, BuildError> { + self.build_many(&[pattern]) + } + + /// Build a lazy DFA from the given patterns. + /// + /// When matches are returned, the pattern ID corresponds to the index of + /// the pattern in the slice given. + #[cfg(feature = "syntax")] + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<DFA, BuildError> { + let nfa = self + .thompson + .clone() + // We can always forcefully disable captures because DFAs do not + // support them. + .configure( + thompson::Config::new() + .which_captures(thompson::WhichCaptures::None), + ) + .build_many(patterns) + .map_err(BuildError::nfa)?; + self.build_from_nfa(nfa) + } + + /// Build a DFA from the given NFA. + /// + /// Note that this requires owning a `thompson::NFA`. While this may force + /// you to clone the NFA, such a clone is not a deep clone. Namely, NFAs + /// are defined internally to support shared ownership such that cloning is + /// very cheap. + /// + /// # Example + /// + /// This example shows how to build a lazy DFA if you already have an NFA + /// in hand. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// nfa::thompson, + /// HalfMatch, Input, + /// }; + /// + /// let haystack = "foo123bar"; + /// + /// // This shows how to set non-default options for building an NFA. + /// let nfa = thompson::Compiler::new() + /// .configure(thompson::Config::new().shrink(true)) + /// .build(r"[0-9]+")?; + /// let dfa = DFA::builder().build_from_nfa(nfa)?; + /// let mut cache = dfa.create_cache(); + /// let expected = Some(HalfMatch::must(0, 6)); + /// let got = dfa.try_search_fwd(&mut cache, &Input::new(haystack))?; + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_from_nfa( + &self, + nfa: thompson::NFA, + ) -> Result<DFA, BuildError> { + let quitset = self.config.quit_set_from_nfa(&nfa)?; + let classes = self.config.byte_classes_from_nfa(&nfa, &quitset); + // Check that we can fit at least a few states into our cache, + // otherwise it's pretty senseless to use the lazy DFA. This does have + // a possible failure mode though. This assumes the maximum size of a + // state in powerset space (so, the total number of NFA states), which + // may never actually materialize, and could be quite a bit larger + // than the actual biggest state. If this turns out to be a problem, + // we could expose a knob that disables this check. But if so, we have + // to be careful not to panic in other areas of the code (the cache + // clearing and init code) that tend to assume some minimum useful + // cache capacity. + let min_cache = minimum_cache_capacity( + &nfa, + &classes, + self.config.get_starts_for_each_pattern(), + ); + let mut cache_capacity = self.config.get_cache_capacity(); + if cache_capacity < min_cache { + // When the caller has asked us to skip the cache capacity check, + // then we simply force the cache capacity to its minimum amount + // and mush on. + if self.config.get_skip_cache_capacity_check() { + debug!( + "given capacity ({cache_capacity}) is too small, \ + since skip_cache_capacity_check is enabled, \ + setting cache capacity to minimum ({min_cache})", + ); + cache_capacity = min_cache; + } else { + return Err(BuildError::insufficient_cache_capacity( + min_cache, + cache_capacity, + )); + } + } + // We also need to check that we can fit at least some small number + // of states in our state ID space. This is unlikely to trigger in + // >=32-bit systems, but 16-bit systems have a pretty small state ID + // space since a number of bits are used up as sentinels. + if let Err(err) = minimum_lazy_state_id(&classes) { + return Err(BuildError::insufficient_state_id_capacity(err)); + } + let stride2 = classes.stride2(); + let start_map = StartByteMap::new(nfa.look_matcher()); + Ok(DFA { + config: self.config.clone(), + nfa, + stride2, + start_map, + classes, + quitset, + cache_capacity, + }) + } + + /// Apply the given lazy DFA configuration options to this builder. + pub fn configure(&mut self, config: Config) -> &mut Builder { + self.config = self.config.overwrite(config); + self + } + + /// Set the syntax configuration for this builder using + /// [`syntax::Config`](crate::util::syntax::Config). + /// + /// This permits setting things like case insensitivity, Unicode and multi + /// line mode. + /// + /// These settings only apply when constructing a lazy DFA directly from a + /// pattern. + #[cfg(feature = "syntax")] + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Builder { + self.thompson.syntax(config); + self + } + + /// Set the Thompson NFA configuration for this builder using + /// [`nfa::thompson::Config`](crate::nfa::thompson::Config). + /// + /// This permits setting things like whether the DFA should match the regex + /// in reverse or if additional time should be spent shrinking the size of + /// the NFA. + /// + /// These settings only apply when constructing a DFA directly from a + /// pattern. + #[cfg(feature = "syntax")] + pub fn thompson(&mut self, config: thompson::Config) -> &mut Builder { + self.thompson.configure(config); + self + } +} + +/// Represents the current state of an overlapping search. +/// +/// This is used for overlapping searches since they need to know something +/// about the previous search. For example, when multiple patterns match at the +/// same position, this state tracks the last reported pattern so that the next +/// search knows whether to report another matching pattern or continue with +/// the search at the next position. Additionally, it also tracks which state +/// the last search call terminated in. +/// +/// This type provides little introspection capabilities. The only thing a +/// caller can do is construct it and pass it around to permit search routines +/// to use it to track state, and also ask whether a match has been found. +/// +/// Callers should always provide a fresh state constructed via +/// [`OverlappingState::start`] when starting a new search. Reusing state from +/// a previous search may result in incorrect results. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OverlappingState { + /// The match reported by the most recent overlapping search to use this + /// state. + /// + /// If a search does not find any matches, then it is expected to clear + /// this value. + pub(crate) mat: Option<HalfMatch>, + /// The state ID of the state at which the search was in when the call + /// terminated. When this is a match state, `last_match` must be set to a + /// non-None value. + /// + /// A `None` value indicates the start state of the corresponding + /// automaton. We cannot use the actual ID, since any one automaton may + /// have many start states, and which one is in use depends on several + /// search-time factors. + pub(crate) id: Option<LazyStateID>, + /// The position of the search. + /// + /// When `id` is None (i.e., we are starting a search), this is set to + /// the beginning of the search as given by the caller regardless of its + /// current value. Subsequent calls to an overlapping search pick up at + /// this offset. + pub(crate) at: usize, + /// The index into the matching patterns of the next match to report if the + /// current state is a match state. Note that this may be 1 greater than + /// the total number of matches to report for the current match state. (In + /// which case, no more matches should be reported at the current position + /// and the search should advance to the next position.) + pub(crate) next_match_index: Option<usize>, + /// This is set to true when a reverse overlapping search has entered its + /// EOI transitions. + /// + /// This isn't used in a forward search because it knows to stop once the + /// position exceeds the end of the search range. In a reverse search, + /// since we use unsigned offsets, we don't "know" once we've gone past + /// `0`. So the only way to detect it is with this extra flag. The reverse + /// overlapping search knows to terminate specifically after it has + /// reported all matches after following the EOI transition. + pub(crate) rev_eoi: bool, +} + +impl OverlappingState { + /// Create a new overlapping state that begins at the start state of any + /// automaton. + pub fn start() -> OverlappingState { + OverlappingState { + mat: None, + id: None, + at: 0, + next_match_index: None, + rev_eoi: false, + } + } + + /// Return the match result of the most recent search to execute with this + /// state. + /// + /// A searches will clear this result automatically, such that if no + /// match is found, this will correctly report `None`. + pub fn get_match(&self) -> Option<HalfMatch> { + self.mat + } +} + +/// Runs the given overlapping `search` function (forwards or backwards) until +/// a match is found whose offset does not split a codepoint. +/// +/// This is *not* always correct to call. It should only be called when the +/// underlying NFA has UTF-8 mode enabled *and* it can produce zero-width +/// matches. Calling this when both of those things aren't true might result +/// in legitimate matches getting skipped. +#[cold] +#[inline(never)] +fn skip_empty_utf8_splits_overlapping<F>( + input: &Input<'_>, + state: &mut OverlappingState, + mut search: F, +) -> Result<(), MatchError> +where + F: FnMut(&Input<'_>, &mut OverlappingState) -> Result<(), MatchError>, +{ + // Note that this routine works for forwards and reverse searches + // even though there's no code here to handle those cases. That's + // because overlapping searches drive themselves to completion via + // `OverlappingState`. So all we have to do is push it until no matches are + // found. + + let mut hm = match state.get_match() { + None => return Ok(()), + Some(hm) => hm, + }; + if input.get_anchored().is_anchored() { + if !input.is_char_boundary(hm.offset()) { + state.mat = None; + } + return Ok(()); + } + while !input.is_char_boundary(hm.offset()) { + search(input, state)?; + hm = match state.get_match() { + None => return Ok(()), + Some(hm) => hm, + }; + } + Ok(()) +} + +/// Based on the minimum number of states required for a useful lazy DFA cache, +/// this returns the minimum lazy state ID that must be representable. +/// +/// It's not likely for this to have any impact 32-bit systems (or higher), but +/// on 16-bit systems, the lazy state ID space is quite constrained and thus +/// may be insufficient if our MIN_STATES value is (for some reason) too high. +fn minimum_lazy_state_id( + classes: &ByteClasses, +) -> Result<LazyStateID, LazyStateIDError> { + let stride = 1 << classes.stride2(); + let min_state_index = MIN_STATES.checked_sub(1).unwrap(); + LazyStateID::new(min_state_index * stride) +} + +/// Based on the minimum number of states required for a useful lazy DFA cache, +/// this returns a heuristic minimum number of bytes of heap space required. +/// +/// This is a "heuristic" because the minimum it returns is likely bigger than +/// the true minimum. Namely, it assumes that each powerset NFA/DFA state uses +/// the maximum number of NFA states (all of them). This is likely bigger +/// than what is required in practice. Computing the true minimum effectively +/// requires determinization, which is probably too much work to do for a +/// simple check like this. +/// +/// One of the issues with this approach IMO is that it requires that this +/// be in sync with the calculation above for computing how much heap memory +/// the DFA cache uses. If we get it wrong, it's possible for example for the +/// minimum to be smaller than the computed heap memory, and thus, it may be +/// the case that we can't add the required minimum number of states. That in +/// turn will make lazy DFA panic because we assume that we can add at least a +/// minimum number of states. +/// +/// Another approach would be to always allow the minimum number of states to +/// be added to the lazy DFA cache, even if it exceeds the configured cache +/// limit. This does mean that the limit isn't really a limit in all cases, +/// which is unfortunate. But it does at least guarantee that the lazy DFA can +/// always make progress, even if it is slow. (This approach is very similar to +/// enabling the 'skip_cache_capacity_check' config knob, except it wouldn't +/// rely on cache size calculation. Instead, it would just always permit a +/// minimum number of states to be added.) +fn minimum_cache_capacity( + nfa: &thompson::NFA, + classes: &ByteClasses, + starts_for_each_pattern: bool, +) -> usize { + const ID_SIZE: usize = size_of::<LazyStateID>(); + const STATE_SIZE: usize = size_of::<State>(); + + let stride = 1 << classes.stride2(); + let states_len = nfa.states().len(); + let sparses = 2 * states_len * NFAStateID::SIZE; + let trans = MIN_STATES * stride * ID_SIZE; + + let mut starts = Start::len() * ID_SIZE; + if starts_for_each_pattern { + starts += (Start::len() * nfa.pattern_len()) * ID_SIZE; + } + + // The min number of states HAS to be at least 4: we have 3 sentinel states + // and then we need space for one more when we save a state after clearing + // the cache. We also need space for one more, otherwise we get stuck in a + // loop where we try to add a 5th state, which gets rejected, which clears + // the cache, which adds back a saved state (4th total state) which then + // tries to add the 5th state again. + assert!(MIN_STATES >= 5, "minimum number of states has to be at least 5"); + // The minimum number of non-sentinel states. We consider this separately + // because sentinel states are much smaller in that they contain no NFA + // states. Given our aggressive calculation here, it's worth being more + // precise with the number of states we need. + let non_sentinel = MIN_STATES.checked_sub(SENTINEL_STATES).unwrap(); + + // Every `State` has 5 bytes for flags, 4 bytes (max) for the number of + // patterns, followed by 32-bit encodings of patterns and then delta + // varint encodings of NFA state IDs. We use the worst case (which isn't + // technically possible) of 5 bytes for each NFA state ID. + // + // HOWEVER, three of the states needed by a lazy DFA are just the sentinel + // unknown, dead and quit states. Those states have a known size and it is + // small. + let dead_state_size = State::dead().memory_usage(); + let max_state_size = 5 + 4 + (nfa.pattern_len() * 4) + (states_len * 5); + let states = (SENTINEL_STATES * (STATE_SIZE + dead_state_size)) + + (non_sentinel * (STATE_SIZE + max_state_size)); + // NOTE: We don't double count heap memory used by State for this map since + // we use reference counting to avoid doubling memory usage. (This tends to + // be where most memory is allocated in the cache.) + let states_to_sid = (MIN_STATES * STATE_SIZE) + (MIN_STATES * ID_SIZE); + let stack = states_len * NFAStateID::SIZE; + let scratch_state_builder = max_state_size; + + trans + + starts + + states + + states_to_sid + + sparses + + stack + + scratch_state_builder +} + +#[cfg(all(test, feature = "syntax"))] +mod tests { + use super::*; + + // Tests that we handle heuristic Unicode word boundary support in reverse + // DFAs in the specific case of contextual searches. + // + // I wrote this test when I discovered a bug in how heuristic word + // boundaries were handled. Namely, that the starting state selection + // didn't consider the DFA's quit byte set when looking at the byte + // immediately before the start of the search (or immediately after the + // end of the search in the case of a reverse search). As a result, it was + // possible for '\bfoo\b' to match 'β123' because the trailing \xB2 byte + // in the 'β' codepoint would be treated as a non-word character. But of + // course, this search should trigger the DFA to quit, since there is a + // non-ASCII byte in consideration. + // + // Thus, I fixed 'start_state_{forward,reverse}' to check the quit byte set + // if it wasn't empty. The forward case is tested in the doc test for the + // Config::unicode_word_boundary API. We test the reverse case here, which + // is sufficiently niche that it doesn't really belong in a doc test. + #[test] + fn heuristic_unicode_reverse() { + let dfa = DFA::builder() + .configure(DFA::config().unicode_word_boundary(true)) + .thompson(thompson::Config::new().reverse(true)) + .build(r"\b[0-9]+\b") + .unwrap(); + let mut cache = dfa.create_cache(); + + let input = Input::new("β123").range(2..); + let expected = MatchError::quit(0xB2, 1); + let got = dfa.try_search_rev(&mut cache, &input); + assert_eq!(Err(expected), got); + + let input = Input::new("123β").range(..3); + let expected = MatchError::quit(0xCE, 3); + let got = dfa.try_search_rev(&mut cache, &input); + assert_eq!(Err(expected), got); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/hybrid/error.rs b/third_party/crates/regex-automata-0.4.18/src/hybrid/error.rs new file mode 100644 index 0000000..93e58dd --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/hybrid/error.rs @@ -0,0 +1,241 @@ +use crate::{hybrid::id::LazyStateIDError, nfa, util::search::Anchored}; + +/// An error that occurs when initial construction of a lazy DFA fails. +/// +/// A build error can occur when insufficient cache capacity is configured or +/// if something about the NFA is unsupported. (For example, if one attempts +/// to build a lazy DFA without heuristic Unicode support but with an NFA that +/// contains a Unicode word boundary.) +/// +/// This error does not provide many introspection capabilities. There are +/// generally only two things you can do with it: +/// +/// * Obtain a human readable message via its `std::fmt::Display` impl. +/// * Access an underlying +/// [`nfa::thompson::BuildError`](crate::nfa::thompson::BuildError) +/// type from its `source` method via the `std::error::Error` trait. This error +/// only occurs when using convenience routines for building a lazy DFA +/// directly from a pattern string. +/// +/// When the `std` feature is enabled, this implements the `std::error::Error` +/// trait. +#[derive(Clone, Debug)] +pub struct BuildError { + kind: BuildErrorKind, +} + +#[derive(Clone, Debug)] +enum BuildErrorKind { + NFA(nfa::thompson::BuildError), + InsufficientCacheCapacity { minimum: usize, given: usize }, + InsufficientStateIDCapacity { err: LazyStateIDError }, + Unsupported(&'static str), +} + +impl BuildError { + pub(crate) fn nfa(err: nfa::thompson::BuildError) -> BuildError { + BuildError { kind: BuildErrorKind::NFA(err) } + } + + pub(crate) fn insufficient_cache_capacity( + minimum: usize, + given: usize, + ) -> BuildError { + BuildError { + kind: BuildErrorKind::InsufficientCacheCapacity { minimum, given }, + } + } + + pub(crate) fn insufficient_state_id_capacity( + err: LazyStateIDError, + ) -> BuildError { + BuildError { + kind: BuildErrorKind::InsufficientStateIDCapacity { err }, + } + } + + pub(crate) fn unsupported_dfa_word_boundary_unicode() -> BuildError { + let msg = "cannot build lazy DFAs for regexes with Unicode word \ + boundaries; switch to ASCII word boundaries, or \ + heuristically enable Unicode word boundaries or use a \ + different regex engine"; + BuildError { kind: BuildErrorKind::Unsupported(msg) } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for BuildError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self.kind { + BuildErrorKind::NFA(ref err) => Some(err), + _ => None, + } + } +} + +impl core::fmt::Display for BuildError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.kind { + BuildErrorKind::NFA(_) => write!(f, "error building NFA"), + BuildErrorKind::InsufficientCacheCapacity { minimum, given } => { + write!( + f, + "given cache capacity ({given}) is smaller than \ + minimum required ({minimum})", + ) + } + BuildErrorKind::InsufficientStateIDCapacity { ref err } => { + err.fmt(f) + } + BuildErrorKind::Unsupported(ref msg) => { + write!(f, "unsupported regex feature for DFAs: {msg}") + } + } + } +} + +/// An error that can occur when computing the start state for a search. +/// +/// Computing a start state can fail for a few reasons, either +/// based on incorrect configuration or even based on whether +/// the look-behind byte triggers a quit state. Typically +/// one does not need to handle this error if you're using +/// [`DFA::start_state_forward`](crate::hybrid::dfa::DFA::start_state_forward) +/// (or its reverse counterpart), as that routine automatically converts +/// `StartError` to a [`MatchError`](crate::MatchError) for you. +/// +/// This error may be returned by the +/// [`DFA::start_state`](crate::hybrid::dfa::DFA::start_state) routine. +/// +/// This error implements the `std::error::Error` trait when the `std` feature +/// is enabled. +/// +/// This error is marked as non-exhaustive. New variants may be added in a +/// semver compatible release. +#[non_exhaustive] +#[derive(Clone, Debug)] +pub enum StartError { + /// An error that occurs when cache inefficiency has dropped below the + /// configured heuristic thresholds. + Cache { + /// The underlying cache error that occurred. + err: CacheError, + }, + /// An error that occurs when a starting configuration's look-behind byte + /// is in this DFA's quit set. + Quit { + /// The quit byte that was found. + byte: u8, + }, + /// An error that occurs when the caller requests an anchored mode that + /// isn't supported by the DFA. + UnsupportedAnchored { + /// The anchored mode given that is unsupported. + mode: Anchored, + }, +} + +impl StartError { + pub(crate) fn cache(err: CacheError) -> StartError { + StartError::Cache { err } + } + + pub(crate) fn quit(byte: u8) -> StartError { + StartError::Quit { byte } + } + + pub(crate) fn unsupported_anchored(mode: Anchored) -> StartError { + StartError::UnsupportedAnchored { mode } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for StartError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match *self { + StartError::Cache { ref err } => Some(err), + _ => None, + } + } +} + +impl core::fmt::Display for StartError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match *self { + StartError::Cache { .. } => write!( + f, + "error computing start state because of cache inefficiency" + ), + StartError::Quit { byte } => write!( + f, + "error computing start state because the look-behind byte \ + {:?} triggered a quit state", + crate::util::escape::DebugByte(byte), + ), + StartError::UnsupportedAnchored { mode: Anchored::Yes } => { + write!( + f, + "error computing start state because \ + anchored searches are not supported or enabled" + ) + } + StartError::UnsupportedAnchored { mode: Anchored::No } => { + write!( + f, + "error computing start state because \ + unanchored searches are not supported or enabled" + ) + } + StartError::UnsupportedAnchored { + mode: Anchored::Pattern(pid), + } => { + write!( + f, + "error computing start state because \ + anchored searches for a specific pattern ({}) \ + are not supported or enabled", + pid.as_usize(), + ) + } + } + } +} + +/// An error that occurs when cache usage has become inefficient. +/// +/// One of the weaknesses of a lazy DFA is that it may need to clear its +/// cache repeatedly if it's not big enough. If this happens too much, then it +/// can slow searching down significantly. A mitigation to this is to use +/// heuristics to detect whether the cache is being used efficiently or not. +/// If not, then a lazy DFA can return a `CacheError`. +/// +/// The default configuration of a lazy DFA in this crate is +/// set such that a `CacheError` will never occur. Instead, +/// callers must opt into this behavior with settings like +/// [`dfa::Config::minimum_cache_clear_count`](crate::hybrid::dfa::Config::minimum_cache_clear_count) +/// and +/// [`dfa::Config::minimum_bytes_per_state`](crate::hybrid::dfa::Config::minimum_bytes_per_state). +/// +/// When the `std` feature is enabled, this implements the `std::error::Error` +/// trait. +#[derive(Clone, Debug)] +pub struct CacheError(()); + +impl CacheError { + pub(crate) fn too_many_cache_clears() -> CacheError { + CacheError(()) + } + + pub(crate) fn bad_efficiency() -> CacheError { + CacheError(()) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for CacheError {} + +impl core::fmt::Display for CacheError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "lazy DFA cache has been cleared too many times") + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/hybrid/id.rs b/third_party/crates/regex-automata-0.4.18/src/hybrid/id.rs new file mode 100644 index 0000000..65d8528 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/hybrid/id.rs @@ -0,0 +1,354 @@ +/// A state identifier specifically tailored for lazy DFAs. +/// +/// A lazy state ID logically represents a pointer to a DFA state. In practice, +/// by limiting the number of DFA states it can address, it reserves some +/// bits of its representation to encode some additional information. That +/// additional information is called a "tag." That tag is used to record +/// whether the state it points to is an unknown, dead, quit, start or match +/// state. +/// +/// When implementing a low level search routine with a lazy DFA, it is +/// necessary to query the type of the current state to know what to do: +/// +/// * **Unknown** - The state has not yet been computed. The +/// parameters used to get this state ID must be re-passed to +/// [`DFA::next_state`](crate::hybrid::dfa::DFA::next_state), which will never +/// return an unknown state ID. +/// * **Dead** - A dead state only has transitions to itself. It indicates that +/// the search cannot do anything else and should stop with whatever result it +/// has. +/// * **Quit** - A quit state indicates that the automaton could not answer +/// whether a match exists or not. Correct search implementations must return a +/// [`MatchError::quit`](crate::MatchError::quit) when a DFA enters a quit +/// state. +/// * **Start** - A start state is a state in which a search can begin. +/// Lazy DFAs usually have more than one start state. Branching on +/// this isn't required for correctness, but a common optimization is +/// to run a prefilter when a search enters a start state. Note that +/// start states are *not* tagged automatically, and one must enable the +/// [`Config::specialize_start_states`](crate::hybrid::dfa::Config::specialize_start_states) +/// setting for start states to be tagged. The reason for this is +/// that a DFA search loop is usually written to execute a prefilter once it +/// enters a start state. But if there is no prefilter, this handling can be +/// quite disastrous as the DFA may ping-pong between the special handling code +/// and a possible optimized hot path for handling untagged states. When start +/// states aren't specialized, then they are untagged and remain in the hot +/// path. +/// * **Match** - A match state indicates that a match has been found. +/// Depending on the semantics of your search implementation, it may either +/// continue until the end of the haystack or a dead state, or it might quit +/// and return the match immediately. +/// +/// As an optimization, the [`is_tagged`](LazyStateID::is_tagged) predicate +/// can be used to determine if a tag exists at all. This is useful to avoid +/// branching on all of the above types for every byte searched. +/// +/// # Example +/// +/// This example shows how `LazyStateID` can be used to implement a correct +/// search routine with minimal branching. In particular, this search routine +/// implements "leftmost" matching, which means that it doesn't immediately +/// stop once a match is found. Instead, it continues until it reaches a dead +/// state. +/// +/// Notice also how a correct search implementation deals with +/// [`CacheError`](crate::hybrid::CacheError)s returned by some of +/// the lazy DFA routines. When a `CacheError` occurs, it returns +/// [`MatchError::gave_up`](crate::MatchError::gave_up). +/// +/// ``` +/// use regex_automata::{ +/// hybrid::dfa::{Cache, DFA}, +/// HalfMatch, MatchError, Input, +/// }; +/// +/// fn find_leftmost_first( +/// dfa: &DFA, +/// cache: &mut Cache, +/// haystack: &[u8], +/// ) -> Result<Option<HalfMatch>, MatchError> { +/// // The start state is determined by inspecting the position and the +/// // initial bytes of the haystack. Note that start states can never +/// // be match states (since DFAs in this crate delay matches by 1 +/// // byte), so we don't need to check if the start state is a match. +/// let mut sid = dfa.start_state_forward( +/// cache, +/// &Input::new(haystack), +/// )?; +/// let mut last_match = None; +/// // Walk all the bytes in the haystack. We can quit early if we see +/// // a dead or a quit state. The former means the automaton will +/// // never transition to any other state. The latter means that the +/// // automaton entered a condition in which its search failed. +/// for (i, &b) in haystack.iter().enumerate() { +/// sid = dfa +/// .next_state(cache, sid, b) +/// .map_err(|_| MatchError::gave_up(i))?; +/// if sid.is_tagged() { +/// if sid.is_match() { +/// last_match = Some(HalfMatch::new( +/// dfa.match_pattern(cache, sid, 0), +/// i, +/// )); +/// } else if sid.is_dead() { +/// return Ok(last_match); +/// } else if sid.is_quit() { +/// // It is possible to enter into a quit state after +/// // observing a match has occurred. In that case, we +/// // should return the match instead of an error. +/// if last_match.is_some() { +/// return Ok(last_match); +/// } +/// return Err(MatchError::quit(b, i)); +/// } +/// // Implementors may also want to check for start states and +/// // handle them differently for performance reasons. But it is +/// // not necessary for correctness. Note that in order to check +/// // for start states, you'll need to enable the +/// // 'specialize_start_states' config knob, otherwise start +/// // states will not be tagged. +/// } +/// } +/// // Matches are always delayed by 1 byte, so we must explicitly walk +/// // the special "EOI" transition at the end of the search. +/// sid = dfa +/// .next_eoi_state(cache, sid) +/// .map_err(|_| MatchError::gave_up(haystack.len()))?; +/// if sid.is_match() { +/// last_match = Some(HalfMatch::new( +/// dfa.match_pattern(cache, sid, 0), +/// haystack.len(), +/// )); +/// } +/// Ok(last_match) +/// } +/// +/// // We use a greedy '+' operator to show how the search doesn't just stop +/// // once a match is detected. It continues extending the match. Using +/// // '[a-z]+?' would also work as expected and stop the search early. +/// // Greediness is built into the automaton. +/// let dfa = DFA::new(r"[a-z]+")?; +/// let mut cache = dfa.create_cache(); +/// let haystack = "123 foobar 4567".as_bytes(); +/// let mat = find_leftmost_first(&dfa, &mut cache, haystack)?.unwrap(); +/// assert_eq!(mat.pattern().as_usize(), 0); +/// assert_eq!(mat.offset(), 10); +/// +/// // Here's another example that tests our handling of the special +/// // EOI transition. This will fail to find a match if we don't call +/// // 'next_eoi_state' at the end of the search since the match isn't found +/// // until the final byte in the haystack. +/// let dfa = DFA::new(r"[0-9]{4}")?; +/// let mut cache = dfa.create_cache(); +/// let haystack = "123 foobar 4567".as_bytes(); +/// let mat = find_leftmost_first(&dfa, &mut cache, haystack)?.unwrap(); +/// assert_eq!(mat.pattern().as_usize(), 0); +/// assert_eq!(mat.offset(), 15); +/// +/// // And note that our search implementation above automatically works +/// // with multi-DFAs. Namely, `dfa.match_pattern(match_state, 0)` selects +/// // the appropriate pattern ID for us. +/// let dfa = DFA::new_many(&[r"[a-z]+", r"[0-9]+"])?; +/// let mut cache = dfa.create_cache(); +/// let haystack = "123 foobar 4567".as_bytes(); +/// let mat = find_leftmost_first(&dfa, &mut cache, haystack)?.unwrap(); +/// assert_eq!(mat.pattern().as_usize(), 1); +/// assert_eq!(mat.offset(), 3); +/// let mat = find_leftmost_first(&dfa, &mut cache, &haystack[3..])?.unwrap(); +/// assert_eq!(mat.pattern().as_usize(), 0); +/// assert_eq!(mat.offset(), 7); +/// let mat = find_leftmost_first(&dfa, &mut cache, &haystack[10..])?.unwrap(); +/// assert_eq!(mat.pattern().as_usize(), 1); +/// assert_eq!(mat.offset(), 5); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive( + Clone, Copy, Debug, Default, Eq, Hash, PartialEq, PartialOrd, Ord, +)] +pub struct LazyStateID(u32); + +impl LazyStateID { + #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))] + const MAX_BIT: usize = 31; + + #[cfg(target_pointer_width = "16")] + const MAX_BIT: usize = 15; + + const MASK_UNKNOWN: usize = 1 << (LazyStateID::MAX_BIT); + const MASK_DEAD: usize = 1 << (LazyStateID::MAX_BIT - 1); + const MASK_QUIT: usize = 1 << (LazyStateID::MAX_BIT - 2); + const MASK_START: usize = 1 << (LazyStateID::MAX_BIT - 3); + const MASK_MATCH: usize = 1 << (LazyStateID::MAX_BIT - 4); + pub(crate) const MAX: usize = LazyStateID::MASK_MATCH - 1; + + /// Create a new lazy state ID. + /// + /// If the given identifier exceeds [`LazyStateID::MAX`], then this returns + /// an error. + #[inline] + pub(crate) fn new(id: usize) -> Result<LazyStateID, LazyStateIDError> { + if id > LazyStateID::MAX { + let attempted = u64::try_from(id).unwrap(); + return Err(LazyStateIDError { attempted }); + } + Ok(LazyStateID::new_unchecked(id)) + } + + /// Create a new lazy state ID without checking whether the given value + /// exceeds [`LazyStateID::MAX`]. + /// + /// While this is unchecked, providing an incorrect value must never + /// sacrifice memory safety. + #[inline] + const fn new_unchecked(id: usize) -> LazyStateID { + // FIXME: Use as_u32() once const functions in traits are stable. + LazyStateID(id as u32) + } + + /// Return this lazy state ID as an untagged `usize`. + /// + /// If this lazy state ID is tagged, then the usize returned is the state + /// ID without the tag. If the ID was not tagged, then the usize returned + /// is equivalent to the state ID. + #[inline] + pub(crate) fn as_usize_untagged(&self) -> usize { + self.as_usize_unchecked() & LazyStateID::MAX + } + + /// Return this lazy state ID as its raw internal `usize` value, which may + /// be tagged (and thus greater than LazyStateID::MAX). + #[inline] + pub(crate) const fn as_usize_unchecked(&self) -> usize { + // FIXME: Use as_usize() once const functions in traits are stable. + self.0 as usize + } + + #[inline] + pub(crate) const fn to_unknown(&self) -> LazyStateID { + LazyStateID::new_unchecked( + self.as_usize_unchecked() | LazyStateID::MASK_UNKNOWN, + ) + } + + #[inline] + pub(crate) const fn to_dead(&self) -> LazyStateID { + LazyStateID::new_unchecked( + self.as_usize_unchecked() | LazyStateID::MASK_DEAD, + ) + } + + #[inline] + pub(crate) const fn to_quit(&self) -> LazyStateID { + LazyStateID::new_unchecked( + self.as_usize_unchecked() | LazyStateID::MASK_QUIT, + ) + } + + /// Return this lazy state ID as a state ID that is tagged as a start + /// state. + #[inline] + pub(crate) const fn to_start(&self) -> LazyStateID { + LazyStateID::new_unchecked( + self.as_usize_unchecked() | LazyStateID::MASK_START, + ) + } + + /// Return this lazy state ID as a lazy state ID that is tagged as a match + /// state. + #[inline] + pub(crate) const fn to_match(&self) -> LazyStateID { + LazyStateID::new_unchecked( + self.as_usize_unchecked() | LazyStateID::MASK_MATCH, + ) + } + + /// Return true if and only if this lazy state ID is tagged. + /// + /// When a lazy state ID is tagged, then one can conclude that it is one + /// of a match, start, dead, quit or unknown state. + #[inline] + pub const fn is_tagged(&self) -> bool { + self.as_usize_unchecked() > LazyStateID::MAX + } + + /// Return true if and only if this represents a lazy state ID that is + /// "unknown." That is, the state has not yet been created. When a caller + /// sees this state ID, it generally means that a state has to be computed + /// in order to proceed. + #[inline] + pub const fn is_unknown(&self) -> bool { + self.as_usize_unchecked() & LazyStateID::MASK_UNKNOWN > 0 + } + + /// Return true if and only if this represents a dead state. A dead state + /// is a state that can never transition to any other state except the + /// dead state. When a dead state is seen, it generally indicates that a + /// search should stop. + #[inline] + pub const fn is_dead(&self) -> bool { + self.as_usize_unchecked() & LazyStateID::MASK_DEAD > 0 + } + + /// Return true if and only if this represents a quit state. A quit state + /// is a state that is representationally equivalent to a dead state, + /// except it indicates the automaton has reached a point at which it can + /// no longer determine whether a match exists or not. In general, this + /// indicates an error during search and the caller must either pass this + /// error up or use a different search technique. + #[inline] + pub const fn is_quit(&self) -> bool { + self.as_usize_unchecked() & LazyStateID::MASK_QUIT > 0 + } + + /// Return true if and only if this lazy state ID has been tagged as a + /// start state. + /// + /// Note that if + /// [`Config::specialize_start_states`](crate::hybrid::dfa::Config) is + /// disabled (which is the default), then this will always return false + /// since start states won't be tagged. + #[inline] + pub const fn is_start(&self) -> bool { + self.as_usize_unchecked() & LazyStateID::MASK_START > 0 + } + + /// Return true if and only if this lazy state ID has been tagged as a + /// match state. + #[inline] + pub const fn is_match(&self) -> bool { + self.as_usize_unchecked() & LazyStateID::MASK_MATCH > 0 + } +} + +/// This error occurs when a lazy state ID could not be constructed. +/// +/// This occurs when given an integer exceeding the maximum lazy state ID +/// value. +/// +/// When the `std` feature is enabled, this implements the `Error` trait. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct LazyStateIDError { + attempted: u64, +} + +impl LazyStateIDError { + /// Returns the value that failed to constructed a lazy state ID. + pub(crate) fn attempted(&self) -> u64 { + self.attempted + } +} + +#[cfg(feature = "std")] +impl std::error::Error for LazyStateIDError {} + +impl core::fmt::Display for LazyStateIDError { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "failed to create LazyStateID from {:?}, which exceeds {:?}", + self.attempted(), + LazyStateID::MAX, + ) + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/hybrid/mod.rs b/third_party/crates/regex-automata-0.4.18/src/hybrid/mod.rs new file mode 100644 index 0000000..2feb839 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/hybrid/mod.rs @@ -0,0 +1,144 @@ +/*! +A module for building and searching with lazy deterministic finite automata +(DFAs). + +Like other modules in this crate, lazy DFAs support a rich regex syntax with +Unicode features. The key feature of a lazy DFA is that it builds itself +incrementally during search, and never uses more than a configured capacity of +memory. Thus, when searching with a lazy DFA, one must supply a mutable "cache" +in which the actual DFA's transition table is stored. + +If you're looking for fully compiled DFAs, then please see the top-level +[`dfa` module](crate::dfa). + +# Overview + +This section gives a brief overview of the primary types in this module: + +* A [`regex::Regex`] provides a way to search for matches of a regular +expression using lazy DFAs. This includes iterating over matches with both the +start and end positions of each match. +* A [`dfa::DFA`] provides direct low level access to a lazy DFA. + +# Example: basic regex searching + +This example shows how to compile a regex using the default configuration +and then use it to find matches in a byte string: + +``` +use regex_automata::{hybrid::regex::Regex, Match}; + +let re = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")?; +let mut cache = re.create_cache(); + +let haystack = "2018-12-24 2016-10-08"; +let matches: Vec<Match> = re.find_iter(&mut cache, haystack).collect(); +assert_eq!(matches, vec![ + Match::must(0, 0..10), + Match::must(0, 11..21), +]); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +# Example: searching with multiple regexes + +The lazy DFAs in this module all fully support searching with multiple regexes +simultaneously. You can use this support with standard leftmost-first style +searching to find non-overlapping matches: + +``` +# if cfg!(miri) { return Ok(()); } // miri takes too long +use regex_automata::{hybrid::regex::Regex, Match}; + +let re = Regex::new_many(&[r"\w+", r"\S+"])?; +let mut cache = re.create_cache(); + +let haystack = "@foo bar"; +let matches: Vec<Match> = re.find_iter(&mut cache, haystack).collect(); +assert_eq!(matches, vec![ + Match::must(1, 0..4), + Match::must(0, 5..8), +]); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +# When should I use this? + +Generally speaking, if you can abide the use of mutable state during search, +and you don't need things like capturing groups or Unicode word boundary +support in non-ASCII text, then a lazy DFA is likely a robust choice with +respect to both search speed and memory usage. Note however that its speed +may be worse than a general purpose regex engine if you don't select a good +[prefilter](crate::util::prefilter). + +If you know ahead of time that your pattern would result in a very large DFA +if it was fully compiled, it may be better to use an NFA simulation instead +of a lazy DFA. Either that, or increase the cache capacity of your lazy DFA +to something that is big enough to hold the state machine (likely through +experimentation). The issue here is that if the cache is too small, then it +could wind up being reset too frequently and this might decrease searching +speed significantly. + +# Differences with fully compiled DFAs + +A [`hybrid::regex::Regex`](crate::hybrid::regex::Regex) and a +[`dfa::regex::Regex`](crate::dfa::regex::Regex) both have the same capabilities +(and similarly for their underlying DFAs), but they achieve them through +different means. The main difference is that a hybrid or "lazy" regex builds +its DFA lazily during search, where as a fully compiled regex will build its +DFA at construction time. While building a DFA at search time might sound like +it's slow, it tends to work out where most bytes seen during a search will +reuse pre-built parts of the DFA and thus can be almost as fast as a fully +compiled DFA. The main downside is that searching requires mutable space to +store the DFA, and, in the worst case, a search can result in a new state being +created for each byte seen, which would make searching quite a bit slower. + +A fully compiled DFA never has to worry about searches being slower once +it's built. (Aside from, say, the transition table being so large that it +is subject to harsh CPU cache effects.) However, of course, building a full +DFA can be quite time consuming and memory hungry. Particularly when large +Unicode character classes are used, which tend to translate into very large +DFAs. + +A lazy DFA strikes a nice balance _in practice_, particularly in the +presence of Unicode mode, by only building what is needed. It avoids the +worst case exponential time complexity of DFA compilation by guaranteeing that +it will only build at most one state per byte searched. While the worst +case here can lead to a very high constant, it will never be exponential. + +# Syntax + +This module supports the same syntax as the `regex` crate, since they share the +same parser. You can find an exhaustive list of supported syntax in the +[documentation for the `regex` crate](https://docs.rs/regex/1/regex/#syntax). + +There are two things that are not supported by the lazy DFAs in this module: + +* Capturing groups. The DFAs (and [`Regex`](regex::Regex)es built on top +of them) can only find the offsets of an entire match, but cannot resolve +the offsets of each capturing group. This is because DFAs do not have the +expressive power necessary. Note that it is okay to build a lazy DFA from an +NFA that contains capture groups. The capture groups will simply be ignored. +* Unicode word boundaries. These present particularly difficult challenges for +DFA construction and would result in an explosion in the number of states. +One can enable [`dfa::Config::unicode_word_boundary`] though, which provides +heuristic support for Unicode word boundaries that only works on ASCII text. +Otherwise, one can use `(?-u:\b)` for an ASCII word boundary, which will work +on any input. + +There are no plans to lift either of these limitations. + +Note that these restrictions are identical to the restrictions on fully +compiled DFAs. +*/ + +pub use self::{ + error::{BuildError, CacheError, StartError}, + id::LazyStateID, +}; + +pub mod dfa; +mod error; +mod id; +pub mod regex; +mod search; diff --git a/third_party/crates/regex-automata-0.4.18/src/hybrid/regex.rs b/third_party/crates/regex-automata-0.4.18/src/hybrid/regex.rs new file mode 100644 index 0000000..b3b1fe3 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/hybrid/regex.rs @@ -0,0 +1,895 @@ +/*! +A lazy DFA backed `Regex`. + +This module provides a [`Regex`] backed by a lazy DFA. A `Regex` implements +convenience routines you might have come to expect, such as finding a match +and iterating over all non-overlapping matches. This `Regex` type is limited +in its capabilities to what a lazy DFA can provide. Therefore, APIs involving +capturing groups, for example, are not provided. + +Internally, a `Regex` is composed of two DFAs. One is a "forward" DFA that +finds the end offset of a match, where as the other is a "reverse" DFA that +find the start offset of a match. + +See the [parent module](crate::hybrid) for examples. +*/ + +use crate::{ + hybrid::{ + dfa::{self, DFA}, + error::BuildError, + }, + nfa::thompson, + util::{ + iter, + search::{Anchored, Input, Match, MatchError, MatchKind}, + }, +}; + +/// A regular expression that uses hybrid NFA/DFAs (also called "lazy DFAs") +/// for searching. +/// +/// A regular expression is comprised of two lazy DFAs, a "forward" DFA and a +/// "reverse" DFA. The forward DFA is responsible for detecting the end of +/// a match while the reverse DFA is responsible for detecting the start +/// of a match. Thus, in order to find the bounds of any given match, a +/// forward search must first be run followed by a reverse search. A match +/// found by the forward DFA guarantees that the reverse DFA will also find +/// a match. +/// +/// # Fallibility +/// +/// Most of the search routines defined on this type will _panic_ when the +/// underlying search fails. This might be because the DFA gave up because it +/// saw a quit byte, whether configured explicitly or via heuristic Unicode +/// word boundary support, although neither are enabled by default. It might +/// also fail if the underlying DFA determines it isn't making effective use of +/// the cache (which also never happens by default). Or it might fail because +/// an invalid `Input` configuration is given, for example, with an unsupported +/// [`Anchored`] mode. +/// +/// If you need to handle these error cases instead of allowing them to trigger +/// a panic, then the lower level [`Regex::try_search`] provides a fallible API +/// that never panics. +/// +/// # Example +/// +/// This example shows how to cause a search to terminate if it sees a +/// `\n` byte, and handle the error returned. This could be useful if, for +/// example, you wanted to prevent a user supplied pattern from matching +/// across a line boundary. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{hybrid::{dfa, regex::Regex}, Input, MatchError}; +/// +/// let re = Regex::builder() +/// .dfa(dfa::Config::new().quit(b'\n', true)) +/// .build(r"foo\p{any}+bar")?; +/// let mut cache = re.create_cache(); +/// +/// let input = Input::new("foo\nbar"); +/// // Normally this would produce a match, since \p{any} contains '\n'. +/// // But since we instructed the automaton to enter a quit state if a +/// // '\n' is observed, this produces a match error instead. +/// let expected = MatchError::quit(b'\n', 3); +/// let got = re.try_search(&mut cache, &input).unwrap_err(); +/// assert_eq!(expected, got); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Debug)] +pub struct Regex { + /// The forward lazy DFA. This can only find the end of a match. + forward: DFA, + /// The reverse lazy DFA. This can only find the start of a match. + /// + /// This is built with 'all' match semantics (instead of leftmost-first) + /// so that it always finds the longest possible match (which corresponds + /// to the leftmost starting position). It is also compiled as an anchored + /// matcher and has 'starts_for_each_pattern' enabled. Including starting + /// states for each pattern is necessary to ensure that we only look for + /// matches of a pattern that matched in the forward direction. Otherwise, + /// we might wind up finding the "leftmost" starting position of a totally + /// different pattern! + reverse: DFA, +} + +/// Convenience routines for regex and cache construction. +impl Regex { + /// Parse the given regular expression using the default configuration and + /// return the corresponding regex. + /// + /// If you want a non-default configuration, then use the [`Builder`] to + /// set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{hybrid::regex::Regex, Match}; + /// + /// let re = Regex::new("foo[0-9]+bar")?; + /// let mut cache = re.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 3..14)), + /// re.find(&mut cache, "zzzfoo12345barzzz"), + /// ); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new(pattern: &str) -> Result<Regex, BuildError> { + Regex::builder().build(pattern) + } + + /// Like `new`, but parses multiple patterns into a single "multi regex." + /// This similarly uses the default regex configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{hybrid::regex::Regex, Match}; + /// + /// let re = Regex::new_many(&["[a-z]+", "[0-9]+"])?; + /// let mut cache = re.create_cache(); + /// + /// let mut it = re.find_iter(&mut cache, "abc 1 foo 4567 0 quux"); + /// assert_eq!(Some(Match::must(0, 0..3)), it.next()); + /// assert_eq!(Some(Match::must(1, 4..5)), it.next()); + /// assert_eq!(Some(Match::must(0, 6..9)), it.next()); + /// assert_eq!(Some(Match::must(1, 10..14)), it.next()); + /// assert_eq!(Some(Match::must(1, 15..16)), it.next()); + /// assert_eq!(Some(Match::must(0, 17..21)), it.next()); + /// assert_eq!(None, it.next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new_many<P: AsRef<str>>( + patterns: &[P], + ) -> Result<Regex, BuildError> { + Regex::builder().build_many(patterns) + } + + /// Return a builder for configuring the construction of a `Regex`. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Builder`] type in common cases. + /// + /// # Example + /// + /// This example shows how to use the builder to disable UTF-8 mode + /// everywhere. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// hybrid::regex::Regex, nfa::thompson, util::syntax, Match, + /// }; + /// + /// let re = Regex::builder() + /// .syntax(syntax::Config::new().utf8(false)) + /// .thompson(thompson::Config::new().utf8(false)) + /// .build(r"foo(?-u:[^b])ar.*")?; + /// let mut cache = re.create_cache(); + /// + /// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"; + /// let expected = Some(Match::must(0, 1..9)); + /// let got = re.find(&mut cache, haystack); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn builder() -> Builder { + Builder::new() + } + + /// Create a new cache for this `Regex`. + /// + /// The cache returned should only be used for searches for this + /// `Regex`. If you want to reuse the cache for another `Regex`, then + /// you must call [`Cache::reset`] with that `Regex` (or, equivalently, + /// [`Regex::reset_cache`]). + pub fn create_cache(&self) -> Cache { + Cache::new(self) + } + + /// Reset the given cache such that it can be used for searching with the + /// this `Regex` (and only this `Regex`). + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different `Regex`. + /// + /// Resetting a cache sets its "clear count" to 0. This is relevant if the + /// `Regex` has been configured to "give up" after it has cleared the cache + /// a certain number of times. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different `Regex`. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::regex::Regex, Match}; + /// + /// let re1 = Regex::new(r"\w")?; + /// let re2 = Regex::new(r"\W")?; + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 0..2)), + /// re1.find(&mut cache, "Δ"), + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the Regex we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// re2.reset_cache(&mut cache); + /// assert_eq!( + /// Some(Match::must(0, 0..3)), + /// re2.find(&mut cache, "☃"), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset_cache(&self, cache: &mut Cache) { + self.forward().reset_cache(&mut cache.forward); + self.reverse().reset_cache(&mut cache.reverse); + } +} + +/// Standard infallible search routines for finding and iterating over matches. +impl Regex { + /// Returns true if and only if this regex matches the given haystack. + /// + /// This routine may short circuit if it knows that scanning future input + /// will never lead to a different result. In particular, if the underlying + /// DFA enters a match state or a dead state, then this routine will return + /// `true` or `false`, respectively, without inspecting any future input. + /// + /// # Panics + /// + /// This routine panics if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search panics, callers cannot know whether a match exists or + /// not. + /// + /// Use [`Regex::try_search`] if you want to handle these error conditions. + /// + /// # Example + /// + /// ``` + /// use regex_automata::hybrid::regex::Regex; + /// + /// let re = Regex::new("foo[0-9]+bar")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.is_match(&mut cache, "foo12345bar")); + /// assert!(!re.is_match(&mut cache, "foobar")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_match<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + ) -> bool { + // Not only can we do an "earliest" search, but we can avoid doing a + // reverse scan too. + self.forward() + .try_search_fwd(&mut cache.forward, &input.into().earliest(true)) + .unwrap() + .is_some() + } + + /// Returns the start and end offset of the leftmost match. If no match + /// exists, then `None` is returned. + /// + /// # Panics + /// + /// This routine panics if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search panics, callers cannot know whether a match exists or + /// not. + /// + /// Use [`Regex::try_search`] if you want to handle these error conditions. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Match, hybrid::regex::Regex}; + /// + /// let re = Regex::new("foo[0-9]+")?; + /// let mut cache = re.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 3..11)), + /// re.find(&mut cache, "zzzfoo12345zzz"), + /// ); + /// + /// // Even though a match is found after reading the first byte (`a`), + /// // the default leftmost-first match semantics demand that we find the + /// // earliest match that prefers earlier parts of the pattern over latter + /// // parts. + /// let re = Regex::new("abc|a")?; + /// let mut cache = re.create_cache(); + /// assert_eq!(Some(Match::must(0, 0..3)), re.find(&mut cache, "abc")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + ) -> Option<Match> { + self.try_search(cache, &input.into()).unwrap() + } + + /// Returns an iterator over all non-overlapping leftmost matches in the + /// given bytes. If no match exists, then the iterator yields no elements. + /// + /// # Panics + /// + /// This routine panics if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search panics, callers cannot know whether a match exists or + /// not. + /// + /// The above conditions also apply to the iterator returned as well. For + /// example, if the lazy DFA gives up or quits during a search using this + /// method, then a panic will occur during iteration. + /// + /// Use [`Regex::try_search`] with [`util::iter::Searcher`](iter::Searcher) + /// if you want to handle these error conditions. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{hybrid::regex::Regex, Match}; + /// + /// let re = Regex::new("foo[0-9]+")?; + /// let mut cache = re.create_cache(); + /// + /// let text = "foo1 foo12 foo123"; + /// let matches: Vec<Match> = re.find_iter(&mut cache, text).collect(); + /// assert_eq!(matches, vec![ + /// Match::must(0, 0..4), + /// Match::must(0, 5..10), + /// Match::must(0, 11..17), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find_iter<'r, 'c, 'h, I: Into<Input<'h>>>( + &'r self, + cache: &'c mut Cache, + input: I, + ) -> FindMatches<'r, 'c, 'h> { + let it = iter::Searcher::new(input.into()); + FindMatches { re: self, cache, it } + } +} + +/// Lower level "search" primitives that accept a `&Input` for cheap reuse +/// and return an error if one occurs instead of panicking. +impl Regex { + /// Returns the start and end offset of the leftmost match. If no match + /// exists, then `None` is returned. + /// + /// This is like [`Regex::find`] but with two differences: + /// + /// 1. It is not generic over `Into<Input>` and instead accepts a + /// `&Input`. This permits reusing the same `Input` for multiple searches + /// without needing to create a new one. This _may_ help with latency. + /// 2. It returns an error if the search could not complete where as + /// [`Regex::find`] will panic. + /// + /// # Errors + /// + /// This routine errors if the search could not complete. This can occur + /// in a number of circumstances: + /// + /// * The configuration of the lazy DFA may permit it to "quit" the search. + /// For example, setting quit bytes or enabling heuristic support for + /// Unicode word boundaries. The default configuration does not enable any + /// option that could result in the lazy DFA quitting. + /// * The configuration of the lazy DFA may also permit it to "give up" + /// on a search if it makes ineffective use of its transition table + /// cache. The default configuration does not enable this by default, + /// although it is typically a good idea to. + /// * When the provided `Input` configuration is not supported. For + /// example, by providing an unsupported anchor mode. + /// + /// When a search returns an error, callers cannot know whether a match + /// exists or not. + #[inline] + pub fn try_search( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<Option<Match>, MatchError> { + let (fcache, rcache) = (&mut cache.forward, &mut cache.reverse); + let end = match self.forward().try_search_fwd(fcache, input)? { + None => return Ok(None), + Some(end) => end, + }; + // This special cases an empty match at the beginning of the search. If + // our end matches our start, then since a reverse DFA can't match past + // the start, it must follow that our starting position is also our end + // position. So short circuit and skip the reverse search. + if input.start() == end.offset() { + return Ok(Some(Match::new( + end.pattern(), + end.offset()..end.offset(), + ))); + } + // We can also skip the reverse search if we know our search was + // anchored. This occurs either when the input config is anchored or + // when we know the regex itself is anchored. In this case, we know the + // start of the match, if one is found, must be the start of the + // search. + if self.is_anchored(input) { + return Ok(Some(Match::new( + end.pattern(), + input.start()..end.offset(), + ))); + } + // N.B. I have tentatively convinced myself that it isn't necessary + // to specify the specific pattern for the reverse search since the + // reverse search will always find the same pattern to match as the + // forward search. But I lack a rigorous proof. Why not just provide + // the pattern anyway? Well, if it is needed, then leaving it out + // gives us a chance to find a witness. (Also, if we don't need to + // specify the pattern, then we don't need to build the reverse DFA + // with 'starts_for_each_pattern' enabled. It doesn't matter too much + // for the lazy DFA, but does make the overall DFA bigger.) + // + // We also need to be careful to disable 'earliest' for the reverse + // search, since it could be enabled for the forward search. In the + // reverse case, to satisfy "leftmost" criteria, we need to match as + // much as we can. We also need to be careful to make the search + // anchored. We don't want the reverse search to report any matches + // other than the one beginning at the end of our forward search. + let revsearch = input + .clone() + .span(input.start()..end.offset()) + .anchored(Anchored::Yes) + .earliest(false); + let start = self + .reverse() + .try_search_rev(rcache, &revsearch)? + .expect("reverse search must match if forward search does"); + debug_assert_eq!( + start.pattern(), + end.pattern(), + "forward and reverse search must match same pattern", + ); + debug_assert!(start.offset() <= end.offset()); + Ok(Some(Match::new(end.pattern(), start.offset()..end.offset()))) + } + + /// Returns true if either the given input specifies an anchored search + /// or if the underlying NFA is always anchored. + fn is_anchored(&self, input: &Input<'_>) -> bool { + match input.get_anchored() { + Anchored::No => { + self.forward().get_nfa().is_always_start_anchored() + } + Anchored::Yes | Anchored::Pattern(_) => true, + } + } +} + +/// Non-search APIs for querying information about the regex and setting a +/// prefilter. +impl Regex { + /// Return the underlying lazy DFA responsible for forward matching. + /// + /// This is useful for accessing the underlying lazy DFA and using it + /// directly if the situation calls for it. + pub fn forward(&self) -> &DFA { + &self.forward + } + + /// Return the underlying lazy DFA responsible for reverse matching. + /// + /// This is useful for accessing the underlying lazy DFA and using it + /// directly if the situation calls for it. + pub fn reverse(&self) -> &DFA { + &self.reverse + } + + /// Returns the total number of patterns matched by this regex. + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::hybrid::regex::Regex; + /// + /// let re = Regex::new_many(&[r"[a-z]+", r"[0-9]+", r"\w+"])?; + /// assert_eq!(3, re.pattern_len()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn pattern_len(&self) -> usize { + assert_eq!(self.forward().pattern_len(), self.reverse().pattern_len()); + self.forward().pattern_len() + } +} + +/// An iterator over all non-overlapping matches for an infallible search. +/// +/// The iterator yields a [`Match`] value until no more matches could be found. +/// If the underlying regex engine returns an error, then a panic occurs. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the regex object. +/// * `'h` represents the lifetime of the haystack being searched. +/// * `'c` represents the lifetime of the regex cache. +/// +/// This iterator can be created with the [`Regex::find_iter`] method. +#[derive(Debug)] +pub struct FindMatches<'r, 'c, 'h> { + re: &'r Regex, + cache: &'c mut Cache, + it: iter::Searcher<'h>, +} + +impl<'r, 'c, 'h> Iterator for FindMatches<'r, 'c, 'h> { + type Item = Match; + + #[inline] + fn next(&mut self) -> Option<Match> { + let FindMatches { re, ref mut cache, ref mut it } = *self; + it.advance(|input| re.try_search(cache, input)) + } +} + +/// A cache represents a partially computed forward and reverse DFA. +/// +/// A cache is the key component that differentiates a classical DFA and a +/// hybrid NFA/DFA (also called a "lazy DFA"). Where a classical DFA builds a +/// complete transition table that can handle all possible inputs, a hybrid +/// NFA/DFA starts with an empty transition table and builds only the parts +/// required during search. The parts that are built are stored in a cache. For +/// this reason, a cache is a required parameter for nearly every operation on +/// a [`Regex`]. +/// +/// Caches can be created from their corresponding `Regex` via +/// [`Regex::create_cache`]. A cache can only be used with either the `Regex` +/// that created it, or the `Regex` that was most recently used to reset it +/// with [`Cache::reset`]. Using a cache with any other `Regex` may result in +/// panics or incorrect results. +#[derive(Debug, Clone)] +pub struct Cache { + forward: dfa::Cache, + reverse: dfa::Cache, +} + +impl Cache { + /// Create a new cache for the given `Regex`. + /// + /// The cache returned should only be used for searches for the given + /// `Regex`. If you want to reuse the cache for another `Regex`, then you + /// must call [`Cache::reset`] with that `Regex`. + pub fn new(re: &Regex) -> Cache { + let forward = dfa::Cache::new(re.forward()); + let reverse = dfa::Cache::new(re.reverse()); + Cache { forward, reverse } + } + + /// Reset this cache such that it can be used for searching with the given + /// `Regex` (and only that `Regex`). + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different `Regex`. + /// + /// Resetting a cache sets its "clear count" to 0. This is relevant if the + /// `Regex` has been configured to "give up" after it has cleared the cache + /// a certain number of times. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different `Regex`. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{hybrid::regex::Regex, Match}; + /// + /// let re1 = Regex::new(r"\w")?; + /// let re2 = Regex::new(r"\W")?; + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 0..2)), + /// re1.find(&mut cache, "Δ"), + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the Regex we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// cache.reset(&re2); + /// assert_eq!( + /// Some(Match::must(0, 0..3)), + /// re2.find(&mut cache, "☃"), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset(&mut self, re: &Regex) { + self.forward.reset(re.forward()); + self.reverse.reset(re.reverse()); + } + + /// Return a reference to the forward cache. + pub fn forward(&mut self) -> &dfa::Cache { + &self.forward + } + + /// Return a reference to the reverse cache. + pub fn reverse(&mut self) -> &dfa::Cache { + &self.reverse + } + + /// Return a mutable reference to the forward cache. + /// + /// If you need mutable references to both the forward and reverse caches, + /// then use [`Cache::as_parts_mut`]. + pub fn forward_mut(&mut self) -> &mut dfa::Cache { + &mut self.forward + } + + /// Return a mutable reference to the reverse cache. + /// + /// If you need mutable references to both the forward and reverse caches, + /// then use [`Cache::as_parts_mut`]. + pub fn reverse_mut(&mut self) -> &mut dfa::Cache { + &mut self.reverse + } + + /// Return references to the forward and reverse caches, respectively. + pub fn as_parts(&self) -> (&dfa::Cache, &dfa::Cache) { + (&self.forward, &self.reverse) + } + + /// Return mutable references to the forward and reverse caches, + /// respectively. + pub fn as_parts_mut(&mut self) -> (&mut dfa::Cache, &mut dfa::Cache) { + (&mut self.forward, &mut self.reverse) + } + + /// Returns the heap memory usage, in bytes, as a sum of the forward and + /// reverse lazy DFA caches. + /// + /// This does **not** include the stack size used up by this cache. To + /// compute that, use `std::mem::size_of::<Cache>()`. + pub fn memory_usage(&self) -> usize { + self.forward.memory_usage() + self.reverse.memory_usage() + } +} + +/// A builder for a regex based on a hybrid NFA/DFA. +/// +/// This builder permits configuring options for the syntax of a pattern, the +/// NFA construction, the lazy DFA construction and finally the regex searching +/// itself. This builder is different from a general purpose regex builder +/// in that it permits fine grain configuration of the construction process. +/// The trade off for this is complexity, and the possibility of setting a +/// configuration that might not make sense. For example, there are two +/// different UTF-8 modes: +/// +/// * [`syntax::Config::utf8`](crate::util::syntax::Config::utf8) controls +/// whether the pattern itself can contain sub-expressions that match invalid +/// UTF-8. +/// * [`thompson::Config::utf8`] controls how the regex iterators themselves +/// advance the starting position of the next search when a match with zero +/// length is found. +/// +/// Generally speaking, callers will want to either enable all of these or +/// disable all of these. +/// +/// Internally, building a regex requires building two hybrid NFA/DFAs, +/// where one is responsible for finding the end of a match and the other is +/// responsible for finding the start of a match. If you only need to detect +/// whether something matched, or only the end of a match, then you should use +/// a [`dfa::Builder`] to construct a single hybrid NFA/DFA, which is cheaper +/// than building two of them. +/// +/// # Example +/// +/// This example shows how to disable UTF-8 mode in the syntax and the regex +/// itself. This is generally what you want for matching on arbitrary bytes. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{ +/// hybrid::regex::Regex, nfa::thompson, util::syntax, Match, +/// }; +/// +/// let re = Regex::builder() +/// .syntax(syntax::Config::new().utf8(false)) +/// .thompson(thompson::Config::new().utf8(false)) +/// .build(r"foo(?-u:[^b])ar.*")?; +/// let mut cache = re.create_cache(); +/// +/// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"; +/// let expected = Some(Match::must(0, 1..9)); +/// let got = re.find(&mut cache, haystack); +/// assert_eq!(expected, got); +/// // Notice that `(?-u:[^b])` matches invalid UTF-8, +/// // but the subsequent `.*` does not! Disabling UTF-8 +/// // on the syntax permits this. +/// assert_eq!(b"foo\xFFarzz", &haystack[got.unwrap().range()]); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Builder { + dfa: dfa::Builder, +} + +impl Builder { + /// Create a new regex builder with the default configuration. + pub fn new() -> Builder { + Builder { dfa: DFA::builder() } + } + + /// Build a regex from the given pattern. + /// + /// If there was a problem parsing or compiling the pattern, then an error + /// is returned. + #[cfg(feature = "syntax")] + pub fn build(&self, pattern: &str) -> Result<Regex, BuildError> { + self.build_many(&[pattern]) + } + + /// Build a regex from the given patterns. + #[cfg(feature = "syntax")] + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<Regex, BuildError> { + let forward = self.dfa.build_many(patterns)?; + let reverse = self + .dfa + .clone() + .configure( + DFA::config() + .prefilter(None) + .specialize_start_states(false) + .match_kind(MatchKind::All), + ) + .thompson(thompson::Config::new().reverse(true)) + .build_many(patterns)?; + Ok(self.build_from_dfas(forward, reverse)) + } + + /// Build a regex from its component forward and reverse hybrid NFA/DFAs. + /// + /// This is useful when you've built a forward and reverse lazy DFA + /// separately, and want to combine them into a single regex. Once build, + /// the individual DFAs given can still be accessed via [`Regex::forward`] + /// and [`Regex::reverse`]. + /// + /// It is important that the reverse lazy DFA be compiled under the + /// following conditions: + /// + /// * It should use [`MatchKind::All`] semantics. + /// * It should match in reverse. + /// * Otherwise, its configuration should match the forward DFA. + /// + /// If these conditions aren't satisfied, then the behavior of searches is + /// unspecified. + /// + /// Note that when using this constructor, no configuration is applied. + /// Since this routine provides the DFAs to the builder, there is no + /// opportunity to apply other configuration options. + /// + /// # Example + /// + /// This shows how to build individual lazy forward and reverse DFAs, and + /// then combine them into a single `Regex`. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::{dfa::DFA, regex::Regex}, + /// nfa::thompson, + /// MatchKind, + /// }; + /// + /// let fwd = DFA::new(r"foo[0-9]+")?; + /// let rev = DFA::builder() + /// .configure(DFA::config().match_kind(MatchKind::All)) + /// .thompson(thompson::Config::new().reverse(true)) + /// .build(r"foo[0-9]+")?; + /// + /// let re = Regex::builder().build_from_dfas(fwd, rev); + /// let mut cache = re.create_cache(); + /// assert_eq!(true, re.is_match(&mut cache, "foo123")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_from_dfas(&self, forward: DFA, reverse: DFA) -> Regex { + Regex { forward, reverse } + } + + /// Set the syntax configuration for this builder using + /// [`syntax::Config`](crate::util::syntax::Config). + /// + /// This permits setting things like case insensitivity, Unicode and multi + /// line mode. + #[cfg(feature = "syntax")] + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Builder { + self.dfa.syntax(config); + self + } + + /// Set the Thompson NFA configuration for this builder using + /// [`nfa::thompson::Config`](thompson::Config). + /// + /// This permits setting things like whether additional time should be + /// spent shrinking the size of the NFA. + #[cfg(feature = "syntax")] + pub fn thompson(&mut self, config: thompson::Config) -> &mut Builder { + self.dfa.thompson(config); + self + } + + /// Set the lazy DFA compilation configuration for this builder using + /// [`dfa::Config`]. + /// + /// This permits setting things like whether Unicode word boundaries should + /// be heuristically supported or settings how the behavior of the cache. + pub fn dfa(&mut self, config: dfa::Config) -> &mut Builder { + self.dfa.configure(config); + self + } +} + +impl Default for Builder { + fn default() -> Builder { + Builder::new() + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/hybrid/search.rs b/third_party/crates/regex-automata-0.4.18/src/hybrid/search.rs new file mode 100644 index 0000000..587337a --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/hybrid/search.rs @@ -0,0 +1,808 @@ +use crate::{ + hybrid::{ + dfa::{Cache, OverlappingState, DFA}, + id::LazyStateID, + }, + util::{ + prefilter::Prefilter, + search::{HalfMatch, Input, MatchError, Span}, + }, +}; + +#[inline(never)] +pub(crate) fn find_fwd( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, +) -> Result<Option<HalfMatch>, MatchError> { + if input.is_done() { + return Ok(None); + } + let pre = if input.get_anchored().is_anchored() { + None + } else { + dfa.get_config().get_prefilter() + }; + // So what we do here is specialize four different versions of 'find_fwd': + // one for each of the combinations for 'has prefilter' and 'is earliest + // search'. The reason for doing this is that both of these things require + // branches and special handling in some code that can be very hot, + // and shaving off as much as we can when we don't need it tends to be + // beneficial in ad hoc benchmarks. To see these differences, you often + // need a query with a high match count. In other words, specializing these + // four routines *tends* to help latency more than throughput. + if pre.is_some() { + if input.get_earliest() { + find_fwd_imp(dfa, cache, input, pre, true) + } else { + find_fwd_imp(dfa, cache, input, pre, false) + } + } else { + if input.get_earliest() { + find_fwd_imp(dfa, cache, input, None, true) + } else { + find_fwd_imp(dfa, cache, input, None, false) + } + } +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn find_fwd_imp( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, + pre: Option<&'_ Prefilter>, + earliest: bool, +) -> Result<Option<HalfMatch>, MatchError> { + // See 'prefilter_restart' docs for explanation. + let universal_start = dfa.get_nfa().look_set_prefix_any().is_empty(); + let mut mat = None; + let mut sid = init_fwd(dfa, cache, input)?; + let mut at = input.start(); + // This could just be a closure, but then I think it would be unsound + // because it would need to be safe to invoke. This way, the lack of safety + // is clearer in the code below. + macro_rules! next_unchecked { + ($sid:expr, $at:expr) => {{ + let byte = *input.haystack().get_unchecked($at); + dfa.next_state_untagged_unchecked(cache, $sid, byte) + }}; + } + + if let Some(ref pre) = pre { + let span = Span::from(at..input.end()); + match pre.find(input.haystack(), span) { + None => return Ok(mat), + Some(ref span) => { + at = span.start; + if !universal_start { + sid = prefilter_restart(dfa, cache, &input, at)?; + } + } + } + } + cache.search_start(at); + while at < input.end() { + if sid.is_tagged() { + cache.search_update(at); + sid = dfa + .next_state(cache, sid, input.haystack()[at]) + .map_err(|_| gave_up(at))?; + } else { + // SAFETY: There are two safety invariants we need to uphold + // here in the loops below: that 'sid' and 'prev_sid' are valid + // state IDs for this DFA, and that 'at' is a valid index into + // 'haystack'. For the former, we rely on the invariant that + // next_state* and start_state_forward always returns a valid state + // ID (given a valid state ID in the former case), and that we are + // only at this place in the code if 'sid' is untagged. Moreover, + // every call to next_state_untagged_unchecked below is guarded by + // a check that sid is untagged. For the latter safety invariant, + // we always guard unchecked access with a check that 'at' is less + // than 'end', where 'end <= haystack.len()'. In the unrolled loop + // below, we ensure that 'at' is always in bounds. + // + // PERF: For justification of omitting bounds checks, it gives us a + // ~10% bump in search time. This was used for a benchmark: + // + // regex-cli find half hybrid -p '(?m)^.+$' -UBb bigfile + // + // PERF: For justification for the loop unrolling, we use a few + // different tests: + // + // regex-cli find half hybrid -p '\w{50}' -UBb bigfile + // regex-cli find half hybrid -p '(?m)^.+$' -UBb bigfile + // regex-cli find half hybrid -p 'ZQZQZQZQ' -UBb bigfile + // + // And there are three different configurations: + // + // nounroll: this entire 'else' block vanishes and we just + // always use 'dfa.next_state(..)'. + // unroll1: just the outer loop below + // unroll2: just the inner loop below + // unroll3: both the outer and inner loops below + // + // This results in a matrix of timings for each of the above + // regexes with each of the above unrolling configurations: + // + // '\w{50}' '(?m)^.+$' 'ZQZQZQZQ' + // nounroll 1.51s 2.34s 1.51s + // unroll1 1.53s 2.32s 1.56s + // unroll2 2.22s 1.50s 0.61s + // unroll3 1.67s 1.45s 0.61s + // + // Ideally we'd be able to find a configuration that yields the + // best time for all regexes, but alas we settle for unroll3 that + // gives us *almost* the best for '\w{50}' and the best for the + // other two regexes. + // + // So what exactly is going on here? The first unrolling (grouping + // together runs of untagged transitions) specifically targets + // our choice of representation. The second unrolling (grouping + // together runs of self-transitions) specifically targets a common + // DFA topology. Let's dig in a little bit by looking at our + // regexes: + // + // '\w{50}': This regex spends a lot of time outside of the DFA's + // start state matching some part of the '\w' repetition. This + // means that it's a bit of a worst case for loop unrolling that + // targets self-transitions since the self-transitions in '\w{50}' + // are not particularly active for this haystack. However, the + // first unrolling (grouping together untagged transitions) + // does apply quite well here since very few transitions hit + // match/dead/quit/unknown states. It is however worth mentioning + // that if start states are configured to be tagged (which you + // typically want to do if you have a prefilter), then this regex + // actually slows way down because it is constantly ping-ponging + // out of the unrolled loop and into the handling of a tagged start + // state below. But when start states aren't tagged, the unrolled + // loop stays hot. (This is why it's imperative that start state + // tagging be disabled when there isn't a prefilter!) + // + // '(?m)^.+$': There are two important aspects of this regex: 1) + // on this haystack, its match count is very high, much higher + // than the other two regex and 2) it spends the vast majority + // of its time matching '.+'. Since Unicode mode is disabled, + // this corresponds to repeatedly following self transitions for + // the vast majority of the input. This does benefit from the + // untagged unrolling since most of the transitions will be to + // untagged states, but the untagged unrolling does more work than + // what is actually required. Namely, it has to keep track of the + // previous and next state IDs, which I guess requires a bit more + // shuffling. This is supported by the fact that nounroll+unroll1 + // are both slower than unroll2+unroll3, where the latter has a + // loop unrolling that specifically targets self-transitions. + // + // 'ZQZQZQZQ': This one is very similar to '(?m)^.+$' because it + // spends the vast majority of its time in self-transitions for + // the (implicit) unanchored prefix. The main difference with + // '(?m)^.+$' is that it has a much lower match count. So there + // isn't much time spent in the overhead of reporting matches. This + // is the primary explainer in the perf difference here. We include + // this regex and the former to make sure we have comparison points + // with high and low match counts. + // + // NOTE: I used 'OpenSubtitles2018.raw.sample.en' for 'bigfile'. + // + // NOTE: In a follow-up, it turns out that the "inner" loop + // mentioned above was a pretty big pessimization in some other + // cases. Namely, it resulted in too much ping-ponging into and out + // of the loop, which resulted in nearly ~2x regressions in search + // time when compared to the originally lazy DFA in the regex crate. + // So I've removed the second loop unrolling that targets the + // self-transition case. + let mut prev_sid = sid; + while at < input.end() { + prev_sid = unsafe { next_unchecked!(sid, at) }; + if prev_sid.is_tagged() || at + 3 >= input.end() { + core::mem::swap(&mut prev_sid, &mut sid); + break; + } + at += 1; + + sid = unsafe { next_unchecked!(prev_sid, at) }; + if sid.is_tagged() { + break; + } + at += 1; + + prev_sid = unsafe { next_unchecked!(sid, at) }; + if prev_sid.is_tagged() { + core::mem::swap(&mut prev_sid, &mut sid); + break; + } + at += 1; + + sid = unsafe { next_unchecked!(prev_sid, at) }; + if sid.is_tagged() { + break; + } + at += 1; + } + // If we quit out of the code above with an unknown state ID at + // any point, then we need to re-compute that transition using + // 'next_state', which will do NFA powerset construction for us. + if sid.is_unknown() { + cache.search_update(at); + sid = dfa + .next_state(cache, prev_sid, input.haystack()[at]) + .map_err(|_| gave_up(at))?; + } + } + if sid.is_tagged() { + if sid.is_start() { + // A prefilter finds possible starts of new matches. Once a + // match is pending, the DFA may still be resolving its + // priority. Skipping ahead here could discard that match and + // replace it with a later one. + if let Some(ref pre) = pre { + if mat.is_none() { + let span = Span::from(at..input.end()); + match pre.find(input.haystack(), span) { + None => { + cache.search_finish(span.end); + return Ok(mat); + } + Some(ref span) => { + // We want to skip any update to 'at' below + // at the end of this iteration and just + // jump immediately back to the next state + // transition at the leading position of the + // candidate match. + // + // ... but only if we actually made progress + // with our prefilter, otherwise if the start + // state has a self-loop, we can get stuck. + if span.start > at { + at = span.start; + if !universal_start { + sid = prefilter_restart( + dfa, cache, &input, at, + )?; + } + continue; + } + } + } + } + } + } else if sid.is_match() { + let pattern = dfa.match_pattern(cache, sid, 0); + // Since slice ranges are inclusive at the beginning and + // exclusive at the end, and since forward searches report + // the end, we can return 'at' as-is. This only works because + // matches are delayed by 1 byte. So by the time we observe a + // match, 'at' has already been set to 1 byte past the actual + // match location, which is precisely the exclusive ending + // bound of the match. + mat = Some(HalfMatch::new(pattern, at)); + if earliest { + cache.search_finish(at); + return Ok(mat); + } + } else if sid.is_dead() { + cache.search_finish(at); + return Ok(mat); + } else if sid.is_quit() { + cache.search_finish(at); + return Err(MatchError::quit(input.haystack()[at], at)); + } else { + debug_assert!(sid.is_unknown()); + unreachable!("sid being unknown is a bug"); + } + } + at += 1; + } + eoi_fwd(dfa, cache, input, &mut sid, &mut mat)?; + cache.search_finish(input.end()); + Ok(mat) +} + +#[inline(never)] +pub(crate) fn find_rev( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, +) -> Result<Option<HalfMatch>, MatchError> { + if input.is_done() { + return Ok(None); + } + if input.get_earliest() { + find_rev_imp(dfa, cache, input, true) + } else { + find_rev_imp(dfa, cache, input, false) + } +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn find_rev_imp( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, + earliest: bool, +) -> Result<Option<HalfMatch>, MatchError> { + let mut mat = None; + let mut sid = init_rev(dfa, cache, input)?; + // In reverse search, the loop below can't handle the case of searching an + // empty slice. Ideally we could write something congruent to the forward + // search, i.e., 'while at >= start', but 'start' might be 0. Since we use + // an unsigned offset, 'at >= 0' is trivially always true. We could avoid + // this extra case handling by using a signed offset, but Rust makes it + // annoying to do. So... We just handle the empty case separately. + if input.start() == input.end() { + eoi_rev(dfa, cache, input, &mut sid, &mut mat)?; + return Ok(mat); + } + + let mut at = input.end() - 1; + macro_rules! next_unchecked { + ($sid:expr, $at:expr) => {{ + let byte = *input.haystack().get_unchecked($at); + dfa.next_state_untagged_unchecked(cache, $sid, byte) + }}; + } + cache.search_start(at); + loop { + if sid.is_tagged() { + cache.search_update(at); + sid = dfa + .next_state(cache, sid, input.haystack()[at]) + .map_err(|_| gave_up(at))?; + } else { + // SAFETY: See comments in 'find_fwd' for a safety argument. + // + // PERF: The comments in 'find_fwd' also provide a justification + // from a performance perspective as to 1) why we elide bounds + // checks and 2) why we do a specialized version of unrolling + // below. The reverse search does have a slightly different + // consideration in that most reverse searches tend to be + // anchored and on shorter haystacks. However, this still makes a + // difference. Take this command for example: + // + // regex-cli find match hybrid -p '(?m)^.+$' -UBb bigfile + // + // (Notice that we use 'find hybrid regex', not 'find hybrid dfa' + // like in the justification for the forward direction. The 'regex' + // sub-command will find start-of-match and thus run the reverse + // direction.) + // + // Without unrolling below, the above command takes around 3.76s. + // But with the unrolling below, we get down to 2.55s. If we keep + // the unrolling but add in bounds checks, then we get 2.86s. + // + // NOTE: I used 'OpenSubtitles2018.raw.sample.en' for 'bigfile'. + let mut prev_sid = sid; + while at >= input.start() { + prev_sid = unsafe { next_unchecked!(sid, at) }; + if prev_sid.is_tagged() + || at <= input.start().saturating_add(3) + { + core::mem::swap(&mut prev_sid, &mut sid); + break; + } + at -= 1; + + sid = unsafe { next_unchecked!(prev_sid, at) }; + if sid.is_tagged() { + break; + } + at -= 1; + + prev_sid = unsafe { next_unchecked!(sid, at) }; + if prev_sid.is_tagged() { + core::mem::swap(&mut prev_sid, &mut sid); + break; + } + at -= 1; + + sid = unsafe { next_unchecked!(prev_sid, at) }; + if sid.is_tagged() { + break; + } + at -= 1; + } + // If we quit out of the code above with an unknown state ID at + // any point, then we need to re-compute that transition using + // 'next_state', which will do NFA powerset construction for us. + if sid.is_unknown() { + cache.search_update(at); + sid = dfa + .next_state(cache, prev_sid, input.haystack()[at]) + .map_err(|_| gave_up(at))?; + } + } + if sid.is_tagged() { + if sid.is_start() { + // do nothing + } else if sid.is_match() { + let pattern = dfa.match_pattern(cache, sid, 0); + // Since reverse searches report the beginning of a match + // and the beginning is inclusive (not exclusive like the + // end of a match), we add 1 to make it inclusive. + mat = Some(HalfMatch::new(pattern, at + 1)); + if earliest { + cache.search_finish(at); + return Ok(mat); + } + } else if sid.is_dead() { + cache.search_finish(at); + return Ok(mat); + } else if sid.is_quit() { + cache.search_finish(at); + return Err(MatchError::quit(input.haystack()[at], at)); + } else { + debug_assert!(sid.is_unknown()); + unreachable!("sid being unknown is a bug"); + } + } + if at == input.start() { + break; + } + at -= 1; + } + cache.search_finish(input.start()); + eoi_rev(dfa, cache, input, &mut sid, &mut mat)?; + Ok(mat) +} + +#[inline(never)] +pub(crate) fn find_overlapping_fwd( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, + state: &mut OverlappingState, +) -> Result<(), MatchError> { + state.mat = None; + if input.is_done() { + return Ok(()); + } + let pre = if input.get_anchored().is_anchored() { + None + } else { + dfa.get_config().get_prefilter() + }; + if pre.is_some() { + find_overlapping_fwd_imp(dfa, cache, input, pre, state) + } else { + find_overlapping_fwd_imp(dfa, cache, input, None, state) + } +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn find_overlapping_fwd_imp( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, + pre: Option<&'_ Prefilter>, + state: &mut OverlappingState, +) -> Result<(), MatchError> { + // See 'prefilter_restart' docs for explanation. + let universal_start = dfa.get_nfa().look_set_prefix_any().is_empty(); + let mut sid = match state.id { + None => { + state.at = input.start(); + init_fwd(dfa, cache, input)? + } + Some(sid) => { + if let Some(match_index) = state.next_match_index { + let match_len = dfa.match_len(cache, sid); + if match_index < match_len { + state.next_match_index = Some(match_index + 1); + let pattern = dfa.match_pattern(cache, sid, match_index); + state.mat = Some(HalfMatch::new(pattern, state.at)); + return Ok(()); + } + } + // Once we've reported all matches at a given position, we need to + // advance the search to the next position. + state.at += 1; + if state.at > input.end() { + return Ok(()); + } + sid + } + }; + + // NOTE: We don't optimize the crap out of this routine primarily because + // it seems like most overlapping searches will have higher match counts, + // and thus, throughput is perhaps not as important. But if you have a use + // case for something faster, feel free to file an issue. + cache.search_start(state.at); + while state.at < input.end() { + sid = dfa + .next_state(cache, sid, input.haystack()[state.at]) + .map_err(|_| gave_up(state.at))?; + if sid.is_tagged() { + state.id = Some(sid); + if sid.is_start() { + if let Some(ref pre) = pre { + let span = Span::from(state.at..input.end()); + match pre.find(input.haystack(), span) { + None => return Ok(()), + Some(ref span) => { + if span.start > state.at { + state.at = span.start; + if !universal_start { + sid = prefilter_restart( + dfa, cache, &input, state.at, + )?; + } + continue; + } + } + } + } + } else if sid.is_match() { + state.next_match_index = Some(1); + let pattern = dfa.match_pattern(cache, sid, 0); + state.mat = Some(HalfMatch::new(pattern, state.at)); + cache.search_finish(state.at); + return Ok(()); + } else if sid.is_dead() { + cache.search_finish(state.at); + return Ok(()); + } else if sid.is_quit() { + cache.search_finish(state.at); + return Err(MatchError::quit( + input.haystack()[state.at], + state.at, + )); + } else { + debug_assert!(sid.is_unknown()); + unreachable!("sid being unknown is a bug"); + } + } + state.at += 1; + cache.search_update(state.at); + } + + let result = eoi_fwd(dfa, cache, input, &mut sid, &mut state.mat); + state.id = Some(sid); + if state.mat.is_some() { + // '1' is always correct here since if we get to this point, this + // always corresponds to the first (index '0') match discovered at + // this position. So the next match to report at this position (if + // it exists) is at index '1'. + state.next_match_index = Some(1); + } + cache.search_finish(input.end()); + result +} + +#[inline(never)] +pub(crate) fn find_overlapping_rev( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, + state: &mut OverlappingState, +) -> Result<(), MatchError> { + state.mat = None; + if input.is_done() { + return Ok(()); + } + let mut sid = match state.id { + None => { + let sid = init_rev(dfa, cache, input)?; + state.id = Some(sid); + if input.start() == input.end() { + state.rev_eoi = true; + } else { + state.at = input.end() - 1; + } + sid + } + Some(sid) => { + if let Some(match_index) = state.next_match_index { + let match_len = dfa.match_len(cache, sid); + if match_index < match_len { + state.next_match_index = Some(match_index + 1); + let pattern = dfa.match_pattern(cache, sid, match_index); + state.mat = Some(HalfMatch::new(pattern, state.at)); + return Ok(()); + } + } + // Once we've reported all matches at a given position, we need + // to advance the search to the next position. However, if we've + // already followed the EOI transition, then we know we're done + // with the search and there cannot be any more matches to report. + if state.rev_eoi { + return Ok(()); + } else if state.at == input.start() { + // At this point, we should follow the EOI transition. This + // will cause us the skip the main loop below and fall through + // to the final 'eoi_rev' transition. + state.rev_eoi = true; + } else { + // We haven't hit the end of the search yet, so move on. + state.at -= 1; + } + sid + } + }; + cache.search_start(state.at); + while !state.rev_eoi { + sid = dfa + .next_state(cache, sid, input.haystack()[state.at]) + .map_err(|_| gave_up(state.at))?; + if sid.is_tagged() { + state.id = Some(sid); + if sid.is_start() { + // do nothing + } else if sid.is_match() { + state.next_match_index = Some(1); + let pattern = dfa.match_pattern(cache, sid, 0); + state.mat = Some(HalfMatch::new(pattern, state.at + 1)); + cache.search_finish(state.at); + return Ok(()); + } else if sid.is_dead() { + cache.search_finish(state.at); + return Ok(()); + } else if sid.is_quit() { + cache.search_finish(state.at); + return Err(MatchError::quit( + input.haystack()[state.at], + state.at, + )); + } else { + debug_assert!(sid.is_unknown()); + unreachable!("sid being unknown is a bug"); + } + } + if state.at == input.start() { + break; + } + state.at -= 1; + cache.search_update(state.at); + } + + let result = eoi_rev(dfa, cache, input, &mut sid, &mut state.mat); + state.rev_eoi = true; + state.id = Some(sid); + if state.mat.is_some() { + // '1' is always correct here since if we get to this point, this + // always corresponds to the first (index '0') match discovered at + // this position. So the next match to report at this position (if + // it exists) is at index '1'. + state.next_match_index = Some(1); + } + cache.search_finish(input.start()); + result +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn init_fwd( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, +) -> Result<LazyStateID, MatchError> { + let sid = dfa.start_state_forward(cache, input)?; + // Start states can never be match states, since all matches are delayed + // by 1 byte. + debug_assert!(!sid.is_match()); + Ok(sid) +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn init_rev( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, +) -> Result<LazyStateID, MatchError> { + let sid = dfa.start_state_reverse(cache, input)?; + // Start states can never be match states, since all matches are delayed + // by 1 byte. + debug_assert!(!sid.is_match()); + Ok(sid) +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn eoi_fwd( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, + sid: &mut LazyStateID, + mat: &mut Option<HalfMatch>, +) -> Result<(), MatchError> { + let sp = input.get_span(); + match input.haystack().get(sp.end) { + Some(&b) => { + *sid = + dfa.next_state(cache, *sid, b).map_err(|_| gave_up(sp.end))?; + if sid.is_match() { + let pattern = dfa.match_pattern(cache, *sid, 0); + *mat = Some(HalfMatch::new(pattern, sp.end)); + } else if sid.is_quit() { + return Err(MatchError::quit(b, sp.end)); + } + } + None => { + *sid = dfa + .next_eoi_state(cache, *sid) + .map_err(|_| gave_up(input.haystack().len()))?; + if sid.is_match() { + let pattern = dfa.match_pattern(cache, *sid, 0); + *mat = Some(HalfMatch::new(pattern, input.haystack().len())); + } + // N.B. We don't have to check 'is_quit' here because the EOI + // transition can never lead to a quit state. + debug_assert!(!sid.is_quit()); + } + } + Ok(()) +} + +#[cfg_attr(feature = "perf-inline", inline(always))] +fn eoi_rev( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, + sid: &mut LazyStateID, + mat: &mut Option<HalfMatch>, +) -> Result<(), MatchError> { + let sp = input.get_span(); + if sp.start > 0 { + let byte = input.haystack()[sp.start - 1]; + *sid = dfa + .next_state(cache, *sid, byte) + .map_err(|_| gave_up(sp.start))?; + if sid.is_match() { + let pattern = dfa.match_pattern(cache, *sid, 0); + *mat = Some(HalfMatch::new(pattern, sp.start)); + } else if sid.is_quit() { + return Err(MatchError::quit(byte, sp.start - 1)); + } + } else { + *sid = + dfa.next_eoi_state(cache, *sid).map_err(|_| gave_up(sp.start))?; + if sid.is_match() { + let pattern = dfa.match_pattern(cache, *sid, 0); + *mat = Some(HalfMatch::new(pattern, 0)); + } + // N.B. We don't have to check 'is_quit' here because the EOI + // transition can never lead to a quit state. + debug_assert!(!sid.is_quit()); + } + Ok(()) +} + +/// Re-compute the starting state that a DFA should be in after finding a +/// prefilter candidate match at the position `at`. +/// +/// It is always correct to call this, but not always necessary. Namely, +/// whenever the DFA has a universal start state, the DFA can remain in the +/// start state that it was in when it ran the prefilter. Why? Because in that +/// case, there is only one start state. +/// +/// When does a DFA have a universal start state? In precisely cases where +/// it has no look-around assertions in its prefix. So for example, `\bfoo` +/// does not have a universal start state because the start state depends on +/// whether the byte immediately before the start position is a word byte or +/// not. However, `foo\b` does have a universal start state because the word +/// boundary does not appear in the pattern's prefix. +/// +/// So... most cases don't need this, but when a pattern doesn't have a +/// universal start state, then after a prefilter candidate has been found, the +/// current state *must* be re-litigated as if computing the start state at the +/// beginning of the search because it might change. That is, not all start +/// states are created equal. +/// +/// Why avoid it? Because while it's not super expensive, it isn't a trivial +/// operation to compute the start state. It is much better to avoid it and +/// just state in the current state if you know it to be correct. +#[cfg_attr(feature = "perf-inline", inline(always))] +fn prefilter_restart( + dfa: &DFA, + cache: &mut Cache, + input: &Input<'_>, + at: usize, +) -> Result<LazyStateID, MatchError> { + let mut input = input.clone(); + input.set_start(at); + init_fwd(dfa, cache, &input) +} + +/// A convenience routine for constructing a "gave up" match error. +#[cfg_attr(feature = "perf-inline", inline(always))] +fn gave_up(offset: usize) -> MatchError { + MatchError::gave_up(offset) +} diff --git a/third_party/crates/regex-automata-0.4.18/src/lib.rs b/third_party/crates/regex-automata-0.4.18/src/lib.rs new file mode 100644 index 0000000..b29f618 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/lib.rs @@ -0,0 +1,651 @@ +/*! +This crate exposes a variety of regex engines used by the `regex` crate. +It provides a vast, sprawling and "expert" level API to each regex engine. +The regex engines provided by this crate focus heavily on finite automata +implementations and specifically guarantee worst case `O(m * n)` time +complexity for all searches. (Where `m ~ len(regex)` and `n ~ len(haystack)`.) + +The primary goal of this crate is to serve as an implementation detail for the +`regex` crate. A secondary goal is to make its internals available for use by +others. + +# Table of contents + +* [Should I be using this crate?](#should-i-be-using-this-crate) gives some +reasons for and against using this crate. +* [Examples](#examples) provides a small selection of things you can do with +this crate. +* [Available regex engines](#available-regex-engines) provides a hyperlinked +list of all regex engines in this crate. +* [API themes](#api-themes) discusses common elements used throughout this +crate. +* [Crate features](#crate-features) documents the extensive list of Cargo +features available. + +# Should I be using this crate? + +If you find yourself here because you just want to use regexes, then you should +first check out whether the [`regex` crate](https://docs.rs/regex) meets +your needs. It provides a streamlined and difficult-to-misuse API for regex +searching. + +If you're here because there is something specific you want to do that can't +be easily done with `regex` crate, then you are perhaps in the right place. +It's most likely that the first stop you'll want to make is to explore the +[`meta` regex APIs](meta). Namely, the `regex` crate is just a light wrapper +over a [`meta::Regex`], so its API will probably be the easiest to transition +to. In contrast to the `regex` crate, the `meta::Regex` API supports more +search parameters and does multi-pattern searches. However, it isn't quite as +ergonomic. + +Otherwise, the following is an inexhaustive list of reasons to use this crate: + +* You want to analyze or use a [Thompson `NFA`](nfa::thompson::NFA) directly. +* You want more powerful multi-pattern search than what is provided by +`RegexSet` in the `regex` crate. All regex engines in this crate support +multi-pattern searches. +* You want to use one of the `regex` crate's internal engines directly because +of some interesting configuration that isn't possible via the `regex` crate. +For example, a [lazy DFA's configuration](hybrid::dfa::Config) exposes a +dizzying number of options for controlling its execution. +* You want to use the lower level search APIs. For example, both the [lazy +DFA](hybrid::dfa) and [fully compiled DFAs](dfa) support searching by exploring +the automaton one state at a time. This might be useful, for example, for +stream searches or searches of strings stored in non-contiguous in memory. +* You want to build a fully compiled DFA and then [use zero-copy +deserialization](dfa::dense::DFA::from_bytes) to load it into memory and use +it for searching. This use case is supported in core-only no-std/no-alloc +environments. +* You want to run [anchored searches](Input::anchored) without using the `^` +anchor in your regex pattern. +* You need to work-around contention issues with +sharing a regex across multiple threads. The +[`meta::Regex::search_with`](meta::Regex::search_with) API permits bypassing +any kind of synchronization at all by requiring the caller to provide the +mutable scratch spaced needed during a search. +* You want to build your own regex engine on top of the `regex` crate's +infrastructure. + +# Examples + +This section tries to identify a few interesting things you can do with this +crate and demonstrates them. + +### Multi-pattern searches with capture groups + +One of the more frustrating limitations of `RegexSet` in the `regex` crate +(at the time of writing) is that it doesn't report match positions. With this +crate, multi-pattern support was intentionally designed in from the beginning, +which means it works in all regex engines and even for capture groups as well. + +This example shows how to search for matches of multiple regexes, where each +regex uses the same capture group names to parse different key-value formats. + +``` +use regex_automata::{meta::Regex, PatternID}; + +let re = Regex::new_many(&[ + r#"(?m)^(?<key>[[:word:]]+)=(?<val>[[:word:]]+)$"#, + r#"(?m)^(?<key>[[:word:]]+)="(?<val>[^"]+)"$"#, + r#"(?m)^(?<key>[[:word:]]+)='(?<val>[^']+)'$"#, + r#"(?m)^(?<key>[[:word:]]+):\s*(?<val>[[:word:]]+)$"#, +])?; +let hay = r#" +best_album="Blow Your Face Out" +best_quote='"then as it was, then again it will be"' +best_year=1973 +best_simpsons_episode: HOMR +"#; +let mut kvs = vec![]; +for caps in re.captures_iter(hay) { + // N.B. One could use capture indices '1' and '2' here + // as well. Capture indices are local to each pattern. + // (Just like names are.) + let key = &hay[caps.get_group_by_name("key").unwrap()]; + let val = &hay[caps.get_group_by_name("val").unwrap()]; + kvs.push((key, val)); +} +assert_eq!(kvs, vec![ + ("best_album", "Blow Your Face Out"), + ("best_quote", "\"then as it was, then again it will be\""), + ("best_year", "1973"), + ("best_simpsons_episode", "HOMR"), +]); + +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +### Build a full DFA and walk it manually + +One of the regex engines in this crate is a fully compiled DFA. It takes worst +case exponential time to build, but once built, it can be easily explored and +used for searches. Here's a simple example that uses its lower level APIs to +implement a simple anchored search by hand. + +``` +use regex_automata::{dfa::{Automaton, dense}, Input}; + +let dfa = dense::DFA::new(r"(?-u)\b[A-Z]\w+z\b")?; +let haystack = "Quartz"; + +// The start state is determined by inspecting the position and the +// initial bytes of the haystack. +let mut state = dfa.start_state_forward(&Input::new(haystack))?; +// Walk all the bytes in the haystack. +for &b in haystack.as_bytes().iter() { + state = dfa.next_state(state, b); +} +// DFAs in this crate require an explicit +// end-of-input transition if a search reaches +// the end of a haystack. +state = dfa.next_eoi_state(state); +assert!(dfa.is_match_state(state)); + +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +Or do the same with a lazy DFA that avoids exponential worst case compile time, +but requires mutable scratch space to lazily build the DFA during the search. + +``` +use regex_automata::{hybrid::dfa::DFA, Input}; + +let dfa = DFA::new(r"(?-u)\b[A-Z]\w+z\b")?; +let mut cache = dfa.create_cache(); +let hay = "Quartz"; + +// The start state is determined by inspecting the position and the +// initial bytes of the haystack. +let mut state = dfa.start_state_forward(&mut cache, &Input::new(hay))?; +// Walk all the bytes in the haystack. +for &b in hay.as_bytes().iter() { + state = dfa.next_state(&mut cache, state, b)?; +} +// DFAs in this crate require an explicit +// end-of-input transition if a search reaches +// the end of a haystack. +state = dfa.next_eoi_state(&mut cache, state)?; +assert!(state.is_match()); + +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +### Find all overlapping matches + +This example shows how to build a DFA and use it to find all possible matches, +including overlapping matches. A similar example will work with a lazy DFA as +well. This also works with multiple patterns and will report all matches at the +same position where multiple patterns match. + +``` +use regex_automata::{ + dfa::{dense, Automaton, OverlappingState}, + Input, MatchKind, +}; + +let dfa = dense::DFA::builder() + .configure(dense::DFA::config().match_kind(MatchKind::All)) + .build(r"(?-u)\w{3,}")?; +let input = Input::new("homer marge bart lisa maggie"); +let mut state = OverlappingState::start(); + +let mut matches = vec![]; +while let Some(hm) = { + dfa.try_search_overlapping_fwd(&input, &mut state)?; + state.get_match() +} { + matches.push(hm.offset()); +} +assert_eq!(matches, vec![ + 3, 4, 5, // hom, home, homer + 9, 10, 11, // mar, marg, marge + 15, 16, // bar, bart + 20, 21, // lis, lisa + 25, 26, 27, 28, // mag, magg, maggi, maggie +]); + +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + +# Available regex engines + +The following is a complete list of all regex engines provided by this crate, +along with a very brief description of it and why you might want to use it. + +* [`dfa::regex::Regex`] is a regex engine that works on top of either +[dense](dfa::dense) or [sparse](dfa::sparse) fully compiled DFAs. You might +use a DFA if you need the fastest possible regex engine in this crate and can +afford the exorbitant memory usage usually required by DFAs. Low level APIs on +fully compiled DFAs are provided by the [`Automaton` trait](dfa::Automaton). +Fully compiled dense DFAs can handle all regexes except for searching a regex +with a Unicode word boundary on non-ASCII haystacks. A fully compiled DFA based +regex can only report the start and end of each match. +* [`hybrid::regex::Regex`] is a regex engine that works on top of a lazily +built DFA. Its performance profile is very similar to that of fully compiled +DFAs, but can be slower in some pathological cases. Fully compiled DFAs are +also amenable to more optimizations, such as state acceleration, that aren't +available in a lazy DFA. You might use this lazy DFA if you can't abide the +worst case exponential compile time of a full DFA, but still want the DFA +search performance in the vast majority of cases. A lazy DFA based regex can +only report the start and end of each match. +* [`dfa::onepass::DFA`] is a regex engine that is implemented as a DFA, but +can report the matches of each capture group in addition to the start and end +of each match. The catch is that it only works on a somewhat small subset of +regexes known as "one-pass." You'll want to use this for cases when you need +capture group matches and the regex is one-pass since it is likely to be faster +than any alternative. A one-pass DFA can handle all types of regexes, but does +have some reasonable limits on the number of capture groups it can handle. +* [`nfa::thompson::backtrack::BoundedBacktracker`] is a regex engine that uses +backtracking, but keeps track of the work it has done to avoid catastrophic +backtracking. Like the one-pass DFA, it provides the matches of each capture +group. It retains the `O(m * n)` worst case time bound. This tends to be slower +than the one-pass DFA regex engine, but faster than the PikeVM. It can handle +all types of regexes, but usually only works well with small haystacks and +small regexes due to the memory required to avoid redoing work. +* [`nfa::thompson::pikevm::PikeVM`] is a regex engine that can handle all +regexes, of all sizes and provides capture group matches. It tends to be a tool +of last resort because it is also usually the slowest regex engine. +* [`meta::Regex`] is the meta regex engine that combines *all* of the above +engines into one. The reason for this is that each of the engines above have +their own caveats such as, "only handles a subset of regexes" or "is generally +slow." The meta regex engine accounts for all of these caveats and composes +the engines in a way that attempts to mitigate each engine's weaknesses while +emphasizing its strengths. For example, it will attempt to run a lazy DFA even +if it might fail. In which case, it will restart the search with a likely +slower but more capable regex engine. The meta regex engine is what you should +default to. Use one of the above engines directly only if you have a specific +reason to. + +# API themes + +While each regex engine has its own APIs and configuration options, there are +some general themes followed by all of them. + +### The `Input` abstraction + +Most search routines in this crate accept anything that implements +`Into<Input>`. Both `&str` and `&[u8]` haystacks satisfy this constraint, which +means that things like `engine.search("foo")` will work as you would expect. + +By virtue of accepting an `Into<Input>` though, callers can provide more than +just a haystack. Indeed, the [`Input`] type has more details, but briefly, +callers can use it to configure various aspects of the search: + +* The span of the haystack to search via [`Input::span`] or [`Input::range`], +which might be a substring of the haystack. +* Whether to run an anchored search or not via [`Input::anchored`]. This +permits one to require matches to start at the same offset that the search +started. +* Whether to ask the regex engine to stop as soon as a match is seen via +[`Input::earliest`]. This can be used to find the offset of a match as soon +as it is known without waiting for the full leftmost-first match to be found. +This can also be used to avoid the worst case `O(m * n^2)` time complexity +of iteration. + +Some lower level search routines accept an `&Input` for performance reasons. +In which case, `&Input::new("haystack")` can be used for a simple search. + +### Error reporting + +Most, but not all, regex engines in this crate can fail to execute a search. +When a search fails, callers cannot determine whether or not a match exists. +That is, the result is indeterminate. + +Search failure, in all cases in this crate, is represented by a [`MatchError`]. +Routines that can fail start with the `try_` prefix in their name. For example, +[`hybrid::regex::Regex::try_search`] can fail for a number of reasons. +Conversely, routines that either can't fail or can panic on failure lack the +`try_` prefix. For example, [`hybrid::regex::Regex::find`] will panic in +cases where [`hybrid::regex::Regex::try_search`] would return an error, and +[`meta::Regex::find`] will never panic. Therefore, callers need to pay close +attention to the panicking conditions in the documentation. + +In most cases, the reasons that a search fails are either predictable or +configurable, albeit at some additional cost. + +An example of predictable failure is +[`BoundedBacktracker::try_search`](nfa::thompson::backtrack::BoundedBacktracker::try_search). +Namely, it fails whenever the multiplication of the haystack, the regex and some +constant exceeds the +[configured visited capacity](nfa::thompson::backtrack::Config::visited_capacity). +Callers can predict the failure in terms of haystack length via the +[`BoundedBacktracker::max_haystack_len`](nfa::thompson::backtrack::BoundedBacktracker::max_haystack_len) +method. While this form of failure is technically avoidable by increasing the +visited capacity, it isn't practical to do so for all inputs because the +memory usage required for larger haystacks becomes impractically large. So in +practice, if one is using the bounded backtracker, you really do have to deal +with the failure. + +An example of configurable failure happens when one enables heuristic support +for Unicode word boundaries in a DFA. Namely, since the DFAs in this crate +(except for the one-pass DFA) do not support Unicode word boundaries on +non-ASCII haystacks, building a DFA from an NFA that contains a Unicode word +boundary will itself fail. However, one can configure DFAs to still be built in +this case by +[configuring heuristic support for Unicode word boundaries](hybrid::dfa::Config::unicode_word_boundary). +If the NFA the DFA is built from contains a Unicode word boundary, then the +DFA will still be built, but special transitions will be added to every state +that cause the DFA to fail if any non-ASCII byte is seen. This failure happens +at search time and it requires the caller to opt into this. + +There are other ways for regex engines to fail in this crate, but the above +two should represent the general theme of failures one can find. Dealing +with these failures is, in part, one the responsibilities of the [meta regex +engine](meta). Notice, for example, that the meta regex engine exposes an API +that never returns an error nor panics. It carefully manages all of the ways +in which the regex engines can fail and either avoids the predictable ones +entirely (e.g., the bounded backtracker) or reacts to configured failures by +falling back to a different engine (e.g., the lazy DFA quitting because it saw +a non-ASCII byte). + +### Configuration and Builders + +Most of the regex engines in this crate come with two types to facilitate +building the regex engine: a `Config` and a `Builder`. A `Config` is usually +specific to that particular regex engine, but other objects such as parsing and +NFA compilation have `Config` types too. A `Builder` is the thing responsible +for taking inputs (either pattern strings or already-parsed patterns or even +NFAs directly) and turning them into an actual regex engine that can be used +for searching. + +The main reason why building a regex engine is a bit complicated is because +of the desire to permit composition with de-coupled components. For example, +you might want to [manually construct a Thompson NFA](nfa::thompson::Builder) +and then build a regex engine from it without ever using a regex parser +at all. On the other hand, you might also want to build a regex engine directly +from the concrete syntax. This demonstrates why regex engine construction is +so flexible: it needs to support not just convenient construction, but also +construction from parts built elsewhere. + +This is also in turn why there are many different `Config` structs in this +crate. Let's look more closely at an example: [`hybrid::regex::Builder`]. It +accepts three different `Config` types for configuring construction of a lazy +DFA regex: + +* [`hybrid::regex::Builder::syntax`] accepts a +[`util::syntax::Config`] for configuring the options found in the +[`regex-syntax`](regex_syntax) crate. For example, whether to match +case insensitively. +* [`hybrid::regex::Builder::thompson`] accepts a [`nfa::thompson::Config`] for +configuring construction of a [Thompson NFA](nfa::thompson::NFA). For example, +whether to build an NFA that matches the reverse language described by the +regex. +* [`hybrid::regex::Builder::dfa`] accept a [`hybrid::dfa::Config`] for +configuring construction of the pair of underlying lazy DFAs that make up the +lazy DFA regex engine. For example, changing the capacity of the cache used to +store the transition table. + +The lazy DFA regex engine uses all three of those configuration objects for +methods like [`hybrid::regex::Builder::build`], which accepts a pattern +string containing the concrete syntax of your regex. It uses the syntax +configuration to parse it into an AST and translate it into an HIR. Then the +NFA configuration when compiling the HIR into an NFA. And then finally the DFA +configuration when lazily determinizing the NFA into a DFA. + +Notice though that the builder also has a +[`hybrid::regex::Builder::build_from_dfas`] constructor. This permits callers +to build the underlying pair of lazy DFAs themselves (one for the forward +searching to find the end of a match and one for the reverse searching to find +the start of a match), and then build the regex engine from them. The lazy +DFAs, in turn, have their own builder that permits [construction directly from +a Thompson NFA](hybrid::dfa::Builder::build_from_nfa). Continuing down the +rabbit hole, a Thompson NFA has its own compiler that permits [construction +directly from an HIR](nfa::thompson::Compiler::build_from_hir). The lazy DFA +regex engine builder lets you follow this rabbit hole all the way down, but +also provides convenience routines that do it for you when you don't need +precise control over every component. + +The [meta regex engine](meta) is a good example of something that utilizes the +full flexibility of these builders. It often needs not only precise control +over each component, but also shares them across multiple regex engines. +(Most sharing is done by internal reference accounting. For example, an +[`NFA`](nfa::thompson::NFA) is reference counted internally which makes cloning +cheap.) + +### Size limits + +Unlike the `regex` crate, the `regex-automata` crate specifically does not +enable any size limits by default. That means users of this crate need to +be quite careful when using untrusted patterns. Namely, because bounded +repetitions can grow exponentially by stacking them, it is possible to build a +very large internal regex object from just a small pattern string. For example, +the NFA built from the pattern `a{10}{10}{10}{10}{10}{10}{10}` is over 240MB. + +There are multiple size limit options in this crate. If one or more size limits +are relevant for the object you're building, they will be configurable via +methods on a corresponding `Config` type. + +# Crate features + +This crate has a dizzying number of features. The main idea is to be able to +control how much stuff you pull in for your specific use case, since the full +crate is quite large and can dramatically increase compile times and binary +size. + +The most barebones but useful configuration is to disable all default features +and enable only `dfa-search`. This will bring in just the DFA deserialization +and search routines without any dependency on `std` or `alloc`. This does +require generating and serializing a DFA, and then storing it somewhere, but +it permits regex searches in freestanding or embedded environments. + +Because there are so many features, they are split into a few groups. + +The default set of features is: `std`, `syntax`, `perf`, `unicode`, `meta`, +`nfa`, `dfa` and `hybrid`. Basically, the default is to enable everything +except for development related features like `logging`. + +### Ecosystem features + +* **std** - Enables use of the standard library. In terms of APIs, this usually +just means that error types implement the `std::error::Error` trait. Otherwise, +`std` sometimes enables the code to be faster, for example, using a `HashMap` +instead of a `BTreeMap`. (The `std` feature matters more for dependencies like +`aho-corasick` and `memchr`, where `std` is required to enable certain classes +of SIMD optimizations.) Enabling `std` automatically enables `alloc`. +* **alloc** - Enables use of the `alloc` library. This is required for most +APIs in this crate. The main exception is deserializing and searching with +fully compiled DFAs. +* **logging** - Adds a dependency on the `log` crate and makes this crate emit +log messages of varying degrees of utility. The log messages are especially +useful in trying to understand what the meta regex engine is doing. + +### Performance features + +**Note**: + To get performance benefits offered by the SIMD, `std` must be enabled. + None of the `perf-*` features will enable `std` implicitly. + +* **perf** - Enables all of the below features. +* **perf-inline** - When enabled, `inline(always)` is used in (many) strategic +locations to help performance at the expense of longer compile times and +increased binary size. +* **perf-literal** - Enables all literal related optimizations. + * **perf-literal-substring** - Enables all single substring literal + optimizations. This includes adding a dependency on the `memchr` crate. + * **perf-literal-multisubstring** - Enables all multiple substring literal + optimizations. This includes adding a dependency on the `aho-corasick` + crate. + +### Unicode features + +* **unicode** - + Enables all Unicode features. This feature is enabled by default, and will + always cover all Unicode features, even if more are added in the future. +* **unicode-age** - + Provide the data for the + [Unicode `Age` property](https://www.unicode.org/reports/tr44/tr44-24.html#Character_Age). + This makes it possible to use classes like `\p{Age:6.0}` to refer to all + codepoints first introduced in Unicode 6.0 +* **unicode-bool** - + Provide the data for numerous Unicode boolean properties. The full list + is not included here, but contains properties like `Alphabetic`, `Emoji`, + `Lowercase`, `Math`, `Uppercase` and `White_Space`. +* **unicode-case** - + Provide the data for case insensitive matching using + [Unicode's "simple loose matches" specification](https://www.unicode.org/reports/tr18/#Simple_Loose_Matches). +* **unicode-gencat** - + Provide the data for + [Unicode general categories](https://www.unicode.org/reports/tr44/tr44-24.html#General_Category_Values). + This includes, but is not limited to, `Decimal_Number`, `Letter`, + `Math_Symbol`, `Number` and `Punctuation`. +* **unicode-perl** - + Provide the data for supporting the Unicode-aware Perl character classes, + corresponding to `\w`, `\s` and `\d`. This is also necessary for using + Unicode-aware word boundary assertions. Note that if this feature is + disabled, the `\s` and `\d` character classes are still available if the + `unicode-bool` and `unicode-gencat` features are enabled, respectively. +* **unicode-script** - + Provide the data for + [Unicode scripts and script extensions](https://www.unicode.org/reports/tr24/). + This includes, but is not limited to, `Arabic`, `Cyrillic`, `Hebrew`, + `Latin` and `Thai`. +* **unicode-segment** - + Provide the data necessary to provide the properties used to implement the + [Unicode text segmentation algorithms](https://www.unicode.org/reports/tr29/). + This enables using classes like `\p{gcb=Extend}`, `\p{wb=Katakana}` and + `\p{sb=ATerm}`. +* **unicode-word-boundary** - + Enables support for Unicode word boundaries, i.e., `\b`, in regexes. When + this and `unicode-perl` are enabled, then data tables from `regex-syntax` are + used to implement Unicode word boundaries. However, if `regex-syntax` isn't + enabled as a dependency then one can still enable this feature. It will + cause `regex-automata` to bundle its own data table that would otherwise be + redundant with `regex-syntax`'s table. + +### Regex engine features + +* **syntax** - Enables a dependency on `regex-syntax`. This makes APIs +for building regex engines from pattern strings available. Without the +`regex-syntax` dependency, the only way to build a regex engine is generally +to deserialize a previously built DFA or to hand assemble an NFA using its +[builder API](nfa::thompson::Builder). Once you have an NFA, you can build any +of the regex engines in this crate. The `syntax` feature also enables `alloc`. +* **meta** - Enables the meta regex engine. This also enables the `syntax` and +`nfa-pikevm` features, as both are the minimal requirements needed. The meta +regex engine benefits from enabling any of the other regex engines and will +use them automatically when appropriate. +* **nfa** - Enables all NFA related features below. + * **nfa-thompson** - Enables the Thompson NFA APIs. This enables `alloc`. + * **nfa-pikevm** - Enables the PikeVM regex engine. This enables + `nfa-thompson`. + * **nfa-backtrack** - Enables the bounded backtracker regex engine. This + enables `nfa-thompson`. +* **dfa** - Enables all DFA related features below. + * **dfa-build** - Enables APIs for determinizing DFAs from NFAs. This + enables `nfa-thompson` and `dfa-search`. + * **dfa-search** - Enables APIs for searching with DFAs. + * **dfa-onepass** - Enables the one-pass DFA API. This enables + `nfa-thompson`. +* **hybrid** - Enables the hybrid NFA/DFA or "lazy DFA" regex engine. This +enables `alloc` and `nfa-thompson`. + +*/ + +// We are no_std. +#![no_std] +// All APIs need docs! +#![deny(missing_docs)] +// Some intra-doc links are broken when certain features are disabled, so we +// only bleat about it when most (all?) features are enabled. But when we do, +// we block the build. Links need to work. +#![cfg_attr( + all( + feature = "std", + feature = "nfa", + feature = "dfa", + feature = "hybrid" + ), + deny(rustdoc::broken_intra_doc_links) +)] +// Broken rustdoc links are very easy to come by when you start disabling +// features. Namely, features tend to change imports, and imports change what's +// available to link to. +// +// Basically, we just don't support rustdoc for anything other than the maximal +// feature configuration. Other configurations will work, they just won't be +// perfect. +// +// So here, we specifically allow them so we don't even get warned about them. +#![cfg_attr( + not(all( + feature = "std", + feature = "nfa", + feature = "dfa", + feature = "hybrid" + )), + allow(rustdoc::broken_intra_doc_links) +)] +// Kinda similar, but eliminating all of the dead code and unused import +// warnings for every feature combo is a fool's errand. Instead, we just +// suppress those, but still let them through in a common configuration when we +// build most of everything. +// +// This does actually suggest that when features are disabled, we are actually +// compiling more code than we need to be. And this is perhaps not so great +// because disabling features is usually done in order to reduce compile times +// by reducing the amount of code one compiles... However, usually, most of the +// time this dead code is a relatively small amount from the 'util' module. +// But... I confess... There isn't a ton of visibility on this. +// +// I'm happy to try to address this in a different way, but "let's annotate +// every function in 'util' with some non-local combination of features" just +// cannot be the way forward. +#![cfg_attr( + not(all( + feature = "std", + feature = "nfa", + feature = "dfa", + feature = "hybrid", + feature = "perf-literal-substring", + feature = "perf-literal-multisubstring", + )), + allow(dead_code, unused_imports, unused_variables) +)] +// We generally want all types to impl Debug. +#![warn(missing_debug_implementations)] +// This adds Cargo feature annotations to items in the rustdoc output. Which is +// sadly hugely beneficial for this crate due to the number of features. +#![cfg_attr(docsrs_regex, feature(doc_cfg))] + +// I have literally never tested this crate on 16-bit, so it is quite +// suspicious to advertise support for it. But... the regex crate, at time +// of writing, at least claims to support it by not doing any conditional +// compilation based on the target pointer width. So I guess I remain +// consistent with that here. +// +// If you are here because you're on a 16-bit system and you were somehow using +// the regex crate previously, please file an issue. Please be prepared to +// provide some kind of reproduction or carve out some path to getting 16-bit +// working in CI. (Via qemu?) +#[cfg(not(any( + target_pointer_width = "16", + target_pointer_width = "32", + target_pointer_width = "64" +)))] +compile_error!("not supported on non-{16,32,64}, please file an issue"); + +#[cfg(any(test, feature = "std"))] +extern crate std; + +#[cfg(feature = "alloc")] +extern crate alloc; + +#[cfg(doctest)] +doc_comment::doctest!("../README.md"); + +#[doc(inline)] +pub use crate::util::primitives::PatternID; +pub use crate::util::search::*; + +#[macro_use] +mod macros; + +#[cfg(any(feature = "dfa-search", feature = "dfa-onepass"))] +pub mod dfa; +#[cfg(feature = "hybrid")] +pub mod hybrid; +#[cfg(feature = "meta")] +pub mod meta; +#[cfg(feature = "nfa-thompson")] +pub mod nfa; +pub mod util; diff --git a/third_party/crates/regex-automata-0.4.18/src/macros.rs b/third_party/crates/regex-automata-0.4.18/src/macros.rs new file mode 100644 index 0000000..31b4ca3 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/macros.rs @@ -0,0 +1,20 @@ +// Some feature combinations result in some of these macros never being used. +// Which is fine. Just squash the warnings. +#![allow(unused_macros)] + +macro_rules! log { + ($($tt:tt)*) => { + #[cfg(feature = "logging")] + { + $($tt)* + } + } +} + +macro_rules! debug { + ($($tt:tt)*) => { log!(log::debug!($($tt)*)) } +} + +macro_rules! trace { + ($($tt:tt)*) => { log!(log::trace!($($tt)*)) } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/error.rs b/third_party/crates/regex-automata-0.4.18/src/meta/error.rs new file mode 100644 index 0000000..9ead729 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/error.rs @@ -0,0 +1,241 @@ +use regex_syntax::{ast, hir}; + +use crate::{nfa, util::search::MatchError, PatternID}; + +/// An error that occurs when construction of a `Regex` fails. +/// +/// A build error is generally a result of one of two possible failure +/// modes. First is a parse or syntax error in the concrete syntax of a +/// pattern. Second is that the construction of the underlying regex matcher +/// fails, usually because it gets too big with respect to limits like +/// [`Config::nfa_size_limit`](crate::meta::Config::nfa_size_limit). +/// +/// This error provides very little introspection capabilities. You can: +/// +/// * Ask for the [`PatternID`] of the pattern that caused an error, if one +/// is available. This is available for things like syntax errors, but not for +/// cases where build limits are exceeded. +/// * Ask for the underlying syntax error, but only if the error is a syntax +/// error. +/// * Ask for a human readable message corresponding to the underlying error. +/// * The `BuildError::source` method (from the `std::error::Error` +/// trait implementation) may be used to query for an underlying error if one +/// exists. There are no API guarantees about which error is returned. +/// +/// When the `std` feature is enabled, this implements `std::error::Error`. +#[derive(Clone, Debug)] +pub struct BuildError { + kind: BuildErrorKind, +} + +#[derive(Clone, Debug)] +enum BuildErrorKind { + Syntax { pid: PatternID, err: regex_syntax::Error }, + NFA(nfa::thompson::BuildError), +} + +impl BuildError { + /// If it is known which pattern ID caused this build error to occur, then + /// this method returns it. + /// + /// Some errors are not associated with a particular pattern. However, any + /// errors that occur as part of parsing a pattern are guaranteed to be + /// associated with a pattern ID. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, PatternID}; + /// + /// let err = Regex::new_many(&["a", "b", r"\p{Foo}", "c"]).unwrap_err(); + /// assert_eq!(Some(PatternID::must(2)), err.pattern()); + /// ``` + pub fn pattern(&self) -> Option<PatternID> { + match self.kind { + BuildErrorKind::Syntax { pid, .. } => Some(pid), + _ => None, + } + } + + /// If this error occurred because the regex exceeded the configured size + /// limit before being built, then this returns the configured size limit. + /// + /// The limit returned is what was configured, and corresponds to the + /// maximum amount of heap usage in bytes. + pub fn size_limit(&self) -> Option<usize> { + match self.kind { + BuildErrorKind::NFA(ref err) => err.size_limit(), + _ => None, + } + } + + /// If this error corresponds to a syntax error, then a reference to it is + /// returned by this method. + pub fn syntax_error(&self) -> Option<®ex_syntax::Error> { + match self.kind { + BuildErrorKind::Syntax { ref err, .. } => Some(err), + _ => None, + } + } + + pub(crate) fn ast(pid: PatternID, err: ast::Error) -> BuildError { + let err = regex_syntax::Error::from(err); + BuildError { kind: BuildErrorKind::Syntax { pid, err } } + } + + pub(crate) fn hir(pid: PatternID, err: hir::Error) -> BuildError { + let err = regex_syntax::Error::from(err); + BuildError { kind: BuildErrorKind::Syntax { pid, err } } + } + + pub(crate) fn nfa(err: nfa::thompson::BuildError) -> BuildError { + BuildError { kind: BuildErrorKind::NFA(err) } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for BuildError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self.kind { + BuildErrorKind::Syntax { ref err, .. } => Some(err), + BuildErrorKind::NFA(ref err) => Some(err), + } + } +} + +impl core::fmt::Display for BuildError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.kind { + BuildErrorKind::Syntax { pid, .. } => { + write!(f, "error parsing pattern {}", pid.as_usize()) + } + BuildErrorKind::NFA(_) => write!(f, "error building NFA"), + } + } +} + +/// An error that occurs when a search should be retried. +/// +/// This retry error distinguishes between two different failure modes. +/// +/// The first is one where potential quadratic behavior has been detected. +/// In this case, whatever optimization that led to this behavior should be +/// stopped, and the next best strategy should be used. +/// +/// The second indicates that the underlying regex engine has failed for some +/// reason. This usually occurs because either a lazy DFA's cache has become +/// ineffective or because a non-ASCII byte has been seen *and* a Unicode word +/// boundary was used in one of the patterns. In this failure case, a different +/// regex engine that won't fail in these ways (PikeVM, backtracker or the +/// one-pass DFA) should be used. +/// +/// This is an internal error only and should never bleed into the public +/// API. +#[derive(Debug)] +pub(crate) enum RetryError { + Quadratic(RetryQuadraticError), + Fail(RetryFailError), +} + +#[cfg(feature = "std")] +impl std::error::Error for RetryError {} + +impl core::fmt::Display for RetryError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match *self { + RetryError::Quadratic(ref err) => err.fmt(f), + RetryError::Fail(ref err) => err.fmt(f), + } + } +} + +impl From<MatchError> for RetryError { + fn from(merr: MatchError) -> RetryError { + RetryError::Fail(RetryFailError::from(merr)) + } +} + +/// An error that occurs when potential quadratic behavior has been detected +/// when applying either the "reverse suffix" or "reverse inner" optimizations. +/// +/// When this error occurs, callers should abandon the "reverse" optimization +/// and use a normal forward search. +#[derive(Debug)] +pub(crate) struct RetryQuadraticError(()); + +impl RetryQuadraticError { + pub(crate) fn new() -> RetryQuadraticError { + RetryQuadraticError(()) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for RetryQuadraticError {} + +impl core::fmt::Display for RetryQuadraticError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "regex engine gave up to avoid quadratic behavior") + } +} + +impl From<RetryQuadraticError> for RetryError { + fn from(err: RetryQuadraticError) -> RetryError { + RetryError::Quadratic(err) + } +} + +/// An error that occurs when a regex engine "gives up" for some reason before +/// finishing a search. Usually this occurs because of heuristic Unicode word +/// boundary support or because of ineffective cache usage in the lazy DFA. +/// +/// When this error occurs, callers should retry the regex search with a +/// different regex engine. +/// +/// Note that this has convenient `From` impls that will automatically +/// convert a `MatchError` into this error. This works because the meta +/// regex engine internals guarantee that errors like `HaystackTooLong` and +/// `UnsupportedAnchored` will never occur. The only errors left are `Quit` and +/// `GaveUp`, which both correspond to this "failure" error. +#[derive(Debug)] +pub(crate) struct RetryFailError { + offset: usize, +} + +impl RetryFailError { + pub(crate) fn from_offset(offset: usize) -> RetryFailError { + RetryFailError { offset } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for RetryFailError {} + +impl core::fmt::Display for RetryFailError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "regex engine failed at offset {:?}", self.offset) + } +} + +impl From<RetryFailError> for RetryError { + fn from(err: RetryFailError) -> RetryError { + RetryError::Fail(err) + } +} + +impl From<MatchError> for RetryFailError { + fn from(merr: MatchError) -> RetryFailError { + use crate::util::search::MatchErrorKind::*; + + match *merr.kind() { + Quit { offset, .. } => RetryFailError::from_offset(offset), + GaveUp { offset } => RetryFailError::from_offset(offset), + // These can never occur because we avoid them by construction + // or with higher level control flow logic. For example, the + // backtracker's wrapper will never hand out a backtracker engine + // when the haystack would be too long. + HaystackTooLong { .. } | UnsupportedAnchored { .. } => { + unreachable!("found impossible error in meta engine: {merr}") + } + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/limited.rs b/third_party/crates/regex-automata-0.4.18/src/meta/limited.rs new file mode 100644 index 0000000..3082a25 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/limited.rs @@ -0,0 +1,251 @@ +/*! +This module defines two bespoke reverse DFA searching routines. (One for the +lazy DFA and one for the fully compiled DFA.) These routines differ from the +usual ones by permitting the caller to specify a minimum starting position. +That is, the search will begin at `input.end()` and will usually stop at +`input.start()`, unless `min_start > input.start()`, in which case, the search +will stop at `min_start`. + +In other words, this lets you say, "no, the search must not extend past this +point, even if it's within the bounds of the given `Input`." And if the search +*does* want to go past that point, it stops and returns a "may be quadratic" +error, which indicates that the caller should retry using some other technique. + +These routines specifically exist to protect against quadratic behavior when +employing the "reverse suffix" and "reverse inner" optimizations. Without the +backstop these routines provide, it is possible for parts of the haystack to +get re-scanned over and over again. The backstop not only prevents this, but +*tells you when it is happening* so that you can change the strategy. + +Why can't we just use the normal search routines? We could use the normal +search routines and just set the start bound on the provided `Input` to our +`min_start` position. The problem here is that it's impossible to distinguish +between "no match because we reached the end of input" and "determined there +was no match well before the end of input." The former case is what we care +about with respect to quadratic behavior. The latter case is totally fine. + +Why don't we modify the normal search routines to report the position at which +the search stops? I considered this, and I still wonder if it is indeed the +right thing to do. However, I think the straight-forward thing to do there +would be to complicate the return type signature of almost every search routine +in this crate, which I really do not want to do. It therefore might make more +sense to provide a richer way for search routines to report meta data, but that +was beyond my bandwidth to work on at the time of writing. + +See the 'opt/reverse-inner' and 'opt/reverse-suffix' benchmarks in rebar for a +real demonstration of how quadratic behavior is mitigated. +*/ + +use crate::{ + meta::error::{RetryError, RetryQuadraticError}, + HalfMatch, Input, MatchError, +}; + +#[cfg(feature = "dfa-build")] +pub(crate) fn dfa_try_search_half_rev( + dfa: &crate::dfa::dense::DFA<alloc::vec::Vec<u32>>, + input: &Input<'_>, + min_start: usize, +) -> Result<Option<HalfMatch>, RetryError> { + use crate::dfa::Automaton; + + let mut mat = None; + let mut sid = dfa.start_state_reverse(input)?; + if input.start() == input.end() { + dfa_eoi_rev(dfa, input, &mut sid, &mut mat)?; + return Ok(mat); + } + let mut at = input.end() - 1; + loop { + sid = dfa.next_state(sid, input.haystack()[at]); + if dfa.is_special_state(sid) { + if dfa.is_match_state(sid) { + let pattern = dfa.match_pattern(sid, 0); + // Since reverse searches report the beginning of a + // match and the beginning is inclusive (not exclusive + // like the end of a match), we add 1 to make it + // inclusive. + mat = Some(HalfMatch::new(pattern, at + 1)); + } else if dfa.is_dead_state(sid) { + return Ok(mat); + } else if dfa.is_quit_state(sid) { + return Err(MatchError::quit(input.haystack()[at], at).into()); + } + } + if at == input.start() { + break; + } + at -= 1; + if at < min_start { + trace!( + "reached position {at} which is before the previous literal \ + match, quitting to avoid quadratic behavior", + ); + return Err(RetryError::Quadratic(RetryQuadraticError::new())); + } + } + let was_dead = dfa.is_dead_state(sid); + dfa_eoi_rev(dfa, input, &mut sid, &mut mat)?; + // If we reach the beginning of the search and we could otherwise still + // potentially keep matching if there was more to match, then we actually + // return an error to indicate giving up on this optimization. Why? Because + // we can't prove that the real match begins at where we would report it. + // + // This only happens when all of the following are true: + // + // 1) We reach the starting point of our search span. + // 2) The match we found is before the starting point. + // 3) The FSM reports we could possibly find a longer match. + // + // We need (1) because otherwise the search stopped before the starting + // point and there is no possible way to find a more leftmost position. + // + // We need (2) because if the match found has an offset equal to the minimum + // possible offset, then there is no possible more leftmost match. + // + // We need (3) because if the FSM couldn't continue anyway (i.e., it's in + // a dead state), then we know we couldn't find anything more leftmost + // than what we have. (We have to check the state we were in prior to the + // EOI transition since the EOI transition will usually bring us to a dead + // state by virtue of it represents the end-of-input.) + if at == input.start() + && mat.map_or(false, |m| m.offset() > input.start()) + && !was_dead + { + trace!( + "reached beginning of search at offset {at} without hitting \ + a dead state, quitting to avoid potential false positive match", + ); + return Err(RetryError::Quadratic(RetryQuadraticError::new())); + } + Ok(mat) +} + +#[cfg(feature = "hybrid")] +pub(crate) fn hybrid_try_search_half_rev( + dfa: &crate::hybrid::dfa::DFA, + cache: &mut crate::hybrid::dfa::Cache, + input: &Input<'_>, + min_start: usize, +) -> Result<Option<HalfMatch>, RetryError> { + let mut mat = None; + let mut sid = dfa.start_state_reverse(cache, input)?; + if input.start() == input.end() { + hybrid_eoi_rev(dfa, cache, input, &mut sid, &mut mat)?; + return Ok(mat); + } + let mut at = input.end() - 1; + loop { + sid = dfa + .next_state(cache, sid, input.haystack()[at]) + .map_err(|_| MatchError::gave_up(at))?; + if sid.is_tagged() { + if sid.is_match() { + let pattern = dfa.match_pattern(cache, sid, 0); + // Since reverse searches report the beginning of a + // match and the beginning is inclusive (not exclusive + // like the end of a match), we add 1 to make it + // inclusive. + mat = Some(HalfMatch::new(pattern, at + 1)); + } else if sid.is_dead() { + return Ok(mat); + } else if sid.is_quit() { + return Err(MatchError::quit(input.haystack()[at], at).into()); + } + } + if at == input.start() { + break; + } + at -= 1; + if at < min_start { + trace!( + "reached position {at} which is before the previous literal \ + match, quitting to avoid quadratic behavior", + ); + return Err(RetryError::Quadratic(RetryQuadraticError::new())); + } + } + let was_dead = sid.is_dead(); + hybrid_eoi_rev(dfa, cache, input, &mut sid, &mut mat)?; + // See the comments in the full DFA routine above for why we need this. + if at == input.start() + && mat.map_or(false, |m| m.offset() > input.start()) + && !was_dead + { + trace!( + "reached beginning of search at offset {at} without hitting \ + a dead state, quitting to avoid potential false positive match", + ); + return Err(RetryError::Quadratic(RetryQuadraticError::new())); + } + Ok(mat) +} + +#[cfg(feature = "dfa-build")] +#[cfg_attr(feature = "perf-inline", inline(always))] +fn dfa_eoi_rev( + dfa: &crate::dfa::dense::DFA<alloc::vec::Vec<u32>>, + input: &Input<'_>, + sid: &mut crate::util::primitives::StateID, + mat: &mut Option<HalfMatch>, +) -> Result<(), MatchError> { + use crate::dfa::Automaton; + + let sp = input.get_span(); + if sp.start > 0 { + let byte = input.haystack()[sp.start - 1]; + *sid = dfa.next_state(*sid, byte); + if dfa.is_match_state(*sid) { + let pattern = dfa.match_pattern(*sid, 0); + *mat = Some(HalfMatch::new(pattern, sp.start)); + } else if dfa.is_quit_state(*sid) { + return Err(MatchError::quit(byte, sp.start - 1)); + } + } else { + *sid = dfa.next_eoi_state(*sid); + if dfa.is_match_state(*sid) { + let pattern = dfa.match_pattern(*sid, 0); + *mat = Some(HalfMatch::new(pattern, 0)); + } + // N.B. We don't have to check 'is_quit' here because the EOI + // transition can never lead to a quit state. + debug_assert!(!dfa.is_quit_state(*sid)); + } + Ok(()) +} + +#[cfg(feature = "hybrid")] +#[cfg_attr(feature = "perf-inline", inline(always))] +fn hybrid_eoi_rev( + dfa: &crate::hybrid::dfa::DFA, + cache: &mut crate::hybrid::dfa::Cache, + input: &Input<'_>, + sid: &mut crate::hybrid::LazyStateID, + mat: &mut Option<HalfMatch>, +) -> Result<(), MatchError> { + let sp = input.get_span(); + if sp.start > 0 { + let byte = input.haystack()[sp.start - 1]; + *sid = dfa + .next_state(cache, *sid, byte) + .map_err(|_| MatchError::gave_up(sp.start))?; + if sid.is_match() { + let pattern = dfa.match_pattern(cache, *sid, 0); + *mat = Some(HalfMatch::new(pattern, sp.start)); + } else if sid.is_quit() { + return Err(MatchError::quit(byte, sp.start - 1)); + } + } else { + *sid = dfa + .next_eoi_state(cache, *sid) + .map_err(|_| MatchError::gave_up(sp.start))?; + if sid.is_match() { + let pattern = dfa.match_pattern(cache, *sid, 0); + *mat = Some(HalfMatch::new(pattern, 0)); + } + // N.B. We don't have to check 'is_quit' here because the EOI + // transition can never lead to a quit state. + debug_assert!(!sid.is_quit()); + } + Ok(()) +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/literal.rs b/third_party/crates/regex-automata-0.4.18/src/meta/literal.rs new file mode 100644 index 0000000..fac68d0 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/literal.rs @@ -0,0 +1,81 @@ +use alloc::{vec, vec::Vec}; + +use regex_syntax::hir::Hir; + +use crate::{meta::regex::RegexInfo, util::search::MatchKind}; + +/// Pull out an alternation of literals from the given sequence of HIR +/// expressions. +/// +/// There are numerous ways for this to fail. Generally, this only applies +/// to regexes of the form 'foo|bar|baz|...|quux'. It can also fail if there +/// are "too few" alternates, in which case, the regex engine is likely faster. +/// +/// And currently, this only returns something when 'hirs.len() == 1'. +pub(crate) fn alternation_literals( + info: &RegexInfo, + hirs: &[&Hir], +) -> Option<Vec<Vec<u8>>> { + use regex_syntax::hir::{HirKind, Literal}; + + // Might as well skip the work below if we know we can't build an + // Aho-Corasick searcher. + if !cfg!(feature = "perf-literal-multisubstring") { + return None; + } + // This is pretty hacky, but basically, if `is_alternation_literal` is + // true, then we can make several assumptions about the structure of our + // HIR. This is what justifies the `unreachable!` statements below. + if hirs.len() != 1 + || !info.props()[0].look_set().is_empty() + || info.props()[0].explicit_captures_len() > 0 + || !info.props()[0].is_alternation_literal() + || info.config().get_match_kind() != MatchKind::LeftmostFirst + { + return None; + } + let hir = &hirs[0]; + let alts = match *hir.kind() { + HirKind::Alternation(ref alts) => alts, + _ => return None, // one literal isn't worth it + }; + + let mut lits = vec![]; + for alt in alts { + let mut lit = vec![]; + match *alt.kind() { + HirKind::Literal(Literal(ref bytes)) => { + lit.extend_from_slice(bytes) + } + HirKind::Concat(ref exprs) => { + for e in exprs { + match *e.kind() { + HirKind::Literal(Literal(ref bytes)) => { + lit.extend_from_slice(bytes); + } + _ => unreachable!("expected literal, got {e:?}"), + } + } + } + _ => unreachable!("expected literal or concat, got {alt:?}"), + } + lits.push(lit); + } + // Why do this? Well, when the number of literals is small, it's likely + // that we'll use the lazy DFA which is in turn likely to be faster than + // Aho-Corasick in such cases. Primarily because Aho-Corasick doesn't have + // a "lazy DFA" but either a contiguous NFA or a full DFA. We rarely use + // the latter because it is so hungry (in time and space), and the former + // is decently fast, but not as fast as a well oiled lazy DFA. + // + // However, once the number starts getting large, the lazy DFA is likely + // to start thrashing because of the modest default cache size. When + // exactly does this happen? Dunno. But at whatever point that is (we make + // a guess below based on ad hoc benchmarking), we'll want to cut over to + // Aho-Corasick, where even the contiguous NFA is likely to do much better. + if lits.len() < 3000 { + debug!("skipping Aho-Corasick because there are too few literals"); + return None; + } + Some(lits) +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/mod.rs b/third_party/crates/regex-automata-0.4.18/src/meta/mod.rs new file mode 100644 index 0000000..31d99dc --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/mod.rs @@ -0,0 +1,64 @@ +/*! +Provides a regex matcher that composes several other regex matchers +automatically. + +This module is home to a meta [`Regex`], which provides a convenient high +level API for executing regular expressions in linear time. + +# Comparison with the `regex` crate + +A meta `Regex` is the implementation used directly by the `regex` crate. +Indeed, the `regex` crate API is essentially just a light wrapper over a meta +`Regex`. This means that if you need the full flexibility offered by this +API, then you should be able to switch to using this API directly without +any changes in match semantics or syntax. However, there are some API level +differences: + +* The `regex` crate API returns match objects that include references to the +haystack itself, which in turn makes it easy to access the matching strings +without having to slice the haystack yourself. In contrast, a meta `Regex` +returns match objects that only have offsets in them. +* At time of writing, a meta `Regex` doesn't have some of the convenience +routines that the `regex` crate has, such as replacements. Note though that +[`Captures::interpolate_string`](crate::util::captures::Captures::interpolate_string) +will handle the replacement string interpolation for you. +* A meta `Regex` supports the [`Input`](crate::Input) abstraction, which +provides a way to configure a search in more ways than is supported by the +`regex` crate. For example, [`Input::anchored`](crate::Input::anchored) can +be used to run an anchored search, regardless of whether the pattern is itself +anchored with a `^`. +* A meta `Regex` supports multi-pattern searching everywhere. +Indeed, every [`Match`](crate::Match) returned by the search APIs +include a [`PatternID`](crate::PatternID) indicating which pattern +matched. In the single pattern case, all matches correspond to +[`PatternID::ZERO`](crate::PatternID::ZERO). In contrast, the `regex` crate +has distinct `Regex` and a `RegexSet` APIs. The former only supports a single +pattern, while the latter supports multiple patterns but cannot report the +offsets of a match. +* A meta `Regex` provides the explicit capability of bypassing its internal +memory pool for automatically acquiring mutable scratch space required by its +internal regex engines. Namely, a [`Cache`] can be explicitly provided to lower +level routines such as [`Regex::search_with`]. + +*/ + +pub use self::{ + error::BuildError, + regex::{ + Builder, Cache, CapturesMatches, Config, FindMatches, Regex, Split, + SplitN, + }, +}; + +mod error; +#[cfg(any(feature = "dfa-build", feature = "hybrid"))] +mod limited; +mod literal; +mod prefix; +mod regex; +mod reverse_inner; +mod reverse_suffix; +#[cfg(any(feature = "dfa-build", feature = "hybrid"))] +mod stopat; +mod strategy; +mod wrappers; diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/prefix.rs b/third_party/crates/regex-automata-0.4.18/src/meta/prefix.rs new file mode 100644 index 0000000..435265e --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/prefix.rs @@ -0,0 +1,236 @@ +/*! +Cheap HIR proofs that literal candidate order is compatible with regex match +order. + +The reverse suffix and reverse inner strategies both scan for a required +literal before confirming the surrounding regex. These proofs are all +conservative. If they cannot establish that the first confirmed candidate is +the match the regex engine would report, then the corresponding reverse +strategy is not used. +*/ + +use regex_syntax::hir::{literal::Literal, Class, Hir, HirKind}; + +/// Return true when `hir` can match some string containing every byte in +/// `lit`. +/// +/// This is deliberately low precision. If every byte in a literal can be +/// consumed somewhere in the HIR, this gives up and reports that the +/// literal might occur internally. That is, this returning true does not +/// necessarily mean that the `Hir` provided definitively matches `lit`. +pub(super) fn hir_can_contain_literal(hir: &Hir, lit: &[u8]) -> bool { + lit.is_empty() || lit.iter().all(|&byte| hir_can_consume_byte(hir, byte)) +} + +/// Returns true when the given `Hir` has a fixed length. +/// +/// That is, when its minimum and maximum lengths are both finite and +/// equivalent. +pub(super) fn hir_has_fixed_length(hir: &Hir) -> bool { + let props = hir.properties(); + props + .minimum_len() + .and_then(|min| props.maximum_len().map(|max| min == max)) + .unwrap_or(false) +} + +/// Return true when an edge of `prefix` provides a required separator between +/// the start of a match and each literal candidate. +/// +/// More precisely, this looks at the first and last consuming children of a +/// top-level concatenation. One of those children must be either a character +/// class or a repetition of a character class that matches at least once. Let +/// that child be `S`. This returns true when both of the following hold: +/// +/// * `S` is disjoint from everything consumed by the other children in +/// `prefix`. +/// * `S` is disjoint from every literal in `literals`. +/// +/// In that case, `S` acts as a separator whose characters cannot be mistaken +/// for characters belonging to either the rest of the prefix or a literal +/// candidate. A reverse search therefore cannot slide `S` across either one +/// and produce a later match start from an earlier literal candidate. +/// +/// For example, consider this prefix and literal: +/// +/// ```text +/// prefix = \w+\s+ +/// literal = Holmes +/// ``` +/// +/// The trailing `\s+` is required, is disjoint from `\w+` and cannot match +/// anything in `Holmes`. Thus this proves the reverse suffix optimization safe +/// for `\w+\s+Holmes`. It also works with multiple literals, such as `Holmes` +/// and `Watson`, provided the separator is disjoint from all of them. +/// +/// The separator may instead be the first consuming child: +/// +/// ```text +/// prefix = \s[A-Za-z]{0,12} +/// literal = ing +/// ``` +/// +/// Here, the leading `\s` is disjoint from both `[A-Za-z]{0,12}` and `ing`. +/// +/// This returns false when the possible separator is optional, overlaps +/// another prefix component or can match a character in any literal. For +/// example, neither `\s*` in `[A-Za-z]*\s*` nor `\w+` in `\w+\w+` provides +/// the required separator. +pub(super) fn has_disjoint_class_separator( + prefix: &Hir, + literals: &[Literal], +) -> bool { + let hirs = match uncapture(prefix).kind() { + HirKind::Concat(hirs) if hirs.len() >= 2 => hirs, + _ => return false, + }; + let Some(first) = hirs.iter().position(|hir| !hir_matches_empty_only(hir)) + else { + return false; + }; + let last = + hirs.iter().rposition(|hir| !hir_matches_empty_only(hir)).unwrap(); + has_disjoint_class_separator_at(hirs, last, literals) + || (first != last + && has_disjoint_class_separator_at(hirs, first, literals)) +} + +fn has_disjoint_class_separator_at( + hirs: &[Hir], + separator: usize, + literals: &[Literal], +) -> bool { + let Some(separator_class) = required_class(&hirs[separator]) else { + return false; + }; + class_is_disjoint_from_literals(separator_class, literals) + && hirs.iter().enumerate().all(|(i, hir)| { + i == separator || hir_is_disjoint_from_class(hir, separator_class) + }) +} + +fn hir_matches_empty_only(hir: &Hir) -> bool { + hir.properties().maximum_len() == Some(0) +} + +fn required_class(hir: &Hir) -> Option<&Class> { + let hir = uncapture(hir); + match hir.kind() { + HirKind::Class(cls) => Some(cls), + HirKind::Repetition(rep) if rep.min > 0 => { + match uncapture(&rep.sub).kind() { + HirKind::Class(cls) => Some(cls), + _ => None, + } + } + _ => None, + } +} + +fn hir_is_disjoint_from_class(hir: &Hir, separator: &Class) -> bool { + match hir.kind() { + HirKind::Empty | HirKind::Look(_) => true, + HirKind::Literal(lit) => { + class_is_disjoint_from_literal(separator, &lit.0) + } + HirKind::Class(cls) => classes_are_disjoint(cls, separator), + HirKind::Repetition(rep) => { + hir_is_disjoint_from_class(&rep.sub, separator) + } + HirKind::Capture(capture) => { + hir_is_disjoint_from_class(&capture.sub, separator) + } + HirKind::Concat(hirs) | HirKind::Alternation(hirs) => { + hirs.iter().all(|hir| hir_is_disjoint_from_class(hir, separator)) + } + } +} + +fn uncapture(mut hir: &Hir) -> &Hir { + while let HirKind::Capture(capture) = hir.kind() { + hir = &capture.sub; + } + hir +} + +fn classes_are_disjoint(left: &Class, right: &Class) -> bool { + match (left, right) { + (Class::Bytes(left), Class::Bytes(right)) => { + let (mut intersection, other) = + if left.ranges().len() <= right.ranges().len() { + (left.clone(), right) + } else { + (right.clone(), left) + }; + intersection.intersect(other); + intersection.ranges().is_empty() + } + (Class::Unicode(left), Class::Unicode(right)) => { + let (mut intersection, other) = + if left.ranges().len() <= right.ranges().len() { + (left.clone(), right) + } else { + (right.clone(), left) + }; + intersection.intersect(other); + intersection.ranges().is_empty() + } + _ => false, + } +} + +fn class_is_disjoint_from_literals(cls: &Class, literals: &[Literal]) -> bool { + literals + .iter() + .all(|lit| class_is_disjoint_from_literal(cls, lit.as_bytes())) +} + +fn class_is_disjoint_from_literal(cls: &Class, lit: &[u8]) -> bool { + match cls { + Class::Bytes(cls) => { + lit.iter().all(|&byte| !byte_class_contains(cls, byte)) + } + Class::Unicode(cls) => core::str::from_utf8(lit) + .map_or(false, |lit| { + lit.chars().all(|ch| !unicode_class_contains(cls, ch)) + }), + } +} + +fn hir_can_consume_byte(hir: &Hir, byte: u8) -> bool { + match hir.kind() { + HirKind::Empty | HirKind::Look(_) => false, + HirKind::Literal(lit) => lit.0.contains(&byte), + HirKind::Class(Class::Bytes(cls)) => byte_class_contains(cls, byte), + HirKind::Class(Class::Unicode(cls)) => { + // We don't check literals based on codepoints, so if + // we have a non-ASCII byte, we assume here that any + // Unicode class will match it. In practice, given + // the prevalence of `\w`, this is not a terrible + // approximation. + if byte > 0x7F { + return true; + } + let ch = char::from(byte); + unicode_class_contains(cls, ch) + } + HirKind::Repetition(rep) => hir_can_consume_byte(&rep.sub, byte), + HirKind::Capture(capture) => hir_can_consume_byte(&capture.sub, byte), + HirKind::Concat(hirs) | HirKind::Alternation(hirs) => { + hirs.iter().any(|hir| hir_can_consume_byte(hir, byte)) + } + } +} + +fn byte_class_contains(cls: ®ex_syntax::hir::ClassBytes, byte: u8) -> bool { + cls.ranges() + .iter() + .any(|range| range.start() <= byte && byte <= range.end()) +} + +fn unicode_class_contains( + cls: ®ex_syntax::hir::ClassUnicode, + ch: char, +) -> bool { + cls.ranges().iter().any(|range| range.start() <= ch && ch <= range.end()) +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/regex.rs b/third_party/crates/regex-automata-0.4.18/src/meta/regex.rs new file mode 100644 index 0000000..453b2af --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/regex.rs @@ -0,0 +1,3781 @@ +use core::{ + borrow::Borrow, + panic::{RefUnwindSafe, UnwindSafe}, +}; + +use alloc::{boxed::Box, sync::Arc, vec, vec::Vec}; + +use regex_syntax::{ + ast, + hir::{self, Hir}, +}; + +use crate::{ + meta::{ + error::BuildError, + strategy::{self, Strategy}, + wrappers, + }, + nfa::thompson::{self, WhichCaptures}, + util::{ + captures::{Captures, GroupInfo}, + iter, + look::LookMatcher, + pool::{Pool, PoolGuard}, + prefilter::Prefilter, + primitives::{NonMaxUsize, PatternID}, + search::{HalfMatch, Input, Match, MatchKind, PatternSet, Span}, + }, +}; + +/// A type alias for our pool of meta::Cache that fixes the type parameters to +/// what we use for the meta regex below. +type CachePool = Pool<Cache, CachePoolFn>; + +/// Same as above, but for the guard returned by a pool. +type CachePoolGuard<'a> = PoolGuard<'a, Cache, CachePoolFn>; + +/// The type of the closure we use to create new caches. We need to spell out +/// all of the marker traits or else we risk leaking !MARKER impls. +type CachePoolFn = + Box<dyn Fn() -> Cache + Send + Sync + UnwindSafe + RefUnwindSafe>; + +/// A regex matcher that works by composing several other regex matchers +/// automatically. +/// +/// In effect, a meta regex papers over a lot of the quirks or performance +/// problems in each of the regex engines in this crate. Its goal is to provide +/// an infallible and simple API that "just does the right thing" in the common +/// case. +/// +/// A meta regex is the implementation of a `Regex` in the `regex` crate. +/// Indeed, the `regex` crate API is essentially just a light wrapper over +/// this type. This includes the `regex` crate's `RegexSet` API! +/// +/// # Composition +/// +/// This is called a "meta" matcher precisely because it uses other regex +/// matchers to provide a convenient high level regex API. Here are some +/// examples of how other regex matchers are composed: +/// +/// * When calling [`Regex::captures`], instead of immediately +/// running a slower but more capable regex engine like the +/// [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM), the meta regex engine +/// will usually first look for the bounds of a match with a higher throughput +/// regex engine like a [lazy DFA](crate::hybrid). Only when a match is found +/// is a slower engine like `PikeVM` used to find the matching span for each +/// capture group. +/// * While higher throughout engines like the lazy DFA cannot handle +/// Unicode word boundaries in general, they can still be used on pure ASCII +/// haystacks by pretending that Unicode word boundaries are just plain ASCII +/// word boundaries. However, if a haystack is not ASCII, the meta regex engine +/// will automatically switch to a (possibly slower) regex engine that supports +/// Unicode word boundaries in general. +/// * In some cases where a regex pattern is just a simple literal or a small +/// set of literals, an actual regex engine won't be used at all. Instead, +/// substring or multi-substring search algorithms will be employed. +/// +/// There are many other forms of composition happening too, but the above +/// should give a general idea. In particular, it may perhaps be surprising +/// that *multiple* regex engines might get executed for a single search. That +/// is, the decision of what regex engine to use is not _just_ based on the +/// pattern, but also based on the dynamic execution of the search itself. +/// +/// The primary reason for this composition is performance. The fundamental +/// tension is that the faster engines tend to be less capable, and the more +/// capable engines tend to be slower. +/// +/// Note that the forms of composition that are allowed are determined by +/// compile time crate features and configuration. For example, if the `hybrid` +/// feature isn't enabled, or if [`Config::hybrid`] has been disabled, then the +/// meta regex engine will never use a lazy DFA. +/// +/// # Synchronization and cloning +/// +/// Most of the regex engines in this crate require some kind of mutable +/// "scratch" space to read and write from while performing a search. Since +/// a meta regex composes these regex engines, a meta regex also requires +/// mutable scratch space. This scratch space is called a [`Cache`]. +/// +/// Most regex engines _also_ usually have a read-only component, typically +/// a [Thompson `NFA`](crate::nfa::thompson::NFA). +/// +/// In order to make the `Regex` API convenient, most of the routines hide +/// the fact that a `Cache` is needed at all. To achieve this, a [memory +/// pool](crate::util::pool::Pool) is used internally to retrieve `Cache` +/// values in a thread safe way that also permits reuse. This in turn implies +/// that every such search call requires some form of synchronization. Usually +/// this synchronization is fast enough to not notice, but in some cases, it +/// can be a bottleneck. This typically occurs when all of the following are +/// true: +/// +/// * The same `Regex` is shared across multiple threads simultaneously, +/// usually via a [`util::lazy::Lazy`](crate::util::lazy::Lazy) or something +/// similar from the `once_cell` or `lazy_static` crates. +/// * The primary unit of work in each thread is a regex search. +/// * Searches are run on very short haystacks. +/// +/// This particular case can lead to high contention on the pool used by a +/// `Regex` internally, which can in turn increase latency to a noticeable +/// effect. This cost can be mitigated in one of the following ways: +/// +/// * Use a distinct copy of a `Regex` in each thread, usually by cloning it. +/// Cloning a `Regex` _does not_ do a deep copy of its read-only component. +/// But it does lead to each `Regex` having its own memory pool, which in +/// turn eliminates the problem of contention. In general, this technique should +/// not result in any additional memory usage when compared to sharing the same +/// `Regex` across multiple threads simultaneously. +/// * Use lower level APIs, like [`Regex::search_with`], which permit passing +/// a `Cache` explicitly. In this case, it is up to you to determine how best +/// to provide a `Cache`. For example, you might put a `Cache` in thread-local +/// storage if your use case allows for it. +/// +/// Overall, this is an issue that happens rarely in practice, but it can +/// happen. +/// +/// # Warning: spin-locks may be used in alloc-only mode +/// +/// When this crate is built without the `std` feature and the high level APIs +/// on a `Regex` are used, then a spin-lock will be used to synchronize access +/// to an internal pool of `Cache` values. This may be undesirable because +/// a spin-lock is [effectively impossible to implement correctly in user +/// space][spinlocks-are-bad]. That is, more concretely, the spin-lock could +/// result in a deadlock. +/// +/// [spinlocks-are-bad]: https://matklad.github.io/2020/01/02/spinlocks-considered-harmful.html +/// +/// If one wants to avoid the use of spin-locks when the `std` feature is +/// disabled, then you must use APIs that accept a `Cache` value explicitly. +/// For example, [`Regex::search_with`]. +/// +/// # Example +/// +/// ``` +/// use regex_automata::meta::Regex; +/// +/// let re = Regex::new(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$")?; +/// assert!(re.is_match("2010-03-14")); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: anchored search +/// +/// This example shows how to use [`Input::anchored`] to run an anchored +/// search, even when the regex pattern itself isn't anchored. An anchored +/// search guarantees that if a match is found, then the start offset of the +/// match corresponds to the offset at which the search was started. +/// +/// ``` +/// use regex_automata::{meta::Regex, Anchored, Input, Match}; +/// +/// let re = Regex::new(r"\bfoo\b")?; +/// let input = Input::new("xx foo xx").range(3..).anchored(Anchored::Yes); +/// // The offsets are in terms of the original haystack. +/// assert_eq!(Some(Match::must(0, 3..6)), re.find(input)); +/// +/// // Notice that no match occurs here, because \b still takes the +/// // surrounding context into account, even if it means looking back +/// // before the start of your search. +/// let hay = "xxfoo xx"; +/// let input = Input::new(hay).range(2..).anchored(Anchored::Yes); +/// assert_eq!(None, re.find(input)); +/// // Indeed, you cannot achieve the above by simply slicing the +/// // haystack itself, since the regex engine can't see the +/// // surrounding context. This is why 'Input' permits setting +/// // the bounds of a search! +/// let input = Input::new(&hay[2..]).anchored(Anchored::Yes); +/// // WRONG! +/// assert_eq!(Some(Match::must(0, 0..3)), re.find(input)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: earliest search +/// +/// This example shows how to use [`Input::earliest`] to run a search that +/// might stop before finding the typical leftmost match. +/// +/// ``` +/// use regex_automata::{meta::Regex, Anchored, Input, Match}; +/// +/// let re = Regex::new(r"[a-z]{3}|b")?; +/// let input = Input::new("abc").earliest(true); +/// assert_eq!(Some(Match::must(0, 1..2)), re.find(input)); +/// +/// // Note that "earliest" isn't really a match semantic unto itself. +/// // Instead, it is merely an instruction to whatever regex engine +/// // gets used internally to quit as soon as it can. For example, +/// // this regex uses a different search technique, and winds up +/// // producing a different (but valid) match! +/// let re = Regex::new(r"abc|b")?; +/// let input = Input::new("abc").earliest(true); +/// assert_eq!(Some(Match::must(0, 0..3)), re.find(input)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: change the line terminator +/// +/// This example shows how to enable multi-line mode by default and change +/// the line terminator to the NUL byte: +/// +/// ``` +/// use regex_automata::{meta::Regex, util::syntax, Match}; +/// +/// let re = Regex::builder() +/// .syntax(syntax::Config::new().multi_line(true)) +/// .configure(Regex::config().line_terminator(b'\x00')) +/// .build(r"^foo$")?; +/// let hay = "\x00foo\x00"; +/// assert_eq!(Some(Match::must(0, 1..4)), re.find(hay)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Debug)] +pub struct Regex { + /// The actual regex implementation. + imp: Arc<RegexI>, + /// A thread safe pool of caches. + /// + /// For the higher level search APIs, a `Cache` is automatically plucked + /// from this pool before running a search. The lower level `with` methods + /// permit the caller to provide their own cache, thereby bypassing + /// accesses to this pool. + /// + /// Note that we put this outside the `Arc` so that cloning a `Regex` + /// results in creating a fresh `CachePool`. This in turn permits callers + /// to clone regexes into separate threads where each such regex gets + /// the pool's "thread owner" optimization. Otherwise, if one shares the + /// `Regex` directly, then the pool will go through a slower mutex path for + /// all threads except for the "owner." + pool: CachePool, +} + +/// The internal implementation of `Regex`, split out so that it can be wrapped +/// in an `Arc`. +#[derive(Debug)] +struct RegexI { + /// The core matching engine. + /// + /// Why is this reference counted when RegexI is already wrapped in an Arc? + /// Well, we need to capture this in a closure to our `Pool` below in order + /// to create new `Cache` values when needed. So since it needs to be in + /// two places, we make it reference counted. + /// + /// We make `RegexI` itself reference counted too so that `Regex` itself + /// stays extremely small and very cheap to clone. + strat: Arc<dyn Strategy>, + /// Metadata about the regexes driving the strategy. The metadata is also + /// usually stored inside the strategy too, but we put it here as well + /// so that we can get quick access to it (without virtual calls) before + /// executing the regex engine. For example, we use this metadata to + /// detect a subset of cases where we know a match is impossible, and can + /// thus avoid calling into the strategy at all. + /// + /// Since `RegexInfo` is stored in multiple places, it is also reference + /// counted. + info: RegexInfo, +} + +/// Convenience constructors for a `Regex` using the default configuration. +impl Regex { + /// Builds a `Regex` from a single pattern string using the default + /// configuration. + /// + /// If there was a problem parsing the pattern or a problem turning it into + /// a regex matcher, then an error is returned. + /// + /// If you want to change the configuration of a `Regex`, use a [`Builder`] + /// with a [`Config`]. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Match}; + /// + /// let re = Regex::new(r"(?Rm)^foo$")?; + /// let hay = "\r\nfoo\r\n"; + /// assert_eq!(Some(Match::must(0, 2..5)), re.find(hay)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new(pattern: &str) -> Result<Regex, BuildError> { + Regex::builder().build(pattern) + } + + /// Builds a `Regex` from many pattern strings using the default + /// configuration. + /// + /// If there was a problem parsing any of the patterns or a problem turning + /// them into a regex matcher, then an error is returned. + /// + /// If you want to change the configuration of a `Regex`, use a [`Builder`] + /// with a [`Config`]. + /// + /// # Example: simple lexer + /// + /// This simplistic example leverages the multi-pattern support to build a + /// simple little lexer. The pattern ID in the match tells you which regex + /// matched, which in turn might be used to map back to the "type" of the + /// token returned by the lexer. + /// + /// ``` + /// use regex_automata::{meta::Regex, Match}; + /// + /// let re = Regex::new_many(&[ + /// r"[[:space:]]", + /// r"[A-Za-z0-9][A-Za-z0-9_]+", + /// r"->", + /// r".", + /// ])?; + /// let haystack = "fn is_boss(bruce: i32, springsteen: String) -> bool;"; + /// let matches: Vec<Match> = re.find_iter(haystack).collect(); + /// assert_eq!(matches, vec![ + /// Match::must(1, 0..2), // 'fn' + /// Match::must(0, 2..3), // ' ' + /// Match::must(1, 3..10), // 'is_boss' + /// Match::must(3, 10..11), // '(' + /// Match::must(1, 11..16), // 'bruce' + /// Match::must(3, 16..17), // ':' + /// Match::must(0, 17..18), // ' ' + /// Match::must(1, 18..21), // 'i32' + /// Match::must(3, 21..22), // ',' + /// Match::must(0, 22..23), // ' ' + /// Match::must(1, 23..34), // 'springsteen' + /// Match::must(3, 34..35), // ':' + /// Match::must(0, 35..36), // ' ' + /// Match::must(1, 36..42), // 'String' + /// Match::must(3, 42..43), // ')' + /// Match::must(0, 43..44), // ' ' + /// Match::must(2, 44..46), // '->' + /// Match::must(0, 46..47), // ' ' + /// Match::must(1, 47..51), // 'bool' + /// Match::must(3, 51..52), // ';' + /// ]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// One can write a lexer like the above using a regex like + /// `(?P<space>[[:space:]])|(?P<ident>[A-Za-z0-9][A-Za-z0-9_]+)|...`, + /// but then you need to ask whether capture group matched to determine + /// which branch in the regex matched, and thus, which token the match + /// corresponds to. In contrast, the above example includes the pattern ID + /// in the match. There's no need to use capture groups at all. + /// + /// # Example: finding the pattern that caused an error + /// + /// When a syntax error occurs, it is possible to ask which pattern + /// caused the syntax error. + /// + /// ``` + /// use regex_automata::{meta::Regex, PatternID}; + /// + /// let err = Regex::new_many(&["a", "b", r"\p{Foo}", "c"]).unwrap_err(); + /// assert_eq!(Some(PatternID::must(2)), err.pattern()); + /// ``` + /// + /// # Example: zero patterns is valid + /// + /// Building a regex with zero patterns results in a regex that never + /// matches anything. Because this routine is generic, passing an empty + /// slice usually requires a turbo-fish (or something else to help type + /// inference). + /// + /// ``` + /// use regex_automata::{meta::Regex, util::syntax, Match}; + /// + /// let re = Regex::new_many::<&str>(&[])?; + /// assert_eq!(None, re.find("")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new_many<P: AsRef<str>>( + patterns: &[P], + ) -> Result<Regex, BuildError> { + Regex::builder().build_many(patterns) + } + + /// Return a default configuration for a `Regex`. + /// + /// This is a convenience routine to avoid needing to import the [`Config`] + /// type when customizing the construction of a `Regex`. + /// + /// # Example: lower the NFA size limit + /// + /// In some cases, the default size limit might be too big. The size limit + /// can be lowered, which will prevent large regex patterns from compiling. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::meta::Regex; + /// + /// let result = Regex::builder() + /// .configure(Regex::config().nfa_size_limit(Some(20 * (1<<10)))) + /// // Not even 20KB is enough to build a single large Unicode class! + /// .build(r"\pL"); + /// assert!(result.is_err()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn config() -> Config { + Config::new() + } + + /// Return a builder for configuring the construction of a `Regex`. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Builder`] type in common cases. + /// + /// # Example: change the line terminator + /// + /// This example shows how to enable multi-line mode by default and change + /// the line terminator to the NUL byte: + /// + /// ``` + /// use regex_automata::{meta::Regex, util::syntax, Match}; + /// + /// let re = Regex::builder() + /// .syntax(syntax::Config::new().multi_line(true)) + /// .configure(Regex::config().line_terminator(b'\x00')) + /// .build(r"^foo$")?; + /// let hay = "\x00foo\x00"; + /// assert_eq!(Some(Match::must(0, 1..4)), re.find(hay)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn builder() -> Builder { + Builder::new() + } +} + +/// High level convenience routines for using a regex to search a haystack. +impl Regex { + /// Returns true if and only if this regex matches the given haystack. + /// + /// This routine may short circuit if it knows that scanning future input + /// will never lead to a different result. (Consider how this might make + /// a difference given the regex `a+` on the haystack `aaaaaaaaaaaaaaa`. + /// This routine _may_ stop after it sees the first `a`, but routines like + /// `find` need to continue searching because `+` is greedy by default.) + /// + /// # Example + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new("foo[0-9]+bar")?; + /// + /// assert!(re.is_match("foo12345bar")); + /// assert!(!re.is_match("foobar")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: consistency with search APIs + /// + /// `is_match` is guaranteed to return `true` whenever `find` returns a + /// match. This includes searches that are executed entirely within a + /// codepoint: + /// + /// ``` + /// use regex_automata::{meta::Regex, Input}; + /// + /// let re = Regex::new("a*")?; + /// + /// // This doesn't match because the default configuration bans empty + /// // matches from splitting a codepoint. + /// assert!(!re.is_match(Input::new("☃").span(1..2))); + /// assert_eq!(None, re.find(Input::new("☃").span(1..2))); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Notice that when UTF-8 mode is disabled, then the above reports a + /// match because the restriction against zero-width matches that split a + /// codepoint has been lifted: + /// + /// ``` + /// use regex_automata::{meta::Regex, Input, Match}; + /// + /// let re = Regex::builder() + /// .configure(Regex::config().utf8_empty(false)) + /// .build("a*")?; + /// + /// assert!(re.is_match(Input::new("☃").span(1..2))); + /// assert_eq!( + /// Some(Match::must(0, 1..1)), + /// re.find(Input::new("☃").span(1..2)), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// A similar idea applies when using line anchors with CRLF mode enabled, + /// which prevents them from matching between a `\r` and a `\n`. + /// + /// ``` + /// use regex_automata::{meta::Regex, Input, Match}; + /// + /// let re = Regex::new(r"(?Rm:$)")?; + /// assert!(!re.is_match(Input::new("\r\n").span(1..1))); + /// // A regular line anchor, which only considers \n as a + /// // line terminator, will match. + /// let re = Regex::new(r"(?m:$)")?; + /// assert!(re.is_match(Input::new("\r\n").span(1..1))); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_match<'h, I: Into<Input<'h>>>(&self, input: I) -> bool { + let input = input.into().earliest(true); + if self.imp.info.is_impossible(&input) { + return false; + } + let mut guard = self.pool.get(); + let result = self.imp.strat.is_match(&mut guard, &input); + // See 'Regex::search' for why we put the guard back explicitly. + PoolGuard::put(guard); + result + } + + /// Executes a leftmost search and returns the first match that is found, + /// if one exists. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Match}; + /// + /// let re = Regex::new("foo[0-9]+")?; + /// assert_eq!(Some(Match::must(0, 0..8)), re.find("foo12345")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find<'h, I: Into<Input<'h>>>(&self, input: I) -> Option<Match> { + self.search(&input.into()) + } + + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided [`Captures`] + /// value. If no match was found, then [`Captures::is_match`] is guaranteed + /// to return `false`. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Span}; + /// + /// let re = Regex::new(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})$")?; + /// let mut caps = re.create_captures(); + /// + /// re.captures("2010-03-14", &mut caps); + /// assert!(caps.is_match()); + /// assert_eq!(Some(Span::from(0..4)), caps.get_group(1)); + /// assert_eq!(Some(Span::from(5..7)), caps.get_group(2)); + /// assert_eq!(Some(Span::from(8..10)), caps.get_group(3)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn captures<'h, I: Into<Input<'h>>>( + &self, + input: I, + caps: &mut Captures, + ) { + self.search_captures(&input.into(), caps) + } + + /// Returns an iterator over all non-overlapping leftmost matches in + /// the given haystack. If no match exists, then the iterator yields no + /// elements. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Match}; + /// + /// let re = Regex::new("foo[0-9]+")?; + /// let haystack = "foo1 foo12 foo123"; + /// let matches: Vec<Match> = re.find_iter(haystack).collect(); + /// assert_eq!(matches, vec![ + /// Match::must(0, 0..4), + /// Match::must(0, 5..10), + /// Match::must(0, 11..17), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find_iter<'r, 'h, I: Into<Input<'h>>>( + &'r self, + input: I, + ) -> FindMatches<'r, 'h> { + let cache = self.pool.get(); + let it = iter::Searcher::new(input.into()); + FindMatches { re: self, cache, it } + } + + /// Returns an iterator over all non-overlapping `Captures` values. If no + /// match exists, then the iterator yields no elements. + /// + /// This yields the same matches as [`Regex::find_iter`], but it includes + /// the spans of all capturing groups that participate in each match. + /// + /// **Tip:** See [`util::iter::Searcher`](crate::util::iter::Searcher) for + /// how to correctly iterate over all matches in a haystack while avoiding + /// the creation of a new `Captures` value for every match. (Which you are + /// forced to do with an `Iterator`.) + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Span}; + /// + /// let re = Regex::new("foo(?P<numbers>[0-9]+)")?; + /// + /// let haystack = "foo1 foo12 foo123"; + /// let matches: Vec<Span> = re + /// .captures_iter(haystack) + /// // The unwrap is OK since 'numbers' matches if the pattern matches. + /// .map(|caps| caps.get_group_by_name("numbers").unwrap()) + /// .collect(); + /// assert_eq!(matches, vec![ + /// Span::from(3..4), + /// Span::from(8..10), + /// Span::from(14..17), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn captures_iter<'r, 'h, I: Into<Input<'h>>>( + &'r self, + input: I, + ) -> CapturesMatches<'r, 'h> { + let cache = self.pool.get(); + let caps = self.create_captures(); + let it = iter::Searcher::new(input.into()); + CapturesMatches { re: self, cache, caps, it } + } + + /// Returns an iterator of spans of the haystack given, delimited by a + /// match of the regex. Namely, each element of the iterator corresponds to + /// a part of the haystack that *isn't* matched by the regular expression. + /// + /// # Example + /// + /// To split a string delimited by arbitrary amounts of spaces or tabs: + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r"[ \t]+")?; + /// let hay = "a b \t c\td e"; + /// let fields: Vec<&str> = re.split(hay).map(|span| &hay[span]).collect(); + /// assert_eq!(fields, vec!["a", "b", "c", "d", "e"]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: more cases + /// + /// Basic usage: + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r" ")?; + /// let hay = "Mary had a little lamb"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["Mary", "had", "a", "little", "lamb"]); + /// + /// let re = Regex::new(r"X")?; + /// let hay = ""; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec![""]); + /// + /// let re = Regex::new(r"X")?; + /// let hay = "lionXXtigerXleopard"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["lion", "", "tiger", "leopard"]); + /// + /// let re = Regex::new(r"::")?; + /// let hay = "lion::tiger::leopard"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["lion", "tiger", "leopard"]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// If a haystack contains multiple contiguous matches, you will end up + /// with empty spans yielded by the iterator: + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r"X")?; + /// let hay = "XXXXaXXbXc"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["", "", "", "", "a", "", "b", "c"]); + /// + /// let re = Regex::new(r"/")?; + /// let hay = "(///)"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["(", "", "", ")"]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Separators at the start or end of a haystack are neighbored by empty + /// spans. + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r"0")?; + /// let hay = "010"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["", "1", ""]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// When the empty string is used as a regex, it splits at every valid + /// UTF-8 boundary by default (which includes the beginning and end of the + /// haystack): + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r"")?; + /// let hay = "rust"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["", "r", "u", "s", "t", ""]); + /// + /// // Splitting by an empty string is UTF-8 aware by default! + /// let re = Regex::new(r"")?; + /// let hay = "☃"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["", "☃", ""]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// But note that UTF-8 mode for empty strings can be disabled, which will + /// then result in a match at every byte offset in the haystack, + /// including between every UTF-8 code unit. + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::builder() + /// .configure(Regex::config().utf8_empty(false)) + /// .build(r"")?; + /// let hay = "☃".as_bytes(); + /// let got: Vec<&[u8]> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec![ + /// // Writing byte string slices is just brutal. The problem is that + /// // b"foo" has type &[u8; 3] instead of &[u8]. + /// &[][..], &[b'\xE2'][..], &[b'\x98'][..], &[b'\x83'][..], &[][..], + /// ]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Contiguous separators (commonly shows up with whitespace), can lead to + /// possibly surprising behavior. For example, this code is correct: + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r" ")?; + /// let hay = " a b c"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["", "", "", "", "a", "", "b", "c"]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// It does *not* give you `["a", "b", "c"]`. For that behavior, you'd want + /// to match contiguous space characters: + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r" +")?; + /// let hay = " a b c"; + /// let got: Vec<&str> = re.split(hay).map(|sp| &hay[sp]).collect(); + /// // N.B. This does still include a leading empty span because ' +' + /// // matches at the beginning of the haystack. + /// assert_eq!(got, vec!["", "a", "b", "c"]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn split<'r, 'h, I: Into<Input<'h>>>( + &'r self, + input: I, + ) -> Split<'r, 'h> { + Split { finder: self.find_iter(input), last: 0 } + } + + /// Returns an iterator of at most `limit` spans of the haystack given, + /// delimited by a match of the regex. (A `limit` of `0` will return no + /// spans.) Namely, each element of the iterator corresponds to a part + /// of the haystack that *isn't* matched by the regular expression. The + /// remainder of the haystack that is not split will be the last element in + /// the iterator. + /// + /// # Example + /// + /// Get the first two words in some haystack: + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r"\W+").unwrap(); + /// let hay = "Hey! How are you?"; + /// let fields: Vec<&str> = + /// re.splitn(hay, 3).map(|span| &hay[span]).collect(); + /// assert_eq!(fields, vec!["Hey", "How", "are you?"]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Examples: more cases + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r" ")?; + /// let hay = "Mary had a little lamb"; + /// let got: Vec<&str> = re.splitn(hay, 3).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["Mary", "had", "a little lamb"]); + /// + /// let re = Regex::new(r"X")?; + /// let hay = ""; + /// let got: Vec<&str> = re.splitn(hay, 3).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec![""]); + /// + /// let re = Regex::new(r"X")?; + /// let hay = "lionXXtigerXleopard"; + /// let got: Vec<&str> = re.splitn(hay, 3).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["lion", "", "tigerXleopard"]); + /// + /// let re = Regex::new(r"::")?; + /// let hay = "lion::tiger::leopard"; + /// let got: Vec<&str> = re.splitn(hay, 2).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["lion", "tiger::leopard"]); + /// + /// let re = Regex::new(r"X")?; + /// let hay = "abcXdef"; + /// let got: Vec<&str> = re.splitn(hay, 1).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["abcXdef"]); + /// + /// let re = Regex::new(r"X")?; + /// let hay = "abcdef"; + /// let got: Vec<&str> = re.splitn(hay, 2).map(|sp| &hay[sp]).collect(); + /// assert_eq!(got, vec!["abcdef"]); + /// + /// let re = Regex::new(r"X")?; + /// let hay = "abcXdef"; + /// let got: Vec<&str> = re.splitn(hay, 0).map(|sp| &hay[sp]).collect(); + /// assert!(got.is_empty()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn splitn<'r, 'h, I: Into<Input<'h>>>( + &'r self, + input: I, + limit: usize, + ) -> SplitN<'r, 'h> { + SplitN { splits: self.split(input), limit } + } +} + +/// Lower level search routines that give more control. +impl Regex { + /// Returns the start and end offset of the leftmost match. If no match + /// exists, then `None` is returned. + /// + /// This is like [`Regex::find`] but, but it accepts a concrete `&Input` + /// instead of an `Into<Input>`. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Input, Match}; + /// + /// let re = Regex::new(r"Samwise|Sam")?; + /// let input = Input::new( + /// "one of the chief characters, Samwise the Brave", + /// ); + /// assert_eq!(Some(Match::must(0, 29..36)), re.search(&input)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search(&self, input: &Input<'_>) -> Option<Match> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return None; + } + let mut guard = self.pool.get(); + let result = self.imp.strat.search(&mut guard, input); + // We do this dance with the guard and explicitly put it back in the + // pool because it seems to result in better codegen. If we let the + // guard's Drop impl put it back in the pool, then functions like + // ptr::drop_in_place get called and they *don't* get inlined. This + // isn't usually a big deal, but in latency sensitive benchmarks the + // extra function call can matter. + // + // I used `rebar measure -f '^grep/every-line$' -e meta` to measure + // the effects here. + // + // Note that this doesn't eliminate the latency effects of using the + // pool. There is still some (minor) cost for the "thread owner" of the + // pool. (i.e., The thread that first calls a regex search routine.) + // However, for other threads using the regex, the pool access can be + // quite expensive as it goes through a mutex. Callers can avoid this + // by either cloning the Regex (which creates a distinct copy of the + // pool), or callers can use the lower level APIs that accept a 'Cache' + // directly and do their own handling. + PoolGuard::put(guard); + result + } + + /// Returns the end offset of the leftmost match. If no match exists, then + /// `None` is returned. + /// + /// This is distinct from [`Regex::search`] in that it only returns the end + /// of a match and not the start of the match. Depending on a variety of + /// implementation details, this _may_ permit the regex engine to do less + /// overall work. For example, if a DFA is being used to execute a search, + /// then the start of a match usually requires running a separate DFA in + /// reverse to the find the start of a match. If one only needs the end of + /// a match, then the separate reverse scan to find the start of a match + /// can be skipped. (Note that the reverse scan is avoided even when using + /// `Regex::search` when possible, for example, in the case of an anchored + /// search.) + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Input, HalfMatch}; + /// + /// let re = Regex::new(r"Samwise|Sam")?; + /// let input = Input::new( + /// "one of the chief characters, Samwise the Brave", + /// ); + /// assert_eq!(Some(HalfMatch::must(0, 36)), re.search_half(&input)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search_half(&self, input: &Input<'_>) -> Option<HalfMatch> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return None; + } + let mut guard = self.pool.get(); + let result = self.imp.strat.search_half(&mut guard, input); + // See 'Regex::search' for why we put the guard back explicitly. + PoolGuard::put(guard); + result + } + + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided [`Captures`] + /// value. If no match was found, then [`Captures::is_match`] is guaranteed + /// to return `false`. + /// + /// This is like [`Regex::captures`], but it accepts a concrete `&Input` + /// instead of an `Into<Input>`. + /// + /// # Example: specific pattern search + /// + /// This example shows how to build a multi-pattern `Regex` that permits + /// searching for specific patterns. + /// + /// ``` + /// use regex_automata::{ + /// meta::Regex, + /// Anchored, Match, PatternID, Input, + /// }; + /// + /// let re = Regex::new_many(&["[a-z0-9]{6}", "[a-z][a-z0-9]{5}"])?; + /// let mut caps = re.create_captures(); + /// let haystack = "foo123"; + /// + /// // Since we are using the default leftmost-first match and both + /// // patterns match at the same starting position, only the first pattern + /// // will be returned in this case when doing a search for any of the + /// // patterns. + /// let expected = Some(Match::must(0, 0..6)); + /// re.search_captures(&Input::new(haystack), &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we want to check whether some other pattern matches, then we + /// // can provide its pattern ID. + /// let expected = Some(Match::must(1, 0..6)); + /// let input = Input::new(haystack) + /// .anchored(Anchored::Pattern(PatternID::must(1))); + /// re.search_captures(&input, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specifying the bounds of a search + /// + /// This example shows how providing the bounds of a search can produce + /// different results than simply sub-slicing the haystack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{meta::Regex, Match, Input}; + /// + /// let re = Regex::new(r"\b[0-9]{3}\b")?; + /// let mut caps = re.create_captures(); + /// let haystack = "foo123bar"; + /// + /// // Since we sub-slice the haystack, the search doesn't know about + /// // the larger context and assumes that `123` is surrounded by word + /// // boundaries. And of course, the match position is reported relative + /// // to the sub-slice as well, which means we get `0..3` instead of + /// // `3..6`. + /// let expected = Some(Match::must(0, 0..3)); + /// let input = Input::new(&haystack[3..6]); + /// re.search_captures(&input, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we provide the bounds of the search within the context of the + /// // entire haystack, then the search can take the surrounding context + /// // into account. (And if we did find a match, it would be reported + /// // as a valid offset into `haystack` instead of its sub-slice.) + /// let expected = None; + /// let input = Input::new(haystack).range(3..6); + /// re.search_captures(&input, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search_captures(&self, input: &Input<'_>, caps: &mut Captures) { + caps.set_pattern(None); + let pid = self.search_slots(input, caps.slots_mut()); + caps.set_pattern(pid); + } + + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided `slots`, and + /// returns the matching pattern ID. The contents of the slots for patterns + /// other than the matching pattern are unspecified. If no match was found, + /// then `None` is returned and the contents of `slots` is unspecified. + /// + /// This is like [`Regex::search`], but it accepts a raw slots slice + /// instead of a `Captures` value. This is useful in contexts where you + /// don't want or need to allocate a `Captures`. + /// + /// It is legal to pass _any_ number of slots to this routine. If the regex + /// engine would otherwise write a slot offset that doesn't fit in the + /// provided slice, then it is simply skipped. In general though, there are + /// usually three slice lengths you might want to use: + /// + /// * An empty slice, if you only care about which pattern matched. + /// * A slice with [`pattern_len() * 2`](Regex::pattern_len) slots, if you + /// only care about the overall match spans for each matching pattern. + /// * A slice with + /// [`slot_len()`](crate::util::captures::GroupInfo::slot_len) slots, which + /// permits recording match offsets for every capturing group in every + /// pattern. + /// + /// # Example + /// + /// This example shows how to find the overall match offsets in a + /// multi-pattern search without allocating a `Captures` value. Indeed, we + /// can put our slots right on the stack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{meta::Regex, PatternID, Input}; + /// + /// let re = Regex::new_many(&[ + /// r"\pL+", + /// r"\d+", + /// ])?; + /// let input = Input::new("!@#123"); + /// + /// // We only care about the overall match offsets here, so we just + /// // allocate two slots for each pattern. Each slot records the start + /// // and end of the match. + /// let mut slots = [None; 4]; + /// let pid = re.search_slots(&input, &mut slots); + /// assert_eq!(Some(PatternID::must(1)), pid); + /// + /// // The overall match offsets are always at 'pid * 2' and 'pid * 2 + 1'. + /// // See 'GroupInfo' for more details on the mapping between groups and + /// // slot indices. + /// let slot_start = pid.unwrap().as_usize() * 2; + /// let slot_end = slot_start + 1; + /// assert_eq!(Some(3), slots[slot_start].map(|s| s.get())); + /// assert_eq!(Some(6), slots[slot_end].map(|s| s.get())); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search_slots( + &self, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return None; + } + let mut guard = self.pool.get(); + let result = self.imp.strat.search_slots(&mut guard, input, slots); + // See 'Regex::search' for why we put the guard back explicitly. + PoolGuard::put(guard); + result + } + + /// Writes the set of patterns that match anywhere in the given search + /// configuration to `patset`. If multiple patterns match at the same + /// position and this `Regex` was configured with [`MatchKind::All`] + /// semantics, then all matching patterns are written to the given set. + /// + /// Unless all of the patterns in this `Regex` are anchored, then generally + /// speaking, this will scan the entire haystack. + /// + /// This search routine *does not* clear the pattern set. This gives some + /// flexibility to the caller (e.g., running multiple searches with the + /// same pattern set), but does make the API bug-prone if you're reusing + /// the same pattern set for multiple searches but intended them to be + /// independent. + /// + /// If a pattern ID matched but the given `PatternSet` does not have + /// sufficient capacity to store it, then it is not inserted and silently + /// dropped. + /// + /// # Example + /// + /// This example shows how to find all matching patterns in a haystack, + /// even when some patterns match at the same position as other patterns. + /// It is important that we configure the `Regex` with [`MatchKind::All`] + /// semantics here, or else overlapping matches will not be reported. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{meta::Regex, Input, MatchKind, PatternSet}; + /// + /// let patterns = &[ + /// r"\w+", r"\d+", r"\pL+", r"foo", r"bar", r"barfoo", r"foobar", + /// ]; + /// let re = Regex::builder() + /// .configure(Regex::config().match_kind(MatchKind::All)) + /// .build_many(patterns)?; + /// + /// let input = Input::new("foobar"); + /// let mut patset = PatternSet::new(re.pattern_len()); + /// re.which_overlapping_matches(&input, &mut patset); + /// let expected = vec![0, 2, 3, 4, 6]; + /// let got: Vec<usize> = patset.iter().map(|p| p.as_usize()).collect(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn which_overlapping_matches( + &self, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + if self.imp.info.is_impossible(input) { + return; + } + let mut guard = self.pool.get(); + let result = self + .imp + .strat + .which_overlapping_matches(&mut guard, input, patset); + // See 'Regex::search' for why we put the guard back explicitly. + PoolGuard::put(guard); + result + } +} + +/// Lower level search routines that give more control, and require the caller +/// to provide an explicit [`Cache`] parameter. +impl Regex { + /// This is like [`Regex::search`], but requires the caller to + /// explicitly pass a [`Cache`]. + /// + /// # Why pass a `Cache` explicitly? + /// + /// Passing a `Cache` explicitly will bypass the use of an internal memory + /// pool used by `Regex` to get a `Cache` for a search. The use of this + /// pool can be slower in some cases when a `Regex` is used from multiple + /// threads simultaneously. Typically, performance only becomes an issue + /// when there is heavy contention, which in turn usually only occurs + /// when each thread's primary unit of work is a regex search on a small + /// haystack. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Input, Match}; + /// + /// let re = Regex::new(r"Samwise|Sam")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new( + /// "one of the chief characters, Samwise the Brave", + /// ); + /// assert_eq!( + /// Some(Match::must(0, 29..36)), + /// re.search_with(&mut cache, &input), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<Match> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return None; + } + self.imp.strat.search(cache, input) + } + + /// This is like [`Regex::search_half`], but requires the caller to + /// explicitly pass a [`Cache`]. + /// + /// # Why pass a `Cache` explicitly? + /// + /// Passing a `Cache` explicitly will bypass the use of an internal memory + /// pool used by `Regex` to get a `Cache` for a search. The use of this + /// pool can be slower in some cases when a `Regex` is used from multiple + /// threads simultaneously. Typically, performance only becomes an issue + /// when there is heavy contention, which in turn usually only occurs + /// when each thread's primary unit of work is a regex search on a small + /// haystack. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Input, HalfMatch}; + /// + /// let re = Regex::new(r"Samwise|Sam")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new( + /// "one of the chief characters, Samwise the Brave", + /// ); + /// assert_eq!( + /// Some(HalfMatch::must(0, 36)), + /// re.search_half_with(&mut cache, &input), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search_half_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<HalfMatch> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return None; + } + self.imp.strat.search_half(cache, input) + } + + /// This is like [`Regex::search_captures`], but requires the caller to + /// explicitly pass a [`Cache`]. + /// + /// # Why pass a `Cache` explicitly? + /// + /// Passing a `Cache` explicitly will bypass the use of an internal memory + /// pool used by `Regex` to get a `Cache` for a search. The use of this + /// pool can be slower in some cases when a `Regex` is used from multiple + /// threads simultaneously. Typically, performance only becomes an issue + /// when there is heavy contention, which in turn usually only occurs + /// when each thread's primary unit of work is a regex search on a small + /// haystack. + /// + /// # Example: specific pattern search + /// + /// This example shows how to build a multi-pattern `Regex` that permits + /// searching for specific patterns. + /// + /// ``` + /// use regex_automata::{ + /// meta::Regex, + /// Anchored, Match, PatternID, Input, + /// }; + /// + /// let re = Regex::new_many(&["[a-z0-9]{6}", "[a-z][a-z0-9]{5}"])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "foo123"; + /// + /// // Since we are using the default leftmost-first match and both + /// // patterns match at the same starting position, only the first pattern + /// // will be returned in this case when doing a search for any of the + /// // patterns. + /// let expected = Some(Match::must(0, 0..6)); + /// re.search_captures_with(&mut cache, &Input::new(haystack), &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we want to check whether some other pattern matches, then we + /// // can provide its pattern ID. + /// let expected = Some(Match::must(1, 0..6)); + /// let input = Input::new(haystack) + /// .anchored(Anchored::Pattern(PatternID::must(1))); + /// re.search_captures_with(&mut cache, &input, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specifying the bounds of a search + /// + /// This example shows how providing the bounds of a search can produce + /// different results than simply sub-slicing the haystack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{meta::Regex, Match, Input}; + /// + /// let re = Regex::new(r"\b[0-9]{3}\b")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "foo123bar"; + /// + /// // Since we sub-slice the haystack, the search doesn't know about + /// // the larger context and assumes that `123` is surrounded by word + /// // boundaries. And of course, the match position is reported relative + /// // to the sub-slice as well, which means we get `0..3` instead of + /// // `3..6`. + /// let expected = Some(Match::must(0, 0..3)); + /// let input = Input::new(&haystack[3..6]); + /// re.search_captures_with(&mut cache, &input, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we provide the bounds of the search within the context of the + /// // entire haystack, then the search can take the surrounding context + /// // into account. (And if we did find a match, it would be reported + /// // as a valid offset into `haystack` instead of its sub-slice.) + /// let expected = None; + /// let input = Input::new(haystack).range(3..6); + /// re.search_captures_with(&mut cache, &input, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search_captures_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + caps: &mut Captures, + ) { + caps.set_pattern(None); + let pid = self.search_slots_with(cache, input, caps.slots_mut()); + caps.set_pattern(pid); + } + + /// This is like [`Regex::search_slots`], but requires the caller to + /// explicitly pass a [`Cache`]. + /// + /// # Why pass a `Cache` explicitly? + /// + /// Passing a `Cache` explicitly will bypass the use of an internal memory + /// pool used by `Regex` to get a `Cache` for a search. The use of this + /// pool can be slower in some cases when a `Regex` is used from multiple + /// threads simultaneously. Typically, performance only becomes an issue + /// when there is heavy contention, which in turn usually only occurs + /// when each thread's primary unit of work is a regex search on a small + /// haystack. + /// + /// # Example + /// + /// This example shows how to find the overall match offsets in a + /// multi-pattern search without allocating a `Captures` value. Indeed, we + /// can put our slots right on the stack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{meta::Regex, PatternID, Input}; + /// + /// let re = Regex::new_many(&[ + /// r"\pL+", + /// r"\d+", + /// ])?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("!@#123"); + /// + /// // We only care about the overall match offsets here, so we just + /// // allocate two slots for each pattern. Each slot records the start + /// // and end of the match. + /// let mut slots = [None; 4]; + /// let pid = re.search_slots_with(&mut cache, &input, &mut slots); + /// assert_eq!(Some(PatternID::must(1)), pid); + /// + /// // The overall match offsets are always at 'pid * 2' and 'pid * 2 + 1'. + /// // See 'GroupInfo' for more details on the mapping between groups and + /// // slot indices. + /// let slot_start = pid.unwrap().as_usize() * 2; + /// let slot_end = slot_start + 1; + /// assert_eq!(Some(3), slots[slot_start].map(|s| s.get())); + /// assert_eq!(Some(6), slots[slot_end].map(|s| s.get())); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search_slots_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + if self.imp.info.captures_disabled() + || self.imp.info.is_impossible(input) + { + return None; + } + self.imp.strat.search_slots(cache, input, slots) + } + + /// This is like [`Regex::which_overlapping_matches`], but requires the + /// caller to explicitly pass a [`Cache`]. + /// + /// Passing a `Cache` explicitly will bypass the use of an internal memory + /// pool used by `Regex` to get a `Cache` for a search. The use of this + /// pool can be slower in some cases when a `Regex` is used from multiple + /// threads simultaneously. Typically, performance only becomes an issue + /// when there is heavy contention, which in turn usually only occurs + /// when each thread's primary unit of work is a regex search on a small + /// haystack. + /// + /// # Why pass a `Cache` explicitly? + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{meta::Regex, Input, MatchKind, PatternSet}; + /// + /// let patterns = &[ + /// r"\w+", r"\d+", r"\pL+", r"foo", r"bar", r"barfoo", r"foobar", + /// ]; + /// let re = Regex::builder() + /// .configure(Regex::config().match_kind(MatchKind::All)) + /// .build_many(patterns)?; + /// let mut cache = re.create_cache(); + /// + /// let input = Input::new("foobar"); + /// let mut patset = PatternSet::new(re.pattern_len()); + /// re.which_overlapping_matches_with(&mut cache, &input, &mut patset); + /// let expected = vec![0, 2, 3, 4, 6]; + /// let got: Vec<usize> = patset.iter().map(|p| p.as_usize()).collect(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn which_overlapping_matches_with( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + if self.imp.info.is_impossible(input) { + return; + } + self.imp.strat.which_overlapping_matches(cache, input, patset) + } +} + +/// Various non-search routines for querying properties of a `Regex` and +/// convenience routines for creating [`Captures`] and [`Cache`] values. +impl Regex { + /// Creates a new object for recording capture group offsets. This is used + /// in search APIs like [`Regex::captures`] and [`Regex::search_captures`]. + /// + /// This is a convenience routine for + /// `Captures::all(re.group_info().clone())`. Callers may build other types + /// of `Captures` values that record less information (and thus require + /// less work from the regex engine) using [`Captures::matches`] and + /// [`Captures::empty`]. + /// + /// # Example + /// + /// This shows some alternatives to [`Regex::create_captures`]: + /// + /// ``` + /// use regex_automata::{ + /// meta::Regex, + /// util::captures::Captures, + /// Match, PatternID, Span, + /// }; + /// + /// let re = Regex::new(r"(?<first>[A-Z][a-z]+) (?<last>[A-Z][a-z]+)")?; + /// + /// // This is equivalent to Regex::create_captures. It stores matching + /// // offsets for all groups in the regex. + /// let mut all = Captures::all(re.group_info().clone()); + /// re.captures("Bruce Springsteen", &mut all); + /// assert_eq!(Some(Match::must(0, 0..17)), all.get_match()); + /// assert_eq!(Some(Span::from(0..5)), all.get_group_by_name("first")); + /// assert_eq!(Some(Span::from(6..17)), all.get_group_by_name("last")); + /// + /// // In this version, we only care about the implicit groups, which + /// // means offsets for the explicit groups will be unavailable. It can + /// // sometimes be faster to ask for fewer groups, since the underlying + /// // regex engine needs to do less work to keep track of them. + /// let mut matches = Captures::matches(re.group_info().clone()); + /// re.captures("Bruce Springsteen", &mut matches); + /// // We still get the overall match info. + /// assert_eq!(Some(Match::must(0, 0..17)), matches.get_match()); + /// // But now the explicit groups are unavailable. + /// assert_eq!(None, matches.get_group_by_name("first")); + /// assert_eq!(None, matches.get_group_by_name("last")); + /// + /// // Finally, in this version, we don't ask to keep track of offsets for + /// // *any* groups. All we get back is whether a match occurred, and if + /// // so, the ID of the pattern that matched. + /// let mut empty = Captures::empty(re.group_info().clone()); + /// re.captures("Bruce Springsteen", &mut empty); + /// // it's a match! + /// assert!(empty.is_match()); + /// // for pattern ID 0 + /// assert_eq!(Some(PatternID::ZERO), empty.pattern()); + /// // Match offsets are unavailable. + /// assert_eq!(None, empty.get_match()); + /// // And of course, explicit groups are unavailable too. + /// assert_eq!(None, empty.get_group_by_name("first")); + /// assert_eq!(None, empty.get_group_by_name("last")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn create_captures(&self) -> Captures { + Captures::all(self.group_info().clone()) + } + + /// Creates a new cache for use with lower level search APIs like + /// [`Regex::search_with`]. + /// + /// The cache returned should only be used for searches for this `Regex`. + /// If you want to reuse the cache for another `Regex`, then you must call + /// [`Cache::reset`] with that `Regex`. + /// + /// This is a convenience routine for [`Cache::new`]. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Input, Match}; + /// + /// let re = Regex::new(r"(?-u)m\w+\s+m\w+")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("crazy janey and her mission man"); + /// assert_eq!( + /// Some(Match::must(0, 20..31)), + /// re.search_with(&mut cache, &input), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn create_cache(&self) -> Cache { + self.imp.strat.create_cache() + } + + /// Returns the total number of patterns in this regex. + /// + /// The standard [`Regex::new`] constructor always results in a `Regex` + /// with a single pattern, but [`Regex::new_many`] permits building a + /// multi-pattern regex. + /// + /// A `Regex` guarantees that the maximum possible `PatternID` returned in + /// any match is `Regex::pattern_len() - 1`. In the case where the number + /// of patterns is `0`, a match is impossible. + /// + /// # Example + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let re = Regex::new(r"(?m)^[a-z]$")?; + /// assert_eq!(1, re.pattern_len()); + /// + /// let re = Regex::new_many::<&str>(&[])?; + /// assert_eq!(0, re.pattern_len()); + /// + /// let re = Regex::new_many(&["a", "b", "c"])?; + /// assert_eq!(3, re.pattern_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn pattern_len(&self) -> usize { + self.imp.info.pattern_len() + } + + /// Returns the total number of capturing groups. + /// + /// This includes the implicit capturing group corresponding to the + /// entire match. Therefore, the minimum value returned is `1`. + /// + /// # Example + /// + /// This shows a few patterns and how many capture groups they have. + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let len = |pattern| { + /// Regex::new(pattern).map(|re| re.captures_len()) + /// }; + /// + /// assert_eq!(1, len("a")?); + /// assert_eq!(2, len("(a)")?); + /// assert_eq!(3, len("(a)|(b)")?); + /// assert_eq!(5, len("(a)(b)|(c)(d)")?); + /// assert_eq!(2, len("(a)|b")?); + /// assert_eq!(2, len("a|(b)")?); + /// assert_eq!(2, len("(b)*")?); + /// assert_eq!(2, len("(b)+")?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: multiple patterns + /// + /// This routine also works for multiple patterns. The total number is + /// the sum of the capture groups of each pattern. + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// let len = |patterns| { + /// Regex::new_many(patterns).map(|re| re.captures_len()) + /// }; + /// + /// assert_eq!(2, len(&["a", "b"])?); + /// assert_eq!(4, len(&["(a)", "(b)"])?); + /// assert_eq!(6, len(&["(a)|(b)", "(c)|(d)"])?); + /// assert_eq!(8, len(&["(a)(b)|(c)(d)", "(x)(y)"])?); + /// assert_eq!(3, len(&["(a)", "b"])?); + /// assert_eq!(3, len(&["a", "(b)"])?); + /// assert_eq!(4, len(&["(a)", "(b)*"])?); + /// assert_eq!(4, len(&["(a)+", "(b)+"])?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn captures_len(&self) -> usize { + self.imp + .info + .props_union() + .explicit_captures_len() + .saturating_add(self.pattern_len()) + } + + /// Returns the total number of capturing groups that appear in every + /// possible match. + /// + /// If the number of capture groups can vary depending on the match, then + /// this returns `None`. That is, a value is only returned when the number + /// of matching groups is invariant or "static." + /// + /// Note that like [`Regex::captures_len`], this **does** include the + /// implicit capturing group corresponding to the entire match. Therefore, + /// when a non-None value is returned, it is guaranteed to be at least `1`. + /// Stated differently, a return value of `Some(0)` is impossible. + /// + /// # Example + /// + /// This shows a few cases where a static number of capture groups is + /// available and a few cases where it is not. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// + /// use regex_automata::meta::Regex; + /// + /// let len = |pattern| { + /// Regex::new(pattern).map(|re| re.static_captures_len()) + /// }; + /// + /// assert_eq!(Some(1), len("a")?); + /// assert_eq!(Some(2), len("(a)")?); + /// assert_eq!(Some(2), len("(a)|(b)")?); + /// assert_eq!(Some(3), len("(a)(b)|(c)(d)")?); + /// assert_eq!(None, len("(a)|b")?); + /// assert_eq!(None, len("a|(b)")?); + /// assert_eq!(None, len("(b)*")?); + /// assert_eq!(Some(2), len("(b)+")?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: multiple patterns + /// + /// This property extends to regexes with multiple patterns as well. In + /// order for their to be a static number of capture groups in this case, + /// every pattern must have the same static number. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// + /// use regex_automata::meta::Regex; + /// + /// let len = |patterns| { + /// Regex::new_many(patterns).map(|re| re.static_captures_len()) + /// }; + /// + /// assert_eq!(Some(1), len(&["a", "b"])?); + /// assert_eq!(Some(2), len(&["(a)", "(b)"])?); + /// assert_eq!(Some(2), len(&["(a)|(b)", "(c)|(d)"])?); + /// assert_eq!(Some(3), len(&["(a)(b)|(c)(d)", "(x)(y)"])?); + /// assert_eq!(None, len(&["(a)", "b"])?); + /// assert_eq!(None, len(&["a", "(b)"])?); + /// assert_eq!(None, len(&["(a)", "(b)*"])?); + /// assert_eq!(Some(2), len(&["(a)+", "(b)+"])?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn static_captures_len(&self) -> Option<usize> { + self.imp + .info + .props_union() + .static_explicit_captures_len() + .map(|len| len.saturating_add(1)) + } + + /// Return information about the capture groups in this `Regex`. + /// + /// A `GroupInfo` is an immutable object that can be cheaply cloned. It + /// is responsible for maintaining a mapping between the capture groups + /// in the concrete syntax of zero or more regex patterns and their + /// internal representation used by some of the regex matchers. It is also + /// responsible for maintaining a mapping between the name of each group + /// (if one exists) and its corresponding group index. + /// + /// A `GroupInfo` is ultimately what is used to build a [`Captures`] value, + /// which is some mutable space where group offsets are stored as a result + /// of a search. + /// + /// # Example + /// + /// This shows some alternatives to [`Regex::create_captures`]: + /// + /// ``` + /// use regex_automata::{ + /// meta::Regex, + /// util::captures::Captures, + /// Match, PatternID, Span, + /// }; + /// + /// let re = Regex::new(r"(?<first>[A-Z][a-z]+) (?<last>[A-Z][a-z]+)")?; + /// + /// // This is equivalent to Regex::create_captures. It stores matching + /// // offsets for all groups in the regex. + /// let mut all = Captures::all(re.group_info().clone()); + /// re.captures("Bruce Springsteen", &mut all); + /// assert_eq!(Some(Match::must(0, 0..17)), all.get_match()); + /// assert_eq!(Some(Span::from(0..5)), all.get_group_by_name("first")); + /// assert_eq!(Some(Span::from(6..17)), all.get_group_by_name("last")); + /// + /// // In this version, we only care about the implicit groups, which + /// // means offsets for the explicit groups will be unavailable. It can + /// // sometimes be faster to ask for fewer groups, since the underlying + /// // regex engine needs to do less work to keep track of them. + /// let mut matches = Captures::matches(re.group_info().clone()); + /// re.captures("Bruce Springsteen", &mut matches); + /// // We still get the overall match info. + /// assert_eq!(Some(Match::must(0, 0..17)), matches.get_match()); + /// // But now the explicit groups are unavailable. + /// assert_eq!(None, matches.get_group_by_name("first")); + /// assert_eq!(None, matches.get_group_by_name("last")); + /// + /// // Finally, in this version, we don't ask to keep track of offsets for + /// // *any* groups. All we get back is whether a match occurred, and if + /// // so, the ID of the pattern that matched. + /// let mut empty = Captures::empty(re.group_info().clone()); + /// re.captures("Bruce Springsteen", &mut empty); + /// // it's a match! + /// assert!(empty.is_match()); + /// // for pattern ID 0 + /// assert_eq!(Some(PatternID::ZERO), empty.pattern()); + /// // Match offsets are unavailable. + /// assert_eq!(None, empty.get_match()); + /// // And of course, explicit groups are unavailable too. + /// assert_eq!(None, empty.get_group_by_name("first")); + /// assert_eq!(None, empty.get_group_by_name("last")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn group_info(&self) -> &GroupInfo { + self.imp.strat.group_info() + } + + /// Returns the configuration object used to build this `Regex`. + /// + /// If no configuration object was explicitly passed, then the + /// configuration returned represents the default. + #[inline] + pub fn get_config(&self) -> &Config { + self.imp.info.config() + } + + /// Returns true if this regex has a high chance of being "accelerated." + /// + /// The precise meaning of "accelerated" is specifically left unspecified, + /// but the general meaning is that the search is a high likelihood of + /// running faster than a character-at-a-time loop inside a standard + /// regex engine. + /// + /// When a regex is accelerated, it is only a *probabilistic* claim. That + /// is, just because the regex is believed to be accelerated, that doesn't + /// mean it will definitely execute searches very fast. Similarly, if a + /// regex is *not* accelerated, that is also a probabilistic claim. That + /// is, a regex for which `is_accelerated` returns `false` could still run + /// searches more quickly than a regex for which `is_accelerated` returns + /// `true`. + /// + /// Whether a regex is marked as accelerated or not is dependent on + /// implementations details that may change in a semver compatible release. + /// That is, a regex that is accelerated in a `x.y.1` release might not be + /// accelerated in a `x.y.2` release. + /// + /// Basically, the value of acceleration boils down to a hedge: a hodge + /// podge of internal heuristics combine to make a probabilistic guess + /// that this regex search may run "fast." The value in knowing this from + /// a caller's perspective is that it may act as a signal that no further + /// work should be done to accelerate a search. For example, a grep-like + /// tool might try to do some extra work extracting literals from a regex + /// to create its own heuristic acceleration strategies. But it might + /// choose to defer to this crate's acceleration strategy if one exists. + /// This routine permits querying whether such a strategy is active for a + /// particular regex. + /// + /// # Example + /// + /// ``` + /// use regex_automata::meta::Regex; + /// + /// // A simple literal is very likely to be accelerated. + /// let re = Regex::new(r"foo")?; + /// assert!(re.is_accelerated()); + /// + /// // A regex with no literals is likely to not be accelerated. + /// let re = Regex::new(r"\w")?; + /// assert!(!re.is_accelerated()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_accelerated(&self) -> bool { + self.imp.strat.is_accelerated() + } + + /// Return the total approximate heap memory, in bytes, used by this `Regex`. + /// + /// Note that currently, there is no high level configuration for setting + /// a limit on the specific value returned by this routine. Instead, the + /// following routines can be used to control heap memory at a bit of a + /// lower level: + /// + /// * [`Config::nfa_size_limit`] controls how big _any_ of the NFAs are + /// allowed to be. + /// * [`Config::onepass_size_limit`] controls how big the one-pass DFA is + /// allowed to be. + /// * [`Config::hybrid_cache_capacity`] controls how much memory the lazy + /// DFA is permitted to allocate to store its transition table. + /// * [`Config::dfa_size_limit`] controls how big a fully compiled DFA is + /// allowed to be. + /// * [`Config::dfa_state_limit`] controls the conditions under which the + /// meta regex engine will even attempt to build a fully compiled DFA. + #[inline] + pub fn memory_usage(&self) -> usize { + self.imp.strat.memory_usage() + } +} + +impl Clone for Regex { + fn clone(&self) -> Regex { + let imp = Arc::clone(&self.imp); + let pool = { + let strat = Arc::clone(&imp.strat); + let create: CachePoolFn = Box::new(move || strat.create_cache()); + Pool::with_capacity( + self.imp.info.config().get_pool_capacity(), + create, + ) + }; + Regex { imp, pool } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct RegexInfo(Arc<RegexInfoI>); + +#[derive(Clone, Debug)] +struct RegexInfoI { + config: Config, + props: Vec<hir::Properties>, + props_union: hir::Properties, +} + +impl RegexInfo { + /// Creates a new `RegexInfo` from the configuration and HIRs that make up + /// a meta regex. + /// + /// This is exported for use in some tests. + pub(super) fn new(config: Config, hirs: &[&Hir]) -> RegexInfo { + // Collect all of the properties from each of the HIRs, and also + // union them into one big set of properties representing all HIRs + // as if they were in one big alternation. + let mut props = vec![]; + for hir in hirs.iter() { + props.push(hir.properties().clone()); + } + let props_union = hir::Properties::union(&props); + + RegexInfo(Arc::new(RegexInfoI { config, props, props_union })) + } + + pub(crate) fn config(&self) -> &Config { + &self.0.config + } + + pub(crate) fn props(&self) -> &[hir::Properties] { + &self.0.props + } + + pub(crate) fn props_union(&self) -> &hir::Properties { + &self.0.props_union + } + + pub(crate) fn pattern_len(&self) -> usize { + self.props().len() + } + + pub(crate) fn memory_usage(&self) -> usize { + self.props().iter().map(|p| p.memory_usage()).sum::<usize>() + + self.props_union().memory_usage() + } + + /// Returns true when the search is guaranteed to be anchored. That is, + /// when a match is reported, its offset is guaranteed to correspond to + /// the start of the search. + /// + /// This includes returning true when `input` _isn't_ anchored but the + /// underlying regex is. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn is_anchored_start(&self, input: &Input<'_>) -> bool { + input.get_anchored().is_anchored() || self.is_always_anchored_start() + } + + /// Returns true when this regex is always anchored to the start of a + /// search. And in particular, that regardless of an `Input` configuration, + /// if any match is reported it must start at `0`. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn is_always_anchored_start(&self) -> bool { + use regex_syntax::hir::Look; + self.props_union().look_set_prefix().contains(Look::Start) + } + + /// Returns true when this regex is always anchored to the end of a + /// search. And in particular, that regardless of an `Input` configuration, + /// if any match is reported it must end at the end of the haystack. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn is_always_anchored_end(&self) -> bool { + use regex_syntax::hir::Look; + self.props_union().look_set_suffix().contains(Look::End) + } + + /// Returns true when the regex's NFA lacks capture states. + /// + /// In this case, some regex engines (like the PikeVM) are unable to report + /// match offsets, while some (like the lazy DFA can). To avoid whether a + /// match or not is reported based on engine selection, routines that + /// return match offsets will _always_ report `None` when this is true. + /// + /// Yes, this is a weird case and it's a little fucked up. But + /// `WhichCaptures::None` comes with an appropriate warning. + fn captures_disabled(&self) -> bool { + matches!(self.config().get_which_captures(), WhichCaptures::None) + } + + /// Returns true if and only if it is known that a match is impossible + /// for the given input. This is useful for short-circuiting and avoiding + /// running the regex engine if it's known no match can be reported. + /// + /// Note that this doesn't necessarily detect every possible case. For + /// example, when `pattern_len() == 0`, a match is impossible, but that + /// case is so rare that it's fine to be handled by the regex engine + /// itself. That is, it's not worth the cost of adding it here in order to + /// make it a little faster. The reason is that this is called for every + /// search. so there is some cost to adding checks here. Arguably, some of + /// the checks that are here already probably shouldn't be here... + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_impossible(&self, input: &Input<'_>) -> bool { + // The underlying regex is anchored, so if we don't start the search + // at position 0, a match is impossible, because the anchor can only + // match at position 0. + if input.start() > 0 && self.is_always_anchored_start() { + return true; + } + // Same idea, but for the end anchor. + if input.end() < input.haystack().len() + && self.is_always_anchored_end() + { + return true; + } + // If the haystack is smaller than the minimum length required, then + // we know there can be no match. + let minlen = match self.props_union().minimum_len() { + None => return false, + Some(minlen) => minlen, + }; + if input.get_span().len() < minlen { + return true; + } + // Same idea as minimum, but for maximum. This is trickier. We can + // only apply the maximum when we know the entire span that we're + // searching *has* to match according to the regex (and possibly the + // input configuration). If we know there is too much for the regex + // to match, we can bail early. + // + // I don't think we can apply the maximum otherwise unfortunately. + if self.is_anchored_start(input) && self.is_always_anchored_end() { + let maxlen = match self.props_union().maximum_len() { + None => return false, + Some(maxlen) => maxlen, + }; + if input.get_span().len() > maxlen { + return true; + } + } + false + } +} + +/// An iterator over all non-overlapping matches. +/// +/// The iterator yields a [`Match`] value until no more matches could be found. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the `Regex` that produced this iterator. +/// * `'h` represents the lifetime of the haystack being searched. +/// +/// This iterator can be created with the [`Regex::find_iter`] method. +#[derive(Debug)] +pub struct FindMatches<'r, 'h> { + re: &'r Regex, + cache: CachePoolGuard<'r>, + it: iter::Searcher<'h>, +} + +impl<'r, 'h> FindMatches<'r, 'h> { + /// Returns the `Regex` value that created this iterator. + #[inline] + pub fn regex(&self) -> &'r Regex { + self.re + } + + /// Returns the current `Input` associated with this iterator. + /// + /// The `start` position on the given `Input` may change during iteration, + /// but all other values are guaranteed to remain invariant. + #[inline] + pub fn input<'s>(&'s self) -> &'s Input<'h> { + self.it.input() + } +} + +impl<'r, 'h> Iterator for FindMatches<'r, 'h> { + type Item = Match; + + #[inline] + fn next(&mut self) -> Option<Match> { + let FindMatches { re, ref mut cache, ref mut it } = *self; + it.advance(|input| Ok(re.search_with(cache, input))) + } + + #[inline] + fn count(self) -> usize { + // If all we care about is a count of matches, then we only need to + // find the end position of each match. This can give us a 2x perf + // boost in some cases, because it avoids needing to do a reverse scan + // to find the start of a match. + let FindMatches { re, mut cache, it } = self; + // This does the deref for PoolGuard once instead of every iter. + let cache = &mut *cache; + it.into_half_matches_iter( + |input| Ok(re.search_half_with(cache, input)), + ) + .count() + } +} + +impl<'r, 'h> core::iter::FusedIterator for FindMatches<'r, 'h> {} + +/// An iterator over all non-overlapping leftmost matches with their capturing +/// groups. +/// +/// The iterator yields a [`Captures`] value until no more matches could be +/// found. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the `Regex` that produced this iterator. +/// * `'h` represents the lifetime of the haystack being searched. +/// +/// This iterator can be created with the [`Regex::captures_iter`] method. +#[derive(Debug)] +pub struct CapturesMatches<'r, 'h> { + re: &'r Regex, + cache: CachePoolGuard<'r>, + caps: Captures, + it: iter::Searcher<'h>, +} + +impl<'r, 'h> CapturesMatches<'r, 'h> { + /// Returns the `Regex` value that created this iterator. + #[inline] + pub fn regex(&self) -> &'r Regex { + self.re + } + + /// Returns the current `Input` associated with this iterator. + /// + /// The `start` position on the given `Input` may change during iteration, + /// but all other values are guaranteed to remain invariant. + #[inline] + pub fn input<'s>(&'s self) -> &'s Input<'h> { + self.it.input() + } +} + +impl<'r, 'h> Iterator for CapturesMatches<'r, 'h> { + type Item = Captures; + + #[inline] + fn next(&mut self) -> Option<Captures> { + // Splitting 'self' apart seems necessary to appease borrowck. + let CapturesMatches { re, ref mut cache, ref mut caps, ref mut it } = + *self; + let _ = it.advance(|input| { + re.search_captures_with(cache, input, caps); + Ok(caps.get_match()) + }); + if caps.is_match() { + Some(caps.clone()) + } else { + None + } + } + + #[inline] + fn count(self) -> usize { + let CapturesMatches { re, mut cache, it, .. } = self; + // This does the deref for PoolGuard once instead of every iter. + let cache = &mut *cache; + it.into_half_matches_iter( + |input| Ok(re.search_half_with(cache, input)), + ) + .count() + } +} + +impl<'r, 'h> core::iter::FusedIterator for CapturesMatches<'r, 'h> {} + +/// Yields all substrings delimited by a regular expression match. +/// +/// The spans correspond to the offsets between matches. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the `Regex` that produced this iterator. +/// * `'h` represents the lifetime of the haystack being searched. +/// +/// This iterator can be created with the [`Regex::split`] method. +#[derive(Debug)] +pub struct Split<'r, 'h> { + finder: FindMatches<'r, 'h>, + last: usize, +} + +impl<'r, 'h> Split<'r, 'h> { + /// Returns the current `Input` associated with this iterator. + /// + /// The `start` position on the given `Input` may change during iteration, + /// but all other values are guaranteed to remain invariant. + #[inline] + pub fn input<'s>(&'s self) -> &'s Input<'h> { + self.finder.input() + } +} + +impl<'r, 'h> Iterator for Split<'r, 'h> { + type Item = Span; + + fn next(&mut self) -> Option<Span> { + match self.finder.next() { + None => { + let len = self.finder.it.input().haystack().len(); + if self.last > len { + None + } else { + let span = Span::from(self.last..len); + self.last = len + 1; // Next call will return None + Some(span) + } + } + Some(m) => { + let span = Span::from(self.last..m.start()); + self.last = m.end(); + Some(span) + } + } + } +} + +impl<'r, 'h> core::iter::FusedIterator for Split<'r, 'h> {} + +/// Yields at most `N` spans delimited by a regular expression match. +/// +/// The spans correspond to the offsets between matches. The last span will be +/// whatever remains after splitting. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the `Regex` that produced this iterator. +/// * `'h` represents the lifetime of the haystack being searched. +/// +/// This iterator can be created with the [`Regex::splitn`] method. +#[derive(Debug)] +pub struct SplitN<'r, 'h> { + splits: Split<'r, 'h>, + limit: usize, +} + +impl<'r, 'h> SplitN<'r, 'h> { + /// Returns the current `Input` associated with this iterator. + /// + /// The `start` position on the given `Input` may change during iteration, + /// but all other values are guaranteed to remain invariant. + #[inline] + pub fn input<'s>(&'s self) -> &'s Input<'h> { + self.splits.input() + } +} + +impl<'r, 'h> Iterator for SplitN<'r, 'h> { + type Item = Span; + + fn next(&mut self) -> Option<Span> { + if self.limit == 0 { + return None; + } + + self.limit -= 1; + if self.limit > 0 { + return self.splits.next(); + } + + let len = self.splits.finder.it.input().haystack().len(); + if self.splits.last > len { + // We've already returned all substrings. + None + } else { + // self.n == 0, so future calls will return None immediately + Some(Span::from(self.splits.last..len)) + } + } + + fn size_hint(&self) -> (usize, Option<usize>) { + (0, Some(self.limit)) + } +} + +impl<'r, 'h> core::iter::FusedIterator for SplitN<'r, 'h> {} + +/// Represents mutable scratch space used by regex engines during a search. +/// +/// Most of the regex engines in this crate require some kind of +/// mutable state in order to execute a search. This mutable state is +/// explicitly separated from the core regex object (such as a +/// [`thompson::NFA`](crate::nfa::thompson::NFA)) so that the read-only regex +/// object can be shared across multiple threads simultaneously without any +/// synchronization. Conversely, a `Cache` must either be duplicated if using +/// the same `Regex` from multiple threads, or else there must be some kind of +/// synchronization that guarantees exclusive access while it's in use by one +/// thread. +/// +/// A `Regex` attempts to do this synchronization for you by using a thread +/// pool internally. Its size scales roughly with the number of simultaneous +/// regex searches. +/// +/// For cases where one does not want to rely on a `Regex`'s internal thread +/// pool, lower level routines such as [`Regex::search_with`] are provided +/// that permit callers to pass a `Cache` into the search routine explicitly. +/// +/// General advice is that the thread pool is often more than good enough. +/// However, it may be possible to observe the effects of its latency, +/// especially when searching many small haystacks from many threads +/// simultaneously. +/// +/// Caches can be created from their corresponding `Regex` via +/// [`Regex::create_cache`]. A cache can only be used with either the `Regex` +/// that created it, or the `Regex` that was most recently used to reset it +/// with [`Cache::reset`]. Using a cache with any other `Regex` may result in +/// panics or incorrect results. +/// +/// # Example +/// +/// ``` +/// use regex_automata::{meta::Regex, Input, Match}; +/// +/// let re = Regex::new(r"(?-u)m\w+\s+m\w+")?; +/// let mut cache = re.create_cache(); +/// let input = Input::new("crazy janey and her mission man"); +/// assert_eq!( +/// Some(Match::must(0, 20..31)), +/// re.search_with(&mut cache, &input), +/// ); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Debug, Clone)] +pub struct Cache { + pub(crate) capmatches: Captures, + pub(crate) pikevm: wrappers::PikeVMCache, + pub(crate) backtrack: wrappers::BoundedBacktrackerCache, + pub(crate) onepass: wrappers::OnePassCache, + pub(crate) hybrid: wrappers::HybridCache, + pub(crate) revhybrid: wrappers::ReverseHybridCache, +} + +impl Cache { + /// Creates a new `Cache` for use with this regex. + /// + /// The cache returned should only be used for searches for the given + /// `Regex`. If you want to reuse the cache for another `Regex`, then you + /// must call [`Cache::reset`] with that `Regex`. + pub fn new(re: &Regex) -> Cache { + re.create_cache() + } + + /// Reset this cache such that it can be used for searching with the given + /// `Regex` (and only that `Regex`). + /// + /// A cache reset permits potentially reusing memory already allocated in + /// this cache with a different `Regex`. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different `Regex`. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{meta::Regex, Match, Input}; + /// + /// let re1 = Regex::new(r"\w")?; + /// let re2 = Regex::new(r"\W")?; + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 0..2)), + /// re1.search_with(&mut cache, &Input::new("Δ")), + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the Regex we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// cache.reset(&re2); + /// assert_eq!( + /// Some(Match::must(0, 0..3)), + /// re2.search_with(&mut cache, &Input::new("☃")), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset(&mut self, re: &Regex) { + re.imp.strat.reset_cache(self) + } + + /// Returns the heap memory usage, in bytes, of this cache. + /// + /// This does **not** include the stack size used up by this cache. To + /// compute that, use `std::mem::size_of::<Cache>()`. + pub fn memory_usage(&self) -> usize { + let mut bytes = 0; + bytes += self.pikevm.memory_usage(); + bytes += self.backtrack.memory_usage(); + bytes += self.onepass.memory_usage(); + bytes += self.hybrid.memory_usage(); + bytes += self.revhybrid.memory_usage(); + bytes + } +} + +/// An object describing the configuration of a `Regex`. +/// +/// This configuration only includes options for the +/// non-syntax behavior of a `Regex`, and can be applied via the +/// [`Builder::configure`] method. For configuring the syntax options, see +/// [`util::syntax::Config`](crate::util::syntax::Config). +/// +/// # Example: lower the NFA size limit +/// +/// In some cases, the default size limit might be too big. The size limit can +/// be lowered, which will prevent large regex patterns from compiling. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::meta::Regex; +/// +/// let result = Regex::builder() +/// .configure(Regex::config().nfa_size_limit(Some(20 * (1<<10)))) +/// // Not even 20KB is enough to build a single large Unicode class! +/// .build(r"\pL"); +/// assert!(result.is_err()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug, Default)] +pub struct Config { + // As with other configuration types in this crate, we put all our knobs + // in options so that we can distinguish between "default" and "not set." + // This makes it possible to easily combine multiple configurations + // without default values overwriting explicitly specified values. See the + // 'overwrite' method. + // + // For docs on the fields below, see the corresponding method setters. + match_kind: Option<MatchKind>, + utf8_empty: Option<bool>, + autopre: Option<bool>, + pre: Option<Option<Prefilter>>, + which_captures: Option<WhichCaptures>, + nfa_size_limit: Option<Option<usize>>, + onepass_size_limit: Option<Option<usize>>, + hybrid_cache_capacity: Option<usize>, + hybrid: Option<bool>, + dfa: Option<bool>, + dfa_size_limit: Option<Option<usize>>, + dfa_state_limit: Option<Option<usize>>, + onepass: Option<bool>, + backtrack: Option<bool>, + byte_classes: Option<bool>, + line_terminator: Option<u8>, + pool_capacity: Option<usize>, +} + +impl Config { + /// Create a new configuration object for a `Regex`. + pub fn new() -> Config { + Config::default() + } + + /// Set the match semantics for a `Regex`. + /// + /// The default value is [`MatchKind::LeftmostFirst`]. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Match, MatchKind}; + /// + /// // By default, leftmost-first semantics are used, which + /// // disambiguates matches at the same position by selecting + /// // the one that corresponds earlier in the pattern. + /// let re = Regex::new("sam|samwise")?; + /// assert_eq!(Some(Match::must(0, 0..3)), re.find("samwise")); + /// + /// // But with 'all' semantics, match priority is ignored + /// // and all match states are included. When coupled with + /// // a leftmost search, the search will report the last + /// // possible match. + /// let re = Regex::builder() + /// .configure(Regex::config().match_kind(MatchKind::All)) + /// .build("sam|samwise")?; + /// assert_eq!(Some(Match::must(0, 0..7)), re.find("samwise")); + /// // Beware that this can lead to skipping matches! + /// // Usually 'all' is used for anchored reverse searches + /// // only, or for overlapping searches. + /// assert_eq!(Some(Match::must(0, 4..11)), re.find("sam samwise")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn match_kind(self, kind: MatchKind) -> Config { + Config { match_kind: Some(kind), ..self } + } + + /// Toggles whether empty matches are permitted to occur between the code + /// units of a UTF-8 encoded codepoint. + /// + /// This should generally be enabled when search a `&str` or anything that + /// you otherwise know is valid UTF-8. It should be disabled in all other + /// cases. Namely, if the haystack is not valid UTF-8 and this is enabled, + /// then behavior is unspecified. + /// + /// By default, this is enabled. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Match}; + /// + /// let re = Regex::new("")?; + /// let got: Vec<Match> = re.find_iter("☃").collect(); + /// // Matches only occur at the beginning and end of the snowman. + /// assert_eq!(got, vec![ + /// Match::must(0, 0..0), + /// Match::must(0, 3..3), + /// ]); + /// + /// let re = Regex::builder() + /// .configure(Regex::config().utf8_empty(false)) + /// .build("")?; + /// let got: Vec<Match> = re.find_iter("☃").collect(); + /// // Matches now occur at every position! + /// assert_eq!(got, vec![ + /// Match::must(0, 0..0), + /// Match::must(0, 1..1), + /// Match::must(0, 2..2), + /// Match::must(0, 3..3), + /// ]); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn utf8_empty(self, yes: bool) -> Config { + Config { utf8_empty: Some(yes), ..self } + } + + /// Toggles whether automatic prefilter support is enabled. + /// + /// If this is disabled and [`Config::prefilter`] is not set, then the + /// meta regex engine will not use any prefilters. This can sometimes + /// be beneficial in cases where you know (or have measured) that the + /// prefilter leads to overall worse search performance. + /// + /// By default, this is enabled. + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{meta::Regex, Match}; + /// + /// let re = Regex::builder() + /// .configure(Regex::config().auto_prefilter(false)) + /// .build(r"Bruce \w+")?; + /// let hay = "Hello Bruce Springsteen!"; + /// assert_eq!(Some(Match::must(0, 6..23)), re.find(hay)); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn auto_prefilter(self, yes: bool) -> Config { + Config { autopre: Some(yes), ..self } + } + + /// Overrides and sets the prefilter to use inside a `Regex`. + /// + /// This permits one to forcefully set a prefilter in cases where the + /// caller knows better than whatever the automatic prefilter logic is + /// capable of. + /// + /// By default, this is set to `None` and an automatic prefilter will be + /// used if one could be built. (Assuming [`Config::auto_prefilter`] is + /// enabled, which it is by default.) + /// + /// # Example + /// + /// This example shows how to set your own prefilter. In the case of a + /// pattern like `Bruce \w+`, the automatic prefilter is likely to be + /// constructed in a way that it will look for occurrences of `Bruce `. + /// In most cases, this is the best choice. But in some cases, it may be + /// the case that running `memchr` on `B` is the best choice. One can + /// achieve that behavior by overriding the automatic prefilter logic + /// and providing a prefilter that just matches `B`. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// meta::Regex, + /// util::prefilter::Prefilter, + /// Match, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["B"]) + /// .expect("a prefilter"); + /// let re = Regex::builder() + /// .configure(Regex::config().prefilter(Some(pre))) + /// .build(r"Bruce \w+")?; + /// let hay = "Hello Bruce Springsteen!"; + /// assert_eq!(Some(Match::must(0, 6..23)), re.find(hay)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: incorrect prefilters can lead to incorrect results! + /// + /// Be warned that setting an incorrect prefilter can lead to missed + /// matches. So if you use this option, ensure your prefilter can _never_ + /// report false negatives. (A false positive is, on the other hand, quite + /// okay and generally unavoidable.) + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// meta::Regex, + /// util::prefilter::Prefilter, + /// Match, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["Z"]) + /// .expect("a prefilter"); + /// let re = Regex::builder() + /// .configure(Regex::config().prefilter(Some(pre))) + /// .build(r"Bruce \w+")?; + /// let hay = "Hello Bruce Springsteen!"; + /// // Oops! No match found, but there should be one! + /// assert_eq!(None, re.find(hay)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn prefilter(self, pre: Option<Prefilter>) -> Config { + Config { pre: Some(pre), ..self } + } + + /// Configures what kinds of groups are compiled as "capturing" in the + /// underlying regex engine. + /// + /// This is set to [`WhichCaptures::All`] by default. Callers may wish to + /// use [`WhichCaptures::Implicit`] in cases where one wants avoid the + /// overhead of capture states for explicit groups. + /// + /// Note that another approach to avoiding the overhead of capture groups + /// is by using non-capturing groups in the regex pattern. That is, + /// `(?:a)` instead of `(a)`. This option is useful when you can't control + /// the concrete syntax but know that you don't need the underlying capture + /// states. For example, using `WhichCaptures::Implicit` will behave as if + /// all explicit capturing groups in the pattern were non-capturing. + /// + /// Setting this to `WhichCaptures::None` is usually not the right thing to + /// do. When no capture states are compiled, some regex engines (such as + /// the `PikeVM`) won't be able to report match offsets. This will manifest + /// as no match being found. Indeed, in order to enforce consistent + /// behavior, the meta regex engine will always report `None` for routines + /// that return match offsets even if one of its regex engines could + /// service the request. This avoids "match or not" behavior from being + /// influenced by user input (since user input can influence the selection + /// of the regex engine). + /// + /// # Example + /// + /// This example demonstrates how the results of capture groups can change + /// based on this option. First we show the default (all capture groups in + /// the pattern are capturing): + /// + /// ``` + /// use regex_automata::{meta::Regex, Match, Span}; + /// + /// let re = Regex::new(r"foo([0-9]+)bar")?; + /// let hay = "foo123bar"; + /// + /// let mut caps = re.create_captures(); + /// re.captures(hay, &mut caps); + /// assert_eq!(Some(Span::from(0..9)), caps.get_group(0)); + /// assert_eq!(Some(Span::from(3..6)), caps.get_group(1)); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And now we show the behavior when we only include implicit capture + /// groups. In this case, we can only find the overall match span, but the + /// spans of any other explicit group don't exist because they are treated + /// as non-capturing. (In effect, when `WhichCaptures::Implicit` is used, + /// there is no real point in using [`Regex::captures`] since it will never + /// be able to report more information than [`Regex::find`].) + /// + /// ``` + /// use regex_automata::{ + /// meta::Regex, + /// nfa::thompson::WhichCaptures, + /// Match, + /// Span, + /// }; + /// + /// let re = Regex::builder() + /// .configure(Regex::config().which_captures(WhichCaptures::Implicit)) + /// .build(r"foo([0-9]+)bar")?; + /// let hay = "foo123bar"; + /// + /// let mut caps = re.create_captures(); + /// re.captures(hay, &mut caps); + /// assert_eq!(Some(Span::from(0..9)), caps.get_group(0)); + /// assert_eq!(None, caps.get_group(1)); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: strange `Regex::find` behavior + /// + /// As noted above, when using [`WhichCaptures::None`], this means that + /// `Regex::is_match` could return `true` while `Regex::find` returns + /// `None`: + /// + /// ``` + /// use regex_automata::{ + /// meta::Regex, + /// nfa::thompson::WhichCaptures, + /// Input, + /// Match, + /// Span, + /// }; + /// + /// let re = Regex::builder() + /// .configure(Regex::config().which_captures(WhichCaptures::None)) + /// .build(r"foo([0-9]+)bar")?; + /// let hay = "foo123bar"; + /// + /// assert!(re.is_match(hay)); + /// assert_eq!(re.find(hay), None); + /// assert_eq!(re.search_half(&Input::new(hay)), None); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn which_captures(mut self, which_captures: WhichCaptures) -> Config { + self.which_captures = Some(which_captures); + self + } + + /// Sets the size limit, in bytes, to enforce on the construction of every + /// NFA build by the meta regex engine. + /// + /// Setting it to `None` disables the limit. This is not recommended if + /// you're compiling untrusted patterns. + /// + /// Note that this limit is applied to _each_ NFA built, and if any of + /// them exceed the limit, then construction will fail. This limit does + /// _not_ correspond to the total memory used by all NFAs in the meta regex + /// engine. + /// + /// This defaults to some reasonable number that permits most reasonable + /// patterns. + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::meta::Regex; + /// + /// let result = Regex::builder() + /// .configure(Regex::config().nfa_size_limit(Some(20 * (1<<10)))) + /// // Not even 20KB is enough to build a single large Unicode class! + /// .build(r"\pL"); + /// assert!(result.is_err()); + /// + /// // But notice that building such a regex with the exact same limit + /// // can succeed depending on other aspects of the configuration. For + /// // example, a single *forward* NFA will (at time of writing) fit into + /// // the 20KB limit, but a *reverse* NFA of the same pattern will not. + /// // So if one configures a meta regex such that a reverse NFA is never + /// // needed and thus never built, then the 20KB limit will be enough for + /// // a pattern like \pL! + /// let result = Regex::builder() + /// .configure(Regex::config() + /// .nfa_size_limit(Some(20 * (1<<10))) + /// // The DFAs are the only thing that (currently) need a reverse + /// // NFA. So if both are disabled, the meta regex engine will + /// // skip building the reverse NFA. Note that this isn't an API + /// // guarantee. A future semver compatible version may introduce + /// // new use cases for a reverse NFA. + /// .hybrid(false) + /// .dfa(false) + /// ) + /// // Not even 20KB is enough to build a single large Unicode class! + /// .build(r"\pL"); + /// assert!(result.is_ok()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn nfa_size_limit(self, limit: Option<usize>) -> Config { + Config { nfa_size_limit: Some(limit), ..self } + } + + /// Sets the size limit, in bytes, for the one-pass DFA. + /// + /// Setting it to `None` disables the limit. Disabling the limit is + /// strongly discouraged when compiling untrusted patterns. Even if the + /// patterns are trusted, it still may not be a good idea, since a one-pass + /// DFA can use a lot of memory. With that said, as the size of a regex + /// increases, the likelihood of it being one-pass likely decreases. + /// + /// This defaults to some reasonable number that permits most reasonable + /// one-pass patterns. + /// + /// # Example + /// + /// This shows how to set the one-pass DFA size limit. Note that since + /// a one-pass DFA is an optional component of the meta regex engine, + /// this size limit only impacts what is built internally and will never + /// determine whether a `Regex` itself fails to build. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::meta::Regex; + /// + /// let result = Regex::builder() + /// .configure(Regex::config().onepass_size_limit(Some(2 * (1<<20)))) + /// .build(r"\pL{5}"); + /// assert!(result.is_ok()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn onepass_size_limit(self, limit: Option<usize>) -> Config { + Config { onepass_size_limit: Some(limit), ..self } + } + + /// Set the cache capacity, in bytes, for the lazy DFA. + /// + /// The cache capacity of the lazy DFA determines approximately how much + /// heap memory it is allowed to use to store its state transitions. The + /// state transitions are computed at search time, and if the cache fills + /// up it, it is cleared. At this point, any previously generated state + /// transitions are lost and are re-generated if they're needed again. + /// + /// This sort of cache filling and clearing works quite well _so long as + /// cache clearing happens infrequently_. If it happens too often, then the + /// meta regex engine will stop using the lazy DFA and switch over to a + /// different regex engine. + /// + /// In cases where the cache is cleared too often, it may be possible to + /// give the cache more space and reduce (or eliminate) how often it is + /// cleared. Similarly, sometimes a regex is so big that the lazy DFA isn't + /// used at all if its cache capacity isn't big enough. + /// + /// The capacity set here is a _limit_ on how much memory is used. The + /// actual memory used is only allocated as it's needed. + /// + /// Determining the right value for this is a little tricky and will likely + /// required some profiling. Enabling the `logging` feature and setting the + /// log level to `trace` will also tell you how often the cache is being + /// cleared. + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::meta::Regex; + /// + /// let result = Regex::builder() + /// .configure(Regex::config().hybrid_cache_capacity(20 * (1<<20))) + /// .build(r"\pL{5}"); + /// assert!(result.is_ok()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn hybrid_cache_capacity(self, limit: usize) -> Config { + Config { hybrid_cache_capacity: Some(limit), ..self } + } + + /// Sets the size limit, in bytes, for heap memory used for a fully + /// compiled DFA. + /// + /// **NOTE:** If you increase this, you'll likely also need to increase + /// [`Config::dfa_state_limit`]. + /// + /// In contrast to the lazy DFA, building a full DFA requires computing + /// all of its state transitions up front. This can be a very expensive + /// process, and runs in worst case `2^n` time and space (where `n` is + /// proportional to the size of the regex). However, a full DFA unlocks + /// some additional optimization opportunities. + /// + /// Because full DFAs can be so expensive, the default limits for them are + /// incredibly small. Generally speaking, if your regex is moderately big + /// or if you're using Unicode features (`\w` is Unicode-aware by default + /// for example), then you can expect that the meta regex engine won't even + /// attempt to build a DFA for it. + /// + /// If this and [`Config::dfa_state_limit`] are set to `None`, then the + /// meta regex will not use any sort of limits when deciding whether to + /// build a DFA. This in turn makes construction of a `Regex` take + /// worst case exponential time and space. Even short patterns can result + /// in huge space blow ups. So it is strongly recommended to keep some kind + /// of limit set! + /// + /// The default is set to a small number that permits some simple regexes + /// to get compiled into DFAs in reasonable time. + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::meta::Regex; + /// + /// let result = Regex::builder() + /// // 100MB is much bigger than the default. + /// .configure(Regex::config() + /// .dfa_size_limit(Some(100 * (1<<20))) + /// // We don't care about size too much here, so just + /// // remove the NFA state limit altogether. + /// .dfa_state_limit(None)) + /// .build(r"\pL{5}"); + /// assert!(result.is_ok()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn dfa_size_limit(self, limit: Option<usize>) -> Config { + Config { dfa_size_limit: Some(limit), ..self } + } + + /// Sets a limit on the total number of NFA states, beyond which, a full + /// DFA is not attempted to be compiled. + /// + /// This limit works in concert with [`Config::dfa_size_limit`]. Namely, + /// where as `Config::dfa_size_limit` is applied by attempting to construct + /// a DFA, this limit is used to avoid the attempt in the first place. This + /// is useful to avoid hefty initialization costs associated with building + /// a DFA for cases where it is obvious the DFA will ultimately be too big. + /// + /// By default, this is set to a very small number. + /// + /// # Example + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::meta::Regex; + /// + /// let result = Regex::builder() + /// .configure(Regex::config() + /// // Sometimes the default state limit rejects DFAs even + /// // if they would fit in the size limit. Here, we disable + /// // the check on the number of NFA states and just rely on + /// // the size limit. + /// .dfa_state_limit(None)) + /// .build(r"(?-u)\w{30}"); + /// assert!(result.is_ok()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn dfa_state_limit(self, limit: Option<usize>) -> Config { + Config { dfa_state_limit: Some(limit), ..self } + } + + /// Whether to attempt to shrink the size of the alphabet for the regex + /// pattern or not. When enabled, the alphabet is shrunk into a set of + /// equivalence classes, where every byte in the same equivalence class + /// cannot discriminate between a match or non-match. + /// + /// **WARNING:** This is only useful for debugging DFAs. Disabling this + /// does not yield any speed advantages. Indeed, disabling it can result + /// in much higher memory usage. Disabling byte classes is useful for + /// debugging the actual generated transitions because it lets one see the + /// transitions defined on actual bytes instead of the equivalence classes. + /// + /// This option is enabled by default and should never be disabled unless + /// one is debugging the meta regex engine's internals. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, Match}; + /// + /// let re = Regex::builder() + /// .configure(Regex::config().byte_classes(false)) + /// .build(r"[a-z]+")?; + /// let hay = "!!quux!!"; + /// assert_eq!(Some(Match::must(0, 2..6)), re.find(hay)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn byte_classes(self, yes: bool) -> Config { + Config { byte_classes: Some(yes), ..self } + } + + /// Set the line terminator to be used by the `^` and `$` anchors in + /// multi-line mode. + /// + /// This option has no effect when CRLF mode is enabled. That is, + /// regardless of this setting, `(?Rm:^)` and `(?Rm:$)` will always treat + /// `\r` and `\n` as line terminators (and will never match between a `\r` + /// and a `\n`). + /// + /// By default, `\n` is the line terminator. + /// + /// **Warning**: This does not change the behavior of `.`. To do that, + /// you'll need to configure the syntax option + /// [`syntax::Config::line_terminator`](crate::util::syntax::Config::line_terminator) + /// in addition to this. Otherwise, `.` will continue to match any + /// character other than `\n`. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{meta::Regex, util::syntax, Match}; + /// + /// let re = Regex::builder() + /// .syntax(syntax::Config::new().multi_line(true)) + /// .configure(Regex::config().line_terminator(b'\x00')) + /// .build(r"^foo$")?; + /// let hay = "\x00foo\x00"; + /// assert_eq!(Some(Match::must(0, 1..4)), re.find(hay)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn line_terminator(self, byte: u8) -> Config { + Config { line_terminator: Some(byte), ..self } + } + + /// Sets the capacity used to manage a pool of [`Cache`] values in the + /// higher level convenience APIs. + /// + /// When not configured explicitly, a reasonable default is selected. It + /// is rarely expected that a number large than the number of logical CPUs + /// makes sense as a value. A smaller number could result in slowdowns if + /// many regex queries are run under contention. + pub fn pool_capacity(self, capacity: usize) -> Config { + Config { pool_capacity: Some(capacity), ..self } + } + + /// Toggle whether the hybrid NFA/DFA (also known as the "lazy DFA") should + /// be available for use by the meta regex engine. + /// + /// Enabling this does not necessarily mean that the lazy DFA will + /// definitely be used. It just means that it will be _available_ for use + /// if the meta regex engine thinks it will be useful. + /// + /// When the `hybrid` crate feature is enabled, then this is enabled by + /// default. Otherwise, if the crate feature is disabled, then this is + /// always disabled, regardless of its setting by the caller. + pub fn hybrid(self, yes: bool) -> Config { + Config { hybrid: Some(yes), ..self } + } + + /// Toggle whether a fully compiled DFA should be available for use by the + /// meta regex engine. + /// + /// Enabling this does not necessarily mean that a DFA will definitely be + /// used. It just means that it will be _available_ for use if the meta + /// regex engine thinks it will be useful. + /// + /// When the `dfa-build` crate feature is enabled, then this is enabled by + /// default. Otherwise, if the crate feature is disabled, then this is + /// always disabled, regardless of its setting by the caller. + pub fn dfa(self, yes: bool) -> Config { + Config { dfa: Some(yes), ..self } + } + + /// Toggle whether a one-pass DFA should be available for use by the meta + /// regex engine. + /// + /// Enabling this does not necessarily mean that a one-pass DFA will + /// definitely be used. It just means that it will be _available_ for + /// use if the meta regex engine thinks it will be useful. (Indeed, a + /// one-pass DFA can only be used when the regex is one-pass. See the + /// [`dfa::onepass`](crate::dfa::onepass) module for more details.) + /// + /// When the `dfa-onepass` crate feature is enabled, then this is enabled + /// by default. Otherwise, if the crate feature is disabled, then this is + /// always disabled, regardless of its setting by the caller. + pub fn onepass(self, yes: bool) -> Config { + Config { onepass: Some(yes), ..self } + } + + /// Toggle whether a bounded backtracking regex engine should be available + /// for use by the meta regex engine. + /// + /// Enabling this does not necessarily mean that a bounded backtracker will + /// definitely be used. It just means that it will be _available_ for use + /// if the meta regex engine thinks it will be useful. + /// + /// When the `nfa-backtrack` crate feature is enabled, then this is enabled + /// by default. Otherwise, if the crate feature is disabled, then this is + /// always disabled, regardless of its setting by the caller. + pub fn backtrack(self, yes: bool) -> Config { + Config { backtrack: Some(yes), ..self } + } + + /// Returns the match kind on this configuration, as set by + /// [`Config::match_kind`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_match_kind(&self) -> MatchKind { + self.match_kind.unwrap_or(MatchKind::LeftmostFirst) + } + + /// Returns whether empty matches must fall on valid UTF-8 boundaries, as + /// set by [`Config::utf8_empty`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_utf8_empty(&self) -> bool { + self.utf8_empty.unwrap_or(true) + } + + /// Returns whether automatic prefilters are enabled, as set by + /// [`Config::auto_prefilter`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_auto_prefilter(&self) -> bool { + self.autopre.unwrap_or(true) + } + + /// Returns a manually set prefilter, if one was set by + /// [`Config::prefilter`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_prefilter(&self) -> Option<&Prefilter> { + self.pre.as_ref().unwrap_or(&None).as_ref() + } + + /// Returns the capture configuration, as set by + /// [`Config::which_captures`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_which_captures(&self) -> WhichCaptures { + self.which_captures.unwrap_or(WhichCaptures::All) + } + + /// Returns NFA size limit, as set by [`Config::nfa_size_limit`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_nfa_size_limit(&self) -> Option<usize> { + self.nfa_size_limit.unwrap_or(Some(10 * (1 << 20))) + } + + /// Returns one-pass DFA size limit, as set by + /// [`Config::onepass_size_limit`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_onepass_size_limit(&self) -> Option<usize> { + self.onepass_size_limit.unwrap_or(Some(1 * (1 << 20))) + } + + /// Returns hybrid NFA/DFA cache capacity, as set by + /// [`Config::hybrid_cache_capacity`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_hybrid_cache_capacity(&self) -> usize { + self.hybrid_cache_capacity.unwrap_or(2 * (1 << 20)) + } + + /// Returns DFA size limit, as set by [`Config::dfa_size_limit`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_dfa_size_limit(&self) -> Option<usize> { + // The default for this is VERY small because building a full DFA is + // ridiculously costly. But for regexes that are very small, it can be + // beneficial to use a full DFA. In particular, a full DFA can enable + // additional optimizations via something called "accelerated" states. + // Namely, when there's a state with only a few outgoing transitions, + // we can temporary suspend walking the transition table and use memchr + // for just those outgoing transitions to skip ahead very quickly. + // + // Generally speaking, if Unicode is enabled in your regex and you're + // using some kind of Unicode feature, then it's going to blow this + // size limit. Moreover, Unicode tends to defeat the "accelerated" + // state optimization too, so it's a double whammy. + // + // We also use a limit on the number of NFA states to avoid even + // starting the DFA construction process. Namely, DFA construction + // itself could make lots of initial allocs proportional to the size + // of the NFA, and if the NFA is large, it doesn't make sense to pay + // that cost if we know it's likely to be blown by a large margin. + self.dfa_size_limit.unwrap_or(Some(40 * (1 << 10))) + } + + /// Returns DFA size limit in terms of the number of states in the NFA, as + /// set by [`Config::dfa_state_limit`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_dfa_state_limit(&self) -> Option<usize> { + // Again, as with the size limit, we keep this very small. + self.dfa_state_limit.unwrap_or(Some(30)) + } + + /// Returns whether byte classes are enabled, as set by + /// [`Config::byte_classes`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_byte_classes(&self) -> bool { + self.byte_classes.unwrap_or(true) + } + + /// Returns the line terminator for this configuration, as set by + /// [`Config::line_terminator`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_line_terminator(&self) -> u8 { + self.line_terminator.unwrap_or(b'\n') + } + + /// Returns the configured pool capacity, as set by + /// [`Config::pool_capacity`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_pool_capacity(&self) -> usize { + // The default is an empirically chosen value that balances memory + // usage with runtime performance. In practice, with `std` enabled, + // we choose a value that matches the total number of CPUs. + const DEFAULT_POOL_CAPACITY: usize = 8; + + self.pool_capacity.unwrap_or_else(|| { + #[cfg(feature = "std")] + { + use crate::util::lazy::Lazy; + + static AVAILABLE_PARALLELISM: Lazy<usize> = Lazy::new(|| { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(DEFAULT_POOL_CAPACITY) + }); + *Lazy::get(&AVAILABLE_PARALLELISM) + } + #[cfg(not(feature = "std"))] + { + DEFAULT_POOL_CAPACITY + } + }) + } + + /// Returns whether the hybrid NFA/DFA regex engine may be used, as set by + /// [`Config::hybrid`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_hybrid(&self) -> bool { + #[cfg(feature = "hybrid")] + { + self.hybrid.unwrap_or(true) + } + #[cfg(not(feature = "hybrid"))] + { + false + } + } + + /// Returns whether the DFA regex engine may be used, as set by + /// [`Config::dfa`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_dfa(&self) -> bool { + #[cfg(feature = "dfa-build")] + { + self.dfa.unwrap_or(true) + } + #[cfg(not(feature = "dfa-build"))] + { + false + } + } + + /// Returns whether the one-pass DFA regex engine may be used, as set by + /// [`Config::onepass`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_onepass(&self) -> bool { + #[cfg(feature = "dfa-onepass")] + { + self.onepass.unwrap_or(true) + } + #[cfg(not(feature = "dfa-onepass"))] + { + false + } + } + + /// Returns whether the bounded backtracking regex engine may be used, as + /// set by [`Config::backtrack`]. + /// + /// If it was not explicitly set, then a default value is returned. + pub fn get_backtrack(&self) -> bool { + #[cfg(feature = "nfa-backtrack")] + { + self.backtrack.unwrap_or(true) + } + #[cfg(not(feature = "nfa-backtrack"))] + { + false + } + } + + /// Returns a "baseline" Thompson configuration for constructing NFAs based + /// on this configuration. + /// + /// This is just a convenience routine to avoid repeating configuration + /// construction. + /// + /// Callers may still need to set other things, like whether the NFA should + /// be compiled in reverse. Callers may also override settings, like + /// forcing no capture states to be included. + pub(crate) fn to_thompson_config(&self) -> thompson::Config { + let mut lookm = LookMatcher::new(); + lookm.set_line_terminator(self.get_line_terminator()); + thompson::Config::new() + .utf8(self.get_utf8_empty()) + .reverse(false) + .nfa_size_limit(self.get_nfa_size_limit()) + .shrink(false) + .which_captures(self.get_which_captures()) + .look_matcher(lookm) + } + + /// Overwrite the default configuration such that the options in `o` are + /// always used. If an option in `o` is not set, then the corresponding + /// option in `self` is used. If it's not set in `self` either, then it + /// remains not set. + pub(crate) fn overwrite(&self, o: Config) -> Config { + Config { + match_kind: o.match_kind.or(self.match_kind), + utf8_empty: o.utf8_empty.or(self.utf8_empty), + autopre: o.autopre.or(self.autopre), + pre: o.pre.or_else(|| self.pre.clone()), + which_captures: o.which_captures.or(self.which_captures), + nfa_size_limit: o.nfa_size_limit.or(self.nfa_size_limit), + onepass_size_limit: o + .onepass_size_limit + .or(self.onepass_size_limit), + hybrid_cache_capacity: o + .hybrid_cache_capacity + .or(self.hybrid_cache_capacity), + hybrid: o.hybrid.or(self.hybrid), + dfa: o.dfa.or(self.dfa), + dfa_size_limit: o.dfa_size_limit.or(self.dfa_size_limit), + dfa_state_limit: o.dfa_state_limit.or(self.dfa_state_limit), + onepass: o.onepass.or(self.onepass), + backtrack: o.backtrack.or(self.backtrack), + byte_classes: o.byte_classes.or(self.byte_classes), + line_terminator: o.line_terminator.or(self.line_terminator), + pool_capacity: o.pool_capacity.or(self.pool_capacity), + } + } +} + +/// A builder for configuring and constructing a `Regex`. +/// +/// The builder permits configuring two different aspects of a `Regex`: +/// +/// * [`Builder::configure`] will set high-level configuration options as +/// described by a [`Config`]. +/// * [`Builder::syntax`] will set the syntax level configuration options +/// as described by a [`util::syntax::Config`](crate::util::syntax::Config). +/// This only applies when building a `Regex` from pattern strings. +/// +/// Once configured, the builder can then be used to construct a `Regex` from +/// one of 4 different inputs: +/// +/// * [`Builder::build`] creates a regex from a single pattern string. +/// * [`Builder::build_many`] creates a regex from many pattern strings. +/// * [`Builder::build_from_hir`] creates a regex from a +/// [`regex-syntax::Hir`](Hir) expression. +/// * [`Builder::build_many_from_hir`] creates a regex from many +/// [`regex-syntax::Hir`](Hir) expressions. +/// +/// The latter two methods in particular provide a way to construct a fully +/// feature regular expression matcher directly from an `Hir` expression +/// without having to first convert it to a string. (This is in contrast to the +/// top-level `regex` crate which intentionally provides no such API in order +/// to avoid making `regex-syntax` a public dependency.) +/// +/// As a convenience, this builder may be created via [`Regex::builder`], which +/// may help avoid an extra import. +/// +/// # Example: change the line terminator +/// +/// This example shows how to enable multi-line mode by default and change the +/// line terminator to the NUL byte: +/// +/// ``` +/// use regex_automata::{meta::Regex, util::syntax, Match}; +/// +/// let re = Regex::builder() +/// .syntax(syntax::Config::new().multi_line(true)) +/// .configure(Regex::config().line_terminator(b'\x00')) +/// .build(r"^foo$")?; +/// let hay = "\x00foo\x00"; +/// assert_eq!(Some(Match::must(0, 1..4)), re.find(hay)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: disable UTF-8 requirement +/// +/// By default, regex patterns are required to match UTF-8. This includes +/// regex patterns that can produce matches of length zero. In the case of an +/// empty match, by default, matches will not appear between the code units of +/// a UTF-8 encoded codepoint. +/// +/// However, it can be useful to disable this requirement, particularly if +/// you're searching things like `&[u8]` that are not known to be valid UTF-8. +/// +/// ``` +/// use regex_automata::{meta::Regex, util::syntax, Match}; +/// +/// let mut builder = Regex::builder(); +/// // Disables the requirement that non-empty matches match UTF-8. +/// builder.syntax(syntax::Config::new().utf8(false)); +/// // Disables the requirement that empty matches match UTF-8 boundaries. +/// builder.configure(Regex::config().utf8_empty(false)); +/// +/// // We can match raw bytes via \xZZ syntax, but we need to disable +/// // Unicode mode to do that. We could disable it everywhere, or just +/// // selectively, as shown here. +/// let re = builder.build(r"(?-u:\xFF)foo(?-u:\xFF)")?; +/// let hay = b"\xFFfoo\xFF"; +/// assert_eq!(Some(Match::must(0, 0..5)), re.find(hay)); +/// +/// // We can also match between code units. +/// let re = builder.build(r"")?; +/// let hay = "☃"; +/// assert_eq!(re.find_iter(hay).collect::<Vec<Match>>(), vec![ +/// Match::must(0, 0..0), +/// Match::must(0, 1..1), +/// Match::must(0, 2..2), +/// Match::must(0, 3..3), +/// ]); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Builder { + config: Config, + ast: ast::parse::ParserBuilder, + hir: hir::translate::TranslatorBuilder, +} + +impl Builder { + /// Creates a new builder for configuring and constructing a [`Regex`]. + pub fn new() -> Builder { + Builder { + config: Config::default(), + ast: ast::parse::ParserBuilder::new(), + hir: hir::translate::TranslatorBuilder::new(), + } + } + + /// Builds a `Regex` from a single pattern string. + /// + /// If there was a problem parsing the pattern or a problem turning it into + /// a regex matcher, then an error is returned. + /// + /// # Example + /// + /// This example shows how to configure syntax options. + /// + /// ``` + /// use regex_automata::{meta::Regex, util::syntax, Match}; + /// + /// let re = Regex::builder() + /// .syntax(syntax::Config::new().crlf(true).multi_line(true)) + /// .build(r"^foo$")?; + /// let hay = "\r\nfoo\r\n"; + /// assert_eq!(Some(Match::must(0, 2..5)), re.find(hay)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build(&self, pattern: &str) -> Result<Regex, BuildError> { + self.build_many(&[pattern]) + } + + /// Builds a `Regex` from many pattern strings. + /// + /// If there was a problem parsing any of the patterns or a problem turning + /// them into a regex matcher, then an error is returned. + /// + /// # Example: finding the pattern that caused an error + /// + /// When a syntax error occurs, it is possible to ask which pattern + /// caused the syntax error. + /// + /// ``` + /// use regex_automata::{meta::Regex, PatternID}; + /// + /// let err = Regex::builder() + /// .build_many(&["a", "b", r"\p{Foo}", "c"]) + /// .unwrap_err(); + /// assert_eq!(Some(PatternID::must(2)), err.pattern()); + /// ``` + /// + /// # Example: zero patterns is valid + /// + /// Building a regex with zero patterns results in a regex that never + /// matches anything. Because this routine is generic, passing an empty + /// slice usually requires a turbo-fish (or something else to help type + /// inference). + /// + /// ``` + /// use regex_automata::{meta::Regex, util::syntax, Match}; + /// + /// let re = Regex::builder() + /// .build_many::<&str>(&[])?; + /// assert_eq!(None, re.find("")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<Regex, BuildError> { + use crate::util::primitives::IteratorIndexExt; + log! { + debug!("building meta regex with {} patterns:", patterns.len()); + for (pid, p) in patterns.iter().with_pattern_ids() { + let p = p.as_ref(); + // We might split a grapheme with this truncation logic, but + // that's fine. We at least avoid splitting a codepoint. + let maxoff = p + .char_indices() + .map(|(i, ch)| i + ch.len_utf8()) + .take(1000) + .last() + .unwrap_or(0); + if maxoff < p.len() { + debug!("{pid:?}: {}[... snip ...]", &p[..maxoff]); + } else { + debug!("{pid:?}: {p}"); + } + } + } + let (mut asts, mut hirs) = (vec![], vec![]); + for (pid, p) in patterns.iter().with_pattern_ids() { + let ast = self + .ast + .build() + .parse(p.as_ref()) + .map_err(|err| BuildError::ast(pid, err))?; + asts.push(ast); + } + for ((pid, p), ast) in + patterns.iter().with_pattern_ids().zip(asts.iter()) + { + let hir = self + .hir + .build() + .translate(p.as_ref(), ast) + .map_err(|err| BuildError::hir(pid, err))?; + hirs.push(hir); + } + self.build_many_from_hir(&hirs) + } + + /// Builds a `Regex` directly from an `Hir` expression. + /// + /// This is useful if you needed to parse a pattern string into an `Hir` + /// for other reasons (such as analysis or transformations). This routine + /// permits building a `Regex` directly from the `Hir` expression instead + /// of first converting the `Hir` back to a pattern string. + /// + /// When using this method, any options set via [`Builder::syntax`] are + /// ignored. Namely, the syntax options only apply when parsing a pattern + /// string, which isn't relevant here. + /// + /// If there was a problem building the underlying regex matcher for the + /// given `Hir`, then an error is returned. + /// + /// # Example + /// + /// This example shows how one can hand-construct an `Hir` expression and + /// build a regex from it without doing any parsing at all. + /// + /// ``` + /// use { + /// regex_automata::{meta::Regex, Match}, + /// regex_syntax::hir::{Hir, Look}, + /// }; + /// + /// // (?Rm)^foo$ + /// let hir = Hir::concat(vec![ + /// Hir::look(Look::StartCRLF), + /// Hir::literal("foo".as_bytes()), + /// Hir::look(Look::EndCRLF), + /// ]); + /// let re = Regex::builder() + /// .build_from_hir(&hir)?; + /// let hay = "\r\nfoo\r\n"; + /// assert_eq!(Some(Match::must(0, 2..5)), re.find(hay)); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_from_hir(&self, hir: &Hir) -> Result<Regex, BuildError> { + self.build_many_from_hir(&[hir]) + } + + /// Builds a `Regex` directly from many `Hir` expressions. + /// + /// This is useful if you needed to parse pattern strings into `Hir` + /// expressions for other reasons (such as analysis or transformations). + /// This routine permits building a `Regex` directly from the `Hir` + /// expressions instead of first converting the `Hir` expressions back to + /// pattern strings. + /// + /// When using this method, any options set via [`Builder::syntax`] are + /// ignored. Namely, the syntax options only apply when parsing a pattern + /// string, which isn't relevant here. + /// + /// If there was a problem building the underlying regex matcher for the + /// given `Hir` expressions, then an error is returned. + /// + /// Note that unlike [`Builder::build_many`], this can only fail as a + /// result of building the underlying matcher. In that case, there is + /// no single `Hir` expression that can be isolated as a reason for the + /// failure. So if this routine fails, it's not possible to determine which + /// `Hir` expression caused the failure. + /// + /// # Example + /// + /// This example shows how one can hand-construct multiple `Hir` + /// expressions and build a single regex from them without doing any + /// parsing at all. + /// + /// ``` + /// use { + /// regex_automata::{meta::Regex, Match}, + /// regex_syntax::hir::{Hir, Look}, + /// }; + /// + /// // (?Rm)^foo$ + /// let hir1 = Hir::concat(vec![ + /// Hir::look(Look::StartCRLF), + /// Hir::literal("foo".as_bytes()), + /// Hir::look(Look::EndCRLF), + /// ]); + /// // (?Rm)^bar$ + /// let hir2 = Hir::concat(vec![ + /// Hir::look(Look::StartCRLF), + /// Hir::literal("bar".as_bytes()), + /// Hir::look(Look::EndCRLF), + /// ]); + /// let re = Regex::builder() + /// .build_many_from_hir(&[&hir1, &hir2])?; + /// let hay = "\r\nfoo\r\nbar"; + /// let got: Vec<Match> = re.find_iter(hay).collect(); + /// let expected = vec![ + /// Match::must(0, 2..5), + /// Match::must(1, 7..10), + /// ]; + /// assert_eq!(expected, got); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_many_from_hir<H: Borrow<Hir>>( + &self, + hirs: &[H], + ) -> Result<Regex, BuildError> { + let config = self.config.clone(); + // We collect the HIRs into a vec so we can write internal routines + // with '&[&Hir]'. i.e., Don't use generics everywhere to keep code + // bloat down.. + let hirs: Vec<&Hir> = hirs.iter().map(|hir| hir.borrow()).collect(); + let info = RegexInfo::new(config, &hirs); + let strat = strategy::new(&info, &hirs)?; + let pool = { + let strat = Arc::clone(&strat); + let create: CachePoolFn = Box::new(move || strat.create_cache()); + Pool::with_capacity(self.config.get_pool_capacity(), create) + }; + Ok(Regex { imp: Arc::new(RegexI { strat, info }), pool }) + } + + /// Configure the behavior of a `Regex`. + /// + /// This configuration controls non-syntax options related to the behavior + /// of a `Regex`. This includes things like whether empty matches can split + /// a codepoint, prefilters, line terminators and a long list of options + /// for configuring which regex engines the meta regex engine will be able + /// to use internally. + /// + /// # Example + /// + /// This example shows how to disable UTF-8 empty mode. This will permit + /// empty matches to occur between the UTF-8 encoding of a codepoint. + /// + /// ``` + /// use regex_automata::{meta::Regex, Match}; + /// + /// let re = Regex::new("")?; + /// let got: Vec<Match> = re.find_iter("☃").collect(); + /// // Matches only occur at the beginning and end of the snowman. + /// assert_eq!(got, vec![ + /// Match::must(0, 0..0), + /// Match::must(0, 3..3), + /// ]); + /// + /// let re = Regex::builder() + /// .configure(Regex::config().utf8_empty(false)) + /// .build("")?; + /// let got: Vec<Match> = re.find_iter("☃").collect(); + /// // Matches now occur at every position! + /// assert_eq!(got, vec![ + /// Match::must(0, 0..0), + /// Match::must(0, 1..1), + /// Match::must(0, 2..2), + /// Match::must(0, 3..3), + /// ]); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn configure(&mut self, config: Config) -> &mut Builder { + self.config = self.config.overwrite(config); + self + } + + /// Configure the syntax options when parsing a pattern string while + /// building a `Regex`. + /// + /// These options _only_ apply when [`Builder::build`] or [`Builder::build_many`] + /// are used. The other build methods accept `Hir` values, which have + /// already been parsed. + /// + /// # Example + /// + /// This example shows how to enable case insensitive mode. + /// + /// ``` + /// use regex_automata::{meta::Regex, util::syntax, Match}; + /// + /// let re = Regex::builder() + /// .syntax(syntax::Config::new().case_insensitive(true)) + /// .build(r"δ")?; + /// assert_eq!(Some(Match::must(0, 0..2)), re.find(r"Δ")); + /// + /// Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Builder { + config.apply_ast(&mut self.ast); + config.apply_hir(&mut self.hir); + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // I found this in the course of building out the benchmark suite for + // rebar. + #[test] + fn regression_suffix_literal_count() { + let _ = env_logger::try_init(); + + let re = Regex::new(r"[a-zA-Z]+ing").unwrap(); + assert_eq!(1, re.find_iter("tingling").count()); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/reverse_inner.rs b/third_party/crates/regex-automata-0.4.18/src/meta/reverse_inner.rs new file mode 100644 index 0000000..4f93a1e --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/reverse_inner.rs @@ -0,0 +1,303 @@ +/*! +A module dedicated to plucking inner literals out of a regex pattern, and +then constructing a prefilter for them. We also include a regex pattern +"prefix" that corresponds to the bits of the regex that need to match before +the literals do. The reverse inner optimization then proceeds by looking for +matches of the inner literal(s), and then doing a reverse search of the prefix +from the start of the literal match to find the overall start position of the +match. + +The essential invariant we want to uphold here is that the literals we return +reflect a set where *at least* one of them must match in order for the overall +regex to match. We also need to maintain the invariant that the regex prefix +returned corresponds to the entirety of the regex up until the literals we +return. + +This somewhat limits what we can do. That is, if we a regex like +`\w+(@!|%%)\w+`, then we can pluck the `{@!, %%}` out and build a prefilter +from it. Then we just need to compile `\w+` in reverse. No fuss no muss. But if +we have a regex like \d+@!|\w+%%`, then we get kind of stymied. Technically, +we could still extract `{@!, %%}`, and it is true that at least of them must +match. But then, what is our regex prefix? Again, in theory, that could be +`\d+|\w+`, but that's not quite right, because the `\d+` only matches when `@!` +matches, and `\w+` only matches when `%%` matches. + +All of that is technically possible to do, but it seemingly requires a lot of +sophistication and machinery. Probably the way to tackle that is with some kind +of formalism and approach this problem more generally. + +For now, the code below basically just looks for a top-level concatenation. +And if it can find one, it looks for literals in each of the direct child +sub-expressions of that concatenation. If some good ones are found, we return +those and a concatenation of the Hir expressions seen up to that point. +*/ + +use alloc::vec::Vec; + +use regex_syntax::hir::{ + self, + literal::{self, Literal}, + Hir, HirKind, +}; + +use crate::{meta::prefix, util::prefilter::Prefilter, MatchKind}; + +/// Returns true when it's impossible for an earlier match to be detected after +/// a literal candidate (corresponding to anything in `literals`) has +/// been found. +/// +/// Specifically, that there is no earlier match than what a reverse scan of +/// `concat_prefix` after a match of `literals` reports. +/// +/// Since this requires a single `Hir`, this implies the reverse inner optimization +/// only works with a single regex. +pub(super) fn has_no_earlier_match( + concat_prefix: &Hir, + literals: &[Literal], +) -> bool { + // let literals = prefix::LiteralSet::many(literals); + if literals.is_empty() || literals.iter().any(|lit| lit.is_empty()) { + debug!( + "reverse inner is not early return safe because \ + no non-empty inner literals were found" + ); + return false; + } + // With one literal, an occurrence crossing the prefix boundary must + // overlap another occurrence of that same literal. Such an overlap + // requires the prefix to consume every distinct byte in the literal. + // This reasoning does not apply when one extracted literal can cross + // the boundary into a different extracted literal. + if literals.len() == 1 { + let prefix_may_contain = prefix::hir_can_contain_literal( + concat_prefix, + literals[0].as_bytes(), + ); + debug!( + "reverse inner prefix can contain inner literals? \ + {prefix_may_contain}" + ); + if !prefix_may_contain { + return true; + } + } + + let fixed_length = prefix::hir_has_fixed_length(concat_prefix); + debug!("reverse inner has fixed length prefix? {fixed_length}"); + if fixed_length { + return true; + } + + let class_separator = + prefix::has_disjoint_class_separator(concat_prefix, &literals); + debug!("reverse inner has disjoint class separator? {class_separator}"); + if class_separator { + return true; + } + + // We couldn't prove that the reverse inner optimization + // was safe, so bail out. + false +} + +/// This attempts to extract an "inner" prefilter from the given HIR +/// expressions. If one was found, then a concatenation of the HIR expressions +/// that precede it is returned. +/// +/// The idea here is that the prefilter returned can be used to find +/// candidate matches. And then the HIR returned can be used to build a +/// reverse regex matcher, which will find the start of the candidate +/// match. Finally, the match still has to be confirmed with a normal +/// anchored forward scan to find the end position of the match. +/// +/// Note that this assumes leftmost-first match semantics, so callers must +/// not call this otherwise. +#[derive(Debug)] +pub(crate) struct InnerPrefilter { + pub(crate) prefix: Hir, + /// The prefilter generated from `literals`. + pub(crate) pre: Prefilter, + /// The actual literals extracted and used to build `pre`. + /// + /// These are used by the meta strategy to prove that the inner prefilter + /// can return after the first confirmed candidate. If that proof fails, we + /// could try extracting a different set of literals. But we don't + /// currently do that. + pub(crate) literals: Vec<Literal>, +} + +impl InnerPrefilter { + pub(crate) fn new(hirs: &[&Hir]) -> Option<InnerPrefilter> { + if hirs.len() != 1 { + debug!( + "skipping reverse inner optimization since it only \ + supports 1 pattern, {} were given", + hirs.len(), + ); + return None; + } + let mut concat = match top_concat(hirs[0]) { + Some(concat) => concat, + None => { + debug!( + "skipping reverse inner optimization because a top-level \ + concatenation could not found", + ); + return None; + } + }; + // We skip the first HIR because if it did have a prefix prefilter in + // it, we probably wouldn't be here looking for an inner prefilter. + for i in 1..concat.len() { + let hir = &concat[i]; + let (pre, lits) = match prefilter_with_literals(hir) { + None => continue, + Some(pre) => pre, + }; + // Even if we got a prefilter, if it isn't consider "fast," then + // we probably don't want to bother with it. Namely, since the + // reverse inner optimization requires some overhead, it likely + // only makes sense if the prefilter scan itself is (believed) to + // be much faster than the regex engine. + if !pre.is_fast() { + debug!( + "skipping extracted inner prefilter because \ + it probably isn't fast" + ); + continue; + } + let concat_suffix = Hir::concat(concat.split_off(i)); + let concat_prefix = Hir::concat(concat); + // Look for a prefilter again. Why? Because above we only looked + // for a prefilter on the individual 'hir', but we might be able + // to find something better and more discriminatory by looking at + // the entire suffix. We don't do this above to avoid making this + // loop worst case quadratic in the length of 'concat'. + let (preinner, inner_literals) = + match prefilter_with_literals(&concat_suffix) { + None => (pre, lits), + Some((pre2, lits2)) => { + if pre2.is_fast() { + (pre2, lits2) + } else { + (pre, lits) + } + } + }; + return Some(InnerPrefilter { + prefix: concat_prefix, + pre: preinner, + literals: inner_literals, + }); + } + debug!( + "skipping reverse inner optimization because a top-level \ + sub-expression with a fast prefilter could not be found" + ); + None + } +} + +/// Attempt to extract a prefilter from an HIR expression. +/// +/// We do a little massaging here to do our best that the prefilter we get out +/// of this is *probably* fast. Basically, the false positive rate has a much +/// higher impact for things like the reverse inner optimization because more +/// work needs to potentially be done for each candidate match. +/// +/// Note that this assumes leftmost-first match semantics, so callers must +/// not call this otherwise. +fn prefilter_with_literals(hir: &Hir) -> Option<(Prefilter, Vec<Literal>)> { + let mut extractor = literal::Extractor::new(); + extractor.kind(literal::ExtractKind::Prefix); + let mut prefixes = extractor.extract(hir); + debug!( + "inner prefixes (len={:?}) extracted before optimization: {:?}", + prefixes.len(), + prefixes + ); + // Since these are inner literals, we know they cannot be exact. But the + // extractor doesn't know this. We mark them as inexact because this might + // impact literal optimization. Namely, optimization weights "all literals + // are exact" as very high, because it presumes that any match results in + // an overall match. But of course, that is not the case here. + // + // In practice, this avoids plucking out a ASCII-only \s as an alternation + // of single-byte whitespace characters. + prefixes.make_inexact(); + prefixes.optimize_for_prefix_by_preference(); + debug!( + "inner prefixes (len={:?}) extracted after optimization: {:?}", + prefixes.len(), + prefixes + ); + let lits = prefixes.literals()?; + let pre = Prefilter::new(MatchKind::LeftmostFirst, lits)?; + Some((pre, lits.to_vec())) +} + +/// Looks for a "top level" HirKind::Concat item in the given HIR. This will +/// try to return one even if it's embedded in a capturing group, but is +/// otherwise pretty conservative in what is returned. +/// +/// The HIR returned is a complete copy of the concat with all capturing +/// groups removed. In effect, the concat returned is "flattened" with respect +/// to capturing groups. This makes the detection logic above for prefixes +/// a bit simpler, and it works because 1) capturing groups never influence +/// whether a match occurs or not and 2) capturing groups are not used when +/// doing the reverse inner search to find the start of the match. +fn top_concat(mut hir: &Hir) -> Option<Vec<Hir>> { + loop { + hir = match hir.kind() { + HirKind::Empty + | HirKind::Literal(_) + | HirKind::Class(_) + | HirKind::Look(_) + | HirKind::Repetition(_) + | HirKind::Alternation(_) => return None, + HirKind::Capture(hir::Capture { ref sub, .. }) => sub, + HirKind::Concat(ref subs) => { + // We are careful to only do the flattening/copy when we know + // we have a "top level" concat we can inspect. This avoids + // doing extra work in cases where we definitely won't use it. + // (This might still be wasted work if we can't go on to find + // some literals to extract.) + let concat = + Hir::concat(subs.iter().map(|h| flatten(h)).collect()); + return match concat.into_kind() { + HirKind::Concat(xs) => Some(xs), + // It is actually possible for this case to occur, because + // 'Hir::concat' might simplify the expression to the point + // that concatenations are actually removed. One wonders + // whether this leads to other cases where we should be + // extracting literals, but in theory, I believe if we do + // get here, then it means that a "real" prefilter failed + // to be extracted and we should probably leave well enough + // alone. (A "real" prefilter is unbothered by "top-level + // concats" and "capturing groups.") + _ => return None, + }; + } + }; + } +} + +/// Returns a copy of the given HIR but with all capturing groups removed. +fn flatten(hir: &Hir) -> Hir { + match hir.kind() { + HirKind::Empty => Hir::empty(), + HirKind::Literal(hir::Literal(ref x)) => Hir::literal(x.clone()), + HirKind::Class(ref x) => Hir::class(x.clone()), + HirKind::Look(ref x) => Hir::look(x.clone()), + HirKind::Repetition(ref x) => Hir::repetition(x.with(flatten(&x.sub))), + // This is the interesting case. We just drop the group information + // entirely and use the child HIR itself. + HirKind::Capture(hir::Capture { ref sub, .. }) => flatten(sub), + HirKind::Alternation(ref xs) => { + Hir::alternation(xs.iter().map(|x| flatten(x)).collect()) + } + HirKind::Concat(ref xs) => { + Hir::concat(xs.iter().map(|x| flatten(x)).collect()) + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/reverse_suffix.rs b/third_party/crates/regex-automata-0.4.18/src/meta/reverse_suffix.rs new file mode 100644 index 0000000..323b12b --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/reverse_suffix.rs @@ -0,0 +1,270 @@ +/*! +Routines specific to the reverse suffix optimization. +*/ + +use alloc::vec::Vec; + +use regex_syntax::hir::{literal::Literal, Hir, HirKind, Repetition}; + +use crate::meta::prefix; + +/// Returns true when it's impossible for an earlier match to be detected after +/// a literal candidate (corresponding to `suffix`) has been found. +/// +/// Specifically, that there is no earlier match than what a reverse scan of +/// `hirs` after a match of `suffix` reports. +/// +/// At present, this always returns `false` when `hirs` has any length except +/// `1`. That is, this optimization does not apply to multi-regex. +pub(super) fn has_no_earlier_match(hirs: &[&Hir], suffix: &[u8]) -> bool { + if hirs.len() != 1 || suffix.is_empty() { + return false; + } + let hir = hirs[0]; + let Some(prefix) = strip_literal_suffix(hir, suffix) else { return false }; + + let fixed_length = prefix::hir_has_fixed_length(&prefix); + debug!("reverse suffix has fixed length prefix? {fixed_length}"); + if fixed_length { + return true; + } + + let class_separator = prefix::has_disjoint_class_separator( + &prefix, + &[Literal::inexact(suffix)], + ); + debug!("reverse suffix has disjoint class separator? {class_separator}"); + if class_separator { + return true; + } + + let internal = prefix::hir_can_contain_literal(&prefix, suffix); + debug!("reverse suffix has internal suffix? {internal}"); + if !internal { + return true; + } + + // We couldn't prove that the reverse suffix optimization + // was safe, so bail out. + false +} + +/// Strip `suffix` from the end of `hir` and return the HIR that remains. +/// +/// This is conservative. It returns `None` when the structure of the HIR does +/// not make the suffix straightforward to remove, even when `suffix` might be +/// required by the pattern. +fn strip_literal_suffix(hir: &Hir, suffix: &[u8]) -> Option<Hir> { + let (prefix, suffix_cursor) = + strip_literal_suffix_at(hir, suffix, suffix.len())?; + if suffix_cursor == 0 { + Some(prefix) + } else { + None + } +} + +/// Strip a terminal part of `suffix[..suffix_cursor]` from the end of `hir`. +/// +/// The returned cursor marks the end of the part of `suffix` that remains to +/// be stripped. Thus, this call removes `suffix[after..suffix_cursor]`, where +/// `after` is the returned cursor. A non-zero cursor may be returned only when +/// the returned HIR cannot consume any bytes. This lets a concatenation +/// continue stripping from its preceding child without losing adjacency. +fn strip_literal_suffix_at( + hir: &Hir, + suffix: &[u8], + suffix_cursor: usize, +) -> Option<(Hir, usize)> { + if suffix_cursor == 0 { + return Some((hir.clone(), 0)); + } + match hir.kind() { + HirKind::Literal(lit) => { + let bytes = &lit.0; + let mut len = 0; + while len < bytes.len() + && len < suffix_cursor + && bytes[bytes.len() - len - 1] + == suffix[suffix_cursor - len - 1] + { + len += 1; + } + if len == 0 || (len < bytes.len() && len < suffix_cursor) { + return None; + } + Some(( + Hir::literal(bytes[..bytes.len() - len].to_vec()), + suffix_cursor - len, + )) + } + HirKind::Capture(capture) => { + strip_literal_suffix_at(&capture.sub, suffix, suffix_cursor) + } + HirKind::Concat(hirs) => { + let mut prefix = hirs.to_vec(); + let mut cursor = suffix_cursor; + for i in (0..prefix.len()).rev() { + let (stripped, after) = + strip_literal_suffix_at(&prefix[i], suffix, cursor)?; + if after == cursor { + return None; + } + prefix[i] = stripped; + cursor = after; + if cursor == 0 { + break; + } + } + Some((Hir::concat(prefix), cursor)) + } + HirKind::Repetition(rep) => { + strip_repetition_literal_suffix(rep, suffix, suffix_cursor) + } + _ => None, + } +} + +fn strip_repetition_literal_suffix( + rep: &Repetition, + suffix: &[u8], + suffix_cursor: usize, +) -> Option<(Hir, usize)> { + let mut min = rep.min; + let mut max = rep.max; + let mut cursor = suffix_cursor; + let mut tail = Vec::new(); + while cursor > 0 { + if min == 0 { + return None; + } + let before = cursor; + let (stripped, after) = + strip_literal_suffix_at(&rep.sub, suffix, cursor)?; + if after == before { + return None; + } + min = min.saturating_sub(1); + max = max.map(|max| max.saturating_sub(1)); + if !matches!(stripped.kind(), HirKind::Empty) { + tail.push(stripped); + } + cursor = after; + } + + let rest = Hir::repetition(Repetition { + min, + max, + greedy: rep.greedy, + sub: rep.sub.clone(), + }); + let mut prefix = Vec::with_capacity(tail.len() + 1); + if !matches!(rest.kind(), HirKind::Empty) { + prefix.push(rest); + } + tail.reverse(); + prefix.extend(tail); + Some((Hir::concat(prefix), 0)) +} + +// We only test when we have `unicode-perl` here since some regexes require +// that. +#[cfg(all(feature = "unicode-perl", not(miri),))] +#[cfg(test)] +mod tests { + use crate::util::syntax; + + use super::*; + + #[track_caller] + fn assert_strip(pattern: &str, suffix: &str, expected: Option<&str>) { + let hir = syntax::parse(pattern).unwrap(); + let got = strip_literal_suffix(&hir, suffix.as_bytes()); + let expected = expected.map(|pattern| syntax::parse(pattern).unwrap()); + assert_eq!(expected, got); + } + + #[test] + fn literal() { + assert_strip("foobar", "bar", Some("foo")); + assert_strip("foobar", "foobar", Some("")); + assert_strip("foobar", "", Some("foobar")); + } + + #[test] + fn capture() { + assert_strip(r".+foo(bar)", "foobar", Some(r".+")); + } + + #[test] + fn concat() { + assert_strip(r".+(?:ab){2}cd", "ababcd", Some(r".+")); + assert_strip(r"a(?:)cd", "acd", Some(r"")); + assert_strip(r"az{0}cd", "acd", Some(r"")); + assert_strip(r"az{2}cd", "acd", None); + assert_strip(r"az{0,2}cd", "acd", None); + } + + #[test] + fn repetition() { + assert_strip(r"x(?:ab){3}", "bab", Some("xaba")); + assert_strip(r"x(?:ab){2,}", "bab", Some(r"x(?:ab)*a")); + } + + #[test] + fn mismatch() { + assert_strip("foobar", "baz", None); + assert_strip("foobar", "xfoobar", None); + assert_strip(r"foo[a-z]", "a", None); + } + + #[test] + fn suffix_must_be_adjacent() { + assert_strip(r"foo(xbar)", "foobar", None); + } + + #[test] + fn repetition_must_be_required() { + assert_strip(r"x(?:ab)*cd", "abcd", None); + } + + #[track_caller] + fn assert_fixed_length_prefix(yes: bool, pattern: &str, suffix: &[u8]) { + let hir = syntax::parse(pattern).unwrap(); + assert_eq!( + yes, + strip_literal_suffix(&hir, suffix) + .map_or(false, |hir| prefix::hir_has_fixed_length(&hir)), + ); + } + + #[track_caller] + fn assert_internal_suffix(yes: bool, pattern: &str, suffix: &[u8]) { + let hir = syntax::parse(pattern).unwrap(); + assert_eq!( + yes, + strip_literal_suffix(&hir, suffix).map_or(false, |hir| { + prefix::hir_can_contain_literal(&hir, suffix) + }), + ); + } + + #[test] + fn reverse_suffix_accepts_prefix_without_internal_suffix() { + assert_internal_suffix(false, r"\d+XYZ", b"XYZ"); + assert_internal_suffix(false, r"[a-q][^u-z]{13}x", b"x"); + } + + #[test] + fn reverse_suffix_accepts_fixed_length_prefix() { + assert_fixed_length_prefix(true, r"(?:ab|cd)XYZ", b"XYZ"); + assert_internal_suffix(true, r"[A-Z][0-9]XYZ", b"XYZ"); + assert_fixed_length_prefix(true, r"[A-Z][0-9]XYZ", b"XYZ"); + } + + #[test] + fn reverse_suffix_fixed_length_prefix_is_conservative() { + assert_fixed_length_prefix(false, r"a{1,3}yy", b"yy"); + assert_fixed_length_prefix(false, r"a*yy", b"yy"); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/stopat.rs b/third_party/crates/regex-automata-0.4.18/src/meta/stopat.rs new file mode 100644 index 0000000..c4dcd79 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/stopat.rs @@ -0,0 +1,212 @@ +/*! +This module defines two bespoke forward DFA search routines. One for the lazy +DFA and one for the fully compiled DFA. These routines differ from the normal +ones by reporting the position at which the search terminates when a match +*isn't* found. + +This position at which a search terminates is useful in contexts where the meta +regex engine runs optimizations that could go quadratic if we aren't careful. +Namely, a regex search *could* scan to the end of the haystack only to report a +non-match. If the caller doesn't know that the search scanned to the end of the +haystack, it might restart the search at the next literal candidate it finds +and repeat the process. + +Providing the caller with the position at which the search stopped provides a +way for the caller to determine the point at which subsequent scans should not +pass. This is principally used in the "reverse inner" optimization, which works +like this: + +1. Look for a match of an inner literal. Say, 'Z' in '\w+Z\d+'. +2. At the spot where 'Z' matches, do a reverse anchored search from there for +'\w+'. +3. If the reverse search matches, it corresponds to the start position of a +(possible) match. At this point, do a forward anchored search to find the end +position. If an end position is found, then we have a match and we know its +bounds. + +If the forward anchored search in (3) searches the entire rest of the haystack +but reports a non-match, then a naive implementation of the above will continue +back at step 1 looking for more candidates. There might still be a match to be +found! It's possible. But we already scanned the whole haystack. So if we keep +repeating the process, then we might wind up taking quadratic time in the size +of the haystack, which is not great. + +So if the forward anchored search in (3) reports the position at which it +stops, then we can detect whether quadratic behavior might be occurring in +steps (1) and (2). For (1), it occurs if the literal candidate found occurs +*before* the end of the previous search in (3), since that means we're now +going to look for another match in a place where the forward search has already +scanned. It is *correct* to do so, but our technique has become inefficient. +For (2), quadratic behavior occurs similarly when its reverse search extends +past the point where the previous forward search in (3) terminated. Indeed, to +implement (2), we use the sibling 'limited' module for ensuring our reverse +scan doesn't go further than we want. + +See the 'opt/reverse-inner' benchmarks in rebar for a real demonstration of +how quadratic behavior is mitigated. +*/ + +use crate::{meta::error::RetryFailError, HalfMatch, Input, MatchError}; + +#[cfg(feature = "dfa-build")] +pub(crate) fn dfa_try_search_half_fwd( + dfa: &crate::dfa::dense::DFA<alloc::vec::Vec<u32>>, + input: &Input<'_>, +) -> Result<Result<HalfMatch, usize>, RetryFailError> { + use crate::dfa::{accel, Automaton}; + + let mut mat = None; + let mut sid = dfa.start_state_forward(input)?; + let mut at = input.start(); + while at < input.end() { + sid = dfa.next_state(sid, input.haystack()[at]); + if dfa.is_special_state(sid) { + if dfa.is_match_state(sid) { + let pattern = dfa.match_pattern(sid, 0); + mat = Some(HalfMatch::new(pattern, at)); + if input.get_earliest() { + return Ok(mat.ok_or(at)); + } + if dfa.is_accel_state(sid) { + let needs = dfa.accelerator(sid); + at = accel::find_fwd(needs, input.haystack(), at) + .unwrap_or(input.end()); + continue; + } + } else if dfa.is_accel_state(sid) { + let needs = dfa.accelerator(sid); + at = accel::find_fwd(needs, input.haystack(), at) + .unwrap_or(input.end()); + continue; + } else if dfa.is_dead_state(sid) { + return Ok(mat.ok_or(at)); + } else if dfa.is_quit_state(sid) { + return Err(MatchError::quit(input.haystack()[at], at).into()); + } else { + // Ideally we wouldn't use a DFA that specialized start states + // and thus 'is_start_state()' could never be true here, but in + // practice we reuse the DFA created for the full regex which + // will specialize start states whenever there is a prefilter. + debug_assert!(dfa.is_start_state(sid)); + } + } + at += 1; + } + dfa_eoi_fwd(dfa, input, &mut sid, &mut mat)?; + Ok(mat.ok_or(at)) +} + +#[cfg(feature = "hybrid")] +pub(crate) fn hybrid_try_search_half_fwd( + dfa: &crate::hybrid::dfa::DFA, + cache: &mut crate::hybrid::dfa::Cache, + input: &Input<'_>, +) -> Result<Result<HalfMatch, usize>, RetryFailError> { + let mut mat = None; + let mut sid = dfa.start_state_forward(cache, input)?; + let mut at = input.start(); + while at < input.end() { + sid = dfa + .next_state(cache, sid, input.haystack()[at]) + .map_err(|_| MatchError::gave_up(at))?; + if sid.is_tagged() { + if sid.is_match() { + let pattern = dfa.match_pattern(cache, sid, 0); + mat = Some(HalfMatch::new(pattern, at)); + if input.get_earliest() { + return Ok(mat.ok_or(at)); + } + } else if sid.is_dead() { + return Ok(mat.ok_or(at)); + } else if sid.is_quit() { + return Err(MatchError::quit(input.haystack()[at], at).into()); + } else { + // We should NEVER get an unknown state ID back from + // dfa.next_state(). + debug_assert!(!sid.is_unknown()); + // Ideally we wouldn't use a lazy DFA that specialized start + // states and thus 'sid.is_start()' could never be true here, + // but in practice we reuse the lazy DFA created for the full + // regex which will specialize start states whenever there is + // a prefilter. + debug_assert!(sid.is_start()); + } + } + at += 1; + } + hybrid_eoi_fwd(dfa, cache, input, &mut sid, &mut mat)?; + Ok(mat.ok_or(at)) +} + +#[cfg(feature = "dfa-build")] +#[cfg_attr(feature = "perf-inline", inline(always))] +fn dfa_eoi_fwd( + dfa: &crate::dfa::dense::DFA<alloc::vec::Vec<u32>>, + input: &Input<'_>, + sid: &mut crate::util::primitives::StateID, + mat: &mut Option<HalfMatch>, +) -> Result<(), MatchError> { + use crate::dfa::Automaton; + + let sp = input.get_span(); + match input.haystack().get(sp.end) { + Some(&b) => { + *sid = dfa.next_state(*sid, b); + if dfa.is_match_state(*sid) { + let pattern = dfa.match_pattern(*sid, 0); + *mat = Some(HalfMatch::new(pattern, sp.end)); + } else if dfa.is_quit_state(*sid) { + return Err(MatchError::quit(b, sp.end)); + } + } + None => { + *sid = dfa.next_eoi_state(*sid); + if dfa.is_match_state(*sid) { + let pattern = dfa.match_pattern(*sid, 0); + *mat = Some(HalfMatch::new(pattern, input.haystack().len())); + } + // N.B. We don't have to check 'is_quit' here because the EOI + // transition can never lead to a quit state. + debug_assert!(!dfa.is_quit_state(*sid)); + } + } + Ok(()) +} + +#[cfg(feature = "hybrid")] +#[cfg_attr(feature = "perf-inline", inline(always))] +fn hybrid_eoi_fwd( + dfa: &crate::hybrid::dfa::DFA, + cache: &mut crate::hybrid::dfa::Cache, + input: &Input<'_>, + sid: &mut crate::hybrid::LazyStateID, + mat: &mut Option<HalfMatch>, +) -> Result<(), MatchError> { + let sp = input.get_span(); + match input.haystack().get(sp.end) { + Some(&b) => { + *sid = dfa + .next_state(cache, *sid, b) + .map_err(|_| MatchError::gave_up(sp.end))?; + if sid.is_match() { + let pattern = dfa.match_pattern(cache, *sid, 0); + *mat = Some(HalfMatch::new(pattern, sp.end)); + } else if sid.is_quit() { + return Err(MatchError::quit(b, sp.end)); + } + } + None => { + *sid = dfa + .next_eoi_state(cache, *sid) + .map_err(|_| MatchError::gave_up(input.haystack().len()))?; + if sid.is_match() { + let pattern = dfa.match_pattern(cache, *sid, 0); + *mat = Some(HalfMatch::new(pattern, input.haystack().len())); + } + // N.B. We don't have to check 'is_quit' here because the EOI + // transition can never lead to a quit state. + debug_assert!(!sid.is_quit()); + } + } + Ok(()) +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/strategy.rs b/third_party/crates/regex-automata-0.4.18/src/meta/strategy.rs new file mode 100644 index 0000000..c121443 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/strategy.rs @@ -0,0 +1,2388 @@ +use core::{ + fmt::Debug, + panic::{RefUnwindSafe, UnwindSafe}, +}; + +use alloc::{borrow::Cow, format, sync::Arc}; + +use regex_syntax::hir::{literal, Hir}; + +use crate::{ + meta::{ + error::{BuildError, RetryError, RetryFailError, RetryQuadraticError}, + regex::{Cache, RegexInfo}, + reverse_inner, reverse_suffix, wrappers, + }, + nfa::thompson::{self, WhichCaptures, NFA}, + util::{ + captures::{Captures, GroupInfo}, + look::LookMatcher, + prefilter::{self, Prefilter, PrefilterI}, + primitives::{NonMaxUsize, PatternID}, + search::{Anchored, HalfMatch, Input, Match, MatchKind, PatternSet}, + }, +}; + +/// A trait that represents a single meta strategy. Its main utility is in +/// providing a way to do dynamic dispatch over a few choices. +/// +/// Why dynamic dispatch? I actually don't have a super compelling reason, and +/// importantly, I have not benchmarked it with the main alternative: an enum. +/// I went with dynamic dispatch initially because the regex engine search code +/// really can't be inlined into caller code in most cases because it's just +/// too big. In other words, it is already expected that every regex search +/// will entail at least the cost of a function call. +/// +/// I do wonder whether using enums would result in better codegen overall +/// though. It's a worthwhile experiment to try. Probably the most interesting +/// benchmark to run in such a case would be one with a high match count. That +/// is, a benchmark to test the overall latency of a search call. +pub(super) trait Strategy: + Debug + Send + Sync + RefUnwindSafe + UnwindSafe + 'static +{ + #[allow(dead_code)] + fn name(&self) -> Cow<'static, str>; + + fn group_info(&self) -> &GroupInfo; + + fn create_cache(&self) -> Cache; + + fn reset_cache(&self, cache: &mut Cache); + + fn is_accelerated(&self) -> bool; + + fn memory_usage(&self) -> usize; + + fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option<Match>; + + fn search_half( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<HalfMatch>; + + fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool; + + fn search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID>; + + fn which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ); +} + +pub(super) fn new( + info: &RegexInfo, + hirs: &[&Hir], +) -> Result<Arc<dyn Strategy>, BuildError> { + // At this point, we're committed to a regex engine of some kind. So pull + // out a prefilter if we can, which will feed to each of the constituent + // regex engines. + let pre = if info.is_always_anchored_start() { + // PERF: I'm not sure we necessarily want to do this... We may want to + // run a prefilter for quickly rejecting in some cases. The problem + // is that anchored searches overlap quite a bit with the use case + // of "run a regex on every line to extract data." In that case, the + // regex always matches, so running a prefilter doesn't really help us + // there. The main place where a prefilter helps in an anchored search + // is if the anchored search is not expected to match frequently. That + // is, the prefilter gives us a way to possibly reject a haystack very + // quickly. + // + // Maybe we should do use a prefilter, but only for longer haystacks? + // Or maybe we should only use a prefilter when we think it's "fast"? + // + // Interestingly, I think we currently lack the infrastructure for + // disabling a prefilter based on haystack length. That would probably + // need to be a new 'Input' option. (Interestingly, an 'Input' used to + // carry a 'Prefilter' with it, but I moved away from that.) + debug!("skipping literal extraction since regex is anchored"); + None + } else if let Some(pre) = info.config().get_prefilter() { + debug!( + "skipping literal extraction since the caller provided a prefilter" + ); + Some(pre.clone()) + } else if info.config().get_auto_prefilter() { + let kind = info.config().get_match_kind(); + let prefixes = crate::util::prefilter::prefixes(kind, hirs); + // If we can build a full `Strategy` from just the extracted prefixes, + // then we can short-circuit and avoid building a regex engine at all. + if let Some(pre) = Pre::from_prefixes(info, &prefixes) { + debug!( + "found that the regex can be broken down to a literal \ + search, avoiding the regex engine entirely", + ); + debug!("using {} strategy", pre.name()); + return Ok(pre); + } + // This now attempts another short-circuit of the regex engine: if we + // have a huge alternation of just plain literals, then we can just use + // Aho-Corasick for that and avoid the regex engine entirely. + // + // You might think this case would just be handled by + // `Pre::from_prefixes`, but that technique relies on heuristic literal + // extraction from the corresponding `Hir`. That works, but part of + // heuristics limit the size and number of literals returned. This case + // will specifically handle patterns with very large alternations. + // + // One wonders if we should just roll this our heuristic literal + // extraction, and then I think this case could disappear entirely. + if let Some(pre) = Pre::from_alternation_literals(info, hirs) { + debug!( + "found plain alternation of literals, \ + avoiding regex engine entirely and using Aho-Corasick" + ); + debug!("using {} strategy", pre.name()); + return Ok(pre); + } + prefixes.literals().and_then(|strings| { + debug!( + "creating prefilter from {} literals: {:?}", + strings.len(), + strings, + ); + Prefilter::new(kind, strings) + }) + } else { + debug!("skipping literal extraction since prefilters were disabled"); + None + }; + let mut core = Core::new(info.clone(), pre.clone(), hirs)?; + // Now that we have our core regex engines built, there are a few cases + // where we can do a little bit better than just a normal "search forward + // and maybe use a prefilter when in a start state." However, these cases + // may not always work or otherwise build on top of the Core searcher. + // For example, the reverse anchored optimization seems like it might + // always work, but only the DFAs support reverse searching and the DFAs + // might give up or quit for reasons. If we had, e.g., a PikeVM that + // supported reverse searching, then we could avoid building a full Core + // engine for this case. + core = match ReverseAnchored::new(core) { + Err(core) => core, + Ok(ra) => { + debug!("using {} strategy", ra.name()); + return Ok(Arc::new(ra)); + } + }; + core = match ReverseSuffix::new(core, hirs) { + Err(core) => core, + Ok(rs) => { + debug!("using {} strategy", rs.name()); + return Ok(Arc::new(rs)); + } + }; + core = match ReverseInner::new(core, hirs) { + Err(core) => core, + Ok(ri) => { + debug!("using {} strategy", ri.name()); + return Ok(Arc::new(ri)); + } + }; + debug!("using {} strategy", core.name()); + Ok(Arc::new(core)) +} + +#[derive(Clone, Debug)] +struct Pre<P> { + pre: P, + group_info: GroupInfo, +} + +impl<P: PrefilterI> Pre<P> { + fn new(pre: P) -> Arc<dyn Strategy> { + // The only thing we support when we use prefilters directly as a + // strategy is the start and end of the overall match for a single + // pattern. In other words, exactly one implicit capturing group. Which + // is exactly what we use here for a GroupInfo. + let group_info = GroupInfo::new([[None::<&str>]]).unwrap(); + Arc::new(Pre { pre, group_info }) + } +} + +// This is a little weird, but we don't actually care about the type parameter +// here because we're selecting which underlying prefilter to use. So we just +// define it on an arbitrary type. +impl Pre<()> { + /// Given a sequence of prefixes, attempt to return a full `Strategy` using + /// just the prefixes. + /// + /// Basically, this occurs when the prefixes given not just prefixes, + /// but an enumeration of the entire language matched by the regular + /// expression. + /// + /// A number of other conditions need to be true too. For example, there + /// can be only one pattern, the number of explicit capture groups is 0, no + /// look-around assertions and so on. + /// + /// Note that this ignores `Config::get_auto_prefilter` because if this + /// returns something, then it isn't a prefilter but a matcher itself. + /// Therefore, it shouldn't suffer from the problems typical to prefilters + /// (such as a high false positive rate). + fn from_prefixes( + info: &RegexInfo, + prefixes: &literal::Seq, + ) -> Option<Arc<dyn Strategy>> { + let kind = info.config().get_match_kind(); + // Check to see if our prefixes are exact, which means we might be + // able to bypass the regex engine entirely and just rely on literal + // searches. + if !prefixes.is_exact() { + return None; + } + // We also require that we have a single regex pattern. Namely, + // we reuse the prefilter infrastructure to implement search and + // prefilters only report spans. Prefilters don't know about pattern + // IDs. The multi-regex case isn't a lost cause, we might still use + // Aho-Corasick and we might still just use a regular prefilter, but + // that's done below. + if info.pattern_len() != 1 { + return None; + } + // We can't have any capture groups either. The literal engines don't + // know how to deal with things like '(foo)(bar)'. In that case, a + // prefilter will just be used and then the regex engine will resolve + // the capture groups. + if info.props()[0].explicit_captures_len() != 0 { + return None; + } + // We also require that it has zero look-around assertions. Namely, + // literal extraction treats look-around assertions as if they match + // *every* empty string. But of course, that isn't true. So for + // example, 'foo\bquux' never matches anything, but 'fooquux' is + // extracted from that as an exact literal. Such cases should just run + // the regex engine. 'fooquux' will be used as a normal prefilter, and + // then the regex engine will try to look for an actual match. + if !info.props()[0].look_set().is_empty() { + return None; + } + // Finally, currently, our prefilters are all oriented around + // leftmost-first match semantics, so don't try to use them if the + // caller asked for anything else. + if kind != MatchKind::LeftmostFirst { + return None; + } + // The above seems like a lot of requirements to meet, but it applies + // to a lot of cases. 'foo', '[abc][123]' and 'foo|bar|quux' all meet + // the above criteria, for example. + // + // Note that this is effectively a latency optimization. If we didn't + // do this, then the extracted literals would still get bundled into + // a prefilter, and every regex engine capable of running unanchored + // searches supports prefilters. So this optimization merely sidesteps + // having to run the regex engine at all to confirm the match. Thus, it + // decreases the latency of a match. + + // OK because we know the set is exact and thus finite. + let prefixes = prefixes.literals().unwrap(); + debug!( + "trying to bypass regex engine by creating \ + prefilter from {} literals: {:?}", + prefixes.len(), + prefixes, + ); + let choice = match prefilter::Choice::new(kind, prefixes) { + Some(choice) => choice, + None => { + debug!( + "regex bypass failed because no prefilter could be built" + ); + return None; + } + }; + let strat: Arc<dyn Strategy> = match choice { + prefilter::Choice::Memchr(pre) => Pre::new(pre), + prefilter::Choice::Memchr2(pre) => Pre::new(pre), + prefilter::Choice::Memchr3(pre) => Pre::new(pre), + prefilter::Choice::Memmem(pre) => Pre::new(pre), + prefilter::Choice::Teddy(pre) => Pre::new(pre), + prefilter::Choice::ByteSet(pre) => Pre::new(pre), + prefilter::Choice::AhoCorasick(pre) => Pre::new(pre), + }; + Some(strat) + } + + /// Attempts to extract an alternation of literals, and if it's deemed + /// worth doing, returns an Aho-Corasick prefilter as a strategy. + /// + /// And currently, this only returns something when 'hirs.len() == 1'. This + /// could in theory do something if there are multiple HIRs where all of + /// them are alternation of literals, but I haven't had the time to go down + /// that path yet. + fn from_alternation_literals( + info: &RegexInfo, + hirs: &[&Hir], + ) -> Option<Arc<dyn Strategy>> { + use crate::util::prefilter::AhoCorasick; + + let lits = crate::meta::literal::alternation_literals(info, hirs)?; + let ac = AhoCorasick::new(MatchKind::LeftmostFirst, &lits)?; + Some(Pre::new(ac)) + } +} + +// This implements Strategy for anything that implements PrefilterI. +// +// Note that this must only be used for regexes of length 1. Multi-regexes +// don't work here. The prefilter interface only provides the span of a match +// and not the pattern ID. (I did consider making it more expressive, but I +// couldn't figure out how to tie everything together elegantly.) Thus, so long +// as the regex only contains one pattern, we can simply assume that a match +// corresponds to PatternID::ZERO. And indeed, that's what we do here. +// +// In practice, since this impl is used to report matches directly and thus +// completely bypasses the regex engine, we only wind up using this under the +// following restrictions: +// +// * There must be only one pattern. As explained above. +// * The literal sequence must be finite and only contain exact literals. +// * There must not be any look-around assertions. If there are, the literals +// extracted might be exact, but a match doesn't necessarily imply an overall +// match. As a trivial example, 'foo\bbar' does not match 'foobar'. +// * The pattern must not have any explicit capturing groups. If it does, the +// caller might expect them to be resolved. e.g., 'foo(bar)'. +// +// So when all of those things are true, we use a prefilter directly as a +// strategy. +// +// In the case where the number of patterns is more than 1, we don't use this +// but do use a special Aho-Corasick strategy if all of the regexes are just +// simple literals or alternations of literals. (We also use the Aho-Corasick +// strategy when len(patterns)==1 if the number of literals is large. In that +// case, literal extraction gives up and will return an infinite set.) +impl<P: PrefilterI> Strategy for Pre<P> { + fn name(&self) -> Cow<'static, str> { + Cow::Owned(format!("prefilter {}", self.pre.name())) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn group_info(&self) -> &GroupInfo { + &self.group_info + } + + fn create_cache(&self) -> Cache { + Cache { + capmatches: Captures::all(self.group_info().clone()), + pikevm: wrappers::PikeVMCache::none(), + backtrack: wrappers::BoundedBacktrackerCache::none(), + onepass: wrappers::OnePassCache::none(), + hybrid: wrappers::HybridCache::none(), + revhybrid: wrappers::ReverseHybridCache::none(), + } + } + + fn reset_cache(&self, _cache: &mut Cache) {} + + fn is_accelerated(&self) -> bool { + self.pre.is_fast() + } + + fn memory_usage(&self) -> usize { + self.pre.memory_usage() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search(&self, _cache: &mut Cache, input: &Input<'_>) -> Option<Match> { + if input.is_done() { + return None; + } + if input.get_anchored().is_anchored() { + return self + .pre + .prefix(input.haystack(), input.get_span()) + .map(|sp| Match::new(PatternID::ZERO, sp)); + } + self.pre + .find(input.haystack(), input.get_span()) + .map(|sp| Match::new(PatternID::ZERO, sp)) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_half( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<HalfMatch> { + self.search(cache, input).map(|m| HalfMatch::new(m.pattern(), m.end())) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + self.search(cache, input).is_some() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + let m = self.search(cache, input)?; + if let Some(slot) = slots.get_mut(0) { + *slot = NonMaxUsize::new(m.start()); + } + if let Some(slot) = slots.get_mut(1) { + *slot = NonMaxUsize::new(m.end()); + } + Some(m.pattern()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + if self.search(cache, input).is_some() { + patset.insert(PatternID::ZERO); + } + } +} + +#[derive(Debug)] +struct Core { + info: RegexInfo, + pre: Option<Prefilter>, + nfa: NFA, + nfarev: Option<NFA>, + pikevm: wrappers::PikeVM, + backtrack: wrappers::BoundedBacktracker, + onepass: wrappers::OnePass, + hybrid: wrappers::Hybrid, + dfa: wrappers::DFA, +} + +impl Core { + fn new( + info: RegexInfo, + pre: Option<Prefilter>, + hirs: &[&Hir], + ) -> Result<Core, BuildError> { + let mut lookm = LookMatcher::new(); + lookm.set_line_terminator(info.config().get_line_terminator()); + let thompson_config = info.config().to_thompson_config(); + let nfa = thompson::Compiler::new() + .configure(thompson_config.clone()) + .build_many_from_hir(hirs) + .map_err(BuildError::nfa)?; + // It's possible for the PikeVM or the BB to fail to build, even though + // at this point, we already have a full NFA in hand. They can fail + // when a Unicode word boundary is used but where Unicode word boundary + // support is disabled at compile time, thus making it impossible to + // match. (Construction can also fail if the NFA was compiled without + // captures, but we always enable that above.) + let pikevm = wrappers::PikeVM::new(&info, pre.clone(), &nfa)?; + let backtrack = + wrappers::BoundedBacktracker::new(&info, pre.clone(), &nfa)?; + // The onepass engine can of course fail to build, but we expect it to + // fail in many cases because it is an optimization that doesn't apply + // to all regexes. The 'OnePass' wrapper encapsulates this failure (and + // logs a message if it occurs). + let onepass = wrappers::OnePass::new(&info, &nfa); + // We try to encapsulate whether a particular regex engine should be + // used within each respective wrapper, but the DFAs need a reverse NFA + // to build itself, and we really do not want to build a reverse NFA if + // we know we aren't going to use the lazy DFA. So we do a config check + // up front, which is in practice the only way we won't try to use the + // DFA. + let (nfarev, hybrid, dfa) = + if !info.config().get_hybrid() && !info.config().get_dfa() { + (None, wrappers::Hybrid::none(), wrappers::DFA::none()) + } else { + // FIXME: Technically, we don't quite yet KNOW that we need + // a reverse NFA. It's possible for the DFAs below to both + // fail to build just based on the forward NFA. In which case, + // building the reverse NFA was totally wasted work. But... + // fixing this requires breaking DFA construction apart into + // two pieces: one for the forward part and another for the + // reverse part. Quite annoying. Making it worse, when building + // both DFAs fails, it's quite likely that the NFA is large and + // that it will take quite some time to build the reverse NFA + // too. So... it's really probably worth it to do this! + let nfarev = thompson::Compiler::new() + // Currently, reverse NFAs don't support capturing groups, + // so we MUST disable them. But even if we didn't have to, + // we would, because nothing in this crate does anything + // useful with capturing groups in reverse. And of course, + // the lazy DFA ignores capturing groups in all cases. + .configure( + thompson_config + .which_captures(WhichCaptures::None) + .reverse(true), + ) + .build_many_from_hir(hirs) + .map_err(BuildError::nfa)?; + let dfa = if !info.config().get_dfa() { + wrappers::DFA::none() + } else { + wrappers::DFA::new(&info, pre.clone(), &nfa, &nfarev) + }; + let hybrid = if !info.config().get_hybrid() { + wrappers::Hybrid::none() + } else if dfa.is_some() { + debug!("skipping lazy DFA because we have a full DFA"); + wrappers::Hybrid::none() + } else { + wrappers::Hybrid::new(&info, pre.clone(), &nfa, &nfarev) + }; + (Some(nfarev), hybrid, dfa) + }; + Ok(Core { + info, + pre, + nfa, + nfarev, + pikevm, + backtrack, + onepass, + hybrid, + dfa, + }) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn try_search_mayfail( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<Result<Option<Match>, RetryFailError>> { + if let Some(e) = self.dfa.get(input) { + trace!("using full DFA for search at {:?}", input.get_span()); + Some(e.try_search(input)) + } else if let Some(e) = self.hybrid.get(input) { + trace!("using lazy DFA for search at {:?}", input.get_span()); + Some(e.try_search(&mut cache.hybrid, input)) + } else { + None + } + } + + fn search_nofail( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<Match> { + let caps = &mut cache.capmatches; + caps.set_pattern(None); + // We manually inline 'try_search_slots_nofail' here because we need to + // borrow from 'cache.capmatches' in this method, but if we do, then + // we can't pass 'cache' wholesale to to 'try_slots_no_hybrid'. It's a + // classic example of how the borrow checker inhibits decomposition. + // There are of course work-arounds (more types and/or interior + // mutability), but that's more annoying than this IMO. + let pid = if let Some(ref e) = self.onepass.get(input) { + trace!("using OnePass for search at {:?}", input.get_span()); + e.search_slots(&mut cache.onepass, input, caps.slots_mut()) + } else if let Some(ref e) = self.backtrack.get(input) { + trace!( + "using BoundedBacktracker for search at {:?}", + input.get_span() + ); + e.search_slots(&mut cache.backtrack, input, caps.slots_mut()) + } else { + trace!("using PikeVM for search at {:?}", input.get_span()); + let e = self.pikevm.get(); + e.search_slots(&mut cache.pikevm, input, caps.slots_mut()) + }; + caps.set_pattern(pid); + caps.get_match() + } + + fn search_half_nofail( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<HalfMatch> { + // Only the lazy/full DFA returns half-matches, since the DFA requires + // a reverse scan to find the start position. These fallback regex + // engines can find the start and end in a single pass, so we just do + // that and throw away the start offset to conform to the API. + let m = self.search_nofail(cache, input)?; + Some(HalfMatch::new(m.pattern(), m.end())) + } + + fn search_slots_nofail( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + if let Some(ref e) = self.onepass.get(input) { + trace!( + "using OnePass for capture search at {:?}", + input.get_span() + ); + e.search_slots(&mut cache.onepass, input, slots) + } else if let Some(ref e) = self.backtrack.get(input) { + trace!( + "using BoundedBacktracker for capture search at {:?}", + input.get_span() + ); + e.search_slots(&mut cache.backtrack, input, slots) + } else { + trace!( + "using PikeVM for capture search at {:?}", + input.get_span() + ); + let e = self.pikevm.get(); + e.search_slots(&mut cache.pikevm, input, slots) + } + } + + fn is_match_nofail(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + if let Some(ref e) = self.onepass.get(input) { + trace!( + "using OnePass for is-match search at {:?}", + input.get_span() + ); + e.search_slots(&mut cache.onepass, input, &mut []).is_some() + } else if let Some(ref e) = self.backtrack.get(input) { + trace!( + "using BoundedBacktracker for is-match search at {:?}", + input.get_span() + ); + e.is_match(&mut cache.backtrack, input) + } else { + trace!( + "using PikeVM for is-match search at {:?}", + input.get_span() + ); + let e = self.pikevm.get(); + e.is_match(&mut cache.pikevm, input) + } + } + + fn is_capture_search_needed(&self, slots_len: usize) -> bool { + slots_len > self.nfa.group_info().implicit_slot_len() + } +} + +impl Strategy for Core { + fn name(&self) -> Cow<'static, str> { + Cow::Borrowed("core") + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn group_info(&self) -> &GroupInfo { + self.nfa.group_info() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn create_cache(&self) -> Cache { + Cache { + capmatches: Captures::all(self.group_info().clone()), + pikevm: self.pikevm.create_cache(), + backtrack: self.backtrack.create_cache(), + onepass: self.onepass.create_cache(), + hybrid: self.hybrid.create_cache(), + revhybrid: wrappers::ReverseHybridCache::none(), + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn reset_cache(&self, cache: &mut Cache) { + cache.pikevm.reset(&self.pikevm); + cache.backtrack.reset(&self.backtrack); + cache.onepass.reset(&self.onepass); + cache.hybrid.reset(&self.hybrid); + } + + fn is_accelerated(&self) -> bool { + self.pre.as_ref().map_or(false, |pre| pre.is_fast()) + } + + fn memory_usage(&self) -> usize { + self.info.memory_usage() + + self.pre.as_ref().map_or(0, |pre| pre.memory_usage()) + + self.nfa.memory_usage() + + self.nfarev.as_ref().map_or(0, |nfa| nfa.memory_usage()) + + self.onepass.memory_usage() + + self.dfa.memory_usage() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option<Match> { + // We manually inline try_search_mayfail here because letting the + // compiler do it seems to produce pretty crappy codegen. + return if let Some(e) = self.dfa.get(input) { + trace!("using full DFA for full search at {:?}", input.get_span()); + match e.try_search(input) { + Ok(x) => x, + Err(_err) => { + trace!("full DFA search failed: {_err}"); + self.search_nofail(cache, input) + } + } + } else if let Some(e) = self.hybrid.get(input) { + trace!("using lazy DFA for full search at {:?}", input.get_span()); + match e.try_search(&mut cache.hybrid, input) { + Ok(x) => x, + Err(_err) => { + trace!("lazy DFA search failed: {_err}"); + self.search_nofail(cache, input) + } + } + } else { + self.search_nofail(cache, input) + }; + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_half( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<HalfMatch> { + // The main difference with 'search' is that if we're using a DFA, we + // can use a single forward scan without needing to run the reverse + // DFA. + if let Some(e) = self.dfa.get(input) { + trace!("using full DFA for half search at {:?}", input.get_span()); + match e.try_search_half_fwd(input) { + Ok(x) => x, + Err(_err) => { + trace!("full DFA half search failed: {_err}"); + self.search_half_nofail(cache, input) + } + } + } else if let Some(e) = self.hybrid.get(input) { + trace!("using lazy DFA for half search at {:?}", input.get_span()); + match e.try_search_half_fwd(&mut cache.hybrid, input) { + Ok(x) => x, + Err(_err) => { + trace!("lazy DFA half search failed: {_err}"); + self.search_half_nofail(cache, input) + } + } + } else { + self.search_half_nofail(cache, input) + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + if let Some(e) = self.dfa.get(input) { + trace!( + "using full DFA for is-match search at {:?}", + input.get_span() + ); + match e.try_search_half_fwd(input) { + Ok(x) => x.is_some(), + Err(_err) => { + trace!("full DFA half search failed: {_err}"); + self.is_match_nofail(cache, input) + } + } + } else if let Some(e) = self.hybrid.get(input) { + trace!( + "using lazy DFA for is-match search at {:?}", + input.get_span() + ); + match e.try_search_half_fwd(&mut cache.hybrid, input) { + Ok(x) => x.is_some(), + Err(_err) => { + trace!("lazy DFA half search failed: {_err}"); + self.is_match_nofail(cache, input) + } + } + } else { + self.is_match_nofail(cache, input) + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + // Even if the regex has explicit capture groups, if the caller didn't + // provide any explicit slots, then it doesn't make sense to try and do + // extra work to get offsets for those slots. Ideally the caller should + // realize this and not call this routine in the first place, but alas, + // we try to save the caller from themselves if they do. + if !self.is_capture_search_needed(slots.len()) { + trace!("asked for slots unnecessarily, trying fast path"); + let m = self.search(cache, input)?; + copy_match_to_slots(m, slots); + return Some(m.pattern()); + } + // If the onepass DFA is available for this search (which only happens + // when it's anchored), then skip running a fallible DFA. The onepass + // DFA isn't as fast as a full or lazy DFA, but it is typically quite + // a bit faster than the backtracker or the PikeVM. So it isn't as + // advantageous to try and do a full/lazy DFA scan first. + // + // We still theorize that it's better to do a full/lazy DFA scan, even + // when it's anchored, because it's usually much faster and permits us + // to say "no match" much more quickly. This does hurt the case of, + // say, parsing each line in a log file into capture groups, because + // in that case, the line always matches. So the lazy DFA scan is + // usually just wasted work. But, the lazy DFA is usually quite fast + // and doesn't cost too much here. + if self.onepass.get(&input).is_some() { + return self.search_slots_nofail(cache, &input, slots); + } + let m = match self.try_search_mayfail(cache, input) { + Some(Ok(Some(m))) => m, + Some(Ok(None)) => return None, + Some(Err(_err)) => { + trace!("fast capture search failed: {_err}"); + return self.search_slots_nofail(cache, input, slots); + } + None => { + return self.search_slots_nofail(cache, input, slots); + } + }; + // At this point, now that we've found the bounds of the + // match, we need to re-run something that can resolve + // capturing groups. But we only need to run on it on the + // match bounds and not the entire haystack. + trace!( + "match found at {}..{} in capture search, \ + using another engine to find captures", + m.start(), + m.end(), + ); + let input = input + .clone() + .span(m.start()..m.end()) + .anchored(Anchored::Pattern(m.pattern())); + Some( + self.search_slots_nofail(cache, &input, slots) + .expect("should find a match"), + ) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + if let Some(e) = self.dfa.get(input) { + trace!( + "using full DFA for overlapping search at {:?}", + input.get_span() + ); + let _err = match e.try_which_overlapping_matches(input, patset) { + Ok(()) => return, + Err(err) => err, + }; + trace!("fast overlapping search failed: {_err}"); + } else if let Some(e) = self.hybrid.get(input) { + trace!( + "using lazy DFA for overlapping search at {:?}", + input.get_span() + ); + let _err = match e.try_which_overlapping_matches( + &mut cache.hybrid, + input, + patset, + ) { + Ok(()) => { + return; + } + Err(err) => err, + }; + trace!("fast overlapping search failed: {_err}"); + } + trace!( + "using PikeVM for overlapping search at {:?}", + input.get_span() + ); + let e = self.pikevm.get(); + e.which_overlapping_matches(&mut cache.pikevm, input, patset) + } +} + +#[derive(Debug)] +struct ReverseAnchored { + core: Core, +} + +impl ReverseAnchored { + fn new(core: Core) -> Result<ReverseAnchored, Core> { + if !core.info.is_always_anchored_end() { + debug!( + "skipping reverse anchored optimization because \ + the regex is not always anchored at the end" + ); + return Err(core); + } + // Note that the caller can still request an anchored search even when + // the regex isn't anchored at the start. We detect that case in the + // search routines below and just fallback to the core engine. This + // is fine because both searches are anchored. It's just a matter of + // picking one. Falling back to the core engine is a little simpler, + // since if we used the reverse anchored approach, we'd have to add an + // extra check to ensure the match reported starts at the place where + // the caller requested the search to start. + if core.info.is_always_anchored_start() { + debug!( + "skipping reverse anchored optimization because \ + the regex is also anchored at the start" + ); + return Err(core); + } + // Only DFAs can do reverse searches (currently), so we need one of + // them in order to do this optimization. It's possible (although + // pretty unlikely) that we have neither and need to give up. + if !core.hybrid.is_some() && !core.dfa.is_some() { + debug!( + "skipping reverse anchored optimization because \ + we don't have a lazy DFA or a full DFA" + ); + return Err(core); + } + Ok(ReverseAnchored { core }) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn try_search_half_anchored_rev( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, RetryFailError> { + // We of course always want an anchored search. In theory, the + // underlying regex engines should automatically enable anchored + // searches since the regex is itself anchored, but this more clearly + // expresses intent and is always correct. + let input = input.clone().anchored(Anchored::Yes); + if let Some(e) = self.core.dfa.get(&input) { + trace!( + "using full DFA for reverse anchored search at {:?}", + input.get_span() + ); + e.try_search_half_rev(&input) + } else if let Some(e) = self.core.hybrid.get(&input) { + trace!( + "using lazy DFA for reverse anchored search at {:?}", + input.get_span() + ); + e.try_search_half_rev(&mut cache.hybrid, &input) + } else { + unreachable!("ReverseAnchored always has a DFA") + } + } +} + +// Note that in this impl, we don't check that 'input.end() == +// input.haystack().len()'. In particular, when that condition is false, a +// match is always impossible because we know that the regex is always anchored +// at the end (or else 'ReverseAnchored' won't be built). We don't check that +// here because the 'Regex' wrapper actually does that for us in all cases. +// Thus, in this impl, we can actually assume that the end position in 'input' +// is equivalent to the length of the haystack. +impl Strategy for ReverseAnchored { + fn name(&self) -> Cow<'static, str> { + Cow::Borrowed("reverse anchored") + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn group_info(&self) -> &GroupInfo { + self.core.group_info() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn create_cache(&self) -> Cache { + self.core.create_cache() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn reset_cache(&self, cache: &mut Cache) { + self.core.reset_cache(cache); + } + + fn is_accelerated(&self) -> bool { + // Since this is anchored at the end, a reverse anchored search is + // almost certainly guaranteed to result in a much faster search than + // a standard forward search. + true + } + + fn memory_usage(&self) -> usize { + self.core.memory_usage() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option<Match> { + if input.get_anchored().is_anchored() { + return self.core.search(cache, input); + } + match self.try_search_half_anchored_rev(cache, input) { + Err(_err) => { + trace!("fast reverse anchored search failed: {_err}"); + self.core.search_nofail(cache, input) + } + Ok(None) => None, + Ok(Some(hm)) => { + Some(Match::new(hm.pattern(), hm.offset()..input.end())) + } + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_half( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<HalfMatch> { + if input.get_anchored().is_anchored() { + return self.core.search_half(cache, input); + } + match self.try_search_half_anchored_rev(cache, input) { + Err(_err) => { + trace!("fast reverse anchored search failed: {_err}"); + self.core.search_half_nofail(cache, input) + } + Ok(None) => None, + Ok(Some(hm)) => { + // Careful here! 'try_search_half' is a *forward* search that + // only cares about the *end* position of a match. But + // 'hm.offset()' is actually the start of the match. So we + // actually just throw that away here and, since we know we + // have a match, return the only possible position at which a + // match can occur: input.end(). + Some(HalfMatch::new(hm.pattern(), input.end())) + } + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + if input.get_anchored().is_anchored() { + return self.core.is_match(cache, input); + } + match self.try_search_half_anchored_rev(cache, input) { + Err(_err) => { + trace!("fast reverse anchored search failed: {_err}"); + self.core.is_match_nofail(cache, input) + } + Ok(None) => false, + Ok(Some(_)) => true, + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + if input.get_anchored().is_anchored() { + return self.core.search_slots(cache, input, slots); + } + match self.try_search_half_anchored_rev(cache, input) { + Err(_err) => { + trace!("fast reverse anchored search failed: {_err}"); + self.core.search_slots_nofail(cache, input, slots) + } + Ok(None) => None, + Ok(Some(hm)) => { + if !self.core.is_capture_search_needed(slots.len()) { + trace!("asked for slots unnecessarily, skipping captures"); + let m = Match::new(hm.pattern(), hm.offset()..input.end()); + copy_match_to_slots(m, slots); + return Some(m.pattern()); + } + let start = hm.offset(); + let input = input + .clone() + .span(start..input.end()) + .anchored(Anchored::Pattern(hm.pattern())); + self.core.search_slots_nofail(cache, &input, slots) + } + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + // It seems like this could probably benefit from a reverse anchored + // optimization, perhaps by doing an overlapping reverse search (which + // the DFAs do support). I haven't given it much thought though, and + // I'm currently focus more on the single pattern case. + self.core.which_overlapping_matches(cache, input, patset) + } +} + +#[derive(Debug)] +struct ReverseSuffix { + core: Core, + pre: Prefilter, +} + +impl ReverseSuffix { + fn new(core: Core, hirs: &[&Hir]) -> Result<ReverseSuffix, Core> { + if !core.info.config().get_auto_prefilter() { + debug!( + "skipping reverse suffix optimization because \ + automatic prefilters are disabled" + ); + return Err(core); + } + // Also like the reverse inner optimization, a reverse suffix encodes + // leftmost-first match semantics. + if core.info.config().get_match_kind() != MatchKind::LeftmostFirst { + debug!( + "skipping reverse suffix optimization because \ + match kind is {:?} but this only supports leftmost-first", + core.info.config().get_match_kind(), + ); + return Err(core); + } + // Like the reverse inner optimization, we don't do this for regexes + // that are always anchored. It could lead to scanning too much, but + // could say "no match" much more quickly than running the regex + // engine if the initial literal scan doesn't match. With that said, + // the reverse suffix optimization has lower overhead, since it only + // requires a reverse scan after a literal match to confirm or reject + // the match. (Although, in the case of confirmation, it then needs to + // do another forward scan to find the end position.) + // + // Note that the caller can still request an anchored search even + // when the regex isn't anchored. We detect that case in the search + // routines below and just fallback to the core engine. Currently this + // optimization assumes all searches are unanchored, so if we do want + // to enable this optimization for anchored searches, it will need a + // little work to support it. + if core.info.is_always_anchored_start() { + debug!( + "skipping reverse suffix optimization because \ + the regex is always anchored at the start", + ); + return Err(core); + } + // Only DFAs can do reverse searches (currently), so we need one of + // them in order to do this optimization. It's possible (although + // pretty unlikely) that we have neither and need to give up. + if !core.hybrid.is_some() && !core.dfa.is_some() { + debug!( + "skipping reverse suffix optimization because \ + we don't have a lazy DFA or a full DFA" + ); + return Err(core); + } + if core.pre.as_ref().map_or(false, |p| p.is_fast()) { + debug!( + "skipping reverse suffix optimization because \ + we already have a prefilter that we think is fast" + ); + return Err(core); + } + let kind = core.info.config().get_match_kind(); + let suffixes = crate::util::prefilter::suffixes(kind, hirs); + let lcs = match suffixes.longest_common_suffix() { + None => { + debug!( + "skipping reverse suffix optimization because \ + a longest common suffix could not be found", + ); + return Err(core); + } + Some(lcs) if lcs.is_empty() => { + debug!( + "skipping reverse suffix optimization because \ + the longest common suffix is the empty string", + ); + return Err(core); + } + Some(lcs) => lcs, + }; + let pre = match Prefilter::new(kind, &[lcs]) { + Some(pre) => pre, + None => { + debug!( + "skipping reverse suffix optimization because \ + a prefilter could not be constructed from the \ + longest common suffix", + ); + return Err(core); + } + }; + if !pre.is_fast() { + debug!( + "skipping reverse suffix optimization because \ + while we have a suffix prefilter, it is not \ + believed to be 'fast'" + ); + return Err(core); + } + if !reverse_suffix::has_no_earlier_match(hirs, &lcs) { + debug!( + "skipping reverse suffix optimization because \ + an earlier suffix match could be a complete match \ + inside of a larger match" + ); + return Err(core); + } + Ok(ReverseSuffix { core, pre }) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn try_search_half_start( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, RetryError> { + let mut span = input.get_span(); + let mut min_start = 0; + loop { + let litmatch = match self.pre.find(input.haystack(), span) { + None => break, + Some(span) => span, + }; + trace!("reverse suffix scan found suffix match at {litmatch:?}"); + let revinput = input + .clone() + .anchored(Anchored::Yes) + .span(input.start()..litmatch.end); + if let Some(hm) = + self.try_search_half_rev_limited(cache, &revinput, min_start)? + { + return Ok(Some(hm)); + } + + if span.start >= span.end { + break; + } + span.start = litmatch.start.checked_add(1).unwrap(); + min_start = litmatch.end; + } + Ok(None) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn try_search_half_fwd( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, RetryFailError> { + if let Some(e) = self.core.dfa.get(&input) { + trace!( + "using full DFA for forward reverse suffix search at {:?}", + input.get_span() + ); + e.try_search_half_fwd(&input) + } else if let Some(e) = self.core.hybrid.get(&input) { + trace!( + "using lazy DFA for forward reverse suffix search at {:?}", + input.get_span() + ); + e.try_search_half_fwd(&mut cache.hybrid, &input) + } else { + unreachable!("ReverseSuffix always has a DFA") + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn try_search_half_rev_limited( + &self, + cache: &mut Cache, + input: &Input<'_>, + min_start: usize, + ) -> Result<Option<HalfMatch>, RetryError> { + if let Some(e) = self.core.dfa.get(&input) { + trace!( + "using full DFA for reverse suffix search at {:?}, \ + but will be stopped at {} to avoid quadratic behavior", + input.get_span(), + min_start, + ); + e.try_search_half_rev_limited(&input, min_start) + } else if let Some(e) = self.core.hybrid.get(&input) { + trace!( + "using lazy DFA for reverse suffix search at {:?}, \ + but will be stopped at {} to avoid quadratic behavior", + input.get_span(), + min_start, + ); + e.try_search_half_rev_limited(&mut cache.hybrid, &input, min_start) + } else { + unreachable!("ReverseSuffix always has a DFA") + } + } +} + +impl Strategy for ReverseSuffix { + fn name(&self) -> Cow<'static, str> { + Cow::Borrowed("reverse suffix") + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn group_info(&self) -> &GroupInfo { + self.core.group_info() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn create_cache(&self) -> Cache { + self.core.create_cache() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn reset_cache(&self, cache: &mut Cache) { + self.core.reset_cache(cache); + } + + fn is_accelerated(&self) -> bool { + self.pre.is_fast() + } + + fn memory_usage(&self) -> usize { + self.core.memory_usage() + self.pre.memory_usage() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option<Match> { + if input.get_anchored().is_anchored() { + return self.core.search(cache, input); + } + match self.try_search_half_start(cache, input) { + Err(RetryError::Quadratic(_err)) => { + trace!("reverse suffix optimization failed: {_err}"); + self.core.search(cache, input) + } + Err(RetryError::Fail(_err)) => { + trace!("reverse suffix reverse fast search failed: {_err}"); + self.core.search_nofail(cache, input) + } + Ok(None) => None, + Ok(Some(hm_start)) => { + let fwdinput = input + .clone() + .anchored(Anchored::Pattern(hm_start.pattern())) + .span(hm_start.offset()..input.end()); + match self.try_search_half_fwd(cache, &fwdinput) { + Err(_err) => { + trace!( + "reverse suffix forward fast search failed: {_err}" + ); + self.core.search_nofail(cache, input) + } + Ok(None) => { + unreachable!( + "suffix match plus reverse match implies \ + there must be a match", + ) + } + Ok(Some(hm_end)) => Some(Match::new( + hm_start.pattern(), + hm_start.offset()..hm_end.offset(), + )), + } + } + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_half( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<HalfMatch> { + if input.get_anchored().is_anchored() { + return self.core.search_half(cache, input); + } + match self.try_search_half_start(cache, input) { + Err(RetryError::Quadratic(_err)) => { + trace!("reverse suffix half optimization failed: {_err}"); + self.core.search_half(cache, input) + } + Err(RetryError::Fail(_err)) => { + trace!( + "reverse suffix reverse fast half search failed: {_err}" + ); + self.core.search_half_nofail(cache, input) + } + Ok(None) => None, + Ok(Some(hm_start)) => { + // This is a bit subtle. It is tempting to just stop searching + // at this point and return a half-match with an offset + // corresponding to where the suffix was found. But the suffix + // match does not necessarily correspond to the end of the + // proper leftmost-first match. Consider /[a-z]+ing/ against + // 'tingling'. The first suffix match is the first 'ing', and + // the /[a-z]+/ matches the 't'. So if we stopped here, then + // we'd report 'ting' as the match. But 'tingling' is the + // correct match because of greediness. + let fwdinput = input + .clone() + .anchored(Anchored::Pattern(hm_start.pattern())) + .span(hm_start.offset()..input.end()); + match self.try_search_half_fwd(cache, &fwdinput) { + Err(_err) => { + trace!( + "reverse suffix forward fast search failed: {_err}" + ); + self.core.search_half_nofail(cache, input) + } + Ok(None) => { + unreachable!( + "suffix match plus reverse match implies \ + there must be a match", + ) + } + Ok(Some(hm_end)) => Some(hm_end), + } + } + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + if input.get_anchored().is_anchored() { + return self.core.is_match(cache, input); + } + match self.try_search_half_start(cache, input) { + Err(RetryError::Quadratic(_err)) => { + trace!("reverse suffix half optimization failed: {_err}"); + self.core.is_match_nofail(cache, input) + } + Err(RetryError::Fail(_err)) => { + trace!( + "reverse suffix reverse fast half search failed: {_err}" + ); + self.core.is_match_nofail(cache, input) + } + Ok(None) => false, + Ok(Some(_)) => true, + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + if input.get_anchored().is_anchored() { + return self.core.search_slots(cache, input, slots); + } + if !self.core.is_capture_search_needed(slots.len()) { + trace!("asked for slots unnecessarily, trying fast path"); + let m = self.search(cache, input)?; + copy_match_to_slots(m, slots); + return Some(m.pattern()); + } + let hm_start = match self.try_search_half_start(cache, input) { + Err(RetryError::Quadratic(_err)) => { + trace!("reverse suffix captures optimization failed: {_err}"); + return self.core.search_slots(cache, input, slots); + } + Err(RetryError::Fail(_err)) => { + trace!( + "reverse suffix reverse fast captures search failed: \ + {_err}" + ); + return self.core.search_slots_nofail(cache, input, slots); + } + Ok(None) => return None, + Ok(Some(hm_start)) => hm_start, + }; + trace!( + "match found at {}..{} in capture search, \ + using another engine to find captures", + hm_start.offset(), + input.end(), + ); + let start = hm_start.offset(); + let input = input + .clone() + .span(start..input.end()) + .anchored(Anchored::Pattern(hm_start.pattern())); + self.core.search_slots_nofail(cache, &input, slots) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + self.core.which_overlapping_matches(cache, input, patset) + } +} + +#[derive(Debug)] +struct ReverseInner { + core: Core, + preinner: Prefilter, + nfarev: NFA, + hybrid: wrappers::ReverseHybrid, + dfa: wrappers::ReverseDFA, +} + +impl ReverseInner { + fn new(core: Core, hirs: &[&Hir]) -> Result<ReverseInner, Core> { + if !core.info.config().get_auto_prefilter() { + debug!( + "skipping reverse inner optimization because \ + automatic prefilters are disabled" + ); + return Err(core); + } + // Currently we hard-code the assumption of leftmost-first match + // semantics. This isn't a huge deal because 'all' semantics tend to + // only be used for forward overlapping searches with multiple regexes, + // and this optimization only supports a single pattern at the moment. + if core.info.config().get_match_kind() != MatchKind::LeftmostFirst { + debug!( + "skipping reverse inner optimization because \ + match kind is {:?} but this only supports leftmost-first", + core.info.config().get_match_kind(), + ); + return Err(core); + } + // It's likely that a reverse inner scan has too much overhead for it + // to be worth it when the regex is anchored at the start. It is + // possible for it to be quite a bit faster if the initial literal + // scan fails to detect a match, in which case, we can say "no match" + // very quickly. But this could be undesirable, e.g., scanning too far + // or when the literal scan matches. If it matches, then confirming the + // match requires a reverse scan followed by a forward scan to confirm + // or reject, which is a fair bit of work. + // + // Note that the caller can still request an anchored search even + // when the regex isn't anchored. We detect that case in the search + // routines below and just fallback to the core engine. Currently this + // optimization assumes all searches are unanchored, so if we do want + // to enable this optimization for anchored searches, it will need a + // little work to support it. + if core.info.is_always_anchored_start() { + debug!( + "skipping reverse inner optimization because \ + the regex is always anchored at the start", + ); + return Err(core); + } + // Only DFAs can do reverse searches (currently), so we need one of + // them in order to do this optimization. It's possible (although + // pretty unlikely) that we have neither and need to give up. + if !core.hybrid.is_some() && !core.dfa.is_some() { + debug!( + "skipping reverse inner optimization because \ + we don't have a lazy DFA or a full DFA" + ); + return Err(core); + } + if core.pre.as_ref().map_or(false, |p| p.is_fast()) { + debug!( + "skipping reverse inner optimization because \ + we already have a prefilter that we think is fast" + ); + return Err(core); + } else if core.pre.is_some() { + debug!( + "core engine has a prefix prefilter, but it is \ + probably not fast, so continuing with attempt to \ + use reverse inner prefilter" + ); + } + let prefilter = match reverse_inner::InnerPrefilter::new(hirs) { + Some(prefilter) => prefilter, + // N.B. the 'new' function emits debug messages explaining + // why we bailed out here. + None => return Err(core), + }; + if !reverse_inner::has_no_earlier_match( + &prefilter.prefix, + &prefilter.literals, + ) { + debug!( + "skipping reverse inner optimization because an inner \ + literal match could be confirmed before an earlier match" + ); + return Err(core); + } + debug!("building reverse NFA for prefix before inner literal"); + let thompson_config = core + .info + .config() + .to_thompson_config() + .reverse(true) + .which_captures(WhichCaptures::None); + let result = thompson::Compiler::new() + .configure(thompson_config) + .build_from_hir(&prefilter.prefix); + let nfarev = match result { + Ok(nfarev) => nfarev, + Err(_err) => { + debug!( + "skipping reverse inner optimization because the \ + reverse NFA failed to build: {}", + _err, + ); + return Err(core); + } + }; + debug!("building reverse DFA for prefix before inner literal"); + let dfa = if !core.info.config().get_dfa() { + wrappers::ReverseDFA::none() + } else { + wrappers::ReverseDFA::new(&core.info, &nfarev) + }; + let hybrid = if !core.info.config().get_hybrid() { + wrappers::ReverseHybrid::none() + } else if dfa.is_some() { + debug!( + "skipping lazy DFA for reverse inner optimization \ + because we have a full DFA" + ); + wrappers::ReverseHybrid::none() + } else { + wrappers::ReverseHybrid::new(&core.info, &nfarev) + }; + Ok(ReverseInner { core, preinner: prefilter.pre, nfarev, hybrid, dfa }) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn try_search_full( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<Option<Match>, RetryError> { + let mut span = input.get_span(); + let mut min_match_start = 0; + let mut min_pre_start = 0; + loop { + let litmatch = match self.preinner.find(input.haystack(), span) { + None => break, + Some(span) => span, + }; + if litmatch.start < min_pre_start { + trace!( + "found inner prefilter match at {litmatch:?}, which starts \ + before the end of the last forward scan at {min_pre_start}, \ + quitting to avoid quadratic behavior", + ); + return Err(RetryError::Quadratic(RetryQuadraticError::new())); + } + trace!("reverse inner scan found inner match at {litmatch:?}"); + let revinput = input + .clone() + .anchored(Anchored::Yes) + .span(input.start()..litmatch.start); + // Note that in addition to the literal search above scanning past + // our minimum start point, this routine can also return an error + // as a result of detecting possible quadratic behavior if the + // reverse scan goes past the minimum start point. That is, the + // literal search might not, but the reverse regex search for the + // prefix might! + if let Some(hm_start) = self.try_search_half_rev_limited( + cache, + &revinput, + min_match_start, + )? { + let fwdinput = input + .clone() + .anchored(Anchored::Pattern(hm_start.pattern())) + .span(hm_start.offset()..input.end()); + match self.try_search_half_fwd_stopat(cache, &fwdinput)? { + Err(stopat) => { + min_pre_start = stopat; + span.start = litmatch.start.checked_add(1).unwrap(); + } + Ok(hm_end) => { + return Ok(Some(Match::new( + hm_start.pattern(), + hm_start.offset()..hm_end.offset(), + ))); + } + } + } + + if span.start >= span.end { + break; + } + span.start = litmatch.start.checked_add(1).unwrap(); + min_match_start = litmatch.end; + } + Ok(None) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn try_search_half_fwd_stopat( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Result<Result<HalfMatch, usize>, RetryFailError> { + if let Some(e) = self.core.dfa.get(&input) { + trace!( + "using full DFA for forward reverse inner search at {:?}", + input.get_span() + ); + e.try_search_half_fwd_stopat(&input) + } else if let Some(e) = self.core.hybrid.get(&input) { + trace!( + "using lazy DFA for forward reverse inner search at {:?}", + input.get_span() + ); + e.try_search_half_fwd_stopat(&mut cache.hybrid, &input) + } else { + unreachable!("ReverseInner always has a DFA") + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn try_search_half_rev_limited( + &self, + cache: &mut Cache, + input: &Input<'_>, + min_start: usize, + ) -> Result<Option<HalfMatch>, RetryError> { + if let Some(e) = self.dfa.get(&input) { + trace!( + "using full DFA for reverse inner search at {:?}, \ + but will be stopped at {} to avoid quadratic behavior", + input.get_span(), + min_start, + ); + e.try_search_half_rev_limited(&input, min_start) + } else if let Some(e) = self.hybrid.get(&input) { + trace!( + "using lazy DFA for reverse inner search at {:?}, \ + but will be stopped at {} to avoid quadratic behavior", + input.get_span(), + min_start, + ); + e.try_search_half_rev_limited( + &mut cache.revhybrid, + &input, + min_start, + ) + } else { + unreachable!("ReverseInner always has a DFA") + } + } +} + +impl Strategy for ReverseInner { + fn name(&self) -> Cow<'static, str> { + Cow::Borrowed("reverse inner") + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn group_info(&self) -> &GroupInfo { + self.core.group_info() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn create_cache(&self) -> Cache { + let mut cache = self.core.create_cache(); + cache.revhybrid = self.hybrid.create_cache(); + cache + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn reset_cache(&self, cache: &mut Cache) { + self.core.reset_cache(cache); + cache.revhybrid.reset(&self.hybrid); + } + + fn is_accelerated(&self) -> bool { + self.preinner.is_fast() + } + + fn memory_usage(&self) -> usize { + self.core.memory_usage() + + self.preinner.memory_usage() + + self.nfarev.memory_usage() + + self.dfa.memory_usage() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search(&self, cache: &mut Cache, input: &Input<'_>) -> Option<Match> { + if input.get_anchored().is_anchored() { + return self.core.search(cache, input); + } + match self.try_search_full(cache, input) { + Err(RetryError::Quadratic(_err)) => { + trace!("reverse inner optimization failed: {_err}"); + self.core.search(cache, input) + } + Err(RetryError::Fail(_err)) => { + trace!("reverse inner fast search failed: {_err}"); + self.core.search_nofail(cache, input) + } + Ok(matornot) => matornot, + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_half( + &self, + cache: &mut Cache, + input: &Input<'_>, + ) -> Option<HalfMatch> { + if input.get_anchored().is_anchored() { + return self.core.search_half(cache, input); + } + match self.try_search_full(cache, input) { + Err(RetryError::Quadratic(_err)) => { + trace!("reverse inner half optimization failed: {_err}"); + self.core.search_half(cache, input) + } + Err(RetryError::Fail(_err)) => { + trace!("reverse inner fast half search failed: {_err}"); + self.core.search_half_nofail(cache, input) + } + Ok(None) => None, + Ok(Some(m)) => Some(HalfMatch::new(m.pattern(), m.end())), + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_match(&self, cache: &mut Cache, input: &Input<'_>) -> bool { + if input.get_anchored().is_anchored() { + return self.core.is_match(cache, input); + } + match self.try_search_full(cache, input) { + Err(RetryError::Quadratic(_err)) => { + trace!("reverse inner half optimization failed: {_err}"); + self.core.is_match_nofail(cache, input) + } + Err(RetryError::Fail(_err)) => { + trace!("reverse inner fast half search failed: {_err}"); + self.core.is_match_nofail(cache, input) + } + Ok(None) => false, + Ok(Some(_)) => true, + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + if input.get_anchored().is_anchored() { + return self.core.search_slots(cache, input, slots); + } + if !self.core.is_capture_search_needed(slots.len()) { + trace!("asked for slots unnecessarily, trying fast path"); + let m = self.search(cache, input)?; + copy_match_to_slots(m, slots); + return Some(m.pattern()); + } + let m = match self.try_search_full(cache, input) { + Err(RetryError::Quadratic(_err)) => { + trace!("reverse inner captures optimization failed: {_err}"); + return self.core.search_slots(cache, input, slots); + } + Err(RetryError::Fail(_err)) => { + trace!("reverse inner fast captures search failed: {_err}"); + return self.core.search_slots_nofail(cache, input, slots); + } + Ok(None) => return None, + Ok(Some(m)) => m, + }; + trace!( + "match found at {}..{} in capture search, \ + using another engine to find captures", + m.start(), + m.end(), + ); + let input = input + .clone() + .span(m.start()..m.end()) + .anchored(Anchored::Pattern(m.pattern())); + self.core.search_slots_nofail(cache, &input, slots) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + self.core.which_overlapping_matches(cache, input, patset) + } +} + +/// Copies the offsets in the given match to the corresponding positions in +/// `slots`. +/// +/// In effect, this sets the slots corresponding to the implicit group for the +/// pattern in the given match. If the indices for the corresponding slots do +/// not exist, then no slots are set. +/// +/// This is useful when the caller provides slots (or captures), but you use a +/// regex engine that doesn't operate on slots (like a lazy DFA). This function +/// lets you map the match you get back to the slots provided by the caller. +#[cfg_attr(feature = "perf-inline", inline(always))] +fn copy_match_to_slots(m: Match, slots: &mut [Option<NonMaxUsize>]) { + let slot_start = m.pattern().as_usize() * 2; + let slot_end = slot_start + 1; + if let Some(slot) = slots.get_mut(slot_start) { + *slot = NonMaxUsize::new(m.start()); + } + if let Some(slot) = slots.get_mut(slot_end) { + *slot = NonMaxUsize::new(m.end()); + } +} + +// We only test which strategy we get when all literal features are enabled. +// Other cases are less substantially less interesting. +// +// We also don't test this on miri since it takes forever. +// +// We also require `unicode-perl` since some regexes use `\w`, `\d` and `\s`. +#[cfg(all( + feature = "perf-literal-substring", + feature = "perf-literal-multisubstring", + feature = "unicode-perl", + not(miri), +))] +#[cfg(test)] +mod tests { + use alloc::{format, string::String, vec::Vec}; + + use crate::{meta::regex::Config, util::syntax}; + + use super::*; + + fn teddy_probably_available() -> bool { + cfg!(any(target_arch = "x86_64", target_arch = "aarch64")) + } + + #[track_caller] + fn strategy_with(patterns: &[&str], config: Config) -> Arc<dyn Strategy> { + let hirs = syntax::parse_many(patterns).unwrap(); + let hirs: Vec<&Hir> = hirs.iter().collect(); + let info = RegexInfo::new(config, &hirs); + new(&info, &hirs).unwrap() + } + + #[track_caller] + fn assert_strategy(name: &'static str, patterns: &[&str]) { + assert_strategy_with(name, patterns, Config::new()); + } + + #[track_caller] + fn assert_strategy_with( + name: &'static str, + patterns: &[&str], + config: Config, + ) { + let strategy = strategy_with(patterns, config); + assert_eq!(name, strategy.name().as_ref()); + } + + fn literal_alternation(count: usize) -> String { + let mut pattern = String::new(); + for i in 0..count { + if i > 0 { + pattern.push('|'); + } + pattern.push_str(&format!("needle{i:05}")); + } + pattern + } + + #[test] + fn pre_from_prefixes_accepts_exact_literal() { + assert_strategy("prefilter memchr", &["a"]); + assert_strategy("prefilter memchr2", &["a|b"]); + assert_strategy("prefilter memchr3", &["a|b|c"]); + assert_strategy("prefilter memmem", &["Sherlock"]); + if teddy_probably_available() { + assert_strategy( + "prefilter teddy", + &["Samwise|Gandalf|Holmes|Watson"], + ); + } + } + + #[test] + fn pre_from_prefixes_rejects_inexact_literal() { + assert_strategy("core", &["a+"]); + } + + #[test] + fn pre_from_prefixes_rejects_empty_literal() { + assert_strategy("core", &[""]); + } + + #[test] + fn pre_from_prefixes_rejects_multiple_patterns() { + assert_strategy("core", &["a", "b"]); + assert_strategy("core", &["a|b", "c|d"]); + } + + #[test] + fn pre_from_prefixes_rejects_captures() { + assert_strategy("core", &["(a)"]); + // ... but not when it's a non-capture. + assert_strategy("prefilter memchr", &["(?:a)"]); + // ... or when there is a capture, but it's optimized out. + assert_strategy("prefilter memchr", &["(){0}a"]); + } + + #[test] + fn pre_from_prefixes_rejects_look_around() { + assert_strategy("core", &[r"a\b"]); + } + + #[test] + fn pre_from_prefixes_rejects_non_leftmost_first() { + assert_strategy_with( + "core", + &["a"], + Config::new().match_kind(MatchKind::All), + ); + } + + #[test] + fn pre_from_alternation_literals_accepts_large_alternation() { + let pattern = literal_alternation(3_000); + assert_strategy("prefilter aho-corasick", &[&pattern]); + } + + #[test] + fn pre_from_alternation_literals_rejects_small_alternation() { + let pattern = literal_alternation(2_999); + assert_strategy("core", &[&pattern]); + } + + #[test] + fn pre_from_alternation_literals_rejects_captures() { + let pattern = format!("({})", literal_alternation(3_000)); + assert_strategy("core", &[&pattern]); + } + + #[test] + fn pre_from_alternation_literals_rejects_look_around() { + let pattern = format!(r"(?:{})\b", literal_alternation(3_000)); + assert_strategy("core", &[&pattern]); + } + + #[test] + fn pre_from_alternation_literals_rejects_non_leftmost_first() { + let pattern = literal_alternation(3_000); + assert_strategy_with( + "core", + &[&pattern], + Config::new().match_kind(MatchKind::All), + ); + } + + #[test] + fn pre_from_alternation_literals_rejects_multiple_patterns() { + let pattern1 = literal_alternation(3_000); + let pattern2 = literal_alternation(3_000); + assert_strategy("core", &[&pattern1, &pattern2]); + } + + #[test] + fn core_selected_when_no_special_strategy_applies() { + assert_strategy("core", &["[a-z]+"]); + } + + #[test] + fn core_selected_when_auto_prefilter_is_disabled() { + assert_strategy_with( + "core", + &[r"\w+Holmes"], + Config::new().auto_prefilter(false), + ); + } + + #[test] + fn core_selected_when_reverse_engines_are_disabled() { + assert_strategy_with( + "core", + &[r"\w+Holmes"], + Config::new().dfa(false).hybrid(false), + ); + } + + #[test] + fn reverse_anchored_accepts_end_anchored() { + assert_strategy("reverse anchored", &[r"\w+Holmes$"]); + } + + #[test] + fn reverse_anchored_accepts_multiple_end_anchored_patterns() { + assert_strategy("reverse anchored", &[r"\w+Holmes$", r"\w+Watson$"]); + } + + #[test] + fn reverse_anchored_accepts_hybrid_without_full_dfa() { + assert_strategy_with( + "reverse anchored", + &[r"\w+Holmes$"], + Config::new().dfa(false), + ); + } + + #[test] + fn reverse_anchored_rejects_start_and_end_anchored() { + assert_strategy("core", &[r"^\w+Holmes$"]); + } + + #[test] + fn reverse_anchored_rejects_without_reverse_engine() { + assert_strategy_with( + "core", + &[r"\w+Holmes$"], + Config::new().dfa(false).hybrid(false), + ); + } + + #[test] + fn reverse_suffix_accepts_prefix_without_internal_suffix() { + assert_strategy("reverse suffix", &[r"\d+XYZ"]); + assert_strategy("reverse suffix", &[r"[a-q][^u-z]{13}x"]); + } + + #[test] + fn reverse_suffix_rejects_internal_suffix_with_word_boundary() { + assert_strategy("core", &[r"\b\w+nn\b"]); + } + + #[test] + fn reverse_suffix_accepts_disjoint_class_separator() { + assert_strategy("reverse suffix", &[r"\w+\s+Holmes"]); + assert_strategy("reverse suffix", &[r"\w+\d*\s+Holmes"]); + assert_strategy("reverse suffix", &[r"(?:\w+|[.-]+)+\s+Holmes"]); + assert_strategy("reverse suffix", &[r"\b\w+\s+Holmes"]); + assert_strategy("reverse suffix", &[r"\w+\sHolmes"]); + assert_strategy("reverse suffix", &[r"[a-z]+[0-9]+Holmes"]); + assert_strategy("reverse suffix", &[r"(\w+)(\s+)Holmes"]); + assert_strategy("reverse suffix", &[r"(?-u:[A-Za-z]+[0-9]+Holmes)"]); + } + + #[test] + fn reverse_suffix_class_separator_is_conservative() { + assert_strategy("core", &[r"\w+\w+Holmes"]); + assert_strategy("core", &[r"\w+\s*Holmes"]); + assert_strategy("core", &[r"[a-z]+[0-9a]+xyz"]); + assert_strategy("core", &[r"[a-z]+[0-9]+a1"]); + assert_strategy("core", &[r"(?:.abc)?[a]+[b]+c"]); + } + + #[test] + fn reverse_suffix_rejects_safe_nfa_overlap() { + assert_strategy("reverse inner", &[r"(a|aa)b"]); + } + + #[test] + fn reverse_suffix_accepts_fixed_length_prefix() { + assert_strategy("reverse suffix", &[r"[A-Z][0-9]XYZ"]); + } + + #[test] + fn reverse_suffix_rejects_multiple_patterns_with_common_suffix() { + assert_strategy("core", &[r"\d+XYZ", r"\w+XYZ"]); + } + + #[test] + fn reverse_suffix_rejects_auto_prefilter_disabled() { + assert_strategy_with( + "core", + &[r"\d+XYZ"], + Config::new().auto_prefilter(false), + ); + } + + #[test] + fn reverse_suffix_rejects_non_leftmost_first() { + assert_strategy_with( + "core", + &[r"\d+XYZ"], + Config::new().match_kind(MatchKind::All), + ); + } + + #[test] + fn reverse_suffix_rejects_anchored_start() { + assert_strategy("core", &[r"^\d+XYZ"]); + } + + #[test] + fn reverse_suffix_rejects_variable_length_prefix() { + assert_strategy("core", &[r"(?:[A-Za-z]ab)?b"]); + } + + #[test] + fn reverse_suffix_rejects_without_reverse_engine() { + assert_strategy_with( + "core", + &[r"\d+XYZ"], + Config::new().dfa(false).hybrid(false), + ); + } + + #[test] + fn reverse_suffix_rejects_existing_fast_prefilter() { + assert_strategy("core", &[r"abc\w+XYZ"]); + } + + #[test] + fn reverse_suffix_rejects_no_literal_suffix() { + assert_strategy("core", &[r"[a-z]+"]); + } + + #[test] + fn reverse_suffix_rejects_multiple_patterns_without_common_suffix() { + assert_strategy("core", &[r"\d+XYZ", r"\w+ABC"]); + } + + #[test] + fn reverse_suffix_rejects_multiple_patterns_when_first_looks_safe() { + assert_strategy("core", &[r"\d+b", r".bb|b"]); + assert_strategy("core", &[r"\d+b", r"ab"]); + } + + #[test] + fn reverse_suffix_rejects_unsafe_overlap() { + assert_strategy("core", &[r".abb|b"]); + assert_strategy("core", &[r".bb|b"]); + } + + #[test] + fn reverse_suffix_rejects_fully_absorbed_literal() { + assert_strategy("core", &[r"(?:[a-wyz]{3}|[a-wyz]).b"]); + } + + #[test] + fn reverse_suffix_rejects_disjoint_trailing_component() { + assert_strategy("core", &[r"(?:[ac-z]{2}b[ac-z])?[ac-z]b"]); + } + + #[test] + fn reverse_suffix_rejects_internal_candidate_before_absorber() { + assert_strategy("core", &[r"(?:[a-z]cb|c)[a-z]*b"]); + } + + #[test] + fn reverse_inner_accepts_disjoint_inner_literal() { + assert_strategy("reverse inner", &[r"\w+@\w+"]); + assert_strategy( + "reverse inner", + &[r"(?P<email>[.\pL]+@(?P<domain>[.\pL]+))"], + ); + } + + #[test] + fn reverse_inner_accepts_fixed_length_prefix() { + assert_strategy("reverse inner", &[r"[A-Z][0-9]@(foo|bar)"]); + } + + #[test] + fn reverse_inner_accepts_disjoint_class_separator() { + assert_strategy("reverse inner", &[r"\w+\s+Holmes\s+\w+"]); + assert_strategy("reverse inner", &[r"\w+\d*\s+Holmes\s+\w+"]); + assert_strategy("reverse inner", &[r"\b\w+\s+Holmes\s+\w+"]); + if teddy_probably_available() { + assert_strategy( + "reverse inner", + &[r"\w+\s+(?:Holmes|Watson)\s+\w+"], + ); + } + } + + #[test] + fn reverse_inner_accepts_leading_disjoint_class_separator() { + assert_strategy("reverse inner", &[r"\s[A-Za-z]{0,12}ing\s"]); + assert_strategy("reverse inner", &[r"\s[A-Za-z]*ing\s"]); + assert_strategy("reverse inner", &[r"\b\s[A-Za-z]{0,12}ing\s"]); + assert_strategy("reverse inner", &[r"(\s)([A-Za-z]{0,12})ing\s"]); + } + + #[test] + fn reverse_inner_leading_class_separator_is_conservative() { + assert_strategy("core", &[r"\s[\sA-Za-z]{0,12}ing\s"]); + assert_strategy("core", &[r"\s*[A-Za-z]{0,12}ing\s"]); + } + + #[test] + fn reverse_inner_rejects_auto_prefilter_disabled() { + assert_strategy_with( + "core", + &[r"\w+@\w+"], + Config::new().auto_prefilter(false), + ); + } + + #[test] + fn reverse_inner_rejects_non_leftmost_first() { + assert_strategy_with( + "core", + &[r"\w+@\w+"], + Config::new().match_kind(MatchKind::All), + ); + } + + #[test] + fn reverse_inner_rejects_anchored_start() { + assert_strategy("core", &[r"^\w+@\w+"]); + } + + #[test] + fn reverse_inner_rejects_without_reverse_engine() { + assert_strategy_with( + "core", + &[r"\w+@\w+"], + Config::new().dfa(false).hybrid(false), + ); + } + + #[test] + fn reverse_inner_rejects_existing_fast_prefilter() { + assert_strategy("core", &[r"abc\w+@\w+"]); + } + + #[test] + fn reverse_inner_rejects_multiple_patterns() { + assert_strategy("core", &[r"\w+@\w+", r"\w+#\w+"]); + } + + #[test] + fn reverse_inner_rejects_no_top_level_concat() { + assert_strategy("core", &[r"\w+"]); + } + + #[test] + fn reverse_inner_rejects_unsafe_overlap() { + assert_strategy("core", &[r"a.*@\w+"]); + } + + #[test] + fn reverse_inner_rejects_literal_crossing_prefix_boundary() { + assert_strategy("core", &[r"(?:[ac-z]{3}|[ac-z])(?:aba|baa)"]); + } + + #[test] + fn reverse_inner_rejects_internal_separator_alignment() { + assert_strategy("core", &[r"(?:.abc)?[a]+[b]+c.*"]); + } + + #[test] + fn rebar_benchmarks() { + assert_strategy("reverse suffix", &[r"[a-z]shing"]); + assert_strategy("reverse suffix", &[r"[a-q][^u-z]{23}x"]); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/meta/wrappers.rs b/third_party/crates/regex-automata-0.4.18/src/meta/wrappers.rs new file mode 100644 index 0000000..8d6f738 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/meta/wrappers.rs @@ -0,0 +1,1336 @@ +/*! +This module contains a boat load of wrappers around each of our internal regex +engines. They encapsulate a few things: + +1. The wrappers manage the conditional existence of the regex engine. Namely, +the PikeVM is the only required regex engine. The rest are optional. These +wrappers present a uniform API regardless of which engines are available. And +availability might be determined by compile time features or by dynamic +configuration via `meta::Config`. Encapsulating the conditional compilation +features is in particular a huge simplification for the higher level code that +composes these engines. +2. The wrappers manage construction of each engine, including skipping it if +the engine is unavailable or configured to not be used. +3. The wrappers manage whether an engine *can* be used for a particular +search configuration. For example, `BoundedBacktracker::get` only returns a +backtracking engine when the haystack is bigger than the maximum supported +length. The wrappers also sometimes take a position on when an engine *ought* +to be used, but only in cases where the logic is extremely local to the engine +itself. Otherwise, things like "choose between the backtracker and the one-pass +DFA" are managed by the higher level meta strategy code. + +There are also corresponding wrappers for the various `Cache` types for each +regex engine that needs them. If an engine is unavailable or not used, then a +cache for it will *not* actually be allocated. +*/ + +use alloc::vec::Vec; + +use crate::{ + meta::{ + error::{BuildError, RetryError, RetryFailError}, + regex::RegexInfo, + }, + nfa::thompson::{pikevm, NFA}, + util::{prefilter::Prefilter, primitives::NonMaxUsize}, + HalfMatch, Input, Match, MatchKind, PatternID, PatternSet, +}; + +#[cfg(feature = "dfa-build")] +use crate::dfa; +#[cfg(feature = "dfa-onepass")] +use crate::dfa::onepass; +#[cfg(feature = "hybrid")] +use crate::hybrid; +#[cfg(feature = "nfa-backtrack")] +use crate::nfa::thompson::backtrack; + +#[derive(Debug)] +pub(crate) struct PikeVM(PikeVMEngine); + +impl PikeVM { + pub(crate) fn new( + info: &RegexInfo, + pre: Option<Prefilter>, + nfa: &NFA, + ) -> Result<PikeVM, BuildError> { + PikeVMEngine::new(info, pre, nfa).map(PikeVM) + } + + pub(crate) fn create_cache(&self) -> PikeVMCache { + PikeVMCache::none() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn get(&self) -> &PikeVMEngine { + &self.0 + } +} + +#[derive(Debug)] +pub(crate) struct PikeVMEngine(pikevm::PikeVM); + +impl PikeVMEngine { + pub(crate) fn new( + info: &RegexInfo, + pre: Option<Prefilter>, + nfa: &NFA, + ) -> Result<PikeVMEngine, BuildError> { + let pikevm_config = pikevm::Config::new() + .match_kind(info.config().get_match_kind()) + .prefilter(pre); + let engine = pikevm::Builder::new() + .configure(pikevm_config) + .build_from_nfa(nfa.clone()) + .map_err(BuildError::nfa)?; + debug!("PikeVM built"); + Ok(PikeVMEngine(engine)) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn is_match( + &self, + cache: &mut PikeVMCache, + input: &Input<'_>, + ) -> bool { + self.0.is_match(cache.get(&self.0), input.clone()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn search_slots( + &self, + cache: &mut PikeVMCache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + self.0.search_slots(cache.get(&self.0), input, slots) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn which_overlapping_matches( + &self, + cache: &mut PikeVMCache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + self.0.which_overlapping_matches(cache.get(&self.0), input, patset) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct PikeVMCache(Option<pikevm::Cache>); + +impl PikeVMCache { + pub(crate) fn none() -> PikeVMCache { + PikeVMCache(None) + } + + pub(crate) fn reset(&mut self, builder: &PikeVM) { + self.get(&builder.get().0).reset(&builder.get().0); + } + + pub(crate) fn memory_usage(&self) -> usize { + self.0.as_ref().map_or(0, |c| c.memory_usage()) + } + + fn get(&mut self, vm: &pikevm::PikeVM) -> &mut pikevm::Cache { + self.0.get_or_insert_with(|| vm.create_cache()) + } +} + +#[derive(Debug)] +pub(crate) struct BoundedBacktracker(Option<BoundedBacktrackerEngine>); + +impl BoundedBacktracker { + pub(crate) fn new( + info: &RegexInfo, + pre: Option<Prefilter>, + nfa: &NFA, + ) -> Result<BoundedBacktracker, BuildError> { + BoundedBacktrackerEngine::new(info, pre, nfa).map(BoundedBacktracker) + } + + pub(crate) fn create_cache(&self) -> BoundedBacktrackerCache { + BoundedBacktrackerCache::none() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn get( + &self, + input: &Input<'_>, + ) -> Option<&BoundedBacktrackerEngine> { + let engine = self.0.as_ref()?; + // It is difficult to make the backtracker give up early if it is + // guaranteed to eventually wind up in a match state. This is because + // of the greedy nature of a backtracker: it just blindly mushes + // forward. Every other regex engine is able to give up more quickly, + // so even if the backtracker might be able to zip through faster than + // (say) the PikeVM, we prefer the theoretical benefit that some other + // engine might be able to scan much less of the haystack than the + // backtracker. + // + // Now, if the haystack is really short already, then we allow the + // backtracker to run. (This hasn't been litigated quantitatively with + // benchmarks. Just a hunch.) + if input.get_earliest() && input.haystack().len() > 128 { + return None; + } + // If the backtracker is just going to return an error because the + // haystack is too long, then obviously do not use it. + if input.get_span().len() > engine.max_haystack_len() { + return None; + } + Some(engine) + } +} + +#[derive(Debug)] +pub(crate) struct BoundedBacktrackerEngine( + #[cfg(feature = "nfa-backtrack")] backtrack::BoundedBacktracker, + #[cfg(not(feature = "nfa-backtrack"))] (), +); + +impl BoundedBacktrackerEngine { + pub(crate) fn new( + info: &RegexInfo, + pre: Option<Prefilter>, + nfa: &NFA, + ) -> Result<Option<BoundedBacktrackerEngine>, BuildError> { + #[cfg(feature = "nfa-backtrack")] + { + if !info.config().get_backtrack() + || info.config().get_match_kind() != MatchKind::LeftmostFirst + { + return Ok(None); + } + let backtrack_config = backtrack::Config::new().prefilter(pre); + let engine = backtrack::Builder::new() + .configure(backtrack_config) + .build_from_nfa(nfa.clone()) + .map_err(BuildError::nfa)?; + debug!( + "BoundedBacktracker built (max haystack length: {:?})", + engine.max_haystack_len() + ); + Ok(Some(BoundedBacktrackerEngine(engine))) + } + #[cfg(not(feature = "nfa-backtrack"))] + { + Ok(None) + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn is_match( + &self, + cache: &mut BoundedBacktrackerCache, + input: &Input<'_>, + ) -> bool { + #[cfg(feature = "nfa-backtrack")] + { + // OK because we only permit access to this engine when we know + // the haystack is short enough for the backtracker to run without + // reporting an error. + self.0.try_is_match(cache.get(&self.0), input.clone()).unwrap() + } + #[cfg(not(feature = "nfa-backtrack"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn search_slots( + &self, + cache: &mut BoundedBacktrackerCache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + #[cfg(feature = "nfa-backtrack")] + { + // OK because we only permit access to this engine when we know + // the haystack is short enough for the backtracker to run without + // reporting an error. + self.0.try_search_slots(cache.get(&self.0), input, slots).unwrap() + } + #[cfg(not(feature = "nfa-backtrack"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn max_haystack_len(&self) -> usize { + #[cfg(feature = "nfa-backtrack")] + { + self.0.max_haystack_len() + } + #[cfg(not(feature = "nfa-backtrack"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct BoundedBacktrackerCache( + #[cfg(feature = "nfa-backtrack")] Option<backtrack::Cache>, + #[cfg(not(feature = "nfa-backtrack"))] (), +); + +impl BoundedBacktrackerCache { + pub(crate) fn none() -> BoundedBacktrackerCache { + #[cfg(feature = "nfa-backtrack")] + { + BoundedBacktrackerCache(None) + } + #[cfg(not(feature = "nfa-backtrack"))] + { + BoundedBacktrackerCache(()) + } + } + + pub(crate) fn reset(&mut self, builder: &BoundedBacktracker) { + #[cfg(feature = "nfa-backtrack")] + if let Some(ref e) = builder.0 { + self.get(&e.0).reset(&e.0); + } + } + + pub(crate) fn memory_usage(&self) -> usize { + #[cfg(feature = "nfa-backtrack")] + { + self.0.as_ref().map_or(0, |c| c.memory_usage()) + } + #[cfg(not(feature = "nfa-backtrack"))] + { + 0 + } + } + + #[cfg(feature = "nfa-backtrack")] + fn get( + &mut self, + bb: &backtrack::BoundedBacktracker, + ) -> &mut backtrack::Cache { + self.0.get_or_insert_with(|| bb.create_cache()) + } +} + +#[derive(Debug)] +pub(crate) struct OnePass(Option<OnePassEngine>); + +impl OnePass { + pub(crate) fn new(info: &RegexInfo, nfa: &NFA) -> OnePass { + OnePass(OnePassEngine::new(info, nfa)) + } + + pub(crate) fn create_cache(&self) -> OnePassCache { + OnePassCache::new(self) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn get(&self, input: &Input<'_>) -> Option<&OnePassEngine> { + let engine = self.0.as_ref()?; + if !input.get_anchored().is_anchored() + && !engine.get_nfa().is_always_start_anchored() + { + return None; + } + Some(engine) + } + + pub(crate) fn memory_usage(&self) -> usize { + self.0.as_ref().map_or(0, |e| e.memory_usage()) + } +} + +#[derive(Debug)] +pub(crate) struct OnePassEngine( + #[cfg(feature = "dfa-onepass")] onepass::DFA, + #[cfg(not(feature = "dfa-onepass"))] (), +); + +impl OnePassEngine { + pub(crate) fn new(info: &RegexInfo, nfa: &NFA) -> Option<OnePassEngine> { + #[cfg(feature = "dfa-onepass")] + { + if !info.config().get_onepass() { + return None; + } + // In order to even attempt building a one-pass DFA, we require + // that we either have at least one explicit capturing group or + // there's a Unicode word boundary somewhere. If we don't have + // either of these things, then the lazy DFA will almost certainly + // be usable and be much faster. The only case where it might + // not is if the lazy DFA isn't utilizing its cache effectively, + // but in those cases, the underlying regex is almost certainly + // not one-pass or is too big to fit within the current one-pass + // implementation limits. + if info.props_union().explicit_captures_len() == 0 + && !info.props_union().look_set().contains_word_unicode() + { + debug!("not building OnePass because it isn't worth it"); + return None; + } + let onepass_config = onepass::Config::new() + .match_kind(info.config().get_match_kind()) + // Like for the lazy DFA, we unconditionally enable this + // because it doesn't cost much and makes the API more + // flexible. + .starts_for_each_pattern(true) + .byte_classes(info.config().get_byte_classes()) + .size_limit(info.config().get_onepass_size_limit()); + let result = onepass::Builder::new() + .configure(onepass_config) + .build_from_nfa(nfa.clone()); + let engine = match result { + Ok(engine) => engine, + Err(_err) => { + debug!("OnePass failed to build: {_err}"); + return None; + } + }; + debug!("OnePass built, {} bytes", engine.memory_usage()); + Some(OnePassEngine(engine)) + } + #[cfg(not(feature = "dfa-onepass"))] + { + None + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn search_slots( + &self, + cache: &mut OnePassCache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + #[cfg(feature = "dfa-onepass")] + { + // OK because we only permit getting a OnePassEngine when we know + // the search is anchored and thus an error cannot occur. + self.0 + .try_search_slots(cache.0.as_mut().unwrap(), input, slots) + .unwrap() + } + #[cfg(not(feature = "dfa-onepass"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + pub(crate) fn memory_usage(&self) -> usize { + #[cfg(feature = "dfa-onepass")] + { + self.0.memory_usage() + } + #[cfg(not(feature = "dfa-onepass"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn get_nfa(&self) -> &NFA { + #[cfg(feature = "dfa-onepass")] + { + self.0.get_nfa() + } + #[cfg(not(feature = "dfa-onepass"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct OnePassCache( + #[cfg(feature = "dfa-onepass")] Option<onepass::Cache>, + #[cfg(not(feature = "dfa-onepass"))] (), +); + +impl OnePassCache { + pub(crate) fn none() -> OnePassCache { + #[cfg(feature = "dfa-onepass")] + { + OnePassCache(None) + } + #[cfg(not(feature = "dfa-onepass"))] + { + OnePassCache(()) + } + } + + pub(crate) fn new(builder: &OnePass) -> OnePassCache { + #[cfg(feature = "dfa-onepass")] + { + OnePassCache(builder.0.as_ref().map(|e| e.0.create_cache())) + } + #[cfg(not(feature = "dfa-onepass"))] + { + OnePassCache(()) + } + } + + pub(crate) fn reset(&mut self, builder: &OnePass) { + #[cfg(feature = "dfa-onepass")] + if let Some(ref e) = builder.0 { + self.0.as_mut().unwrap().reset(&e.0); + } + } + + pub(crate) fn memory_usage(&self) -> usize { + #[cfg(feature = "dfa-onepass")] + { + self.0.as_ref().map_or(0, |c| c.memory_usage()) + } + #[cfg(not(feature = "dfa-onepass"))] + { + 0 + } + } +} + +#[derive(Debug)] +pub(crate) struct Hybrid(Option<HybridEngine>); + +impl Hybrid { + pub(crate) fn none() -> Hybrid { + Hybrid(None) + } + + pub(crate) fn new( + info: &RegexInfo, + pre: Option<Prefilter>, + nfa: &NFA, + nfarev: &NFA, + ) -> Hybrid { + Hybrid(HybridEngine::new(info, pre, nfa, nfarev)) + } + + pub(crate) fn create_cache(&self) -> HybridCache { + HybridCache::new(self) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn get(&self, _input: &Input<'_>) -> Option<&HybridEngine> { + let engine = self.0.as_ref()?; + Some(engine) + } + + pub(crate) fn is_some(&self) -> bool { + self.0.is_some() + } +} + +#[derive(Debug)] +pub(crate) struct HybridEngine( + #[cfg(feature = "hybrid")] hybrid::regex::Regex, + #[cfg(not(feature = "hybrid"))] (), +); + +impl HybridEngine { + pub(crate) fn new( + info: &RegexInfo, + pre: Option<Prefilter>, + nfa: &NFA, + nfarev: &NFA, + ) -> Option<HybridEngine> { + #[cfg(feature = "hybrid")] + { + if !info.config().get_hybrid() { + return None; + } + let dfa_config = hybrid::dfa::Config::new() + .match_kind(info.config().get_match_kind()) + .prefilter(pre.clone()) + // Enabling this is necessary for ensuring we can service any + // kind of 'Input' search without error. For the lazy DFA, + // this is not particularly costly, since the start states are + // generated lazily. + .starts_for_each_pattern(true) + .byte_classes(info.config().get_byte_classes()) + .unicode_word_boundary(true) + .specialize_start_states(pre.is_some()) + .cache_capacity(info.config().get_hybrid_cache_capacity()) + // This makes it possible for building a lazy DFA to + // fail even though the NFA has already been built. Namely, + // if the cache capacity is too small to fit some minimum + // number of states (which is small, like 4 or 5), then the + // DFA will refuse to build. + // + // We shouldn't enable this to make building always work, since + // this could cause the allocation of a cache bigger than the + // provided capacity amount. + // + // This is effectively the only reason why building a lazy DFA + // could fail. If it does, then we simply suppress the error + // and return None. + .skip_cache_capacity_check(false) + // This and enabling heuristic Unicode word boundary support + // above make it so the lazy DFA can quit at match time. + .minimum_cache_clear_count(Some(3)) + .minimum_bytes_per_state(Some(10)); + let result = hybrid::dfa::Builder::new() + .configure(dfa_config.clone()) + .build_from_nfa(nfa.clone()); + let fwd = match result { + Ok(fwd) => fwd, + Err(_err) => { + debug!("forward lazy DFA failed to build: {_err}"); + return None; + } + }; + let result = hybrid::dfa::Builder::new() + .configure( + dfa_config + .clone() + .match_kind(MatchKind::All) + .prefilter(None) + .specialize_start_states(false), + ) + .build_from_nfa(nfarev.clone()); + let rev = match result { + Ok(rev) => rev, + Err(_err) => { + debug!("reverse lazy DFA failed to build: {_err}"); + return None; + } + }; + let engine = + hybrid::regex::Builder::new().build_from_dfas(fwd, rev); + debug!("lazy DFA built"); + Some(HybridEngine(engine)) + } + #[cfg(not(feature = "hybrid"))] + { + None + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search( + &self, + cache: &mut HybridCache, + input: &Input<'_>, + ) -> Result<Option<Match>, RetryFailError> { + #[cfg(feature = "hybrid")] + { + let cache = cache.0.as_mut().unwrap(); + self.0.try_search(cache, input).map_err(|e| e.into()) + } + #[cfg(not(feature = "hybrid"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_fwd( + &self, + cache: &mut HybridCache, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, RetryFailError> { + #[cfg(feature = "hybrid")] + { + let fwd = self.0.forward(); + let mut fwdcache = cache.0.as_mut().unwrap().as_parts_mut().0; + fwd.try_search_fwd(&mut fwdcache, input).map_err(|e| e.into()) + } + #[cfg(not(feature = "hybrid"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_fwd_stopat( + &self, + cache: &mut HybridCache, + input: &Input<'_>, + ) -> Result<Result<HalfMatch, usize>, RetryFailError> { + #[cfg(feature = "hybrid")] + { + let dfa = self.0.forward(); + let mut cache = cache.0.as_mut().unwrap().as_parts_mut().0; + crate::meta::stopat::hybrid_try_search_half_fwd( + dfa, &mut cache, input, + ) + } + #[cfg(not(feature = "hybrid"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_rev( + &self, + cache: &mut HybridCache, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, RetryFailError> { + #[cfg(feature = "hybrid")] + { + let rev = self.0.reverse(); + let mut revcache = cache.0.as_mut().unwrap().as_parts_mut().1; + rev.try_search_rev(&mut revcache, input).map_err(|e| e.into()) + } + #[cfg(not(feature = "hybrid"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_rev_limited( + &self, + cache: &mut HybridCache, + input: &Input<'_>, + min_start: usize, + ) -> Result<Option<HalfMatch>, RetryError> { + #[cfg(feature = "hybrid")] + { + let dfa = self.0.reverse(); + let mut cache = cache.0.as_mut().unwrap().as_parts_mut().1; + crate::meta::limited::hybrid_try_search_half_rev( + dfa, &mut cache, input, min_start, + ) + } + #[cfg(not(feature = "hybrid"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[inline] + pub(crate) fn try_which_overlapping_matches( + &self, + cache: &mut HybridCache, + input: &Input<'_>, + patset: &mut PatternSet, + ) -> Result<(), RetryFailError> { + #[cfg(feature = "hybrid")] + { + let fwd = self.0.forward(); + let mut fwdcache = cache.0.as_mut().unwrap().as_parts_mut().0; + fwd.try_which_overlapping_matches(&mut fwdcache, input, patset) + .map_err(|e| e.into()) + } + #[cfg(not(feature = "hybrid"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct HybridCache( + #[cfg(feature = "hybrid")] Option<hybrid::regex::Cache>, + #[cfg(not(feature = "hybrid"))] (), +); + +impl HybridCache { + pub(crate) fn none() -> HybridCache { + #[cfg(feature = "hybrid")] + { + HybridCache(None) + } + #[cfg(not(feature = "hybrid"))] + { + HybridCache(()) + } + } + + pub(crate) fn new(builder: &Hybrid) -> HybridCache { + #[cfg(feature = "hybrid")] + { + HybridCache(builder.0.as_ref().map(|e| e.0.create_cache())) + } + #[cfg(not(feature = "hybrid"))] + { + HybridCache(()) + } + } + + pub(crate) fn reset(&mut self, builder: &Hybrid) { + #[cfg(feature = "hybrid")] + if let Some(ref e) = builder.0 { + self.0.as_mut().unwrap().reset(&e.0); + } + } + + pub(crate) fn memory_usage(&self) -> usize { + #[cfg(feature = "hybrid")] + { + self.0.as_ref().map_or(0, |c| c.memory_usage()) + } + #[cfg(not(feature = "hybrid"))] + { + 0 + } + } +} + +#[derive(Debug)] +pub(crate) struct DFA(Option<DFAEngine>); + +impl DFA { + pub(crate) fn none() -> DFA { + DFA(None) + } + + pub(crate) fn new( + info: &RegexInfo, + pre: Option<Prefilter>, + nfa: &NFA, + nfarev: &NFA, + ) -> DFA { + DFA(DFAEngine::new(info, pre, nfa, nfarev)) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn get(&self, _input: &Input<'_>) -> Option<&DFAEngine> { + let engine = self.0.as_ref()?; + Some(engine) + } + + pub(crate) fn is_some(&self) -> bool { + self.0.is_some() + } + + pub(crate) fn memory_usage(&self) -> usize { + self.0.as_ref().map_or(0, |e| e.memory_usage()) + } +} + +#[derive(Debug)] +pub(crate) struct DFAEngine( + #[cfg(feature = "dfa-build")] dfa::regex::Regex, + #[cfg(not(feature = "dfa-build"))] (), +); + +impl DFAEngine { + pub(crate) fn new( + info: &RegexInfo, + pre: Option<Prefilter>, + nfa: &NFA, + nfarev: &NFA, + ) -> Option<DFAEngine> { + #[cfg(feature = "dfa-build")] + { + if !info.config().get_dfa() { + return None; + } + // If our NFA is anything but small, don't even bother with a DFA. + if let Some(state_limit) = info.config().get_dfa_state_limit() { + if nfa.states().len() > state_limit { + debug!( + "skipping full DFA because NFA has {} states, \ + which exceeds the heuristic limit of {}", + nfa.states().len(), + state_limit, + ); + return None; + } + } + // We cut the size limit in four because the total heap used by + // DFA construction is determinization aux memory and the DFA + // itself, and those things are configured independently in the + // lower level DFA builder API. And then split that in two because + // of forward and reverse DFAs. + let size_limit = info.config().get_dfa_size_limit().map(|n| n / 4); + let dfa_config = dfa::dense::Config::new() + .match_kind(info.config().get_match_kind()) + .prefilter(pre.clone()) + // Enabling this is necessary for ensuring we can service any + // kind of 'Input' search without error. For the full DFA, this + // can be quite costly. But since we have such a small bound + // on the size of the DFA, in practice, any multi-regexes are + // probably going to blow the limit anyway. + .starts_for_each_pattern(true) + .byte_classes(info.config().get_byte_classes()) + .unicode_word_boundary(true) + .specialize_start_states(pre.is_some()) + .determinize_size_limit(size_limit) + .dfa_size_limit(size_limit); + let result = dfa::dense::Builder::new() + .configure(dfa_config.clone()) + .build_from_nfa(&nfa); + let fwd = match result { + Ok(fwd) => fwd, + Err(_err) => { + debug!("forward full DFA failed to build: {_err}"); + return None; + } + }; + let result = dfa::dense::Builder::new() + .configure( + dfa_config + .clone() + // We never need unanchored reverse searches, so + // there's no point in building it into the DFA, which + // WILL take more space. (This isn't done for the lazy + // DFA because the DFA is, well, lazy. It doesn't pay + // the cost for supporting unanchored searches unless + // you actually do an unanchored search, which we + // don't.) + .start_kind(dfa::StartKind::Anchored) + .match_kind(MatchKind::All) + .prefilter(None) + .specialize_start_states(false), + ) + .build_from_nfa(&nfarev); + let rev = match result { + Ok(rev) => rev, + Err(_err) => { + debug!("reverse full DFA failed to build: {_err}"); + return None; + } + }; + let engine = dfa::regex::Builder::new().build_from_dfas(fwd, rev); + debug!( + "fully compiled forward and reverse DFAs built, {} bytes", + engine.forward().memory_usage() + + engine.reverse().memory_usage(), + ); + Some(DFAEngine(engine)) + } + #[cfg(not(feature = "dfa-build"))] + { + None + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search( + &self, + input: &Input<'_>, + ) -> Result<Option<Match>, RetryFailError> { + #[cfg(feature = "dfa-build")] + { + self.0.try_search(input).map_err(|e| e.into()) + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_fwd( + &self, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, RetryFailError> { + #[cfg(feature = "dfa-build")] + { + use crate::dfa::Automaton; + self.0.forward().try_search_fwd(input).map_err(|e| e.into()) + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_fwd_stopat( + &self, + input: &Input<'_>, + ) -> Result<Result<HalfMatch, usize>, RetryFailError> { + #[cfg(feature = "dfa-build")] + { + let dfa = self.0.forward(); + crate::meta::stopat::dfa_try_search_half_fwd(dfa, input) + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_rev( + &self, + input: &Input<'_>, + ) -> Result<Option<HalfMatch>, RetryFailError> { + #[cfg(feature = "dfa-build")] + { + use crate::dfa::Automaton; + self.0.reverse().try_search_rev(&input).map_err(|e| e.into()) + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_rev_limited( + &self, + input: &Input<'_>, + min_start: usize, + ) -> Result<Option<HalfMatch>, RetryError> { + #[cfg(feature = "dfa-build")] + { + let dfa = self.0.reverse(); + crate::meta::limited::dfa_try_search_half_rev( + dfa, input, min_start, + ) + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + #[inline] + pub(crate) fn try_which_overlapping_matches( + &self, + input: &Input<'_>, + patset: &mut PatternSet, + ) -> Result<(), RetryFailError> { + #[cfg(feature = "dfa-build")] + { + use crate::dfa::Automaton; + self.0 + .forward() + .try_which_overlapping_matches(input, patset) + .map_err(|e| e.into()) + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + pub(crate) fn memory_usage(&self) -> usize { + #[cfg(feature = "dfa-build")] + { + self.0.forward().memory_usage() + self.0.reverse().memory_usage() + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } +} + +#[derive(Debug)] +pub(crate) struct ReverseHybrid(Option<ReverseHybridEngine>); + +impl ReverseHybrid { + pub(crate) fn none() -> ReverseHybrid { + ReverseHybrid(None) + } + + pub(crate) fn new(info: &RegexInfo, nfarev: &NFA) -> ReverseHybrid { + ReverseHybrid(ReverseHybridEngine::new(info, nfarev)) + } + + pub(crate) fn create_cache(&self) -> ReverseHybridCache { + ReverseHybridCache::new(self) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn get( + &self, + _input: &Input<'_>, + ) -> Option<&ReverseHybridEngine> { + let engine = self.0.as_ref()?; + Some(engine) + } +} + +#[derive(Debug)] +pub(crate) struct ReverseHybridEngine( + #[cfg(feature = "hybrid")] hybrid::dfa::DFA, + #[cfg(not(feature = "hybrid"))] (), +); + +impl ReverseHybridEngine { + pub(crate) fn new( + info: &RegexInfo, + nfarev: &NFA, + ) -> Option<ReverseHybridEngine> { + #[cfg(feature = "hybrid")] + { + if !info.config().get_hybrid() { + return None; + } + // Since we only use this for reverse searches, we can hard-code + // a number of things like match semantics, prefilters, starts + // for each pattern and so on. + let dfa_config = hybrid::dfa::Config::new() + .match_kind(MatchKind::All) + .prefilter(None) + .starts_for_each_pattern(false) + .byte_classes(info.config().get_byte_classes()) + .unicode_word_boundary(true) + .specialize_start_states(false) + .cache_capacity(info.config().get_hybrid_cache_capacity()) + .skip_cache_capacity_check(false) + .minimum_cache_clear_count(Some(3)) + .minimum_bytes_per_state(Some(10)); + let result = hybrid::dfa::Builder::new() + .configure(dfa_config) + .build_from_nfa(nfarev.clone()); + let rev = match result { + Ok(rev) => rev, + Err(_err) => { + debug!("lazy reverse DFA failed to build: {_err}"); + return None; + } + }; + debug!("lazy reverse DFA built"); + Some(ReverseHybridEngine(rev)) + } + #[cfg(not(feature = "hybrid"))] + { + None + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_rev_limited( + &self, + cache: &mut ReverseHybridCache, + input: &Input<'_>, + min_start: usize, + ) -> Result<Option<HalfMatch>, RetryError> { + #[cfg(feature = "hybrid")] + { + let dfa = &self.0; + let mut cache = cache.0.as_mut().unwrap(); + crate::meta::limited::hybrid_try_search_half_rev( + dfa, &mut cache, input, min_start, + ) + } + #[cfg(not(feature = "hybrid"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct ReverseHybridCache( + #[cfg(feature = "hybrid")] Option<hybrid::dfa::Cache>, + #[cfg(not(feature = "hybrid"))] (), +); + +impl ReverseHybridCache { + pub(crate) fn none() -> ReverseHybridCache { + #[cfg(feature = "hybrid")] + { + ReverseHybridCache(None) + } + #[cfg(not(feature = "hybrid"))] + { + ReverseHybridCache(()) + } + } + + pub(crate) fn new(builder: &ReverseHybrid) -> ReverseHybridCache { + #[cfg(feature = "hybrid")] + { + ReverseHybridCache(builder.0.as_ref().map(|e| e.0.create_cache())) + } + #[cfg(not(feature = "hybrid"))] + { + ReverseHybridCache(()) + } + } + + pub(crate) fn reset(&mut self, builder: &ReverseHybrid) { + #[cfg(feature = "hybrid")] + if let Some(ref e) = builder.0 { + self.0.as_mut().unwrap().reset(&e.0); + } + } + + pub(crate) fn memory_usage(&self) -> usize { + #[cfg(feature = "hybrid")] + { + self.0.as_ref().map_or(0, |c| c.memory_usage()) + } + #[cfg(not(feature = "hybrid"))] + { + 0 + } + } +} + +#[derive(Debug)] +pub(crate) struct ReverseDFA(Option<ReverseDFAEngine>); + +impl ReverseDFA { + pub(crate) fn none() -> ReverseDFA { + ReverseDFA(None) + } + + pub(crate) fn new(info: &RegexInfo, nfarev: &NFA) -> ReverseDFA { + ReverseDFA(ReverseDFAEngine::new(info, nfarev)) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn get(&self, _input: &Input<'_>) -> Option<&ReverseDFAEngine> { + let engine = self.0.as_ref()?; + Some(engine) + } + + pub(crate) fn is_some(&self) -> bool { + self.0.is_some() + } + + pub(crate) fn memory_usage(&self) -> usize { + self.0.as_ref().map_or(0, |e| e.memory_usage()) + } +} + +#[derive(Debug)] +pub(crate) struct ReverseDFAEngine( + #[cfg(feature = "dfa-build")] dfa::dense::DFA<Vec<u32>>, + #[cfg(not(feature = "dfa-build"))] (), +); + +impl ReverseDFAEngine { + pub(crate) fn new( + info: &RegexInfo, + nfarev: &NFA, + ) -> Option<ReverseDFAEngine> { + #[cfg(feature = "dfa-build")] + { + if !info.config().get_dfa() { + return None; + } + // If our NFA is anything but small, don't even bother with a DFA. + if let Some(state_limit) = info.config().get_dfa_state_limit() { + if nfarev.states().len() > state_limit { + debug!( + "skipping full reverse DFA because NFA has {} states, \ + which exceeds the heuristic limit of {}", + nfarev.states().len(), + state_limit, + ); + return None; + } + } + // We cut the size limit in two because the total heap used by DFA + // construction is determinization aux memory and the DFA itself, + // and those things are configured independently in the lower level + // DFA builder API. + let size_limit = info.config().get_dfa_size_limit().map(|n| n / 2); + // Since we only use this for reverse searches, we can hard-code + // a number of things like match semantics, prefilters, starts + // for each pattern and so on. We also disable acceleration since + // it's incompatible with limited searches (which is the only + // operation we support for this kind of engine at the moment). + let dfa_config = dfa::dense::Config::new() + .match_kind(MatchKind::All) + .prefilter(None) + .accelerate(false) + .start_kind(dfa::StartKind::Anchored) + .starts_for_each_pattern(false) + .byte_classes(info.config().get_byte_classes()) + .unicode_word_boundary(true) + .specialize_start_states(false) + .determinize_size_limit(size_limit) + .dfa_size_limit(size_limit); + let result = dfa::dense::Builder::new() + .configure(dfa_config) + .build_from_nfa(&nfarev); + let rev = match result { + Ok(rev) => rev, + Err(_err) => { + debug!("full reverse DFA failed to build: {_err}"); + return None; + } + }; + debug!( + "fully compiled reverse DFA built, {} bytes", + rev.memory_usage() + ); + Some(ReverseDFAEngine(rev)) + } + #[cfg(not(feature = "dfa-build"))] + { + None + } + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn try_search_half_rev_limited( + &self, + input: &Input<'_>, + min_start: usize, + ) -> Result<Option<HalfMatch>, RetryError> { + #[cfg(feature = "dfa-build")] + { + let dfa = &self.0; + crate::meta::limited::dfa_try_search_half_rev( + dfa, input, min_start, + ) + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } + + pub(crate) fn memory_usage(&self) -> usize { + #[cfg(feature = "dfa-build")] + { + self.0.memory_usage() + } + #[cfg(not(feature = "dfa-build"))] + { + // Impossible to reach because this engine is never constructed + // if the requisite features aren't enabled. + unreachable!() + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/mod.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/mod.rs new file mode 100644 index 0000000..14a0c30 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/mod.rs @@ -0,0 +1,55 @@ +/*! +Provides non-deterministic finite automata (NFA) and regex engines that use +them. + +While NFAs and DFAs (deterministic finite automata) have equivalent *theoretical* +power, their usage in practice tends to result in different engineering trade +offs. While this isn't meant to be a comprehensive treatment of the topic, here +are a few key trade offs that are, at minimum, true for this crate: + +* NFAs tend to be represented sparsely where as DFAs are represented densely. +Sparse representations use less memory, but are slower to traverse. Conversely, +dense representations use more memory, but are faster to traverse. (Sometimes +these lines are blurred. For example, an `NFA` might choose to represent a +particular state in a dense fashion, and a DFA can be built using a sparse +representation via [`sparse::DFA`](crate::dfa::sparse::DFA). +* NFAs have epsilon transitions and DFAs don't. In practice, this means that +handling a single byte in a haystack with an NFA at search time may require +visiting multiple NFA states. In a DFA, each byte only requires visiting +a single state. Stated differently, NFAs require a variable number of CPU +instructions to process one byte in a haystack where as a DFA uses a constant +number of CPU instructions to process one byte. +* NFAs are generally easier to amend with secondary storage. For example, the +[`thompson::pikevm::PikeVM`] uses an NFA to match, but also uses additional +memory beyond the model of a finite state machine to track offsets for matching +capturing groups. Conversely, the most a DFA can do is report the offset (and +pattern ID) at which a match occurred. This is generally why we also compile +DFAs in reverse, so that we can run them after finding the end of a match to +also find the start of a match. +* NFAs take worst case linear time to build, but DFAs take worst case +exponential time to build. The [hybrid NFA/DFA](crate::hybrid) mitigates this +challenge for DFAs in many practical cases. + +There are likely other differences, but the bottom line is that NFAs tend to be +more memory efficient and give easier opportunities for increasing expressive +power, where as DFAs are faster to search with. + +# Why only a Thompson NFA? + +Currently, the only kind of NFA we support in this crate is a [Thompson +NFA](https://en.wikipedia.org/wiki/Thompson%27s_construction). This refers +to a specific construction algorithm that takes the syntax of a regex +pattern and converts it to an NFA. Specifically, it makes gratuitous use of +epsilon transitions in order to keep its structure simple. In exchange, its +construction time is linear in the size of the regex. A Thompson NFA also makes +the guarantee that given any state and a character in a haystack, there is at +most one transition defined for it. (Although there may be many epsilon +transitions.) + +It's possible that other types of NFAs will be added in the future, such as a +[Glushkov NFA](https://en.wikipedia.org/wiki/Glushkov%27s_construction_algorithm). +But currently, this crate only provides a Thompson NFA. +*/ + +#[cfg(feature = "nfa-thompson")] +pub mod thompson; diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/backtrack.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/backtrack.rs new file mode 100644 index 0000000..df99e45 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/backtrack.rs @@ -0,0 +1,1908 @@ +/*! +An NFA backed bounded backtracker for executing regex searches with capturing +groups. + +This module provides a [`BoundedBacktracker`] that works by simulating an NFA +using the classical backtracking algorithm with a twist: it avoids redoing +work that it has done before and thereby avoids worst case exponential time. +In exchange, it can only be used on "short" haystacks. Its advantage is that +is can be faster than the [`PikeVM`](thompson::pikevm::PikeVM) in many cases +because it does less book-keeping. +*/ + +use alloc::{vec, vec::Vec}; + +use crate::{ + nfa::thompson::{self, BuildError, State, NFA}, + util::{ + captures::Captures, + empty, iter, + prefilter::Prefilter, + primitives::{NonMaxUsize, PatternID, SmallIndex, StateID}, + search::{Anchored, HalfMatch, Input, Match, MatchError, Span}, + }, +}; + +/// Returns the minimum visited capacity for the given haystack. +/// +/// This function can be used as the argument to [`Config::visited_capacity`] +/// in order to guarantee that a backtracking search for the given `input` +/// won't return an error when using a [`BoundedBacktracker`] built from the +/// given `NFA`. +/// +/// This routine exists primarily as a way to test that the bounded backtracker +/// works correctly when its capacity is set to the smallest possible amount. +/// Still, it may be useful in cases where you know you want to use the bounded +/// backtracker for a specific input, and just need to know what visited +/// capacity to provide to make it work. +/// +/// Be warned that this number could be quite large as it is multiplicative in +/// the size the given NFA and haystack. +pub fn min_visited_capacity(nfa: &NFA, input: &Input<'_>) -> usize { + div_ceil(nfa.states().len() * (input.get_span().len() + 1), 8) +} + +/// The configuration used for building a bounded backtracker. +/// +/// A bounded backtracker configuration is a simple data object that is +/// typically used with [`Builder::configure`]. +#[derive(Clone, Debug, Default)] +pub struct Config { + pre: Option<Option<Prefilter>>, + visited_capacity: Option<usize>, +} + +impl Config { + /// Return a new default regex configuration. + pub fn new() -> Config { + Config::default() + } + + /// Set a prefilter to be used whenever a start state is entered. + /// + /// A [`Prefilter`] in this context is meant to accelerate searches by + /// looking for literal prefixes that every match for the corresponding + /// pattern (or patterns) must start with. Once a prefilter produces a + /// match, the underlying search routine continues on to try and confirm + /// the match. + /// + /// Be warned that setting a prefilter does not guarantee that the search + /// will be faster. While it's usually a good bet, if the prefilter + /// produces a lot of false positive candidates (i.e., positions matched + /// by the prefilter but not by the regex), then the overall result can + /// be slower than if you had just executed the regex engine without any + /// prefilters. + /// + /// By default no prefilter is set. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// util::prefilter::Prefilter, + /// Input, Match, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["foo", "bar"]); + /// let re = BoundedBacktracker::builder() + /// .configure(BoundedBacktracker::config().prefilter(pre)) + /// .build(r"(foo|bar)[a-z]+")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("foo1 barfox bar"); + /// assert_eq!( + /// Some(Match::must(0, 5..11)), + /// re.try_find(&mut cache, input)?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Be warned though that an incorrect prefilter can lead to incorrect + /// results! + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// util::prefilter::Prefilter, + /// Input, HalfMatch, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["foo", "car"]); + /// let re = BoundedBacktracker::builder() + /// .configure(BoundedBacktracker::config().prefilter(pre)) + /// .build(r"(foo|bar)[a-z]+")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("foo1 barfox bar"); + /// // No match reported even though there clearly is one! + /// assert_eq!(None, re.try_find(&mut cache, input)?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn prefilter(mut self, pre: Option<Prefilter>) -> Config { + self.pre = Some(pre); + self + } + + /// Set the visited capacity used to bound backtracking. + /// + /// The visited capacity represents the amount of heap memory (in bytes) to + /// allocate toward tracking which parts of the backtracking search have + /// been done before. The heap memory needed for any particular search is + /// proportional to `haystack.len() * nfa.states().len()`, which an be + /// quite large. Therefore, the bounded backtracker is typically only able + /// to run on shorter haystacks. + /// + /// For a given regex, increasing the visited capacity means that the + /// maximum haystack length that can be searched is increased. The + /// [`BoundedBacktracker::max_haystack_len`] method returns that maximum. + /// + /// The default capacity is a reasonable but empirically chosen size. + /// + /// # Example + /// + /// As with other regex engines, Unicode is what tends to make the bounded + /// backtracker less useful by making the maximum haystack length quite + /// small. If necessary, increasing the visited capacity using this routine + /// will increase the maximum haystack length at the cost of using more + /// memory. + /// + /// Note though that the specific maximum values here are not an API + /// guarantee. The default visited capacity is subject to change and not + /// covered by semver. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::nfa::thompson::backtrack::BoundedBacktracker; + /// + /// // Unicode inflates the size of the underlying NFA quite a bit, and + /// // thus means that the backtracker can only handle smaller haystacks, + /// // assuming that the visited capacity remains unchanged. + /// let re = BoundedBacktracker::new(r"\w+")?; + /// assert!(re.max_haystack_len() <= 7_000); + /// // But we can increase the visited capacity to handle bigger haystacks! + /// let re = BoundedBacktracker::builder() + /// .configure(BoundedBacktracker::config().visited_capacity(1<<20)) + /// .build(r"\w+")?; + /// assert!(re.max_haystack_len() >= 25_000); + /// assert!(re.max_haystack_len() <= 28_000); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn visited_capacity(mut self, capacity: usize) -> Config { + self.visited_capacity = Some(capacity); + self + } + + /// Returns the prefilter set in this configuration, if one at all. + pub fn get_prefilter(&self) -> Option<&Prefilter> { + self.pre.as_ref().unwrap_or(&None).as_ref() + } + + /// Returns the configured visited capacity. + /// + /// Note that the actual capacity used may be slightly bigger than the + /// configured capacity. + pub fn get_visited_capacity(&self) -> usize { + const DEFAULT: usize = 256 * (1 << 10); // 256 KB + self.visited_capacity.unwrap_or(DEFAULT) + } + + /// Overwrite the default configuration such that the options in `o` are + /// always used. If an option in `o` is not set, then the corresponding + /// option in `self` is used. If it's not set in `self` either, then it + /// remains not set. + pub(crate) fn overwrite(&self, o: Config) -> Config { + Config { + pre: o.pre.or_else(|| self.pre.clone()), + visited_capacity: o.visited_capacity.or(self.visited_capacity), + } + } +} + +/// A builder for a bounded backtracker. +/// +/// This builder permits configuring options for the syntax of a pattern, the +/// NFA construction and the `BoundedBacktracker` construction. This builder +/// is different from a general purpose regex builder in that it permits fine +/// grain configuration of the construction process. The trade off for this is +/// complexity, and the possibility of setting a configuration that might not +/// make sense. For example, there are two different UTF-8 modes: +/// +/// * [`syntax::Config::utf8`](crate::util::syntax::Config::utf8) controls +/// whether the pattern itself can contain sub-expressions that match invalid +/// UTF-8. +/// * [`thompson::Config::utf8`] controls how the regex iterators themselves +/// advance the starting position of the next search when a match with zero +/// length is found. +/// +/// Generally speaking, callers will want to either enable all of these or +/// disable all of these. +/// +/// # Example +/// +/// This example shows how to disable UTF-8 mode in the syntax and the regex +/// itself. This is generally what you want for matching on arbitrary bytes. +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::{self, backtrack::BoundedBacktracker}, +/// util::syntax, +/// Match, +/// }; +/// +/// let re = BoundedBacktracker::builder() +/// .syntax(syntax::Config::new().utf8(false)) +/// .thompson(thompson::Config::new().utf8(false)) +/// .build(r"foo(?-u:[^b])ar.*")?; +/// let mut cache = re.create_cache(); +/// +/// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"; +/// let expected = Some(Ok(Match::must(0, 1..9))); +/// let got = re.try_find_iter(&mut cache, haystack).next(); +/// assert_eq!(expected, got); +/// // Notice that `(?-u:[^b])` matches invalid UTF-8, +/// // but the subsequent `.*` does not! Disabling UTF-8 +/// // on the syntax permits this. +/// // +/// // N.B. This example does not show the impact of +/// // disabling UTF-8 mode on a BoundedBacktracker Config, since that +/// // only impacts regexes that can produce matches of +/// // length 0. +/// assert_eq!(b"foo\xFFarzz", &haystack[got.unwrap()?.range()]); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Builder { + config: Config, + #[cfg(feature = "syntax")] + thompson: thompson::Compiler, +} + +impl Builder { + /// Create a new BoundedBacktracker builder with its default configuration. + pub fn new() -> Builder { + Builder { + config: Config::default(), + #[cfg(feature = "syntax")] + thompson: thompson::Compiler::new(), + } + } + + /// Build a `BoundedBacktracker` from the given pattern. + /// + /// If there was a problem parsing or compiling the pattern, then an error + /// is returned. + #[cfg(feature = "syntax")] + pub fn build( + &self, + pattern: &str, + ) -> Result<BoundedBacktracker, BuildError> { + self.build_many(&[pattern]) + } + + /// Build a `BoundedBacktracker` from the given patterns. + #[cfg(feature = "syntax")] + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<BoundedBacktracker, BuildError> { + let nfa = self.thompson.build_many(patterns)?; + self.build_from_nfa(nfa) + } + + /// Build a `BoundedBacktracker` directly from its NFA. + /// + /// Note that when using this method, any configuration that applies to the + /// construction of the NFA itself will of course be ignored, since the NFA + /// given here is already built. + pub fn build_from_nfa( + &self, + nfa: NFA, + ) -> Result<BoundedBacktracker, BuildError> { + nfa.look_set_any().available().map_err(BuildError::word)?; + Ok(BoundedBacktracker { config: self.config.clone(), nfa }) + } + + /// Apply the given `BoundedBacktracker` configuration options to this + /// builder. + pub fn configure(&mut self, config: Config) -> &mut Builder { + self.config = self.config.overwrite(config); + self + } + + /// Set the syntax configuration for this builder using + /// [`syntax::Config`](crate::util::syntax::Config). + /// + /// This permits setting things like case insensitivity, Unicode and multi + /// line mode. + /// + /// These settings only apply when constructing a `BoundedBacktracker` + /// directly from a pattern. + #[cfg(feature = "syntax")] + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Builder { + self.thompson.syntax(config); + self + } + + /// Set the Thompson NFA configuration for this builder using + /// [`nfa::thompson::Config`](crate::nfa::thompson::Config). + /// + /// This permits setting things like if additional time should be spent + /// shrinking the size of the NFA. + /// + /// These settings only apply when constructing a `BoundedBacktracker` + /// directly from a pattern. + #[cfg(feature = "syntax")] + pub fn thompson(&mut self, config: thompson::Config) -> &mut Builder { + self.thompson.configure(config); + self + } +} + +/// A backtracking regex engine that bounds its execution to avoid exponential +/// blow-up. +/// +/// This regex engine only implements leftmost-first match semantics and +/// only supports leftmost searches. It effectively does the same thing as a +/// [`PikeVM`](thompson::pikevm::PikeVM), but typically does it faster because +/// it doesn't have to worry about copying capturing group spans for most NFA +/// states. Instead, the backtracker can maintain one set of captures (provided +/// by the caller) and never needs to copy them. In exchange, the backtracker +/// bounds itself to ensure it doesn't exhibit worst case exponential time. +/// This results in the backtracker only being able to handle short haystacks +/// given reasonable memory usage. +/// +/// # Searches may return an error! +/// +/// By design, this backtracking regex engine is bounded. This bound is +/// implemented by not visiting any combination of NFA state ID and position +/// in a haystack more than once. Thus, the total memory required to bound +/// backtracking is proportional to `haystack.len() * nfa.states().len()`. +/// This can obviously get quite large, since large haystacks aren't terribly +/// uncommon. To avoid using exorbitant memory, the capacity is bounded by +/// a fixed limit set via [`Config::visited_capacity`]. Thus, if the total +/// capacity required for a particular regex and a haystack exceeds this +/// capacity, then the search routine will return an error. +/// +/// Unlike other regex engines that may return an error at search time (like +/// the DFA or the hybrid NFA/DFA), there is no way to guarantee that a bounded +/// backtracker will work for every haystack. Therefore, this regex engine +/// _only_ exposes fallible search routines to avoid the footgun of panicking +/// when running a search on a haystack that is too big. +/// +/// If one wants to use the fallible search APIs without handling the +/// error, the only way to guarantee an error won't occur from the +/// haystack length is to ensure the haystack length does not exceed +/// [`BoundedBacktracker::max_haystack_len`]. +/// +/// # Example: Unicode word boundaries +/// +/// This example shows that the bounded backtracker implements Unicode word +/// boundaries correctly by default. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{nfa::thompson::backtrack::BoundedBacktracker, Match}; +/// +/// let re = BoundedBacktracker::new(r"\b\w+\b")?; +/// let mut cache = re.create_cache(); +/// +/// let mut it = re.try_find_iter(&mut cache, "Шерлок Холмс"); +/// assert_eq!(Some(Ok(Match::must(0, 0..12))), it.next()); +/// assert_eq!(Some(Ok(Match::must(0, 13..23))), it.next()); +/// assert_eq!(None, it.next()); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: multiple regex patterns +/// +/// The bounded backtracker supports searching for multiple patterns +/// simultaneously, just like other regex engines. Note though that because it +/// uses a backtracking strategy, this regex engine is unlikely to scale well +/// as more patterns are added. But then again, as more patterns are added, the +/// maximum haystack length allowed will also shorten (assuming the visited +/// capacity remains invariant). +/// +/// ``` +/// use regex_automata::{nfa::thompson::backtrack::BoundedBacktracker, Match}; +/// +/// let re = BoundedBacktracker::new_many(&["[a-z]+", "[0-9]+"])?; +/// let mut cache = re.create_cache(); +/// +/// let mut it = re.try_find_iter(&mut cache, "abc 1 foo 4567 0 quux"); +/// assert_eq!(Some(Ok(Match::must(0, 0..3))), it.next()); +/// assert_eq!(Some(Ok(Match::must(1, 4..5))), it.next()); +/// assert_eq!(Some(Ok(Match::must(0, 6..9))), it.next()); +/// assert_eq!(Some(Ok(Match::must(1, 10..14))), it.next()); +/// assert_eq!(Some(Ok(Match::must(1, 15..16))), it.next()); +/// assert_eq!(Some(Ok(Match::must(0, 17..21))), it.next()); +/// assert_eq!(None, it.next()); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct BoundedBacktracker { + config: Config, + nfa: NFA, +} + +impl BoundedBacktracker { + /// Parse the given regular expression using the default configuration and + /// return the corresponding `BoundedBacktracker`. + /// + /// If you want a non-default configuration, then use the [`Builder`] to + /// set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, + /// }; + /// + /// let re = BoundedBacktracker::new("foo[0-9]+bar")?; + /// let mut cache = re.create_cache(); + /// assert_eq!( + /// Some(Ok(Match::must(0, 3..14))), + /// re.try_find_iter(&mut cache, "zzzfoo12345barzzz").next(), + /// ); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new(pattern: &str) -> Result<BoundedBacktracker, BuildError> { + BoundedBacktracker::builder().build(pattern) + } + + /// Like `new`, but parses multiple patterns into a single "multi regex." + /// This similarly uses the default regex configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, + /// }; + /// + /// let re = BoundedBacktracker::new_many(&["[a-z]+", "[0-9]+"])?; + /// let mut cache = re.create_cache(); + /// + /// let mut it = re.try_find_iter(&mut cache, "abc 1 foo 4567 0 quux"); + /// assert_eq!(Some(Ok(Match::must(0, 0..3))), it.next()); + /// assert_eq!(Some(Ok(Match::must(1, 4..5))), it.next()); + /// assert_eq!(Some(Ok(Match::must(0, 6..9))), it.next()); + /// assert_eq!(Some(Ok(Match::must(1, 10..14))), it.next()); + /// assert_eq!(Some(Ok(Match::must(1, 15..16))), it.next()); + /// assert_eq!(Some(Ok(Match::must(0, 17..21))), it.next()); + /// assert_eq!(None, it.next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new_many<P: AsRef<str>>( + patterns: &[P], + ) -> Result<BoundedBacktracker, BuildError> { + BoundedBacktracker::builder().build_many(patterns) + } + + /// # Example + /// + /// This shows how to hand assemble a regular expression via its HIR, + /// compile an NFA from it and build a BoundedBacktracker from the NFA. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{NFA, backtrack::BoundedBacktracker}, + /// Match, + /// }; + /// use regex_syntax::hir::{Hir, Class, ClassBytes, ClassBytesRange}; + /// + /// let hir = Hir::class(Class::Bytes(ClassBytes::new(vec![ + /// ClassBytesRange::new(b'0', b'9'), + /// ClassBytesRange::new(b'A', b'Z'), + /// ClassBytesRange::new(b'_', b'_'), + /// ClassBytesRange::new(b'a', b'z'), + /// ]))); + /// + /// let config = NFA::config().nfa_size_limit(Some(1_000)); + /// let nfa = NFA::compiler().configure(config).build_from_hir(&hir)?; + /// + /// let re = BoundedBacktracker::new_from_nfa(nfa)?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let expected = Some(Match::must(0, 3..4)); + /// re.try_captures(&mut cache, "!@#A#@!", &mut caps)?; + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new_from_nfa(nfa: NFA) -> Result<BoundedBacktracker, BuildError> { + BoundedBacktracker::builder().build_from_nfa(nfa) + } + + /// Create a new `BoundedBacktracker` that matches every input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, + /// }; + /// + /// let re = BoundedBacktracker::always_match()?; + /// let mut cache = re.create_cache(); + /// + /// let expected = Some(Ok(Match::must(0, 0..0))); + /// assert_eq!(expected, re.try_find_iter(&mut cache, "").next()); + /// assert_eq!(expected, re.try_find_iter(&mut cache, "foo").next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn always_match() -> Result<BoundedBacktracker, BuildError> { + let nfa = thompson::NFA::always_match(); + BoundedBacktracker::new_from_nfa(nfa) + } + + /// Create a new `BoundedBacktracker` that never matches any input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::backtrack::BoundedBacktracker; + /// + /// let re = BoundedBacktracker::never_match()?; + /// let mut cache = re.create_cache(); + /// + /// assert_eq!(None, re.try_find_iter(&mut cache, "").next()); + /// assert_eq!(None, re.try_find_iter(&mut cache, "foo").next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn never_match() -> Result<BoundedBacktracker, BuildError> { + let nfa = thompson::NFA::never_match(); + BoundedBacktracker::new_from_nfa(nfa) + } + + /// Return a default configuration for a `BoundedBacktracker`. + /// + /// This is a convenience routine to avoid needing to import the `Config` + /// type when customizing the construction of a `BoundedBacktracker`. + /// + /// # Example + /// + /// This example shows how to disable UTF-8 mode. When UTF-8 mode is + /// disabled, zero-width matches that split a codepoint are allowed. + /// Otherwise they are never reported. + /// + /// In the code below, notice that `""` is permitted to match positions + /// that split the encoding of a codepoint. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{self, backtrack::BoundedBacktracker}, + /// Match, + /// }; + /// + /// let re = BoundedBacktracker::builder() + /// .thompson(thompson::Config::new().utf8(false)) + /// .build(r"")?; + /// let mut cache = re.create_cache(); + /// + /// let haystack = "a☃z"; + /// let mut it = re.try_find_iter(&mut cache, haystack); + /// assert_eq!(Some(Ok(Match::must(0, 0..0))), it.next()); + /// assert_eq!(Some(Ok(Match::must(0, 1..1))), it.next()); + /// assert_eq!(Some(Ok(Match::must(0, 2..2))), it.next()); + /// assert_eq!(Some(Ok(Match::must(0, 3..3))), it.next()); + /// assert_eq!(Some(Ok(Match::must(0, 4..4))), it.next()); + /// assert_eq!(Some(Ok(Match::must(0, 5..5))), it.next()); + /// assert_eq!(None, it.next()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn config() -> Config { + Config::new() + } + + /// Return a builder for configuring the construction of a + /// `BoundedBacktracker`. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Builder`] type in common cases. + /// + /// # Example + /// + /// This example shows how to use the builder to disable UTF-8 mode + /// everywhere. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// nfa::thompson::{self, backtrack::BoundedBacktracker}, + /// util::syntax, + /// Match, + /// }; + /// + /// let re = BoundedBacktracker::builder() + /// .syntax(syntax::Config::new().utf8(false)) + /// .thompson(thompson::Config::new().utf8(false)) + /// .build(r"foo(?-u:[^b])ar.*")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"; + /// let expected = Some(Match::must(0, 1..9)); + /// re.try_captures(&mut cache, haystack, &mut caps)?; + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn builder() -> Builder { + Builder::new() + } + + /// Create a new cache for this regex. + /// + /// The cache returned should only be used for searches for this + /// regex. If you want to reuse the cache for another regex, then you + /// must call [`Cache::reset`] with that regex (or, equivalently, + /// [`BoundedBacktracker::reset_cache`]). + pub fn create_cache(&self) -> Cache { + Cache::new(self) + } + + /// Create a new empty set of capturing groups that is guaranteed to be + /// valid for the search APIs on this `BoundedBacktracker`. + /// + /// A `Captures` value created for a specific `BoundedBacktracker` cannot + /// be used with any other `BoundedBacktracker`. + /// + /// This is a convenience function for [`Captures::all`]. See the + /// [`Captures`] documentation for an explanation of its alternative + /// constructors that permit the `BoundedBacktracker` to do less work + /// during a search, and thus might make it faster. + pub fn create_captures(&self) -> Captures { + Captures::all(self.get_nfa().group_info().clone()) + } + + /// Reset the given cache such that it can be used for searching with the + /// this `BoundedBacktracker` (and only this `BoundedBacktracker`). + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different `BoundedBacktracker`. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different + /// `BoundedBacktracker`. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, + /// }; + /// + /// let re1 = BoundedBacktracker::new(r"\w")?; + /// let re2 = BoundedBacktracker::new(r"\W")?; + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Ok(Match::must(0, 0..2))), + /// re1.try_find_iter(&mut cache, "Δ").next(), + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the BoundedBacktracker we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// cache.reset(&re2); + /// assert_eq!( + /// Some(Ok(Match::must(0, 0..3))), + /// re2.try_find_iter(&mut cache, "☃").next(), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset_cache(&self, cache: &mut Cache) { + cache.reset(self); + } + + /// Returns the total number of patterns compiled into this + /// `BoundedBacktracker`. + /// + /// In the case of a `BoundedBacktracker` that contains no patterns, this + /// returns `0`. + /// + /// # Example + /// + /// This example shows the pattern length for a `BoundedBacktracker` that + /// never matches: + /// + /// ``` + /// use regex_automata::nfa::thompson::backtrack::BoundedBacktracker; + /// + /// let re = BoundedBacktracker::never_match()?; + /// assert_eq!(re.pattern_len(), 0); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And another example for a `BoundedBacktracker` that matches at every + /// position: + /// + /// ``` + /// use regex_automata::nfa::thompson::backtrack::BoundedBacktracker; + /// + /// let re = BoundedBacktracker::always_match()?; + /// assert_eq!(re.pattern_len(), 1); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And finally, a `BoundedBacktracker` that was constructed from multiple + /// patterns: + /// + /// ``` + /// use regex_automata::nfa::thompson::backtrack::BoundedBacktracker; + /// + /// let re = BoundedBacktracker::new_many(&["[0-9]+", "[a-z]+", "[A-Z]+"])?; + /// assert_eq!(re.pattern_len(), 3); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn pattern_len(&self) -> usize { + self.nfa.pattern_len() + } + + /// Return the config for this `BoundedBacktracker`. + #[inline] + pub fn get_config(&self) -> &Config { + &self.config + } + + /// Returns a reference to the underlying NFA. + #[inline] + pub fn get_nfa(&self) -> &NFA { + &self.nfa + } + + /// Returns the maximum haystack length supported by this backtracker. + /// + /// This routine is a function of both [`Config::visited_capacity`] and the + /// internal size of the backtracker's NFA. + /// + /// # Example + /// + /// This example shows how the maximum haystack length can vary depending + /// on the size of the regex itself. Note though that the specific maximum + /// values here are not an API guarantee. The default visited capacity is + /// subject to change and not covered by semver. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, MatchError, + /// }; + /// + /// // If you're only using ASCII, you get a big budget. + /// let re = BoundedBacktracker::new(r"(?-u)\w+")?; + /// let mut cache = re.create_cache(); + /// assert_eq!(re.max_haystack_len(), 299_592); + /// // Things work up to the max. + /// let mut haystack = "a".repeat(299_592); + /// let expected = Some(Ok(Match::must(0, 0..299_592))); + /// assert_eq!(expected, re.try_find_iter(&mut cache, &haystack).next()); + /// // But you'll get an error if you provide a haystack that's too big. + /// // Notice that we use the 'try_find_iter' routine instead, which + /// // yields Result<Match, MatchError> instead of Match. + /// haystack.push('a'); + /// let expected = Some(Err(MatchError::haystack_too_long(299_593))); + /// assert_eq!(expected, re.try_find_iter(&mut cache, &haystack).next()); + /// + /// // Unicode inflates the size of the underlying NFA quite a bit, and + /// // thus means that the backtracker can only handle smaller haystacks, + /// // assuming that the visited capacity remains unchanged. + /// let re = BoundedBacktracker::new(r"\w+")?; + /// assert!(re.max_haystack_len() <= 7_000); + /// // But we can increase the visited capacity to handle bigger haystacks! + /// let re = BoundedBacktracker::builder() + /// .configure(BoundedBacktracker::config().visited_capacity(1<<20)) + /// .build(r"\w+")?; + /// assert!(re.max_haystack_len() >= 25_000); + /// assert!(re.max_haystack_len() <= 28_000); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn max_haystack_len(&self) -> usize { + // The capacity given in the config is "bytes of heap memory," but the + // capacity we use here is "number of bits." So convert the capacity in + // bytes to the capacity in bits. + let capacity = 8 * self.get_config().get_visited_capacity(); + let blocks = div_ceil(capacity, Visited::BLOCK_SIZE); + let real_capacity = blocks.saturating_mul(Visited::BLOCK_SIZE); + // It's possible for `real_capacity` to be smaller than the number of + // NFA states for particularly large regexes, so we saturate towards + // zero. + (real_capacity / self.nfa.states().len()).saturating_sub(1) + } +} + +impl BoundedBacktracker { + /// Returns true if and only if this regex matches the given haystack. + /// + /// In the case of a backtracking regex engine, and unlike most other + /// regex engines in this crate, short circuiting isn't practical. However, + /// this routine may still be faster because it instructs backtracking to + /// not keep track of any capturing groups. + /// + /// # Errors + /// + /// This routine only errors if the search could not complete. For this + /// backtracking regex engine, this only occurs when the haystack length + /// exceeds [`BoundedBacktracker::max_haystack_len`]. + /// + /// When a search cannot complete, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::backtrack::BoundedBacktracker; + /// + /// let re = BoundedBacktracker::new("foo[0-9]+bar")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.try_is_match(&mut cache, "foo12345bar")?); + /// assert!(!re.try_is_match(&mut cache, "foobar")?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: consistency with search APIs + /// + /// `is_match` is guaranteed to return `true` whenever `find` returns a + /// match. This includes searches that are executed entirely within a + /// codepoint: + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Input, + /// }; + /// + /// let re = BoundedBacktracker::new("a*")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(!re.try_is_match(&mut cache, Input::new("☃").span(1..2))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Notice that when UTF-8 mode is disabled, then the above reports a + /// match because the restriction against zero-width matches that split a + /// codepoint has been lifted: + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{backtrack::BoundedBacktracker, NFA}, + /// Input, + /// }; + /// + /// let re = BoundedBacktracker::builder() + /// .thompson(NFA::config().utf8(false)) + /// .build("a*")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.try_is_match(&mut cache, Input::new("☃").span(1..2))?); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_is_match<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + ) -> Result<bool, MatchError> { + let input = input.into().earliest(true); + self.try_search_slots(cache, &input, &mut []).map(|pid| pid.is_some()) + } + + /// Executes a leftmost forward search and returns a `Match` if one exists. + /// + /// This routine only includes the overall match span. To get + /// access to the individual spans of each capturing group, use + /// [`BoundedBacktracker::try_captures`]. + /// + /// # Errors + /// + /// This routine only errors if the search could not complete. For this + /// backtracking regex engine, this only occurs when the haystack length + /// exceeds [`BoundedBacktracker::max_haystack_len`]. + /// + /// When a search cannot complete, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, + /// }; + /// + /// let re = BoundedBacktracker::new("foo[0-9]+")?; + /// let mut cache = re.create_cache(); + /// let expected = Match::must(0, 0..8); + /// assert_eq!(Some(expected), re.try_find(&mut cache, "foo12345")?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_find<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + ) -> Result<Option<Match>, MatchError> { + let input = input.into(); + if self.get_nfa().pattern_len() == 1 { + let mut slots = [None, None]; + let pid = match self.try_search_slots(cache, &input, &mut slots)? { + None => return Ok(None), + Some(pid) => pid, + }; + let start = match slots[0] { + None => return Ok(None), + Some(s) => s.get(), + }; + let end = match slots[1] { + None => return Ok(None), + Some(s) => s.get(), + }; + return Ok(Some(Match::new(pid, Span { start, end }))); + } + let ginfo = self.get_nfa().group_info(); + let slots_len = ginfo.implicit_slot_len(); + let mut slots = vec![None; slots_len]; + let pid = match self.try_search_slots(cache, &input, &mut slots)? { + None => return Ok(None), + Some(pid) => pid, + }; + let start = match slots[pid.as_usize() * 2] { + None => return Ok(None), + Some(s) => s.get(), + }; + let end = match slots[pid.as_usize() * 2 + 1] { + None => return Ok(None), + Some(s) => s.get(), + }; + Ok(Some(Match::new(pid, Span { start, end }))) + } + + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided [`Captures`] + /// value. If no match was found, then [`Captures::is_match`] is guaranteed + /// to return `false`. + /// + /// # Errors + /// + /// This routine only errors if the search could not complete. For this + /// backtracking regex engine, this only occurs when the haystack length + /// exceeds [`BoundedBacktracker::max_haystack_len`]. + /// + /// When a search cannot complete, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Span, + /// }; + /// + /// let re = BoundedBacktracker::new( + /// r"^([0-9]{4})-([0-9]{2})-([0-9]{2})$", + /// )?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.try_captures(&mut cache, "2010-03-14", &mut caps)?; + /// assert!(caps.is_match()); + /// assert_eq!(Some(Span::from(0..4)), caps.get_group(1)); + /// assert_eq!(Some(Span::from(5..7)), caps.get_group(2)); + /// assert_eq!(Some(Span::from(8..10)), caps.get_group(3)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_captures<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + caps: &mut Captures, + ) -> Result<(), MatchError> { + self.try_search(cache, &input.into(), caps) + } + + /// Returns an iterator over all non-overlapping leftmost matches in the + /// given bytes. If no match exists, then the iterator yields no elements. + /// + /// If the regex engine returns an error at any point, then the iterator + /// will yield that error. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, MatchError, + /// }; + /// + /// let re = BoundedBacktracker::new("foo[0-9]+")?; + /// let mut cache = re.create_cache(); + /// + /// let text = "foo1 foo12 foo123"; + /// let result: Result<Vec<Match>, MatchError> = re + /// .try_find_iter(&mut cache, text) + /// .collect(); + /// let matches = result?; + /// assert_eq!(matches, vec![ + /// Match::must(0, 0..4), + /// Match::must(0, 5..10), + /// Match::must(0, 11..17), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_find_iter<'r, 'c, 'h, I: Into<Input<'h>>>( + &'r self, + cache: &'c mut Cache, + input: I, + ) -> TryFindMatches<'r, 'c, 'h> { + let caps = Captures::matches(self.get_nfa().group_info().clone()); + let it = iter::Searcher::new(input.into()); + TryFindMatches { re: self, cache, caps, it } + } + + /// Returns an iterator over all non-overlapping `Captures` values. If no + /// match exists, then the iterator yields no elements. + /// + /// This yields the same matches as [`BoundedBacktracker::try_find_iter`], + /// but it includes the spans of all capturing groups that participate in + /// each match. + /// + /// If the regex engine returns an error at any point, then the iterator + /// will yield that error. + /// + /// **Tip:** See [`util::iter::Searcher`](crate::util::iter::Searcher) for + /// how to correctly iterate over all matches in a haystack while avoiding + /// the creation of a new `Captures` value for every match. (Which you are + /// forced to do with an `Iterator`.) + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Span, + /// }; + /// + /// let re = BoundedBacktracker::new("foo(?P<numbers>[0-9]+)")?; + /// let mut cache = re.create_cache(); + /// + /// let text = "foo1 foo12 foo123"; + /// let mut spans = vec![]; + /// for result in re.try_captures_iter(&mut cache, text) { + /// let caps = result?; + /// // The unwrap is OK since 'numbers' matches if the pattern matches. + /// spans.push(caps.get_group_by_name("numbers").unwrap()); + /// } + /// assert_eq!(spans, vec![ + /// Span::from(3..4), + /// Span::from(8..10), + /// Span::from(14..17), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_captures_iter<'r, 'c, 'h, I: Into<Input<'h>>>( + &'r self, + cache: &'c mut Cache, + input: I, + ) -> TryCapturesMatches<'r, 'c, 'h> { + let caps = self.create_captures(); + let it = iter::Searcher::new(input.into()); + TryCapturesMatches { re: self, cache, caps, it } + } +} + +impl BoundedBacktracker { + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided [`Captures`] + /// value. If no match was found, then [`Captures::is_match`] is guaranteed + /// to return `false`. + /// + /// This is like [`BoundedBacktracker::try_captures`], but it accepts a + /// concrete `&Input` instead of an `Into<Input>`. + /// + /// # Errors + /// + /// This routine only errors if the search could not complete. For this + /// backtracking regex engine, this only occurs when the haystack length + /// exceeds [`BoundedBacktracker::max_haystack_len`]. + /// + /// When a search cannot complete, callers cannot know whether a match + /// exists or not. + /// + /// # Example: specific pattern search + /// + /// This example shows how to build a multi bounded backtracker that + /// permits searching for specific patterns. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Anchored, Input, Match, PatternID, + /// }; + /// + /// let re = BoundedBacktracker::new_many(&[ + /// "[a-z0-9]{6}", + /// "[a-z][a-z0-9]{5}", + /// ])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "foo123"; + /// + /// // Since we are using the default leftmost-first match and both + /// // patterns match at the same starting position, only the first pattern + /// // will be returned in this case when doing a search for any of the + /// // patterns. + /// let expected = Some(Match::must(0, 0..6)); + /// re.try_search(&mut cache, &Input::new(haystack), &mut caps)?; + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we want to check whether some other pattern matches, then we + /// // can provide its pattern ID. + /// let expected = Some(Match::must(1, 0..6)); + /// let input = Input::new(haystack) + /// .anchored(Anchored::Pattern(PatternID::must(1))); + /// re.try_search(&mut cache, &input, &mut caps)?; + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specifying the bounds of a search + /// + /// This example shows how providing the bounds of a search can produce + /// different results than simply sub-slicing the haystack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, Input, + /// }; + /// + /// let re = BoundedBacktracker::new(r"\b[0-9]{3}\b")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "foo123bar"; + /// + /// // Since we sub-slice the haystack, the search doesn't know about + /// // the larger context and assumes that `123` is surrounded by word + /// // boundaries. And of course, the match position is reported relative + /// // to the sub-slice as well, which means we get `0..3` instead of + /// // `3..6`. + /// let expected = Some(Match::must(0, 0..3)); + /// re.try_search(&mut cache, &Input::new(&haystack[3..6]), &mut caps)?; + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we provide the bounds of the search within the context of the + /// // entire haystack, then the search can take the surrounding context + /// // into account. (And if we did find a match, it would be reported + /// // as a valid offset into `haystack` instead of its sub-slice.) + /// let expected = None; + /// re.try_search( + /// &mut cache, &Input::new(haystack).range(3..6), &mut caps, + /// )?; + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_search( + &self, + cache: &mut Cache, + input: &Input<'_>, + caps: &mut Captures, + ) -> Result<(), MatchError> { + caps.set_pattern(None); + let pid = self.try_search_slots(cache, input, caps.slots_mut())?; + caps.set_pattern(pid); + Ok(()) + } + + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided `slots`, and + /// returns the matching pattern ID. The contents of the slots for patterns + /// other than the matching pattern are unspecified. If no match was found, + /// then `None` is returned and the contents of all `slots` is unspecified. + /// + /// This is like [`BoundedBacktracker::try_search`], but it accepts a raw + /// slots slice instead of a `Captures` value. This is useful in contexts + /// where you don't want or need to allocate a `Captures`. + /// + /// It is legal to pass _any_ number of slots to this routine. If the regex + /// engine would otherwise write a slot offset that doesn't fit in the + /// provided slice, then it is simply skipped. In general though, there are + /// usually three slice lengths you might want to use: + /// + /// * An empty slice, if you only care about which pattern matched. + /// * A slice with + /// [`pattern_len() * 2`](crate::nfa::thompson::NFA::pattern_len) + /// slots, if you only care about the overall match spans for each matching + /// pattern. + /// * A slice with + /// [`slot_len()`](crate::util::captures::GroupInfo::slot_len) slots, which + /// permits recording match offsets for every capturing group in every + /// pattern. + /// + /// # Errors + /// + /// This routine only errors if the search could not complete. For this + /// backtracking regex engine, this only occurs when the haystack length + /// exceeds [`BoundedBacktracker::max_haystack_len`]. + /// + /// When a search cannot complete, callers cannot know whether a match + /// exists or not. + /// + /// # Example + /// + /// This example shows how to find the overall match offsets in a + /// multi-pattern search without allocating a `Captures` value. Indeed, we + /// can put our slots right on the stack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// PatternID, Input, + /// }; + /// + /// let re = BoundedBacktracker::new_many(&[ + /// r"\pL+", + /// r"\d+", + /// ])?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("!@#123"); + /// + /// // We only care about the overall match offsets here, so we just + /// // allocate two slots for each pattern. Each slot records the start + /// // and end of the match. + /// let mut slots = [None; 4]; + /// let pid = re.try_search_slots(&mut cache, &input, &mut slots)?; + /// assert_eq!(Some(PatternID::must(1)), pid); + /// + /// // The overall match offsets are always at 'pid * 2' and 'pid * 2 + 1'. + /// // See 'GroupInfo' for more details on the mapping between groups and + /// // slot indices. + /// let slot_start = pid.unwrap().as_usize() * 2; + /// let slot_end = slot_start + 1; + /// assert_eq!(Some(3), slots[slot_start].map(|s| s.get())); + /// assert_eq!(Some(6), slots[slot_end].map(|s| s.get())); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn try_search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Result<Option<PatternID>, MatchError> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + if !utf8empty { + let maybe_hm = self.try_search_slots_imp(cache, input, slots)?; + return Ok(maybe_hm.map(|hm| hm.pattern())); + } + // See PikeVM::try_search_slots for why we do this. + let min = self.get_nfa().group_info().implicit_slot_len(); + if slots.len() >= min { + let maybe_hm = self.try_search_slots_imp(cache, input, slots)?; + return Ok(maybe_hm.map(|hm| hm.pattern())); + } + if self.get_nfa().pattern_len() == 1 { + let mut enough = [None, None]; + let got = self.try_search_slots_imp(cache, input, &mut enough)?; + // This is OK because we know `enough_slots` is strictly bigger + // than `slots`, otherwise this special case isn't reached. + slots.copy_from_slice(&enough[..slots.len()]); + return Ok(got.map(|hm| hm.pattern())); + } + let mut enough = vec![None; min]; + let got = self.try_search_slots_imp(cache, input, &mut enough)?; + // This is OK because we know `enough_slots` is strictly bigger than + // `slots`, otherwise this special case isn't reached. + slots.copy_from_slice(&enough[..slots.len()]); + Ok(got.map(|hm| hm.pattern())) + } + + /// This is the actual implementation of `try_search_slots_imp` that + /// doesn't account for the special case when 1) the NFA has UTF-8 mode + /// enabled, 2) the NFA can match the empty string and 3) the caller has + /// provided an insufficient number of slots to record match offsets. + #[inline(never)] + fn try_search_slots_imp( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Result<Option<HalfMatch>, MatchError> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + let hm = match self.search_imp(cache, input, slots)? { + None => return Ok(None), + Some(hm) if !utf8empty => return Ok(Some(hm)), + Some(hm) => hm, + }; + empty::skip_splits_fwd(input, hm, hm.offset(), |input| { + Ok(self + .search_imp(cache, input, slots)? + .map(|hm| (hm, hm.offset()))) + }) + } + + /// The implementation of standard leftmost backtracking search. + /// + /// Capturing group spans are written to 'caps', but only if requested. + /// 'caps' can be one of three things: 1) totally empty, in which case, we + /// only report the pattern that matched or 2) only has slots for recording + /// the overall match offsets for any pattern or 3) has all slots available + /// for recording the spans of any groups participating in a match. + fn search_imp( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Result<Option<HalfMatch>, MatchError> { + // Unlike in the PikeVM, we write our capturing group spans directly + // into the caller's captures groups. So we have to make sure we're + // starting with a blank slate first. In the PikeVM, we avoid this + // by construction: the spans that are copied to every slot in the + // 'Captures' value already account for presence/absence. In this + // backtracker, we write directly into the caller provided slots, where + // as in the PikeVM, we write into scratch space first and only copy + // them to the caller provided slots when a match is found. + for slot in slots.iter_mut() { + *slot = None; + } + cache.setup_search(&self, input)?; + if input.is_done() { + return Ok(None); + } + let (anchored, start_id) = match input.get_anchored() { + // Only way we're unanchored is if both the caller asked for an + // unanchored search *and* the pattern is itself not anchored. + Anchored::No => ( + self.nfa.is_always_start_anchored(), + // We always use the anchored starting state here, even if + // doing an unanchored search. The "unanchored" part of it is + // implemented in the loop below, by simply trying the next + // byte offset if the previous backtracking exploration failed. + self.nfa.start_anchored(), + ), + Anchored::Yes => (true, self.nfa.start_anchored()), + Anchored::Pattern(pid) => match self.nfa.start_pattern(pid) { + None => return Ok(None), + Some(sid) => (true, sid), + }, + }; + if anchored { + let at = input.start(); + return Ok(self.backtrack(cache, input, at, start_id, slots)); + } + let pre = self.get_config().get_prefilter(); + let mut at = input.start(); + while at <= input.end() { + if let Some(ref pre) = pre { + let span = Span::from(at..input.end()); + match pre.find(input.haystack(), span) { + None => break, + Some(ref span) => at = span.start, + } + } + if let Some(hm) = self.backtrack(cache, input, at, start_id, slots) + { + return Ok(Some(hm)); + } + at += 1; + } + Ok(None) + } + + /// Look for a match starting at `at` in `input` and write the matching + /// pattern ID and group spans to `caps`. The search uses `start_id` as its + /// starting state in the underlying NFA. + /// + /// If no match was found, then the caller should increment `at` and try + /// at the next position. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn backtrack( + &self, + cache: &mut Cache, + input: &Input<'_>, + at: usize, + start_id: StateID, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<HalfMatch> { + cache.stack.push(Frame::Step { sid: start_id, at }); + while let Some(frame) = cache.stack.pop() { + match frame { + Frame::Step { sid, at } => { + if let Some(hm) = self.step(cache, input, sid, at, slots) { + return Some(hm); + } + } + Frame::RestoreCapture { slot, offset } => { + slots[slot] = offset; + } + } + } + None + } + + // LAMENTATION: The actual backtracking search is implemented in about + // 75 lines below. Yet this file is over 2,000 lines long. What have I + // done? + + /// Execute a "step" in the backtracing algorithm. + /// + /// A "step" is somewhat of a misnomer, because this routine keeps going + /// until it either runs out of things to try or fins a match. In the + /// former case, it may have pushed some things on to the backtracking + /// stack, in which case, those will be tried next as part of the + /// 'backtrack' routine above. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn step( + &self, + cache: &mut Cache, + input: &Input<'_>, + mut sid: StateID, + mut at: usize, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<HalfMatch> { + loop { + if !cache.visited.insert(sid, at - input.start()) { + return None; + } + match *self.nfa.state(sid) { + State::ByteRange { ref trans } => { + // Why do we need this? Unlike other regex engines in this + // crate, the backtracker can steam roll ahead in the + // haystack outside of the main loop over the bytes in the + // haystack. While 'trans.matches()' below handles the case + // of 'at' being out of bounds of 'input.haystack()', we + // also need to handle the case of 'at' going out of bounds + // of the span the caller asked to search. + // + // We should perhaps make the 'trans.matches()' API accept + // an '&Input' instead of a '&[u8]'. Or at least, add a new + // API that does it. + if at >= input.end() { + return None; + } + if !trans.matches(input.haystack(), at) { + return None; + } + sid = trans.next; + at += 1; + } + State::Sparse(ref sparse) => { + if at >= input.end() { + return None; + } + sid = sparse.matches(input.haystack(), at)?; + at += 1; + } + State::Dense(ref dense) => { + if at >= input.end() { + return None; + } + sid = dense.matches(input.haystack(), at)?; + at += 1; + } + State::Look { look, next } => { + // OK because we don't permit building a searcher with a + // Unicode word boundary if the requisite Unicode data is + // unavailable. + if !self.nfa.look_matcher().matches_inline( + look, + input.haystack(), + at, + ) { + return None; + } + sid = next; + } + State::Union { ref alternates } => { + sid = match alternates.get(0) { + None => return None, + Some(&sid) => sid, + }; + cache.stack.extend( + alternates[1..] + .iter() + .copied() + .rev() + .map(|sid| Frame::Step { sid, at }), + ); + } + State::BinaryUnion { alt1, alt2 } => { + sid = alt1; + cache.stack.push(Frame::Step { sid: alt2, at }); + } + State::Capture { next, slot, .. } => { + if slot.as_usize() < slots.len() { + cache.stack.push(Frame::RestoreCapture { + slot, + offset: slots[slot], + }); + slots[slot] = NonMaxUsize::new(at); + } + sid = next; + } + State::Fail => return None, + State::Match { pattern_id } => { + return Some(HalfMatch::new(pattern_id, at)); + } + } + } + } +} + +/// An iterator over all non-overlapping matches for a fallible search. +/// +/// The iterator yields a `Result<Match, MatchError` value until no more +/// matches could be found. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the BoundedBacktracker. +/// * `'c` represents the lifetime of the BoundedBacktracker's cache. +/// * `'h` represents the lifetime of the haystack being searched. +/// +/// This iterator can be created with the [`BoundedBacktracker::try_find_iter`] +/// method. +#[derive(Debug)] +pub struct TryFindMatches<'r, 'c, 'h> { + re: &'r BoundedBacktracker, + cache: &'c mut Cache, + caps: Captures, + it: iter::Searcher<'h>, +} + +impl<'r, 'c, 'h> Iterator for TryFindMatches<'r, 'c, 'h> { + type Item = Result<Match, MatchError>; + + #[inline] + fn next(&mut self) -> Option<Result<Match, MatchError>> { + // Splitting 'self' apart seems necessary to appease borrowck. + let TryFindMatches { re, ref mut cache, ref mut caps, ref mut it } = + *self; + it.try_advance(|input| { + re.try_search(cache, input, caps)?; + Ok(caps.get_match()) + }) + .transpose() + } +} + +/// An iterator over all non-overlapping leftmost matches, with their capturing +/// groups, for a fallible search. +/// +/// The iterator yields a `Result<Captures, MatchError>` value until no more +/// matches could be found. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the BoundedBacktracker. +/// * `'c` represents the lifetime of the BoundedBacktracker's cache. +/// * `'h` represents the lifetime of the haystack being searched. +/// +/// This iterator can be created with the +/// [`BoundedBacktracker::try_captures_iter`] method. +#[derive(Debug)] +pub struct TryCapturesMatches<'r, 'c, 'h> { + re: &'r BoundedBacktracker, + cache: &'c mut Cache, + caps: Captures, + it: iter::Searcher<'h>, +} + +impl<'r, 'c, 'h> Iterator for TryCapturesMatches<'r, 'c, 'h> { + type Item = Result<Captures, MatchError>; + + #[inline] + fn next(&mut self) -> Option<Result<Captures, MatchError>> { + // Splitting 'self' apart seems necessary to appease borrowck. + let TryCapturesMatches { re, ref mut cache, ref mut caps, ref mut it } = + *self; + let _ = it + .try_advance(|input| { + re.try_search(cache, input, caps)?; + Ok(caps.get_match()) + }) + .transpose()?; + if caps.is_match() { + Some(Ok(caps.clone())) + } else { + None + } + } +} + +/// A cache represents mutable state that a [`BoundedBacktracker`] requires +/// during a search. +/// +/// For a given [`BoundedBacktracker`], its corresponding cache may be created +/// either via [`BoundedBacktracker::create_cache`], or via [`Cache::new`]. +/// They are equivalent in every way, except the former does not require +/// explicitly importing `Cache`. +/// +/// A particular `Cache` is coupled with the [`BoundedBacktracker`] from which +/// it was created. It may only be used with that `BoundedBacktracker`. A cache +/// and its allocations may be re-purposed via [`Cache::reset`], in which case, +/// it can only be used with the new `BoundedBacktracker` (and not the old +/// one). +#[derive(Clone, Debug)] +pub struct Cache { + /// Stack used on the heap for doing backtracking instead of the + /// traditional recursive approach. We don't want recursion because then + /// we're likely to hit a stack overflow for bigger regexes. + stack: Vec<Frame>, + /// The set of (StateID, HaystackOffset) pairs that have been visited + /// by the backtracker within a single search. If such a pair has been + /// visited, then we avoid doing the work for that pair again. This is + /// what "bounds" the backtracking and prevents it from having worst case + /// exponential time. + visited: Visited, +} + +impl Cache { + /// Create a new [`BoundedBacktracker`] cache. + /// + /// A potentially more convenient routine to create a cache is + /// [`BoundedBacktracker::create_cache`], as it does not require also + /// importing the `Cache` type. + /// + /// If you want to reuse the returned `Cache` with some other + /// `BoundedBacktracker`, then you must call [`Cache::reset`] with the + /// desired `BoundedBacktracker`. + pub fn new(re: &BoundedBacktracker) -> Cache { + Cache { stack: vec![], visited: Visited::new(re) } + } + + /// Reset this cache such that it can be used for searching with different + /// [`BoundedBacktracker`]. + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different `BoundedBacktracker`. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different + /// `BoundedBacktracker`. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// nfa::thompson::backtrack::BoundedBacktracker, + /// Match, + /// }; + /// + /// let re1 = BoundedBacktracker::new(r"\w")?; + /// let re2 = BoundedBacktracker::new(r"\W")?; + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Ok(Match::must(0, 0..2))), + /// re1.try_find_iter(&mut cache, "Δ").next(), + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the BoundedBacktracker we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// cache.reset(&re2); + /// assert_eq!( + /// Some(Ok(Match::must(0, 0..3))), + /// re2.try_find_iter(&mut cache, "☃").next(), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset(&mut self, re: &BoundedBacktracker) { + self.visited.reset(re); + } + + /// Returns the heap memory usage, in bytes, of this cache. + /// + /// This does **not** include the stack size used up by this cache. To + /// compute that, use `std::mem::size_of::<Cache>()`. + pub fn memory_usage(&self) -> usize { + self.stack.len() * core::mem::size_of::<Frame>() + + self.visited.memory_usage() + } + + /// Clears this cache. This should be called at the start of every search + /// to ensure we start with a clean slate. + /// + /// This also sets the length of the capturing groups used in the current + /// search. This permits an optimization where by 'SlotTable::for_state' + /// only returns the number of slots equivalent to the number of slots + /// given in the 'Captures' value. This may be less than the total number + /// of possible slots, e.g., when one only wants to track overall match + /// offsets. This in turn permits less copying of capturing group spans + /// in the BoundedBacktracker. + fn setup_search( + &mut self, + re: &BoundedBacktracker, + input: &Input<'_>, + ) -> Result<(), MatchError> { + self.stack.clear(); + self.visited.setup_search(re, input)?; + Ok(()) + } +} + +/// Represents a stack frame on the heap while doing backtracking. +/// +/// Instead of using explicit recursion for backtracking, we use a stack on +/// the heap to keep track of things that we want to explore if the current +/// backtracking branch turns out to not lead to a match. +#[derive(Clone, Debug)] +enum Frame { + /// Look for a match starting at `sid` and the given position in the + /// haystack. + Step { sid: StateID, at: usize }, + /// Reset the given `slot` to the given `offset` (which might be `None`). + /// This effectively gives a "scope" to capturing groups, such that an + /// offset for a particular group only gets returned if the match goes + /// through that capturing group. If backtracking ends up going down a + /// different branch that results in a different offset (or perhaps none at + /// all), then this "restore capture" frame will cause the offset to get + /// reset. + RestoreCapture { slot: SmallIndex, offset: Option<NonMaxUsize> }, +} + +/// A bitset that keeps track of whether a particular (StateID, offset) has +/// been considered during backtracking. If it has already been visited, then +/// backtracking skips it. This is what gives backtracking its "bound." +#[derive(Clone, Debug)] +struct Visited { + /// The actual underlying bitset. Each element in the bitset corresponds + /// to a particular (StateID, offset) pair. States correspond to the rows + /// and the offsets correspond to the columns. + /// + /// If our underlying NFA has N states and the haystack we're searching + /// has M bytes, then we have N*(M+1) entries in our bitset table. The + /// M+1 occurs because our matches are delayed by one byte (to support + /// look-around), and so we need to handle the end position itself rather + /// than stopping just before the end. (If there is no end position, then + /// it's treated as "end-of-input," which is matched by things like '$'.) + /// + /// Given BITS=N*(M+1), we wind up with div_ceil(BITS, sizeof(usize)) + /// blocks. + /// + /// We use 'usize' to represent our blocks because it makes some of the + /// arithmetic in 'insert' a bit nicer. For example, if we used 'u32' for + /// our block, we'd either need to cast u32s to usizes or usizes to u32s. + bitset: Vec<usize>, + /// The stride represents one plus length of the haystack we're searching + /// (as described above). The stride must be initialized for each search. + stride: usize, +} + +impl Visited { + /// The size of each block, in bits. + const BLOCK_SIZE: usize = 8 * core::mem::size_of::<usize>(); + + /// Create a new visited set for the given backtracker. + /// + /// The set is ready to use, but must be setup at the beginning of each + /// search by calling `setup_search`. + fn new(re: &BoundedBacktracker) -> Visited { + let mut visited = Visited { bitset: vec![], stride: 0 }; + visited.reset(re); + visited + } + + /// Insert the given (StateID, offset) pair into this set. If it already + /// exists, then this is a no-op and it returns false. Otherwise this + /// returns true. + fn insert(&mut self, sid: StateID, at: usize) -> bool { + let table_index = sid.as_usize() * self.stride + at; + let block_index = table_index / Visited::BLOCK_SIZE; + let bit = table_index % Visited::BLOCK_SIZE; + let block_with_bit = 1 << bit; + if self.bitset[block_index] & block_with_bit != 0 { + return false; + } + self.bitset[block_index] |= block_with_bit; + true + } + + /// Reset this visited set to work with the given bounded backtracker. + fn reset(&mut self, _: &BoundedBacktracker) { + self.bitset.truncate(0); + } + + /// Setup this visited set to work for a search using the given NFA + /// and input configuration. The NFA must be the same NFA used by the + /// BoundedBacktracker given to Visited::reset. Failing to call this might + /// result in panics or silently incorrect search behavior. + fn setup_search( + &mut self, + re: &BoundedBacktracker, + input: &Input<'_>, + ) -> Result<(), MatchError> { + // Our haystack length is only the length of the span of the entire + // haystack that we'll be searching. + let haylen = input.get_span().len(); + let err = || MatchError::haystack_too_long(haylen); + // Our stride is one more than the length of the input because our main + // search loop includes the position at input.end(). (And it does this + // because matches are delayed by one byte to account for look-around.) + self.stride = haylen + 1; + let needed_capacity = + match re.get_nfa().states().len().checked_mul(self.stride) { + None => return Err(err()), + Some(capacity) => capacity, + }; + let max_capacity = 8 * re.get_config().get_visited_capacity(); + if needed_capacity > max_capacity { + return Err(err()); + } + let needed_blocks = div_ceil(needed_capacity, Visited::BLOCK_SIZE); + self.bitset.truncate(needed_blocks); + for block in self.bitset.iter_mut() { + *block = 0; + } + if needed_blocks > self.bitset.len() { + self.bitset.resize(needed_blocks, 0); + } + Ok(()) + } + + /// Return the heap memory usage, in bytes, of this visited set. + fn memory_usage(&self) -> usize { + self.bitset.len() * core::mem::size_of::<usize>() + } +} + +/// Integer division, but rounds up instead of down. +fn div_ceil(lhs: usize, rhs: usize) -> usize { + if lhs % rhs == 0 { + lhs / rhs + } else { + (lhs / rhs) + 1 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // This is a regression test for the maximum haystack length computation. + // Previously, it assumed that the total capacity of the backtracker's + // bitset would always be greater than the number of NFA states. But there + // is of course no guarantee that this is true. This regression test + // ensures that not only does `max_haystack_len` not panic, but that it + // should return `0`. + #[cfg(feature = "syntax")] + #[test] + fn max_haystack_len_overflow() { + let re = BoundedBacktracker::builder() + .configure(BoundedBacktracker::config().visited_capacity(10)) + .build(r"[0-9A-Za-z]{100}") + .unwrap(); + assert_eq!(0, re.max_haystack_len()); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/builder.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/builder.rs new file mode 100644 index 0000000..6b69e87 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/builder.rs @@ -0,0 +1,1337 @@ +use core::mem; + +use alloc::{sync::Arc, vec, vec::Vec}; + +use crate::{ + nfa::thompson::{ + error::BuildError, + nfa::{self, SparseTransitions, Transition, NFA}, + }, + util::{ + look::{Look, LookMatcher}, + primitives::{IteratorIndexExt, PatternID, SmallIndex, StateID}, + }, +}; + +/// An intermediate NFA state used during construction. +/// +/// During construction of an NFA, it is often convenient to work with states +/// that are amenable to mutation and other carry more information than we +/// otherwise need once an NFA has been built. This type represents those +/// needs. +/// +/// Once construction is finished, the builder will convert these states to a +/// [`nfa::thompson::State`](crate::nfa::thompson::State). This conversion not +/// only results in a simpler representation, but in some cases, entire classes +/// of states are completely removed (such as [`State::Empty`]). +#[derive(Clone, Debug, Eq, PartialEq)] +enum State { + /// An empty state whose only purpose is to forward the automaton to + /// another state via an unconditional epsilon transition. + /// + /// Unconditional epsilon transitions are quite useful during the + /// construction of an NFA, as they permit the insertion of no-op + /// placeholders that make it easier to compose NFA sub-graphs. When + /// the Thompson NFA builder produces a final NFA, all unconditional + /// epsilon transitions are removed, and state identifiers are remapped + /// accordingly. + Empty { + /// The next state that this state should transition to. + next: StateID, + }, + /// A state that only transitions to another state if the current input + /// byte is in a particular range of bytes. + ByteRange { trans: Transition }, + /// A state with possibly many transitions, represented in a sparse + /// fashion. Transitions must be ordered lexicographically by input range + /// and be non-overlapping. As such, this may only be used when every + /// transition has equal priority. (In practice, this is only used for + /// encoding large UTF-8 automata.) In contrast, a `Union` state has each + /// alternate in order of priority. Priority is used to implement greedy + /// matching and also alternations themselves, e.g., `abc|a` where `abc` + /// has priority over `a`. + /// + /// To clarify, it is possible to remove `Sparse` and represent all things + /// that `Sparse` is used for via `Union`. But this creates a more bloated + /// NFA with more epsilon transitions than is necessary in the special case + /// of character classes. + Sparse { transitions: Vec<Transition> }, + /// A conditional epsilon transition satisfied via some sort of + /// look-around. + Look { look: Look, next: StateID }, + /// An empty state that records the start of a capture location. This is an + /// unconditional epsilon transition like `Empty`, except it can be used to + /// record position information for a capture group when using the NFA for + /// search. + CaptureStart { + /// The ID of the pattern that this capture was defined. + pattern_id: PatternID, + /// The capture group index that this capture state corresponds to. + /// The capture group index is always relative to its corresponding + /// pattern. Therefore, in the presence of multiple patterns, both the + /// pattern ID and the capture group index are required to uniquely + /// identify a capturing group. + group_index: SmallIndex, + /// The next state that this state should transition to. + next: StateID, + }, + /// An empty state that records the end of a capture location. This is an + /// unconditional epsilon transition like `Empty`, except it can be used to + /// record position information for a capture group when using the NFA for + /// search. + CaptureEnd { + /// The ID of the pattern that this capture was defined. + pattern_id: PatternID, + /// The capture group index that this capture state corresponds to. + /// The capture group index is always relative to its corresponding + /// pattern. Therefore, in the presence of multiple patterns, both the + /// pattern ID and the capture group index are required to uniquely + /// identify a capturing group. + group_index: SmallIndex, + /// The next state that this state should transition to. + next: StateID, + }, + /// An alternation such that there exists an epsilon transition to all + /// states in `alternates`, where matches found via earlier transitions + /// are preferred over later transitions. + Union { alternates: Vec<StateID> }, + /// An alternation such that there exists an epsilon transition to all + /// states in `alternates`, where matches found via later transitions are + /// preferred over earlier transitions. + /// + /// This "reverse" state exists for convenience during compilation that + /// permits easy construction of non-greedy combinations of NFA states. At + /// the end of compilation, Union and UnionReverse states are merged into + /// one Union type of state, where the latter has its epsilon transitions + /// reversed to reflect the priority inversion. + /// + /// The "convenience" here arises from the fact that as new states are + /// added to the list of `alternates`, we would like that add operation + /// to be amortized constant time. But if we used a `Union`, we'd need to + /// prepend the state, which takes O(n) time. There are other approaches we + /// could use to solve this, but this seems simple enough. + UnionReverse { alternates: Vec<StateID> }, + /// A state that cannot be transitioned out of. This is useful for cases + /// where you want to prevent matching from occurring. For example, if your + /// regex parser permits empty character classes, then one could choose a + /// `Fail` state to represent it. + Fail, + /// A match state. There is at most one such occurrence of this state in + /// an NFA for each pattern compiled into the NFA. At time of writing, a + /// match state is always produced for every pattern given, but in theory, + /// if a pattern can never lead to a match, then the match state could be + /// omitted. + /// + /// `pattern_id` refers to the ID of the pattern itself, which corresponds + /// to the pattern's index (starting at 0). + Match { pattern_id: PatternID }, +} + +impl State { + /// If this state is an unconditional epsilon transition, then this returns + /// the target of the transition. + fn goto(&self) -> Option<StateID> { + match *self { + State::Empty { next } => Some(next), + State::Union { ref alternates } if alternates.len() == 1 => { + Some(alternates[0]) + } + State::UnionReverse { ref alternates } + if alternates.len() == 1 => + { + Some(alternates[0]) + } + _ => None, + } + } + + /// Returns the heap memory usage, in bytes, of this state. + fn memory_usage(&self) -> usize { + match *self { + State::Empty { .. } + | State::ByteRange { .. } + | State::Look { .. } + | State::CaptureStart { .. } + | State::CaptureEnd { .. } + | State::Fail + | State::Match { .. } => 0, + State::Sparse { ref transitions } => { + transitions.len() * mem::size_of::<Transition>() + } + State::Union { ref alternates } => { + alternates.len() * mem::size_of::<StateID>() + } + State::UnionReverse { ref alternates } => { + alternates.len() * mem::size_of::<StateID>() + } + } + } +} + +/// An abstraction for building Thompson NFAs by hand. +/// +/// A builder is what a [`thompson::Compiler`](crate::nfa::thompson::Compiler) +/// uses internally to translate a regex's high-level intermediate +/// representation into an [`NFA`]. +/// +/// The primary function of this builder is to abstract away the internal +/// representation of an NFA and make it difficult to produce NFAs are that +/// internally invalid or inconsistent. This builder also provides a way to +/// add "empty" states (which can be thought of as unconditional epsilon +/// transitions), despite the fact that [`thompson::State`](nfa::State) does +/// not have any "empty" representation. The advantage of "empty" states is +/// that they make the code for constructing a Thompson NFA logically simpler. +/// +/// Many of the routines on this builder may panic or return errors. Generally +/// speaking, panics occur when an invalid sequence of method calls were made, +/// where as an error occurs if things get too big. (Where "too big" might mean +/// exhausting identifier space or using up too much heap memory in accordance +/// with the configured [`size_limit`](Builder::set_size_limit).) +/// +/// # Overview +/// +/// ## Adding multiple patterns +/// +/// Each pattern you add to an NFA should correspond to a pair of +/// [`Builder::start_pattern`] and [`Builder::finish_pattern`] calls, with +/// calls inbetween that add NFA states for that pattern. NFA states may be +/// added without first calling `start_pattern`, with the exception of adding +/// capturing states. +/// +/// ## Adding NFA states +/// +/// Here is a very brief overview of each of the methods that add NFA states. +/// Every method adds a single state. +/// +/// * [`add_empty`](Builder::add_empty): Add a state with a single +/// unconditional epsilon transition to another state. +/// * [`add_union`](Builder::add_union): Adds a state with unconditional +/// epsilon transitions to two or more states, with earlier transitions +/// preferred over later ones. +/// * [`add_union_reverse`](Builder::add_union_reverse): Adds a state with +/// unconditional epsilon transitions to two or more states, with later +/// transitions preferred over earlier ones. +/// * [`add_range`](Builder::add_range): Adds a state with a single transition +/// to another state that can only be followed if the current input byte is +/// within the range given. +/// * [`add_sparse`](Builder::add_sparse): Adds a state with two or more +/// range transitions to other states, where a transition is only followed +/// if the current input byte is within one of the ranges. All transitions +/// in this state have equal priority, and the corresponding ranges must be +/// non-overlapping. +/// * [`add_look`](Builder::add_look): Adds a state with a single *conditional* +/// epsilon transition to another state, where the condition depends on a +/// limited look-around property. +/// * [`add_capture_start`](Builder::add_capture_start): Adds a state with +/// a single unconditional epsilon transition that also instructs an NFA +/// simulation to record the current input position to a specific location in +/// memory. This is intended to represent the starting location of a capturing +/// group. +/// * [`add_capture_end`](Builder::add_capture_end): Adds a state with +/// a single unconditional epsilon transition that also instructs an NFA +/// simulation to record the current input position to a specific location in +/// memory. This is intended to represent the ending location of a capturing +/// group. +/// * [`add_fail`](Builder::add_fail): Adds a state that never transitions to +/// another state. +/// * [`add_match`](Builder::add_match): Add a state that indicates a match has +/// been found for a particular pattern. A match state is a final state with +/// no outgoing transitions. +/// +/// ## Setting transitions between NFA states +/// +/// The [`Builder::patch`] method creates a transition from one state to the +/// next. If the `from` state corresponds to a state that supports multiple +/// outgoing transitions (such as "union"), then this adds the corresponding +/// transition. Otherwise, it sets the single transition. (This routine panics +/// if `from` corresponds to a state added by `add_sparse`, since sparse states +/// need more specialized handling.) +/// +/// # Example +/// +/// This annotated example shows how to hand construct the regex `[a-z]+` +/// (without an unanchored prefix). +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::{pikevm::PikeVM, Builder, Transition}, +/// util::primitives::StateID, +/// Match, +/// }; +/// +/// let mut builder = Builder::new(); +/// // Before adding NFA states for our pattern, we need to tell the builder +/// // that we are starting the pattern. +/// builder.start_pattern()?; +/// // Since we use the Pike VM below for searching, we need to add capturing +/// // states. If you're just going to build a DFA from the NFA, then capturing +/// // states do not need to be added. +/// let start = builder.add_capture_start(StateID::ZERO, 0, None)?; +/// let range = builder.add_range(Transition { +/// // We don't know the state ID of the 'next' state yet, so we just fill +/// // in a dummy 'ZERO' value. +/// start: b'a', end: b'z', next: StateID::ZERO, +/// })?; +/// // This state will point back to 'range', but also enable us to move ahead. +/// // That is, this implements the '+' repetition operator. We add 'range' and +/// // then 'end' below to this alternation. +/// let alt = builder.add_union(vec![])?; +/// // The final state before the match state, which serves to capture the +/// // end location of the match. +/// let end = builder.add_capture_end(StateID::ZERO, 0)?; +/// // The match state for our pattern. +/// let mat = builder.add_match()?; +/// // Now we fill in the transitions between states. +/// builder.patch(start, range)?; +/// builder.patch(range, alt)?; +/// // If we added 'end' before 'range', then we'd implement non-greedy +/// // matching, i.e., '+?'. +/// builder.patch(alt, range)?; +/// builder.patch(alt, end)?; +/// builder.patch(end, mat)?; +/// // We must explicitly finish pattern and provide the starting state ID for +/// // this particular pattern. +/// builder.finish_pattern(start)?; +/// // Finally, when we build the NFA, we provide the anchored and unanchored +/// // starting state IDs. Since we didn't bother with an unanchored prefix +/// // here, we only support anchored searching. Thus, both starting states are +/// // the same. +/// let nfa = builder.build(start, start)?; +/// +/// // Now build a Pike VM from our NFA, and use it for searching. This shows +/// // how we can use a regex engine without ever worrying about syntax! +/// let re = PikeVM::new_from_nfa(nfa)?; +/// let mut cache = re.create_cache(); +/// let mut caps = re.create_captures(); +/// let expected = Some(Match::must(0, 0..3)); +/// re.captures(&mut cache, "foo0", &mut caps); +/// assert_eq!(expected, caps.get_match()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug, Default)] +pub struct Builder { + /// The ID of the pattern that we're currently building. + /// + /// Callers are required to set (and unset) this by calling + /// {start,finish}_pattern. Otherwise, most methods will panic. + pattern_id: Option<PatternID>, + /// A sequence of intermediate NFA states. Once a state is added to this + /// sequence, it is assigned a state ID equivalent to its index. Once a + /// state is added, it is still expected to be mutated, e.g., to set its + /// transition to a state that didn't exist at the time it was added. + states: Vec<State>, + /// The starting states for each individual pattern. Starting at any + /// of these states will result in only an anchored search for the + /// corresponding pattern. The vec is indexed by pattern ID. When the NFA + /// contains a single regex, then `start_pattern[0]` and `start_anchored` + /// are always equivalent. + start_pattern: Vec<StateID>, + /// A map from pattern ID to capture group index to name. (If no name + /// exists, then a None entry is present. Thus, all capturing groups are + /// present in this mapping.) + /// + /// The outer vec is indexed by pattern ID, while the inner vec is indexed + /// by capture index offset for the corresponding pattern. + /// + /// The first capture group for each pattern is always unnamed and is thus + /// always None. + captures: Vec<Vec<Option<Arc<str>>>>, + /// The combined memory used by each of the 'State's in 'states'. This + /// only includes heap usage by each state, and not the size of the state + /// itself. In other words, this tracks heap memory used that isn't + /// captured via `size_of::<State>() * states.len()`. + memory_states: usize, + /// Whether this NFA only matches UTF-8 and whether regex engines using + /// this NFA for searching should report empty matches that split a + /// codepoint. + utf8: bool, + /// Whether this NFA should be matched in reverse or not. + reverse: bool, + /// The matcher to use for look-around assertions. + look_matcher: LookMatcher, + /// A size limit to respect when building an NFA. If the total heap memory + /// of the intermediate NFA states exceeds (or would exceed) this amount, + /// then an error is returned. + size_limit: Option<usize>, +} + +impl Builder { + /// Create a new builder for hand-assembling NFAs. + pub fn new() -> Builder { + Builder::default() + } + + /// Clear this builder. + /// + /// Clearing removes all state associated with building an NFA, but does + /// not reset configuration (such as size limits and whether the NFA + /// should only match UTF-8). After clearing, the builder can be reused to + /// assemble an entirely new NFA. + pub fn clear(&mut self) { + self.pattern_id = None; + self.states.clear(); + self.start_pattern.clear(); + self.captures.clear(); + self.memory_states = 0; + } + + /// Assemble a [`NFA`] from the states added so far. + /// + /// After building an NFA, more states may be added and `build` may be + /// called again. To reuse a builder to produce an entirely new NFA from + /// scratch, call the [`clear`](Builder::clear) method first. + /// + /// `start_anchored` refers to the ID of the starting state that anchored + /// searches should use. That is, searches who matches are limited to the + /// starting position of the search. + /// + /// `start_unanchored` refers to the ID of the starting state that + /// unanchored searches should use. This permits searches to report matches + /// that start after the beginning of the search. In cases where unanchored + /// searches are not supported, the unanchored starting state ID must be + /// the same as the anchored starting state ID. + /// + /// # Errors + /// + /// This returns an error if there was a problem producing the final NFA. + /// In particular, this might include an error if the capturing groups + /// added to this builder violate any of the invariants documented on + /// [`GroupInfo`](crate::util::captures::GroupInfo). + /// + /// # Panics + /// + /// If `start_pattern` was called, then `finish_pattern` must be called + /// before `build`, otherwise this panics. + /// + /// This may panic for other invalid uses of a builder. For example, if + /// a "start capture" state was added without a corresponding "end capture" + /// state. + pub fn build( + &self, + start_anchored: StateID, + start_unanchored: StateID, + ) -> Result<NFA, BuildError> { + assert!(self.pattern_id.is_none(), "must call 'finish_pattern' first"); + debug!( + "intermediate NFA compilation via builder is complete, \ + intermediate NFA size: {} states, {} bytes on heap", + self.states.len(), + self.memory_usage(), + ); + + let mut nfa = nfa::Inner::default(); + nfa.set_utf8(self.utf8); + nfa.set_reverse(self.reverse); + nfa.set_look_matcher(self.look_matcher.clone()); + // A set of compiler internal state IDs that correspond to states + // that are exclusively epsilon transitions, i.e., goto instructions, + // combined with the state that they point to. This is used to + // record said states while transforming the compiler's internal NFA + // representation to the external form. + let mut empties = vec![]; + // A map used to re-map state IDs when translating this builder's + // internal NFA state representation to the final NFA representation. + let mut remap = vec![]; + remap.resize(self.states.len(), StateID::ZERO); + + nfa.set_starts(start_anchored, start_unanchored, &self.start_pattern); + nfa.set_captures(&self.captures).map_err(BuildError::captures)?; + // The idea here is to convert our intermediate states to their final + // form. The only real complexity here is the process of converting + // transitions, which are expressed in terms of state IDs. The new + // set of states will be smaller because of partial epsilon removal, + // so the state IDs will not be the same. + for (sid, state) in self.states.iter().with_state_ids() { + match *state { + State::Empty { next } => { + // Since we're removing empty states, we need to handle + // them later since we don't yet know which new state this + // empty state will be mapped to. + empties.push((sid, next)); + } + State::ByteRange { trans } => { + remap[sid] = nfa.add(nfa::State::ByteRange { trans }); + } + State::Sparse { ref transitions } => { + remap[sid] = match transitions.len() { + 0 => nfa.add(nfa::State::Fail), + 1 => nfa.add(nfa::State::ByteRange { + trans: transitions[0], + }), + _ => { + let transitions = + transitions.to_vec().into_boxed_slice(); + let sparse = SparseTransitions { transitions }; + nfa.add(nfa::State::Sparse(sparse)) + } + } + } + State::Look { look, next } => { + remap[sid] = nfa.add(nfa::State::Look { look, next }); + } + State::CaptureStart { pattern_id, group_index, next } => { + // We can't remove this empty state because of the side + // effect of capturing an offset for this capture slot. + let slot = nfa + .group_info() + .slot(pattern_id, group_index.as_usize()) + .expect("invalid capture index"); + let slot = + SmallIndex::new(slot).expect("a small enough slot"); + remap[sid] = nfa.add(nfa::State::Capture { + next, + pattern_id, + group_index, + slot, + }); + } + State::CaptureEnd { pattern_id, group_index, next } => { + // We can't remove this empty state because of the side + // effect of capturing an offset for this capture slot. + // Also, this always succeeds because we check that all + // slot indices are valid for all capture indices when they + // are initially added. + let slot = nfa + .group_info() + .slot(pattern_id, group_index.as_usize()) + .expect("invalid capture index") + .checked_add(1) + .unwrap(); + let slot = + SmallIndex::new(slot).expect("a small enough slot"); + remap[sid] = nfa.add(nfa::State::Capture { + next, + pattern_id, + group_index, + slot, + }); + } + State::Union { ref alternates } => { + if alternates.is_empty() { + remap[sid] = nfa.add(nfa::State::Fail); + } else if alternates.len() == 1 { + empties.push((sid, alternates[0])); + remap[sid] = alternates[0]; + } else if alternates.len() == 2 { + remap[sid] = nfa.add(nfa::State::BinaryUnion { + alt1: alternates[0], + alt2: alternates[1], + }); + } else { + let alternates = + alternates.to_vec().into_boxed_slice(); + remap[sid] = nfa.add(nfa::State::Union { alternates }); + } + } + State::UnionReverse { ref alternates } => { + if alternates.is_empty() { + remap[sid] = nfa.add(nfa::State::Fail); + } else if alternates.len() == 1 { + empties.push((sid, alternates[0])); + remap[sid] = alternates[0]; + } else if alternates.len() == 2 { + remap[sid] = nfa.add(nfa::State::BinaryUnion { + alt1: alternates[1], + alt2: alternates[0], + }); + } else { + let mut alternates = + alternates.to_vec().into_boxed_slice(); + alternates.reverse(); + remap[sid] = nfa.add(nfa::State::Union { alternates }); + } + } + State::Fail => { + remap[sid] = nfa.add(nfa::State::Fail); + } + State::Match { pattern_id } => { + remap[sid] = nfa.add(nfa::State::Match { pattern_id }); + } + } + } + // Some of the new states still point to empty state IDs, so we need to + // follow each of them and remap the empty state IDs to their non-empty + // state IDs. + // + // We also keep track of which states we've already mapped. This helps + // avoid quadratic behavior in a long chain of empty states. For + // example, in 'a{0}{50000}'. + let mut remapped = vec![false; self.states.len()]; + for &(empty_id, empty_next) in empties.iter() { + if remapped[empty_id] { + continue; + } + // empty states can point to other empty states, forming a chain. + // So we must follow the chain until the end, which must end at + // a non-empty state, and therefore, a state that is correctly + // remapped. We are guaranteed to terminate because our compiler + // never builds a loop among only empty states. + let mut new_next = empty_next; + while let Some(next) = self.states[new_next].goto() { + new_next = next; + } + remap[empty_id] = remap[new_next]; + remapped[empty_id] = true; + + // Now that we've remapped the main 'empty_id' above, we re-follow + // the chain from above and remap every empty state we found along + // the way to our ultimate non-empty target. We are careful to set + // 'remapped' to true for each such state. We thus will not need + // to re-compute this chain for any subsequent empty states in + // 'empties' that are part of this chain. + let mut next2 = empty_next; + while let Some(next) = self.states[next2].goto() { + remap[next2] = remap[new_next]; + remapped[next2] = true; + next2 = next; + } + } + // Finally remap all of the state IDs. + nfa.remap(&remap); + let final_nfa = nfa.into_nfa(); + debug!( + "NFA compilation via builder complete, \ + final NFA size: {} states, {} bytes on heap, \ + has empty? {:?}, utf8? {:?}", + final_nfa.states().len(), + final_nfa.memory_usage(), + final_nfa.has_empty(), + final_nfa.is_utf8(), + ); + Ok(final_nfa) + } + + /// Start the assembly of a pattern in this NFA. + /// + /// Upon success, this returns the identifier for the new pattern. + /// Identifiers start at `0` and are incremented by 1 for each new pattern. + /// + /// It is necessary to call this routine before adding capturing states. + /// Otherwise, any other NFA state may be added before starting a pattern. + /// + /// # Errors + /// + /// If the pattern identifier space is exhausted, then this returns an + /// error. + /// + /// # Panics + /// + /// If this is called while assembling another pattern (i.e., before + /// `finish_pattern` is called), then this panics. + pub fn start_pattern(&mut self) -> Result<PatternID, BuildError> { + assert!(self.pattern_id.is_none(), "must call 'finish_pattern' first"); + + let proposed = self.start_pattern.len(); + let pid = PatternID::new(proposed) + .map_err(|_| BuildError::too_many_patterns(proposed))?; + self.pattern_id = Some(pid); + // This gets filled in when 'finish_pattern' is called. + self.start_pattern.push(StateID::ZERO); + Ok(pid) + } + + /// Finish the assembly of a pattern in this NFA. + /// + /// Upon success, this returns the identifier for the new pattern. + /// Identifiers start at `0` and are incremented by 1 for each new + /// pattern. This is the same identifier returned by the corresponding + /// `start_pattern` call. + /// + /// Note that `start_pattern` and `finish_pattern` pairs cannot be + /// interleaved or nested. A correct `finish_pattern` call _always_ + /// corresponds to the most recently called `start_pattern` routine. + /// + /// # Errors + /// + /// This currently never returns an error, but this is subject to change. + /// + /// # Panics + /// + /// If this is called without a corresponding `start_pattern` call, then + /// this panics. + pub fn finish_pattern( + &mut self, + start_id: StateID, + ) -> Result<PatternID, BuildError> { + let pid = self.current_pattern_id(); + self.start_pattern[pid] = start_id; + self.pattern_id = None; + Ok(pid) + } + + /// Returns the pattern identifier of the current pattern. + /// + /// # Panics + /// + /// If this doesn't occur after a `start_pattern` call and before the + /// corresponding `finish_pattern` call, then this panics. + pub fn current_pattern_id(&self) -> PatternID { + self.pattern_id.expect("must call 'start_pattern' first") + } + + /// Returns the number of patterns added to this builder so far. + /// + /// This only includes patterns that have had `finish_pattern` called + /// for them. + pub fn pattern_len(&self) -> usize { + self.start_pattern.len() + } + + /// Add an "empty" NFA state. + /// + /// An "empty" NFA state is a state with a single unconditional epsilon + /// transition to another NFA state. Such empty states are removed before + /// building the final [`NFA`] (which has no such "empty" states), but they + /// can be quite useful in the construction process of an NFA. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded. + pub fn add_empty(&mut self) -> Result<StateID, BuildError> { + self.add(State::Empty { next: StateID::ZERO }) + } + + /// Add a "union" NFA state. + /// + /// A "union" NFA state that contains zero or more unconditional epsilon + /// transitions to other NFA states. The order of these transitions + /// reflects a priority order where earlier transitions are preferred over + /// later transitions. + /// + /// Callers may provide an empty set of alternates to this method call, and + /// then later add transitions via `patch`. At final build time, a "union" + /// state with no alternates is converted to a "fail" state, and a "union" + /// state with exactly one alternate is treated as if it were an "empty" + /// state. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded. + pub fn add_union( + &mut self, + alternates: Vec<StateID>, + ) -> Result<StateID, BuildError> { + self.add(State::Union { alternates }) + } + + /// Add a "reverse union" NFA state. + /// + /// A "reverse union" NFA state contains zero or more unconditional epsilon + /// transitions to other NFA states. The order of these transitions + /// reflects a priority order where later transitions are preferred + /// over earlier transitions. This is an inverted priority order when + /// compared to `add_union`. This is useful, for example, for implementing + /// non-greedy repetition operators. + /// + /// Callers may provide an empty set of alternates to this method call, and + /// then later add transitions via `patch`. At final build time, a "reverse + /// union" state with no alternates is converted to a "fail" state, and a + /// "reverse union" state with exactly one alternate is treated as if it + /// were an "empty" state. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded. + pub fn add_union_reverse( + &mut self, + alternates: Vec<StateID>, + ) -> Result<StateID, BuildError> { + self.add(State::UnionReverse { alternates }) + } + + /// Add a "range" NFA state. + /// + /// A "range" NFA state is a state with one outgoing transition to another + /// state, where that transition may only be followed if the current input + /// byte falls between a range of bytes given. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded. + pub fn add_range( + &mut self, + trans: Transition, + ) -> Result<StateID, BuildError> { + self.add(State::ByteRange { trans }) + } + + /// Add a "sparse" NFA state. + /// + /// A "sparse" NFA state contains zero or more outgoing transitions, where + /// the transition to be followed (if any) is chosen based on whether the + /// current input byte falls in the range of one such transition. The + /// transitions given *must* be non-overlapping and in ascending order. (A + /// "sparse" state with no transitions is equivalent to a "fail" state.) + /// + /// A "sparse" state is like adding a "union" state and pointing it at a + /// bunch of "range" states, except that the different alternates have + /// equal priority. + /// + /// Note that a "sparse" state is the only state that cannot be patched. + /// This is because a "sparse" state has many transitions, each of which + /// may point to a different NFA state. Moreover, adding more such + /// transitions requires more than just an NFA state ID to point to. It + /// also requires a byte range. The `patch` routine does not support the + /// additional information required. Therefore, callers must ensure that + /// all outgoing transitions for this state are included when `add_sparse` + /// is called. There is no way to add more later. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded. + /// + /// # Panics + /// + /// This routine _may_ panic if the transitions given overlap or are not + /// in ascending order. + pub fn add_sparse( + &mut self, + transitions: Vec<Transition>, + ) -> Result<StateID, BuildError> { + self.add(State::Sparse { transitions }) + } + + /// Add a "look" NFA state. + /// + /// A "look" NFA state corresponds to a state with exactly one + /// *conditional* epsilon transition to another NFA state. Namely, it + /// represents one of a small set of simplistic look-around operators. + /// + /// Callers may provide a "dummy" state ID (typically [`StateID::ZERO`]), + /// and then change it later with [`patch`](Builder::patch). + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded. + pub fn add_look( + &mut self, + next: StateID, + look: Look, + ) -> Result<StateID, BuildError> { + self.add(State::Look { look, next }) + } + + /// Add a "start capture" NFA state. + /// + /// A "start capture" NFA state corresponds to a state with exactly one + /// outgoing unconditional epsilon transition to another state. Unlike + /// "empty" states, a "start capture" state also carries with it an + /// instruction for saving the current position of input to a particular + /// location in memory. NFA simulations, like the Pike VM, may use this + /// information to report the match locations of capturing groups in a + /// regex pattern. + /// + /// If the corresponding capturing group has a name, then callers should + /// include it here. + /// + /// Callers may provide a "dummy" state ID (typically [`StateID::ZERO`]), + /// and then change it later with [`patch`](Builder::patch). + /// + /// Note that unlike `start_pattern`/`finish_pattern`, capturing start and + /// end states may be interleaved. Indeed, it is typical for many "start + /// capture" NFA states to appear before the first "end capture" state. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded or if the given + /// capture index overflows `usize`. + /// + /// While the above are the only conditions in which this routine can + /// currently return an error, it is possible to call this method with an + /// inputs that results in the final `build()` step failing to produce an + /// NFA. For example, if one adds two distinct capturing groups with the + /// same name, then that will result in `build()` failing with an error. + /// + /// See the [`GroupInfo`](crate::util::captures::GroupInfo) type for + /// more information on what qualifies as valid capturing groups. + /// + /// # Example + /// + /// This example shows that an error occurs when one tries to add multiple + /// capturing groups with the same name to the same pattern. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::Builder, + /// util::primitives::StateID, + /// }; + /// + /// let name = Some(std::sync::Arc::from("foo")); + /// let mut builder = Builder::new(); + /// builder.start_pattern()?; + /// // 0th capture group should always be unnamed. + /// let start = builder.add_capture_start(StateID::ZERO, 0, None)?; + /// // OK + /// builder.add_capture_start(StateID::ZERO, 1, name.clone())?; + /// // This is not OK, but 'add_capture_start' still succeeds. We don't + /// // get an error until we call 'build' below. Without this call, the + /// // call to 'build' below would succeed. + /// builder.add_capture_start(StateID::ZERO, 2, name.clone())?; + /// // Finish our pattern so we can try to build the NFA. + /// builder.finish_pattern(start)?; + /// let result = builder.build(start, start); + /// assert!(result.is_err()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// However, adding multiple capturing groups with the same name to + /// distinct patterns is okay: + /// + /// ``` + /// use std::sync::Arc; + /// + /// use regex_automata::{ + /// nfa::thompson::{pikevm::PikeVM, Builder, Transition}, + /// util::{ + /// captures::Captures, + /// primitives::{PatternID, StateID}, + /// }, + /// Span, + /// }; + /// + /// // Hand-compile the patterns '(?P<foo>[a-z])' and '(?P<foo>[A-Z])'. + /// let mut builder = Builder::new(); + /// // We compile them to support an unanchored search, which requires + /// // adding an implicit '(?s-u:.)*?' prefix before adding either pattern. + /// let unanchored_prefix = builder.add_union_reverse(vec![])?; + /// let any = builder.add_range(Transition { + /// start: b'\x00', end: b'\xFF', next: StateID::ZERO, + /// })?; + /// builder.patch(unanchored_prefix, any)?; + /// builder.patch(any, unanchored_prefix)?; + /// + /// // Compile an alternation that permits matching multiple patterns. + /// let alt = builder.add_union(vec![])?; + /// builder.patch(unanchored_prefix, alt)?; + /// + /// // Compile '(?P<foo>[a-z]+)'. + /// builder.start_pattern()?; + /// let start0 = builder.add_capture_start(StateID::ZERO, 0, None)?; + /// // N.B. 0th capture group must always be unnamed. + /// let foo_start0 = builder.add_capture_start( + /// StateID::ZERO, 1, Some(Arc::from("foo")), + /// )?; + /// let lowercase = builder.add_range(Transition { + /// start: b'a', end: b'z', next: StateID::ZERO, + /// })?; + /// let foo_end0 = builder.add_capture_end(StateID::ZERO, 1)?; + /// let end0 = builder.add_capture_end(StateID::ZERO, 0)?; + /// let match0 = builder.add_match()?; + /// builder.patch(start0, foo_start0)?; + /// builder.patch(foo_start0, lowercase)?; + /// builder.patch(lowercase, foo_end0)?; + /// builder.patch(foo_end0, end0)?; + /// builder.patch(end0, match0)?; + /// builder.finish_pattern(start0)?; + /// + /// // Compile '(?P<foo>[A-Z]+)'. + /// builder.start_pattern()?; + /// let start1 = builder.add_capture_start(StateID::ZERO, 0, None)?; + /// // N.B. 0th capture group must always be unnamed. + /// let foo_start1 = builder.add_capture_start( + /// StateID::ZERO, 1, Some(Arc::from("foo")), + /// )?; + /// let uppercase = builder.add_range(Transition { + /// start: b'A', end: b'Z', next: StateID::ZERO, + /// })?; + /// let foo_end1 = builder.add_capture_end(StateID::ZERO, 1)?; + /// let end1 = builder.add_capture_end(StateID::ZERO, 0)?; + /// let match1 = builder.add_match()?; + /// builder.patch(start1, foo_start1)?; + /// builder.patch(foo_start1, uppercase)?; + /// builder.patch(uppercase, foo_end1)?; + /// builder.patch(foo_end1, end1)?; + /// builder.patch(end1, match1)?; + /// builder.finish_pattern(start1)?; + /// + /// // Now add the patterns to our alternation that we started above. + /// builder.patch(alt, start0)?; + /// builder.patch(alt, start1)?; + /// + /// // Finally build the NFA. The first argument is the anchored starting + /// // state (the pattern alternation) where as the second is the + /// // unanchored starting state (the unanchored prefix). + /// let nfa = builder.build(alt, unanchored_prefix)?; + /// + /// // Now build a Pike VM from our NFA and access the 'foo' capture + /// // group regardless of which pattern matched, since it is defined + /// // for both patterns. + /// let vm = PikeVM::new_from_nfa(nfa)?; + /// let mut cache = vm.create_cache(); + /// let caps: Vec<Captures> = + /// vm.captures_iter(&mut cache, "0123aAaAA").collect(); + /// assert_eq!(5, caps.len()); + /// + /// assert_eq!(Some(PatternID::must(0)), caps[0].pattern()); + /// assert_eq!(Some(Span::from(4..5)), caps[0].get_group_by_name("foo")); + /// + /// assert_eq!(Some(PatternID::must(1)), caps[1].pattern()); + /// assert_eq!(Some(Span::from(5..6)), caps[1].get_group_by_name("foo")); + /// + /// assert_eq!(Some(PatternID::must(0)), caps[2].pattern()); + /// assert_eq!(Some(Span::from(6..7)), caps[2].get_group_by_name("foo")); + /// + /// assert_eq!(Some(PatternID::must(1)), caps[3].pattern()); + /// assert_eq!(Some(Span::from(7..8)), caps[3].get_group_by_name("foo")); + /// + /// assert_eq!(Some(PatternID::must(1)), caps[4].pattern()); + /// assert_eq!(Some(Span::from(8..9)), caps[4].get_group_by_name("foo")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn add_capture_start( + &mut self, + next: StateID, + group_index: u32, + name: Option<Arc<str>>, + ) -> Result<StateID, BuildError> { + let pid = self.current_pattern_id(); + let group_index = match SmallIndex::try_from(group_index) { + Err(_) => { + return Err(BuildError::invalid_capture_index(group_index)) + } + Ok(group_index) => group_index, + }; + // Make sure we have space to insert our (pid,index)|-->name mapping. + if pid.as_usize() >= self.captures.len() { + for _ in 0..=(pid.as_usize() - self.captures.len()) { + self.captures.push(vec![]); + } + } + // In the case where 'group_index < self.captures[pid].len()', it means + // that we are adding a duplicate capture group. This is somewhat + // weird, but permissible because the capture group itself can be + // repeated in the syntax. For example, '([a-z]){4}' will produce 4 + // capture groups. In practice, only the last will be set at search + // time when a match occurs. For duplicates, we don't need to push + // anything other than a CaptureStart NFA state. + if group_index.as_usize() >= self.captures[pid].len() { + // For discontiguous indices, push placeholders for earlier capture + // groups that weren't explicitly added. + for _ in 0..(group_index.as_usize() - self.captures[pid].len()) { + self.captures[pid].push(None); + } + self.captures[pid].push(name); + } + self.add(State::CaptureStart { pattern_id: pid, group_index, next }) + } + + /// Add a "end capture" NFA state. + /// + /// A "end capture" NFA state corresponds to a state with exactly one + /// outgoing unconditional epsilon transition to another state. Unlike + /// "empty" states, a "end capture" state also carries with it an + /// instruction for saving the current position of input to a particular + /// location in memory. NFA simulations, like the Pike VM, may use this + /// information to report the match locations of capturing groups in a + /// + /// Callers may provide a "dummy" state ID (typically [`StateID::ZERO`]), + /// and then change it later with [`patch`](Builder::patch). + /// + /// Note that unlike `start_pattern`/`finish_pattern`, capturing start and + /// end states may be interleaved. Indeed, it is typical for many "start + /// capture" NFA states to appear before the first "end capture" state. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded or if the given + /// capture index overflows `usize`. + /// + /// While the above are the only conditions in which this routine can + /// currently return an error, it is possible to call this method with an + /// inputs that results in the final `build()` step failing to produce an + /// NFA. For example, if one adds two distinct capturing groups with the + /// same name, then that will result in `build()` failing with an error. + /// + /// See the [`GroupInfo`](crate::util::captures::GroupInfo) type for + /// more information on what qualifies as valid capturing groups. + pub fn add_capture_end( + &mut self, + next: StateID, + group_index: u32, + ) -> Result<StateID, BuildError> { + let pid = self.current_pattern_id(); + let group_index = match SmallIndex::try_from(group_index) { + Err(_) => { + return Err(BuildError::invalid_capture_index(group_index)) + } + Ok(group_index) => group_index, + }; + self.add(State::CaptureEnd { pattern_id: pid, group_index, next }) + } + + /// Adds a "fail" NFA state. + /// + /// A "fail" state is simply a state that has no outgoing transitions. It + /// acts as a way to cause a search to stop without reporting a match. + /// For example, one way to represent an NFA with zero patterns is with a + /// single "fail" state. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded. + pub fn add_fail(&mut self) -> Result<StateID, BuildError> { + self.add(State::Fail) + } + + /// Adds a "match" NFA state. + /// + /// A "match" state has no outgoing transitions (just like a "fail" + /// state), but it has special significance in that if a search enters + /// this state, then a match has been found. The match state that is added + /// automatically has the current pattern ID associated with it. This is + /// used to report the matching pattern ID at search time. + /// + /// # Errors + /// + /// This returns an error if the state identifier space is exhausted, or if + /// the configured heap size limit has been exceeded. + /// + /// # Panics + /// + /// This must be called after a `start_pattern` call but before the + /// corresponding `finish_pattern` call. Otherwise, it panics. + pub fn add_match(&mut self) -> Result<StateID, BuildError> { + let pattern_id = self.current_pattern_id(); + let sid = self.add(State::Match { pattern_id })?; + Ok(sid) + } + + /// The common implementation of "add a state." It handles the common + /// error cases of state ID exhausting (by owning state ID allocation) and + /// whether the size limit has been exceeded. + fn add(&mut self, state: State) -> Result<StateID, BuildError> { + let id = StateID::new(self.states.len()) + .map_err(|_| BuildError::too_many_states(self.states.len()))?; + self.memory_states += state.memory_usage(); + self.states.push(state); + self.check_size_limit()?; + Ok(id) + } + + /// Add a transition from one state to another. + /// + /// This routine is called "patch" since it is very common to add the + /// states you want, typically with "dummy" state ID transitions, and then + /// "patch" in the real state IDs later. This is because you don't always + /// know all of the necessary state IDs to add because they might not + /// exist yet. + /// + /// # Errors + /// + /// This may error if patching leads to an increase in heap usage beyond + /// the configured size limit. Heap usage only grows when patching adds a + /// new transition (as in the case of a "union" state). + /// + /// # Panics + /// + /// This panics if `from` corresponds to a "sparse" state. When "sparse" + /// states are added, there is no way to patch them after-the-fact. (If you + /// have a use case where this would be helpful, please file an issue. It + /// will likely require a new API.) + pub fn patch( + &mut self, + from: StateID, + to: StateID, + ) -> Result<(), BuildError> { + let old_memory_states = self.memory_states; + match self.states[from] { + State::Empty { ref mut next } => { + *next = to; + } + State::ByteRange { ref mut trans } => { + trans.next = to; + } + State::Sparse { .. } => { + panic!("cannot patch from a sparse NFA state") + } + State::Look { ref mut next, .. } => { + *next = to; + } + State::Union { ref mut alternates } => { + alternates.push(to); + self.memory_states += mem::size_of::<StateID>(); + } + State::UnionReverse { ref mut alternates } => { + alternates.push(to); + self.memory_states += mem::size_of::<StateID>(); + } + State::CaptureStart { ref mut next, .. } => { + *next = to; + } + State::CaptureEnd { ref mut next, .. } => { + *next = to; + } + State::Fail => {} + State::Match { .. } => {} + } + if old_memory_states != self.memory_states { + self.check_size_limit()?; + } + Ok(()) + } + + /// Set whether the NFA produced by this builder should only match UTF-8. + /// + /// This should be set when both of the following are true: + /// + /// 1. The caller guarantees that the NFA created by this build will only + /// report non-empty matches with spans that are valid UTF-8. + /// 2. The caller desires regex engines using this NFA to avoid reporting + /// empty matches with a span that splits a valid UTF-8 encoded codepoint. + /// + /// Property (1) is not checked. Instead, this requires the caller to + /// promise that it is true. Property (2) corresponds to the behavior of + /// regex engines using the NFA created by this builder. Namely, there + /// is no way in the NFA's graph itself to say that empty matches found + /// by, for example, the regex `a*` will fall on valid UTF-8 boundaries. + /// Instead, this option is used to communicate the UTF-8 semantic to regex + /// engines that will typically implement it as a post-processing step by + /// filtering out empty matches that don't fall on UTF-8 boundaries. + /// + /// If you're building an NFA from an HIR (and not using a + /// [`thompson::Compiler`](crate::nfa::thompson::Compiler)), then you can + /// use the [`syntax::Config::utf8`](crate::util::syntax::Config::utf8) + /// option to guarantee that if the HIR detects a non-empty match, then it + /// is guaranteed to be valid UTF-8. + /// + /// Note that property (2) does *not* specify the behavior of executing + /// a search on a haystack that is not valid UTF-8. Therefore, if you're + /// *not* running this NFA on strings that are guaranteed to be valid + /// UTF-8, you almost certainly do not want to enable this option. + /// Similarly, if you are running the NFA on strings that *are* guaranteed + /// to be valid UTF-8, then you almost certainly want to enable this option + /// unless you can guarantee that your NFA will never produce a zero-width + /// match. + /// + /// It is disabled by default. + pub fn set_utf8(&mut self, yes: bool) { + self.utf8 = yes; + } + + /// Returns whether UTF-8 mode is enabled for this builder. + /// + /// See [`Builder::set_utf8`] for more details about what "UTF-8 mode" is. + pub fn get_utf8(&self) -> bool { + self.utf8 + } + + /// Sets whether the NFA produced by this builder should be matched in + /// reverse or not. Generally speaking, when enabled, the NFA produced + /// should be matched by moving backwards through a haystack, from a higher + /// memory address to a lower memory address. + /// + /// See also [`NFA::is_reverse`] for more details. + /// + /// This is disabled by default, which means NFAs are by default matched + /// in the forward direction. + pub fn set_reverse(&mut self, yes: bool) { + self.reverse = yes; + } + + /// Returns whether reverse mode is enabled for this builder. + /// + /// See [`Builder::set_reverse`] for more details about what "reverse mode" + /// is. + pub fn get_reverse(&self) -> bool { + self.reverse + } + + /// Sets the look-around matcher that should be used for the resulting NFA. + /// + /// A look-around matcher can be used to configure how look-around + /// assertions are matched. For example, a matcher might carry + /// configuration that changes the line terminator used for `(?m:^)` and + /// `(?m:$)` assertions. + pub fn set_look_matcher(&mut self, m: LookMatcher) { + self.look_matcher = m; + } + + /// Returns the look-around matcher used for this builder. + /// + /// If a matcher was not explicitly set, then `LookMatcher::default()` is + /// returned. + pub fn get_look_matcher(&self) -> &LookMatcher { + &self.look_matcher + } + + /// Set the size limit on this builder. + /// + /// Setting the size limit will also check whether the NFA built so far + /// fits within the given size limit. If it doesn't, then an error is + /// returned. + /// + /// By default, there is no configured size limit. + pub fn set_size_limit( + &mut self, + limit: Option<usize>, + ) -> Result<(), BuildError> { + self.size_limit = limit; + self.check_size_limit() + } + + /// Return the currently configured size limit. + /// + /// By default, this returns `None`, which corresponds to no configured + /// size limit. + pub fn get_size_limit(&self) -> Option<usize> { + self.size_limit + } + + /// Returns the heap memory usage, in bytes, used by the NFA states added + /// so far. + /// + /// Note that this is an approximation of how big the final NFA will be. + /// In practice, the final NFA will likely be a bit smaller because of + /// its simpler state representation. (For example, using things like + /// `Box<[StateID]>` instead of `Vec<StateID>`.) + pub fn memory_usage(&self) -> usize { + self.states.len() * mem::size_of::<State>() + self.memory_states + } + + fn check_size_limit(&self) -> Result<(), BuildError> { + if let Some(limit) = self.size_limit { + if self.memory_usage() > limit { + return Err(BuildError::exceeded_size_limit(limit)); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // This asserts that a builder state doesn't have its size changed. It is + // *really* easy to accidentally increase the size, and thus potentially + // dramatically increase the memory usage of NFA builder. + // + // This assert doesn't mean we absolutely cannot increase the size of a + // builder state. We can. It's just here to make sure we do it knowingly + // and intentionally. + // + // A builder state is unfortunately a little bigger than an NFA state, + // since we really want to support adding things to a pre-existing state. + // i.e., We use Vec<thing> instead of Box<[thing]>. So we end up using an + // extra 8 bytes per state. Sad, but at least it gets freed once the NFA + // is built. + #[test] + fn state_has_small_size() { + #[cfg(target_pointer_width = "64")] + assert_eq!(32, core::mem::size_of::<State>()); + #[cfg(target_pointer_width = "32")] + assert_eq!(16, core::mem::size_of::<State>()); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/compiler.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/compiler.rs new file mode 100644 index 0000000..96a39ac --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/compiler.rs @@ -0,0 +1,2368 @@ +use core::{borrow::Borrow, cell::RefCell}; + +use alloc::{sync::Arc, vec, vec::Vec}; + +use regex_syntax::{ + hir::{self, Hir}, + utf8::{Utf8Range, Utf8Sequences}, + ParserBuilder, +}; + +use crate::{ + nfa::thompson::{ + builder::Builder, + error::BuildError, + literal_trie::LiteralTrie, + map::{Utf8BoundedMap, Utf8SuffixKey, Utf8SuffixMap}, + nfa::{Transition, NFA}, + range_trie::RangeTrie, + }, + util::{ + look::{Look, LookMatcher}, + primitives::{PatternID, StateID}, + }, +}; + +/// The configuration used for a Thompson NFA compiler. +#[derive(Clone, Debug, Default)] +pub struct Config { + utf8: Option<bool>, + reverse: Option<bool>, + nfa_size_limit: Option<Option<usize>>, + shrink: Option<bool>, + which_captures: Option<WhichCaptures>, + look_matcher: Option<LookMatcher>, + #[cfg(test)] + unanchored_prefix: Option<bool>, +} + +impl Config { + /// Return a new default Thompson NFA compiler configuration. + pub fn new() -> Config { + Config::default() + } + + /// Whether to enable UTF-8 mode during search or not. + /// + /// A regex engine is said to be in UTF-8 mode when it guarantees that + /// all matches returned by it have spans consisting of only valid UTF-8. + /// That is, it is impossible for a match span to be returned that + /// contains any invalid UTF-8. + /// + /// UTF-8 mode generally consists of two things: + /// + /// 1. Whether the NFA's states are constructed such that all paths to a + /// match state that consume at least one byte always correspond to valid + /// UTF-8. + /// 2. Whether all paths to a match state that do _not_ consume any bytes + /// should always correspond to valid UTF-8 boundaries. + /// + /// (1) is a guarantee made by whoever constructs the NFA. + /// If you're parsing a regex from its concrete syntax, then + /// [`syntax::Config::utf8`](crate::util::syntax::Config::utf8) can make + /// this guarantee for you. It does it by returning an error if the regex + /// pattern could every report a non-empty match span that contains invalid + /// UTF-8. So long as `syntax::Config::utf8` mode is enabled and your regex + /// successfully parses, then you're guaranteed that the corresponding NFA + /// will only ever report non-empty match spans containing valid UTF-8. + /// + /// (2) is a trickier guarantee because it cannot be enforced by the NFA + /// state graph itself. Consider, for example, the regex `a*`. It matches + /// the empty strings in `☃` at positions `0`, `1`, `2` and `3`, where + /// positions `1` and `2` occur within the UTF-8 encoding of a codepoint, + /// and thus correspond to invalid UTF-8 boundaries. Therefore, this + /// guarantee must be made at a higher level than the NFA state graph + /// itself. This crate deals with this case in each regex engine. Namely, + /// when a zero-width match that splits a codepoint is found and UTF-8 + /// mode enabled, then it is ignored and the engine moves on looking for + /// the next match. + /// + /// Thus, UTF-8 mode is both a promise that the NFA built only reports + /// non-empty matches that are valid UTF-8, and an *instruction* to regex + /// engines that empty matches that split codepoints should be banned. + /// + /// Because UTF-8 mode is fundamentally about avoiding invalid UTF-8 spans, + /// it only makes sense to enable this option when you *know* your haystack + /// is valid UTF-8. (For example, a `&str`.) Enabling UTF-8 mode and + /// searching a haystack that contains invalid UTF-8 leads to **unspecified + /// behavior**. + /// + /// Therefore, it may make sense to enable `syntax::Config::utf8` while + /// simultaneously *disabling* this option. That would ensure all non-empty + /// match spans are valid UTF-8, but that empty match spans may still split + /// a codepoint or match at other places that aren't valid UTF-8. + /// + /// In general, this mode is only relevant if your regex can match the + /// empty string. Most regexes don't. + /// + /// This is enabled by default. + /// + /// # Example + /// + /// This example shows how UTF-8 mode can impact the match spans that may + /// be reported in certain cases. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{self, pikevm::PikeVM}, + /// Match, Input, + /// }; + /// + /// let re = PikeVM::new("")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// // UTF-8 mode is enabled by default. + /// let mut input = Input::new("☃"); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 0..0)), caps.get_match()); + /// + /// // Even though an empty regex matches at 1..1, our next match is + /// // 3..3 because 1..1 and 2..2 split the snowman codepoint (which is + /// // three bytes long). + /// input.set_start(1); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 3..3)), caps.get_match()); + /// + /// // But if we disable UTF-8, then we'll get matches at 1..1 and 2..2: + /// let re = PikeVM::builder() + /// .thompson(thompson::Config::new().utf8(false)) + /// .build("")?; + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 1..1)), caps.get_match()); + /// + /// input.set_start(2); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 2..2)), caps.get_match()); + /// + /// input.set_start(3); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 3..3)), caps.get_match()); + /// + /// input.set_start(4); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(None, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn utf8(mut self, yes: bool) -> Config { + self.utf8 = Some(yes); + self + } + + /// Reverse the NFA. + /// + /// A NFA reversal is performed by reversing all of the concatenated + /// sub-expressions in the original pattern, recursively. (Look around + /// operators are also inverted.) The resulting NFA can be used to match + /// the pattern starting from the end of a string instead of the beginning + /// of a string. + /// + /// Reversing the NFA is useful for building a reverse DFA, which is most + /// useful for finding the start of a match after its ending position has + /// been found. NFA execution engines typically do not work on reverse + /// NFAs. For example, currently, the Pike VM reports the starting location + /// of matches without a reverse NFA. + /// + /// Currently, enabling this setting requires disabling the + /// [`captures`](Config::captures) setting. If both are enabled, then the + /// compiler will return an error. It is expected that this limitation will + /// be lifted in the future. + /// + /// This is disabled by default. + /// + /// # Example + /// + /// This example shows how to build a DFA from a reverse NFA, and then use + /// the DFA to search backwards. + /// + /// ``` + /// use regex_automata::{ + /// dfa::{self, Automaton}, + /// nfa::thompson::{NFA, WhichCaptures}, + /// HalfMatch, Input, + /// }; + /// + /// let dfa = dfa::dense::Builder::new() + /// .thompson(NFA::config() + /// .which_captures(WhichCaptures::None) + /// .reverse(true) + /// ) + /// .build("baz[0-9]+")?; + /// let expected = Some(HalfMatch::must(0, 3)); + /// assert_eq!( + /// expected, + /// dfa.try_search_rev(&Input::new("foobaz12345bar"))?, + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reverse(mut self, yes: bool) -> Config { + self.reverse = Some(yes); + self + } + + /// Sets an approximate size limit on the total heap used by the NFA being + /// compiled. + /// + /// This permits imposing constraints on the size of a compiled NFA. This + /// may be useful in contexts where the regex pattern is untrusted and one + /// wants to avoid using too much memory. + /// + /// This size limit does not apply to auxiliary heap used during + /// compilation that is not part of the built NFA. + /// + /// Note that this size limit is applied during compilation in order for + /// the limit to prevent too much heap from being used. However, the + /// implementation may use an intermediate NFA representation that is + /// otherwise slightly bigger than the final public form. Since the size + /// limit may be applied to an intermediate representation, there is not + /// necessarily a precise correspondence between the configured size limit + /// and the heap usage of the final NFA. + /// + /// There is no size limit by default. + /// + /// # Example + /// + /// This example demonstrates how Unicode mode can greatly increase the + /// size of the NFA. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::nfa::thompson::NFA; + /// + /// // 300KB isn't enough! + /// NFA::compiler() + /// .configure(NFA::config().nfa_size_limit(Some(300_000))) + /// .build(r"\w{20}") + /// .unwrap_err(); + /// + /// // ... but 500KB probably is. + /// let nfa = NFA::compiler() + /// .configure(NFA::config().nfa_size_limit(Some(500_000))) + /// .build(r"\w{20}")?; + /// + /// assert_eq!(nfa.pattern_len(), 1); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn nfa_size_limit(mut self, bytes: Option<usize>) -> Config { + self.nfa_size_limit = Some(bytes); + self + } + + /// Apply best effort heuristics to shrink the NFA at the expense of more + /// time/memory. + /// + /// Generally speaking, if one is using an NFA to compile a DFA, then the + /// extra time used to shrink the NFA will be more than made up for during + /// DFA construction (potentially by a lot). In other words, enabling this + /// can substantially decrease the overall amount of time it takes to build + /// a DFA. + /// + /// A reason to keep this disabled is if you want to compile an NFA and + /// start using it as quickly as possible without needing to build a DFA, + /// and you don't mind using a bit of extra memory for the NFA. e.g., for + /// an NFA simulation or for a lazy DFA. + /// + /// NFA shrinking is currently most useful when compiling a reverse + /// NFA with large Unicode character classes. In particular, it trades + /// additional CPU time during NFA compilation in favor of generating fewer + /// NFA states. + /// + /// This is disabled by default because it can increase compile times + /// quite a bit if you aren't building a full DFA. + /// + /// # Example + /// + /// This example shows that NFA shrinking can lead to substantial space + /// savings in some cases. Notice that, as noted above, we build a reverse + /// DFA and use a pattern with a large Unicode character class. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::nfa::thompson::{NFA, WhichCaptures}; + /// + /// // Currently we have to disable captures when enabling reverse NFA. + /// let config = NFA::config() + /// .which_captures(WhichCaptures::None) + /// .reverse(true); + /// let not_shrunk = NFA::compiler() + /// .configure(config.clone().shrink(false)) + /// .build(r"\w")?; + /// let shrunk = NFA::compiler() + /// .configure(config.clone().shrink(true)) + /// .build(r"\w")?; + /// + /// // While a specific shrink factor is not guaranteed, the savings can be + /// // considerable in some cases. + /// assert!(shrunk.states().len() * 2 < not_shrunk.states().len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn shrink(mut self, yes: bool) -> Config { + self.shrink = Some(yes); + self + } + + /// Whether to include 'Capture' states in the NFA. + /// + /// Currently, enabling this setting requires disabling the + /// [`reverse`](Config::reverse) setting. If both are enabled, then the + /// compiler will return an error. It is expected that this limitation will + /// be lifted in the future. + /// + /// This is enabled by default. + /// + /// # Example + /// + /// This example demonstrates that some regex engines, like the Pike VM, + /// require capturing states to be present in the NFA to report match + /// offsets. + /// + /// (Note that since this method is deprecated, the example below uses + /// [`Config::which_captures`] to disable capture states.) + /// + /// ``` + /// use regex_automata::nfa::thompson::{ + /// pikevm::PikeVM, + /// NFA, + /// WhichCaptures, + /// }; + /// + /// let re = PikeVM::builder() + /// .thompson(NFA::config().which_captures(WhichCaptures::None)) + /// .build(r"[a-z]+")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.is_match(&mut cache, "abc")); + /// assert_eq!(None, re.find(&mut cache, "abc")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[deprecated(since = "0.3.5", note = "use which_captures instead")] + pub fn captures(self, yes: bool) -> Config { + self.which_captures(if yes { + WhichCaptures::All + } else { + WhichCaptures::None + }) + } + + /// Configures what kinds of capture groups are compiled into + /// [`State::Capture`](crate::nfa::thompson::State::Capture) states in a + /// Thompson NFA. + /// + /// Currently, using any option except for [`WhichCaptures::None`] requires + /// disabling the [`reverse`](Config::reverse) setting. If both are + /// enabled, then the compiler will return an error. It is expected that + /// this limitation will be lifted in the future. + /// + /// This is set to [`WhichCaptures::All`] by default. Callers may wish to + /// use [`WhichCaptures::Implicit`] in cases where one wants avoid the + /// overhead of capture states for explicit groups. Usually this occurs + /// when one wants to use the `PikeVM` only for determining the overall + /// match. Otherwise, the `PikeVM` could use much more memory than is + /// necessary. + /// + /// # Example + /// + /// This example demonstrates that some regex engines, like the Pike VM, + /// require capturing states to be present in the NFA to report match + /// offsets. + /// + /// ``` + /// use regex_automata::nfa::thompson::{ + /// pikevm::PikeVM, + /// NFA, + /// WhichCaptures, + /// }; + /// + /// let re = PikeVM::builder() + /// .thompson(NFA::config().which_captures(WhichCaptures::None)) + /// .build(r"[a-z]+")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.is_match(&mut cache, "abc")); + /// assert_eq!(None, re.find(&mut cache, "abc")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// The same applies to the bounded backtracker: + /// + /// ``` + /// use regex_automata::nfa::thompson::{ + /// backtrack::BoundedBacktracker, + /// NFA, + /// WhichCaptures, + /// }; + /// + /// let re = BoundedBacktracker::builder() + /// .thompson(NFA::config().which_captures(WhichCaptures::None)) + /// .build(r"[a-z]+")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.try_is_match(&mut cache, "abc")?); + /// assert_eq!(None, re.try_find(&mut cache, "abc")?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn which_captures(mut self, which_captures: WhichCaptures) -> Config { + self.which_captures = Some(which_captures); + self + } + + /// Sets the look-around matcher that should be used with this NFA. + /// + /// A look-around matcher determines how to match look-around assertions. + /// In particular, some assertions are configurable. For example, the + /// `(?m:^)` and `(?m:$)` assertions can have their line terminator changed + /// from the default of `\n` to any other byte. + /// + /// # Example + /// + /// This shows how to change the line terminator for multi-line assertions. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{self, pikevm::PikeVM}, + /// util::look::LookMatcher, + /// Match, Input, + /// }; + /// + /// let mut lookm = LookMatcher::new(); + /// lookm.set_line_terminator(b'\x00'); + /// + /// let re = PikeVM::builder() + /// .thompson(thompson::Config::new().look_matcher(lookm)) + /// .build(r"(?m)^[a-z]+$")?; + /// let mut cache = re.create_cache(); + /// + /// // Multi-line assertions now use NUL as a terminator. + /// assert_eq!( + /// Some(Match::must(0, 1..4)), + /// re.find(&mut cache, b"\x00abc\x00"), + /// ); + /// // ... and \n is no longer recognized as a terminator. + /// assert_eq!( + /// None, + /// re.find(&mut cache, b"\nabc\n"), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn look_matcher(mut self, m: LookMatcher) -> Config { + self.look_matcher = Some(m); + self + } + + /// Whether to compile an unanchored prefix into this NFA. + /// + /// This is enabled by default. It is made available for tests only to make + /// it easier to unit test the output of the compiler. + #[cfg(test)] + fn unanchored_prefix(mut self, yes: bool) -> Config { + self.unanchored_prefix = Some(yes); + self + } + + /// Returns whether this configuration has enabled UTF-8 mode. + pub fn get_utf8(&self) -> bool { + self.utf8.unwrap_or(true) + } + + /// Returns whether this configuration has enabled reverse NFA compilation. + pub fn get_reverse(&self) -> bool { + self.reverse.unwrap_or(false) + } + + /// Return the configured NFA size limit, if it exists, in the number of + /// bytes of heap used. + pub fn get_nfa_size_limit(&self) -> Option<usize> { + self.nfa_size_limit.unwrap_or(None) + } + + /// Return whether NFA shrinking is enabled. + pub fn get_shrink(&self) -> bool { + self.shrink.unwrap_or(false) + } + + /// Return whether NFA compilation is configured to produce capture states. + #[deprecated(since = "0.3.5", note = "use get_which_captures instead")] + pub fn get_captures(&self) -> bool { + self.get_which_captures().is_any() + } + + /// Return what kinds of capture states will be compiled into an NFA. + pub fn get_which_captures(&self) -> WhichCaptures { + self.which_captures.unwrap_or(WhichCaptures::All) + } + + /// Return the look-around matcher for this NFA. + pub fn get_look_matcher(&self) -> LookMatcher { + self.look_matcher.clone().unwrap_or(LookMatcher::default()) + } + + /// Return whether NFA compilation is configured to include an unanchored + /// prefix. + /// + /// This is always false when not in test mode. + fn get_unanchored_prefix(&self) -> bool { + #[cfg(test)] + { + self.unanchored_prefix.unwrap_or(true) + } + #[cfg(not(test))] + { + true + } + } + + /// Overwrite the default configuration such that the options in `o` are + /// always used. If an option in `o` is not set, then the corresponding + /// option in `self` is used. If it's not set in `self` either, then it + /// remains not set. + pub(crate) fn overwrite(&self, o: Config) -> Config { + Config { + utf8: o.utf8.or(self.utf8), + reverse: o.reverse.or(self.reverse), + nfa_size_limit: o.nfa_size_limit.or(self.nfa_size_limit), + shrink: o.shrink.or(self.shrink), + which_captures: o.which_captures.or(self.which_captures), + look_matcher: o.look_matcher.or_else(|| self.look_matcher.clone()), + #[cfg(test)] + unanchored_prefix: o.unanchored_prefix.or(self.unanchored_prefix), + } + } +} + +/// A configuration indicating which kinds of +/// [`State::Capture`](crate::nfa::thompson::State::Capture) states to include. +/// +/// This configuration can be used with [`Config::which_captures`] to control +/// which capture states are compiled into a Thompson NFA. +/// +/// The default configuration is [`WhichCaptures::All`]. +#[derive(Clone, Copy, Debug)] +pub enum WhichCaptures { + /// All capture states, including those corresponding to both implicit and + /// explicit capture groups, are included in the Thompson NFA. + All, + /// Only capture states corresponding to implicit capture groups are + /// included. Implicit capture groups appear in every pattern implicitly + /// and correspond to the overall match of a pattern. + /// + /// This is useful when one only cares about the overall match of a + /// pattern. By excluding capture states from explicit capture groups, + /// one might be able to reduce the memory usage of a multi-pattern regex + /// substantially if it was otherwise written to have many explicit capture + /// groups. + Implicit, + /// No capture states are compiled into the Thompson NFA. + /// + /// This is useful when capture states are either not needed (for example, + /// if one is only trying to build a DFA) or if they aren't supported (for + /// example, a reverse NFA). + /// + /// # Warning + /// + /// Callers must be exceedingly careful when using this + /// option. In particular, not all regex engines support + /// reporting match spans when using this option (for example, + /// [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM) or + /// [`BoundedBacktracker`](crate::nfa::thompson::backtrack::BoundedBacktracker)). + /// + /// Perhaps more confusingly, using this option with such an + /// engine means that an `is_match` routine could report `true` + /// when `find` reports `None`. This is generally not something + /// that _should_ happen, but the low level control provided by + /// this crate makes it possible. + /// + /// Similarly, any regex engines (like [`meta::Regex`](crate::meta::Regex)) + /// should always return `None` from `find` routines when this option is + /// used, even if _some_ of its internal engines could find the match + /// boundaries. This is because inputs from user data could influence + /// engine selection, and thus influence whether a match is found or not. + /// Indeed, `meta::Regex::find` will always return `None` when configured + /// with this option. + None, +} + +impl Default for WhichCaptures { + fn default() -> WhichCaptures { + WhichCaptures::All + } +} + +impl WhichCaptures { + /// Returns true if this configuration indicates that no capture states + /// should be produced in an NFA. + pub fn is_none(&self) -> bool { + matches!(*self, WhichCaptures::None) + } + + /// Returns true if this configuration indicates that some capture states + /// should be added to an NFA. Note that this might only include capture + /// states for implicit capture groups. + pub fn is_any(&self) -> bool { + !self.is_none() + } +} + +/* +This compiler below uses Thompson's construction algorithm. The compiler takes +a regex-syntax::Hir as input and emits an NFA graph as output. The NFA graph +is structured in a way that permits it to be executed by a virtual machine and +also used to efficiently build a DFA. + +The compiler deals with a slightly expanded set of NFA states than what is +in a final NFA (as exhibited by builder::State and nfa::State). Notably a +compiler state includes an empty node that has exactly one unconditional +epsilon transition to the next state. In other words, it's a "goto" instruction +if one views Thompson's NFA as a set of bytecode instructions. These goto +instructions are removed in a subsequent phase before returning the NFA to the +caller. The purpose of these empty nodes is that they make the construction +algorithm substantially simpler to implement. We remove them before returning +to the caller because they can represent substantial overhead when traversing +the NFA graph (either while searching using the NFA directly or while building +a DFA). + +In the future, it would be nice to provide a Glushkov compiler as well, as it +would work well as a bit-parallel NFA for smaller regexes. But the Thompson +construction is one I'm more familiar with and seems more straight-forward to +deal with when it comes to large Unicode character classes. + +Internally, the compiler uses interior mutability to improve composition in the +face of the borrow checker. In particular, we'd really like to be able to write +things like this: + + self.c_concat(exprs.iter().map(|e| self.c(e))) + +Which elegantly uses iterators to build up a sequence of compiled regex +sub-expressions and then hands it off to the concatenating compiler routine. +Without interior mutability, the borrow checker won't let us borrow `self` +mutably both inside and outside the closure at the same time. +*/ + +/// A builder for compiling an NFA from a regex's high-level intermediate +/// representation (HIR). +/// +/// This compiler provides a way to translate a parsed regex pattern into an +/// NFA state graph. The NFA state graph can either be used directly to execute +/// a search (e.g., with a Pike VM), or it can be further used to build a DFA. +/// +/// This compiler provides APIs both for compiling regex patterns directly from +/// their concrete syntax, or via a [`regex_syntax::hir::Hir`]. +/// +/// This compiler has various options that may be configured via +/// [`thompson::Config`](Config). +/// +/// Note that a compiler is not the same as a [`thompson::Builder`](Builder). +/// A `Builder` provides a lower level API that is uncoupled from a regex +/// pattern's concrete syntax or even its HIR. Instead, it permits stitching +/// together an NFA by hand. See its docs for examples. +/// +/// # Example: compilation from concrete syntax +/// +/// This shows how to compile an NFA from a pattern string while setting a size +/// limit on how big the NFA is allowed to be (in terms of bytes of heap used). +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::{NFA, pikevm::PikeVM}, +/// Match, +/// }; +/// +/// let config = NFA::config().nfa_size_limit(Some(1_000)); +/// let nfa = NFA::compiler().configure(config).build(r"(?-u)\w")?; +/// +/// let re = PikeVM::new_from_nfa(nfa)?; +/// let mut cache = re.create_cache(); +/// let mut caps = re.create_captures(); +/// let expected = Some(Match::must(0, 3..4)); +/// re.captures(&mut cache, "!@#A#@!", &mut caps); +/// assert_eq!(expected, caps.get_match()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: compilation from HIR +/// +/// This shows how to hand assemble a regular expression via its HIR, and then +/// compile an NFA directly from it. +/// +/// ``` +/// use regex_automata::{nfa::thompson::{NFA, pikevm::PikeVM}, Match}; +/// use regex_syntax::hir::{Hir, Class, ClassBytes, ClassBytesRange}; +/// +/// let hir = Hir::class(Class::Bytes(ClassBytes::new(vec![ +/// ClassBytesRange::new(b'0', b'9'), +/// ClassBytesRange::new(b'A', b'Z'), +/// ClassBytesRange::new(b'_', b'_'), +/// ClassBytesRange::new(b'a', b'z'), +/// ]))); +/// +/// let config = NFA::config().nfa_size_limit(Some(1_000)); +/// let nfa = NFA::compiler().configure(config).build_from_hir(&hir)?; +/// +/// let re = PikeVM::new_from_nfa(nfa)?; +/// let mut cache = re.create_cache(); +/// let mut caps = re.create_captures(); +/// let expected = Some(Match::must(0, 3..4)); +/// re.captures(&mut cache, "!@#A#@!", &mut caps); +/// assert_eq!(expected, caps.get_match()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Compiler { + /// A regex parser, used when compiling an NFA directly from a pattern + /// string. + parser: ParserBuilder, + /// The compiler configuration. + config: Config, + /// The builder for actually constructing an NFA. This provides a + /// convenient abstraction for writing a compiler. + builder: RefCell<Builder>, + /// State used for compiling character classes to UTF-8 byte automata. + /// State is not retained between character class compilations. This just + /// serves to amortize allocation to the extent possible. + utf8_state: RefCell<Utf8State>, + /// State used for arranging character classes in reverse into a trie. + trie_state: RefCell<RangeTrie>, + /// State used for caching common suffixes when compiling reverse UTF-8 + /// automata (for Unicode character classes). + utf8_suffix: RefCell<Utf8SuffixMap>, +} + +impl Compiler { + /// Create a new NFA builder with its default configuration. + pub fn new() -> Compiler { + Compiler { + parser: ParserBuilder::new(), + config: Config::default(), + builder: RefCell::new(Builder::new()), + utf8_state: RefCell::new(Utf8State::new()), + trie_state: RefCell::new(RangeTrie::new()), + utf8_suffix: RefCell::new(Utf8SuffixMap::new(1000)), + } + } + + /// Compile the given regular expression pattern into an NFA. + /// + /// If there was a problem parsing the regex, then that error is returned. + /// + /// Otherwise, if there was a problem building the NFA, then an error is + /// returned. The only error that can occur is if the compiled regex would + /// exceed the size limits configured on this builder, or if any part of + /// the NFA would exceed the integer representations used. (For example, + /// too many states might plausibly occur on a 16-bit target.) + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, pikevm::PikeVM}, Match}; + /// + /// let config = NFA::config().nfa_size_limit(Some(1_000)); + /// let nfa = NFA::compiler().configure(config).build(r"(?-u)\w")?; + /// + /// let re = PikeVM::new_from_nfa(nfa)?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// let expected = Some(Match::must(0, 3..4)); + /// re.captures(&mut cache, "!@#A#@!", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build(&self, pattern: &str) -> Result<NFA, BuildError> { + self.build_many(&[pattern]) + } + + /// Compile the given regular expression patterns into a single NFA. + /// + /// When matches are returned, the pattern ID corresponds to the index of + /// the pattern in the slice given. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, pikevm::PikeVM}, Match}; + /// + /// let config = NFA::config().nfa_size_limit(Some(1_000)); + /// let nfa = NFA::compiler().configure(config).build_many(&[ + /// r"(?-u)\s", + /// r"(?-u)\w", + /// ])?; + /// + /// let re = PikeVM::new_from_nfa(nfa)?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// let expected = Some(Match::must(1, 1..2)); + /// re.captures(&mut cache, "!A! !A!", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<NFA, BuildError> { + let mut hirs = vec![]; + for p in patterns { + hirs.push( + self.parser + .build() + .parse(p.as_ref()) + .map_err(BuildError::syntax)?, + ); + debug!("parsed: {:?}", p.as_ref()); + } + self.build_many_from_hir(&hirs) + } + + /// Compile the given high level intermediate representation of a regular + /// expression into an NFA. + /// + /// If there was a problem building the NFA, then an error is returned. The + /// only error that can occur is if the compiled regex would exceed the + /// size limits configured on this builder, or if any part of the NFA would + /// exceed the integer representations used. (For example, too many states + /// might plausibly occur on a 16-bit target.) + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, pikevm::PikeVM}, Match}; + /// use regex_syntax::hir::{Hir, Class, ClassBytes, ClassBytesRange}; + /// + /// let hir = Hir::class(Class::Bytes(ClassBytes::new(vec![ + /// ClassBytesRange::new(b'0', b'9'), + /// ClassBytesRange::new(b'A', b'Z'), + /// ClassBytesRange::new(b'_', b'_'), + /// ClassBytesRange::new(b'a', b'z'), + /// ]))); + /// + /// let config = NFA::config().nfa_size_limit(Some(1_000)); + /// let nfa = NFA::compiler().configure(config).build_from_hir(&hir)?; + /// + /// let re = PikeVM::new_from_nfa(nfa)?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// let expected = Some(Match::must(0, 3..4)); + /// re.captures(&mut cache, "!@#A#@!", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_from_hir(&self, expr: &Hir) -> Result<NFA, BuildError> { + self.build_many_from_hir(&[expr]) + } + + /// Compile the given high level intermediate representations of regular + /// expressions into a single NFA. + /// + /// When matches are returned, the pattern ID corresponds to the index of + /// the pattern in the slice given. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, pikevm::PikeVM}, Match}; + /// use regex_syntax::hir::{Hir, Class, ClassBytes, ClassBytesRange}; + /// + /// let hirs = &[ + /// Hir::class(Class::Bytes(ClassBytes::new(vec![ + /// ClassBytesRange::new(b'\t', b'\r'), + /// ClassBytesRange::new(b' ', b' '), + /// ]))), + /// Hir::class(Class::Bytes(ClassBytes::new(vec![ + /// ClassBytesRange::new(b'0', b'9'), + /// ClassBytesRange::new(b'A', b'Z'), + /// ClassBytesRange::new(b'_', b'_'), + /// ClassBytesRange::new(b'a', b'z'), + /// ]))), + /// ]; + /// + /// let config = NFA::config().nfa_size_limit(Some(1_000)); + /// let nfa = NFA::compiler().configure(config).build_many_from_hir(hirs)?; + /// + /// let re = PikeVM::new_from_nfa(nfa)?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// let expected = Some(Match::must(1, 1..2)); + /// re.captures(&mut cache, "!A! !A!", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn build_many_from_hir<H: Borrow<Hir>>( + &self, + exprs: &[H], + ) -> Result<NFA, BuildError> { + self.compile(exprs) + } + + /// Apply the given NFA configuration options to this builder. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::NFA; + /// + /// let config = NFA::config().nfa_size_limit(Some(1_000)); + /// let nfa = NFA::compiler().configure(config).build(r"(?-u)\w")?; + /// assert_eq!(nfa.pattern_len(), 1); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn configure(&mut self, config: Config) -> &mut Compiler { + self.config = self.config.overwrite(config); + self + } + + /// Set the syntax configuration for this builder using + /// [`syntax::Config`](crate::util::syntax::Config). + /// + /// This permits setting things like case insensitivity, Unicode and multi + /// line mode. + /// + /// This syntax configuration only applies when an NFA is built directly + /// from a pattern string. If an NFA is built from an HIR, then all syntax + /// settings are ignored. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, util::syntax}; + /// + /// let syntax_config = syntax::Config::new().unicode(false); + /// let nfa = NFA::compiler().syntax(syntax_config).build(r"\w")?; + /// // If Unicode were enabled, the number of states would be much bigger. + /// assert!(nfa.states().len() < 15); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Compiler { + config.apply(&mut self.parser); + self + } +} + +impl Compiler { + /// Compile the sequence of HIR expressions given. Pattern IDs are + /// allocated starting from 0, in correspondence with the slice given. + /// + /// It is legal to provide an empty slice. In that case, the NFA returned + /// has no patterns and will never match anything. + fn compile<H: Borrow<Hir>>(&self, exprs: &[H]) -> Result<NFA, BuildError> { + if exprs.len() > PatternID::LIMIT { + return Err(BuildError::too_many_patterns(exprs.len())); + } + if self.config.get_reverse() + && self.config.get_which_captures().is_any() + { + return Err(BuildError::unsupported_captures()); + } + + self.builder.borrow_mut().clear(); + self.builder.borrow_mut().set_utf8(self.config.get_utf8()); + self.builder.borrow_mut().set_reverse(self.config.get_reverse()); + self.builder + .borrow_mut() + .set_look_matcher(self.config.get_look_matcher()); + self.builder + .borrow_mut() + .set_size_limit(self.config.get_nfa_size_limit())?; + + // We always add an unanchored prefix unless we were specifically told + // not to (for tests only), or if we know that the regex is anchored + // for all matches. When an unanchored prefix is not added, then the + // NFA's anchored and unanchored start states are equivalent. + let all_anchored = exprs.iter().all(|e| { + let props = e.borrow().properties(); + if self.config.get_reverse() { + props.look_set_suffix().contains(hir::Look::End) + } else { + props.look_set_prefix().contains(hir::Look::Start) + } + }); + let anchored = !self.config.get_unanchored_prefix() || all_anchored; + let unanchored_prefix = if anchored { + self.c_empty()? + } else { + self.c_at_least(&Hir::dot(hir::Dot::AnyByte), false, 0)? + }; + + let compiled = self.c_alt_iter(exprs.iter().map(|e| { + let _ = self.start_pattern()?; + let one = self.c_cap(0, None, e.borrow())?; + let match_state_id = self.add_match()?; + self.patch(one.end, match_state_id)?; + let _ = self.finish_pattern(one.start)?; + Ok(ThompsonRef { start: one.start, end: match_state_id }) + }))?; + self.patch(unanchored_prefix.end, compiled.start)?; + let nfa = self + .builder + .borrow_mut() + .build(compiled.start, unanchored_prefix.start)?; + + debug!("HIR-to-NFA compilation complete, config: {:?}", self.config); + Ok(nfa) + } + + /// Compile an arbitrary HIR expression. + fn c(&self, expr: &Hir) -> Result<ThompsonRef, BuildError> { + use regex_syntax::hir::{Class, HirKind::*}; + + match *expr.kind() { + Empty => self.c_empty(), + Literal(hir::Literal(ref bytes)) => self.c_literal(bytes), + Class(Class::Bytes(ref c)) => self.c_byte_class(c), + Class(Class::Unicode(ref c)) => self.c_unicode_class(c), + Look(ref look) => self.c_look(look), + Repetition(ref rep) => self.c_repetition(rep), + Capture(ref c) => self.c_cap(c.index, c.name.as_deref(), &c.sub), + Concat(ref es) => self.c_concat(es.iter().map(|e| self.c(e))), + Alternation(ref es) => self.c_alt_slice(es), + } + } + + /// Compile a concatenation of the sub-expressions yielded by the given + /// iterator. If the iterator yields no elements, then this compiles down + /// to an "empty" state that always matches. + /// + /// If the compiler is in reverse mode, then the expressions given are + /// automatically compiled in reverse. + fn c_concat<I>(&self, mut it: I) -> Result<ThompsonRef, BuildError> + where + I: DoubleEndedIterator<Item = Result<ThompsonRef, BuildError>>, + { + let first = if self.is_reverse() { it.next_back() } else { it.next() }; + let ThompsonRef { start, mut end } = match first { + Some(result) => result?, + None => return self.c_empty(), + }; + loop { + let next = + if self.is_reverse() { it.next_back() } else { it.next() }; + let compiled = match next { + Some(result) => result?, + None => break, + }; + self.patch(end, compiled.start)?; + end = compiled.end; + } + Ok(ThompsonRef { start, end }) + } + + /// Compile an alternation of the given HIR values. + /// + /// This is like 'c_alt_iter', but it accepts a slice of HIR values instead + /// of an iterator of compiled NFA sub-graphs. The point of accepting a + /// slice here is that it opens up some optimization opportunities. For + /// example, if all of the HIR values are literals, then this routine might + /// re-shuffle them to make NFA epsilon closures substantially faster. + fn c_alt_slice(&self, exprs: &[Hir]) -> Result<ThompsonRef, BuildError> { + // self.c_alt_iter(exprs.iter().map(|e| self.c(e))) + let literal_count = exprs + .iter() + .filter(|e| { + matches!(*e.kind(), hir::HirKind::Literal(hir::Literal(_))) + }) + .count(); + if literal_count <= 1 || literal_count < exprs.len() { + return self.c_alt_iter(exprs.iter().map(|e| self.c(e))); + } + + let mut trie = if self.is_reverse() { + LiteralTrie::reverse() + } else { + LiteralTrie::forward() + }; + for expr in exprs.iter() { + let literal = match *expr.kind() { + hir::HirKind::Literal(hir::Literal(ref bytes)) => bytes, + _ => unreachable!(), + }; + trie.add(literal)?; + } + trie.compile(&mut self.builder.borrow_mut()) + } + + /// Compile an alternation, where each element yielded by the given + /// iterator represents an item in the alternation. If the iterator yields + /// no elements, then this compiles down to a "fail" state. + /// + /// In an alternation, expressions appearing earlier are "preferred" at + /// match time over expressions appearing later. At least, this is true + /// when using "leftmost first" match semantics. (If "leftmost longest" are + /// ever added in the future, then this preference order of priority would + /// not apply in that mode.) + fn c_alt_iter<I>(&self, mut it: I) -> Result<ThompsonRef, BuildError> + where + I: Iterator<Item = Result<ThompsonRef, BuildError>>, + { + let first = match it.next() { + None => return self.c_fail(), + Some(result) => result?, + }; + let second = match it.next() { + None => return Ok(first), + Some(result) => result?, + }; + + let union = self.add_union()?; + let end = self.add_empty()?; + self.patch(union, first.start)?; + self.patch(first.end, end)?; + self.patch(union, second.start)?; + self.patch(second.end, end)?; + for result in it { + let compiled = result?; + self.patch(union, compiled.start)?; + self.patch(compiled.end, end)?; + } + Ok(ThompsonRef { start: union, end }) + } + + /// Compile the given capture sub-expression. `expr` should be the + /// sub-expression contained inside the capture. If "capture" states are + /// enabled, then they are added as appropriate. + /// + /// This accepts the pieces of a capture instead of a `hir::Capture` so + /// that it's easy to manufacture a "fake" group when necessary, e.g., for + /// adding the entire pattern as if it were a group in order to create + /// appropriate "capture" states in the NFA. + fn c_cap( + &self, + index: u32, + name: Option<&str>, + expr: &Hir, + ) -> Result<ThompsonRef, BuildError> { + match self.config.get_which_captures() { + // No capture states means we always skip them. + WhichCaptures::None => return self.c(expr), + // Implicit captures states means we only add when index==0 since + // index==0 implies the group is implicit. + WhichCaptures::Implicit if index > 0 => return self.c(expr), + _ => {} + } + + let start = self.add_capture_start(index, name)?; + let inner = self.c(expr)?; + let end = self.add_capture_end(index)?; + self.patch(start, inner.start)?; + self.patch(inner.end, end)?; + Ok(ThompsonRef { start, end }) + } + + /// Compile the given repetition expression. This handles all types of + /// repetitions and greediness. + fn c_repetition( + &self, + rep: &hir::Repetition, + ) -> Result<ThompsonRef, BuildError> { + match (rep.min, rep.max) { + (0, Some(1)) => self.c_zero_or_one(&rep.sub, rep.greedy), + (min, None) => self.c_at_least(&rep.sub, rep.greedy, min), + (min, Some(max)) if min == max => self.c_exactly(&rep.sub, min), + (min, Some(max)) => self.c_bounded(&rep.sub, rep.greedy, min, max), + } + } + + /// Compile the given expression such that it matches at least `min` times, + /// but no more than `max` times. + /// + /// When `greedy` is true, then the preference is for the expression to + /// match as much as possible. Otherwise, it will match as little as + /// possible. + fn c_bounded( + &self, + expr: &Hir, + greedy: bool, + min: u32, + max: u32, + ) -> Result<ThompsonRef, BuildError> { + let prefix = self.c_exactly(expr, min)?; + if min == max { + return Ok(prefix); + } + + // It is tempting here to compile the rest here as a concatenation + // of zero-or-one matches. i.e., for `a{2,5}`, compile it as if it + // were `aaa?a?a?`. The problem here is that it leads to this program: + // + // >000000: 61 => 01 + // 000001: 61 => 02 + // 000002: union(03, 04) + // 000003: 61 => 04 + // 000004: union(05, 06) + // 000005: 61 => 06 + // 000006: union(07, 08) + // 000007: 61 => 08 + // 000008: MATCH + // + // And effectively, once you hit state 2, the epsilon closure will + // include states 3, 5, 6, 7 and 8, which is quite a bit. It is better + // to instead compile it like so: + // + // >000000: 61 => 01 + // 000001: 61 => 02 + // 000002: union(03, 08) + // 000003: 61 => 04 + // 000004: union(05, 08) + // 000005: 61 => 06 + // 000006: union(07, 08) + // 000007: 61 => 08 + // 000008: MATCH + // + // So that the epsilon closure of state 2 is now just 3 and 8. + let empty = self.add_empty()?; + let mut prev_end = prefix.end; + for _ in min..max { + let union = if greedy { + self.add_union() + } else { + self.add_union_reverse() + }?; + let compiled = self.c(expr)?; + self.patch(prev_end, union)?; + self.patch(union, compiled.start)?; + self.patch(union, empty)?; + prev_end = compiled.end; + } + self.patch(prev_end, empty)?; + Ok(ThompsonRef { start: prefix.start, end: empty }) + } + + /// Compile the given expression such that it may be matched `n` or more + /// times, where `n` can be any integer. (Although a particularly large + /// integer is likely to run afoul of any configured size limits.) + /// + /// When `greedy` is true, then the preference is for the expression to + /// match as much as possible. Otherwise, it will match as little as + /// possible. + fn c_at_least( + &self, + expr: &Hir, + greedy: bool, + n: u32, + ) -> Result<ThompsonRef, BuildError> { + if n == 0 { + // When the expression cannot match the empty string, then we + // can get away with something much simpler: just one 'alt' + // instruction that optionally repeats itself. But if the expr + // can match the empty string... see below. + if expr.properties().minimum_len().map_or(false, |len| len > 0) { + let union = if greedy { + self.add_union() + } else { + self.add_union_reverse() + }?; + let compiled = self.c(expr)?; + self.patch(union, compiled.start)?; + self.patch(compiled.end, union)?; + return Ok(ThompsonRef { start: union, end: union }); + } + + // What's going on here? Shouldn't x* be simpler than this? It + // turns out that when implementing leftmost-first (Perl-like) + // match semantics, x* results in an incorrect preference order + // when computing the transitive closure of states if and only if + // 'x' can match the empty string. So instead, we compile x* as + // (x+)?, which preserves the correct preference order. + // + // See: https://github.com/rust-lang/regex/issues/779 + let compiled = self.c(expr)?; + let plus = if greedy { + self.add_union() + } else { + self.add_union_reverse() + }?; + self.patch(compiled.end, plus)?; + self.patch(plus, compiled.start)?; + + let question = if greedy { + self.add_union() + } else { + self.add_union_reverse() + }?; + let empty = self.add_empty()?; + self.patch(question, compiled.start)?; + self.patch(question, empty)?; + self.patch(plus, empty)?; + Ok(ThompsonRef { start: question, end: empty }) + } else if n == 1 { + let compiled = self.c(expr)?; + let union = if greedy { + self.add_union() + } else { + self.add_union_reverse() + }?; + self.patch(compiled.end, union)?; + self.patch(union, compiled.start)?; + Ok(ThompsonRef { start: compiled.start, end: union }) + } else { + let prefix = self.c_exactly(expr, n - 1)?; + let last = self.c(expr)?; + let union = if greedy { + self.add_union() + } else { + self.add_union_reverse() + }?; + self.patch(prefix.end, last.start)?; + self.patch(last.end, union)?; + self.patch(union, last.start)?; + Ok(ThompsonRef { start: prefix.start, end: union }) + } + } + + /// Compile the given expression such that it may be matched zero or one + /// times. + /// + /// When `greedy` is true, then the preference is for the expression to + /// match as much as possible. Otherwise, it will match as little as + /// possible. + fn c_zero_or_one( + &self, + expr: &Hir, + greedy: bool, + ) -> Result<ThompsonRef, BuildError> { + let union = + if greedy { self.add_union() } else { self.add_union_reverse() }?; + let compiled = self.c(expr)?; + let empty = self.add_empty()?; + self.patch(union, compiled.start)?; + self.patch(union, empty)?; + self.patch(compiled.end, empty)?; + Ok(ThompsonRef { start: union, end: empty }) + } + + /// Compile the given HIR expression exactly `n` times. + fn c_exactly( + &self, + expr: &Hir, + n: u32, + ) -> Result<ThompsonRef, BuildError> { + let it = (0..n).map(|_| self.c(expr)); + self.c_concat(it) + } + + /// Compile the given byte oriented character class. + /// + /// This uses "sparse" states to represent an alternation between ranges in + /// this character class. We can use "sparse" states instead of stitching + /// together a "union" state because all ranges in a character class have + /// equal priority *and* are non-overlapping (thus, only one can match, so + /// there's never a question of priority in the first place). This saves a + /// fair bit of overhead when traversing an NFA. + /// + /// This routine compiles an empty character class into a "fail" state. + fn c_byte_class( + &self, + cls: &hir::ClassBytes, + ) -> Result<ThompsonRef, BuildError> { + let end = self.add_empty()?; + let mut trans = Vec::with_capacity(cls.ranges().len()); + for r in cls.iter() { + trans.push(Transition { + start: r.start(), + end: r.end(), + next: end, + }); + } + Ok(ThompsonRef { start: self.add_sparse(trans)?, end }) + } + + /// Compile the given Unicode character class. + /// + /// This routine specifically tries to use various types of compression, + /// since UTF-8 automata of large classes can get quite large. The specific + /// type of compression used depends on forward vs reverse compilation, and + /// whether NFA shrinking is enabled or not. + /// + /// Aside from repetitions causing lots of repeat group, this is like the + /// single most expensive part of regex compilation. Therefore, a large part + /// of the expense of compilation may be reduce by disabling Unicode in the + /// pattern. + /// + /// This routine compiles an empty character class into a "fail" state. + fn c_unicode_class( + &self, + cls: &hir::ClassUnicode, + ) -> Result<ThompsonRef, BuildError> { + // If all we have are ASCII ranges wrapped in a Unicode package, then + // there is zero reason to bring out the big guns. We can fit all ASCII + // ranges within a single sparse state. + if cls.is_ascii() { + let end = self.add_empty()?; + let mut trans = Vec::with_capacity(cls.ranges().len()); + for r in cls.iter() { + // The unwraps below are OK because we've verified that this + // class only contains ASCII codepoints. + trans.push(Transition { + // FIXME(1.59): use the 'TryFrom<char> for u8' impl. + start: u8::try_from(u32::from(r.start())).unwrap(), + end: u8::try_from(u32::from(r.end())).unwrap(), + next: end, + }); + } + Ok(ThompsonRef { start: self.add_sparse(trans)?, end }) + } else if self.is_reverse() { + if !self.config.get_shrink() { + // When we don't want to spend the extra time shrinking, we + // compile the UTF-8 automaton in reverse using something like + // the "naive" approach, but will attempt to re-use common + // suffixes. + self.c_unicode_class_reverse_with_suffix(cls) + } else { + // When we want to shrink our NFA for reverse UTF-8 automata, + // we cannot feed UTF-8 sequences directly to the UTF-8 + // compiler, since the UTF-8 compiler requires all sequences + // to be lexicographically sorted. Instead, we organize our + // sequences into a range trie, which can then output our + // sequences in the correct order. Unfortunately, building the + // range trie is fairly expensive (but not nearly as expensive + // as building a DFA). Hence the reason why the 'shrink' option + // exists, so that this path can be toggled off. For example, + // we might want to turn this off if we know we won't be + // compiling a DFA. + let mut trie = self.trie_state.borrow_mut(); + trie.clear(); + + for rng in cls.iter() { + for mut seq in Utf8Sequences::new(rng.start(), rng.end()) { + seq.reverse(); + trie.insert(seq.as_slice()); + } + } + let mut builder = self.builder.borrow_mut(); + let mut utf8_state = self.utf8_state.borrow_mut(); + let mut utf8c = + Utf8Compiler::new(&mut *builder, &mut *utf8_state)?; + trie.iter(|seq| { + utf8c.add(&seq)?; + Ok(()) + })?; + utf8c.finish() + } + } else { + // In the forward direction, we always shrink our UTF-8 automata + // because we can stream it right into the UTF-8 compiler. There + // is almost no downside (in either memory or time) to using this + // approach. + let mut builder = self.builder.borrow_mut(); + let mut utf8_state = self.utf8_state.borrow_mut(); + let mut utf8c = + Utf8Compiler::new(&mut *builder, &mut *utf8_state)?; + for rng in cls.iter() { + for seq in Utf8Sequences::new(rng.start(), rng.end()) { + utf8c.add(seq.as_slice())?; + } + } + utf8c.finish() + } + + // For reference, the code below is the "naive" version of compiling a + // UTF-8 automaton. It is deliciously simple (and works for both the + // forward and reverse cases), but will unfortunately produce very + // large NFAs. When compiling a forward automaton, the size difference + // can sometimes be an order of magnitude. For example, the '\w' regex + // will generate about ~3000 NFA states using the naive approach below, + // but only 283 states when using the approach above. This is because + // the approach above actually compiles a *minimal* (or near minimal, + // because of the bounded hashmap for reusing equivalent states) UTF-8 + // automaton. + // + // The code below is kept as a reference point in order to make it + // easier to understand the higher level goal here. Although, it will + // almost certainly bit-rot, so keep that in mind. Also, if you try to + // use it, some of the tests in this module will fail because they look + // for terser byte code produce by the more optimized handling above. + // But the integration test suite should still pass. + // + // One good example of the substantial difference this can make is to + // compare and contrast performance of the Pike VM when the code below + // is active vs the code above. Here's an example to try: + // + // regex-cli find match pikevm -b -p '(?m)^\w{20}' non-ascii-file + // + // With Unicode classes generated below, this search takes about 45s on + // my machine. But with the compressed version above, the search takes + // only around 1.4s. The NFA is also 20% smaller. This is in part due + // to the compression, but also because of the utilization of 'sparse' + // NFA states. They lead to much less state shuffling during the NFA + // search. + /* + let it = cls + .iter() + .flat_map(|rng| Utf8Sequences::new(rng.start(), rng.end())) + .map(|seq| { + let it = seq + .as_slice() + .iter() + .map(|rng| self.c_range(rng.start, rng.end)); + self.c_concat(it) + }); + self.c_alt_iter(it) + */ + } + + /// Compile the given Unicode character class in reverse with suffix + /// caching. + /// + /// This is a "quick" way to compile large Unicode classes into reverse + /// UTF-8 automata while doing a small amount of compression on that + /// automata by reusing common suffixes. + /// + /// A more comprehensive compression scheme can be accomplished by using + /// a range trie to efficiently sort a reverse sequence of UTF-8 byte + /// ranges, and then use Daciuk's algorithm via `Utf8Compiler`. + /// + /// This is the technique used when "NFA shrinking" is disabled. + /// + /// (This also tries to use "sparse" states where possible, just like + /// `c_byte_class` does.) + fn c_unicode_class_reverse_with_suffix( + &self, + cls: &hir::ClassUnicode, + ) -> Result<ThompsonRef, BuildError> { + // N.B. It would likely be better to cache common *prefixes* in the + // reverse direction, but it's not quite clear how to do that. The + // advantage of caching suffixes is that it does give us a win, and + // has a very small additional overhead. + let mut cache = self.utf8_suffix.borrow_mut(); + cache.clear(); + + let union = self.add_union()?; + let alt_end = self.add_empty()?; + for urng in cls.iter() { + for seq in Utf8Sequences::new(urng.start(), urng.end()) { + let mut end = alt_end; + for brng in seq.as_slice() { + let key = Utf8SuffixKey { + from: end, + start: brng.start, + end: brng.end, + }; + let hash = cache.hash(&key); + if let Some(id) = cache.get(&key, hash) { + end = id; + continue; + } + + let compiled = self.c_range(brng.start, brng.end)?; + self.patch(compiled.end, end)?; + end = compiled.start; + cache.set(key, hash, end); + } + self.patch(union, end)?; + } + } + Ok(ThompsonRef { start: union, end: alt_end }) + } + + /// Compile the given HIR look-around assertion to an NFA look-around + /// assertion. + fn c_look(&self, anchor: &hir::Look) -> Result<ThompsonRef, BuildError> { + let look = match *anchor { + hir::Look::Start => Look::Start, + hir::Look::End => Look::End, + hir::Look::StartLF => Look::StartLF, + hir::Look::EndLF => Look::EndLF, + hir::Look::StartCRLF => Look::StartCRLF, + hir::Look::EndCRLF => Look::EndCRLF, + hir::Look::WordAscii => Look::WordAscii, + hir::Look::WordAsciiNegate => Look::WordAsciiNegate, + hir::Look::WordUnicode => Look::WordUnicode, + hir::Look::WordUnicodeNegate => Look::WordUnicodeNegate, + hir::Look::WordStartAscii => Look::WordStartAscii, + hir::Look::WordEndAscii => Look::WordEndAscii, + hir::Look::WordStartUnicode => Look::WordStartUnicode, + hir::Look::WordEndUnicode => Look::WordEndUnicode, + hir::Look::WordStartHalfAscii => Look::WordStartHalfAscii, + hir::Look::WordEndHalfAscii => Look::WordEndHalfAscii, + hir::Look::WordStartHalfUnicode => Look::WordStartHalfUnicode, + hir::Look::WordEndHalfUnicode => Look::WordEndHalfUnicode, + }; + let id = self.add_look(look)?; + Ok(ThompsonRef { start: id, end: id }) + } + + /// Compile the given byte string to a concatenation of bytes. + fn c_literal(&self, bytes: &[u8]) -> Result<ThompsonRef, BuildError> { + self.c_concat(bytes.iter().copied().map(|b| self.c_range(b, b))) + } + + /// Compile a "range" state with one transition that may only be followed + /// if the input byte is in the (inclusive) range given. + /// + /// Both the `start` and `end` locations point to the state created. + /// Callers will likely want to keep the `start`, but patch the `end` to + /// point to some other state. + fn c_range(&self, start: u8, end: u8) -> Result<ThompsonRef, BuildError> { + let id = self.add_range(start, end)?; + Ok(ThompsonRef { start: id, end: id }) + } + + /// Compile an "empty" state with one unconditional epsilon transition. + /// + /// Both the `start` and `end` locations point to the state created. + /// Callers will likely want to keep the `start`, but patch the `end` to + /// point to some other state. + fn c_empty(&self) -> Result<ThompsonRef, BuildError> { + let id = self.add_empty()?; + Ok(ThompsonRef { start: id, end: id }) + } + + /// Compile a "fail" state that can never have any outgoing transitions. + fn c_fail(&self) -> Result<ThompsonRef, BuildError> { + let id = self.add_fail()?; + Ok(ThompsonRef { start: id, end: id }) + } + + // The below helpers are meant to be simple wrappers around the + // corresponding Builder methods. For the most part, they let us write + // 'self.add_foo()' instead of 'self.builder.borrow_mut().add_foo()', where + // the latter is a mouthful. Some of the methods do inject a little bit + // of extra logic. e.g., Flipping look-around operators when compiling in + // reverse mode. + + fn patch(&self, from: StateID, to: StateID) -> Result<(), BuildError> { + self.builder.borrow_mut().patch(from, to) + } + + fn start_pattern(&self) -> Result<PatternID, BuildError> { + self.builder.borrow_mut().start_pattern() + } + + fn finish_pattern( + &self, + start_id: StateID, + ) -> Result<PatternID, BuildError> { + self.builder.borrow_mut().finish_pattern(start_id) + } + + fn add_empty(&self) -> Result<StateID, BuildError> { + self.builder.borrow_mut().add_empty() + } + + fn add_range(&self, start: u8, end: u8) -> Result<StateID, BuildError> { + self.builder.borrow_mut().add_range(Transition { + start, + end, + next: StateID::ZERO, + }) + } + + fn add_sparse( + &self, + ranges: Vec<Transition>, + ) -> Result<StateID, BuildError> { + self.builder.borrow_mut().add_sparse(ranges) + } + + fn add_look(&self, mut look: Look) -> Result<StateID, BuildError> { + if self.is_reverse() { + look = look.reversed(); + } + self.builder.borrow_mut().add_look(StateID::ZERO, look) + } + + fn add_union(&self) -> Result<StateID, BuildError> { + self.builder.borrow_mut().add_union(vec![]) + } + + fn add_union_reverse(&self) -> Result<StateID, BuildError> { + self.builder.borrow_mut().add_union_reverse(vec![]) + } + + fn add_capture_start( + &self, + capture_index: u32, + name: Option<&str>, + ) -> Result<StateID, BuildError> { + let name = name.map(Arc::from); + self.builder.borrow_mut().add_capture_start( + StateID::ZERO, + capture_index, + name, + ) + } + + fn add_capture_end( + &self, + capture_index: u32, + ) -> Result<StateID, BuildError> { + self.builder.borrow_mut().add_capture_end(StateID::ZERO, capture_index) + } + + fn add_fail(&self) -> Result<StateID, BuildError> { + self.builder.borrow_mut().add_fail() + } + + fn add_match(&self) -> Result<StateID, BuildError> { + self.builder.borrow_mut().add_match() + } + + fn is_reverse(&self) -> bool { + self.config.get_reverse() + } +} + +/// A value that represents the result of compiling a sub-expression of a +/// regex's HIR. Specifically, this represents a sub-graph of the NFA that +/// has an initial state at `start` and a final state at `end`. +#[derive(Clone, Copy, Debug)] +pub(crate) struct ThompsonRef { + pub(crate) start: StateID, + pub(crate) end: StateID, +} + +/// A UTF-8 compiler based on Daciuk's algorithm for compiling minimal DFAs +/// from a lexicographically sorted sequence of strings in linear time. +/// +/// The trick here is that any Unicode codepoint range can be converted to +/// a sequence of byte ranges that form a UTF-8 automaton. Connecting them +/// together via an alternation is trivial, and indeed, it works. However, +/// there is a lot of redundant structure in many UTF-8 automatons. Since our +/// UTF-8 ranges are in lexicographic order, we can use Daciuk's algorithm +/// to build nearly minimal DFAs in linear time. (They are guaranteed to be +/// minimal because we use a bounded cache of previously build DFA states.) +/// +/// The drawback is that this sadly doesn't work for reverse automata, since +/// the ranges are no longer in lexicographic order. For that, we invented the +/// range trie (which gets its own module). Once a range trie is built, we then +/// use this same Utf8Compiler to build a reverse UTF-8 automaton. +/// +/// The high level idea is described here: +/// https://blog.burntsushi.net/transducers/#finite-state-machines-as-data-structures +/// +/// There is also another implementation of this in the `fst` crate. +#[derive(Debug)] +struct Utf8Compiler<'a> { + builder: &'a mut Builder, + state: &'a mut Utf8State, + target: StateID, +} + +#[derive(Clone, Debug)] +struct Utf8State { + compiled: Utf8BoundedMap, + uncompiled: Vec<Utf8Node>, +} + +#[derive(Clone, Debug)] +struct Utf8Node { + trans: Vec<Transition>, + last: Option<Utf8LastTransition>, +} + +#[derive(Clone, Debug)] +struct Utf8LastTransition { + start: u8, + end: u8, +} + +impl Utf8State { + fn new() -> Utf8State { + Utf8State { compiled: Utf8BoundedMap::new(10_000), uncompiled: vec![] } + } + + fn clear(&mut self) { + self.compiled.clear(); + self.uncompiled.clear(); + } +} + +impl<'a> Utf8Compiler<'a> { + fn new( + builder: &'a mut Builder, + state: &'a mut Utf8State, + ) -> Result<Utf8Compiler<'a>, BuildError> { + let target = builder.add_empty()?; + state.clear(); + let mut utf8c = Utf8Compiler { builder, state, target }; + utf8c.add_empty(); + Ok(utf8c) + } + + fn finish(&mut self) -> Result<ThompsonRef, BuildError> { + self.compile_from(0)?; + let node = self.pop_root(); + let start = self.compile(node)?; + Ok(ThompsonRef { start, end: self.target }) + } + + fn add(&mut self, ranges: &[Utf8Range]) -> Result<(), BuildError> { + let prefix_len = ranges + .iter() + .zip(&self.state.uncompiled) + .take_while(|&(range, node)| { + node.last.as_ref().map_or(false, |t| { + (t.start, t.end) == (range.start, range.end) + }) + }) + .count(); + assert!(prefix_len < ranges.len()); + self.compile_from(prefix_len)?; + self.add_suffix(&ranges[prefix_len..]); + Ok(()) + } + + fn compile_from(&mut self, from: usize) -> Result<(), BuildError> { + let mut next = self.target; + while from + 1 < self.state.uncompiled.len() { + let node = self.pop_freeze(next); + next = self.compile(node)?; + } + self.top_last_freeze(next); + Ok(()) + } + + fn compile( + &mut self, + node: Vec<Transition>, + ) -> Result<StateID, BuildError> { + let hash = self.state.compiled.hash(&node); + if let Some(id) = self.state.compiled.get(&node, hash) { + return Ok(id); + } + let id = self.builder.add_sparse(node.clone())?; + self.state.compiled.set(node, hash, id); + Ok(id) + } + + fn add_suffix(&mut self, ranges: &[Utf8Range]) { + assert!(!ranges.is_empty()); + let last = self + .state + .uncompiled + .len() + .checked_sub(1) + .expect("non-empty nodes"); + assert!(self.state.uncompiled[last].last.is_none()); + self.state.uncompiled[last].last = Some(Utf8LastTransition { + start: ranges[0].start, + end: ranges[0].end, + }); + for r in &ranges[1..] { + self.state.uncompiled.push(Utf8Node { + trans: vec![], + last: Some(Utf8LastTransition { start: r.start, end: r.end }), + }); + } + } + + fn add_empty(&mut self) { + self.state.uncompiled.push(Utf8Node { trans: vec![], last: None }); + } + + fn pop_freeze(&mut self, next: StateID) -> Vec<Transition> { + let mut uncompiled = self.state.uncompiled.pop().unwrap(); + uncompiled.set_last_transition(next); + uncompiled.trans + } + + fn pop_root(&mut self) -> Vec<Transition> { + assert_eq!(self.state.uncompiled.len(), 1); + assert!(self.state.uncompiled[0].last.is_none()); + self.state.uncompiled.pop().expect("non-empty nodes").trans + } + + fn top_last_freeze(&mut self, next: StateID) { + let last = self + .state + .uncompiled + .len() + .checked_sub(1) + .expect("non-empty nodes"); + self.state.uncompiled[last].set_last_transition(next); + } +} + +impl Utf8Node { + fn set_last_transition(&mut self, next: StateID) { + if let Some(last) = self.last.take() { + self.trans.push(Transition { + start: last.start, + end: last.end, + next, + }); + } + } +} + +#[cfg(test)] +mod tests { + use alloc::vec; + + use crate::{ + nfa::thompson::{SparseTransitions, State}, + util::primitives::SmallIndex, + }; + + use super::*; + + fn build(pattern: &str) -> NFA { + NFA::compiler() + .configure( + NFA::config() + .which_captures(WhichCaptures::None) + .unanchored_prefix(false), + ) + .build(pattern) + .unwrap() + } + + fn pid(id: usize) -> PatternID { + PatternID::new(id).unwrap() + } + + fn sid(id: usize) -> StateID { + StateID::new(id).unwrap() + } + + fn s_byte(byte: u8, next: usize) -> State { + let next = sid(next); + let trans = Transition { start: byte, end: byte, next }; + State::ByteRange { trans } + } + + fn s_range(start: u8, end: u8, next: usize) -> State { + let next = sid(next); + let trans = Transition { start, end, next }; + State::ByteRange { trans } + } + + fn s_sparse(transitions: &[(u8, u8, usize)]) -> State { + let transitions = transitions + .iter() + .map(|&(start, end, next)| Transition { + start, + end, + next: sid(next), + }) + .collect(); + State::Sparse(SparseTransitions { transitions }) + } + + fn s_look(look: Look, next: usize) -> State { + let next = sid(next); + State::Look { look, next } + } + + fn s_bin_union(alt1: usize, alt2: usize) -> State { + State::BinaryUnion { alt1: sid(alt1), alt2: sid(alt2) } + } + + fn s_union(alts: &[usize]) -> State { + State::Union { + alternates: alts + .iter() + .map(|&id| sid(id)) + .collect::<Vec<StateID>>() + .into_boxed_slice(), + } + } + + fn s_cap(next: usize, pattern: usize, index: usize, slot: usize) -> State { + State::Capture { + next: sid(next), + pattern_id: pid(pattern), + group_index: SmallIndex::new(index).unwrap(), + slot: SmallIndex::new(slot).unwrap(), + } + } + + fn s_fail() -> State { + State::Fail + } + + fn s_match(id: usize) -> State { + State::Match { pattern_id: pid(id) } + } + + // Test that building an unanchored NFA has an appropriate `(?s:.)*?` + // prefix. + #[test] + fn compile_unanchored_prefix() { + let nfa = NFA::compiler() + .configure(NFA::config().which_captures(WhichCaptures::None)) + .build(r"a") + .unwrap(); + assert_eq!( + nfa.states(), + &[ + s_bin_union(2, 1), + s_range(0, 255, 0), + s_byte(b'a', 3), + s_match(0), + ] + ); + } + + #[test] + fn compile_no_unanchored_prefix_with_start_anchor() { + let nfa = NFA::compiler() + .configure(NFA::config().which_captures(WhichCaptures::None)) + .build(r"^a") + .unwrap(); + assert_eq!( + nfa.states(), + &[s_look(Look::Start, 1), s_byte(b'a', 2), s_match(0)] + ); + } + + #[test] + fn compile_yes_unanchored_prefix_with_end_anchor() { + let nfa = NFA::compiler() + .configure(NFA::config().which_captures(WhichCaptures::None)) + .build(r"a$") + .unwrap(); + assert_eq!( + nfa.states(), + &[ + s_bin_union(2, 1), + s_range(0, 255, 0), + s_byte(b'a', 3), + s_look(Look::End, 4), + s_match(0), + ] + ); + } + + #[test] + fn compile_yes_reverse_unanchored_prefix_with_start_anchor() { + let nfa = NFA::compiler() + .configure( + NFA::config() + .reverse(true) + .which_captures(WhichCaptures::None), + ) + .build(r"^a") + .unwrap(); + assert_eq!( + nfa.states(), + &[ + s_bin_union(2, 1), + s_range(0, 255, 0), + s_byte(b'a', 3), + // Anchors get flipped in a reverse automaton. + s_look(Look::End, 4), + s_match(0), + ], + ); + } + + #[test] + fn compile_no_reverse_unanchored_prefix_with_end_anchor() { + let nfa = NFA::compiler() + .configure( + NFA::config() + .reverse(true) + .which_captures(WhichCaptures::None), + ) + .build(r"a$") + .unwrap(); + assert_eq!( + nfa.states(), + &[ + // Anchors get flipped in a reverse automaton. + s_look(Look::Start, 1), + s_byte(b'a', 2), + s_match(0), + ], + ); + } + + #[test] + fn compile_empty() { + assert_eq!(build("").states(), &[s_match(0),]); + } + + #[test] + fn compile_literal() { + assert_eq!(build("a").states(), &[s_byte(b'a', 1), s_match(0),]); + assert_eq!( + build("ab").states(), + &[s_byte(b'a', 1), s_byte(b'b', 2), s_match(0),] + ); + assert_eq!( + build("☃").states(), + &[s_byte(0xE2, 1), s_byte(0x98, 2), s_byte(0x83, 3), s_match(0)] + ); + + // Check that non-UTF-8 literals work. + let nfa = NFA::compiler() + .configure( + NFA::config() + .which_captures(WhichCaptures::None) + .unanchored_prefix(false), + ) + .syntax(crate::util::syntax::Config::new().utf8(false)) + .build(r"(?-u)\xFF") + .unwrap(); + assert_eq!(nfa.states(), &[s_byte(b'\xFF', 1), s_match(0),]); + } + + #[test] + fn compile_class_ascii() { + assert_eq!( + build(r"[a-z]").states(), + &[s_range(b'a', b'z', 1), s_match(0),] + ); + assert_eq!( + build(r"[x-za-c]").states(), + &[s_sparse(&[(b'a', b'c', 1), (b'x', b'z', 1)]), s_match(0)] + ); + } + + #[test] + #[cfg(not(miri))] + fn compile_class_unicode() { + assert_eq!( + build(r"[\u03B1-\u03B4]").states(), + &[s_range(0xB1, 0xB4, 2), s_byte(0xCE, 0), s_match(0)] + ); + assert_eq!( + build(r"[\u03B1-\u03B4\u{1F919}-\u{1F91E}]").states(), + &[ + s_range(0xB1, 0xB4, 5), + s_range(0x99, 0x9E, 5), + s_byte(0xA4, 1), + s_byte(0x9F, 2), + s_sparse(&[(0xCE, 0xCE, 0), (0xF0, 0xF0, 3)]), + s_match(0), + ] + ); + assert_eq!( + build(r"[a-z☃]").states(), + &[ + s_byte(0x83, 3), + s_byte(0x98, 0), + s_sparse(&[(b'a', b'z', 3), (0xE2, 0xE2, 1)]), + s_match(0), + ] + ); + } + + #[test] + fn compile_repetition() { + assert_eq!( + build(r"a?").states(), + &[s_bin_union(1, 2), s_byte(b'a', 2), s_match(0),] + ); + assert_eq!( + build(r"a??").states(), + &[s_bin_union(2, 1), s_byte(b'a', 2), s_match(0),] + ); + } + + #[test] + fn compile_group() { + assert_eq!( + build(r"ab+").states(), + &[s_byte(b'a', 1), s_byte(b'b', 2), s_bin_union(1, 3), s_match(0)] + ); + assert_eq!( + build(r"(ab)").states(), + &[s_byte(b'a', 1), s_byte(b'b', 2), s_match(0)] + ); + assert_eq!( + build(r"(ab)+").states(), + &[s_byte(b'a', 1), s_byte(b'b', 2), s_bin_union(0, 3), s_match(0)] + ); + } + + #[test] + fn compile_alternation() { + assert_eq!( + build(r"a|b").states(), + &[s_range(b'a', b'b', 1), s_match(0)] + ); + assert_eq!( + build(r"ab|cd").states(), + &[ + s_byte(b'b', 3), + s_byte(b'd', 3), + s_sparse(&[(b'a', b'a', 0), (b'c', b'c', 1)]), + s_match(0) + ], + ); + assert_eq!( + build(r"|b").states(), + &[s_byte(b'b', 2), s_bin_union(2, 0), s_match(0)] + ); + assert_eq!( + build(r"a|").states(), + &[s_byte(b'a', 2), s_bin_union(0, 2), s_match(0)] + ); + } + + // This tests the use of a non-binary union, i.e., a state with more than + // 2 unconditional epsilon transitions. The only place they tend to appear + // is in reverse NFAs when shrinking is disabled. Otherwise, 'binary-union' + // and 'sparse' tend to cover all other cases of alternation. + #[test] + fn compile_non_binary_union() { + let nfa = NFA::compiler() + .configure( + NFA::config() + .which_captures(WhichCaptures::None) + .reverse(true) + .shrink(false) + .unanchored_prefix(false), + ) + .build(r"[\u1000\u2000\u3000]") + .unwrap(); + assert_eq!( + nfa.states(), + &[ + s_union(&[3, 6, 9]), + s_byte(0xE1, 10), + s_byte(0x80, 1), + s_byte(0x80, 2), + s_byte(0xE2, 10), + s_byte(0x80, 4), + s_byte(0x80, 5), + s_byte(0xE3, 10), + s_byte(0x80, 7), + s_byte(0x80, 8), + s_match(0), + ] + ); + } + + #[test] + fn compile_many_start_pattern() { + let nfa = NFA::compiler() + .configure( + NFA::config() + .which_captures(WhichCaptures::None) + .unanchored_prefix(false), + ) + .build_many(&["a", "b"]) + .unwrap(); + assert_eq!( + nfa.states(), + &[ + s_byte(b'a', 1), + s_match(0), + s_byte(b'b', 3), + s_match(1), + s_bin_union(0, 2), + ] + ); + assert_eq!(nfa.start_anchored().as_usize(), 4); + assert_eq!(nfa.start_unanchored().as_usize(), 4); + // Test that the start states for each individual pattern are correct. + assert_eq!(nfa.start_pattern(pid(0)).unwrap(), sid(0)); + assert_eq!(nfa.start_pattern(pid(1)).unwrap(), sid(2)); + } + + // This tests that our compiler can handle an empty character class. At the + // time of writing, the regex parser forbids it, so the only way to test it + // is to provide a hand written HIR. + #[test] + fn empty_class_bytes() { + use regex_syntax::hir::{Class, ClassBytes, Hir}; + + let hir = Hir::class(Class::Bytes(ClassBytes::new(vec![]))); + let config = NFA::config() + .which_captures(WhichCaptures::None) + .unanchored_prefix(false); + let nfa = + NFA::compiler().configure(config).build_from_hir(&hir).unwrap(); + assert_eq!(nfa.states(), &[s_fail(), s_match(0)]); + } + + // Like empty_class_bytes, but for a Unicode class. + #[test] + fn empty_class_unicode() { + use regex_syntax::hir::{Class, ClassUnicode, Hir}; + + let hir = Hir::class(Class::Unicode(ClassUnicode::new(vec![]))); + let config = NFA::config() + .which_captures(WhichCaptures::None) + .unanchored_prefix(false); + let nfa = + NFA::compiler().configure(config).build_from_hir(&hir).unwrap(); + assert_eq!(nfa.states(), &[s_fail(), s_match(0)]); + } + + #[test] + fn compile_captures_all() { + let nfa = NFA::compiler() + .configure( + NFA::config() + .unanchored_prefix(false) + .which_captures(WhichCaptures::All), + ) + .build("a(b)c") + .unwrap(); + assert_eq!( + nfa.states(), + &[ + s_cap(1, 0, 0, 0), + s_byte(b'a', 2), + s_cap(3, 0, 1, 2), + s_byte(b'b', 4), + s_cap(5, 0, 1, 3), + s_byte(b'c', 6), + s_cap(7, 0, 0, 1), + s_match(0) + ] + ); + let ginfo = nfa.group_info(); + assert_eq!(2, ginfo.all_group_len()); + } + + #[test] + fn compile_captures_implicit() { + let nfa = NFA::compiler() + .configure( + NFA::config() + .unanchored_prefix(false) + .which_captures(WhichCaptures::Implicit), + ) + .build("a(b)c") + .unwrap(); + assert_eq!( + nfa.states(), + &[ + s_cap(1, 0, 0, 0), + s_byte(b'a', 2), + s_byte(b'b', 3), + s_byte(b'c', 4), + s_cap(5, 0, 0, 1), + s_match(0) + ] + ); + let ginfo = nfa.group_info(); + assert_eq!(1, ginfo.all_group_len()); + } + + #[test] + fn compile_captures_none() { + let nfa = NFA::compiler() + .configure( + NFA::config() + .unanchored_prefix(false) + .which_captures(WhichCaptures::None), + ) + .build("a(b)c") + .unwrap(); + assert_eq!( + nfa.states(), + &[s_byte(b'a', 1), s_byte(b'b', 2), s_byte(b'c', 3), s_match(0)] + ); + let ginfo = nfa.group_info(); + assert_eq!(0, ginfo.all_group_len()); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/error.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/error.rs new file mode 100644 index 0000000..9f884ff --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/error.rs @@ -0,0 +1,182 @@ +use crate::util::{ + captures, look, + primitives::{PatternID, StateID}, +}; + +/// An error that can occurred during the construction of a thompson NFA. +/// +/// This error does not provide many introspection capabilities. There are +/// generally only two things you can do with it: +/// +/// * Obtain a human readable message via its `std::fmt::Display` impl. +/// * Access an underlying [`regex_syntax::Error`] type from its `source` +/// method via the `std::error::Error` trait. This error only occurs when using +/// convenience routines for building an NFA directly from a pattern string. +/// +/// Otherwise, errors typically occur when a limit has been breached. For +/// example, if the total heap usage of the compiled NFA exceeds the limit +/// set by [`Config::nfa_size_limit`](crate::nfa::thompson::Config), then +/// building the NFA will fail. +#[derive(Clone, Debug)] +pub struct BuildError { + kind: BuildErrorKind, +} + +/// The kind of error that occurred during the construction of a thompson NFA. +#[derive(Clone, Debug)] +enum BuildErrorKind { + /// An error that occurred while parsing a regular expression. Note that + /// this error may be printed over multiple lines, and is generally + /// intended to be end user readable on its own. + #[cfg(feature = "syntax")] + Syntax(regex_syntax::Error), + /// An error that occurs if the capturing groups provided to an NFA builder + /// do not satisfy the documented invariants. For example, things like + /// too many groups, missing groups, having the first (zeroth) group be + /// named or duplicate group names within the same pattern. + Captures(captures::GroupInfoError), + /// An error that occurs when an NFA contains a Unicode word boundary, but + /// where the crate was compiled without the necessary data for dealing + /// with Unicode word boundaries. + Word(look::UnicodeWordBoundaryError), + /// An error that occurs if too many patterns were given to the NFA + /// compiler. + TooManyPatterns { + /// The number of patterns given, which exceeds the limit. + given: usize, + /// The limit on the number of patterns. + limit: usize, + }, + /// An error that occurs if too states are produced while building an NFA. + TooManyStates { + /// The minimum number of states that are desired, which exceeds the + /// limit. + given: usize, + /// The limit on the number of states. + limit: usize, + }, + /// An error that occurs when NFA compilation exceeds a configured heap + /// limit. + ExceededSizeLimit { + /// The configured limit, in bytes. + limit: usize, + }, + /// An error that occurs when an invalid capture group index is added to + /// the NFA. An "invalid" index can be one that would otherwise overflow + /// a `usize` on the current target. + InvalidCaptureIndex { + /// The invalid index that was given. + index: u32, + }, + /// An error that occurs when one tries to build a reverse NFA with + /// captures enabled. Currently, this isn't supported, but we probably + /// should support it at some point. + #[cfg(feature = "syntax")] + UnsupportedCaptures, +} + +impl BuildError { + /// If this error occurred because the NFA exceeded the configured size + /// limit before being built, then this returns the configured size limit. + /// + /// The limit returned is what was configured, and corresponds to the + /// maximum amount of heap usage in bytes. + pub fn size_limit(&self) -> Option<usize> { + match self.kind { + BuildErrorKind::ExceededSizeLimit { limit } => Some(limit), + _ => None, + } + } + + fn kind(&self) -> &BuildErrorKind { + &self.kind + } + + #[cfg(feature = "syntax")] + pub(crate) fn syntax(err: regex_syntax::Error) -> BuildError { + BuildError { kind: BuildErrorKind::Syntax(err) } + } + + pub(crate) fn captures(err: captures::GroupInfoError) -> BuildError { + BuildError { kind: BuildErrorKind::Captures(err) } + } + + pub(crate) fn word(err: look::UnicodeWordBoundaryError) -> BuildError { + BuildError { kind: BuildErrorKind::Word(err) } + } + + pub(crate) fn too_many_patterns(given: usize) -> BuildError { + let limit = PatternID::LIMIT; + BuildError { kind: BuildErrorKind::TooManyPatterns { given, limit } } + } + + pub(crate) fn too_many_states(given: usize) -> BuildError { + let limit = StateID::LIMIT; + BuildError { kind: BuildErrorKind::TooManyStates { given, limit } } + } + + pub(crate) fn exceeded_size_limit(limit: usize) -> BuildError { + BuildError { kind: BuildErrorKind::ExceededSizeLimit { limit } } + } + + pub(crate) fn invalid_capture_index(index: u32) -> BuildError { + BuildError { kind: BuildErrorKind::InvalidCaptureIndex { index } } + } + + #[cfg(feature = "syntax")] + pub(crate) fn unsupported_captures() -> BuildError { + BuildError { kind: BuildErrorKind::UnsupportedCaptures } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for BuildError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self.kind() { + #[cfg(feature = "syntax")] + BuildErrorKind::Syntax(ref err) => Some(err), + BuildErrorKind::Captures(ref err) => Some(err), + _ => None, + } + } +} + +impl core::fmt::Display for BuildError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.kind() { + #[cfg(feature = "syntax")] + BuildErrorKind::Syntax(_) => write!(f, "error parsing regex"), + BuildErrorKind::Captures(_) => { + write!(f, "error with capture groups") + } + BuildErrorKind::Word(_) => { + write!(f, "NFA contains Unicode word boundary") + } + BuildErrorKind::TooManyPatterns { given, limit } => write!( + f, + "attempted to compile {given} patterns, \ + which exceeds the limit of {limit}", + ), + BuildErrorKind::TooManyStates { given, limit } => write!( + f, + "attempted to compile {given} NFA states, \ + which exceeds the limit of {limit}", + ), + BuildErrorKind::ExceededSizeLimit { limit } => write!( + f, + "heap usage during NFA compilation exceeded limit of {limit}", + ), + BuildErrorKind::InvalidCaptureIndex { index } => write!( + f, + "capture group index {index} is invalid \ + (too big or discontinuous)", + ), + #[cfg(feature = "syntax")] + BuildErrorKind::UnsupportedCaptures => write!( + f, + "currently captures must be disabled when compiling \ + a reverse NFA", + ), + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/literal_trie.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/literal_trie.rs new file mode 100644 index 0000000..08793cd --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/literal_trie.rs @@ -0,0 +1,528 @@ +use core::mem; + +use alloc::{vec, vec::Vec}; + +use crate::{ + nfa::thompson::{self, compiler::ThompsonRef, BuildError, Builder}, + util::primitives::{IteratorIndexExt, StateID}, +}; + +/// A trie that preserves leftmost-first match semantics. +/// +/// This is a purpose-built data structure for optimizing 'lit1|lit2|..|litN' +/// patterns. It can *only* handle alternations of literals, which makes it +/// somewhat restricted in its scope, but literal alternations are fairly +/// common. +/// +/// At a 5,000 foot level, the main idea of this trie is make an alternation of +/// literals look more like a DFA than an NFA via epsilon removal. +/// +/// More precisely, the main issue is in how alternations are compiled into +/// a Thompson NFA. Namely, each alternation gets a single NFA "union" state +/// with an epsilon transition for every branch of the alternation pointing to +/// an NFA state corresponding to the start of that branch. The main problem +/// with this representation is the cost of computing an epsilon closure. Once +/// you hit the alternation's start state, it acts as a sort of "clog" that +/// requires you to traverse all of the epsilon transitions to compute the full +/// closure. +/// +/// While fixing such clogs in the general case is pretty tricky without going +/// to a DFA (or perhaps a Glushkov NFA, but that comes with other problems). +/// But at least in the case of an alternation of literals, we can convert +/// that to a prefix trie without too much cost. In theory, that's all you +/// really need to do: build the trie and then compile it to a Thompson NFA. +/// For example, if you have the pattern 'bar|baz|foo', then using a trie, it +/// is transformed to something like 'b(a(r|z))|f'. This reduces the clog by +/// reducing the number of epsilon transitions out of the alternation's start +/// state from 3 to 2 (it actually gets down to 1 when you use a sparse state, +/// which we do below). It's a small effect here, but when your alternation is +/// huge, the savings is also huge. +/// +/// And that is... essentially what a LiteralTrie does. But there is one +/// hiccup. Consider a regex like 'sam|samwise'. How does a prefix trie compile +/// that when leftmost-first semantics are used? If 'sam|samwise' was the +/// entire regex, then you could just drop the 'samwise' branch entirely since +/// it is impossible to match ('sam' will always take priority, and since it +/// is a prefix of 'samwise', 'samwise' will never match). But what about the +/// regex '\b(sam|samwise)\b'? In that case, you can't remove 'samwise' because +/// it might match when 'sam' doesn't fall on a word boundary. +/// +/// The main idea is that 'sam|samwise' can be translated to 'sam(?:|wise)', +/// which is a precisely equivalent regex that also gets rid of the clog. +/// +/// Another example is 'zapper|z|zap'. That gets translated to +/// 'z(?:apper||ap)'. +/// +/// We accomplish this by giving each state in the trie multiple "chunks" of +/// transitions. Each chunk barrier represents a match. The idea is that once +/// you know a match occurs, none of the transitions after the match can be +/// re-ordered and mixed in with the transitions before the match. Otherwise, +/// the match semantics could be changed. +/// +/// See the 'State' data type for a bit more detail. +/// +/// Future work: +/// +/// * In theory, it would be nice to generalize the idea of removing clogs and +/// apply it to the NFA graph itself. Then this could in theory work for +/// case insensitive alternations of literals, or even just alternations where +/// each branch starts with a non-epsilon transition. +/// * Could we instead use the Aho-Corasick algorithm here? The aho-corasick +/// crate deals with leftmost-first matches correctly, but I think this implies +/// encoding failure transitions into a Thompson NFA somehow. Which seems fine, +/// because failure transitions are just unconditional epsilon transitions? +/// * Or perhaps even better, could we use an aho_corasick::AhoCorasick +/// directly? At time of writing, 0.7 is the current version of the +/// aho-corasick crate, and that definitely cannot be used as-is. But if we +/// expose the underlying finite state machine API, then could we use it? That +/// would be super. If we could figure that out, it might also lend itself to +/// more general composition of finite state machines. +#[derive(Clone)] +pub(crate) struct LiteralTrie { + /// The set of trie states. Each state contains one or more chunks, where + /// each chunk is a sparse set of transitions to other states. A leaf state + /// is always a match state that contains only empty chunks (i.e., no + /// transitions). + states: Vec<State>, + /// Whether to add literals in reverse to the trie. Useful when building + /// a reverse NFA automaton. + rev: bool, +} + +impl LiteralTrie { + /// Create a new literal trie that adds literals in the forward direction. + pub(crate) fn forward() -> LiteralTrie { + let root = State::default(); + LiteralTrie { states: vec![root], rev: false } + } + + /// Create a new literal trie that adds literals in reverse. + pub(crate) fn reverse() -> LiteralTrie { + let root = State::default(); + LiteralTrie { states: vec![root], rev: true } + } + + /// Add the given literal to this trie. + /// + /// If the literal could not be added because the `StateID` space was + /// exhausted, then an error is returned. If an error returns, the trie + /// is in an unspecified state. + pub(crate) fn add(&mut self, bytes: &[u8]) -> Result<(), BuildError> { + let mut prev = StateID::ZERO; + let mut it = bytes.iter().copied(); + while let Some(b) = if self.rev { it.next_back() } else { it.next() } { + prev = self.get_or_add_state(prev, b)?; + } + self.states[prev].add_match(); + Ok(()) + } + + /// If the given transition is defined, then return the next state ID. + /// Otherwise, add the transition to `from` and point it to a new state. + /// + /// If a new state ID could not be allocated, then an error is returned. + fn get_or_add_state( + &mut self, + from: StateID, + byte: u8, + ) -> Result<StateID, BuildError> { + let active = self.states[from].active_chunk(); + match active.binary_search_by_key(&byte, |t| t.byte) { + Ok(i) => Ok(active[i].next), + Err(i) => { + // Add a new state and get its ID. + let next = StateID::new(self.states.len()).map_err(|_| { + BuildError::too_many_states(self.states.len()) + })?; + self.states.push(State::default()); + // Offset our position to account for all transitions and not + // just the ones in the active chunk. + let i = self.states[from].active_chunk_start() + i; + let t = Transition { byte, next }; + self.states[from].transitions.insert(i, t); + Ok(next) + } + } + } + + /// Compile this literal trie to the NFA builder given. + /// + /// This forwards any errors that may occur while using the given builder. + pub(crate) fn compile( + &self, + builder: &mut Builder, + ) -> Result<ThompsonRef, BuildError> { + // Compilation proceeds via depth-first traversal of the trie. + // + // This is overall pretty brutal. The recursive version of this is + // deliciously simple. (See 'compile_to_hir' below for what it might + // look like.) But recursion on a trie means your call stack grows + // in accordance with the longest literal, which just does not seem + // appropriate. So we push the call stack to the heap. But as a result, + // the trie traversal becomes pretty brutal because we essentially + // have to encode the state of a double for-loop into an explicit call + // frame. If someone can simplify this without using recursion, that'd + // be great. + + // 'end' is our match state for this trie, but represented in the the + // NFA. Any time we see a match in the trie, we insert a transition + // from the current state we're in to 'end'. + let end = builder.add_empty()?; + let mut stack = vec![]; + let mut f = Frame::new(&self.states[StateID::ZERO]); + loop { + if let Some(t) = f.transitions.next() { + if self.states[t.next].is_leaf() { + f.sparse.push(thompson::Transition { + start: t.byte, + end: t.byte, + next: end, + }); + } else { + f.sparse.push(thompson::Transition { + start: t.byte, + end: t.byte, + // This is a little funny, but when the frame we create + // below completes, it will pop this parent frame off + // and modify this transition to point to the correct + // state. + next: StateID::ZERO, + }); + stack.push(f); + f = Frame::new(&self.states[t.next]); + } + continue; + } + // At this point, we have visited all transitions in f.chunk, so + // add it as a sparse NFA state. Unless the chunk was empty, in + // which case, we don't do anything. + if !f.sparse.is_empty() { + let chunk_id = if f.sparse.len() == 1 { + builder.add_range(f.sparse.pop().unwrap())? + } else { + let sparse = mem::replace(&mut f.sparse, vec![]); + builder.add_sparse(sparse)? + }; + f.union.push(chunk_id); + } + // Now we need to look to see if there are other chunks to visit. + if let Some(chunk) = f.chunks.next() { + // If we're here, it means we're on the second (or greater) + // chunk, which implies there is a match at this point. So + // connect this state to the final end state. + f.union.push(end); + // Advance to the next chunk. + f.transitions = chunk.iter(); + continue; + } + // Now that we are out of chunks, we have completely visited + // this state. So turn our union of chunks into an NFA union + // state, and add that union state to the parent state's current + // sparse state. (If there is no parent, we're done.) + let start = builder.add_union(f.union)?; + match stack.pop() { + None => { + return Ok(ThompsonRef { start, end }); + } + Some(mut parent) => { + // OK because the only way a frame gets pushed on to the + // stack (aside from the root) is when a transition has + // been added to 'sparse'. + parent.sparse.last_mut().unwrap().next = start; + f = parent; + } + } + } + } + + /// Converts this trie to an equivalent HIR expression. + /// + /// We don't actually use this, but it's useful for tests. In particular, + /// it provides a (somewhat) human readable representation of the trie + /// itself. + #[cfg(test)] + fn compile_to_hir(&self) -> regex_syntax::hir::Hir { + self.compile_state_to_hir(StateID::ZERO) + } + + /// The recursive implementation of 'to_hir'. + /// + /// Notice how simple this is compared to 'compile' above. 'compile' could + /// be similarly simple, but we opt to not use recursion in order to avoid + /// overflowing the stack in the case of a longer literal. + #[cfg(test)] + fn compile_state_to_hir(&self, sid: StateID) -> regex_syntax::hir::Hir { + use regex_syntax::hir::Hir; + + let mut alt = vec![]; + for (i, chunk) in self.states[sid].chunks().enumerate() { + if i > 0 { + alt.push(Hir::empty()); + } + if chunk.is_empty() { + continue; + } + let mut chunk_alt = vec![]; + for t in chunk.iter() { + chunk_alt.push(Hir::concat(vec![ + Hir::literal(vec![t.byte]), + self.compile_state_to_hir(t.next), + ])); + } + alt.push(Hir::alternation(chunk_alt)); + } + Hir::alternation(alt) + } +} + +impl core::fmt::Debug for LiteralTrie { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + writeln!(f, "LiteralTrie(")?; + for (sid, state) in self.states.iter().with_state_ids() { + writeln!(f, "{:06?}: {:?}", sid.as_usize(), state)?; + } + writeln!(f, ")")?; + Ok(()) + } +} + +/// An explicit stack frame used for traversing the trie without using +/// recursion. +/// +/// Each frame is tied to the traversal of a single trie state. The frame is +/// dropped once the entire state (and all of its children) have been visited. +/// The "output" of compiling a state is the 'union' vector, which is turn +/// converted to a NFA union state. Each branch of the union corresponds to a +/// chunk in the trie state. +/// +/// 'sparse' corresponds to the set of transitions for a particular chunk in a +/// trie state. It is ultimately converted to an NFA sparse state. The 'sparse' +/// field, after being converted to a sparse NFA state, is reused for any +/// subsequent chunks in the trie state, if any exist. +#[derive(Debug)] +struct Frame<'a> { + /// The remaining chunks to visit for a trie state. + chunks: StateChunksIter<'a>, + /// The transitions of the current chunk that we're iterating over. Since + /// every trie state has at least one chunk, every frame is initialized + /// with the first chunk's transitions ready to be consumed. + transitions: core::slice::Iter<'a, Transition>, + /// The NFA state IDs pointing to the start of each chunk compiled by + /// this trie state. This ultimately gets converted to an NFA union once + /// the entire trie state (and all of its children) have been compiled. + /// The order of these matters for leftmost-first match semantics, since + /// earlier matches in the union are preferred over later ones. + union: Vec<StateID>, + /// The actual NFA transitions for a single chunk in a trie state. This + /// gets converted to an NFA sparse state, and its corresponding NFA state + /// ID should get added to 'union'. + sparse: Vec<thompson::Transition>, +} + +impl<'a> Frame<'a> { + /// Create a new stack frame for trie traversal. This initializes the + /// 'transitions' iterator to the transitions for the first chunk, with the + /// 'chunks' iterator being every chunk after the first one. + fn new(state: &'a State) -> Frame<'a> { + let mut chunks = state.chunks(); + // every state has at least 1 chunk + let chunk = chunks.next().unwrap(); + let transitions = chunk.iter(); + Frame { chunks, transitions, union: vec![], sparse: vec![] } + } +} + +/// A state in a trie. +/// +/// This uses a sparse representation. Since we don't use literal tries +/// for searching, and ultimately (and compilation requires visiting every +/// transition anyway), we use a sparse representation for transitions. This +/// means we save on memory, at the expense of 'LiteralTrie::add' being perhaps +/// a bit slower. +/// +/// While 'transitions' is pretty standard as far as tries goes, the 'chunks' +/// piece here is more unusual. In effect, 'chunks' defines a partitioning +/// of 'transitions', where each chunk corresponds to a distinct set of +/// transitions. The key invariant is that a transition in one chunk cannot +/// be moved to another chunk. This is the secret sauce that preserve +/// leftmost-first match semantics. +/// +/// A new chunk is added whenever we mark a state as a match state. Once a +/// new chunk is added, the old active chunk is frozen and is never mutated +/// again. The new chunk becomes the active chunk, which is defined as +/// '&transitions[chunks.last().map_or(0, |c| c.1)..]'. Thus, a state where +/// 'chunks' is empty actually contains one chunk. Thus, every state contains +/// at least one (possibly empty) chunk. +/// +/// A "leaf" state is a state that has no outgoing transitions (so +/// 'transitions' is empty). Note that there is no way for a leaf state to be a +/// non-matching state. (Although while building the trie, within 'add', a leaf +/// state may exist while not containing any matches. But this invariant is +/// only broken within 'add'. Once 'add' returns, the invariant is upheld.) +#[derive(Clone, Default)] +struct State { + transitions: Vec<Transition>, + chunks: Vec<(usize, usize)>, +} + +impl State { + /// Mark this state as a match state and freeze the active chunk such that + /// it can not be further mutated. + fn add_match(&mut self) { + // This is not strictly necessary, but there's no point in recording + // another match by adding another chunk if the state has no + // transitions. Note though that we only skip this if we already know + // this is a match state, which is only true if 'chunks' is not empty. + // Basically, if we didn't do this, nothing semantically would change, + // but we'd end up pushing another chunk and potentially triggering an + // alloc. + if self.transitions.is_empty() && !self.chunks.is_empty() { + return; + } + let chunk_start = self.active_chunk_start(); + let chunk_end = self.transitions.len(); + self.chunks.push((chunk_start, chunk_end)); + } + + /// Returns true if and only if this state is a leaf state. That is, a + /// state that has no outgoing transitions. + fn is_leaf(&self) -> bool { + self.transitions.is_empty() + } + + /// Returns an iterator over all of the chunks (including the currently + /// active chunk) in this state. Since the active chunk is included, the + /// iterator is guaranteed to always yield at least one chunk (although the + /// chunk may be empty). + fn chunks(&self) -> StateChunksIter<'_> { + StateChunksIter { + transitions: &*self.transitions, + chunks: self.chunks.iter(), + active: Some(self.active_chunk()), + } + } + + /// Returns the active chunk as a slice of transitions. + fn active_chunk(&self) -> &[Transition] { + let start = self.active_chunk_start(); + &self.transitions[start..] + } + + /// Returns the index into 'transitions' where the active chunk starts. + fn active_chunk_start(&self) -> usize { + self.chunks.last().map_or(0, |&(_, end)| end) + } +} + +impl core::fmt::Debug for State { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut spacing = " "; + for (i, chunk) in self.chunks().enumerate() { + if i > 0 { + write!(f, "{spacing}MATCH")?; + } + spacing = ""; + for (j, t) in chunk.iter().enumerate() { + spacing = " "; + if j == 0 && i > 0 { + write!(f, " ")?; + } else if j > 0 { + write!(f, ", ")?; + } + write!(f, "{t:?}")?; + } + } + Ok(()) + } +} + +/// An iterator over all of the chunks in a state, including the active chunk. +/// +/// This iterator is created by `State::chunks`. We name this iterator so that +/// we can include it in the `Frame` type for non-recursive trie traversal. +#[derive(Debug)] +struct StateChunksIter<'a> { + transitions: &'a [Transition], + chunks: core::slice::Iter<'a, (usize, usize)>, + active: Option<&'a [Transition]>, +} + +impl<'a> Iterator for StateChunksIter<'a> { + type Item = &'a [Transition]; + + fn next(&mut self) -> Option<&'a [Transition]> { + if let Some(&(start, end)) = self.chunks.next() { + return Some(&self.transitions[start..end]); + } + if let Some(chunk) = self.active.take() { + return Some(chunk); + } + None + } +} + +/// A single transition in a trie to another state. +#[derive(Clone, Copy)] +struct Transition { + byte: u8, + next: StateID, +} + +impl core::fmt::Debug for Transition { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "{:?} => {}", + crate::util::escape::DebugByte(self.byte), + self.next.as_usize() + ) + } +} + +#[cfg(test)] +mod tests { + use bstr::B; + use regex_syntax::hir::Hir; + + use super::*; + + #[test] + fn zap() { + let mut trie = LiteralTrie::forward(); + trie.add(b"zapper").unwrap(); + trie.add(b"z").unwrap(); + trie.add(b"zap").unwrap(); + + let got = trie.compile_to_hir(); + let expected = Hir::concat(vec![ + Hir::literal(B("z")), + Hir::alternation(vec![ + Hir::literal(B("apper")), + Hir::empty(), + Hir::literal(B("ap")), + ]), + ]); + assert_eq!(expected, got); + } + + #[test] + fn maker() { + let mut trie = LiteralTrie::forward(); + trie.add(b"make").unwrap(); + trie.add(b"maple").unwrap(); + trie.add(b"maker").unwrap(); + + let got = trie.compile_to_hir(); + let expected = Hir::concat(vec![ + Hir::literal(B("ma")), + Hir::alternation(vec![ + Hir::concat(vec![ + Hir::literal(B("ke")), + Hir::alternation(vec![Hir::empty(), Hir::literal(B("r"))]), + ]), + Hir::literal(B("ple")), + ]), + ]); + assert_eq!(expected, got); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/map.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/map.rs new file mode 100644 index 0000000..7f074a3 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/map.rs @@ -0,0 +1,296 @@ +// This module contains a couple simple and purpose built hash maps. The key +// trade off they make is that they serve as caches rather than true maps. That +// is, inserting a new entry may cause eviction of another entry. This gives +// us two things. First, there's less overhead associated with inserts and +// lookups. Secondly, it lets us control our memory usage. +// +// These maps are used in some fairly hot code when generating NFA states for +// large Unicode character classes. +// +// Instead of exposing a rich hashmap entry API, we just permit the caller to +// produce a hash of the key directly. The hash can then be reused for both +// lookups and insertions at the cost of leaking abstraction a bit. But these +// are for internal use only, so it's fine. +// +// The Utf8BoundedMap is used for Daciuk's algorithm for constructing a +// (almost) minimal DFA for large Unicode character classes in linear time. +// (Daciuk's algorithm is always used when compiling forward NFAs. For reverse +// NFAs, it's only used when the compiler is configured to 'shrink' the NFA, +// since there's a bit more expense in the reverse direction.) +// +// The Utf8SuffixMap is used when compiling large Unicode character classes for +// reverse NFAs when 'shrink' is disabled. Specifically, it augments the naive +// construction of UTF-8 automata by caching common suffixes. This doesn't +// get the same space savings as Daciuk's algorithm, but it's basically as +// fast as the naive approach and typically winds up using less memory (since +// it generates smaller NFAs) despite the presence of the cache. +// +// These maps effectively represent caching mechanisms for sparse and +// byte-range NFA states, respectively. The former represents a single NFA +// state with many transitions of equivalent priority while the latter +// represents a single NFA state with a single transition. (Neither state ever +// has or is an epsilon transition.) Thus, they have different key types. It's +// likely we could make one generic map, but the machinery didn't seem worth +// it. They are simple enough. + +use alloc::{vec, vec::Vec}; + +use crate::{ + nfa::thompson::Transition, + util::{ + int::{Usize, U64}, + primitives::StateID, + }, +}; + +// Basic FNV-1a hash constants as described in: +// https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function +const PRIME: u64 = 1099511628211; +const INIT: u64 = 14695981039346656037; + +/// A bounded hash map where the key is a sequence of NFA transitions and the +/// value is a pre-existing NFA state ID. +/// +/// std's hashmap can be used for this, however, this map has two important +/// advantages. Firstly, it has lower overhead. Secondly, it permits us to +/// control our memory usage by limited the number of slots. In general, the +/// cost here is that this map acts as a cache. That is, inserting a new entry +/// may remove an old entry. We are okay with this, since it does not impact +/// correctness in the cases where it is used. The only effect that dropping +/// states from the cache has is that the resulting NFA generated may be bigger +/// than it otherwise would be. +/// +/// This improves benchmarks that compile large Unicode character classes, +/// since it makes the generation of (almost) minimal UTF-8 automaton faster. +/// Specifically, one could observe the difference with std's hashmap via +/// something like the following benchmark: +/// +/// hyperfine "regex-cli debug thompson -qr --captures none '\w{90} ecurB'" +/// +/// But to observe that difference, you'd have to modify the code to use +/// std's hashmap. +/// +/// It is quite possible that there is a better way to approach this problem. +/// For example, if there happens to be a very common state that collides with +/// a lot of less frequent states, then we could wind up with very poor caching +/// behavior. Alas, the effectiveness of this cache has not been measured. +/// Instead, ad hoc experiments suggest that it is "good enough." Additional +/// smarts (such as an LRU eviction policy) have to be weighed against the +/// amount of extra time they cost. +#[derive(Clone, Debug)] +pub struct Utf8BoundedMap { + /// The current version of this map. Only entries with matching versions + /// are considered during lookups. If an entry is found with a mismatched + /// version, then the map behaves as if the entry does not exist. + /// + /// This makes it possible to clear the map by simply incrementing the + /// version number instead of actually deallocating any storage. + version: u16, + /// The total number of entries this map can store. + capacity: usize, + /// The actual entries, keyed by hash. Collisions between different states + /// result in the old state being dropped. + map: Vec<Utf8BoundedEntry>, +} + +/// An entry in this map. +#[derive(Clone, Debug, Default)] +struct Utf8BoundedEntry { + /// The version of the map used to produce this entry. If this entry's + /// version does not match the current version of the map, then the map + /// should behave as if this entry does not exist. + version: u16, + /// The key, which is a sorted sequence of non-overlapping NFA transitions. + key: Vec<Transition>, + /// The state ID corresponding to the state containing the transitions in + /// this entry. + val: StateID, +} + +impl Utf8BoundedMap { + /// Create a new bounded map with the given capacity. The map will never + /// grow beyond the given size. + /// + /// Note that this does not allocate. Instead, callers must call `clear` + /// before using this map. `clear` will allocate space if necessary. + /// + /// This avoids the need to pay for the allocation of this map when + /// compiling regexes that lack large Unicode character classes. + pub fn new(capacity: usize) -> Utf8BoundedMap { + assert!(capacity > 0); + Utf8BoundedMap { version: 0, capacity, map: vec![] } + } + + /// Clear this map of all entries, but permit the reuse of allocation + /// if possible. + /// + /// This must be called before the map can be used. + pub fn clear(&mut self) { + if self.map.is_empty() { + self.map = vec![Utf8BoundedEntry::default(); self.capacity]; + } else { + self.version = self.version.wrapping_add(1); + // If we loop back to version 0, then we forcefully clear the + // entire map. Otherwise, it might be possible to incorrectly + // match entries used to generate other NFAs. + if self.version == 0 { + self.map = vec![Utf8BoundedEntry::default(); self.capacity]; + } + } + } + + /// Return a hash of the given transitions. + pub fn hash(&self, key: &[Transition]) -> usize { + let mut h = INIT; + for t in key { + h = (h ^ u64::from(t.start)).wrapping_mul(PRIME); + h = (h ^ u64::from(t.end)).wrapping_mul(PRIME); + h = (h ^ t.next.as_u64()).wrapping_mul(PRIME); + } + (h % self.map.len().as_u64()).as_usize() + } + + /// Retrieve the cached state ID corresponding to the given key. The hash + /// given must have been computed with `hash` using the same key value. + /// + /// If there is no cached state with the given transitions, then None is + /// returned. + pub fn get(&mut self, key: &[Transition], hash: usize) -> Option<StateID> { + let entry = &self.map[hash]; + if entry.version != self.version { + return None; + } + // There may be a hash collision, so we need to confirm real equality. + if entry.key != key { + return None; + } + Some(entry.val) + } + + /// Add a cached state to this map with the given key. Callers should + /// ensure that `state_id` points to a state that contains precisely the + /// NFA transitions given. + /// + /// `hash` must have been computed using the `hash` method with the same + /// key. + pub fn set( + &mut self, + key: Vec<Transition>, + hash: usize, + state_id: StateID, + ) { + self.map[hash] = + Utf8BoundedEntry { version: self.version, key, val: state_id }; + } +} + +/// A cache of suffixes used to modestly compress UTF-8 automata for large +/// Unicode character classes. +#[derive(Clone, Debug)] +pub struct Utf8SuffixMap { + /// The current version of this map. Only entries with matching versions + /// are considered during lookups. If an entry is found with a mismatched + /// version, then the map behaves as if the entry does not exist. + version: u16, + /// The total number of entries this map can store. + capacity: usize, + /// The actual entries, keyed by hash. Collisions between different states + /// result in the old state being dropped. + map: Vec<Utf8SuffixEntry>, +} + +/// A key that uniquely identifies an NFA state. It is a triple that represents +/// a transition from one state for a particular byte range. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Utf8SuffixKey { + pub from: StateID, + pub start: u8, + pub end: u8, +} + +/// An entry in this map. +#[derive(Clone, Debug, Default)] +struct Utf8SuffixEntry { + /// The version of the map used to produce this entry. If this entry's + /// version does not match the current version of the map, then the map + /// should behave as if this entry does not exist. + version: u16, + /// The key, which consists of a transition in a particular state. + key: Utf8SuffixKey, + /// The identifier that the transition in the key maps to. + val: StateID, +} + +impl Utf8SuffixMap { + /// Create a new bounded map with the given capacity. The map will never + /// grow beyond the given size. + /// + /// Note that this does not allocate. Instead, callers must call `clear` + /// before using this map. `clear` will allocate space if necessary. + /// + /// This avoids the need to pay for the allocation of this map when + /// compiling regexes that lack large Unicode character classes. + pub fn new(capacity: usize) -> Utf8SuffixMap { + assert!(capacity > 0); + Utf8SuffixMap { version: 0, capacity, map: vec![] } + } + + /// Clear this map of all entries, but permit the reuse of allocation + /// if possible. + /// + /// This must be called before the map can be used. + pub fn clear(&mut self) { + if self.map.is_empty() { + self.map = vec![Utf8SuffixEntry::default(); self.capacity]; + } else { + self.version = self.version.wrapping_add(1); + if self.version == 0 { + self.map = vec![Utf8SuffixEntry::default(); self.capacity]; + } + } + } + + /// Return a hash of the given transition. + pub fn hash(&self, key: &Utf8SuffixKey) -> usize { + // Basic FNV-1a hash as described: + // https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function + const PRIME: u64 = 1099511628211; + const INIT: u64 = 14695981039346656037; + + let mut h = INIT; + h = (h ^ key.from.as_u64()).wrapping_mul(PRIME); + h = (h ^ u64::from(key.start)).wrapping_mul(PRIME); + h = (h ^ u64::from(key.end)).wrapping_mul(PRIME); + (h % self.map.len().as_u64()).as_usize() + } + + /// Retrieve the cached state ID corresponding to the given key. The hash + /// given must have been computed with `hash` using the same key value. + /// + /// If there is no cached state with the given key, then None is returned. + pub fn get( + &mut self, + key: &Utf8SuffixKey, + hash: usize, + ) -> Option<StateID> { + let entry = &self.map[hash]; + if entry.version != self.version { + return None; + } + if key != &entry.key { + return None; + } + Some(entry.val) + } + + /// Add a cached state to this map with the given key. Callers should + /// ensure that `state_id` points to a state that contains precisely the + /// NFA transition given. + /// + /// `hash` must have been computed using the `hash` method with the same + /// key. + pub fn set(&mut self, key: Utf8SuffixKey, hash: usize, state_id: StateID) { + self.map[hash] = + Utf8SuffixEntry { version: self.version, key, val: state_id }; + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/mod.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/mod.rs new file mode 100644 index 0000000..dc7effe --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/mod.rs @@ -0,0 +1,81 @@ +/*! +Defines a Thompson NFA and provides the [`PikeVM`](pikevm::PikeVM) and +[`BoundedBacktracker`](backtrack::BoundedBacktracker) regex engines. + +A Thompson NFA (non-deterministic finite automaton) is arguably _the_ central +data type in this library. It is the result of what is commonly referred to as +"regex compilation." That is, turning a regex pattern from its concrete syntax +string into something that can run a search looks roughly like this: + +* A `&str` is parsed into a [`regex-syntax::ast::Ast`](regex_syntax::ast::Ast). +* An `Ast` is translated into a [`regex-syntax::hir::Hir`](regex_syntax::hir::Hir). +* An `Hir` is compiled into a [`NFA`]. +* The `NFA` is then used to build one of a few different regex engines: + * An `NFA` is used directly in the `PikeVM` and `BoundedBacktracker` engines. + * An `NFA` is used by a [hybrid NFA/DFA](crate::hybrid) to build out a DFA's + transition table at search time. + * An `NFA`, assuming it is one-pass, is used to build a full + [one-pass DFA](crate::dfa::onepass) ahead of time. + * An `NFA` is used to build a [full DFA](crate::dfa) ahead of time. + +The [`meta`](crate::meta) regex engine makes all of these choices for you based +on various criteria. However, if you have a lower level use case, _you_ can +build any of the above regex engines and use them directly. But you must start +here by building an `NFA`. + +# Details + +It is perhaps worth expanding a bit more on what it means to go through the +`&str`->`Ast`->`Hir`->`NFA` process. + +* Parsing a string into an `Ast` gives it a structured representation. +Crucially, the size and amount of work done in this step is proportional to the +size of the original string. No optimization or Unicode handling is done at +this point. This means that parsing into an `Ast` has very predictable costs. +Moreover, an `Ast` can be round-tripped back to its original pattern string as +written. +* Translating an `Ast` into an `Hir` is a process by which the structured +representation is simplified down to its most fundamental components. +Translation deals with flags such as case insensitivity by converting things +like `(?i:a)` to `[Aa]`. Translation is also where Unicode tables are consulted +to resolve things like `\p{Emoji}` and `\p{Greek}`. It also flattens each +character class, regardless of how deeply nested it is, into a single sequence +of non-overlapping ranges. All the various literal forms are thrown out in +favor of one common representation. Overall, the `Hir` is small enough to fit +into your head and makes analysis and other tasks much simpler. +* Compiling an `Hir` into an `NFA` formulates the regex into a finite state +machine whose transitions are defined over bytes. For example, an `Hir` might +have a Unicode character class corresponding to a sequence of ranges defined +in terms of `char`. Compilation is then responsible for turning those ranges +into a UTF-8 automaton. That is, an automaton that matches the UTF-8 encoding +of just the codepoints specified by those ranges. Otherwise, the main job of +an `NFA` is to serve as a byte-code of sorts for a virtual machine. It can be +seen as a sequence of instructions for how to match a regex. +*/ + +#[cfg(feature = "nfa-backtrack")] +pub mod backtrack; +mod builder; +#[cfg(feature = "syntax")] +mod compiler; +mod error; +#[cfg(feature = "syntax")] +mod literal_trie; +#[cfg(feature = "syntax")] +mod map; +mod nfa; +#[cfg(feature = "nfa-pikevm")] +pub mod pikevm; +#[cfg(feature = "syntax")] +mod range_trie; + +pub use self::{ + builder::Builder, + error::BuildError, + nfa::{ + DenseTransitions, PatternIter, SparseTransitions, State, Transition, + NFA, + }, +}; +#[cfg(feature = "syntax")] +pub use compiler::{Compiler, Config, WhichCaptures}; diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/nfa.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/nfa.rs new file mode 100644 index 0000000..405aa75 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/nfa.rs @@ -0,0 +1,2098 @@ +use core::{fmt, mem}; + +use alloc::{boxed::Box, format, string::String, sync::Arc, vec, vec::Vec}; + +#[cfg(feature = "syntax")] +use crate::nfa::thompson::{ + compiler::{Compiler, Config}, + error::BuildError, +}; +use crate::{ + nfa::thompson::builder::Builder, + util::{ + alphabet::{self, ByteClassSet, ByteClasses}, + captures::{GroupInfo, GroupInfoError}, + look::{Look, LookMatcher, LookSet}, + primitives::{ + IteratorIndexExt, PatternID, PatternIDIter, SmallIndex, StateID, + }, + sparse_set::SparseSet, + }, +}; + +/// A byte oriented Thompson non-deterministic finite automaton (NFA). +/// +/// A Thompson NFA is a finite state machine that permits unconditional epsilon +/// transitions, but guarantees that there exists at most one non-epsilon +/// transition for each element in the alphabet for each state. +/// +/// An NFA may be used directly for searching, for analysis or to build +/// a deterministic finite automaton (DFA). +/// +/// # Cheap clones +/// +/// Since an NFA is a core data type in this crate that many other regex +/// engines are based on top of, it is convenient to give ownership of an NFA +/// to said regex engines. Because of this, an NFA uses reference counting +/// internally. Therefore, it is cheap to clone and it is encouraged to do so. +/// +/// # Capabilities +/// +/// Using an NFA for searching via the +/// [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM) provides the most amount +/// of "power" of any regex engine in this crate. Namely, it supports the +/// following in all cases: +/// +/// 1. Detection of a match. +/// 2. Location of a match, including both the start and end offset, in a +/// single pass of the haystack. +/// 3. Location of matching capturing groups. +/// 4. Handles multiple patterns, including (1)-(3) when multiple patterns are +/// present. +/// +/// # Capturing Groups +/// +/// Groups refer to parenthesized expressions inside a regex pattern. They look +/// like this, where `exp` is an arbitrary regex: +/// +/// * `(exp)` - An unnamed capturing group. +/// * `(?P<name>exp)` or `(?<name>exp)` - A named capturing group. +/// * `(?:exp)` - A non-capturing group. +/// * `(?i:exp)` - A non-capturing group that sets flags. +/// +/// Only the first two forms are said to be _capturing_. Capturing +/// means that the last position at which they match is reportable. The +/// [`Captures`](crate::util::captures::Captures) type provides convenient +/// access to the match positions of capturing groups, which includes looking +/// up capturing groups by their name. +/// +/// # Byte oriented +/// +/// This NFA is byte oriented, which means that all of its transitions are +/// defined on bytes. In other words, the alphabet of an NFA consists of the +/// 256 different byte values. +/// +/// While DFAs nearly demand that they be byte oriented for performance +/// reasons, an NFA could conceivably be *Unicode codepoint* oriented. Indeed, +/// a previous version of this NFA supported both byte and codepoint oriented +/// modes. A codepoint oriented mode can work because an NFA fundamentally uses +/// a sparse representation of transitions, which works well with the large +/// sparse space of Unicode codepoints. +/// +/// Nevertheless, this NFA is only byte oriented. This choice is primarily +/// driven by implementation simplicity, and also in part memory usage. In +/// practice, performance between the two is roughly comparable. However, +/// building a DFA (including a hybrid DFA) really wants a byte oriented NFA. +/// So if we do have a codepoint oriented NFA, then we also need to generate +/// byte oriented NFA in order to build an hybrid NFA/DFA. Thus, by only +/// generating byte oriented NFAs, we can produce one less NFA. In other words, +/// if we made our NFA codepoint oriented, we'd need to *also* make it support +/// a byte oriented mode, which is more complicated. But a byte oriented mode +/// can support everything. +/// +/// # Differences with DFAs +/// +/// At the theoretical level, the precise difference between an NFA and a DFA +/// is that, in a DFA, for every state, an input symbol unambiguously refers +/// to a single transition _and_ that an input symbol is required for each +/// transition. At a practical level, this permits DFA implementations to be +/// implemented at their core with a small constant number of CPU instructions +/// for each byte of input searched. In practice, this makes them quite a bit +/// faster than NFAs _in general_. Namely, in order to execute a search for any +/// Thompson NFA, one needs to keep track of a _set_ of states, and execute +/// the possible transitions on all of those states for each input symbol. +/// Overall, this results in much more overhead. To a first approximation, one +/// can expect DFA searches to be about an order of magnitude faster. +/// +/// So why use an NFA at all? The main advantage of an NFA is that it takes +/// linear time (in the size of the pattern string after repetitions have been +/// expanded) to build and linear memory usage. A DFA, on the other hand, may +/// take exponential time and/or space to build. Even in non-pathological +/// cases, DFAs often take quite a bit more memory than their NFA counterparts, +/// _especially_ if large Unicode character classes are involved. Of course, +/// an NFA also provides additional capabilities. For example, it can match +/// Unicode word boundaries on non-ASCII text and resolve the positions of +/// capturing groups. +/// +/// Note that a [`hybrid::regex::Regex`](crate::hybrid::regex::Regex) strikes a +/// good balance between an NFA and a DFA. It avoids the exponential build time +/// of a DFA while maintaining its fast search time. The downside of a hybrid +/// NFA/DFA is that in some cases it can be slower at search time than the NFA. +/// (It also has less functionality than a pure NFA. It cannot handle Unicode +/// word boundaries on non-ASCII text and cannot resolve capturing groups.) +/// +/// # Example +/// +/// This shows how to build an NFA with the default configuration and execute a +/// search using the Pike VM. +/// +/// ``` +/// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; +/// +/// let re = PikeVM::new(r"foo[0-9]+")?; +/// let mut cache = re.create_cache(); +/// let mut caps = re.create_captures(); +/// +/// let expected = Some(Match::must(0, 0..8)); +/// re.captures(&mut cache, b"foo12345", &mut caps); +/// assert_eq!(expected, caps.get_match()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: resolving capturing groups +/// +/// This example shows how to parse some simple dates and extract the +/// components of each date via capturing groups. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{ +/// nfa::thompson::pikevm::PikeVM, +/// util::captures::Captures, +/// }; +/// +/// let vm = PikeVM::new(r"(?P<y>\d{4})-(?P<m>\d{2})-(?P<d>\d{2})")?; +/// let mut cache = vm.create_cache(); +/// +/// let haystack = "2012-03-14, 2013-01-01 and 2014-07-05"; +/// let all: Vec<Captures> = vm.captures_iter( +/// &mut cache, haystack.as_bytes() +/// ).collect(); +/// // There should be a total of 3 matches. +/// assert_eq!(3, all.len()); +/// // The year from the second match is '2013'. +/// let span = all[1].get_group_by_name("y").unwrap(); +/// assert_eq!("2013", &haystack[span]); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// This example shows that only the last match of a capturing group is +/// reported, even if it had to match multiple times for an overall match +/// to occur. +/// +/// ``` +/// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span}; +/// +/// let re = PikeVM::new(r"([a-z]){4}")?; +/// let mut cache = re.create_cache(); +/// let mut caps = re.create_captures(); +/// +/// let haystack = b"quux"; +/// re.captures(&mut cache, haystack, &mut caps); +/// assert!(caps.is_match()); +/// assert_eq!(Some(Span::from(3..4)), caps.get_group(1)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone)] +pub struct NFA( + // We make NFAs reference counted primarily for two reasons. First is that + // the NFA type itself is quite large (at least 0.5KB), and so it makes + // sense to put it on the heap by default anyway. Second is that, for Arc + // specifically, this enables cheap clones. This tends to be useful because + // several structures (the backtracker, the Pike VM, the hybrid NFA/DFA) + // all want to hang on to an NFA for use during search time. We could + // provide the NFA at search time via a function argument, but this makes + // for an unnecessarily annoying API. Instead, we just let each structure + // share ownership of the NFA. Using a deep clone would not be smart, since + // the NFA can use quite a bit of heap space. + Arc<Inner>, +); + +impl NFA { + /// Parse the given regular expression using a default configuration and + /// build an NFA from it. + /// + /// If you want a non-default configuration, then use the NFA + /// [`Compiler`] with a [`Config`]. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re = PikeVM::new(r"foo[0-9]+")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// let expected = Some(Match::must(0, 0..8)); + /// re.captures(&mut cache, b"foo12345", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new(pattern: &str) -> Result<NFA, BuildError> { + NFA::compiler().build(pattern) + } + + /// Parse the given regular expressions using a default configuration and + /// build a multi-NFA from them. + /// + /// If you want a non-default configuration, then use the NFA + /// [`Compiler`] with a [`Config`]. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re = PikeVM::new_many(&["[0-9]+", "[a-z]+"])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// let expected = Some(Match::must(1, 0..3)); + /// re.captures(&mut cache, b"foo12345bar", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new_many<P: AsRef<str>>(patterns: &[P]) -> Result<NFA, BuildError> { + NFA::compiler().build_many(patterns) + } + + /// Returns an NFA with a single regex pattern that always matches at every + /// position. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, pikevm::PikeVM}, Match}; + /// + /// let re = PikeVM::new_from_nfa(NFA::always_match())?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// let expected = Some(Match::must(0, 0..0)); + /// re.captures(&mut cache, b"", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// re.captures(&mut cache, b"foo", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn always_match() -> NFA { + // We could use NFA::new("") here and we'd get the same semantics, but + // hand-assembling the NFA (as below) does the same thing with a fewer + // number of states. It also avoids needing the 'syntax' feature + // enabled. + // + // Technically all we need is the "match" state, but we add the + // "capture" states so that the PikeVM can use this NFA. + // + // The unwraps below are OK because we add so few states that they will + // never exhaust any default limits in any environment. + let mut builder = Builder::new(); + let pid = builder.start_pattern().unwrap(); + assert_eq!(pid.as_usize(), 0); + let start_id = + builder.add_capture_start(StateID::ZERO, 0, None).unwrap(); + let end_id = builder.add_capture_end(StateID::ZERO, 0).unwrap(); + let match_id = builder.add_match().unwrap(); + builder.patch(start_id, end_id).unwrap(); + builder.patch(end_id, match_id).unwrap(); + let pid = builder.finish_pattern(start_id).unwrap(); + assert_eq!(pid.as_usize(), 0); + builder.build(start_id, start_id).unwrap() + } + + /// Returns an NFA that never matches at any position. + /// + /// This is a convenience routine for creating an NFA with zero patterns. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::{NFA, pikevm::PikeVM}; + /// + /// let re = PikeVM::new_from_nfa(NFA::never_match())?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, b"", &mut caps); + /// assert!(!caps.is_match()); + /// re.captures(&mut cache, b"foo", &mut caps); + /// assert!(!caps.is_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn never_match() -> NFA { + // This always succeeds because it only requires one NFA state, which + // will never exhaust any (default) limits. + let mut builder = Builder::new(); + let sid = builder.add_fail().unwrap(); + builder.build(sid, sid).unwrap() + } + + /// Return a default configuration for an `NFA`. + /// + /// This is a convenience routine to avoid needing to import the `Config` + /// type when customizing the construction of an NFA. + /// + /// # Example + /// + /// This example shows how to build an NFA with a small size limit that + /// results in a compilation error for any regex that tries to use more + /// heap memory than the configured limit. + /// + /// ``` + /// use regex_automata::nfa::thompson::{NFA, pikevm::PikeVM}; + /// + /// let result = PikeVM::builder() + /// .thompson(NFA::config().nfa_size_limit(Some(1_000))) + /// // Remember, \w is Unicode-aware by default and thus huge. + /// .build(r"\w+"); + /// assert!(result.is_err()); + /// ``` + #[cfg(feature = "syntax")] + pub fn config() -> Config { + Config::new() + } + + /// Return a compiler for configuring the construction of an `NFA`. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Compiler`] type in common cases. + /// + /// # Example + /// + /// This example shows how to build an NFA that is permitted match invalid + /// UTF-8. Without the additional syntax configuration here, compilation of + /// `(?-u:.)` would fail because it is permitted to match invalid UTF-8. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::syntax, + /// Match, + /// }; + /// + /// let re = PikeVM::builder() + /// .syntax(syntax::Config::new().utf8(false)) + /// .build(r"[a-z]+(?-u:.)")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// let expected = Some(Match::must(0, 1..5)); + /// re.captures(&mut cache, b"\xFFabc\xFF", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn compiler() -> Compiler { + Compiler::new() + } + + /// Returns an iterator over all pattern identifiers in this NFA. + /// + /// Pattern IDs are allocated in sequential order starting from zero, + /// where the order corresponds to the order of patterns provided to the + /// [`NFA::new_many`] constructor. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let nfa = NFA::new_many(&["[0-9]+", "[a-z]+", "[A-Z]+"])?; + /// let pids: Vec<PatternID> = nfa.patterns().collect(); + /// assert_eq!(pids, vec![ + /// PatternID::must(0), + /// PatternID::must(1), + /// PatternID::must(2), + /// ]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn patterns(&self) -> PatternIter<'_> { + PatternIter { + it: PatternID::iter(self.pattern_len()), + _marker: core::marker::PhantomData, + } + } + + /// Returns the total number of regex patterns in this NFA. + /// + /// This may return zero if the NFA was constructed with no patterns. In + /// this case, the NFA can never produce a match for any input. + /// + /// This is guaranteed to be no bigger than [`PatternID::LIMIT`] because + /// NFA construction will fail if too many patterns are added. + /// + /// It is always true that `nfa.patterns().count() == nfa.pattern_len()`. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::NFA; + /// + /// let nfa = NFA::new_many(&["[0-9]+", "[a-z]+", "[A-Z]+"])?; + /// assert_eq!(3, nfa.pattern_len()); + /// + /// let nfa = NFA::never_match(); + /// assert_eq!(0, nfa.pattern_len()); + /// + /// let nfa = NFA::always_match(); + /// assert_eq!(1, nfa.pattern_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn pattern_len(&self) -> usize { + self.0.start_pattern.len() + } + + /// Return the state identifier of the initial anchored state of this NFA. + /// + /// The returned identifier is guaranteed to be a valid index into the + /// slice returned by [`NFA::states`], and is also a valid argument to + /// [`NFA::state`]. + /// + /// # Example + /// + /// This example shows a somewhat contrived example where we can easily + /// predict the anchored starting state. + /// + /// ``` + /// use regex_automata::nfa::thompson::{NFA, State, WhichCaptures}; + /// + /// let nfa = NFA::compiler() + /// .configure(NFA::config().which_captures(WhichCaptures::None)) + /// .build("a")?; + /// let state = nfa.state(nfa.start_anchored()); + /// match *state { + /// State::ByteRange { trans } => { + /// assert_eq!(b'a', trans.start); + /// assert_eq!(b'a', trans.end); + /// } + /// _ => unreachable!("unexpected state"), + /// } + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn start_anchored(&self) -> StateID { + self.0.start_anchored + } + + /// Return the state identifier of the initial unanchored state of this + /// NFA. + /// + /// This is equivalent to the identifier returned by + /// [`NFA::start_anchored`] when the NFA has no unanchored starting state. + /// + /// The returned identifier is guaranteed to be a valid index into the + /// slice returned by [`NFA::states`], and is also a valid argument to + /// [`NFA::state`]. + /// + /// # Example + /// + /// This example shows that the anchored and unanchored starting states + /// are equivalent when an anchored NFA is built. + /// + /// ``` + /// use regex_automata::nfa::thompson::NFA; + /// + /// let nfa = NFA::new("^a")?; + /// assert_eq!(nfa.start_anchored(), nfa.start_unanchored()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn start_unanchored(&self) -> StateID { + self.0.start_unanchored + } + + /// Return the state identifier of the initial anchored state for the given + /// pattern, or `None` if there is no pattern corresponding to the given + /// identifier. + /// + /// If one uses the starting state for a particular pattern, then the only + /// match that can be returned is for the corresponding pattern. + /// + /// The returned identifier is guaranteed to be a valid index into the + /// slice returned by [`NFA::states`], and is also a valid argument to + /// [`NFA::state`]. + /// + /// # Errors + /// + /// If the pattern doesn't exist in this NFA, then this returns an error. + /// This occurs when `pid.as_usize() >= nfa.pattern_len()`. + /// + /// # Example + /// + /// This example shows that the anchored and unanchored starting states + /// are equivalent when an anchored NFA is built. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let nfa = NFA::new_many(&["^a", "^b"])?; + /// // The anchored and unanchored states for the entire NFA are the same, + /// // since all of the patterns are anchored. + /// assert_eq!(nfa.start_anchored(), nfa.start_unanchored()); + /// // But the anchored starting states for each pattern are distinct, + /// // because these starting states can only lead to matches for the + /// // corresponding pattern. + /// let anchored = Some(nfa.start_anchored()); + /// assert_ne!(anchored, nfa.start_pattern(PatternID::must(0))); + /// assert_ne!(anchored, nfa.start_pattern(PatternID::must(1))); + /// // Requesting a pattern not in the NFA will result in None: + /// assert_eq!(None, nfa.start_pattern(PatternID::must(2))); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn start_pattern(&self, pid: PatternID) -> Option<StateID> { + self.0.start_pattern.get(pid.as_usize()).copied() + } + + /// Get the byte class set for this NFA. + /// + /// A byte class set is a partitioning of this NFA's alphabet into + /// equivalence classes. Any two bytes in the same equivalence class are + /// guaranteed to never discriminate between a match or a non-match. (The + /// partitioning may not be minimal.) + /// + /// Byte classes are used internally by this crate when building DFAs. + /// Namely, among other optimizations, they enable a space optimization + /// where the DFA's internal alphabet is defined over the equivalence + /// classes of bytes instead of all possible byte values. The former is + /// often quite a bit smaller than the latter, which permits the DFA to use + /// less space for its transition table. + #[inline] + pub(crate) fn byte_class_set(&self) -> &ByteClassSet { + &self.0.byte_class_set + } + + /// Get the byte classes for this NFA. + /// + /// Byte classes represent a partitioning of this NFA's alphabet into + /// equivalence classes. Any two bytes in the same equivalence class are + /// guaranteed to never discriminate between a match or a non-match. (The + /// partitioning may not be minimal.) + /// + /// Byte classes are used internally by this crate when building DFAs. + /// Namely, among other optimizations, they enable a space optimization + /// where the DFA's internal alphabet is defined over the equivalence + /// classes of bytes instead of all possible byte values. The former is + /// often quite a bit smaller than the latter, which permits the DFA to use + /// less space for its transition table. + /// + /// # Example + /// + /// This example shows how to query the class of various bytes. + /// + /// ``` + /// use regex_automata::nfa::thompson::NFA; + /// + /// let nfa = NFA::new("[a-z]+")?; + /// let classes = nfa.byte_classes(); + /// // 'a' and 'z' are in the same class for this regex. + /// assert_eq!(classes.get(b'a'), classes.get(b'z')); + /// // But 'a' and 'A' are not. + /// assert_ne!(classes.get(b'a'), classes.get(b'A')); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn byte_classes(&self) -> &ByteClasses { + &self.0.byte_classes + } + + /// Return a reference to the NFA state corresponding to the given ID. + /// + /// This is a convenience routine for `nfa.states()[id]`. + /// + /// # Panics + /// + /// This panics when the given identifier does not reference a valid state. + /// That is, when `id.as_usize() >= nfa.states().len()`. + /// + /// # Example + /// + /// The anchored state for a pattern will typically correspond to a + /// capturing state for that pattern. (Although, this is not an API + /// guarantee!) + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, State}, PatternID}; + /// + /// let nfa = NFA::new("a")?; + /// let state = nfa.state(nfa.start_pattern(PatternID::ZERO).unwrap()); + /// match *state { + /// State::Capture { slot, .. } => { + /// assert_eq!(0, slot.as_usize()); + /// } + /// _ => unreachable!("unexpected state"), + /// } + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn state(&self, id: StateID) -> &State { + &self.states()[id] + } + + /// Returns a slice of all states in this NFA. + /// + /// The slice returned is indexed by `StateID`. This provides a convenient + /// way to access states while following transitions among those states. + /// + /// # Example + /// + /// This demonstrates that disabling UTF-8 mode can shrink the size of the + /// NFA considerably in some cases, especially when using Unicode character + /// classes. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::nfa::thompson::NFA; + /// + /// let nfa_unicode = NFA::new(r"\w")?; + /// let nfa_ascii = NFA::new(r"(?-u)\w")?; + /// // Yes, a factor of 45 difference. No lie. + /// assert!(40 * nfa_ascii.states().len() < nfa_unicode.states().len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn states(&self) -> &[State] { + &self.0.states + } + + /// Returns the capturing group info for this NFA. + /// + /// The [`GroupInfo`] provides a way to map to and from capture index + /// and capture name for each pattern. It also provides a mapping from + /// each of the capturing groups in every pattern to their corresponding + /// slot offsets encoded in [`State::Capture`] states. + /// + /// Note that `GroupInfo` uses reference counting internally, such that + /// cloning a `GroupInfo` is very cheap. + /// + /// # Example + /// + /// This example shows how to get a list of all capture group names for + /// a particular pattern. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let nfa = NFA::new(r"(a)(?P<foo>b)(c)(d)(?P<bar>e)")?; + /// // The first is the implicit group that is always unnamed. The next + /// // 5 groups are the explicit groups found in the concrete syntax above. + /// let expected = vec![None, None, Some("foo"), None, None, Some("bar")]; + /// let got: Vec<Option<&str>> = + /// nfa.group_info().pattern_names(PatternID::ZERO).collect(); + /// assert_eq!(expected, got); + /// + /// // Using an invalid pattern ID will result in nothing yielded. + /// let got = nfa.group_info().pattern_names(PatternID::must(999)).count(); + /// assert_eq!(0, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn group_info(&self) -> &GroupInfo { + &self.0.group_info() + } + + /// Returns true if and only if this NFA has at least one + /// [`Capture`](State::Capture) in its sequence of states. + /// + /// This is useful as a way to perform a quick test before attempting + /// something that does or does not require capture states. For example, + /// some regex engines (like the PikeVM) require capture states in order to + /// work at all. + /// + /// # Example + /// + /// This example shows a few different NFAs and whether they have captures + /// or not. + /// + /// ``` + /// use regex_automata::nfa::thompson::{NFA, WhichCaptures}; + /// + /// // Obviously has capture states. + /// let nfa = NFA::new("(a)")?; + /// assert!(nfa.has_capture()); + /// + /// // Less obviously has capture states, because every pattern has at + /// // least one anonymous capture group corresponding to the match for the + /// // entire pattern. + /// let nfa = NFA::new("a")?; + /// assert!(nfa.has_capture()); + /// + /// // Other than hand building your own NFA, this is the only way to build + /// // an NFA without capturing groups. In general, you should only do this + /// // if you don't intend to use any of the NFA-oriented regex engines. + /// // Overall, capturing groups don't have many downsides. Although they + /// // can add a bit of noise to simple NFAs, so it can be nice to disable + /// // them for debugging purposes. + /// // + /// // Notice that 'has_capture' is false here even when we have an + /// // explicit capture group in the pattern. + /// let nfa = NFA::compiler() + /// .configure(NFA::config().which_captures(WhichCaptures::None)) + /// .build("(a)")?; + /// assert!(!nfa.has_capture()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn has_capture(&self) -> bool { + self.0.has_capture + } + + /// Returns true if and only if this NFA can match the empty string. + /// When it returns false, all possible matches are guaranteed to have a + /// non-zero length. + /// + /// This is useful as cheap way to know whether code needs to handle the + /// case of a zero length match. This is particularly important when UTF-8 + /// modes are enabled, as when UTF-8 mode is enabled, empty matches that + /// split a codepoint must never be reported. This extra handling can + /// sometimes be costly, and since regexes matching an empty string are + /// somewhat rare, it can be beneficial to treat such regexes specially. + /// + /// # Example + /// + /// This example shows a few different NFAs and whether they match the + /// empty string or not. Notice the empty string isn't merely a matter + /// of a string of length literally `0`, but rather, whether a match can + /// occur between specific pairs of bytes. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, util::syntax}; + /// + /// // The empty regex matches the empty string. + /// let nfa = NFA::new("")?; + /// assert!(nfa.has_empty(), "empty matches empty"); + /// // The '+' repetition operator requires at least one match, and so + /// // does not match the empty string. + /// let nfa = NFA::new("a+")?; + /// assert!(!nfa.has_empty(), "+ does not match empty"); + /// // But the '*' repetition operator does. + /// let nfa = NFA::new("a*")?; + /// assert!(nfa.has_empty(), "* does match empty"); + /// // And wrapping '+' in an operator that can match an empty string also + /// // causes it to match the empty string too. + /// let nfa = NFA::new("(a+)*")?; + /// assert!(nfa.has_empty(), "+ inside of * matches empty"); + /// + /// // If a regex is just made of a look-around assertion, even if the + /// // assertion requires some kind of non-empty string around it (such as + /// // \b), then it is still treated as if it matches the empty string. + /// // Namely, if a match occurs of just a look-around assertion, then the + /// // match returned is empty. + /// let nfa = NFA::compiler() + /// .syntax(syntax::Config::new().utf8(false)) + /// .build(r"^$\A\z\b\B(?-u:\b\B)")?; + /// assert!(nfa.has_empty(), "assertions match empty"); + /// // Even when an assertion is wrapped in a '+', it still matches the + /// // empty string. + /// let nfa = NFA::new(r"\b+")?; + /// assert!(nfa.has_empty(), "+ of an assertion matches empty"); + /// + /// // An alternation with even one branch that can match the empty string + /// // is also said to match the empty string overall. + /// let nfa = NFA::new("foo|(bar)?|quux")?; + /// assert!(nfa.has_empty(), "alternations can match empty"); + /// + /// // An NFA that matches nothing does not match the empty string. + /// let nfa = NFA::new("[a&&b]")?; + /// assert!(!nfa.has_empty(), "never matching means not matching empty"); + /// // But if it's wrapped in something that doesn't require a match at + /// // all, then it can match the empty string! + /// let nfa = NFA::new("[a&&b]*")?; + /// assert!(nfa.has_empty(), "* on never-match still matches empty"); + /// // Since a '+' requires a match, using it on something that can never + /// // match will itself produce a regex that can never match anything, + /// // and thus does not match the empty string. + /// let nfa = NFA::new("[a&&b]+")?; + /// assert!(!nfa.has_empty(), "+ on never-match still matches nothing"); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn has_empty(&self) -> bool { + self.0.has_empty + } + + /// Whether UTF-8 mode is enabled for this NFA or not. + /// + /// When UTF-8 mode is enabled, all matches reported by a regex engine + /// derived from this NFA are guaranteed to correspond to spans of valid + /// UTF-8. This includes zero-width matches. For example, the regex engine + /// must guarantee that the empty regex will not match at the positions + /// between code units in the UTF-8 encoding of a single codepoint. + /// + /// See [`Config::utf8`] for more information. + /// + /// This is enabled by default. + /// + /// # Example + /// + /// This example shows how UTF-8 mode can impact the match spans that may + /// be reported in certain cases. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{self, pikevm::PikeVM}, + /// Match, Input, + /// }; + /// + /// let re = PikeVM::new("")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// // UTF-8 mode is enabled by default. + /// let mut input = Input::new("☃"); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 0..0)), caps.get_match()); + /// + /// // Even though an empty regex matches at 1..1, our next match is + /// // 3..3 because 1..1 and 2..2 split the snowman codepoint (which is + /// // three bytes long). + /// input.set_start(1); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 3..3)), caps.get_match()); + /// + /// // But if we disable UTF-8, then we'll get matches at 1..1 and 2..2: + /// let re = PikeVM::builder() + /// .thompson(thompson::Config::new().utf8(false)) + /// .build("")?; + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 1..1)), caps.get_match()); + /// + /// input.set_start(2); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 2..2)), caps.get_match()); + /// + /// input.set_start(3); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 3..3)), caps.get_match()); + /// + /// input.set_start(4); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(None, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_utf8(&self) -> bool { + self.0.utf8 + } + + /// Returns true when this NFA is meant to be matched in reverse. + /// + /// Generally speaking, when this is true, it means the NFA is supposed to + /// be used in conjunction with moving backwards through the haystack. That + /// is, from a higher memory address to a lower memory address. + /// + /// It is often the case that lower level routines dealing with an NFA + /// don't need to care about whether it is "meant" to be matched in reverse + /// or not. However, there are some specific cases where it matters. For + /// example, the implementation of CRLF-aware `^` and `$` line anchors + /// needs to know whether the search is in the forward or reverse + /// direction. In the forward direction, neither `^` nor `$` should match + /// when a `\r` has been seen previously and a `\n` is next. However, in + /// the reverse direction, neither `^` nor `$` should match when a `\n` + /// has been seen previously and a `\r` is next. This fundamentally changes + /// how the state machine is constructed, and thus needs to be altered + /// based on the direction of the search. + /// + /// This is automatically set when using a [`Compiler`] with a configuration + /// where [`Config::reverse`] is enabled. If you're building your own NFA + /// by hand via a [`Builder`] + #[inline] + pub fn is_reverse(&self) -> bool { + self.0.reverse + } + + /// Returns true if and only if all starting states for this NFA correspond + /// to the beginning of an anchored search. + /// + /// Typically, an NFA will have both an anchored and an unanchored starting + /// state. Namely, because it tends to be useful to have both and the cost + /// of having an unanchored starting state is almost zero (for an NFA). + /// However, if all patterns in the NFA are themselves anchored, then even + /// the unanchored starting state will correspond to an anchored search + /// since the pattern doesn't permit anything else. + /// + /// # Example + /// + /// This example shows a few different scenarios where this method's + /// return value varies. + /// + /// ``` + /// use regex_automata::nfa::thompson::NFA; + /// + /// // The unanchored starting state permits matching this pattern anywhere + /// // in a haystack, instead of just at the beginning. + /// let nfa = NFA::new("a")?; + /// assert!(!nfa.is_always_start_anchored()); + /// + /// // In this case, the pattern is itself anchored, so there is no way + /// // to run an unanchored search. + /// let nfa = NFA::new("^a")?; + /// assert!(nfa.is_always_start_anchored()); + /// + /// // When multiline mode is enabled, '^' can match at the start of a line + /// // in addition to the start of a haystack, so an unanchored search is + /// // actually possible. + /// let nfa = NFA::new("(?m)^a")?; + /// assert!(!nfa.is_always_start_anchored()); + /// + /// // Weird cases also work. A pattern is only considered anchored if all + /// // matches may only occur at the start of a haystack. + /// let nfa = NFA::new("(^a)|a")?; + /// assert!(!nfa.is_always_start_anchored()); + /// + /// // When multiple patterns are present, if they are all anchored, then + /// // the NFA is always anchored too. + /// let nfa = NFA::new_many(&["^a", "^b", "^c"])?; + /// assert!(nfa.is_always_start_anchored()); + /// + /// // But if one pattern is unanchored, then the NFA must permit an + /// // unanchored search. + /// let nfa = NFA::new_many(&["^a", "b", "^c"])?; + /// assert!(!nfa.is_always_start_anchored()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_always_start_anchored(&self) -> bool { + self.start_anchored() == self.start_unanchored() + } + + /// Returns the look-around matcher associated with this NFA. + /// + /// A look-around matcher determines how to match look-around assertions. + /// In particular, some assertions are configurable. For example, the + /// `(?m:^)` and `(?m:$)` assertions can have their line terminator changed + /// from the default of `\n` to any other byte. + /// + /// If the NFA was built using a [`Compiler`], then this matcher + /// can be set via the [`Config::look_matcher`] configuration + /// knob. Otherwise, if you've built an NFA by hand, it is set via + /// [`Builder::set_look_matcher`]. + /// + /// # Example + /// + /// This shows how to change the line terminator for multi-line assertions. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{self, pikevm::PikeVM}, + /// util::look::LookMatcher, + /// Match, Input, + /// }; + /// + /// let mut lookm = LookMatcher::new(); + /// lookm.set_line_terminator(b'\x00'); + /// + /// let re = PikeVM::builder() + /// .thompson(thompson::Config::new().look_matcher(lookm)) + /// .build(r"(?m)^[a-z]+$")?; + /// let mut cache = re.create_cache(); + /// + /// // Multi-line assertions now use NUL as a terminator. + /// assert_eq!( + /// Some(Match::must(0, 1..4)), + /// re.find(&mut cache, b"\x00abc\x00"), + /// ); + /// // ... and \n is no longer recognized as a terminator. + /// assert_eq!( + /// None, + /// re.find(&mut cache, b"\nabc\n"), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn look_matcher(&self) -> &LookMatcher { + &self.0.look_matcher + } + + /// Returns the union of all look-around assertions used throughout this + /// NFA. When the returned set is empty, it implies that the NFA has no + /// look-around assertions and thus zero conditional epsilon transitions. + /// + /// This is useful in some cases enabling optimizations. It is not + /// unusual, for example, for optimizations to be of the form, "for any + /// regex with zero conditional epsilon transitions, do ..." where "..." + /// is some kind of optimization. + /// + /// This isn't only helpful for optimizations either. Sometimes look-around + /// assertions are difficult to support. For example, many of the DFAs in + /// this crate don't support Unicode word boundaries or handle them using + /// heuristics. Handling that correctly typically requires some kind of + /// cheap check of whether the NFA has a Unicode word boundary in the first + /// place. + /// + /// # Example + /// + /// This example shows how this routine varies based on the regex pattern: + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, util::look::Look}; + /// + /// // No look-around at all. + /// let nfa = NFA::new("a")?; + /// assert!(nfa.look_set_any().is_empty()); + /// + /// // When multiple patterns are present, since this returns the union, + /// // it will include look-around assertions that only appear in one + /// // pattern. + /// let nfa = NFA::new_many(&["a", "b", "a^b", "c"])?; + /// assert!(nfa.look_set_any().contains(Look::Start)); + /// + /// // Some groups of assertions have various shortcuts. For example: + /// let nfa = NFA::new(r"(?-u:\b)")?; + /// assert!(nfa.look_set_any().contains_word()); + /// assert!(!nfa.look_set_any().contains_word_unicode()); + /// assert!(nfa.look_set_any().contains_word_ascii()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn look_set_any(&self) -> LookSet { + self.0.look_set_any + } + + /// Returns the union of all prefix look-around assertions for every + /// pattern in this NFA. When the returned set is empty, it implies none of + /// the patterns require moving through a conditional epsilon transition + /// before inspecting the first byte in the haystack. + /// + /// This can be useful for determining what kinds of assertions need to be + /// satisfied at the beginning of a search. For example, typically DFAs + /// in this crate will build a distinct starting state for each possible + /// starting configuration that might result in look-around assertions + /// being satisfied differently. However, if the set returned here is + /// empty, then you know that the start state is invariant because there + /// are no conditional epsilon transitions to consider. + /// + /// # Example + /// + /// This example shows how this routine varies based on the regex pattern: + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, util::look::Look}; + /// + /// // No look-around at all. + /// let nfa = NFA::new("a")?; + /// assert!(nfa.look_set_prefix_any().is_empty()); + /// + /// // When multiple patterns are present, since this returns the union, + /// // it will include look-around assertions that only appear in one + /// // pattern. But it will only include assertions that are in the prefix + /// // of a pattern. For example, this includes '^' but not '$' even though + /// // '$' does appear. + /// let nfa = NFA::new_many(&["a", "b", "^ab$", "c"])?; + /// assert!(nfa.look_set_prefix_any().contains(Look::Start)); + /// assert!(!nfa.look_set_prefix_any().contains(Look::End)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn look_set_prefix_any(&self) -> LookSet { + self.0.look_set_prefix_any + } + + // FIXME: The `look_set_prefix_all` computation was not correct, and it + // seemed a little tricky to fix it. Since I wasn't actually using it for + // anything, I just decided to remove it in the run up to the regex 1.9 + // release. If you need this, please file an issue. + /* + /// Returns the intersection of all prefix look-around assertions for every + /// pattern in this NFA. When the returned set is empty, it implies at + /// least one of the patterns does not require moving through a conditional + /// epsilon transition before inspecting the first byte in the haystack. + /// Conversely, when the set contains an assertion, it implies that every + /// pattern in the NFA also contains that assertion in its prefix. + /// + /// This can be useful for determining what kinds of assertions need to be + /// satisfied at the beginning of a search. For example, if you know that + /// [`Look::Start`] is in the prefix intersection set returned here, then + /// you know that all searches, regardless of input configuration, will be + /// anchored. + /// + /// # Example + /// + /// This example shows how this routine varies based on the regex pattern: + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, util::look::Look}; + /// + /// // No look-around at all. + /// let nfa = NFA::new("a")?; + /// assert!(nfa.look_set_prefix_all().is_empty()); + /// + /// // When multiple patterns are present, since this returns the + /// // intersection, it will only include assertions present in every + /// // prefix, and only the prefix. + /// let nfa = NFA::new_many(&["^a$", "^b$", "$^ab$", "^c$"])?; + /// assert!(nfa.look_set_prefix_all().contains(Look::Start)); + /// assert!(!nfa.look_set_prefix_all().contains(Look::End)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn look_set_prefix_all(&self) -> LookSet { + self.0.look_set_prefix_all + } + */ + + /// Returns the memory usage, in bytes, of this NFA. + /// + /// This does **not** include the stack size used up by this NFA. To + /// compute that, use `std::mem::size_of::<NFA>()`. + /// + /// # Example + /// + /// This example shows that large Unicode character classes can use quite + /// a bit of memory. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::nfa::thompson::NFA; + /// + /// let nfa_unicode = NFA::new(r"\w")?; + /// let nfa_ascii = NFA::new(r"(?-u:\w)")?; + /// + /// assert!(10 * nfa_ascii.memory_usage() < nfa_unicode.memory_usage()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn memory_usage(&self) -> usize { + use core::mem::size_of; + + size_of::<Inner>() // allocated on the heap via Arc + + self.0.states.len() * size_of::<State>() + + self.0.start_pattern.len() * size_of::<StateID>() + + self.0.group_info.memory_usage() + + self.0.memory_extra + } +} + +impl fmt::Debug for NFA { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// The "inner" part of the NFA. We split this part out so that we can easily +/// wrap it in an `Arc` above in the definition of `NFA`. +/// +/// See builder.rs for the code that actually builds this type. This module +/// does provide (internal) mutable methods for adding things to this +/// NFA before finalizing it, but the high level construction process is +/// controlled by the builder abstraction. (Which is complicated enough to +/// get its own module.) +#[derive(Default)] +pub(super) struct Inner { + /// The state sequence. This sequence is guaranteed to be indexable by all + /// starting state IDs, and it is also guaranteed to contain at most one + /// `Match` state for each pattern compiled into this NFA. (A pattern may + /// not have a corresponding `Match` state if a `Match` state is impossible + /// to reach.) + states: Vec<State>, + /// The anchored starting state of this NFA. + start_anchored: StateID, + /// The unanchored starting state of this NFA. + start_unanchored: StateID, + /// The starting states for each individual pattern. Starting at any + /// of these states will result in only an anchored search for the + /// corresponding pattern. The vec is indexed by pattern ID. When the NFA + /// contains a single regex, then `start_pattern[0]` and `start_anchored` + /// are always equivalent. + start_pattern: Vec<StateID>, + /// Info about the capturing groups in this NFA. This is responsible for + /// mapping groups to slots, mapping groups to names and names to groups. + group_info: GroupInfo, + /// A representation of equivalence classes over the transitions in this + /// NFA. Two bytes in the same equivalence class must not discriminate + /// between a match or a non-match. This map can be used to shrink the + /// total size of a DFA's transition table with a small match-time cost. + /// + /// Note that the NFA's transitions are *not* defined in terms of these + /// equivalence classes. The NFA's transitions are defined on the original + /// byte values. For the most part, this is because they wouldn't really + /// help the NFA much since the NFA already uses a sparse representation + /// to represent transitions. Byte classes are most effective in a dense + /// representation. + byte_class_set: ByteClassSet, + /// This is generated from `byte_class_set`, and essentially represents the + /// same thing but supports different access patterns. Namely, this permits + /// looking up the equivalence class of a byte very cheaply. + /// + /// Ideally we would just store this, but because of annoying code + /// structure reasons, we keep both this and `byte_class_set` around for + /// now. I think I would prefer that `byte_class_set` were computed in the + /// `Builder`, but right now, we compute it as states are added to the + /// `NFA`. + byte_classes: ByteClasses, + /// Whether this NFA has a `Capture` state anywhere. + has_capture: bool, + /// When the empty string is in the language matched by this NFA. + has_empty: bool, + /// Whether UTF-8 mode is enabled for this NFA. Briefly, this means that + /// all non-empty matches produced by this NFA correspond to spans of valid + /// UTF-8, and any empty matches produced by this NFA that split a UTF-8 + /// encoded codepoint should be filtered out by the corresponding regex + /// engine. + utf8: bool, + /// Whether this NFA is meant to be matched in reverse or not. + reverse: bool, + /// The matcher to be used for look-around assertions. + look_matcher: LookMatcher, + /// The union of all look-around assertions that occur anywhere within + /// this NFA. If this set is empty, then it means there are precisely zero + /// conditional epsilon transitions in the NFA. + look_set_any: LookSet, + /// The union of all look-around assertions that occur as a zero-length + /// prefix for any of the patterns in this NFA. + look_set_prefix_any: LookSet, + /* + /// The intersection of all look-around assertions that occur as a + /// zero-length prefix for any of the patterns in this NFA. + look_set_prefix_all: LookSet, + */ + /// Heap memory used indirectly by NFA states and other things (like the + /// various capturing group representations above). Since each state + /// might use a different amount of heap, we need to keep track of this + /// incrementally. + memory_extra: usize, +} + +impl Inner { + /// Runs any last finalization bits and turns this into a full NFA. + pub(super) fn into_nfa(mut self) -> NFA { + self.byte_classes = self.byte_class_set.byte_classes(); + // Do epsilon closure from the start state of every pattern in order + // to compute various properties such as look-around assertions and + // whether the empty string can be matched. + let mut stack = vec![]; + let mut seen = SparseSet::new(self.states.len()); + for &start_id in self.start_pattern.iter() { + stack.push(start_id); + seen.clear(); + // let mut prefix_all = LookSet::full(); + let mut prefix_any = LookSet::empty(); + while let Some(sid) = stack.pop() { + if !seen.insert(sid) { + continue; + } + match self.states[sid] { + State::ByteRange { .. } + | State::Dense { .. } + | State::Fail => continue, + State::Sparse(_) => { + // This snippet below will rewrite this sparse state + // as a dense state. By doing it here, we apply this + // optimization to all hot "sparse" states since these + // are the states that are reachable from the start + // state via an epsilon closure. + // + // Unfortunately, this optimization did not seem to + // help much in some very limited ad hoc benchmarking. + // + // I left the 'Dense' state type in place in case we + // want to revisit this, but I suspect the real way + // to make forward progress is a more fundamental + // re-architecting of how data in the NFA is laid out. + // I think we should consider a single contiguous + // allocation instead of all this indirection and + // potential heap allocations for every state. But this + // is a large re-design and will require API breaking + // changes. + // self.memory_extra -= self.states[sid].memory_usage(); + // let trans = DenseTransitions::from_sparse(sparse); + // self.states[sid] = State::Dense(trans); + // self.memory_extra += self.states[sid].memory_usage(); + continue; + } + State::Match { .. } => self.has_empty = true, + State::Look { look, next } => { + prefix_any = prefix_any.insert(look); + stack.push(next); + } + State::Union { ref alternates } => { + // Order doesn't matter here, since we're just dealing + // with look-around sets. But if we do richer analysis + // here that needs to care about preference order, then + // this should be done in reverse. + stack.extend(alternates.iter()); + } + State::BinaryUnion { alt1, alt2 } => { + stack.push(alt2); + stack.push(alt1); + } + State::Capture { next, .. } => { + stack.push(next); + } + } + } + self.look_set_prefix_any = + self.look_set_prefix_any.union(prefix_any); + } + self.states.shrink_to_fit(); + self.start_pattern.shrink_to_fit(); + NFA(Arc::new(self)) + } + + /// Returns the capturing group info for this NFA. + pub(super) fn group_info(&self) -> &GroupInfo { + &self.group_info + } + + /// Add the given state to this NFA after allocating a fresh identifier for + /// it. + /// + /// This panics if too many states are added such that a fresh identifier + /// could not be created. (Currently, the only caller of this routine is + /// a `Builder`, and it upholds this invariant.) + pub(super) fn add(&mut self, state: State) -> StateID { + match state { + State::ByteRange { ref trans } => { + self.byte_class_set.set_range(trans.start, trans.end); + } + State::Sparse(ref sparse) => { + for trans in sparse.transitions.iter() { + self.byte_class_set.set_range(trans.start, trans.end); + } + } + State::Dense { .. } => unreachable!(), + State::Look { look, .. } => { + self.look_matcher + .add_to_byteset(look, &mut self.byte_class_set); + self.look_set_any = self.look_set_any.insert(look); + } + State::Capture { .. } => { + self.has_capture = true; + } + State::Union { .. } + | State::BinaryUnion { .. } + | State::Fail + | State::Match { .. } => {} + } + + let id = StateID::new(self.states.len()).unwrap(); + self.memory_extra += state.memory_usage(); + self.states.push(state); + id + } + + /// Set the starting state identifiers for this NFA. + /// + /// `start_anchored` and `start_unanchored` may be equivalent. When they + /// are, then the NFA can only execute anchored searches. This might + /// occur, for example, for patterns that are unconditionally anchored. + /// e.g., `^foo`. + pub(super) fn set_starts( + &mut self, + start_anchored: StateID, + start_unanchored: StateID, + start_pattern: &[StateID], + ) { + self.start_anchored = start_anchored; + self.start_unanchored = start_unanchored; + self.start_pattern = start_pattern.to_vec(); + } + + /// Sets the UTF-8 mode of this NFA. + pub(super) fn set_utf8(&mut self, yes: bool) { + self.utf8 = yes; + } + + /// Sets the reverse mode of this NFA. + pub(super) fn set_reverse(&mut self, yes: bool) { + self.reverse = yes; + } + + /// Sets the look-around assertion matcher for this NFA. + pub(super) fn set_look_matcher(&mut self, m: LookMatcher) { + self.look_matcher = m; + } + + /// Set the capturing groups for this NFA. + /// + /// The given slice should contain the capturing groups for each pattern, + /// The capturing groups in turn should correspond to the total number of + /// capturing groups in the pattern, including the anonymous first capture + /// group for each pattern. If a capturing group does have a name, then it + /// should be provided as a Arc<str>. + /// + /// This returns an error if a corresponding `GroupInfo` could not be + /// built. + pub(super) fn set_captures( + &mut self, + captures: &[Vec<Option<Arc<str>>>], + ) -> Result<(), GroupInfoError> { + self.group_info = GroupInfo::new( + captures.iter().map(|x| x.iter().map(|y| y.as_ref())), + )?; + Ok(()) + } + + /// Remap the transitions in every state of this NFA using the given map. + /// The given map should be indexed according to state ID namespace used by + /// the transitions of the states currently in this NFA. + /// + /// This is particularly useful to the NFA builder, since it is convenient + /// to add NFA states in order to produce their final IDs. Then, after all + /// of the intermediate "empty" states (unconditional epsilon transitions) + /// have been removed from the builder's representation, we can re-map all + /// of the transitions in the states already added to their final IDs. + pub(super) fn remap(&mut self, old_to_new: &[StateID]) { + for state in &mut self.states { + state.remap(old_to_new); + } + self.start_anchored = old_to_new[self.start_anchored]; + self.start_unanchored = old_to_new[self.start_unanchored]; + for id in self.start_pattern.iter_mut() { + *id = old_to_new[*id]; + } + } +} + +impl fmt::Debug for Inner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "thompson::NFA(")?; + for (sid, state) in self.states.iter().with_state_ids() { + let status = if sid == self.start_anchored { + '^' + } else if sid == self.start_unanchored { + '>' + } else { + ' ' + }; + writeln!(f, "{}{:06?}: {:?}", status, sid.as_usize(), state)?; + } + let pattern_len = self.start_pattern.len(); + if pattern_len > 1 { + writeln!(f)?; + for pid in 0..pattern_len { + let sid = self.start_pattern[pid]; + writeln!(f, "START({:06?}): {:?}", pid, sid.as_usize())?; + } + } + writeln!(f)?; + writeln!( + f, + "transition equivalence classes: {:?}", + self.byte_classes, + )?; + writeln!(f, ")")?; + Ok(()) + } +} + +/// A state in an NFA. +/// +/// In theory, it can help to conceptualize an `NFA` as a graph consisting of +/// `State`s. Each `State` contains its complete set of outgoing transitions. +/// +/// In practice, it can help to conceptualize an `NFA` as a sequence of +/// instructions for a virtual machine. Each `State` says what to do and where +/// to go next. +/// +/// Strictly speaking, the practical interpretation is the most correct one, +/// because of the [`Capture`](State::Capture) state. Namely, a `Capture` +/// state always forwards execution to another state unconditionally. Its only +/// purpose is to cause a side effect: the recording of the current input +/// position at a particular location in memory. In this sense, an `NFA` +/// has more power than a theoretical non-deterministic finite automaton. +/// +/// For most uses of this crate, it is likely that one may never even need to +/// be aware of this type at all. The main use cases for looking at `State`s +/// directly are if you need to write your own search implementation or if you +/// need to do some kind of analysis on the NFA. +#[derive(Clone, Eq, PartialEq)] +pub enum State { + /// A state with a single transition that can only be taken if the current + /// input symbol is in a particular range of bytes. + ByteRange { + /// The transition from this state to the next. + trans: Transition, + }, + /// A state with possibly many transitions represented in a sparse fashion. + /// Transitions are non-overlapping and ordered lexicographically by input + /// range. + /// + /// In practice, this is used for encoding UTF-8 automata. Its presence is + /// primarily an optimization that avoids many additional unconditional + /// epsilon transitions (via [`Union`](State::Union) states), and thus + /// decreases the overhead of traversing the NFA. This can improve both + /// matching time and DFA construction time. + Sparse(SparseTransitions), + /// A dense representation of a state with multiple transitions. + Dense(DenseTransitions), + /// A conditional epsilon transition satisfied via some sort of + /// look-around. Look-around is limited to anchor and word boundary + /// assertions. + /// + /// Look-around states are meant to be evaluated while performing epsilon + /// closure (computing the set of states reachable from a particular state + /// via only epsilon transitions). If the current position in the haystack + /// satisfies the look-around assertion, then you're permitted to follow + /// that epsilon transition. + Look { + /// The look-around assertion that must be satisfied before moving + /// to `next`. + look: Look, + /// The state to transition to if the look-around assertion is + /// satisfied. + next: StateID, + }, + /// An alternation such that there exists an epsilon transition to all + /// states in `alternates`, where matches found via earlier transitions + /// are preferred over later transitions. + Union { + /// An ordered sequence of unconditional epsilon transitions to other + /// states. Transitions earlier in the sequence are preferred over + /// transitions later in the sequence. + alternates: Box<[StateID]>, + }, + /// An alternation such that there exists precisely two unconditional + /// epsilon transitions, where matches found via `alt1` are preferred over + /// matches found via `alt2`. + /// + /// This state exists as a common special case of Union where there are + /// only two alternates. In this case, we don't need any allocations to + /// represent the state. This saves a bit of memory and also saves an + /// additional memory access when traversing the NFA. + BinaryUnion { + /// An unconditional epsilon transition to another NFA state. This + /// is preferred over `alt2`. + alt1: StateID, + /// An unconditional epsilon transition to another NFA state. Matches + /// reported via this transition should only be reported if no matches + /// were found by following `alt1`. + alt2: StateID, + }, + /// An empty state that records a capture location. + /// + /// From the perspective of finite automata, this is precisely equivalent + /// to an unconditional epsilon transition, but serves the purpose of + /// instructing NFA simulations to record additional state when the finite + /// state machine passes through this epsilon transition. + /// + /// `slot` in this context refers to the specific capture group slot + /// offset that is being recorded. Each capturing group has two slots + /// corresponding to the start and end of the matching portion of that + /// group. + /// + /// The pattern ID and capture group index are also included in this state + /// in case they are useful. But mostly, all you'll need is `next` and + /// `slot`. + Capture { + /// The state to transition to, unconditionally. + next: StateID, + /// The pattern ID that this capture belongs to. + pattern_id: PatternID, + /// The capture group index that this capture belongs to. Capture group + /// indices are local to each pattern. For example, when capturing + /// groups are enabled, every pattern has a capture group at index + /// `0`. + group_index: SmallIndex, + /// The slot index for this capture. Every capturing group has two + /// slots: one for the start haystack offset and one for the end + /// haystack offset. Unlike capture group indices, slot indices are + /// global across all patterns in this NFA. That is, each slot belongs + /// to a single pattern, but there is only one slot at index `i`. + slot: SmallIndex, + }, + /// A state that cannot be transitioned out of. This is useful for cases + /// where you want to prevent matching from occurring. For example, if your + /// regex parser permits empty character classes, then one could choose + /// a `Fail` state to represent them. (An empty character class can be + /// thought of as an empty set. Since nothing is in an empty set, they can + /// never match anything.) + Fail, + /// A match state. There is at least one such occurrence of this state for + /// each regex that can match that is in this NFA. + Match { + /// The matching pattern ID. + pattern_id: PatternID, + }, +} + +impl State { + /// Returns true if and only if this state contains one or more epsilon + /// transitions. + /// + /// In practice, a state has no outgoing transitions (like `Match`), has + /// only non-epsilon transitions (like `ByteRange`) or has only epsilon + /// transitions (like `Union`). + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{State, Transition}, + /// util::primitives::{PatternID, StateID, SmallIndex}, + /// }; + /// + /// // Capture states are epsilon transitions. + /// let state = State::Capture { + /// next: StateID::ZERO, + /// pattern_id: PatternID::ZERO, + /// group_index: SmallIndex::ZERO, + /// slot: SmallIndex::ZERO, + /// }; + /// assert!(state.is_epsilon()); + /// + /// // ByteRange states are not. + /// let state = State::ByteRange { + /// trans: Transition { start: b'a', end: b'z', next: StateID::ZERO }, + /// }; + /// assert!(!state.is_epsilon()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_epsilon(&self) -> bool { + match *self { + State::ByteRange { .. } + | State::Sparse { .. } + | State::Dense { .. } + | State::Fail + | State::Match { .. } => false, + State::Look { .. } + | State::Union { .. } + | State::BinaryUnion { .. } + | State::Capture { .. } => true, + } + } + + /// Returns the heap memory usage of this NFA state in bytes. + fn memory_usage(&self) -> usize { + match *self { + State::ByteRange { .. } + | State::Look { .. } + | State::BinaryUnion { .. } + | State::Capture { .. } + | State::Match { .. } + | State::Fail => 0, + State::Sparse(SparseTransitions { ref transitions }) => { + transitions.len() * mem::size_of::<Transition>() + } + State::Dense { .. } => 256 * mem::size_of::<StateID>(), + State::Union { ref alternates } => { + alternates.len() * mem::size_of::<StateID>() + } + } + } + + /// Remap the transitions in this state using the given map. Namely, the + /// given map should be indexed according to the transitions currently + /// in this state. + /// + /// This is used during the final phase of the NFA compiler, which turns + /// its intermediate NFA into the final NFA. + fn remap(&mut self, remap: &[StateID]) { + match *self { + State::ByteRange { ref mut trans } => { + trans.next = remap[trans.next] + } + State::Sparse(SparseTransitions { ref mut transitions }) => { + for t in transitions.iter_mut() { + t.next = remap[t.next]; + } + } + State::Dense(DenseTransitions { ref mut transitions }) => { + for sid in transitions.iter_mut() { + *sid = remap[*sid]; + } + } + State::Look { ref mut next, .. } => *next = remap[*next], + State::Union { ref mut alternates } => { + for alt in alternates.iter_mut() { + *alt = remap[*alt]; + } + } + State::BinaryUnion { ref mut alt1, ref mut alt2 } => { + *alt1 = remap[*alt1]; + *alt2 = remap[*alt2]; + } + State::Capture { ref mut next, .. } => *next = remap[*next], + State::Fail => {} + State::Match { .. } => {} + } + } +} + +impl fmt::Debug for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + State::ByteRange { ref trans } => trans.fmt(f), + State::Sparse(SparseTransitions { ref transitions }) => { + let rs = transitions + .iter() + .map(|t| format!("{t:?}")) + .collect::<Vec<String>>() + .join(", "); + write!(f, "sparse({rs})") + } + State::Dense(ref dense) => { + write!(f, "dense(")?; + for (i, t) in dense.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{t:?}")?; + } + write!(f, ")") + } + State::Look { ref look, next } => { + write!(f, "{:?} => {:?}", look, next.as_usize()) + } + State::Union { ref alternates } => { + let alts = alternates + .iter() + .map(|id| format!("{:?}", id.as_usize())) + .collect::<Vec<String>>() + .join(", "); + write!(f, "union({alts})") + } + State::BinaryUnion { alt1, alt2 } => { + write!( + f, + "binary-union({}, {})", + alt1.as_usize(), + alt2.as_usize() + ) + } + State::Capture { next, pattern_id, group_index, slot } => { + write!( + f, + "capture(pid={:?}, group={:?}, slot={:?}) => {:?}", + pattern_id.as_usize(), + group_index.as_usize(), + slot.as_usize(), + next.as_usize(), + ) + } + State::Fail => write!(f, "FAIL"), + State::Match { pattern_id } => { + write!(f, "MATCH({:?})", pattern_id.as_usize()) + } + } + } +} + +/// A sequence of transitions used to represent a sparse state. +/// +/// This is the primary representation of a [`Sparse`](State::Sparse) state. +/// It corresponds to a sorted sequence of transitions with non-overlapping +/// byte ranges. If the byte at the current position in the haystack matches +/// one of the byte ranges, then the finite state machine should take the +/// corresponding transition. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseTransitions { + /// The sorted sequence of non-overlapping transitions. + pub transitions: Box<[Transition]>, +} + +impl SparseTransitions { + /// This follows the matching transition for a particular byte. + /// + /// The matching transition is found by looking for a matching byte + /// range (there is at most one) corresponding to the position `at` in + /// `haystack`. + /// + /// If `at >= haystack.len()`, then this returns `None`. + #[inline] + pub fn matches(&self, haystack: &[u8], at: usize) -> Option<StateID> { + haystack.get(at).and_then(|&b| self.matches_byte(b)) + } + + /// This follows the matching transition for any member of the alphabet. + /// + /// The matching transition is found by looking for a matching byte + /// range (there is at most one) corresponding to the position `at` in + /// `haystack`. If the given alphabet unit is [`EOI`](alphabet::Unit::eoi), + /// then this always returns `None`. + #[inline] + pub(crate) fn matches_unit( + &self, + unit: alphabet::Unit, + ) -> Option<StateID> { + unit.as_u8().and_then(|byte| self.matches_byte(byte)) + } + + /// This follows the matching transition for a particular byte. + /// + /// The matching transition is found by looking for a matching byte range + /// (there is at most one) corresponding to the byte given. + #[inline] + pub fn matches_byte(&self, byte: u8) -> Option<StateID> { + for t in self.transitions.iter() { + if t.start > byte { + break; + } else if t.matches_byte(byte) { + return Some(t.next); + } + } + None + + /* + // This is an alternative implementation that uses binary search. In + // some ad hoc experiments, like + // + // regex-cli find match pikevm -b -p '\b\w+\b' non-ascii-file + // + // I could not observe any improvement, and in fact, things seemed to + // be a bit slower. I can see an improvement in at least one benchmark: + // + // regex-cli find match pikevm -b -p '\pL{100}' all-codepoints-utf8 + // + // Where total search time goes from 3.2s to 2.4s when using binary + // search. + self.transitions + .binary_search_by(|t| { + if t.end < byte { + core::cmp::Ordering::Less + } else if t.start > byte { + core::cmp::Ordering::Greater + } else { + core::cmp::Ordering::Equal + } + }) + .ok() + .map(|i| self.transitions[i].next) + */ + } +} + +/// A sequence of transitions used to represent a dense state. +/// +/// This is the primary representation of a [`Dense`](State::Dense) state. It +/// provides constant time matching. That is, given a byte in a haystack and +/// a `DenseTransitions`, one can determine if the state matches in constant +/// time. +/// +/// This is in contrast to `SparseTransitions`, whose time complexity is +/// necessarily bigger than constant time. Also in contrast, `DenseTransitions` +/// usually requires (much) more heap memory. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DenseTransitions { + /// A dense representation of this state's transitions on the heap. This + /// always has length 256. + pub transitions: Box<[StateID]>, +} + +impl DenseTransitions { + /// This follows the matching transition for a particular byte. + /// + /// The matching transition is found by looking for a transition that + /// doesn't correspond to `StateID::ZERO` for the byte `at` the given + /// position in `haystack`. + /// + /// If `at >= haystack.len()`, then this returns `None`. + #[inline] + pub fn matches(&self, haystack: &[u8], at: usize) -> Option<StateID> { + haystack.get(at).and_then(|&b| self.matches_byte(b)) + } + + /// This follows the matching transition for any member of the alphabet. + /// + /// The matching transition is found by looking for a transition that + /// doesn't correspond to `StateID::ZERO` for the given alphabet `unit`. + /// + /// If the given alphabet unit is [`EOI`](alphabet::Unit::eoi), then + /// this returns `None`. + #[inline] + pub(crate) fn matches_unit( + &self, + unit: alphabet::Unit, + ) -> Option<StateID> { + unit.as_u8().and_then(|byte| self.matches_byte(byte)) + } + + /// This follows the matching transition for a particular byte. + /// + /// The matching transition is found by looking for a transition that + /// doesn't correspond to `StateID::ZERO` for the given `byte`. + #[inline] + pub fn matches_byte(&self, byte: u8) -> Option<StateID> { + let next = self.transitions[usize::from(byte)]; + if next == StateID::ZERO { + None + } else { + Some(next) + } + } + + /* + /// The dense state optimization isn't currently enabled, so permit a + /// little bit of dead code. + pub(crate) fn from_sparse(sparse: &SparseTransitions) -> DenseTransitions { + let mut dense = vec![StateID::ZERO; 256]; + for t in sparse.transitions.iter() { + for b in t.start..=t.end { + dense[usize::from(b)] = t.next; + } + } + DenseTransitions { transitions: dense.into_boxed_slice() } + } + */ + + /// Returns an iterator over all transitions that don't point to + /// `StateID::ZERO`. + pub(crate) fn iter(&self) -> impl Iterator<Item = Transition> + '_ { + use crate::util::int::Usize; + self.transitions + .iter() + .enumerate() + .filter(|&(_, &sid)| sid != StateID::ZERO) + .map(|(byte, &next)| Transition { + start: byte.as_u8(), + end: byte.as_u8(), + next, + }) + } +} + +/// A single transition to another state. +/// +/// This transition may only be followed if the current byte in the haystack +/// falls in the inclusive range of bytes specified. +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub struct Transition { + /// The inclusive start of the byte range. + pub start: u8, + /// The inclusive end of the byte range. + pub end: u8, + /// The identifier of the state to transition to. + pub next: StateID, +} + +impl Transition { + /// Returns true if the position `at` in `haystack` falls in this + /// transition's range of bytes. + /// + /// If `at >= haystack.len()`, then this returns `false`. + pub fn matches(&self, haystack: &[u8], at: usize) -> bool { + haystack.get(at).map_or(false, |&b| self.matches_byte(b)) + } + + /// Returns true if the given alphabet unit falls in this transition's + /// range of bytes. If the given unit is [`EOI`](alphabet::Unit::eoi), then + /// this returns `false`. + pub fn matches_unit(&self, unit: alphabet::Unit) -> bool { + unit.as_u8().map_or(false, |byte| self.matches_byte(byte)) + } + + /// Returns true if the given byte falls in this transition's range of + /// bytes. + pub fn matches_byte(&self, byte: u8) -> bool { + self.start <= byte && byte <= self.end + } +} + +impl fmt::Debug for Transition { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use crate::util::escape::DebugByte; + + let Transition { start, end, next } = *self; + if self.start == self.end { + write!(f, "{:?} => {:?}", DebugByte(start), next.as_usize()) + } else { + write!( + f, + "{:?}-{:?} => {:?}", + DebugByte(start), + DebugByte(end), + next.as_usize(), + ) + } + } +} + +/// An iterator over all pattern IDs in an NFA. +/// +/// This iterator is created by [`NFA::patterns`]. +/// +/// The lifetime parameter `'a` refers to the lifetime of the NFA from which +/// this pattern iterator was created. +#[derive(Debug)] +pub struct PatternIter<'a> { + it: PatternIDIter, + /// We explicitly associate a lifetime with this iterator even though we + /// don't actually borrow anything from the NFA. We do this for backward + /// compatibility purposes. If we ever do need to borrow something from + /// the NFA, then we can and just get rid of this marker without breaking + /// the public API. + _marker: core::marker::PhantomData<&'a ()>, +} + +impl<'a> Iterator for PatternIter<'a> { + type Item = PatternID; + + fn next(&mut self) -> Option<PatternID> { + self.it.next() + } +} + +#[cfg(all(test, feature = "nfa-pikevm"))] +mod tests { + use super::*; + use crate::{nfa::thompson::pikevm::PikeVM, Input}; + + // This asserts that an NFA state doesn't have its size changed. It is + // *really* easy to accidentally increase the size, and thus potentially + // dramatically increase the memory usage of every NFA. + // + // This assert doesn't mean we absolutely cannot increase the size of an + // NFA state. We can. It's just here to make sure we do it knowingly and + // intentionally. + #[test] + fn state_has_small_size() { + #[cfg(target_pointer_width = "64")] + assert_eq!(24, core::mem::size_of::<State>()); + #[cfg(target_pointer_width = "32")] + assert_eq!(20, core::mem::size_of::<State>()); + } + + #[test] + fn always_match() { + let re = PikeVM::new_from_nfa(NFA::always_match()).unwrap(); + let mut cache = re.create_cache(); + let mut caps = re.create_captures(); + let mut find = |haystack, start, end| { + let input = Input::new(haystack).range(start..end); + re.search(&mut cache, &input, &mut caps); + caps.get_match().map(|m| m.end()) + }; + + assert_eq!(Some(0), find("", 0, 0)); + assert_eq!(Some(0), find("a", 0, 1)); + assert_eq!(Some(1), find("a", 1, 1)); + assert_eq!(Some(0), find("ab", 0, 2)); + assert_eq!(Some(1), find("ab", 1, 2)); + assert_eq!(Some(2), find("ab", 2, 2)); + } + + #[test] + fn never_match() { + let re = PikeVM::new_from_nfa(NFA::never_match()).unwrap(); + let mut cache = re.create_cache(); + let mut caps = re.create_captures(); + let mut find = |haystack, start, end| { + let input = Input::new(haystack).range(start..end); + re.search(&mut cache, &input, &mut caps); + caps.get_match().map(|m| m.end()) + }; + + assert_eq!(None, find("", 0, 0)); + assert_eq!(None, find("a", 0, 1)); + assert_eq!(None, find("a", 1, 1)); + assert_eq!(None, find("ab", 0, 2)); + assert_eq!(None, find("ab", 1, 2)); + assert_eq!(None, find("ab", 2, 2)); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/pikevm.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/pikevm.rs new file mode 100644 index 0000000..a5cd708 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/pikevm.rs @@ -0,0 +1,2359 @@ +/*! +An NFA backed Pike VM for executing regex searches with capturing groups. + +This module provides a [`PikeVM`] that works by simulating an NFA and +resolving all spans of capturing groups that participate in a match. +*/ + +#[cfg(feature = "internal-instrument-pikevm")] +use core::cell::RefCell; + +use alloc::{vec, vec::Vec}; + +use crate::{ + nfa::thompson::{self, BuildError, State, NFA}, + util::{ + captures::Captures, + empty, iter, + prefilter::Prefilter, + primitives::{NonMaxUsize, PatternID, SmallIndex, StateID}, + search::{ + Anchored, HalfMatch, Input, Match, MatchKind, PatternSet, Span, + }, + sparse_set::SparseSet, + }, +}; + +/// A simple macro for conditionally executing instrumentation logic when +/// the 'trace' log level is enabled. This is a compile-time no-op when the +/// 'internal-instrument-pikevm' feature isn't enabled. The intent here is that +/// this makes it easier to avoid doing extra work when instrumentation isn't +/// enabled. +/// +/// This macro accepts a closure of type `|&mut Counters|`. The closure can +/// then increment counters (or whatever) in accordance with what one wants +/// to track. +macro_rules! instrument { + ($fun:expr) => { + #[cfg(feature = "internal-instrument-pikevm")] + { + let fun: &mut dyn FnMut(&mut Counters) = &mut $fun; + COUNTERS.with(|c: &RefCell<Counters>| fun(&mut *c.borrow_mut())); + } + }; +} + +#[cfg(feature = "internal-instrument-pikevm")] +std::thread_local! { + /// Effectively global state used to keep track of instrumentation + /// counters. The "proper" way to do this is to thread it through the + /// PikeVM, but it makes the code quite icky. Since this is just a + /// debugging feature, we're content to relegate it to thread local + /// state. When instrumentation is enabled, the counters are reset at the + /// beginning of every search and printed (with the 'trace' log level) at + /// the end of every search. + static COUNTERS: RefCell<Counters> = RefCell::new(Counters::empty()); +} + +/// The configuration used for building a [`PikeVM`]. +/// +/// A PikeVM configuration is a simple data object that is typically used with +/// [`Builder::configure`]. It can be cheaply cloned. +/// +/// A default configuration can be created either with `Config::new`, or +/// perhaps more conveniently, with [`PikeVM::config`]. +#[derive(Clone, Debug, Default)] +pub struct Config { + match_kind: Option<MatchKind>, + pre: Option<Option<Prefilter>>, +} + +impl Config { + /// Return a new default PikeVM configuration. + pub fn new() -> Config { + Config::default() + } + + /// Set the desired match semantics. + /// + /// The default is [`MatchKind::LeftmostFirst`], which corresponds to the + /// match semantics of Perl-like regex engines. That is, when multiple + /// patterns would match at the same leftmost position, the pattern that + /// appears first in the concrete syntax is chosen. + /// + /// Currently, the only other kind of match semantics supported is + /// [`MatchKind::All`]. This corresponds to "classical DFA" construction + /// where all possible matches are visited in the NFA by the `PikeVM`. + /// + /// Typically, `All` is used when one wants to execute an overlapping + /// search and `LeftmostFirst` otherwise. In particular, it rarely makes + /// sense to use `All` with the various "leftmost" find routines, since the + /// leftmost routines depend on the `LeftmostFirst` automata construction + /// strategy. Specifically, `LeftmostFirst` results in the `PikeVM` + /// simulating dead states as a way to terminate the search and report a + /// match. `LeftmostFirst` also supports non-greedy matches using this + /// strategy where as `All` does not. + pub fn match_kind(mut self, kind: MatchKind) -> Config { + self.match_kind = Some(kind); + self + } + + /// Set a prefilter to be used whenever a start state is entered. + /// + /// A [`Prefilter`] in this context is meant to accelerate searches by + /// looking for literal prefixes that every match for the corresponding + /// pattern (or patterns) must start with. Once a prefilter produces a + /// match, the underlying search routine continues on to try and confirm + /// the match. + /// + /// Be warned that setting a prefilter does not guarantee that the search + /// will be faster. While it's usually a good bet, if the prefilter + /// produces a lot of false positive candidates (i.e., positions matched + /// by the prefilter but not by the regex), then the overall result can + /// be slower than if you had just executed the regex engine without any + /// prefilters. + /// + /// By default no prefilter is set. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::prefilter::Prefilter, + /// Input, Match, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["foo", "bar"]); + /// let re = PikeVM::builder() + /// .configure(PikeVM::config().prefilter(pre)) + /// .build(r"(foo|bar)[a-z]+")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("foo1 barfox bar"); + /// assert_eq!(Some(Match::must(0, 5..11)), re.find(&mut cache, input)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Be warned though that an incorrect prefilter can lead to incorrect + /// results! + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::prefilter::Prefilter, + /// Input, HalfMatch, MatchKind, + /// }; + /// + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["foo", "car"]); + /// let re = PikeVM::builder() + /// .configure(PikeVM::config().prefilter(pre)) + /// .build(r"(foo|bar)[a-z]+")?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("foo1 barfox bar"); + /// // No match reported even though there clearly is one! + /// assert_eq!(None, re.find(&mut cache, input)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn prefilter(mut self, pre: Option<Prefilter>) -> Config { + self.pre = Some(pre); + self + } + + /// Returns the match semantics set in this configuration. + pub fn get_match_kind(&self) -> MatchKind { + self.match_kind.unwrap_or(MatchKind::LeftmostFirst) + } + + /// Returns the prefilter set in this configuration, if one at all. + pub fn get_prefilter(&self) -> Option<&Prefilter> { + self.pre.as_ref().unwrap_or(&None).as_ref() + } + + /// Overwrite the default configuration such that the options in `o` are + /// always used. If an option in `o` is not set, then the corresponding + /// option in `self` is used. If it's not set in `self` either, then it + /// remains not set. + pub(crate) fn overwrite(&self, o: Config) -> Config { + Config { + match_kind: o.match_kind.or(self.match_kind), + pre: o.pre.or_else(|| self.pre.clone()), + } + } +} + +/// A builder for a `PikeVM`. +/// +/// This builder permits configuring options for the syntax of a pattern, +/// the NFA construction and the `PikeVM` construction. This builder is +/// different from a general purpose regex builder in that it permits fine +/// grain configuration of the construction process. The trade off for this is +/// complexity, and the possibility of setting a configuration that might not +/// make sense. For example, there are two different UTF-8 modes: +/// +/// * [`util::syntax::Config::utf8`](crate::util::syntax::Config::utf8) +/// controls whether the pattern itself can contain sub-expressions that match +/// invalid UTF-8. +/// * [`thompson::Config::utf8`] controls whether empty matches that split a +/// Unicode codepoint are reported or not. +/// +/// Generally speaking, callers will want to either enable all of these or +/// disable all of these. +/// +/// # Example +/// +/// This example shows how to disable UTF-8 mode in the syntax and the regex +/// itself. This is generally what you want for matching on arbitrary bytes. +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::{self, pikevm::PikeVM}, +/// util::syntax, +/// Match, +/// }; +/// +/// let re = PikeVM::builder() +/// .syntax(syntax::Config::new().utf8(false)) +/// .thompson(thompson::Config::new().utf8(false)) +/// .build(r"foo(?-u:[^b])ar.*")?; +/// let mut cache = re.create_cache(); +/// +/// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"; +/// let expected = Some(Match::must(0, 1..9)); +/// let got = re.find_iter(&mut cache, haystack).next(); +/// assert_eq!(expected, got); +/// // Notice that `(?-u:[^b])` matches invalid UTF-8, +/// // but the subsequent `.*` does not! Disabling UTF-8 +/// // on the syntax permits this. +/// // +/// // N.B. This example does not show the impact of +/// // disabling UTF-8 mode on a PikeVM Config, since that +/// // only impacts regexes that can produce matches of +/// // length 0. +/// assert_eq!(b"foo\xFFarzz", &haystack[got.unwrap().range()]); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Builder { + config: Config, + #[cfg(feature = "syntax")] + thompson: thompson::Compiler, +} + +impl Builder { + /// Create a new PikeVM builder with its default configuration. + pub fn new() -> Builder { + Builder { + config: Config::default(), + #[cfg(feature = "syntax")] + thompson: thompson::Compiler::new(), + } + } + + /// Build a `PikeVM` from the given pattern. + /// + /// If there was a problem parsing or compiling the pattern, then an error + /// is returned. + #[cfg(feature = "syntax")] + pub fn build(&self, pattern: &str) -> Result<PikeVM, BuildError> { + self.build_many(&[pattern]) + } + + /// Build a `PikeVM` from the given patterns. + #[cfg(feature = "syntax")] + pub fn build_many<P: AsRef<str>>( + &self, + patterns: &[P], + ) -> Result<PikeVM, BuildError> { + let nfa = self.thompson.build_many(patterns)?; + self.build_from_nfa(nfa) + } + + /// Build a `PikeVM` directly from its NFA. + /// + /// Note that when using this method, any configuration that applies to the + /// construction of the NFA itself will of course be ignored, since the NFA + /// given here is already built. + pub fn build_from_nfa(&self, nfa: NFA) -> Result<PikeVM, BuildError> { + nfa.look_set_any().available().map_err(BuildError::word)?; + Ok(PikeVM { config: self.config.clone(), nfa }) + } + + /// Apply the given `PikeVM` configuration options to this builder. + pub fn configure(&mut self, config: Config) -> &mut Builder { + self.config = self.config.overwrite(config); + self + } + + /// Set the syntax configuration for this builder using + /// [`syntax::Config`](crate::util::syntax::Config). + /// + /// This permits setting things like case insensitivity, Unicode and multi + /// line mode. + /// + /// These settings only apply when constructing a PikeVM directly from a + /// pattern. + #[cfg(feature = "syntax")] + pub fn syntax( + &mut self, + config: crate::util::syntax::Config, + ) -> &mut Builder { + self.thompson.syntax(config); + self + } + + /// Set the Thompson NFA configuration for this builder using + /// [`nfa::thompson::Config`](crate::nfa::thompson::Config). + /// + /// This permits setting things like if additional time should be spent + /// shrinking the size of the NFA. + /// + /// These settings only apply when constructing a PikeVM directly from a + /// pattern. + #[cfg(feature = "syntax")] + pub fn thompson(&mut self, config: thompson::Config) -> &mut Builder { + self.thompson.configure(config); + self + } +} + +/// A virtual machine for executing regex searches with capturing groups. +/// +/// # Infallible APIs +/// +/// Unlike most other regex engines in this crate, a `PikeVM` never returns an +/// error at search time. It supports all [`Anchored`] configurations, never +/// quits and works on haystacks of arbitrary length. +/// +/// There are two caveats to mention though: +/// +/// * If an invalid pattern ID is given to a search via [`Anchored::Pattern`], +/// then the PikeVM will report "no match." This is consistent with all other +/// regex engines in this crate. +/// * When using [`PikeVM::which_overlapping_matches`] with a [`PatternSet`] +/// that has insufficient capacity to store all valid pattern IDs, then if a +/// match occurs for a `PatternID` that cannot be inserted, it is silently +/// dropped as if it did not match. +/// +/// # Advice +/// +/// The `PikeVM` is generally the most "powerful" regex engine in this crate. +/// "Powerful" in this context means that it can handle any regular expression +/// that is parseable by `regex-syntax` and any size haystack. Regrettably, +/// the `PikeVM` is also simultaneously often the _slowest_ regex engine in +/// practice. This results in an annoying situation where one generally tries +/// to pick any other regex engine (or perhaps none at all) before being +/// forced to fall back to a `PikeVM`. +/// +/// For example, a common strategy for dealing with capturing groups is to +/// actually look for the overall match of the regex using a faster regex +/// engine, like a [lazy DFA](crate::hybrid::regex::Regex). Once the overall +/// match is found, one can then run the `PikeVM` on just the match span to +/// find the spans of the capturing groups. In this way, the faster regex +/// engine does the majority of the work, while the `PikeVM` only lends its +/// power in a more limited role. +/// +/// Unfortunately, this isn't always possible because the faster regex engines +/// don't support all of the regex features in `regex-syntax`. This notably +/// includes (and is currently limited to) Unicode word boundaries. So if +/// your pattern has Unicode word boundaries, you typically can't use a +/// DFA-based regex engine at all (unless you [enable heuristic support for +/// it](crate::hybrid::dfa::Config::unicode_word_boundary)). (The [one-pass +/// DFA](crate::dfa::onepass::DFA) can handle Unicode word boundaries for +/// anchored searches only, but in a cruel sort of joke, many Unicode features +/// tend to result in making the regex _not_ one-pass.) +/// +/// # Example +/// +/// This example shows that the `PikeVM` implements Unicode word boundaries +/// correctly by default. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; +/// +/// let re = PikeVM::new(r"\b\w+\b")?; +/// let mut cache = re.create_cache(); +/// +/// let mut it = re.find_iter(&mut cache, "Шерлок Холмс"); +/// assert_eq!(Some(Match::must(0, 0..12)), it.next()); +/// assert_eq!(Some(Match::must(0, 13..23)), it.next()); +/// assert_eq!(None, it.next()); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct PikeVM { + config: Config, + nfa: NFA, +} + +impl PikeVM { + /// Parse the given regular expression using the default configuration and + /// return the corresponding `PikeVM`. + /// + /// If you want a non-default configuration, then use the [`Builder`] to + /// set your own configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re = PikeVM::new("foo[0-9]+bar")?; + /// let mut cache = re.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 3..14)), + /// re.find_iter(&mut cache, "zzzfoo12345barzzz").next(), + /// ); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new(pattern: &str) -> Result<PikeVM, BuildError> { + PikeVM::builder().build(pattern) + } + + /// Like `new`, but parses multiple patterns into a single "multi regex." + /// This similarly uses the default regex configuration. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re = PikeVM::new_many(&["[a-z]+", "[0-9]+"])?; + /// let mut cache = re.create_cache(); + /// + /// let mut it = re.find_iter(&mut cache, "abc 1 foo 4567 0 quux"); + /// assert_eq!(Some(Match::must(0, 0..3)), it.next()); + /// assert_eq!(Some(Match::must(1, 4..5)), it.next()); + /// assert_eq!(Some(Match::must(0, 6..9)), it.next()); + /// assert_eq!(Some(Match::must(1, 10..14)), it.next()); + /// assert_eq!(Some(Match::must(1, 15..16)), it.next()); + /// assert_eq!(Some(Match::must(0, 17..21)), it.next()); + /// assert_eq!(None, it.next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn new_many<P: AsRef<str>>( + patterns: &[P], + ) -> Result<PikeVM, BuildError> { + PikeVM::builder().build_many(patterns) + } + + /// Like `new`, but builds a PikeVM directly from an NFA. This is useful + /// if you already have an NFA, or even if you hand-assembled the NFA. + /// + /// # Example + /// + /// This shows how to hand assemble a regular expression via its HIR, + /// compile an NFA from it and build a PikeVM from the NFA. + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, pikevm::PikeVM}, Match}; + /// use regex_syntax::hir::{Hir, Class, ClassBytes, ClassBytesRange}; + /// + /// let hir = Hir::class(Class::Bytes(ClassBytes::new(vec![ + /// ClassBytesRange::new(b'0', b'9'), + /// ClassBytesRange::new(b'A', b'Z'), + /// ClassBytesRange::new(b'_', b'_'), + /// ClassBytesRange::new(b'a', b'z'), + /// ]))); + /// + /// let config = NFA::config().nfa_size_limit(Some(1_000)); + /// let nfa = NFA::compiler().configure(config).build_from_hir(&hir)?; + /// + /// let re = PikeVM::new_from_nfa(nfa)?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let expected = Some(Match::must(0, 3..4)); + /// re.captures(&mut cache, "!@#A#@!", &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new_from_nfa(nfa: NFA) -> Result<PikeVM, BuildError> { + PikeVM::builder().build_from_nfa(nfa) + } + + /// Create a new `PikeVM` that matches every input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re = PikeVM::always_match()?; + /// let mut cache = re.create_cache(); + /// + /// let expected = Match::must(0, 0..0); + /// assert_eq!(Some(expected), re.find_iter(&mut cache, "").next()); + /// assert_eq!(Some(expected), re.find_iter(&mut cache, "foo").next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn always_match() -> Result<PikeVM, BuildError> { + let nfa = thompson::NFA::always_match(); + PikeVM::new_from_nfa(nfa) + } + + /// Create a new `PikeVM` that never matches any input. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::never_match()?; + /// let mut cache = re.create_cache(); + /// + /// assert_eq!(None, re.find_iter(&mut cache, "").next()); + /// assert_eq!(None, re.find_iter(&mut cache, "foo").next()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn never_match() -> Result<PikeVM, BuildError> { + let nfa = thompson::NFA::never_match(); + PikeVM::new_from_nfa(nfa) + } + + /// Return a default configuration for a `PikeVM`. + /// + /// This is a convenience routine to avoid needing to import the `Config` + /// type when customizing the construction of a `PikeVM`. + /// + /// # Example + /// + /// This example shows how to disable UTF-8 mode. When UTF-8 mode is + /// disabled, zero-width matches that split a codepoint are allowed. + /// Otherwise they are never reported. + /// + /// In the code below, notice that `""` is permitted to match positions + /// that split the encoding of a codepoint. + /// + /// ``` + /// use regex_automata::{nfa::thompson::{self, pikevm::PikeVM}, Match}; + /// + /// let re = PikeVM::builder() + /// .thompson(thompson::Config::new().utf8(false)) + /// .build(r"")?; + /// let mut cache = re.create_cache(); + /// + /// let haystack = "a☃z"; + /// let mut it = re.find_iter(&mut cache, haystack); + /// assert_eq!(Some(Match::must(0, 0..0)), it.next()); + /// assert_eq!(Some(Match::must(0, 1..1)), it.next()); + /// assert_eq!(Some(Match::must(0, 2..2)), it.next()); + /// assert_eq!(Some(Match::must(0, 3..3)), it.next()); + /// assert_eq!(Some(Match::must(0, 4..4)), it.next()); + /// assert_eq!(Some(Match::must(0, 5..5)), it.next()); + /// assert_eq!(None, it.next()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn config() -> Config { + Config::new() + } + + /// Return a builder for configuring the construction of a `PikeVM`. + /// + /// This is a convenience routine to avoid needing to import the + /// [`Builder`] type in common cases. + /// + /// # Example + /// + /// This example shows how to use the builder to disable UTF-8 mode + /// everywhere. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::{self, pikevm::PikeVM}, + /// util::syntax, + /// Match, + /// }; + /// + /// let re = PikeVM::builder() + /// .syntax(syntax::Config::new().utf8(false)) + /// .thompson(thompson::Config::new().utf8(false)) + /// .build(r"foo(?-u:[^b])ar.*")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// let haystack = b"\xFEfoo\xFFarzz\xE2\x98\xFF\n"; + /// let expected = Some(Match::must(0, 1..9)); + /// re.captures(&mut cache, haystack, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn builder() -> Builder { + Builder::new() + } + + /// Create a new empty set of capturing groups that is guaranteed to be + /// valid for the search APIs on this `PikeVM`. + /// + /// A `Captures` value created for a specific `PikeVM` cannot be used with + /// any other `PikeVM`. + /// + /// This is a convenience function for [`Captures::all`]. See the + /// [`Captures`] documentation for an explanation of its alternative + /// constructors that permit the `PikeVM` to do less work during a search, + /// and thus might make it faster. + pub fn create_captures(&self) -> Captures { + Captures::all(self.get_nfa().group_info().clone()) + } + + /// Create a new cache for this `PikeVM`. + /// + /// The cache returned should only be used for searches for this + /// `PikeVM`. If you want to reuse the cache for another `PikeVM`, then + /// you must call [`Cache::reset`] with that `PikeVM` (or, equivalently, + /// [`PikeVM::reset_cache`]). + pub fn create_cache(&self) -> Cache { + Cache::new(self) + } + + /// Reset the given cache such that it can be used for searching with the + /// this `PikeVM` (and only this `PikeVM`). + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different `PikeVM`. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different `PikeVM`. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re1 = PikeVM::new(r"\w")?; + /// let re2 = PikeVM::new(r"\W")?; + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 0..2)), + /// re1.find_iter(&mut cache, "Δ").next(), + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the PikeVM we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// re2.reset_cache(&mut cache); + /// assert_eq!( + /// Some(Match::must(0, 0..3)), + /// re2.find_iter(&mut cache, "☃").next(), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset_cache(&self, cache: &mut Cache) { + cache.reset(self); + } + + /// Returns the total number of patterns compiled into this `PikeVM`. + /// + /// In the case of a `PikeVM` that contains no patterns, this returns `0`. + /// + /// # Example + /// + /// This example shows the pattern length for a `PikeVM` that never + /// matches: + /// + /// ``` + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::never_match()?; + /// assert_eq!(re.pattern_len(), 0); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And another example for a `PikeVM` that matches at every position: + /// + /// ``` + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::always_match()?; + /// assert_eq!(re.pattern_len(), 1); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// And finally, a `PikeVM` that was constructed from multiple patterns: + /// + /// ``` + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::new_many(&["[0-9]+", "[a-z]+", "[A-Z]+"])?; + /// assert_eq!(re.pattern_len(), 3); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn pattern_len(&self) -> usize { + self.nfa.pattern_len() + } + + /// Return the config for this `PikeVM`. + #[inline] + pub fn get_config(&self) -> &Config { + &self.config + } + + /// Returns a reference to the underlying NFA. + #[inline] + pub fn get_nfa(&self) -> &NFA { + &self.nfa + } +} + +impl PikeVM { + /// Returns true if and only if this `PikeVM` matches the given haystack. + /// + /// This routine may short circuit if it knows that scanning future + /// input will never lead to a different result. In particular, if the + /// underlying NFA enters a match state, then this routine will return + /// `true` immediately without inspecting any future input. (Consider how + /// this might make a difference given the regex `a+` on the haystack + /// `aaaaaaaaaaaaaaa`. This routine can stop after it sees the first `a`, + /// but routines like `find` need to continue searching because `+` is + /// greedy by default.) + /// + /// # Example + /// + /// This shows basic usage: + /// + /// ``` + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::new("foo[0-9]+bar")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.is_match(&mut cache, "foo12345bar")); + /// assert!(!re.is_match(&mut cache, "foobar")); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: consistency with search APIs + /// + /// `is_match` is guaranteed to return `true` whenever `find` returns a + /// match. This includes searches that are executed entirely within a + /// codepoint: + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Input}; + /// + /// let re = PikeVM::new("a*")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(!re.is_match(&mut cache, Input::new("☃").span(1..2))); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Notice that when UTF-8 mode is disabled, then the above reports a + /// match because the restriction against zero-width matches that split a + /// codepoint has been lifted: + /// + /// ``` + /// use regex_automata::{nfa::thompson::{pikevm::PikeVM, NFA}, Input}; + /// + /// let re = PikeVM::builder() + /// .thompson(NFA::config().utf8(false)) + /// .build("a*")?; + /// let mut cache = re.create_cache(); + /// + /// assert!(re.is_match(&mut cache, Input::new("☃").span(1..2))); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_match<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + ) -> bool { + let input = input.into().earliest(true); + self.search_slots(cache, &input, &mut []).is_some() + } + + /// Executes a leftmost forward search and returns a `Match` if one exists. + /// + /// This routine only includes the overall match span. To get access to the + /// individual spans of each capturing group, use [`PikeVM::captures`]. + /// + /// # Example + /// + /// Leftmost first match semantics corresponds to the match with the + /// smallest starting offset, but where the end offset is determined by + /// preferring earlier branches in the original regular expression. For + /// example, `Sam|Samwise` will match `Sam` in `Samwise`, but `Samwise|Sam` + /// will match `Samwise` in `Samwise`. + /// + /// Generally speaking, the "leftmost first" match is how most backtracking + /// regular expressions tend to work. This is in contrast to POSIX-style + /// regular expressions that yield "leftmost longest" matches. Namely, + /// both `Sam|Samwise` and `Samwise|Sam` match `Samwise` when using + /// leftmost longest semantics. (This crate does not currently support + /// leftmost longest semantics.) + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re = PikeVM::new("foo[0-9]+")?; + /// let mut cache = re.create_cache(); + /// let expected = Match::must(0, 0..8); + /// assert_eq!(Some(expected), re.find(&mut cache, "foo12345")); + /// + /// // Even though a match is found after reading the first byte (`a`), + /// // the leftmost first match semantics demand that we find the earliest + /// // match that prefers earlier parts of the pattern over later parts. + /// let re = PikeVM::new("abc|a")?; + /// let mut cache = re.create_cache(); + /// let expected = Match::must(0, 0..3); + /// assert_eq!(Some(expected), re.find(&mut cache, "abc")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + ) -> Option<Match> { + let input = input.into(); + if self.get_nfa().pattern_len() == 1 { + let mut slots = [None, None]; + let pid = self.search_slots(cache, &input, &mut slots)?; + let start = slots[0]?.get(); + let end = slots[1]?.get(); + return Some(Match::new(pid, Span { start, end })); + } + let ginfo = self.get_nfa().group_info(); + let slots_len = ginfo.implicit_slot_len(); + let mut slots = vec![None; slots_len]; + let pid = self.search_slots(cache, &input, &mut slots)?; + let start = slots[pid.as_usize() * 2]?.get(); + let end = slots[pid.as_usize() * 2 + 1]?.get(); + Some(Match::new(pid, Span { start, end })) + } + + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided [`Captures`] + /// value. If no match was found, then [`Captures::is_match`] is guaranteed + /// to return `false`. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span}; + /// + /// let re = PikeVM::new(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})$")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "2010-03-14", &mut caps); + /// assert!(caps.is_match()); + /// assert_eq!(Some(Span::from(0..4)), caps.get_group(1)); + /// assert_eq!(Some(Span::from(5..7)), caps.get_group(2)); + /// assert_eq!(Some(Span::from(8..10)), caps.get_group(3)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn captures<'h, I: Into<Input<'h>>>( + &self, + cache: &mut Cache, + input: I, + caps: &mut Captures, + ) { + self.search(cache, &input.into(), caps) + } + + /// Returns an iterator over all non-overlapping leftmost matches in the + /// given bytes. If no match exists, then the iterator yields no elements. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re = PikeVM::new("foo[0-9]+")?; + /// let mut cache = re.create_cache(); + /// + /// let text = "foo1 foo12 foo123"; + /// let matches: Vec<Match> = re.find_iter(&mut cache, text).collect(); + /// assert_eq!(matches, vec![ + /// Match::must(0, 0..4), + /// Match::must(0, 5..10), + /// Match::must(0, 11..17), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find_iter<'r, 'c, 'h, I: Into<Input<'h>>>( + &'r self, + cache: &'c mut Cache, + input: I, + ) -> FindMatches<'r, 'c, 'h> { + let caps = Captures::matches(self.get_nfa().group_info().clone()); + let it = iter::Searcher::new(input.into()); + FindMatches { re: self, cache, caps, it } + } + + /// Returns an iterator over all non-overlapping `Captures` values. If no + /// match exists, then the iterator yields no elements. + /// + /// This yields the same matches as [`PikeVM::find_iter`], but it includes + /// the spans of all capturing groups that participate in each match. + /// + /// **Tip:** See [`util::iter::Searcher`](crate::util::iter::Searcher) for + /// how to correctly iterate over all matches in a haystack while avoiding + /// the creation of a new `Captures` value for every match. (Which you are + /// forced to do with an `Iterator`.) + /// + /// # Example + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span}; + /// + /// let re = PikeVM::new("foo(?P<numbers>[0-9]+)")?; + /// let mut cache = re.create_cache(); + /// + /// let text = "foo1 foo12 foo123"; + /// let matches: Vec<Span> = re + /// .captures_iter(&mut cache, text) + /// // The unwrap is OK since 'numbers' matches if the pattern matches. + /// .map(|caps| caps.get_group_by_name("numbers").unwrap()) + /// .collect(); + /// assert_eq!(matches, vec![ + /// Span::from(3..4), + /// Span::from(8..10), + /// Span::from(14..17), + /// ]); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn captures_iter<'r, 'c, 'h, I: Into<Input<'h>>>( + &'r self, + cache: &'c mut Cache, + input: I, + ) -> CapturesMatches<'r, 'c, 'h> { + let caps = self.create_captures(); + let it = iter::Searcher::new(input.into()); + CapturesMatches { re: self, cache, caps, it } + } +} + +impl PikeVM { + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided [`Captures`] + /// value. If no match was found, then [`Captures::is_match`] is guaranteed + /// to return `false`. + /// + /// This is like [`PikeVM::captures`], but it accepts a concrete `&Input` + /// instead of an `Into<Input>`. + /// + /// # Example: specific pattern search + /// + /// This example shows how to build a multi-PikeVM that permits searching + /// for specific patterns. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// Anchored, Match, PatternID, Input, + /// }; + /// + /// let re = PikeVM::new_many(&["[a-z0-9]{6}", "[a-z][a-z0-9]{5}"])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "foo123"; + /// + /// // Since we are using the default leftmost-first match and both + /// // patterns match at the same starting position, only the first pattern + /// // will be returned in this case when doing a search for any of the + /// // patterns. + /// let expected = Some(Match::must(0, 0..6)); + /// re.search(&mut cache, &Input::new(haystack), &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we want to check whether some other pattern matches, then we + /// // can provide its pattern ID. + /// let expected = Some(Match::must(1, 0..6)); + /// let input = Input::new(haystack) + /// .anchored(Anchored::Pattern(PatternID::must(1))); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: specifying the bounds of a search + /// + /// This example shows how providing the bounds of a search can produce + /// different results than simply sub-slicing the haystack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match, Input}; + /// + /// let re = PikeVM::new(r"\b[0-9]{3}\b")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let haystack = "foo123bar"; + /// + /// // Since we sub-slice the haystack, the search doesn't know about + /// // the larger context and assumes that `123` is surrounded by word + /// // boundaries. And of course, the match position is reported relative + /// // to the sub-slice as well, which means we get `0..3` instead of + /// // `3..6`. + /// let expected = Some(Match::must(0, 0..3)); + /// re.search(&mut cache, &Input::new(&haystack[3..6]), &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// // But if we provide the bounds of the search within the context of the + /// // entire haystack, then the search can take the surrounding context + /// // into account. (And if we did find a match, it would be reported + /// // as a valid offset into `haystack` instead of its sub-slice.) + /// let expected = None; + /// let input = Input::new(haystack).range(3..6); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search( + &self, + cache: &mut Cache, + input: &Input<'_>, + caps: &mut Captures, + ) { + caps.set_pattern(None); + let pid = self.search_slots(cache, input, caps.slots_mut()); + caps.set_pattern(pid); + } + + /// Executes a leftmost forward search and writes the spans of capturing + /// groups that participated in a match into the provided `slots`, and + /// returns the matching pattern ID. The contents of the slots for patterns + /// other than the matching pattern are unspecified. If no match was found, + /// then `None` is returned and the contents of `slots` is unspecified. + /// + /// This is like [`PikeVM::search`], but it accepts a raw slots slice + /// instead of a `Captures` value. This is useful in contexts where you + /// don't want or need to allocate a `Captures`. + /// + /// It is legal to pass _any_ number of slots to this routine. If the regex + /// engine would otherwise write a slot offset that doesn't fit in the + /// provided slice, then it is simply skipped. In general though, there are + /// usually three slice lengths you might want to use: + /// + /// * An empty slice, if you only care about which pattern matched. + /// * A slice with + /// [`pattern_len() * 2`](crate::nfa::thompson::NFA::pattern_len) + /// slots, if you only care about the overall match spans for each matching + /// pattern. + /// * A slice with + /// [`slot_len()`](crate::util::captures::GroupInfo::slot_len) slots, which + /// permits recording match offsets for every capturing group in every + /// pattern. + /// + /// # Example + /// + /// This example shows how to find the overall match offsets in a + /// multi-pattern search without allocating a `Captures` value. Indeed, we + /// can put our slots right on the stack. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, PatternID, Input}; + /// + /// let re = PikeVM::new_many(&[ + /// r"\pL+", + /// r"\d+", + /// ])?; + /// let mut cache = re.create_cache(); + /// let input = Input::new("!@#123"); + /// + /// // We only care about the overall match offsets here, so we just + /// // allocate two slots for each pattern. Each slot records the start + /// // and end of the match. + /// let mut slots = [None; 4]; + /// let pid = re.search_slots(&mut cache, &input, &mut slots); + /// assert_eq!(Some(PatternID::must(1)), pid); + /// + /// // The overall match offsets are always at 'pid * 2' and 'pid * 2 + 1'. + /// // See 'GroupInfo' for more details on the mapping between groups and + /// // slot indices. + /// let slot_start = pid.unwrap().as_usize() * 2; + /// let slot_end = slot_start + 1; + /// assert_eq!(Some(3), slots[slot_start].map(|s| s.get())); + /// assert_eq!(Some(6), slots[slot_end].map(|s| s.get())); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn search_slots( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + if !utf8empty { + let hm = self.search_slots_imp(cache, input, slots)?; + return Some(hm.pattern()); + } + // There is an unfortunate special case where if the regex can + // match the empty string and UTF-8 mode is enabled, the search + // implementation requires that the slots have at least as much space + // to report the bounds of any match. This is so zero-width matches + // that split a codepoint can be filtered out. + // + // Note that if utf8empty is true, we specialize the case for when + // the number of patterns is 1. In that case, we can just use a stack + // allocation. Otherwise we resort to a heap allocation, which we + // convince ourselves we're fine with due to the pathological nature of + // this case. + let min = self.get_nfa().group_info().implicit_slot_len(); + if slots.len() >= min { + let hm = self.search_slots_imp(cache, input, slots)?; + return Some(hm.pattern()); + } + if self.get_nfa().pattern_len() == 1 { + let mut enough = [None, None]; + let got = self.search_slots_imp(cache, input, &mut enough); + // This is OK because we know `enough` is strictly bigger than + // `slots`, otherwise this special case isn't reached. + slots.copy_from_slice(&enough[..slots.len()]); + return got.map(|hm| hm.pattern()); + } + let mut enough = vec![None; min]; + let got = self.search_slots_imp(cache, input, &mut enough); + // This is OK because we know `enough` is strictly bigger than `slots`, + // otherwise this special case isn't reached. + slots.copy_from_slice(&enough[..slots.len()]); + got.map(|hm| hm.pattern()) + } + + /// This is the actual implementation of `search_slots_imp` that + /// doesn't account for the special case when 1) the NFA has UTF-8 mode + /// enabled, 2) the NFA can match the empty string and 3) the caller has + /// provided an insufficient number of slots to record match offsets. + #[inline(never)] + fn search_slots_imp( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<HalfMatch> { + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + let hm = match self.search_imp(cache, input, slots) { + None => return None, + Some(hm) if !utf8empty => return Some(hm), + Some(hm) => hm, + }; + empty::skip_splits_fwd(input, hm, hm.offset(), |input| { + Ok(self + .search_imp(cache, input, slots) + .map(|hm| (hm, hm.offset()))) + }) + // OK because the PikeVM never errors. + .unwrap() + } + + /// Writes the set of patterns that match anywhere in the given search + /// configuration to `patset`. If multiple patterns match at the same + /// position and this `PikeVM` was configured with [`MatchKind::All`] + /// semantics, then all matching patterns are written to the given set. + /// + /// Unless all of the patterns in this `PikeVM` are anchored, then + /// generally speaking, this will visit every byte in the haystack. + /// + /// This search routine *does not* clear the pattern set. This gives some + /// flexibility to the caller (e.g., running multiple searches with the + /// same pattern set), but does make the API bug-prone if you're reusing + /// the same pattern set for multiple searches but intended them to be + /// independent. + /// + /// If a pattern ID matched but the given `PatternSet` does not have + /// sufficient capacity to store it, then it is not inserted and silently + /// dropped. + /// + /// # Example + /// + /// This example shows how to find all matching patterns in a haystack, + /// even when some patterns match at the same position as other patterns. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// Input, MatchKind, PatternSet, + /// }; + /// + /// let patterns = &[ + /// r"\w+", r"\d+", r"\pL+", r"foo", r"bar", r"barfoo", r"foobar", + /// ]; + /// let re = PikeVM::builder() + /// .configure(PikeVM::config().match_kind(MatchKind::All)) + /// .build_many(patterns)?; + /// let mut cache = re.create_cache(); + /// + /// let input = Input::new("foobar"); + /// let mut patset = PatternSet::new(re.pattern_len()); + /// re.which_overlapping_matches(&mut cache, &input, &mut patset); + /// let expected = vec![0, 2, 3, 4, 6]; + /// let got: Vec<usize> = patset.iter().map(|p| p.as_usize()).collect(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn which_overlapping_matches( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + self.which_overlapping_imp(cache, input, patset) + } +} + +impl PikeVM { + /// The implementation of standard leftmost search. + /// + /// Capturing group spans are written to `slots`, but only if requested. + /// `slots` can be any length. Any slot in the NFA that is activated but + /// which is out of bounds for the given `slots` is ignored. + fn search_imp( + &self, + cache: &mut Cache, + input: &Input<'_>, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<HalfMatch> { + cache.setup_search(slots.len()); + if input.is_done() { + return None; + } + // Why do we even care about this? Well, in our 'Captures' + // representation, we use usize::MAX as a sentinel to indicate "no + // match." This isn't problematic so long as our haystack doesn't have + // a maximal length. Byte slices are guaranteed by Rust to have a + // length that fits into isize, and so this assert should always pass. + // But we put it here to make our assumption explicit. + assert!( + input.haystack().len() < core::usize::MAX, + "byte slice lengths must be less than usize MAX", + ); + instrument!(|c| c.reset(&self.nfa)); + + // Whether we want to visit all match states instead of emulating the + // 'leftmost' semantics of typical backtracking regex engines. + let allmatches = + self.config.get_match_kind().continue_past_first_match(); + let (anchored, start_id) = match self.start_config(input) { + None => return None, + Some(config) => config, + }; + + let pre = + if anchored { None } else { self.get_config().get_prefilter() }; + let Cache { ref mut stack, ref mut curr, ref mut next } = cache; + let mut hm = None; + // Yes, our search doesn't end at input.end(), but includes it. This + // is necessary because matches are delayed by one byte, just like + // how the DFA engines work. The delay is used to handle look-behind + // assertions. In the case of the PikeVM, the delay is implemented + // by not considering a match to exist until it is visited in + // 'steps'. Technically, we know a match exists in the previous + // iteration via 'epsilon_closure'. (It's the same thing in NFA-to-DFA + // determinization. We don't mark a DFA state as a match state if it + // contains an NFA match state, but rather, whether the DFA state was + // generated by a transition from a DFA state that contains an NFA + // match state.) + let mut at = input.start(); + while at <= input.end() { + // If we have no states left to visit, then there are some cases + // where we know we can quit early or even skip ahead. + if curr.set.is_empty() { + // We have a match and we haven't been instructed to continue + // on even after finding a match, so we can quit. + if hm.is_some() && !allmatches { + break; + } + // If we're running an anchored search and we've advanced + // beyond the start position with no other states to try, then + // we will never observe a match and thus can stop. + if anchored && at > input.start() { + break; + } + // If there no states left to explore at this position and we + // know we can't terminate early, then we are effectively at + // the starting state of the NFA. If we fell through here, + // we'd end up adding our '(?s-u:.)*?' prefix and it would be + // the only thing in 'curr'. So we might as well just skip + // ahead until we find something that we know might advance us + // forward. + if let Some(pre) = pre { + let span = Span::from(at..input.end()); + match pre.find(input.haystack(), span) { + None => break, + Some(ref span) => at = span.start, + } + } + } + // Instead of using the NFA's unanchored start state, we actually + // always use its anchored starting state. As a result, when doing + // an unanchored search, we need to simulate our own '(?s-u:.)*?' + // prefix, to permit a match to appear anywhere. + // + // Now, we don't *have* to do things this way. We could use the + // NFA's unanchored starting state and do one 'epsilon_closure' + // call from that starting state before the main loop here. And + // that is just as correct. However, it turns out to be slower + // than our approach here because it slightly increases the cost + // of processing each byte by requiring us to visit more NFA + // states to deal with the additional NFA states in the unanchored + // prefix. By simulating it explicitly here, we lower those costs + // substantially. The cost is itself small, but it adds up for + // large haystacks. + // + // In order to simulate the '(?s-u:.)*?' prefix---which is not + // greedy---we are careful not to perform an epsilon closure on + // the start state if we already have a match. Namely, if we + // did otherwise, we would never reach a terminating condition + // because there would always be additional states to process. + // In effect, the exclusion of running 'epsilon_closure' when + // we have a match corresponds to the "dead" states we have in + // our DFA regex engines. Namely, in a DFA, match states merely + // instruct the search execution to record the current offset as + // the most recently seen match. It is the dead state that actually + // indicates when to stop the search (other than EOF or quit + // states). + // + // However, when 'allmatches' is true, the caller has asked us to + // leave in every possible match state. This tends not to make a + // whole lot of sense in unanchored searches, because it means the + // search really cannot terminate until EOF. And often, in that + // case, you wind up skipping over a bunch of matches and are left + // with the "last" match. Arguably, it just doesn't make a lot of + // sense to run a 'leftmost' search (which is what this routine is) + // with 'allmatches' set to true. But the DFAs support it and this + // matches their behavior. (Generally, 'allmatches' is useful for + // overlapping searches or leftmost anchored searches to find the + // longest possible match by ignoring match priority.) + // + // Additionally, when we're running an anchored search, this + // epsilon closure should only be computed at the beginning of the + // search. If we re-computed it at every position, we would be + // simulating an unanchored search when we were tasked to perform + // an anchored search. + if (hm.is_none() || allmatches) + && (!anchored || at == input.start()) + { + // Since we are adding to the 'curr' active states and since + // this is for the start ID, we use a slots slice that is + // guaranteed to have the right length but where every element + // is absent. This is exactly what we want, because this + // epsilon closure is responsible for simulating an unanchored + // '(?s:.)*?' prefix. It is specifically outside of any + // capturing groups, and thus, using slots that are always + // absent is correct. + // + // Note though that we can't just use '&mut []' here, since + // this epsilon closure may traverse through 'Captures' epsilon + // transitions, and thus must be able to write offsets to the + // slots given which are later copied to slot values in 'curr'. + let slots = next.slot_table.all_absent(); + self.epsilon_closure(stack, slots, curr, input, at, start_id); + } + if let Some(pid) = self.nexts(stack, curr, next, input, at, slots) + { + hm = Some(HalfMatch::new(pid, at)); + } + // Unless the caller asked us to return early, we need to mush on + // to see if we can extend our match. (But note that 'nexts' will + // quit right after seeing a match when match_kind==LeftmostFirst, + // as is consistent with leftmost-first match priority.) + if input.get_earliest() && hm.is_some() { + break; + } + core::mem::swap(curr, next); + next.set.clear(); + at += 1; + } + instrument!(|c| c.eprint(&self.nfa)); + hm + } + + /// The implementation for the 'which_overlapping_matches' API. Basically, + /// we do a single scan through the entire haystack (unless our regex + /// or search is anchored) and record every pattern that matched. In + /// particular, when MatchKind::All is used, this supports overlapping + /// matches. So if we have the regexes 'sam' and 'samwise', they will + /// *both* be reported in the pattern set when searching the haystack + /// 'samwise'. + fn which_overlapping_imp( + &self, + cache: &mut Cache, + input: &Input<'_>, + patset: &mut PatternSet, + ) { + // NOTE: This is effectively a copy of 'search_imp' above, but with no + // captures support and instead writes patterns that matched directly + // to 'patset'. See that routine for better commentary about what's + // going on in this routine. We probably could unify the routines using + // generics or more helper routines, but I'm not sure it's worth it. + // + // NOTE: We somewhat go out of our way here to support things like + // 'input.get_earliest()' and 'leftmost-first' match semantics. Neither + // of those seem particularly relevant to this routine, but they are + // both supported by the DFA analogs of this routine by construction + // and composition, so it seems like good sense to have the PikeVM + // match that behavior. + + cache.setup_search(0); + if input.is_done() { + return; + } + assert!( + input.haystack().len() < core::usize::MAX, + "byte slice lengths must be less than usize MAX", + ); + instrument!(|c| c.reset(&self.nfa)); + + let allmatches = + self.config.get_match_kind().continue_past_first_match(); + let (anchored, start_id) = match self.start_config(input) { + None => return, + Some(config) => config, + }; + + let Cache { ref mut stack, ref mut curr, ref mut next } = cache; + for at in input.start()..=input.end() { + let any_matches = !patset.is_empty(); + if curr.set.is_empty() { + if any_matches && !allmatches { + break; + } + if anchored && at > input.start() { + break; + } + } + if !any_matches || allmatches { + let slots = &mut []; + self.epsilon_closure(stack, slots, curr, input, at, start_id); + } + self.nexts_overlapping(stack, curr, next, input, at, patset); + // If we found a match and filled our set, then there is no more + // additional info that we can provide. Thus, we can quit. We also + // quit if the caller asked us to stop at the earliest point that + // we know a match exists. + if patset.is_full() || input.get_earliest() { + break; + } + core::mem::swap(curr, next); + next.set.clear(); + } + instrument!(|c| c.eprint(&self.nfa)); + } + + /// Process the active states in 'curr' to find the states (written to + /// 'next') we should process for the next byte in the haystack. + /// + /// 'stack' is used to perform a depth first traversal of the NFA when + /// computing an epsilon closure. + /// + /// When a match is found, the slots for that match state (in 'curr') are + /// copied to 'caps'. Moreover, once a match is seen, processing for 'curr' + /// stops (unless the PikeVM was configured with MatchKind::All semantics). + #[cfg_attr(feature = "perf-inline", inline(always))] + fn nexts( + &self, + stack: &mut Vec<FollowEpsilon>, + curr: &mut ActiveStates, + next: &mut ActiveStates, + input: &Input<'_>, + at: usize, + slots: &mut [Option<NonMaxUsize>], + ) -> Option<PatternID> { + instrument!(|c| c.record_state_set(&curr.set)); + let mut pid = None; + let ActiveStates { ref set, ref mut slot_table } = *curr; + for sid in set.iter() { + pid = match self.next(stack, slot_table, next, input, at, sid) { + None => continue, + Some(pid) => Some(pid), + }; + slots.copy_from_slice(slot_table.for_state(sid)); + if !self.config.get_match_kind().continue_past_first_match() { + break; + } + } + pid + } + + /// Like 'nexts', but for the overlapping case. This doesn't write any + /// slots, and instead just writes which pattern matched in 'patset'. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn nexts_overlapping( + &self, + stack: &mut Vec<FollowEpsilon>, + curr: &mut ActiveStates, + next: &mut ActiveStates, + input: &Input<'_>, + at: usize, + patset: &mut PatternSet, + ) { + instrument!(|c| c.record_state_set(&curr.set)); + let utf8empty = self.get_nfa().has_empty() && self.get_nfa().is_utf8(); + let ActiveStates { ref set, ref mut slot_table } = *curr; + for sid in set.iter() { + let pid = match self.next(stack, slot_table, next, input, at, sid) + { + None => continue, + Some(pid) => pid, + }; + // This handles the case of finding a zero-width match that splits + // a codepoint. Namely, if we're in UTF-8 mode AND we know we can + // match the empty string, then the only valid way of getting to + // this point with an offset that splits a codepoint is when we + // have an empty match. Such matches, in UTF-8 mode, must not be + // reported. So we just skip them here and pretend as if we did + // not see a match. + if utf8empty && !input.is_char_boundary(at) { + continue; + } + let _ = patset.try_insert(pid); + if !self.config.get_match_kind().continue_past_first_match() { + break; + } + } + } + + /// Starting from 'sid', if the position 'at' in the 'input' haystack has a + /// transition defined out of 'sid', then add the state transitioned to and + /// its epsilon closure to the 'next' set of states to explore. + /// + /// 'stack' is used by the epsilon closure computation to perform a depth + /// first traversal of the NFA. + /// + /// 'curr_slot_table' should be the table of slots for the current set of + /// states being explored. If there is a transition out of 'sid', then + /// sid's row in the slot table is used to perform the epsilon closure. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn next( + &self, + stack: &mut Vec<FollowEpsilon>, + curr_slot_table: &mut SlotTable, + next: &mut ActiveStates, + input: &Input<'_>, + at: usize, + sid: StateID, + ) -> Option<PatternID> { + instrument!(|c| c.record_step(sid)); + match *self.nfa.state(sid) { + State::Fail + | State::Look { .. } + | State::Union { .. } + | State::BinaryUnion { .. } + | State::Capture { .. } => None, + State::ByteRange { ref trans } => { + if trans.matches(input.haystack(), at) { + let slots = curr_slot_table.for_state(sid); + // OK because 'at <= haystack.len() < usize::MAX', so + // adding 1 will never wrap. + let at = at.wrapping_add(1); + self.epsilon_closure( + stack, slots, next, input, at, trans.next, + ); + } + None + } + State::Sparse(ref sparse) => { + if let Some(next_sid) = sparse.matches(input.haystack(), at) { + let slots = curr_slot_table.for_state(sid); + // OK because 'at <= haystack.len() < usize::MAX', so + // adding 1 will never wrap. + let at = at.wrapping_add(1); + self.epsilon_closure( + stack, slots, next, input, at, next_sid, + ); + } + None + } + State::Dense(ref dense) => { + if let Some(next_sid) = dense.matches(input.haystack(), at) { + let slots = curr_slot_table.for_state(sid); + // OK because 'at <= haystack.len() < usize::MAX', so + // adding 1 will never wrap. + let at = at.wrapping_add(1); + self.epsilon_closure( + stack, slots, next, input, at, next_sid, + ); + } + None + } + State::Match { pattern_id } => Some(pattern_id), + } + } + + /// Compute the epsilon closure of 'sid', writing the closure into 'next' + /// while copying slot values from 'curr_slots' into corresponding states + /// in 'next'. 'curr_slots' should be the slot values corresponding to + /// 'sid'. + /// + /// The given 'stack' is used to perform a depth first traversal of the + /// NFA by recursively following all epsilon transitions out of 'sid'. + /// Conditional epsilon transitions are followed if and only if they are + /// satisfied for the position 'at' in the 'input' haystack. + /// + /// While this routine may write to 'curr_slots', once it returns, any + /// writes are undone and the original values (even if absent) are + /// restored. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn epsilon_closure( + &self, + stack: &mut Vec<FollowEpsilon>, + curr_slots: &mut [Option<NonMaxUsize>], + next: &mut ActiveStates, + input: &Input<'_>, + at: usize, + sid: StateID, + ) { + instrument!(|c| { + c.record_closure(sid); + c.record_stack_push(sid); + }); + stack.push(FollowEpsilon::Explore(sid)); + while let Some(frame) = stack.pop() { + match frame { + FollowEpsilon::RestoreCapture { slot, offset: pos } => { + curr_slots[slot] = pos; + } + FollowEpsilon::Explore(sid) => { + self.epsilon_closure_explore( + stack, curr_slots, next, input, at, sid, + ); + } + } + } + } + + /// Explore all of the epsilon transitions out of 'sid'. This is mostly + /// split out from 'epsilon_closure' in order to clearly delineate + /// the actual work of computing an epsilon closure from the stack + /// book-keeping. + /// + /// This will push any additional explorations needed on to 'stack'. + /// + /// 'curr_slots' should refer to the slots for the currently active NFA + /// state. That is, the current state we are stepping through. These + /// slots are mutated in place as new 'Captures' states are traversed + /// during epsilon closure, but the slots are restored to their original + /// values once the full epsilon closure is completed. The ultimate use of + /// 'curr_slots' is to copy them to the corresponding 'next_slots', so that + /// the capturing group spans are forwarded from the currently active state + /// to the next. + /// + /// 'next' refers to the next set of active states. Computing an epsilon + /// closure may increase the next set of active states. + /// + /// 'input' refers to the caller's input configuration and 'at' refers to + /// the current position in the haystack. These are used to check whether + /// conditional epsilon transitions (like look-around) are satisfied at + /// the current position. If they aren't, then the epsilon closure won't + /// include them. + #[cfg_attr(feature = "perf-inline", inline(always))] + fn epsilon_closure_explore( + &self, + stack: &mut Vec<FollowEpsilon>, + curr_slots: &mut [Option<NonMaxUsize>], + next: &mut ActiveStates, + input: &Input<'_>, + at: usize, + mut sid: StateID, + ) { + // We can avoid pushing some state IDs on to our stack in precisely + // the cases where a 'push(x)' would be immediately followed by a 'x + // = pop()'. This is achieved by this outer-loop. We simply set 'sid' + // to be the next state ID we want to explore once we're done with + // our initial exploration. In practice, this avoids a lot of stack + // thrashing. + loop { + instrument!(|c| c.record_set_insert(sid)); + // Record this state as part of our next set of active states. If + // we've already explored it, then no need to do it again. + if !next.set.insert(sid) { + return; + } + match *self.nfa.state(sid) { + State::Fail + | State::Match { .. } + | State::ByteRange { .. } + | State::Sparse { .. } + | State::Dense { .. } => { + next.slot_table.for_state(sid).copy_from_slice(curr_slots); + return; + } + State::Look { look, next } => { + // OK because we don't permit building a searcher with a + // Unicode word boundary if the requisite Unicode data is + // unavailable. + if !self.nfa.look_matcher().matches_inline( + look, + input.haystack(), + at, + ) { + return; + } + sid = next; + } + State::Union { ref alternates } => { + sid = match alternates.get(0) { + None => return, + Some(&sid) => sid, + }; + instrument!(|c| { + for &alt in &alternates[1..] { + c.record_stack_push(alt); + } + }); + stack.extend( + alternates[1..] + .iter() + .copied() + .rev() + .map(FollowEpsilon::Explore), + ); + } + State::BinaryUnion { alt1, alt2 } => { + sid = alt1; + instrument!(|c| c.record_stack_push(sid)); + stack.push(FollowEpsilon::Explore(alt2)); + } + State::Capture { next, slot, .. } => { + // There's no need to do anything with slots that + // ultimately won't be copied into the caller-provided + // 'Captures' value. So we just skip dealing with them at + // all. + if slot.as_usize() < curr_slots.len() { + instrument!(|c| c.record_stack_push(sid)); + stack.push(FollowEpsilon::RestoreCapture { + slot, + offset: curr_slots[slot], + }); + // OK because length of a slice must fit into an isize. + curr_slots[slot] = Some(NonMaxUsize::new(at).unwrap()); + } + sid = next; + } + } + } + } + + /// Return the starting configuration of a PikeVM search. + /// + /// The "start config" is basically whether the search should be anchored + /// or not and the NFA state ID at which to begin the search. The state ID + /// returned always corresponds to an anchored starting state even when the + /// search is unanchored. This is because the PikeVM search loop deals with + /// unanchored searches with an explicit epsilon closure out of the start + /// state. + /// + /// This routine accounts for both the caller's `Input` configuration + /// and the pattern itself. For example, even if the caller asks for an + /// unanchored search, if the pattern itself is anchored, then this will + /// always return 'true' because implementing an unanchored search in that + /// case would be incorrect. + /// + /// Similarly, if the caller requests an anchored search for a particular + /// pattern, then the starting state ID returned will reflect that. + /// + /// If a pattern ID is given in the input configuration that is not in + /// this regex, then `None` is returned. + fn start_config(&self, input: &Input<'_>) -> Option<(bool, StateID)> { + match input.get_anchored() { + // Only way we're unanchored is if both the caller asked for an + // unanchored search *and* the pattern is itself not anchored. + Anchored::No => Some(( + self.nfa.is_always_start_anchored(), + self.nfa.start_anchored(), + )), + Anchored::Yes => Some((true, self.nfa.start_anchored())), + Anchored::Pattern(pid) => { + Some((true, self.nfa.start_pattern(pid)?)) + } + } + } +} + +/// An iterator over all non-overlapping matches for a particular search. +/// +/// The iterator yields a [`Match`] value until no more matches could be found. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the PikeVM. +/// * `'c` represents the lifetime of the PikeVM's cache. +/// * `'h` represents the lifetime of the haystack being searched. +/// +/// This iterator can be created with the [`PikeVM::find_iter`] method. +#[derive(Debug)] +pub struct FindMatches<'r, 'c, 'h> { + re: &'r PikeVM, + cache: &'c mut Cache, + caps: Captures, + it: iter::Searcher<'h>, +} + +impl<'r, 'c, 'h> Iterator for FindMatches<'r, 'c, 'h> { + type Item = Match; + + #[inline] + fn next(&mut self) -> Option<Match> { + // Splitting 'self' apart seems necessary to appease borrowck. + let FindMatches { re, ref mut cache, ref mut caps, ref mut it } = + *self; + // 'advance' converts errors into panics, which is OK here because + // the PikeVM can never return an error. + it.advance(|input| { + re.search(cache, input, caps); + Ok(caps.get_match()) + }) + } +} + +/// An iterator over all non-overlapping leftmost matches, with their capturing +/// groups, for a particular search. +/// +/// The iterator yields a [`Captures`] value until no more matches could be +/// found. +/// +/// The lifetime parameters are as follows: +/// +/// * `'r` represents the lifetime of the PikeVM. +/// * `'c` represents the lifetime of the PikeVM's cache. +/// * `'h` represents the lifetime of the haystack being searched. +/// +/// This iterator can be created with the [`PikeVM::captures_iter`] method. +#[derive(Debug)] +pub struct CapturesMatches<'r, 'c, 'h> { + re: &'r PikeVM, + cache: &'c mut Cache, + caps: Captures, + it: iter::Searcher<'h>, +} + +impl<'r, 'c, 'h> Iterator for CapturesMatches<'r, 'c, 'h> { + type Item = Captures; + + #[inline] + fn next(&mut self) -> Option<Captures> { + // Splitting 'self' apart seems necessary to appease borrowck. + let CapturesMatches { re, ref mut cache, ref mut caps, ref mut it } = + *self; + // 'advance' converts errors into panics, which is OK here because + // the PikeVM can never return an error. + it.advance(|input| { + re.search(cache, input, caps); + Ok(caps.get_match()) + }); + if caps.is_match() { + Some(caps.clone()) + } else { + None + } + } +} + +/// A cache represents mutable state that a [`PikeVM`] requires during a +/// search. +/// +/// For a given [`PikeVM`], its corresponding cache may be created either via +/// [`PikeVM::create_cache`], or via [`Cache::new`]. They are equivalent in +/// every way, except the former does not require explicitly importing `Cache`. +/// +/// A particular `Cache` is coupled with the [`PikeVM`] from which it +/// was created. It may only be used with that `PikeVM`. A cache and its +/// allocations may be re-purposed via [`Cache::reset`], in which case, it can +/// only be used with the new `PikeVM` (and not the old one). +#[derive(Clone, Debug)] +pub struct Cache { + /// Stack used while computing epsilon closure. This effectively lets us + /// move what is more naturally expressed through recursion to a stack + /// on the heap. + stack: Vec<FollowEpsilon>, + /// The current active states being explored for the current byte in the + /// haystack. + curr: ActiveStates, + /// The next set of states we're building that will be explored for the + /// next byte in the haystack. + next: ActiveStates, +} + +impl Cache { + /// Create a new [`PikeVM`] cache. + /// + /// A potentially more convenient routine to create a cache is + /// [`PikeVM::create_cache`], as it does not require also importing the + /// `Cache` type. + /// + /// If you want to reuse the returned `Cache` with some other `PikeVM`, + /// then you must call [`Cache::reset`] with the desired `PikeVM`. + pub fn new(re: &PikeVM) -> Cache { + Cache { + stack: vec![], + curr: ActiveStates::new(re), + next: ActiveStates::new(re), + } + } + + /// Reset this cache such that it can be used for searching with a + /// different [`PikeVM`]. + /// + /// A cache reset permits reusing memory already allocated in this cache + /// with a different `PikeVM`. + /// + /// # Example + /// + /// This shows how to re-purpose a cache for use with a different `PikeVM`. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re1 = PikeVM::new(r"\w")?; + /// let re2 = PikeVM::new(r"\W")?; + /// + /// let mut cache = re1.create_cache(); + /// assert_eq!( + /// Some(Match::must(0, 0..2)), + /// re1.find_iter(&mut cache, "Δ").next(), + /// ); + /// + /// // Using 'cache' with re2 is not allowed. It may result in panics or + /// // incorrect results. In order to re-purpose the cache, we must reset + /// // it with the PikeVM we'd like to use it with. + /// // + /// // Similarly, after this reset, using the cache with 're1' is also not + /// // allowed. + /// cache.reset(&re2); + /// assert_eq!( + /// Some(Match::must(0, 0..3)), + /// re2.find_iter(&mut cache, "☃").next(), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn reset(&mut self, re: &PikeVM) { + self.curr.reset(re); + self.next.reset(re); + } + + /// Returns the heap memory usage, in bytes, of this cache. + /// + /// This does **not** include the stack size used up by this cache. To + /// compute that, use `std::mem::size_of::<Cache>()`. + pub fn memory_usage(&self) -> usize { + use core::mem::size_of; + (self.stack.len() * size_of::<FollowEpsilon>()) + + self.curr.memory_usage() + + self.next.memory_usage() + } + + /// Clears this cache. This should be called at the start of every search + /// to ensure we start with a clean slate. + /// + /// This also sets the length of the capturing groups used in the current + /// search. This permits an optimization where by 'SlotTable::for_state' + /// only returns the number of slots equivalent to the number of slots + /// given in the 'Captures' value. This may be less than the total number + /// of possible slots, e.g., when one only wants to track overall match + /// offsets. This in turn permits less copying of capturing group spans + /// in the PikeVM. + fn setup_search(&mut self, captures_slot_len: usize) { + self.stack.clear(); + self.curr.setup_search(captures_slot_len); + self.next.setup_search(captures_slot_len); + } +} + +/// A set of active states used to "simulate" the execution of an NFA via the +/// PikeVM. +/// +/// There are two sets of these used during NFA simulation. One set corresponds +/// to the "current" set of states being traversed for the current position +/// in a haystack. The other set corresponds to the "next" set of states being +/// built, which will become the new "current" set for the next position in the +/// haystack. These two sets correspond to CLIST and NLIST in Thompson's +/// original paper regexes: https://dl.acm.org/doi/pdf/10.1145/363347.363387 +/// +/// In addition to representing a set of NFA states, this also maintains slot +/// values for each state. These slot values are what turn the NFA simulation +/// into the "Pike VM." Namely, they track capturing group values for each +/// state. During the computation of epsilon closure, we copy slot values from +/// states in the "current" set to the "next" set. Eventually, once a match +/// is found, the slot values for that match state are what we write to the +/// caller provided 'Captures' value. +#[derive(Clone, Debug)] +struct ActiveStates { + /// The set of active NFA states. This set preserves insertion order, which + /// is critical for simulating the match semantics of backtracking regex + /// engines. + set: SparseSet, + /// The slots for every NFA state, where each slot stores a (possibly + /// absent) offset. Every capturing group has two slots. One for a start + /// offset and one for an end offset. + slot_table: SlotTable, +} + +impl ActiveStates { + /// Create a new set of active states for the given PikeVM. The active + /// states returned may only be used with the given PikeVM. (Use 'reset' + /// to re-purpose the allocation for a different PikeVM.) + fn new(re: &PikeVM) -> ActiveStates { + let mut active = ActiveStates { + set: SparseSet::new(0), + slot_table: SlotTable::new(), + }; + active.reset(re); + active + } + + /// Reset this set of active states such that it can be used with the given + /// PikeVM (and only that PikeVM). + fn reset(&mut self, re: &PikeVM) { + self.set.resize(re.get_nfa().states().len()); + self.slot_table.reset(re); + } + + /// Return the heap memory usage, in bytes, used by this set of active + /// states. + /// + /// This does not include the stack size of this value. + fn memory_usage(&self) -> usize { + self.set.memory_usage() + self.slot_table.memory_usage() + } + + /// Setup this set of active states for a new search. The given slot + /// length should be the number of slots in a caller provided 'Captures' + /// (and may be zero). + fn setup_search(&mut self, captures_slot_len: usize) { + self.set.clear(); + self.slot_table.setup_search(captures_slot_len); + } +} + +/// A table of slots, where each row represent a state in an NFA. Thus, the +/// table has room for storing slots for every single state in an NFA. +/// +/// This table is represented with a single contiguous allocation. In general, +/// the notion of "capturing group" doesn't really exist at this level of +/// abstraction, hence the name "slot" instead. (Indeed, every capturing group +/// maps to a pair of slots, one for the start offset and one for the end +/// offset.) Slots are indexed by the 'Captures' NFA state. +/// +/// N.B. Not every state actually needs a row of slots. Namely, states that +/// only have epsilon transitions currently never have anything written to +/// their rows in this table. Thus, the table is somewhat wasteful in its heap +/// usage. However, it is important to maintain fast random access by state +/// ID, which means one giant table tends to work well. RE2 takes a different +/// approach here and allocates each row as its own reference counted thing. +/// I explored such a strategy at one point here, but couldn't get it to work +/// well using entirely safe code. (To the ambitious reader: I encourage you to +/// re-litigate that experiment.) I very much wanted to stick to safe code, but +/// could be convinced otherwise if there was a solid argument and the safety +/// was encapsulated well. +#[derive(Clone, Debug)] +struct SlotTable { + /// The actual table of offsets. + table: Vec<Option<NonMaxUsize>>, + /// The number of slots per state, i.e., the table's stride or the length + /// of each row. + slots_per_state: usize, + /// The number of slots in the caller-provided 'Captures' value for the + /// current search. Setting this to 'slots_per_state' is always correct, + /// but may be wasteful. + slots_for_captures: usize, +} + +impl SlotTable { + /// Create a new slot table. + /// + /// One should call 'reset' with the corresponding PikeVM before use. + fn new() -> SlotTable { + SlotTable { table: vec![], slots_for_captures: 0, slots_per_state: 0 } + } + + /// Reset this slot table such that it can be used with the given PikeVM + /// (and only that PikeVM). + fn reset(&mut self, re: &PikeVM) { + let nfa = re.get_nfa(); + self.slots_per_state = nfa.group_info().slot_len(); + // This is always correct, but may be reduced for a particular search + // if a 'Captures' has fewer slots, e.g., none at all or only slots + // for tracking the overall match instead of all slots for every + // group. + self.slots_for_captures = core::cmp::max( + self.slots_per_state, + nfa.pattern_len().checked_mul(2).unwrap(), + ); + let len = nfa + .states() + .len() + .checked_mul(self.slots_per_state) + // Add space to account for scratch space used during a search. + .and_then(|x| x.checked_add(self.slots_for_captures)) + // It seems like this could actually panic on legitimate inputs on + // 32-bit targets, and very likely to panic on 16-bit. Should we + // somehow convert this to an error? What about something similar + // for the lazy DFA cache? If you're tripping this assert, please + // file a bug. + .expect("slot table length doesn't overflow"); + // This happens about as often as a regex is compiled, so it probably + // should be at debug level, but I found it quite distracting and not + // particularly useful. + trace!( + "resizing PikeVM active states table to {} entries \ + (slots_per_state={})", + len, + self.slots_per_state, + ); + self.table.resize(len, None); + } + + /// Return the heap memory usage, in bytes, used by this slot table. + /// + /// This does not include the stack size of this value. + fn memory_usage(&self) -> usize { + self.table.len() * core::mem::size_of::<Option<NonMaxUsize>>() + } + + /// Perform any per-search setup for this slot table. + /// + /// In particular, this sets the length of the number of slots used in the + /// 'Captures' given by the caller (if any at all). This number may be + /// smaller than the total number of slots available, e.g., when the caller + /// is only interested in tracking the overall match and not the spans of + /// every matching capturing group. Only tracking the overall match can + /// save a substantial amount of time copying capturing spans during a + /// search. + fn setup_search(&mut self, captures_slot_len: usize) { + self.slots_for_captures = captures_slot_len; + } + + /// Return a mutable slice of the slots for the given state. + /// + /// Note that the length of the slice returned may be less than the total + /// number of slots available for this state. In particular, the length + /// always matches the number of slots indicated via 'setup_search'. + fn for_state(&mut self, sid: StateID) -> &mut [Option<NonMaxUsize>] { + let i = sid.as_usize() * self.slots_per_state; + &mut self.table[i..i + self.slots_for_captures] + } + + /// Return a slice of slots of appropriate length where every slot offset + /// is guaranteed to be absent. This is useful in cases where you need to + /// compute an epsilon closure outside of the user supplied regex, and thus + /// never want it to have any capturing slots set. + fn all_absent(&mut self) -> &mut [Option<NonMaxUsize>] { + let i = self.table.len() - self.slots_for_captures; + &mut self.table[i..i + self.slots_for_captures] + } +} + +/// Represents a stack frame for use while computing an epsilon closure. +/// +/// (An "epsilon closure" refers to the set of reachable NFA states from a +/// single state without consuming any input. That is, the set of all epsilon +/// transitions not only from that single state, but from every other state +/// reachable by an epsilon transition as well. This is why it's called a +/// "closure." Computing an epsilon closure is also done during DFA +/// determinization! Compare and contrast the epsilon closure here in this +/// PikeVM and the one used for determinization in crate::util::determinize.) +/// +/// Computing the epsilon closure in a Thompson NFA proceeds via a depth +/// first traversal over all epsilon transitions from a particular state. +/// (A depth first traversal is important because it emulates the same priority +/// of matches that is typically found in backtracking regex engines.) This +/// depth first traversal is naturally expressed using recursion, but to avoid +/// a call stack size proportional to the size of a regex, we put our stack on +/// the heap instead. +/// +/// This stack thus consists of call frames. The typical call frame is +/// `Explore`, which instructs epsilon closure to explore the epsilon +/// transitions from that state. (Subsequent epsilon transitions are then +/// pushed on to the stack as more `Explore` frames.) If the state ID being +/// explored has no epsilon transitions, then the capturing group slots are +/// copied from the original state that sparked the epsilon closure (from the +/// 'step' routine) to the state ID being explored. This way, capturing group +/// slots are forwarded from the previous state to the next. +/// +/// The other stack frame, `RestoreCaptures`, instructs the epsilon closure to +/// set the position for a particular slot back to some particular offset. This +/// frame is pushed when `Explore` sees a `Capture` transition. `Explore` will +/// set the offset of the slot indicated in `Capture` to the current offset, +/// and then push the old offset on to the stack as a `RestoreCapture` frame. +/// Thus, the new offset is only used until the epsilon closure reverts back to +/// the `RestoreCapture` frame. In effect, this gives the `Capture` epsilon +/// transition its "scope" to only states that come "after" it during depth +/// first traversal. +#[derive(Clone, Debug)] +enum FollowEpsilon { + /// Explore the epsilon transitions from a state ID. + Explore(StateID), + /// Reset the given `slot` to the given `offset` (which might be `None`). + RestoreCapture { slot: SmallIndex, offset: Option<NonMaxUsize> }, +} + +/// A set of counters that "instruments" a PikeVM search. To enable this, you +/// must enable the 'internal-instrument-pikevm' feature. Then run your Rust +/// program with RUST_LOG=regex_automata::nfa::thompson::pikevm=trace set in +/// the environment. The metrics collected will be dumped automatically for +/// every search executed by the PikeVM. +/// +/// NOTE: When 'internal-instrument-pikevm' is enabled, it will likely cause an +/// absolute decrease in wall-clock performance, even if the 'trace' log level +/// isn't enabled. (Although, we do try to avoid extra costs when 'trace' isn't +/// enabled.) The main point of instrumentation is to get counts of various +/// events that occur during the PikeVM's execution. +/// +/// This is a somewhat hacked together collection of metrics that are useful +/// to gather from a PikeVM search. In particular, it lets us scrutinize the +/// performance profile of a search beyond what general purpose profiling tools +/// give us. Namely, we orient the profiling data around the specific states of +/// the NFA. +/// +/// In other words, this lets us see which parts of the NFA graph are most +/// frequently activated. This then provides direction for optimization +/// opportunities. +/// +/// The really sad part about this is that it absolutely clutters up the PikeVM +/// implementation. :'( Another approach would be to just manually add this +/// code in whenever I want this kind of profiling data, but it's complicated +/// and tedious enough that I went with this approach... for now. +/// +/// When instrumentation is enabled (which also turns on 'logging'), then a +/// `Counters` is initialized for every search and `trace`'d just before the +/// search returns to the caller. +/// +/// Tip: When debugging performance problems with the PikeVM, it's best to try +/// to work with an NFA that is as small as possible. Otherwise the state graph +/// is likely to be too big to digest. +#[cfg(feature = "internal-instrument-pikevm")] +#[derive(Clone, Debug)] +struct Counters { + /// The number of times the NFA is in a particular permutation of states. + state_sets: alloc::collections::BTreeMap<Vec<StateID>, u64>, + /// The number of times 'step' is called for a particular state ID (which + /// indexes this array). + steps: Vec<u64>, + /// The number of times an epsilon closure was computed for a state. + closures: Vec<u64>, + /// The number of times a particular state ID is pushed on to a stack while + /// computing an epsilon closure. + stack_pushes: Vec<u64>, + /// The number of times a particular state ID is inserted into a sparse set + /// while computing an epsilon closure. + set_inserts: Vec<u64>, +} + +#[cfg(feature = "internal-instrument-pikevm")] +impl Counters { + fn empty() -> Counters { + Counters { + state_sets: alloc::collections::BTreeMap::new(), + steps: vec![], + closures: vec![], + stack_pushes: vec![], + set_inserts: vec![], + } + } + + fn reset(&mut self, nfa: &NFA) { + let len = nfa.states().len(); + + self.state_sets.clear(); + + self.steps.clear(); + self.steps.resize(len, 0); + + self.closures.clear(); + self.closures.resize(len, 0); + + self.stack_pushes.clear(); + self.stack_pushes.resize(len, 0); + + self.set_inserts.clear(); + self.set_inserts.resize(len, 0); + } + + fn eprint(&self, nfa: &NFA) { + trace!("===== START PikeVM Instrumentation Output ====="); + // We take the top-K most occurring state sets. Otherwise the output + // is likely to be overwhelming. And we probably only care about the + // most frequently occurring ones anyway. + const LIMIT: usize = 20; + let mut set_counts = + self.state_sets.iter().collect::<Vec<(&Vec<StateID>, &u64)>>(); + set_counts.sort_by_key(|(_, &count)| core::cmp::Reverse(count)); + trace!("## PikeVM frequency of state sets (top {LIMIT})"); + for (set, count) in set_counts.iter().take(LIMIT) { + trace!("{set:?}: {count}"); + } + if set_counts.len() > LIMIT { + trace!( + "... {} sets omitted (out of {} total)", + set_counts.len() - LIMIT, + set_counts.len(), + ); + } + + trace!(""); + trace!("## PikeVM total frequency of events"); + trace!( + "steps: {}, closures: {}, stack-pushes: {}, set-inserts: {}", + self.steps.iter().copied().sum::<u64>(), + self.closures.iter().copied().sum::<u64>(), + self.stack_pushes.iter().copied().sum::<u64>(), + self.set_inserts.iter().copied().sum::<u64>(), + ); + + trace!(""); + trace!("## PikeVM frequency of events broken down by state"); + for sid in 0..self.steps.len() { + trace!( + "{:06}: steps: {}, closures: {}, \ + stack-pushes: {}, set-inserts: {}", + sid, + self.steps[sid], + self.closures[sid], + self.stack_pushes[sid], + self.set_inserts[sid], + ); + } + + trace!(""); + trace!("## NFA debug display"); + trace!("{nfa:?}"); + trace!("===== END PikeVM Instrumentation Output ====="); + } + + fn record_state_set(&mut self, set: &SparseSet) { + let set = set.iter().collect::<Vec<StateID>>(); + *self.state_sets.entry(set).or_insert(0) += 1; + } + + fn record_step(&mut self, sid: StateID) { + self.steps[sid] += 1; + } + + fn record_closure(&mut self, sid: StateID) { + self.closures[sid] += 1; + } + + fn record_stack_push(&mut self, sid: StateID) { + self.stack_pushes[sid] += 1; + } + + fn record_set_insert(&mut self, sid: StateID) { + self.set_inserts[sid] += 1; + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/range_trie.rs b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/range_trie.rs new file mode 100644 index 0000000..57ae322 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/nfa/thompson/range_trie.rs @@ -0,0 +1,1051 @@ +/* +I've called the primary data structure in this module a "range trie." As far +as I can tell, there is no prior art on a data structure like this, however, +it's likely someone somewhere has built something like it. Searching for +"range trie" turns up the paper "Range Tries for Scalable Address Lookup," +but it does not appear relevant. + +The range trie is just like a trie in that it is a special case of a +deterministic finite state machine. It has states and each state has a set +of transitions to other states. It is acyclic, and, like a normal trie, +it makes no attempt to reuse common suffixes among its elements. The key +difference between a normal trie and a range trie below is that a range trie +operates on *contiguous sequences* of bytes instead of singleton bytes. +One could say say that our alphabet is ranges of bytes instead of bytes +themselves, except a key part of range trie construction is splitting ranges +apart to ensure there is at most one transition that can be taken for any +byte in a given state. + +I've tried to explain the details of how the range trie works below, so +for now, we are left with trying to understand what problem we're trying to +solve. Which is itself fairly involved! + +At the highest level, here's what we want to do. We want to convert a +sequence of Unicode codepoints into a finite state machine whose transitions +are over *bytes* and *not* Unicode codepoints. We want this because it makes +said finite state machines much smaller and much faster to execute. As a +simple example, consider a byte oriented automaton for all Unicode scalar +values (0x00 through 0x10FFFF, not including surrogate codepoints): + + [00-7F] + [C2-DF][80-BF] + [E0-E0][A0-BF][80-BF] + [E1-EC][80-BF][80-BF] + [ED-ED][80-9F][80-BF] + [EE-EF][80-BF][80-BF] + [F0-F0][90-BF][80-BF][80-BF] + [F1-F3][80-BF][80-BF][80-BF] + [F4-F4][80-8F][80-BF][80-BF] + +(These byte ranges are generated via the regex-syntax::utf8 module, which +was based on Russ Cox's code in RE2, which was in turn based on Ken +Thompson's implementation of the same idea in his Plan9 implementation of +grep.) + +It should be fairly straight-forward to see how one could compile this into +a DFA. The sequences are sorted and non-overlapping. Essentially, you could +build a trie from this fairly easy. The problem comes when your initial +range (in this case, 0x00-0x10FFFF) isn't so nice. For example, the class +represented by '\w' contains only a tenth of the codepoints that +0x00-0x10FFFF contains, but if we were to write out the byte based ranges +as we did above, the list would stretch to 892 entries! This turns into +quite a large NFA with a few thousand states. Turning this beast into a DFA +takes quite a bit of time. We are thus left with trying to trim down the +number of states we produce as early as possible. + +One approach (used by RE2 and still by the regex crate, at time of writing) +is to try to find common suffixes while building NFA states for the above +and reuse them. This is very cheap to do and one can control precisely how +much extra memory you want to use for the cache. + +Another approach, however, is to reuse an algorithm for constructing a +*minimal* DFA from a sorted sequence of inputs. I don't want to go into +the full details here, but I explain it in more depth in my blog post on +FSTs[1]. Note that the algorithm was not invented by me, but was published +in paper by Daciuk et al. in 2000 called "Incremental Construction of +MinimalAcyclic Finite-State Automata." Like the suffix cache approach above, +it is also possible to control the amount of extra memory one uses, although +this usually comes with the cost of sacrificing true minimality. (But it's +typically close enough with a reasonably sized cache of states.) + +The catch is that Daciuk's algorithm only works if you add your keys in +lexicographic ascending order. In our case, since we're dealing with ranges, +we also need the additional requirement that ranges are either equivalent +or do not overlap at all. For example, if one were given the following byte +ranges: + + [BC-BF][80-BF] + [BC-BF][90-BF] + +Then Daciuk's algorithm would not work, since there is nothing to handle the +fact that the ranges overlap. They would need to be split apart. Thankfully, +Thompson's algorithm for producing byte ranges for Unicode codepoint ranges +meets both of our requirements. (A proof for this eludes me, but it appears +true.) + +... however, we would also like to be able to compile UTF-8 automata in +reverse. We want this because in order to find the starting location of a +match using a DFA, we need to run a second DFA---a reversed version of the +forward DFA---backwards to discover the match location. Unfortunately, if +we reverse our byte sequences for 0x00-0x10FFFF, we get sequences that are +can overlap, even if they are sorted: + + [00-7F] + [80-BF][80-9F][ED-ED] + [80-BF][80-BF][80-8F][F4-F4] + [80-BF][80-BF][80-BF][F1-F3] + [80-BF][80-BF][90-BF][F0-F0] + [80-BF][80-BF][E1-EC] + [80-BF][80-BF][EE-EF] + [80-BF][A0-BF][E0-E0] + [80-BF][C2-DF] + +For example, '[80-BF][80-BF][EE-EF]' and '[80-BF][A0-BF][E0-E0]' have +overlapping ranges between '[80-BF]' and '[A0-BF]'. Thus, there is no +simple way to apply Daciuk's algorithm. + +And thus, the range trie was born. The range trie's only purpose is to take +sequences of byte ranges like the ones above, collect them into a trie and then +spit them out in a sorted fashion with no overlapping ranges. For example, +0x00-0x10FFFF gets translated to: + + [0-7F] + [80-BF][80-9F][80-8F][F1-F3] + [80-BF][80-9F][80-8F][F4] + [80-BF][80-9F][90-BF][F0] + [80-BF][80-9F][90-BF][F1-F3] + [80-BF][80-9F][E1-EC] + [80-BF][80-9F][ED] + [80-BF][80-9F][EE-EF] + [80-BF][A0-BF][80-8F][F1-F3] + [80-BF][A0-BF][80-8F][F4] + [80-BF][A0-BF][90-BF][F0] + [80-BF][A0-BF][90-BF][F1-F3] + [80-BF][A0-BF][E0] + [80-BF][A0-BF][E1-EC] + [80-BF][A0-BF][EE-EF] + [80-BF][C2-DF] + +We've thus satisfied our requirements for running Daciuk's algorithm. All +sequences of ranges are sorted, and any corresponding ranges are either +exactly equivalent or non-overlapping. + +In effect, a range trie is building a DFA from a sequence of arbitrary byte +ranges. But it uses an algorithm custom tailored to its input, so it is not as +costly as traditional DFA construction. While it is still quite a bit more +costly than the forward case (which only needs Daciuk's algorithm), it winds +up saving a substantial amount of time if one is doing a full DFA powerset +construction later by virtue of producing a much much smaller NFA. + +[1] - https://blog.burntsushi.net/transducers/ +[2] - https://www.mitpressjournals.org/doi/pdfplus/10.1162/089120100561601 +*/ + +use core::{cell::RefCell, fmt, mem, ops::RangeInclusive}; + +use alloc::{format, string::String, vec, vec::Vec}; + +use regex_syntax::utf8::Utf8Range; + +use crate::util::primitives::StateID; + +/// There is only one final state in this trie. Every sequence of byte ranges +/// added shares the same final state. +const FINAL: StateID = StateID::ZERO; + +/// The root state of the trie. +const ROOT: StateID = StateID::new_unchecked(1); + +/// A range trie represents an ordered set of sequences of bytes. +/// +/// A range trie accepts as input a sequence of byte ranges and merges +/// them into the existing set such that the trie can produce a sorted +/// non-overlapping sequence of byte ranges. The sequence emitted corresponds +/// precisely to the sequence of bytes matched by the given keys, although the +/// byte ranges themselves may be split at different boundaries. +/// +/// The order complexity of this data structure seems difficult to analyze. +/// If the size of a byte is held as a constant, then insertion is clearly +/// O(n) where n is the number of byte ranges in the input key. However, if +/// k=256 is our alphabet size, then insertion could be O(k^2 * n). In +/// particular it seems possible for pathological inputs to cause insertion +/// to do a lot of work. However, for what we use this data structure for, +/// there should be no pathological inputs since the ultimate source is always +/// a sorted set of Unicode scalar value ranges. +/// +/// Internally, this trie is setup like a finite state machine. Note though +/// that it is acyclic. +#[derive(Clone)] +pub struct RangeTrie { + /// The states in this trie. The first is always the shared final state. + /// The second is always the root state. Otherwise, there is no + /// particular order. + states: Vec<State>, + /// A free-list of states. When a range trie is cleared, all of its states + /// are added to this list. Creating a new state reuses states from this + /// list before allocating a new one. + free: Vec<State>, + /// A stack for traversing this trie to yield sequences of byte ranges in + /// lexicographic order. + iter_stack: RefCell<Vec<NextIter>>, + /// A buffer that stores the current sequence during iteration. + iter_ranges: RefCell<Vec<Utf8Range>>, + /// A stack used for traversing the trie in order to (deeply) duplicate + /// a state. States are recursively duplicated when ranges are split. + dupe_stack: Vec<NextDupe>, + /// A stack used for traversing the trie during insertion of a new + /// sequence of byte ranges. + insert_stack: Vec<NextInsert>, +} + +/// A single state in this trie. +#[derive(Clone)] +struct State { + /// A sorted sequence of non-overlapping transitions to other states. Each + /// transition corresponds to a single range of bytes. + transitions: Vec<Transition>, +} + +/// A transition is a single range of bytes. If a particular byte is in this +/// range, then the corresponding machine may transition to the state pointed +/// to by `next_id`. +#[derive(Clone)] +struct Transition { + /// The byte range. + range: Utf8Range, + /// The next state to transition to. + next_id: StateID, +} + +impl RangeTrie { + /// Create a new empty range trie. + pub fn new() -> RangeTrie { + let mut trie = RangeTrie { + states: vec![], + free: vec![], + iter_stack: RefCell::new(vec![]), + iter_ranges: RefCell::new(vec![]), + dupe_stack: vec![], + insert_stack: vec![], + }; + trie.clear(); + trie + } + + /// Clear this range trie such that it is empty. Clearing a range trie + /// and reusing it can beneficial because this may reuse allocations. + pub fn clear(&mut self) { + self.free.append(&mut self.states); + self.add_empty(); // final + self.add_empty(); // root + } + + /// Iterate over all of the sequences of byte ranges in this trie, and + /// call the provided function for each sequence. Iteration occurs in + /// lexicographic order. + pub fn iter<E, F: FnMut(&[Utf8Range]) -> Result<(), E>>( + &self, + mut f: F, + ) -> Result<(), E> { + let mut stack = self.iter_stack.borrow_mut(); + stack.clear(); + let mut ranges = self.iter_ranges.borrow_mut(); + ranges.clear(); + + // We do iteration in a way that permits us to use a single buffer + // for our keys. We iterate in a depth first fashion, while being + // careful to expand our frontier as we move deeper in the trie. + stack.push(NextIter { state_id: ROOT, tidx: 0 }); + while let Some(NextIter { mut state_id, mut tidx }) = stack.pop() { + // This could be implemented more simply without an inner loop + // here, but at the cost of more stack pushes. + loop { + let state = self.state(state_id); + // If we've visited all transitions in this state, then pop + // back to the parent state. + if tidx >= state.transitions.len() { + ranges.pop(); + break; + } + + let t = &state.transitions[tidx]; + ranges.push(t.range); + if t.next_id == FINAL { + f(&ranges)?; + ranges.pop(); + tidx += 1; + } else { + // Expand our frontier. Once we come back to this state + // via the stack, start in on the next transition. + stack.push(NextIter { state_id, tidx: tidx + 1 }); + // Otherwise, move to the first transition of the next + // state. + state_id = t.next_id; + tidx = 0; + } + } + } + Ok(()) + } + + /// Inserts a new sequence of ranges into this trie. + /// + /// The sequence given must be non-empty and must not have a length + /// exceeding 4. + pub fn insert(&mut self, ranges: &[Utf8Range]) { + assert!(!ranges.is_empty()); + assert!(ranges.len() <= 4); + + let mut stack = core::mem::replace(&mut self.insert_stack, vec![]); + stack.clear(); + + stack.push(NextInsert::new(ROOT, ranges)); + while let Some(next) = stack.pop() { + let (state_id, ranges) = (next.state_id(), next.ranges()); + assert!(!ranges.is_empty()); + + let (mut new, rest) = (ranges[0], &ranges[1..]); + + // i corresponds to the position of the existing transition on + // which we are operating. Typically, the result is to remove the + // transition and replace it with two or more new transitions + // corresponding to the partitions generated by splitting the + // 'new' with the ith transition's range. + let mut i = self.state(state_id).find(new); + + // In this case, there is no overlap *and* the new range is greater + // than all existing ranges. So we can just add it to the end. + if i == self.state(state_id).transitions.len() { + let next_id = NextInsert::push(self, &mut stack, rest); + self.add_transition(state_id, new, next_id); + continue; + } + + // The need for this loop is a bit subtle, buf basically, after + // we've handled the partitions from our initial split, it's + // possible that there will be a partition leftover that overlaps + // with a subsequent transition. If so, then we have to repeat + // the split process again with the leftovers and that subsequent + // transition. + 'OUTER: loop { + let old = self.state(state_id).transitions[i].clone(); + let split = match Split::new(old.range, new) { + Some(split) => split, + None => { + let next_id = NextInsert::push(self, &mut stack, rest); + self.add_transition_at(i, state_id, new, next_id); + continue; + } + }; + let splits = split.as_slice(); + // If we only have one partition, then the ranges must be + // equivalent. There's nothing to do here for this state, so + // just move on to the next one. + if splits.len() == 1 { + // ... but only if we have anything left to do. + if !rest.is_empty() { + stack.push(NextInsert::new(old.next_id, rest)); + } + break; + } + // At this point, we know that 'split' is non-empty and there + // must be some overlap AND that the two ranges are not + // equivalent. Therefore, the existing range MUST be removed + // and split up somehow. Instead of actually doing the removal + // and then a subsequent insertion---with all the memory + // shuffling that entails---we simply overwrite the transition + // at position `i` for the first new transition we want to + // insert. After that, we're forced to do expensive inserts. + let mut first = true; + let mut add_trans = + |trie: &mut RangeTrie, pos, from, range, to| { + if first { + trie.set_transition_at(pos, from, range, to); + first = false; + } else { + trie.add_transition_at(pos, from, range, to); + } + }; + for (j, &srange) in splits.iter().enumerate() { + match srange { + SplitRange::Old(r) => { + // Deep clone the state pointed to by the ith + // transition. This is always necessary since 'old' + // is always coupled with at least a 'both' + // partition. We don't want any new changes made + // via the 'both' partition to impact the part of + // the transition that doesn't overlap with the + // new range. + let dup_id = self.duplicate(old.next_id); + add_trans(self, i, state_id, r, dup_id); + } + SplitRange::New(r) => { + // This is a bit subtle, but if this happens to be + // the last partition in our split, it is possible + // that this overlaps with a subsequent transition. + // If it does, then we must repeat the whole + // splitting process over again with `r` and the + // subsequent transition. + { + let trans = &self.state(state_id).transitions; + if j + 1 == splits.len() + && i < trans.len() + && intersects(r, trans[i].range) + { + new = r; + continue 'OUTER; + } + } + + // ... otherwise, setup exploration for a new + // empty state and add a brand new transition for + // this new range. + let next_id = + NextInsert::push(self, &mut stack, rest); + add_trans(self, i, state_id, r, next_id); + } + SplitRange::Both(r) => { + // Continue adding the remaining ranges on this + // path and update the transition with the new + // range. + if !rest.is_empty() { + stack.push(NextInsert::new(old.next_id, rest)); + } + add_trans(self, i, state_id, r, old.next_id); + } + } + i += 1; + } + // If we've reached this point, then we know that there are + // no subsequent transitions with any overlap. Therefore, we + // can stop processing this range and move on to the next one. + break; + } + } + self.insert_stack = stack; + } + + pub fn add_empty(&mut self) -> StateID { + let id = match StateID::try_from(self.states.len()) { + Ok(id) => id, + Err(_) => { + // This generally should not happen since a range trie is + // only ever used to compile a single sequence of Unicode + // scalar values. If we ever got to this point, we would, at + // *minimum*, be using 96GB in just the range trie alone. + panic!("too many sequences added to range trie"); + } + }; + // If we have some free states available, then use them to avoid + // more allocations. + if let Some(mut state) = self.free.pop() { + state.clear(); + self.states.push(state); + } else { + self.states.push(State { transitions: vec![] }); + } + id + } + + /// Performs a deep clone of the given state and returns the duplicate's + /// state ID. + /// + /// A "deep clone" in this context means that the state given along with + /// recursively all states that it points to are copied. Once complete, + /// the given state ID and the returned state ID share nothing. + /// + /// This is useful during range trie insertion when a new range overlaps + /// with an existing range that is bigger than the new one. The part + /// of the existing range that does *not* overlap with the new one is + /// duplicated so that adding the new range to the overlap doesn't disturb + /// the non-overlapping portion. + /// + /// There's one exception: if old_id is the final state, then it is not + /// duplicated and the same final state is returned. This is because all + /// final states in this trie are equivalent. + fn duplicate(&mut self, old_id: StateID) -> StateID { + if old_id == FINAL { + return FINAL; + } + + let mut stack = mem::replace(&mut self.dupe_stack, vec![]); + stack.clear(); + + let new_id = self.add_empty(); + // old_id is the state we're cloning and new_id is the ID of the + // duplicated state for old_id. + stack.push(NextDupe { old_id, new_id }); + while let Some(NextDupe { old_id, new_id }) = stack.pop() { + for i in 0..self.state(old_id).transitions.len() { + let t = self.state(old_id).transitions[i].clone(); + if t.next_id == FINAL { + // All final states are the same, so there's no need to + // duplicate it. + self.add_transition(new_id, t.range, FINAL); + continue; + } + + let new_child_id = self.add_empty(); + self.add_transition(new_id, t.range, new_child_id); + stack.push(NextDupe { + old_id: t.next_id, + new_id: new_child_id, + }); + } + } + self.dupe_stack = stack; + new_id + } + + /// Adds the given transition to the given state. + /// + /// Callers must ensure that all previous transitions in this state + /// are lexicographically smaller than the given range. + fn add_transition( + &mut self, + from_id: StateID, + range: Utf8Range, + next_id: StateID, + ) { + self.state_mut(from_id) + .transitions + .push(Transition { range, next_id }); + } + + /// Like `add_transition`, except this inserts the transition just before + /// the ith transition. + fn add_transition_at( + &mut self, + i: usize, + from_id: StateID, + range: Utf8Range, + next_id: StateID, + ) { + self.state_mut(from_id) + .transitions + .insert(i, Transition { range, next_id }); + } + + /// Overwrites the transition at position i with the given transition. + fn set_transition_at( + &mut self, + i: usize, + from_id: StateID, + range: Utf8Range, + next_id: StateID, + ) { + self.state_mut(from_id).transitions[i] = Transition { range, next_id }; + } + + /// Return an immutable borrow for the state with the given ID. + fn state(&self, id: StateID) -> &State { + &self.states[id] + } + + /// Return a mutable borrow for the state with the given ID. + fn state_mut(&mut self, id: StateID) -> &mut State { + &mut self.states[id] + } +} + +impl State { + /// Find the position at which the given range should be inserted in this + /// state. + /// + /// The position returned is always in the inclusive range + /// [0, transitions.len()]. If 'transitions.len()' is returned, then the + /// given range overlaps with no other range in this state *and* is greater + /// than all of them. + /// + /// For all other possible positions, the given range either overlaps + /// with the transition at that position or is otherwise less than it + /// with no overlap (and is greater than the previous transition). In the + /// former case, careful attention must be paid to inserting this range + /// as a new transition. In the latter case, the range can be inserted as + /// a new transition at the given position without disrupting any other + /// transitions. + fn find(&self, range: Utf8Range) -> usize { + /// Returns the position `i` at which `pred(xs[i])` first returns true + /// such that for all `j >= i`, `pred(xs[j]) == true`. If `pred` never + /// returns true, then `xs.len()` is returned. + /// + /// We roll our own binary search because it doesn't seem like the + /// standard library's binary search can be used here. Namely, if + /// there is an overlapping range, then we want to find the first such + /// occurrence, but there may be many. Or at least, it's not quite + /// clear to me how to do it. + fn binary_search<T, F>(xs: &[T], mut pred: F) -> usize + where + F: FnMut(&T) -> bool, + { + let (mut left, mut right) = (0, xs.len()); + while left < right { + // Overflow is impossible because xs.len() <= 256. + let mid = (left + right) / 2; + if pred(&xs[mid]) { + right = mid; + } else { + left = mid + 1; + } + } + left + } + + // Benchmarks suggest that binary search is just a bit faster than + // straight linear search. Specifically when using the debug tool: + // + // hyperfine "regex-cli debug thompson -qr --captures none '\w{90} ecurB'" + binary_search(&self.transitions, |t| range.start <= t.range.end) + } + + /// Clear this state such that it has zero transitions. + fn clear(&mut self) { + self.transitions.clear(); + } +} + +/// The next state to process during duplication. +#[derive(Clone, Debug)] +struct NextDupe { + /// The state we want to duplicate. + old_id: StateID, + /// The ID of the new state that is a duplicate of old_id. + new_id: StateID, +} + +/// The next state (and its corresponding transition) that we want to visit +/// during iteration in lexicographic order. +#[derive(Clone, Debug)] +struct NextIter { + state_id: StateID, + tidx: usize, +} + +/// The next state to process during insertion and any remaining ranges that we +/// want to add for a particular sequence of ranges. The first such instance +/// is always the root state along with all ranges given. +#[derive(Clone, Debug)] +struct NextInsert { + /// The next state to begin inserting ranges. This state should be the + /// state at which `ranges[0]` should be inserted. + state_id: StateID, + /// The ranges to insert. We used a fixed-size array here to avoid an + /// allocation. + ranges: [Utf8Range; 4], + /// The number of valid ranges in the above array. + len: u8, +} + +impl NextInsert { + /// Create the next item to visit. The given state ID should correspond + /// to the state at which the first range in the given slice should be + /// inserted. The slice given must not be empty and it must be no longer + /// than 4. + fn new(state_id: StateID, ranges: &[Utf8Range]) -> NextInsert { + let len = ranges.len(); + assert!(len > 0); + assert!(len <= 4); + + let mut tmp = [Utf8Range { start: 0, end: 0 }; 4]; + tmp[..len].copy_from_slice(ranges); + NextInsert { state_id, ranges: tmp, len: u8::try_from(len).unwrap() } + } + + /// Push a new empty state to visit along with any remaining ranges that + /// still need to be inserted. The ID of the new empty state is returned. + /// + /// If ranges is empty, then no new state is created and FINAL is returned. + fn push( + trie: &mut RangeTrie, + stack: &mut Vec<NextInsert>, + ranges: &[Utf8Range], + ) -> StateID { + if ranges.is_empty() { + FINAL + } else { + let next_id = trie.add_empty(); + stack.push(NextInsert::new(next_id, ranges)); + next_id + } + } + + /// Return the ID of the state to visit. + fn state_id(&self) -> StateID { + self.state_id + } + + /// Return the remaining ranges to insert. + fn ranges(&self) -> &[Utf8Range] { + &self.ranges[..usize::try_from(self.len).unwrap()] + } +} + +/// Split represents a partitioning of two ranges into one or more ranges. This +/// is the secret sauce that makes a range trie work, as it's what tells us +/// how to deal with two overlapping but unequal ranges during insertion. +/// +/// Essentially, either two ranges overlap or they don't. If they don't, then +/// handling insertion is easy: just insert the new range into its +/// lexicographically correct position. Since it does not overlap with anything +/// else, no other transitions are impacted by the new range. +/// +/// If they do overlap though, there are generally three possible cases to +/// handle: +/// +/// 1. The part where the two ranges actually overlap. i.e., The intersection. +/// 2. The part of the existing range that is not in the new range. +/// 3. The part of the new range that is not in the old range. +/// +/// (1) is guaranteed to always occur since all overlapping ranges have a +/// non-empty intersection. If the two ranges are not equivalent, then at +/// least one of (2) or (3) is guaranteed to occur as well. In some cases, +/// e.g., `[0-4]` and `[4-9]`, all three cases will occur. +/// +/// This `Split` type is responsible for providing (1), (2) and (3) for any +/// possible pair of byte ranges. +/// +/// As for insertion, for the overlap in (1), the remaining ranges to insert +/// should be added by following the corresponding transition. However, this +/// should only be done for the overlapping parts of the range. If there was +/// a part of the existing range that was not in the new range, then that +/// existing part must be split off from the transition and duplicated. The +/// remaining parts of the overlap can then be added to using the new ranges +/// without disturbing the existing range. +/// +/// Handling the case for the part of a new range that is not in an existing +/// range is seemingly easy. Just treat it as if it were a non-overlapping +/// range. The problem here is that if this new non-overlapping range occurs +/// after both (1) and (2), then it's possible that it can overlap with the +/// next transition in the current state. If it does, then the whole process +/// must be repeated! +/// +/// # Details of the 3 cases +/// +/// The following details the various cases that are implemented in code +/// below. It's plausible that the number of cases is not actually minimal, +/// but it's important for this code to remain at least somewhat readable. +/// +/// Given [a,b] and [x,y], where a <= b, x <= y, b < 256 and y < 256, we define +/// the follow distinct relationships where at least one must apply. The order +/// of these matters, since multiple can match. The first to match applies. +/// +/// 1. b < x <=> [a,b] < [x,y] +/// 2. y < a <=> [x,y] < [a,b] +/// +/// In the case of (1) and (2), these are the only cases where there is no +/// overlap. Or otherwise, the intersection of [a,b] and [x,y] is empty. In +/// order to compute the intersection, one can do [max(a,x), min(b,y)]. The +/// intersection in all of the following cases is non-empty. +/// +/// 3. a = x && b = y <=> [a,b] == [x,y] +/// 4. a = x && b < y <=> [x,y] right-extends [a,b] +/// 5. b = y && a > x <=> [x,y] left-extends [a,b] +/// 6. x = a && y < b <=> [a,b] right-extends [x,y] +/// 7. y = b && x > a <=> [a,b] left-extends [x,y] +/// 8. a > x && b < y <=> [x,y] covers [a,b] +/// 9. x > a && y < b <=> [a,b] covers [x,y] +/// 10. b = x && a < y <=> [a,b] is left-adjacent to [x,y] +/// 11. y = a && x < b <=> [x,y] is left-adjacent to [a,b] +/// 12. b > x && b < y <=> [a,b] left-overlaps [x,y] +/// 13. y > a && y < b <=> [x,y] left-overlaps [a,b] +/// +/// In cases 3-13, we can form rules that partition the ranges into a +/// non-overlapping ordered sequence of ranges: +/// +/// 3. [a,b] +/// 4. [a,b], [b+1,y] +/// 5. [x,a-1], [a,b] +/// 6. [x,y], [y+1,b] +/// 7. [a,x-1], [x,y] +/// 8. [x,a-1], [a,b], [b+1,y] +/// 9. [a,x-1], [x,y], [y+1,b] +/// 10. [a,b-1], [b,b], [b+1,y] +/// 11. [x,y-1], [y,y], [y+1,b] +/// 12. [a,x-1], [x,b], [b+1,y] +/// 13. [x,a-1], [a,y], [y+1,b] +/// +/// In the code below, we go a step further and identify each of the above +/// outputs as belonging either to the overlap of the two ranges or to one +/// of [a,b] or [x,y] exclusively. +#[derive(Clone, Debug, Eq, PartialEq)] +struct Split { + partitions: [SplitRange; 3], + len: usize, +} + +/// A tagged range indicating how it was derived from a pair of ranges. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SplitRange { + Old(Utf8Range), + New(Utf8Range), + Both(Utf8Range), +} + +impl Split { + /// Create a partitioning of the given ranges. + /// + /// If the given ranges have an empty intersection, then None is returned. + fn new(o: Utf8Range, n: Utf8Range) -> Option<Split> { + let range = |r: RangeInclusive<u8>| Utf8Range { + start: *r.start(), + end: *r.end(), + }; + let old = |r| SplitRange::Old(range(r)); + let new = |r| SplitRange::New(range(r)); + let both = |r| SplitRange::Both(range(r)); + + // Use same names as the comment above to make it easier to compare. + let (a, b, x, y) = (o.start, o.end, n.start, n.end); + + if b < x || y < a { + // case 1, case 2 + None + } else if a == x && b == y { + // case 3 + Some(Split::parts1(both(a..=b))) + } else if a == x && b < y { + // case 4 + Some(Split::parts2(both(a..=b), new(b + 1..=y))) + } else if b == y && a > x { + // case 5 + Some(Split::parts2(new(x..=a - 1), both(a..=b))) + } else if x == a && y < b { + // case 6 + Some(Split::parts2(both(x..=y), old(y + 1..=b))) + } else if y == b && x > a { + // case 7 + Some(Split::parts2(old(a..=x - 1), both(x..=y))) + } else if a > x && b < y { + // case 8 + Some(Split::parts3(new(x..=a - 1), both(a..=b), new(b + 1..=y))) + } else if x > a && y < b { + // case 9 + Some(Split::parts3(old(a..=x - 1), both(x..=y), old(y + 1..=b))) + } else if b == x && a < y { + // case 10 + Some(Split::parts3(old(a..=b - 1), both(b..=b), new(b + 1..=y))) + } else if y == a && x < b { + // case 11 + Some(Split::parts3(new(x..=y - 1), both(y..=y), old(y + 1..=b))) + } else if b > x && b < y { + // case 12 + Some(Split::parts3(old(a..=x - 1), both(x..=b), new(b + 1..=y))) + } else if y > a && y < b { + // case 13 + Some(Split::parts3(new(x..=a - 1), both(a..=y), old(y + 1..=b))) + } else { + unreachable!() + } + } + + /// Create a new split with a single partition. This only occurs when two + /// ranges are equivalent. + fn parts1(r1: SplitRange) -> Split { + // This value doesn't matter since it is never accessed. + let nada = SplitRange::Old(Utf8Range { start: 0, end: 0 }); + Split { partitions: [r1, nada, nada], len: 1 } + } + + /// Create a new split with two partitions. + fn parts2(r1: SplitRange, r2: SplitRange) -> Split { + // This value doesn't matter since it is never accessed. + let nada = SplitRange::Old(Utf8Range { start: 0, end: 0 }); + Split { partitions: [r1, r2, nada], len: 2 } + } + + /// Create a new split with three partitions. + fn parts3(r1: SplitRange, r2: SplitRange, r3: SplitRange) -> Split { + Split { partitions: [r1, r2, r3], len: 3 } + } + + /// Return the partitions in this split as a slice. + fn as_slice(&self) -> &[SplitRange] { + &self.partitions[..self.len] + } +} + +impl fmt::Debug for RangeTrie { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f)?; + for (i, state) in self.states.iter().enumerate() { + let status = if i == FINAL.as_usize() { '*' } else { ' ' }; + writeln!(f, "{status}{i:06}: {state:?}")?; + } + Ok(()) + } +} + +impl fmt::Debug for State { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let rs = self + .transitions + .iter() + .map(|t| format!("{t:?}")) + .collect::<Vec<String>>() + .join(", "); + write!(f, "{rs}") + } +} + +impl fmt::Debug for Transition { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.range.start == self.range.end { + write!( + f, + "{:02X} => {:02X}", + self.range.start, + self.next_id.as_usize(), + ) + } else { + write!( + f, + "{:02X}-{:02X} => {:02X}", + self.range.start, + self.range.end, + self.next_id.as_usize(), + ) + } + } +} + +/// Returns true if and only if the given ranges intersect. +fn intersects(r1: Utf8Range, r2: Utf8Range) -> bool { + !(r1.end < r2.start || r2.end < r1.start) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn r(range: RangeInclusive<u8>) -> Utf8Range { + Utf8Range { start: *range.start(), end: *range.end() } + } + + fn split_maybe( + old: RangeInclusive<u8>, + new: RangeInclusive<u8>, + ) -> Option<Split> { + Split::new(r(old), r(new)) + } + + fn split( + old: RangeInclusive<u8>, + new: RangeInclusive<u8>, + ) -> Vec<SplitRange> { + split_maybe(old, new).unwrap().as_slice().to_vec() + } + + #[test] + fn no_splits() { + // case 1 + assert_eq!(None, split_maybe(0..=1, 2..=3)); + // case 2 + assert_eq!(None, split_maybe(2..=3, 0..=1)); + } + + #[test] + fn splits() { + let range = |r: RangeInclusive<u8>| Utf8Range { + start: *r.start(), + end: *r.end(), + }; + let old = |r| SplitRange::Old(range(r)); + let new = |r| SplitRange::New(range(r)); + let both = |r| SplitRange::Both(range(r)); + + // case 3 + assert_eq!(split(0..=0, 0..=0), vec![both(0..=0)]); + assert_eq!(split(9..=9, 9..=9), vec![both(9..=9)]); + + // case 4 + assert_eq!(split(0..=5, 0..=6), vec![both(0..=5), new(6..=6)]); + assert_eq!(split(0..=5, 0..=8), vec![both(0..=5), new(6..=8)]); + assert_eq!(split(5..=5, 5..=8), vec![both(5..=5), new(6..=8)]); + + // case 5 + assert_eq!(split(1..=5, 0..=5), vec![new(0..=0), both(1..=5)]); + assert_eq!(split(3..=5, 0..=5), vec![new(0..=2), both(3..=5)]); + assert_eq!(split(5..=5, 0..=5), vec![new(0..=4), both(5..=5)]); + + // case 6 + assert_eq!(split(0..=6, 0..=5), vec![both(0..=5), old(6..=6)]); + assert_eq!(split(0..=8, 0..=5), vec![both(0..=5), old(6..=8)]); + assert_eq!(split(5..=8, 5..=5), vec![both(5..=5), old(6..=8)]); + + // case 7 + assert_eq!(split(0..=5, 1..=5), vec![old(0..=0), both(1..=5)]); + assert_eq!(split(0..=5, 3..=5), vec![old(0..=2), both(3..=5)]); + assert_eq!(split(0..=5, 5..=5), vec![old(0..=4), both(5..=5)]); + + // case 8 + assert_eq!( + split(3..=6, 2..=7), + vec![new(2..=2), both(3..=6), new(7..=7)], + ); + assert_eq!( + split(3..=6, 1..=8), + vec![new(1..=2), both(3..=6), new(7..=8)], + ); + + // case 9 + assert_eq!( + split(2..=7, 3..=6), + vec![old(2..=2), both(3..=6), old(7..=7)], + ); + assert_eq!( + split(1..=8, 3..=6), + vec![old(1..=2), both(3..=6), old(7..=8)], + ); + + // case 10 + assert_eq!( + split(3..=6, 6..=7), + vec![old(3..=5), both(6..=6), new(7..=7)], + ); + assert_eq!( + split(3..=6, 6..=8), + vec![old(3..=5), both(6..=6), new(7..=8)], + ); + assert_eq!( + split(5..=6, 6..=7), + vec![old(5..=5), both(6..=6), new(7..=7)], + ); + + // case 11 + assert_eq!( + split(6..=7, 3..=6), + vec![new(3..=5), both(6..=6), old(7..=7)], + ); + assert_eq!( + split(6..=8, 3..=6), + vec![new(3..=5), both(6..=6), old(7..=8)], + ); + assert_eq!( + split(6..=7, 5..=6), + vec![new(5..=5), both(6..=6), old(7..=7)], + ); + + // case 12 + assert_eq!( + split(3..=7, 5..=9), + vec![old(3..=4), both(5..=7), new(8..=9)], + ); + assert_eq!( + split(3..=5, 4..=6), + vec![old(3..=3), both(4..=5), new(6..=6)], + ); + + // case 13 + assert_eq!( + split(5..=9, 3..=7), + vec![new(3..=4), both(5..=7), old(8..=9)], + ); + assert_eq!( + split(4..=6, 3..=5), + vec![new(3..=3), both(4..=5), old(6..=6)], + ); + } + + // Arguably there should be more tests here, but in practice, this data + // structure is well covered by the huge number of regex tests. +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/alphabet.rs b/third_party/crates/regex-automata-0.4.18/src/util/alphabet.rs new file mode 100644 index 0000000..475f951 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/alphabet.rs @@ -0,0 +1,1139 @@ +/*! +This module provides APIs for dealing with the alphabets of finite state +machines. + +There are two principal types in this module, [`ByteClasses`] and [`Unit`]. +The former defines the alphabet of a finite state machine while the latter +represents an element of that alphabet. + +To a first approximation, the alphabet of all automata in this crate is just +a `u8`. Namely, every distinct byte value. All 256 of them. In practice, this +can be quite wasteful when building a transition table for a DFA, since it +requires storing a state identifier for each element in the alphabet. Instead, +we collapse the alphabet of an automaton down into equivalence classes, where +every byte in the same equivalence class never discriminates between a match or +a non-match from any other byte in the same class. For example, in the regex +`[a-z]+`, then you could consider it having an alphabet consisting of two +equivalence classes: `a-z` and everything else. In terms of the transitions on +an automaton, it doesn't actually require representing every distinct byte. +Just the equivalence classes. + +The downside of equivalence classes is that, of course, searching a haystack +deals with individual byte values. Those byte values need to be mapped to +their corresponding equivalence class. This is what `ByteClasses` does. In +practice, doing this for every state transition has negligible impact on modern +CPUs. Moreover, it helps make more efficient use of the CPU cache by (possibly +considerably) shrinking the size of the transition table. + +One last hiccup concerns `Unit`. Namely, because of look-around and how the +DFAs in this crate work, we need to add a sentinel value to our alphabet +of equivalence classes that represents the "end" of a search. We call that +sentinel [`Unit::eoi`] or "end of input." Thus, a `Unit` is either an +equivalence class corresponding to a set of bytes, or it is a special "end of +input" sentinel. + +In general, you should not expect to need either of these types unless you're +doing lower level shenanigans with DFAs, or even building your own DFAs. +(Although, you don't have to use these types to build your own DFAs of course.) +For example, if you're walking a DFA's state graph, it's probably useful to +make use of [`ByteClasses`] to visit each element in the DFA's alphabet instead +of just visiting every distinct `u8` value. The latter isn't necessarily wrong, +but it could be potentially very wasteful. +*/ +use crate::util::{ + escape::DebugByte, + wire::{self, DeserializeError, SerializeError}, +}; + +/// Unit represents a single unit of haystack for DFA based regex engines. +/// +/// It is not expected for consumers of this crate to need to use this type +/// unless they are implementing their own DFA. And even then, it's not +/// required: implementors may use other techniques to handle haystack units. +/// +/// Typically, a single unit of haystack for a DFA would be a single byte. +/// However, for the DFAs in this crate, matches are delayed by a single byte +/// in order to handle look-ahead assertions (`\b`, `$` and `\z`). Thus, once +/// we have consumed the haystack, we must run the DFA through one additional +/// transition using a unit that indicates the haystack has ended. +/// +/// There is no way to represent a sentinel with a `u8` since all possible +/// values *may* be valid haystack units to a DFA, therefore this type +/// explicitly adds room for a sentinel value. +/// +/// The sentinel EOI value is always its own equivalence class and is +/// ultimately represented by adding 1 to the maximum equivalence class value. +/// So for example, the regex `^[a-z]+$` might be split into the following +/// equivalence classes: +/// +/// ```text +/// 0 => [\x00-`] +/// 1 => [a-z] +/// 2 => [{-\xFF] +/// 3 => [EOI] +/// ``` +/// +/// Where EOI is the special sentinel value that is always in its own +/// singleton equivalence class. +#[derive(Clone, Copy, Eq, PartialEq, PartialOrd, Ord)] +pub struct Unit(UnitKind); + +#[derive(Clone, Copy, Eq, PartialEq, PartialOrd, Ord)] +enum UnitKind { + /// Represents a byte value, or more typically, an equivalence class + /// represented as a byte value. + U8(u8), + /// Represents the "end of input" sentinel. We regrettably use a `u16` + /// here since the maximum sentinel value is `256`. Thankfully, we don't + /// actually store a `Unit` anywhere, so this extra space shouldn't be too + /// bad. + EOI(u16), +} + +impl Unit { + /// Create a new haystack unit from a byte value. + /// + /// All possible byte values are legal. However, when creating a haystack + /// unit for a specific DFA, one should be careful to only construct units + /// that are in that DFA's alphabet. Namely, one way to compact a DFA's + /// in-memory representation is to collapse its transitions to a set of + /// equivalence classes into a set of all possible byte values. If a DFA + /// uses equivalence classes instead of byte values, then the byte given + /// here should be the equivalence class. + pub fn u8(byte: u8) -> Unit { + Unit(UnitKind::U8(byte)) + } + + /// Create a new "end of input" haystack unit. + /// + /// The value given is the sentinel value used by this unit to represent + /// the "end of input." The value should be the total number of equivalence + /// classes in the corresponding alphabet. Its maximum value is `256`, + /// which occurs when every byte is its own equivalence class. + /// + /// # Panics + /// + /// This panics when `num_byte_equiv_classes` is greater than `256`. + pub fn eoi(num_byte_equiv_classes: usize) -> Unit { + assert!( + num_byte_equiv_classes <= 256, + "max number of byte-based equivalent classes is 256, but got \ + {num_byte_equiv_classes}", + ); + Unit(UnitKind::EOI(u16::try_from(num_byte_equiv_classes).unwrap())) + } + + /// If this unit is not an "end of input" sentinel, then returns its + /// underlying byte value. Otherwise return `None`. + pub fn as_u8(self) -> Option<u8> { + match self.0 { + UnitKind::U8(b) => Some(b), + UnitKind::EOI(_) => None, + } + } + + /// If this unit is an "end of input" sentinel, then return the underlying + /// sentinel value that was given to [`Unit::eoi`]. Otherwise return + /// `None`. + pub fn as_eoi(self) -> Option<u16> { + match self.0 { + UnitKind::U8(_) => None, + UnitKind::EOI(sentinel) => Some(sentinel), + } + } + + /// Return this unit as a `usize`, regardless of whether it is a byte value + /// or an "end of input" sentinel. In the latter case, the underlying + /// sentinel value given to [`Unit::eoi`] is returned. + pub fn as_usize(self) -> usize { + match self.0 { + UnitKind::U8(b) => usize::from(b), + UnitKind::EOI(eoi) => usize::from(eoi), + } + } + + /// Returns true if and only of this unit is a byte value equivalent to the + /// byte given. This always returns false when this is an "end of input" + /// sentinel. + pub fn is_byte(self, byte: u8) -> bool { + self.as_u8().map_or(false, |b| b == byte) + } + + /// Returns true when this unit represents an "end of input" sentinel. + pub fn is_eoi(self) -> bool { + self.as_eoi().is_some() + } + + /// Returns true when this unit corresponds to an ASCII word byte. + /// + /// This always returns false when this unit represents an "end of input" + /// sentinel. + pub fn is_word_byte(self) -> bool { + self.as_u8().map_or(false, crate::util::utf8::is_word_byte) + } +} + +impl core::fmt::Debug for Unit { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + match self.0 { + UnitKind::U8(b) => write!(f, "{:?}", DebugByte(b)), + UnitKind::EOI(_) => write!(f, "EOI"), + } + } +} + +/// A representation of byte oriented equivalence classes. +/// +/// This is used in a DFA to reduce the size of the transition table. This can +/// have a particularly large impact not only on the total size of a dense DFA, +/// but also on compile times. +/// +/// The essential idea here is that the alphabet of a DFA is shrunk from the +/// usual 256 distinct byte values down to a set of equivalence classes. The +/// guarantee you get is that any byte belonging to the same equivalence class +/// can be treated as if it were any other byte in the same class, and the +/// result of a search wouldn't change. +/// +/// # Example +/// +/// This example shows how to get byte classes from an +/// [`NFA`](crate::nfa::thompson::NFA) and ask for the class of various bytes. +/// +/// ``` +/// use regex_automata::nfa::thompson::NFA; +/// +/// let nfa = NFA::new("[a-z]+")?; +/// let classes = nfa.byte_classes(); +/// // 'a' and 'z' are in the same class for this regex. +/// assert_eq!(classes.get(b'a'), classes.get(b'z')); +/// // But 'a' and 'A' are not. +/// assert_ne!(classes.get(b'a'), classes.get(b'A')); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Copy)] +pub struct ByteClasses([u8; 256]); + +impl ByteClasses { + /// Creates a new set of equivalence classes where all bytes are mapped to + /// the same class. + #[inline] + pub fn empty() -> ByteClasses { + ByteClasses([0; 256]) + } + + /// Creates a new set of equivalence classes where each byte belongs to + /// its own equivalence class. + #[inline] + pub fn singletons() -> ByteClasses { + let mut classes = ByteClasses::empty(); + for b in 0..=255 { + classes.set(b, b); + } + classes + } + + /// Deserializes a byte class map from the given slice. If the slice is of + /// insufficient length or otherwise contains an impossible mapping, then + /// an error is returned. Upon success, the number of bytes read along with + /// the map are returned. The number of bytes read is always a multiple of + /// 8. + pub(crate) fn from_bytes( + slice: &[u8], + ) -> Result<(ByteClasses, usize), DeserializeError> { + wire::check_slice_len(slice, 256, "byte class map")?; + let mut classes = ByteClasses::empty(); + for (b, &class) in slice[..256].iter().enumerate() { + classes.set(u8::try_from(b).unwrap(), class); + } + // We specifically don't use 'classes.iter()' here because that + // iterator depends on 'classes.alphabet_len()' being correct. But that + // is precisely the thing we're trying to verify below! + for &b in classes.0.iter() { + if usize::from(b) >= classes.alphabet_len() { + return Err(DeserializeError::generic( + "found equivalence class greater than alphabet len", + )); + } + } + Ok((classes, 256)) + } + + /// Writes this byte class map to the given byte buffer. if the given + /// buffer is too small, then an error is returned. Upon success, the total + /// number of bytes written is returned. The number of bytes written is + /// guaranteed to be a multiple of 8. + pub(crate) fn write_to( + &self, + mut dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("byte class map")); + } + for b in 0..=255 { + dst[0] = self.get(b); + dst = &mut dst[1..]; + } + Ok(nwrite) + } + + /// Returns the total number of bytes written by `write_to`. + pub(crate) fn write_to_len(&self) -> usize { + 256 + } + + /// Set the equivalence class for the given byte. + #[inline] + pub fn set(&mut self, byte: u8, class: u8) { + self.0[usize::from(byte)] = class; + } + + /// Get the equivalence class for the given byte. + #[inline] + pub fn get(&self, byte: u8) -> u8 { + self.0[usize::from(byte)] + } + + /// Get the equivalence class for the given haystack unit and return the + /// class as a `usize`. + #[inline] + pub fn get_by_unit(&self, unit: Unit) -> usize { + match unit.0 { + UnitKind::U8(b) => usize::from(self.get(b)), + UnitKind::EOI(b) => usize::from(b), + } + } + + /// Create a unit that represents the "end of input" sentinel based on the + /// number of equivalence classes. + #[inline] + pub fn eoi(&self) -> Unit { + // The alphabet length already includes the EOI sentinel, hence why + // we subtract 1. + Unit::eoi(self.alphabet_len().checked_sub(1).unwrap()) + } + + /// Return the total number of elements in the alphabet represented by + /// these equivalence classes. Equivalently, this returns the total number + /// of equivalence classes. + #[inline] + pub fn alphabet_len(&self) -> usize { + // Add one since the number of equivalence classes is one bigger than + // the last one. But add another to account for the final EOI class + // that isn't explicitly represented. + usize::from(self.0[255]) + 1 + 1 + } + + /// Returns the stride, as a base-2 exponent, required for these + /// equivalence classes. + /// + /// The stride is always the smallest power of 2 that is greater than or + /// equal to the alphabet length, and the `stride2` returned here is the + /// exponent applied to `2` to get the smallest power. This is done so that + /// converting between premultiplied state IDs and indices can be done with + /// shifts alone, which is much faster than integer division. + #[inline] + pub fn stride2(&self) -> usize { + let zeros = self.alphabet_len().next_power_of_two().trailing_zeros(); + usize::try_from(zeros).unwrap() + } + + /// Returns true if and only if every byte in this class maps to its own + /// equivalence class. Equivalently, there are 257 equivalence classes + /// and each class contains either exactly one byte or corresponds to the + /// singleton class containing the "end of input" sentinel. + #[inline] + pub fn is_singleton(&self) -> bool { + self.alphabet_len() == 257 + } + + /// Returns an iterator over all equivalence classes in this set. + #[inline] + pub fn iter(&self) -> ByteClassIter<'_> { + ByteClassIter { classes: self, i: 0 } + } + + /// Returns an iterator over a sequence of representative bytes from each + /// equivalence class within the range of bytes given. + /// + /// When the given range is unbounded on both sides, the iterator yields + /// exactly N items, where N is equivalent to the number of equivalence + /// classes. Each item is an arbitrary byte drawn from each equivalence + /// class. + /// + /// This is useful when one is determinizing an NFA and the NFA's alphabet + /// hasn't been converted to equivalence classes. Picking an arbitrary byte + /// from each equivalence class then permits a full exploration of the NFA + /// instead of using every possible byte value and thus potentially saves + /// quite a lot of redundant work. + /// + /// # Example + /// + /// This shows an example of what a complete sequence of representatives + /// might look like from a real example. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, util::alphabet::Unit}; + /// + /// let nfa = NFA::new("[a-z]+")?; + /// let classes = nfa.byte_classes(); + /// let reps: Vec<Unit> = classes.representatives(..).collect(); + /// // Note that the specific byte values yielded are not guaranteed! + /// let expected = vec![ + /// Unit::u8(b'\x00'), + /// Unit::u8(b'a'), + /// Unit::u8(b'{'), + /// Unit::eoi(3), + /// ]; + /// assert_eq!(expected, reps); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Note though, that you can ask for an arbitrary range of bytes, and only + /// representatives for that range will be returned: + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, util::alphabet::Unit}; + /// + /// let nfa = NFA::new("[a-z]+")?; + /// let classes = nfa.byte_classes(); + /// let reps: Vec<Unit> = classes.representatives(b'A'..=b'z').collect(); + /// // Note that the specific byte values yielded are not guaranteed! + /// let expected = vec![ + /// Unit::u8(b'A'), + /// Unit::u8(b'a'), + /// ]; + /// assert_eq!(expected, reps); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn representatives<R: core::ops::RangeBounds<u8>>( + &self, + range: R, + ) -> ByteClassRepresentatives<'_> { + use core::ops::Bound; + + let cur_byte = match range.start_bound() { + Bound::Included(&i) => usize::from(i), + Bound::Excluded(&i) => usize::from(i).checked_add(1).unwrap(), + Bound::Unbounded => 0, + }; + let end_byte = match range.end_bound() { + Bound::Included(&i) => { + Some(usize::from(i).checked_add(1).unwrap()) + } + Bound::Excluded(&i) => Some(usize::from(i)), + Bound::Unbounded => None, + }; + assert_ne!( + cur_byte, + usize::MAX, + "start range must be less than usize::MAX", + ); + ByteClassRepresentatives { + classes: self, + cur_byte, + end_byte, + last_class: None, + } + } + + /// Returns an iterator of the bytes in the given equivalence class. + /// + /// This is useful when one needs to know the actual bytes that belong to + /// an equivalence class. For example, conceptually speaking, accelerating + /// a DFA state occurs when a state only has a few outgoing transitions. + /// But in reality, what is required is that there are only a small + /// number of distinct bytes that can lead to an outgoing transition. The + /// difference is that any one transition can correspond to an equivalence + /// class which may contains many bytes. Therefore, DFA state acceleration + /// considers the actual elements in each equivalence class of each + /// outgoing transition. + /// + /// # Example + /// + /// This shows an example of how to get all of the elements in an + /// equivalence class. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, util::alphabet::Unit}; + /// + /// let nfa = NFA::new("[a-z]+")?; + /// let classes = nfa.byte_classes(); + /// let elements: Vec<Unit> = classes.elements(Unit::u8(1)).collect(); + /// let expected: Vec<Unit> = (b'a'..=b'z').map(Unit::u8).collect(); + /// assert_eq!(expected, elements); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn elements(&self, class: Unit) -> ByteClassElements<'_> { + ByteClassElements { classes: self, class, byte: 0 } + } + + /// Returns an iterator of byte ranges in the given equivalence class. + /// + /// That is, a sequence of contiguous ranges are returned. Typically, every + /// class maps to a single contiguous range. + fn element_ranges(&self, class: Unit) -> ByteClassElementRanges<'_> { + ByteClassElementRanges { elements: self.elements(class), range: None } + } +} + +impl Default for ByteClasses { + fn default() -> ByteClasses { + ByteClasses::singletons() + } +} + +impl core::fmt::Debug for ByteClasses { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + if self.is_singleton() { + write!(f, "ByteClasses({{singletons}})") + } else { + write!(f, "ByteClasses(")?; + for (i, class) in self.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{:?} => [", class.as_usize())?; + for (start, end) in self.element_ranges(class) { + if start == end { + write!(f, "{start:?}")?; + } else { + write!(f, "{start:?}-{end:?}")?; + } + } + write!(f, "]")?; + } + write!(f, ")") + } + } +} + +/// An iterator over each equivalence class. +/// +/// The last element in this iterator always corresponds to [`Unit::eoi`]. +/// +/// This is created by the [`ByteClasses::iter`] method. +/// +/// The lifetime `'a` refers to the lifetime of the byte classes that this +/// iterator was created from. +#[derive(Debug)] +pub struct ByteClassIter<'a> { + classes: &'a ByteClasses, + i: usize, +} + +impl<'a> Iterator for ByteClassIter<'a> { + type Item = Unit; + + fn next(&mut self) -> Option<Unit> { + if self.i + 1 == self.classes.alphabet_len() { + self.i += 1; + Some(self.classes.eoi()) + } else if self.i < self.classes.alphabet_len() { + let class = u8::try_from(self.i).unwrap(); + self.i += 1; + Some(Unit::u8(class)) + } else { + None + } + } +} + +/// An iterator over representative bytes from each equivalence class. +/// +/// This is created by the [`ByteClasses::representatives`] method. +/// +/// The lifetime `'a` refers to the lifetime of the byte classes that this +/// iterator was created from. +#[derive(Debug)] +pub struct ByteClassRepresentatives<'a> { + classes: &'a ByteClasses, + cur_byte: usize, + end_byte: Option<usize>, + last_class: Option<u8>, +} + +impl<'a> Iterator for ByteClassRepresentatives<'a> { + type Item = Unit; + + fn next(&mut self) -> Option<Unit> { + while self.cur_byte < self.end_byte.unwrap_or(256) { + let byte = u8::try_from(self.cur_byte).unwrap(); + let class = self.classes.get(byte); + self.cur_byte += 1; + + if self.last_class != Some(class) { + self.last_class = Some(class); + return Some(Unit::u8(byte)); + } + } + if self.cur_byte != usize::MAX && self.end_byte.is_none() { + // Using usize::MAX as a sentinel is OK because we ban usize::MAX + // from appearing as a start bound in iterator construction. But + // why do it this way? Well, we want to return the EOI class + // whenever the end of the given range is unbounded because EOI + // isn't really a "byte" per se, so the only way it should be + // excluded is if there is a bounded end to the range. Therefore, + // when the end is unbounded, we just need to know whether we've + // reported EOI or not. When we do, we set cur_byte to a value it + // can never otherwise be. + self.cur_byte = usize::MAX; + return Some(self.classes.eoi()); + } + None + } +} + +/// An iterator over all elements in an equivalence class. +/// +/// This is created by the [`ByteClasses::elements`] method. +/// +/// The lifetime `'a` refers to the lifetime of the byte classes that this +/// iterator was created from. +#[derive(Debug)] +pub struct ByteClassElements<'a> { + classes: &'a ByteClasses, + class: Unit, + byte: usize, +} + +impl<'a> Iterator for ByteClassElements<'a> { + type Item = Unit; + + fn next(&mut self) -> Option<Unit> { + while self.byte < 256 { + let byte = u8::try_from(self.byte).unwrap(); + self.byte += 1; + if self.class.is_byte(self.classes.get(byte)) { + return Some(Unit::u8(byte)); + } + } + if self.byte < 257 { + self.byte += 1; + if self.class.is_eoi() { + return Some(Unit::eoi(256)); + } + } + None + } +} + +/// An iterator over all elements in an equivalence class expressed as a +/// sequence of contiguous ranges. +#[derive(Debug)] +struct ByteClassElementRanges<'a> { + elements: ByteClassElements<'a>, + range: Option<(Unit, Unit)>, +} + +impl<'a> Iterator for ByteClassElementRanges<'a> { + type Item = (Unit, Unit); + + fn next(&mut self) -> Option<(Unit, Unit)> { + loop { + let element = match self.elements.next() { + None => return self.range.take(), + Some(element) => element, + }; + match self.range.take() { + None => { + self.range = Some((element, element)); + } + Some((start, end)) => { + if end.as_usize() + 1 != element.as_usize() + || element.is_eoi() + { + self.range = Some((element, element)); + return Some((start, end)); + } + self.range = Some((start, element)); + } + } + } + } +} + +/// A partitioning of bytes into equivalence classes. +/// +/// A byte class set keeps track of an *approximation* of equivalence classes +/// of bytes during NFA construction. That is, every byte in an equivalence +/// class cannot discriminate between a match and a non-match. +/// +/// For example, in the regex `[ab]+`, the bytes `a` and `b` would be in the +/// same equivalence class because it never matters whether an `a` or a `b` is +/// seen, and no combination of `a`s and `b`s in the text can discriminate a +/// match. +/// +/// Note though that this does not compute the minimal set of equivalence +/// classes. For example, in the regex `[ac]+`, both `a` and `c` are in the +/// same equivalence class for the same reason that `a` and `b` are in the +/// same equivalence class in the aforementioned regex. However, in this +/// implementation, `a` and `c` are put into distinct equivalence classes. The +/// reason for this is implementation complexity. In the future, we should +/// endeavor to compute the minimal equivalence classes since they can have a +/// rather large impact on the size of the DFA. (Doing this will likely require +/// rethinking how equivalence classes are computed, including changing the +/// representation here, which is only able to group contiguous bytes into the +/// same equivalence class.) +#[cfg(feature = "alloc")] +#[derive(Clone, Debug)] +pub(crate) struct ByteClassSet(ByteSet); + +#[cfg(feature = "alloc")] +impl Default for ByteClassSet { + fn default() -> ByteClassSet { + ByteClassSet::empty() + } +} + +#[cfg(feature = "alloc")] +impl ByteClassSet { + /// Create a new set of byte classes where all bytes are part of the same + /// equivalence class. + pub(crate) fn empty() -> Self { + ByteClassSet(ByteSet::empty()) + } + + /// Indicate the range of byte given (inclusive) can discriminate a + /// match between it and all other bytes outside of the range. + pub(crate) fn set_range(&mut self, start: u8, end: u8) { + debug_assert!(start <= end); + if start > 0 { + self.0.add(start - 1); + } + self.0.add(end); + } + + /// Add the contiguous ranges in the set given to this byte class set. + pub(crate) fn add_set(&mut self, set: &ByteSet) { + for (start, end) in set.iter_ranges() { + self.set_range(start, end); + } + } + + /// Convert this boolean set to a map that maps all byte values to their + /// corresponding equivalence class. The last mapping indicates the largest + /// equivalence class identifier (which is never bigger than 255). + pub(crate) fn byte_classes(&self) -> ByteClasses { + let mut classes = ByteClasses::empty(); + let mut class = 0u8; + let mut b = 0u8; + loop { + classes.set(b, class); + if b == 255 { + break; + } + if self.0.contains(b) { + class = class.checked_add(1).unwrap(); + } + b = b.checked_add(1).unwrap(); + } + classes + } +} + +/// A simple set of bytes that is reasonably cheap to copy and allocation free. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct ByteSet { + bits: BitSet, +} + +/// The representation of a byte set. Split out so that we can define a +/// convenient Debug impl for it while keeping "ByteSet" in the output. +#[derive(Clone, Copy, Default, Eq, PartialEq)] +struct BitSet([u128; 2]); + +impl ByteSet { + /// Create an empty set of bytes. + pub(crate) fn empty() -> ByteSet { + ByteSet { bits: BitSet([0; 2]) } + } + + /// Add a byte to this set. + /// + /// If the given byte already belongs to this set, then this is a no-op. + pub(crate) fn add(&mut self, byte: u8) { + let bucket = byte / 128; + let bit = byte % 128; + self.bits.0[usize::from(bucket)] |= 1 << bit; + } + + /// Remove a byte from this set. + /// + /// If the given byte is not in this set, then this is a no-op. + pub(crate) fn remove(&mut self, byte: u8) { + let bucket = byte / 128; + let bit = byte % 128; + self.bits.0[usize::from(bucket)] &= !(1 << bit); + } + + /// Return true if and only if the given byte is in this set. + pub(crate) fn contains(&self, byte: u8) -> bool { + let bucket = byte / 128; + let bit = byte % 128; + self.bits.0[usize::from(bucket)] & (1 << bit) > 0 + } + + /// Return true if and only if the given inclusive range of bytes is in + /// this set. + pub(crate) fn contains_range(&self, start: u8, end: u8) -> bool { + (start..=end).all(|b| self.contains(b)) + } + + /// Returns an iterator over all bytes in this set. + pub(crate) fn iter(&self) -> ByteSetIter<'_> { + ByteSetIter { set: self, b: 0 } + } + + /// Returns an iterator over all contiguous ranges of bytes in this set. + pub(crate) fn iter_ranges(&self) -> ByteSetRangeIter<'_> { + ByteSetRangeIter { set: self, b: 0 } + } + + /// Return true if and only if this set is empty. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn is_empty(&self) -> bool { + self.bits.0 == [0, 0] + } + + /// Deserializes a byte set from the given slice. If the slice is of + /// incorrect length or is otherwise malformed, then an error is returned. + /// Upon success, the number of bytes read along with the set are returned. + /// The number of bytes read is always a multiple of 8. + pub(crate) fn from_bytes( + slice: &[u8], + ) -> Result<(ByteSet, usize), DeserializeError> { + use core::mem::size_of; + + wire::check_slice_len(slice, 2 * size_of::<u128>(), "byte set")?; + let mut nread = 0; + let (low, nr) = wire::try_read_u128(slice, "byte set low bucket")?; + nread += nr; + let (high, nr) = wire::try_read_u128(slice, "byte set high bucket")?; + nread += nr; + Ok((ByteSet { bits: BitSet([low, high]) }, nread)) + } + + /// Writes this byte set to the given byte buffer. If the given buffer is + /// too small, then an error is returned. Upon success, the total number of + /// bytes written is returned. The number of bytes written is guaranteed to + /// be a multiple of 8. + pub(crate) fn write_to<E: crate::util::wire::Endian>( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + use core::mem::size_of; + + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("byte set")); + } + let mut nw = 0; + E::write_u128(self.bits.0[0], &mut dst[nw..]); + nw += size_of::<u128>(); + E::write_u128(self.bits.0[1], &mut dst[nw..]); + nw += size_of::<u128>(); + assert_eq!(nwrite, nw, "expected to write certain number of bytes",); + assert_eq!( + nw % 8, + 0, + "expected to write multiple of 8 bytes for byte set", + ); + Ok(nw) + } + + /// Returns the total number of bytes written by `write_to`. + pub(crate) fn write_to_len(&self) -> usize { + 2 * core::mem::size_of::<u128>() + } +} + +impl core::fmt::Debug for BitSet { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut fmtd = f.debug_set(); + for b in 0u8..=255 { + if (ByteSet { bits: *self }).contains(b) { + fmtd.entry(&b); + } + } + fmtd.finish() + } +} + +#[derive(Debug)] +pub(crate) struct ByteSetIter<'a> { + set: &'a ByteSet, + b: usize, +} + +impl<'a> Iterator for ByteSetIter<'a> { + type Item = u8; + + fn next(&mut self) -> Option<u8> { + while self.b <= 255 { + let b = u8::try_from(self.b).unwrap(); + self.b += 1; + if self.set.contains(b) { + return Some(b); + } + } + None + } +} + +#[derive(Debug)] +pub(crate) struct ByteSetRangeIter<'a> { + set: &'a ByteSet, + b: usize, +} + +impl<'a> Iterator for ByteSetRangeIter<'a> { + type Item = (u8, u8); + + fn next(&mut self) -> Option<(u8, u8)> { + let asu8 = |n: usize| u8::try_from(n).unwrap(); + while self.b <= 255 { + let start = asu8(self.b); + self.b += 1; + if !self.set.contains(start) { + continue; + } + + let mut end = start; + while self.b <= 255 && self.set.contains(asu8(self.b)) { + end = asu8(self.b); + self.b += 1; + } + return Some((start, end)); + } + None + } +} + +#[cfg(all(test, feature = "alloc"))] +mod tests { + use alloc::{vec, vec::Vec}; + + use super::*; + + #[test] + fn byte_classes() { + let mut set = ByteClassSet::empty(); + set.set_range(b'a', b'z'); + + let classes = set.byte_classes(); + assert_eq!(classes.get(0), 0); + assert_eq!(classes.get(1), 0); + assert_eq!(classes.get(2), 0); + assert_eq!(classes.get(b'a' - 1), 0); + assert_eq!(classes.get(b'a'), 1); + assert_eq!(classes.get(b'm'), 1); + assert_eq!(classes.get(b'z'), 1); + assert_eq!(classes.get(b'z' + 1), 2); + assert_eq!(classes.get(254), 2); + assert_eq!(classes.get(255), 2); + + let mut set = ByteClassSet::empty(); + set.set_range(0, 2); + set.set_range(4, 6); + let classes = set.byte_classes(); + assert_eq!(classes.get(0), 0); + assert_eq!(classes.get(1), 0); + assert_eq!(classes.get(2), 0); + assert_eq!(classes.get(3), 1); + assert_eq!(classes.get(4), 2); + assert_eq!(classes.get(5), 2); + assert_eq!(classes.get(6), 2); + assert_eq!(classes.get(7), 3); + assert_eq!(classes.get(255), 3); + } + + #[test] + fn full_byte_classes() { + let mut set = ByteClassSet::empty(); + for b in 0u8..=255 { + set.set_range(b, b); + } + assert_eq!(set.byte_classes().alphabet_len(), 257); + } + + #[test] + fn elements_typical() { + let mut set = ByteClassSet::empty(); + set.set_range(b'b', b'd'); + set.set_range(b'g', b'm'); + set.set_range(b'z', b'z'); + let classes = set.byte_classes(); + // class 0: \x00-a + // class 1: b-d + // class 2: e-f + // class 3: g-m + // class 4: n-y + // class 5: z-z + // class 6: \x7B-\xFF + // class 7: EOI + assert_eq!(classes.alphabet_len(), 8); + + let elements = classes.elements(Unit::u8(0)).collect::<Vec<_>>(); + assert_eq!(elements.len(), 98); + assert_eq!(elements[0], Unit::u8(b'\x00')); + assert_eq!(elements[97], Unit::u8(b'a')); + + let elements = classes.elements(Unit::u8(1)).collect::<Vec<_>>(); + assert_eq!( + elements, + vec![Unit::u8(b'b'), Unit::u8(b'c'), Unit::u8(b'd')], + ); + + let elements = classes.elements(Unit::u8(2)).collect::<Vec<_>>(); + assert_eq!(elements, vec![Unit::u8(b'e'), Unit::u8(b'f')],); + + let elements = classes.elements(Unit::u8(3)).collect::<Vec<_>>(); + assert_eq!( + elements, + vec![ + Unit::u8(b'g'), + Unit::u8(b'h'), + Unit::u8(b'i'), + Unit::u8(b'j'), + Unit::u8(b'k'), + Unit::u8(b'l'), + Unit::u8(b'm'), + ], + ); + + let elements = classes.elements(Unit::u8(4)).collect::<Vec<_>>(); + assert_eq!(elements.len(), 12); + assert_eq!(elements[0], Unit::u8(b'n')); + assert_eq!(elements[11], Unit::u8(b'y')); + + let elements = classes.elements(Unit::u8(5)).collect::<Vec<_>>(); + assert_eq!(elements, vec![Unit::u8(b'z')]); + + let elements = classes.elements(Unit::u8(6)).collect::<Vec<_>>(); + assert_eq!(elements.len(), 133); + assert_eq!(elements[0], Unit::u8(b'\x7B')); + assert_eq!(elements[132], Unit::u8(b'\xFF')); + + let elements = classes.elements(Unit::eoi(7)).collect::<Vec<_>>(); + assert_eq!(elements, vec![Unit::eoi(256)]); + } + + #[test] + fn elements_singletons() { + let classes = ByteClasses::singletons(); + assert_eq!(classes.alphabet_len(), 257); + + let elements = classes.elements(Unit::u8(b'a')).collect::<Vec<_>>(); + assert_eq!(elements, vec![Unit::u8(b'a')]); + + let elements = classes.elements(Unit::eoi(5)).collect::<Vec<_>>(); + assert_eq!(elements, vec![Unit::eoi(256)]); + } + + #[test] + fn elements_empty() { + let classes = ByteClasses::empty(); + assert_eq!(classes.alphabet_len(), 2); + + let elements = classes.elements(Unit::u8(0)).collect::<Vec<_>>(); + assert_eq!(elements.len(), 256); + assert_eq!(elements[0], Unit::u8(b'\x00')); + assert_eq!(elements[255], Unit::u8(b'\xFF')); + + let elements = classes.elements(Unit::eoi(1)).collect::<Vec<_>>(); + assert_eq!(elements, vec![Unit::eoi(256)]); + } + + #[test] + fn representatives() { + let mut set = ByteClassSet::empty(); + set.set_range(b'b', b'd'); + set.set_range(b'g', b'm'); + set.set_range(b'z', b'z'); + let classes = set.byte_classes(); + + let got: Vec<Unit> = classes.representatives(..).collect(); + let expected = vec![ + Unit::u8(b'\x00'), + Unit::u8(b'b'), + Unit::u8(b'e'), + Unit::u8(b'g'), + Unit::u8(b'n'), + Unit::u8(b'z'), + Unit::u8(b'\x7B'), + Unit::eoi(7), + ]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(..0).collect(); + assert!(got.is_empty()); + let got: Vec<Unit> = classes.representatives(1..1).collect(); + assert!(got.is_empty()); + let got: Vec<Unit> = classes.representatives(255..255).collect(); + assert!(got.is_empty()); + + // A weird case that is the only guaranteed to way to get an iterator + // of just the EOI class by excluding all possible byte values. + let got: Vec<Unit> = classes + .representatives(( + core::ops::Bound::Excluded(255), + core::ops::Bound::Unbounded, + )) + .collect(); + let expected = vec![Unit::eoi(7)]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(..=255).collect(); + let expected = vec![ + Unit::u8(b'\x00'), + Unit::u8(b'b'), + Unit::u8(b'e'), + Unit::u8(b'g'), + Unit::u8(b'n'), + Unit::u8(b'z'), + Unit::u8(b'\x7B'), + ]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(b'b'..=b'd').collect(); + let expected = vec![Unit::u8(b'b')]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(b'a'..=b'd').collect(); + let expected = vec![Unit::u8(b'a'), Unit::u8(b'b')]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(b'b'..=b'e').collect(); + let expected = vec![Unit::u8(b'b'), Unit::u8(b'e')]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(b'A'..=b'Z').collect(); + let expected = vec![Unit::u8(b'A')]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(b'A'..=b'z').collect(); + let expected = vec![ + Unit::u8(b'A'), + Unit::u8(b'b'), + Unit::u8(b'e'), + Unit::u8(b'g'), + Unit::u8(b'n'), + Unit::u8(b'z'), + ]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(b'z'..).collect(); + let expected = vec![Unit::u8(b'z'), Unit::u8(b'\x7B'), Unit::eoi(7)]; + assert_eq!(expected, got); + + let got: Vec<Unit> = classes.representatives(b'z'..=0xFF).collect(); + let expected = vec![Unit::u8(b'z'), Unit::u8(b'\x7B')]; + assert_eq!(expected, got); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/captures.rs b/third_party/crates/regex-automata-0.4.18/src/util/captures.rs new file mode 100644 index 0000000..5376f34 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/captures.rs @@ -0,0 +1,2551 @@ +/*! +Provides types for dealing with capturing groups. + +Capturing groups refer to sub-patterns of regexes that some regex engines can +report matching offsets for. For example, matching `[a-z]([0-9]+)` against +`a789` would give `a789` as the overall match (for the implicit capturing group +at index `0`) and `789` as the match for the capturing group `([0-9]+)` (an +explicit capturing group at index `1`). + +Not all regex engines can report match offsets for capturing groups. Indeed, +to a first approximation, regex engines that can report capturing group offsets +tend to be quite a bit slower than regex engines that can't. This is because +tracking capturing groups at search time usually requires more "power" that +in turn adds overhead. + +Other regex implementations might call capturing groups "submatches." + +# Overview + +The main types in this module are: + +* [`Captures`] records the capturing group offsets found during a search. It +provides convenience routines for looking up capturing group offsets by either +index or name. +* [`GroupInfo`] records the mapping between capturing groups and "slots," +where the latter are how capturing groups are recorded during a regex search. +This also keeps a mapping from capturing group name to index, and capture +group index to name. A `GroupInfo` is used by `Captures` internally to +provide a convenient API. It is unlikely that you'll use a `GroupInfo` +directly, but for example, if you've compiled an Thompson NFA, then you can use +[`thompson::NFA::group_info`](crate::nfa::thompson::NFA::group_info) to get its +underlying `GroupInfo`. +*/ + +use alloc::{string::String, sync::Arc, vec, vec::Vec}; + +use crate::util::{ + interpolate, + primitives::{ + NonMaxUsize, PatternID, PatternIDError, PatternIDIter, SmallIndex, + }, + search::{Match, Span}, +}; + +/// The span offsets of capturing groups after a match has been found. +/// +/// This type represents the output of regex engines that can report the +/// offsets at which capturing groups matches or "submatches" occur. For +/// example, the [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM). When a match +/// occurs, it will at minimum contain the [`PatternID`] of the pattern that +/// matched. Depending upon how it was constructed, it may also contain the +/// start/end offsets of the entire match of the pattern and the start/end +/// offsets of each capturing group that participated in the match. +/// +/// Values of this type are always created for a specific [`GroupInfo`]. It is +/// unspecified behavior to use a `Captures` value in a search with any regex +/// engine that has a different `GroupInfo` than the one the `Captures` were +/// created with. +/// +/// # Constructors +/// +/// There are three constructors for this type that control what kind of +/// information is available upon a match: +/// +/// * [`Captures::all`]: Will store overall pattern match offsets in addition +/// to the offsets of capturing groups that participated in the match. +/// * [`Captures::matches`]: Will store only the overall pattern +/// match offsets. The offsets of capturing groups (even ones that participated +/// in the match) are not available. +/// * [`Captures::empty`]: Will only store the pattern ID that matched. No +/// match offsets are available at all. +/// +/// If you aren't sure which to choose, then pick the first one. The first one +/// is what convenience routines like, +/// [`PikeVM::create_captures`](crate::nfa::thompson::pikevm::PikeVM::create_captures), +/// will use automatically. +/// +/// The main difference between these choices is performance. Namely, if you +/// ask for _less_ information, then the execution of regex search may be able +/// to run more quickly. +/// +/// # Notes +/// +/// It is worth pointing out that this type is not coupled to any one specific +/// regex engine. Instead, its coupling is with [`GroupInfo`], which is the +/// thing that is responsible for mapping capturing groups to "slot" offsets. +/// Slot offsets are indices into a single sequence of memory at which matching +/// haystack offsets for the corresponding group are written by regex engines. +/// +/// # Example +/// +/// This example shows how to parse a simple date and extract the components of +/// the date via capturing groups: +/// +/// ``` +/// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span}; +/// +/// let re = PikeVM::new(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})$")?; +/// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); +/// +/// re.captures(&mut cache, "2010-03-14", &mut caps); +/// assert!(caps.is_match()); +/// assert_eq!(Some(Span::from(0..4)), caps.get_group(1)); +/// assert_eq!(Some(Span::from(5..7)), caps.get_group(2)); +/// assert_eq!(Some(Span::from(8..10)), caps.get_group(3)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: named capturing groups +/// +/// This example is like the one above, but leverages the ability to name +/// capturing groups in order to make the code a bit clearer: +/// +/// ``` +/// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span}; +/// +/// let re = PikeVM::new(r"^(?P<y>[0-9]{4})-(?P<m>[0-9]{2})-(?P<d>[0-9]{2})$")?; +/// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); +/// +/// re.captures(&mut cache, "2010-03-14", &mut caps); +/// assert!(caps.is_match()); +/// assert_eq!(Some(Span::from(0..4)), caps.get_group_by_name("y")); +/// assert_eq!(Some(Span::from(5..7)), caps.get_group_by_name("m")); +/// assert_eq!(Some(Span::from(8..10)), caps.get_group_by_name("d")); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone)] +pub struct Captures { + /// The group info that these capture groups are coupled to. This is what + /// gives the "convenience" of the `Captures` API. Namely, it provides the + /// slot mapping and the name|-->index mapping for capture lookups by name. + group_info: GroupInfo, + /// The ID of the pattern that matched. Regex engines must set this to + /// None when no match occurs. + pid: Option<PatternID>, + /// The slot values, i.e., submatch offsets. + /// + /// In theory, the smallest sequence of slots would be something like + /// `max(groups(pattern) for pattern in regex) * 2`, but instead, we use + /// `sum(groups(pattern) for pattern in regex) * 2`. Why? + /// + /// Well, the former could be used in theory, because we don't generally + /// have any overlapping APIs that involve capturing groups. Therefore, + /// there's technically never any need to have slots set for multiple + /// patterns. However, this might change some day, in which case, we would + /// need to have slots available. + /// + /// The other reason is that during the execution of some regex engines, + /// there exists a point in time where multiple slots for different + /// patterns may be written to before knowing which pattern has matched. + /// Therefore, the regex engines themselves, in order to support multiple + /// patterns correctly, must have all slots available. If `Captures` + /// doesn't have all slots available, then regex engines can't write + /// directly into the caller provided `Captures` and must instead write + /// into some other storage and then copy the slots involved in the match + /// at the end of the search. + /// + /// So overall, at least as of the time of writing, it seems like the path + /// of least resistance is to just require allocating all possible slots + /// instead of the conceptual minimum. Another way to justify this is that + /// the most common case is a single pattern, in which case, there is no + /// inefficiency here since the 'max' and 'sum' calculations above are + /// equivalent in that case. + /// + /// N.B. The mapping from group index to slot is maintained by `GroupInfo` + /// and is considered an API guarantee. See `GroupInfo` for more details on + /// that mapping. + /// + /// N.B. `Option<NonMaxUsize>` has the same size as a `usize`. + slots: Vec<Option<NonMaxUsize>>, +} + +impl Captures { + /// Create new storage for the offsets of all matching capturing groups. + /// + /// This routine provides the most information for matches---namely, the + /// spans of matching capturing groups---but also requires the regex search + /// routines to do the most work. + /// + /// It is unspecified behavior to use the returned `Captures` value in a + /// search with a `GroupInfo` other than the one that is provided to this + /// constructor. + /// + /// # Example + /// + /// This example shows that all capturing groups---but only ones that + /// participated in a match---are available to query after a match has + /// been found: + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::captures::Captures, + /// Span, Match, + /// }; + /// + /// let re = PikeVM::new( + /// r"^(?:(?P<lower>[a-z]+)|(?P<upper>[A-Z]+))(?P<digits>[0-9]+)$", + /// )?; + /// let mut cache = re.create_cache(); + /// let mut caps = Captures::all(re.get_nfa().group_info().clone()); + /// + /// re.captures(&mut cache, "ABC123", &mut caps); + /// assert!(caps.is_match()); + /// assert_eq!(Some(Match::must(0, 0..6)), caps.get_match()); + /// // The 'lower' group didn't match, so it won't have any offsets. + /// assert_eq!(None, caps.get_group_by_name("lower")); + /// assert_eq!(Some(Span::from(0..3)), caps.get_group_by_name("upper")); + /// assert_eq!(Some(Span::from(3..6)), caps.get_group_by_name("digits")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn all(group_info: GroupInfo) -> Captures { + let slots = group_info.slot_len(); + Captures { group_info, pid: None, slots: vec![None; slots] } + } + + /// Create new storage for only the full match spans of a pattern. This + /// does not include any capturing group offsets. + /// + /// It is unspecified behavior to use the returned `Captures` value in a + /// search with a `GroupInfo` other than the one that is provided to this + /// constructor. + /// + /// # Example + /// + /// This example shows that only overall match offsets are reported when + /// this constructor is used. Accessing any capturing groups other than + /// the 0th will always return `None`. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::captures::Captures, + /// Match, + /// }; + /// + /// let re = PikeVM::new( + /// r"^(?:(?P<lower>[a-z]+)|(?P<upper>[A-Z]+))(?P<digits>[0-9]+)$", + /// )?; + /// let mut cache = re.create_cache(); + /// let mut caps = Captures::matches(re.get_nfa().group_info().clone()); + /// + /// re.captures(&mut cache, "ABC123", &mut caps); + /// assert!(caps.is_match()); + /// assert_eq!(Some(Match::must(0, 0..6)), caps.get_match()); + /// // We didn't ask for capturing group offsets, so they aren't available. + /// assert_eq!(None, caps.get_group_by_name("lower")); + /// assert_eq!(None, caps.get_group_by_name("upper")); + /// assert_eq!(None, caps.get_group_by_name("digits")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn matches(group_info: GroupInfo) -> Captures { + // This is OK because we know there are at least this many slots, + // and GroupInfo construction guarantees that the number of slots fits + // into a usize. + let slots = group_info.pattern_len().checked_mul(2).unwrap(); + Captures { group_info, pid: None, slots: vec![None; slots] } + } + + /// Create new storage for only tracking which pattern matched. No offsets + /// are stored at all. + /// + /// It is unspecified behavior to use the returned `Captures` value in a + /// search with a `GroupInfo` other than the one that is provided to this + /// constructor. + /// + /// # Example + /// + /// This example shows that only the pattern that matched can be accessed + /// from a `Captures` value created via this constructor. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::captures::Captures, + /// PatternID, + /// }; + /// + /// let re = PikeVM::new_many(&[r"[a-z]+", r"[A-Z]+"])?; + /// let mut cache = re.create_cache(); + /// let mut caps = Captures::empty(re.get_nfa().group_info().clone()); + /// + /// re.captures(&mut cache, "aABCz", &mut caps); + /// assert!(caps.is_match()); + /// assert_eq!(Some(PatternID::must(0)), caps.pattern()); + /// // We didn't ask for any offsets, so they aren't available. + /// assert_eq!(None, caps.get_match()); + /// + /// re.captures(&mut cache, &"aABCz"[1..], &mut caps); + /// assert!(caps.is_match()); + /// assert_eq!(Some(PatternID::must(1)), caps.pattern()); + /// // We didn't ask for any offsets, so they aren't available. + /// assert_eq!(None, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn empty(group_info: GroupInfo) -> Captures { + Captures { group_info, pid: None, slots: vec![] } + } + + /// Returns true if and only if this capturing group represents a match. + /// + /// This is a convenience routine for `caps.pattern().is_some()`. + /// + /// # Example + /// + /// When using the PikeVM (for example), the lightest weight way of + /// detecting whether a match exists is to create capturing groups that + /// only track the ID of the pattern that match (if any): + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::captures::Captures, + /// }; + /// + /// let re = PikeVM::new(r"[a-z]+")?; + /// let mut cache = re.create_cache(); + /// let mut caps = Captures::empty(re.get_nfa().group_info().clone()); + /// + /// re.captures(&mut cache, "aABCz", &mut caps); + /// assert!(caps.is_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_match(&self) -> bool { + self.pid.is_some() + } + + /// Returns the identifier of the pattern that matched when this + /// capturing group represents a match. If no match was found, then this + /// always returns `None`. + /// + /// This returns a pattern ID in precisely the cases in which `is_match` + /// returns `true`. Similarly, the pattern ID returned is always the + /// same pattern ID found in the `Match` returned by `get_match`. + /// + /// # Example + /// + /// When using the PikeVM (for example), the lightest weight way of + /// detecting which pattern matched is to create capturing groups that only + /// track the ID of the pattern that match (if any): + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::captures::Captures, + /// PatternID, + /// }; + /// + /// let re = PikeVM::new_many(&[r"[a-z]+", r"[A-Z]+"])?; + /// let mut cache = re.create_cache(); + /// let mut caps = Captures::empty(re.get_nfa().group_info().clone()); + /// + /// re.captures(&mut cache, "ABC", &mut caps); + /// assert_eq!(Some(PatternID::must(1)), caps.pattern()); + /// // Recall that offsets are only available when using a non-empty + /// // Captures value. So even though a match occurred, this returns None! + /// assert_eq!(None, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn pattern(&self) -> Option<PatternID> { + self.pid + } + + /// Returns the pattern ID and the span of the match, if one occurred. + /// + /// This always returns `None` when `Captures` was created with + /// [`Captures::empty`], even if a match was found. + /// + /// If this routine returns a non-`None` value, then `is_match` is + /// guaranteed to return `true` and `pattern` is also guaranteed to return + /// a non-`None` value. + /// + /// # Example + /// + /// This example shows how to get the full match from a search: + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match}; + /// + /// let re = PikeVM::new_many(&[r"[a-z]+", r"[A-Z]+"])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "ABC", &mut caps); + /// assert_eq!(Some(Match::must(1, 0..3)), caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn get_match(&self) -> Option<Match> { + Some(Match::new(self.pattern()?, self.get_group(0)?)) + } + + /// Returns the span of a capturing group match corresponding to the group + /// index given, only if both the overall pattern matched and the capturing + /// group participated in that match. + /// + /// This returns `None` if `index` is invalid. `index` is valid if and only + /// if it's less than [`Captures::group_len`] for the matching pattern. + /// + /// This always returns `None` when `Captures` was created with + /// [`Captures::empty`], even if a match was found. This also always + /// returns `None` for any `index > 0` when `Captures` was created with + /// [`Captures::matches`]. + /// + /// If this routine returns a non-`None` value, then `is_match` is + /// guaranteed to return `true`, `pattern` is guaranteed to return a + /// non-`None` value and `get_match` is guaranteed to return a non-`None` + /// value. + /// + /// By convention, the 0th capture group will always return the same + /// span as the span returned by `get_match`. This is because the 0th + /// capture group always corresponds to the entirety of the pattern's + /// match. (It is similarly always unnamed because it is implicit.) This + /// isn't necessarily true of all regex engines. For example, one can + /// hand-compile a [`thompson::NFA`](crate::nfa::thompson::NFA) via a + /// [`thompson::Builder`](crate::nfa::thompson::Builder), which isn't + /// technically forced to make the 0th capturing group always correspond to + /// the entire match. + /// + /// # Example + /// + /// This example shows how to get the capturing groups, by index, from a + /// match: + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span, Match}; + /// + /// let re = PikeVM::new(r"^(?P<first>\pL+)\s+(?P<last>\pL+)$")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "Bruce Springsteen", &mut caps); + /// assert_eq!(Some(Match::must(0, 0..17)), caps.get_match()); + /// assert_eq!(Some(Span::from(0..5)), caps.get_group(1)); + /// assert_eq!(Some(Span::from(6..17)), caps.get_group(2)); + /// // Looking for a non-existent capturing group will return None: + /// assert_eq!(None, caps.get_group(3)); + /// # // literals are too big for 32-bit usize: #1039 + /// # #[cfg(target_pointer_width = "64")] + /// assert_eq!(None, caps.get_group(9944060567225171988)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn get_group(&self, index: usize) -> Option<Span> { + let pid = self.pattern()?; + // There's a little bit of work needed to map captures to slots in the + // fully general case. But in the overwhelming common case of a single + // pattern, we can just do some simple arithmetic. + let (slot_start, slot_end) = if self.group_info().pattern_len() == 1 { + (index.checked_mul(2)?, index.checked_mul(2)?.checked_add(1)?) + } else { + self.group_info().slots(pid, index)? + }; + let start = self.slots.get(slot_start).copied()??; + let end = self.slots.get(slot_end).copied()??; + Some(Span { start: start.get(), end: end.get() }) + } + + /// Returns the span of a capturing group match corresponding to the group + /// name given, only if both the overall pattern matched and the capturing + /// group participated in that match. + /// + /// This returns `None` if `name` does not correspond to a valid capturing + /// group for the pattern that matched. + /// + /// This always returns `None` when `Captures` was created with + /// [`Captures::empty`], even if a match was found. This also always + /// returns `None` for any `index > 0` when `Captures` was created with + /// [`Captures::matches`]. + /// + /// If this routine returns a non-`None` value, then `is_match` is + /// guaranteed to return `true`, `pattern` is guaranteed to return a + /// non-`None` value and `get_match` is guaranteed to return a non-`None` + /// value. + /// + /// # Example + /// + /// This example shows how to get the capturing groups, by name, from a + /// match: + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span, Match}; + /// + /// let re = PikeVM::new(r"^(?P<first>\pL+)\s+(?P<last>\pL+)$")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "Bruce Springsteen", &mut caps); + /// assert_eq!(Some(Match::must(0, 0..17)), caps.get_match()); + /// assert_eq!(Some(Span::from(0..5)), caps.get_group_by_name("first")); + /// assert_eq!(Some(Span::from(6..17)), caps.get_group_by_name("last")); + /// // Looking for a non-existent capturing group will return None: + /// assert_eq!(None, caps.get_group_by_name("middle")); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn get_group_by_name(&self, name: &str) -> Option<Span> { + let index = self.group_info().to_index(self.pattern()?, name)?; + self.get_group(index) + } + + /// Returns an iterator of possible spans for every capturing group in the + /// matching pattern. + /// + /// If this `Captures` value does not correspond to a match, then the + /// iterator returned yields no elements. + /// + /// Note that the iterator returned yields elements of type `Option<Span>`. + /// A span is present if and only if it corresponds to a capturing group + /// that participated in a match. + /// + /// # Example + /// + /// This example shows how to collect all capturing groups: + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span}; + /// + /// let re = PikeVM::new( + /// // Matches first/last names, with an optional middle name. + /// r"^(?P<first>\pL+)\s+(?:(?P<middle>\pL+)\s+)?(?P<last>\pL+)$", + /// )?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "Harry James Potter", &mut caps); + /// assert!(caps.is_match()); + /// let groups: Vec<Option<Span>> = caps.iter().collect(); + /// assert_eq!(groups, vec![ + /// Some(Span::from(0..18)), + /// Some(Span::from(0..5)), + /// Some(Span::from(6..11)), + /// Some(Span::from(12..18)), + /// ]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// This example uses the same regex as the previous example, but with a + /// haystack that omits the middle name. This results in a capturing group + /// that is present in the elements yielded by the iterator but without a + /// match: + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Span}; + /// + /// let re = PikeVM::new( + /// // Matches first/last names, with an optional middle name. + /// r"^(?P<first>\pL+)\s+(?:(?P<middle>\pL+)\s+)?(?P<last>\pL+)$", + /// )?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "Harry Potter", &mut caps); + /// assert!(caps.is_match()); + /// let groups: Vec<Option<Span>> = caps.iter().collect(); + /// assert_eq!(groups, vec![ + /// Some(Span::from(0..12)), + /// Some(Span::from(0..5)), + /// None, + /// Some(Span::from(6..12)), + /// ]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn iter(&self) -> CapturesPatternIter<'_> { + let names = self + .pattern() + .map_or(GroupInfoPatternNames::empty().enumerate(), |pid| { + self.group_info().pattern_names(pid).enumerate() + }); + CapturesPatternIter { caps: self, names } + } + + /// Return the total number of capturing groups for the matching pattern. + /// + /// If this `Captures` value does not correspond to a match, then this + /// always returns `0`. + /// + /// This always returns the same number of elements yielded by + /// [`Captures::iter`]. That is, the number includes capturing groups even + /// if they don't participate in the match. + /// + /// # Example + /// + /// This example shows how to count the total number of capturing groups + /// associated with a pattern. Notice that it includes groups that did not + /// participate in a match (just like `Captures::iter` does). + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::new( + /// // Matches first/last names, with an optional middle name. + /// r"^(?P<first>\pL+)\s+(?:(?P<middle>\pL+)\s+)?(?P<last>\pL+)$", + /// )?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "Harry Potter", &mut caps); + /// assert_eq!(4, caps.group_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn group_len(&self) -> usize { + let pid = match self.pattern() { + None => return 0, + Some(pid) => pid, + }; + self.group_info().group_len(pid) + } + + /// Returns a reference to the underlying group info on which these + /// captures are based. + /// + /// The difference between `GroupInfo` and `Captures` is that the former + /// defines the structure of capturing groups where as the latter is what + /// stores the actual match information. So where as `Captures` only gives + /// you access to the current match, `GroupInfo` lets you query any + /// information about all capturing groups, even ones for patterns that + /// weren't involved in a match. + /// + /// Note that a `GroupInfo` uses reference counting internally, so it may + /// be cloned cheaply. + /// + /// # Example + /// + /// This example shows how to get all capturing group names from the + /// underlying `GroupInfo`. Notice that we don't even need to run a + /// search. + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, PatternID}; + /// + /// let re = PikeVM::new_many(&[ + /// r"(?P<foo>a)", + /// r"(a)(b)", + /// r"ab", + /// r"(?P<bar>a)(?P<quux>a)", + /// r"(?P<foo>z)", + /// ])?; + /// let caps = re.create_captures(); + /// + /// let expected = vec![ + /// (PatternID::must(0), 0, None), + /// (PatternID::must(0), 1, Some("foo")), + /// (PatternID::must(1), 0, None), + /// (PatternID::must(1), 1, None), + /// (PatternID::must(1), 2, None), + /// (PatternID::must(2), 0, None), + /// (PatternID::must(3), 0, None), + /// (PatternID::must(3), 1, Some("bar")), + /// (PatternID::must(3), 2, Some("quux")), + /// (PatternID::must(4), 0, None), + /// (PatternID::must(4), 1, Some("foo")), + /// ]; + /// // We could also just use 're.get_nfa().group_info()'. + /// let got: Vec<(PatternID, usize, Option<&str>)> = + /// caps.group_info().all_names().collect(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn group_info(&self) -> &GroupInfo { + &self.group_info + } + + /// Interpolates the capture references in `replacement` with the + /// corresponding substrings in `haystack` matched by each reference. The + /// interpolated string is returned. + /// + /// See the [`interpolate` module](interpolate) for documentation on the + /// format of the replacement string. + /// + /// # Example + /// + /// This example shows how to use interpolation, and also shows how it + /// can work with multi-pattern regexes. + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, PatternID}; + /// + /// let re = PikeVM::new_many(&[ + /// r"(?<day>[0-9]{2})-(?<month>[0-9]{2})-(?<year>[0-9]{4})", + /// r"(?<year>[0-9]{4})-(?<month>[0-9]{2})-(?<day>[0-9]{2})", + /// ])?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// let replacement = "year=$year, month=$month, day=$day"; + /// + /// // This matches the first pattern. + /// let hay = "On 14-03-2010, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// let result = caps.interpolate_string(hay, replacement); + /// assert_eq!("year=2010, month=03, day=14", result); + /// + /// // And this matches the second pattern. + /// let hay = "On 2010-03-14, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// let result = caps.interpolate_string(hay, replacement); + /// assert_eq!("year=2010, month=03, day=14", result); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn interpolate_string( + &self, + haystack: &str, + replacement: &str, + ) -> String { + let mut dst = String::new(); + self.interpolate_string_into(haystack, replacement, &mut dst); + dst + } + + /// Interpolates the capture references in `replacement` with the + /// corresponding substrings in `haystack` matched by each reference. The + /// interpolated string is written to `dst`. + /// + /// See the [`interpolate` module](interpolate) for documentation on the + /// format of the replacement string. + /// + /// # Example + /// + /// This example shows how to use interpolation, and also shows how it + /// can work with multi-pattern regexes. + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, PatternID}; + /// + /// let re = PikeVM::new_many(&[ + /// r"(?<day>[0-9]{2})-(?<month>[0-9]{2})-(?<year>[0-9]{4})", + /// r"(?<year>[0-9]{4})-(?<month>[0-9]{2})-(?<day>[0-9]{2})", + /// ])?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// let replacement = "year=$year, month=$month, day=$day"; + /// + /// // This matches the first pattern. + /// let hay = "On 14-03-2010, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// let mut dst = String::new(); + /// caps.interpolate_string_into(hay, replacement, &mut dst); + /// assert_eq!("year=2010, month=03, day=14", dst); + /// + /// // And this matches the second pattern. + /// let hay = "On 2010-03-14, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// let mut dst = String::new(); + /// caps.interpolate_string_into(hay, replacement, &mut dst); + /// assert_eq!("year=2010, month=03, day=14", dst); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn interpolate_string_into( + &self, + haystack: &str, + replacement: &str, + dst: &mut String, + ) { + interpolate::string( + replacement, + |index, dst| { + let span = match self.get_group(index) { + None => return, + Some(span) => span, + }; + dst.push_str(&haystack[span]); + }, + |name| self.group_info().to_index(self.pattern()?, name), + dst, + ); + } + + /// Interpolates the capture references in `replacement` with the + /// corresponding substrings in `haystack` matched by each reference. The + /// interpolated byte string is returned. + /// + /// See the [`interpolate` module](interpolate) for documentation on the + /// format of the replacement string. + /// + /// # Example + /// + /// This example shows how to use interpolation, and also shows how it + /// can work with multi-pattern regexes. + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, PatternID}; + /// + /// let re = PikeVM::new_many(&[ + /// r"(?<day>[0-9]{2})-(?<month>[0-9]{2})-(?<year>[0-9]{4})", + /// r"(?<year>[0-9]{4})-(?<month>[0-9]{2})-(?<day>[0-9]{2})", + /// ])?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// let replacement = b"year=$year, month=$month, day=$day"; + /// + /// // This matches the first pattern. + /// let hay = b"On 14-03-2010, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// let result = caps.interpolate_bytes(hay, replacement); + /// assert_eq!(&b"year=2010, month=03, day=14"[..], result); + /// + /// // And this matches the second pattern. + /// let hay = b"On 2010-03-14, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// let result = caps.interpolate_bytes(hay, replacement); + /// assert_eq!(&b"year=2010, month=03, day=14"[..], result); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn interpolate_bytes( + &self, + haystack: &[u8], + replacement: &[u8], + ) -> Vec<u8> { + let mut dst = vec![]; + self.interpolate_bytes_into(haystack, replacement, &mut dst); + dst + } + + /// Interpolates the capture references in `replacement` with the + /// corresponding substrings in `haystack` matched by each reference. The + /// interpolated byte string is written to `dst`. + /// + /// See the [`interpolate` module](interpolate) for documentation on the + /// format of the replacement string. + /// + /// # Example + /// + /// This example shows how to use interpolation, and also shows how it + /// can work with multi-pattern regexes. + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, PatternID}; + /// + /// let re = PikeVM::new_many(&[ + /// r"(?<day>[0-9]{2})-(?<month>[0-9]{2})-(?<year>[0-9]{4})", + /// r"(?<year>[0-9]{4})-(?<month>[0-9]{2})-(?<day>[0-9]{2})", + /// ])?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// let replacement = b"year=$year, month=$month, day=$day"; + /// + /// // This matches the first pattern. + /// let hay = b"On 14-03-2010, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// let mut dst = vec![]; + /// caps.interpolate_bytes_into(hay, replacement, &mut dst); + /// assert_eq!(&b"year=2010, month=03, day=14"[..], dst); + /// + /// // And this matches the second pattern. + /// let hay = b"On 2010-03-14, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// let mut dst = vec![]; + /// caps.interpolate_bytes_into(hay, replacement, &mut dst); + /// assert_eq!(&b"year=2010, month=03, day=14"[..], dst); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn interpolate_bytes_into( + &self, + haystack: &[u8], + replacement: &[u8], + dst: &mut Vec<u8>, + ) { + interpolate::bytes( + replacement, + |index, dst| { + let span = match self.get_group(index) { + None => return, + Some(span) => span, + }; + dst.extend_from_slice(&haystack[span]); + }, + |name| self.group_info().to_index(self.pattern()?, name), + dst, + ); + } + + /// This is a convenience routine for extracting the substrings + /// corresponding to matching capture groups in the given `haystack`. The + /// `haystack` should be the same substring used to find the match spans in + /// this `Captures` value. + /// + /// This is identical to [`Captures::extract_bytes`], except it works with + /// `&str` instead of `&[u8]`. + /// + /// # Panics + /// + /// This panics if the number of explicit matching groups in this + /// `Captures` value is less than `N`. This also panics if this `Captures` + /// value does not correspond to a match. + /// + /// Note that this does *not* panic if the number of explicit matching + /// groups is bigger than `N`. In that case, only the first `N` matching + /// groups are extracted. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::new(r"([0-9]{4})-([0-9]{2})-([0-9]{2})")?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// let hay = "On 2010-03-14, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// assert!(caps.is_match()); + /// let (full, [year, month, day]) = caps.extract(hay); + /// assert_eq!("2010-03-14", full); + /// assert_eq!("2010", year); + /// assert_eq!("03", month); + /// assert_eq!("14", day); + /// + /// // We can also ask for fewer than all capture groups. + /// let (full, [year]) = caps.extract(hay); + /// assert_eq!("2010-03-14", full); + /// assert_eq!("2010", year); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn extract<'h, const N: usize>( + &self, + haystack: &'h str, + ) -> (&'h str, [&'h str; N]) { + let mut matched = self.iter().flatten(); + let whole_match = &haystack[matched.next().expect("a match")]; + let group_matches = [0; N].map(|_| { + let sp = matched.next().expect("too few matching groups"); + &haystack[sp] + }); + (whole_match, group_matches) + } + + /// This is a convenience routine for extracting the substrings + /// corresponding to matching capture groups in the given `haystack`. The + /// `haystack` should be the same substring used to find the match spans in + /// this `Captures` value. + /// + /// This is identical to [`Captures::extract`], except it works with + /// `&[u8]` instead of `&str`. + /// + /// # Panics + /// + /// This panics if the number of explicit matching groups in this + /// `Captures` value is less than `N`. This also panics if this `Captures` + /// value does not correspond to a match. + /// + /// Note that this does *not* panic if the number of explicit matching + /// groups is bigger than `N`. In that case, only the first `N` matching + /// groups are extracted. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::new(r"([0-9]{4})-([0-9]{2})-([0-9]{2})")?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// let hay = b"On 2010-03-14, I became a Tennessee lamb."; + /// re.captures(&mut cache, hay, &mut caps); + /// assert!(caps.is_match()); + /// let (full, [year, month, day]) = caps.extract_bytes(hay); + /// assert_eq!(b"2010-03-14", full); + /// assert_eq!(b"2010", year); + /// assert_eq!(b"03", month); + /// assert_eq!(b"14", day); + /// + /// // We can also ask for fewer than all capture groups. + /// let (full, [year]) = caps.extract_bytes(hay); + /// assert_eq!(b"2010-03-14", full); + /// assert_eq!(b"2010", year); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn extract_bytes<'h, const N: usize>( + &self, + haystack: &'h [u8], + ) -> (&'h [u8], [&'h [u8]; N]) { + let mut matched = self.iter().flatten(); + let whole_match = &haystack[matched.next().expect("a match")]; + let group_matches = [0; N].map(|_| { + let sp = matched.next().expect("too few matching groups"); + &haystack[sp] + }); + (whole_match, group_matches) + } +} + +/// Lower level "slot" oriented APIs. One does not typically need to use these +/// when executing a search. They are instead mostly intended for folks that +/// are writing their own regex engine while reusing this `Captures` type. +impl Captures { + /// Clear this `Captures` value. + /// + /// After clearing, all slots inside this `Captures` value will be set to + /// `None`. Similarly, any pattern ID that it was previously associated + /// with (for a match) is erased. + /// + /// It is not usually necessary to call this routine. Namely, a `Captures` + /// value only provides high level access to the capturing groups of the + /// pattern that matched, and only low level access to individual slots. + /// Thus, even if slots corresponding to groups that aren't associated + /// with the matching pattern are set, then it won't impact the higher + /// level APIs. Namely, higher level APIs like [`Captures::get_group`] will + /// return `None` if no pattern ID is present, even if there are spans set + /// in the underlying slots. + /// + /// Thus, to "clear" a `Captures` value of a match, it is usually only + /// necessary to call [`Captures::set_pattern`] with `None`. + /// + /// # Example + /// + /// This example shows what happens when a `Captures` value is cleared. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::new(r"^(?P<first>\pL+)\s+(?P<last>\pL+)$")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "Bruce Springsteen", &mut caps); + /// assert!(caps.is_match()); + /// let slots: Vec<Option<usize>> = + /// caps.slots().iter().map(|s| s.map(|x| x.get())).collect(); + /// // Note that the following ordering is considered an API guarantee. + /// assert_eq!(slots, vec![ + /// Some(0), + /// Some(17), + /// Some(0), + /// Some(5), + /// Some(6), + /// Some(17), + /// ]); + /// + /// // Now clear the slots. Everything is gone and it is no longer a match. + /// caps.clear(); + /// assert!(!caps.is_match()); + /// let slots: Vec<Option<usize>> = + /// caps.slots().iter().map(|s| s.map(|x| x.get())).collect(); + /// assert_eq!(slots, vec![ + /// None, + /// None, + /// None, + /// None, + /// None, + /// None, + /// ]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn clear(&mut self) { + self.pid = None; + for slot in self.slots.iter_mut() { + *slot = None; + } + } + + /// Set the pattern on this `Captures` value. + /// + /// When the pattern ID is `None`, then this `Captures` value does not + /// correspond to a match (`is_match` will return `false`). Otherwise, it + /// corresponds to a match. + /// + /// This is useful in search implementations where you might want to + /// initially call `set_pattern(None)` in order to avoid the cost of + /// calling `clear()` if it turns out to not be necessary. + /// + /// # Example + /// + /// This example shows that `set_pattern` merely overwrites the pattern ID. + /// It does not actually change the underlying slot values. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::nfa::thompson::pikevm::PikeVM; + /// + /// let re = PikeVM::new(r"^(?P<first>\pL+)\s+(?P<last>\pL+)$")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "Bruce Springsteen", &mut caps); + /// assert!(caps.is_match()); + /// assert!(caps.pattern().is_some()); + /// let slots: Vec<Option<usize>> = + /// caps.slots().iter().map(|s| s.map(|x| x.get())).collect(); + /// // Note that the following ordering is considered an API guarantee. + /// assert_eq!(slots, vec![ + /// Some(0), + /// Some(17), + /// Some(0), + /// Some(5), + /// Some(6), + /// Some(17), + /// ]); + /// + /// // Now set the pattern to None. Note that the slot values remain. + /// caps.set_pattern(None); + /// assert!(!caps.is_match()); + /// assert!(!caps.pattern().is_some()); + /// let slots: Vec<Option<usize>> = + /// caps.slots().iter().map(|s| s.map(|x| x.get())).collect(); + /// // Note that the following ordering is considered an API guarantee. + /// assert_eq!(slots, vec![ + /// Some(0), + /// Some(17), + /// Some(0), + /// Some(5), + /// Some(6), + /// Some(17), + /// ]); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn set_pattern(&mut self, pid: Option<PatternID>) { + self.pid = pid; + } + + /// Returns the underlying slots, where each slot stores a single offset. + /// + /// Every matching capturing group generally corresponds to two slots: one + /// slot for the starting position and another for the ending position. + /// Typically, either both are present or neither are. (The weasel word + /// "typically" is used here because it really depends on the regex engine + /// implementation. Every sensible regex engine likely adheres to this + /// invariant, and every regex engine in this crate is sensible.) + /// + /// Generally speaking, callers should prefer to use higher level routines + /// like [`Captures::get_match`] or [`Captures::get_group`]. + /// + /// An important note here is that a regex engine may not reset all of the + /// slots to `None` values when no match occurs, or even when a match of + /// a different pattern occurs. But this depends on how the regex engine + /// implementation deals with slots. + /// + /// # Example + /// + /// This example shows how to get the underlying slots from a regex match. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::primitives::{PatternID, NonMaxUsize}, + /// }; + /// + /// let re = PikeVM::new_many(&[ + /// r"[a-z]+", + /// r"[0-9]+", + /// ])?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// re.captures(&mut cache, "123", &mut caps); + /// assert_eq!(Some(PatternID::must(1)), caps.pattern()); + /// // Note that the only guarantee we have here is that slots 2 and 3 + /// // are set to correct values. The contents of the first two slots are + /// // unspecified since the 0th pattern did not match. + /// let expected = &[ + /// None, + /// None, + /// NonMaxUsize::new(0), + /// NonMaxUsize::new(3), + /// ]; + /// assert_eq!(expected, caps.slots()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn slots(&self) -> &[Option<NonMaxUsize>] { + &self.slots + } + + /// Returns the underlying slots as a mutable slice, where each slot stores + /// a single offset. + /// + /// This tends to be most useful for regex engine implementations for + /// writing offsets for matching capturing groups to slots. + /// + /// See [`Captures::slots`] for more information about slots. + #[inline] + pub fn slots_mut(&mut self) -> &mut [Option<NonMaxUsize>] { + &mut self.slots + } +} + +impl core::fmt::Debug for Captures { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut dstruct = f.debug_struct("Captures"); + dstruct.field("pid", &self.pid); + if let Some(pid) = self.pid { + dstruct.field("spans", &CapturesDebugMap { pid, caps: self }); + } + dstruct.finish() + } +} + +/// A little helper type to provide a nice map-like debug representation for +/// our capturing group spans. +struct CapturesDebugMap<'a> { + pid: PatternID, + caps: &'a Captures, +} + +impl<'a> core::fmt::Debug for CapturesDebugMap<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + struct Key<'a>(usize, Option<&'a str>); + + impl<'a> core::fmt::Debug for Key<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{}", self.0)?; + if let Some(name) = self.1 { + write!(f, "/{name:?}")?; + } + Ok(()) + } + } + + let mut map = f.debug_map(); + let names = self.caps.group_info().pattern_names(self.pid); + for (group_index, maybe_name) in names.enumerate() { + let key = Key(group_index, maybe_name); + match self.caps.get_group(group_index) { + None => map.entry(&key, &None::<()>), + Some(span) => map.entry(&key, &span), + }; + } + map.finish() + } +} + +/// An iterator over all capturing groups in a `Captures` value. +/// +/// This iterator includes capturing groups that did not participate in a +/// match. See the [`Captures::iter`] method documentation for more details +/// and examples. +/// +/// The lifetime parameter `'a` refers to the lifetime of the underlying +/// `Captures` value. +#[derive(Clone, Debug)] +pub struct CapturesPatternIter<'a> { + caps: &'a Captures, + names: core::iter::Enumerate<GroupInfoPatternNames<'a>>, +} + +impl<'a> Iterator for CapturesPatternIter<'a> { + type Item = Option<Span>; + + fn next(&mut self) -> Option<Option<Span>> { + let (group_index, _) = self.names.next()?; + Some(self.caps.get_group(group_index)) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + self.names.size_hint() + } + + fn count(self) -> usize { + self.names.count() + } +} + +impl<'a> ExactSizeIterator for CapturesPatternIter<'a> {} +impl<'a> core::iter::FusedIterator for CapturesPatternIter<'a> {} + +/// Represents information about capturing groups in a compiled regex. +/// +/// The information encapsulated by this type consists of the following. For +/// each pattern: +/// +/// * A map from every capture group name to its corresponding capture group +/// index. +/// * A map from every capture group index to its corresponding capture group +/// name. +/// * A map from capture group index to its corresponding slot index. A slot +/// refers to one half of a capturing group. That is, a capture slot is either +/// the start or end of a capturing group. A slot is usually the mechanism +/// by which a regex engine records offsets for each capturing group during a +/// search. +/// +/// A `GroupInfo` uses reference counting internally and is thus cheap to +/// clone. +/// +/// # Mapping from capture groups to slots +/// +/// One of the main responsibilities of a `GroupInfo` is to build a mapping +/// from `(PatternID, u32)` (where the `u32` is a capture index) to something +/// called a "slot." As mentioned above, a slot refers to one half of a +/// capturing group. Both combined provide the start and end offsets of +/// a capturing group that participated in a match. +/// +/// **The mapping between group indices and slots is an API guarantee.** That +/// is, the mapping won't change within a semver compatible release. +/// +/// Slots exist primarily because this is a convenient mechanism by which +/// regex engines report group offsets at search time. For example, the +/// [`nfa::thompson::State::Capture`](crate::nfa::thompson::State::Capture) +/// NFA state includes the slot index. When a regex engine transitions through +/// this state, it will likely use the slot index to write the current haystack +/// offset to some region of memory. When a match is found, those slots are +/// then reported to the caller, typically via a convenient abstraction like a +/// [`Captures`] value. +/// +/// Because this crate provides first class support for multi-pattern regexes, +/// and because of some performance related reasons, the mapping between +/// capturing groups and slots is a little complex. However, in the case of a +/// single pattern, the mapping can be described very simply: for all capture +/// group indices `i`, its corresponding slots are at `i * 2` and `i * 2 + 1`. +/// Notice that the pattern ID isn't involved at all here, because it only +/// applies to a single-pattern regex, it is therefore always `0`. +/// +/// In the multi-pattern case, the mapping is a bit more complicated. To talk +/// about it, we must define what we mean by "implicit" vs "explicit" +/// capturing groups: +/// +/// * An **implicit** capturing group refers to the capturing group that is +/// present for every pattern automatically, and corresponds to the overall +/// match of a pattern. Every pattern has precisely one implicit capturing +/// group. It is always unnamed and it always corresponds to the capture group +/// index `0`. +/// * An **explicit** capturing group refers to any capturing group that +/// appears in the concrete syntax of the pattern. (Or, if an NFA was hand +/// built without any concrete syntax, it refers to any capturing group with an +/// index greater than `0`.) +/// +/// Some examples: +/// +/// * `\w+` has one implicit capturing group and zero explicit capturing +/// groups. +/// * `(\w+)` has one implicit group and one explicit group. +/// * `foo(\d+)(?:\pL+)(\d+)` has one implicit group and two explicit groups. +/// +/// Turning back to the slot mapping, we can now state it as follows: +/// +/// * Given a pattern ID `pid`, the slots for its implicit group are always +/// at `pid * 2` and `pid * 2 + 1`. +/// * Given a pattern ID `0`, the slots for its explicit groups start +/// at `group_info.pattern_len() * 2`. +/// * Given a pattern ID `pid > 0`, the slots for its explicit groups start +/// immediately following where the slots for the explicit groups of `pid - 1` +/// end. +/// +/// In particular, while there is a concrete formula one can use to determine +/// where the slots for the implicit group of any pattern are, there is no +/// general formula for determining where the slots for explicit capturing +/// groups are. This is because each pattern can contain a different number +/// of groups. +/// +/// The intended way of getting the slots for a particular capturing group +/// (whether implicit or explicit) is via the [`GroupInfo::slot`] or +/// [`GroupInfo::slots`] method. +/// +/// See below for a concrete example of how capturing groups get mapped to +/// slots. +/// +/// # Example +/// +/// This example shows how to build a new `GroupInfo` and query it for +/// information. +/// +/// ``` +/// use regex_automata::util::{captures::GroupInfo, primitives::PatternID}; +/// +/// let info = GroupInfo::new(vec![ +/// vec![None, Some("foo")], +/// vec![None], +/// vec![None, None, None, Some("bar"), None], +/// vec![None, None, Some("foo")], +/// ])?; +/// // The number of patterns being tracked. +/// assert_eq!(4, info.pattern_len()); +/// // We can query the number of groups for any pattern. +/// assert_eq!(2, info.group_len(PatternID::must(0))); +/// assert_eq!(1, info.group_len(PatternID::must(1))); +/// assert_eq!(5, info.group_len(PatternID::must(2))); +/// assert_eq!(3, info.group_len(PatternID::must(3))); +/// // An invalid pattern always has zero groups. +/// assert_eq!(0, info.group_len(PatternID::must(999))); +/// // 2 slots per group +/// assert_eq!(22, info.slot_len()); +/// +/// // We can map a group index for a particular pattern to its name, if +/// // one exists. +/// assert_eq!(Some("foo"), info.to_name(PatternID::must(3), 2)); +/// assert_eq!(None, info.to_name(PatternID::must(2), 4)); +/// // Or map a name to its group index. +/// assert_eq!(Some(1), info.to_index(PatternID::must(0), "foo")); +/// assert_eq!(Some(2), info.to_index(PatternID::must(3), "foo")); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # Example: mapping from capture groups to slots +/// +/// This example shows the specific mapping from capture group indices for +/// each pattern to their corresponding slots. The slot values shown in this +/// example are considered an API guarantee. +/// +/// ``` +/// use regex_automata::util::{captures::GroupInfo, primitives::PatternID}; +/// +/// let info = GroupInfo::new(vec![ +/// vec![None, Some("foo")], +/// vec![None], +/// vec![None, None, None, Some("bar"), None], +/// vec![None, None, Some("foo")], +/// ])?; +/// +/// // We first show the slots for each pattern's implicit group. +/// assert_eq!(Some((0, 1)), info.slots(PatternID::must(0), 0)); +/// assert_eq!(Some((2, 3)), info.slots(PatternID::must(1), 0)); +/// assert_eq!(Some((4, 5)), info.slots(PatternID::must(2), 0)); +/// assert_eq!(Some((6, 7)), info.slots(PatternID::must(3), 0)); +/// +/// // And now we show the slots for each pattern's explicit group. +/// assert_eq!(Some((8, 9)), info.slots(PatternID::must(0), 1)); +/// assert_eq!(Some((10, 11)), info.slots(PatternID::must(2), 1)); +/// assert_eq!(Some((12, 13)), info.slots(PatternID::must(2), 2)); +/// assert_eq!(Some((14, 15)), info.slots(PatternID::must(2), 3)); +/// assert_eq!(Some((16, 17)), info.slots(PatternID::must(2), 4)); +/// assert_eq!(Some((18, 19)), info.slots(PatternID::must(3), 1)); +/// assert_eq!(Some((20, 21)), info.slots(PatternID::must(3), 2)); +/// +/// // Asking for the slots for an invalid pattern ID or even for an invalid +/// // group index for a specific pattern will return None. So for example, +/// // you're guaranteed to not get the slots for a different pattern than the +/// // one requested. +/// assert_eq!(None, info.slots(PatternID::must(5), 0)); +/// assert_eq!(None, info.slots(PatternID::must(1), 1)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug, Default)] +pub struct GroupInfo(Arc<GroupInfoInner>); + +impl GroupInfo { + /// Creates a new group info from a sequence of patterns, where each + /// sequence of patterns yields a sequence of possible group names. The + /// index of each pattern in the sequence corresponds to its `PatternID`, + /// and the index of each group in each pattern's sequence corresponds to + /// its corresponding group index. + /// + /// While this constructor is very generic and therefore perhaps hard to + /// chew on, an example of a valid concrete type that can be passed to + /// this constructor is `Vec<Vec<Option<String>>>`. The outer `Vec` + /// corresponds to the patterns, i.e., one `Vec<Option<String>>` per + /// pattern. The inner `Vec` corresponds to the capturing groups for + /// each pattern. The `Option<String>` corresponds to the name of the + /// capturing group, if present. + /// + /// It is legal to pass an empty iterator to this constructor. It will + /// return an empty group info with zero slots. An empty group info is + /// useful for cases where you have no patterns or for cases where slots + /// aren't being used at all (e.g., for most DFAs in this crate). + /// + /// # Errors + /// + /// This constructor returns an error if the given capturing groups are + /// invalid in some way. Those reasons include, but are not necessarily + /// limited to: + /// + /// * Too many patterns (i.e., `PatternID` would overflow). + /// * Too many capturing groups (e.g., `u32` would overflow). + /// * A pattern is given that has no capturing groups. (All patterns must + /// have at least an implicit capturing group at index `0`.) + /// * The capturing group at index `0` has a name. It must be unnamed. + /// * There are duplicate capturing group names within the same pattern. + /// (Multiple capturing groups with the same name may exist, but they + /// must be in different patterns.) + /// + /// An example below shows how to trigger some of the above error + /// conditions. + /// + /// # Example + /// + /// This example shows how to build a new `GroupInfo` and query it for + /// information. + /// + /// ``` + /// use regex_automata::util::captures::GroupInfo; + /// + /// let info = GroupInfo::new(vec![ + /// vec![None, Some("foo")], + /// vec![None], + /// vec![None, None, None, Some("bar"), None], + /// vec![None, None, Some("foo")], + /// ])?; + /// // The number of patterns being tracked. + /// assert_eq!(4, info.pattern_len()); + /// // 2 slots per group + /// assert_eq!(22, info.slot_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: empty `GroupInfo` + /// + /// This example shows how to build a new `GroupInfo` and query it for + /// information. + /// + /// ``` + /// use regex_automata::util::captures::GroupInfo; + /// + /// let info = GroupInfo::empty(); + /// // Everything is zero. + /// assert_eq!(0, info.pattern_len()); + /// assert_eq!(0, info.slot_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// # Example: error conditions + /// + /// This example shows how to provoke some of the ways in which building + /// a `GroupInfo` can fail. + /// + /// ``` + /// use regex_automata::util::captures::GroupInfo; + /// + /// // Either the group info is empty, or all patterns must have at least + /// // one capturing group. + /// assert!(GroupInfo::new(vec![ + /// vec![None, Some("a")], // ok + /// vec![None], // ok + /// vec![], // not ok + /// ]).is_err()); + /// // Note that building an empty group info is OK. + /// assert!(GroupInfo::new(Vec::<Vec<Option<String>>>::new()).is_ok()); + /// + /// // The first group in each pattern must correspond to an implicit + /// // anonymous group. i.e., One that is not named. By convention, this + /// // group corresponds to the overall match of a regex. Every other group + /// // in a pattern is explicit and optional. + /// assert!(GroupInfo::new(vec![vec![Some("foo")]]).is_err()); + /// + /// // There must not be duplicate group names within the same pattern. + /// assert!(GroupInfo::new(vec![ + /// vec![None, Some("foo"), Some("foo")], + /// ]).is_err()); + /// // But duplicate names across distinct patterns is OK. + /// assert!(GroupInfo::new(vec![ + /// vec![None, Some("foo")], + /// vec![None, Some("foo")], + /// ]).is_ok()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// There are other ways for building a `GroupInfo` to fail but are + /// difficult to show. For example, if the number of patterns given would + /// overflow `PatternID`. + pub fn new<P, G, N>(pattern_groups: P) -> Result<GroupInfo, GroupInfoError> + where + P: IntoIterator<Item = G>, + G: IntoIterator<Item = Option<N>>, + N: AsRef<str>, + { + let mut group_info = GroupInfoInner { + slot_ranges: vec![], + name_to_index: vec![], + index_to_name: vec![], + memory_extra: 0, + }; + for (pattern_index, groups) in pattern_groups.into_iter().enumerate() { + // If we can't convert the pattern index to an ID, then the caller + // tried to build capture info for too many patterns. + let pid = PatternID::new(pattern_index) + .map_err(GroupInfoError::too_many_patterns)?; + + let mut groups_iter = groups.into_iter().enumerate(); + match groups_iter.next() { + None => return Err(GroupInfoError::missing_groups(pid)), + Some((_, Some(_))) => { + return Err(GroupInfoError::first_must_be_unnamed(pid)) + } + Some((_, None)) => {} + } + group_info.add_first_group(pid); + // Now iterate over the rest, which correspond to all of the + // (conventionally) explicit capture groups in a regex pattern. + for (group_index, maybe_name) in groups_iter { + // Just like for patterns, if the group index can't be + // converted to a "small" index, then the caller has given too + // many groups for a particular pattern. + let group = SmallIndex::new(group_index).map_err(|_| { + GroupInfoError::too_many_groups(pid, group_index) + })?; + group_info.add_explicit_group(pid, group, maybe_name)?; + } + } + group_info.fixup_slot_ranges()?; + group_info.slot_ranges.shrink_to_fit(); + group_info.name_to_index.shrink_to_fit(); + group_info.index_to_name.shrink_to_fit(); + Ok(GroupInfo(Arc::new(group_info))) + } + + /// This creates an empty `GroupInfo`. + /// + /// This is a convenience routine for calling `GroupInfo::new` with an + /// iterator that yields no elements. + /// + /// # Example + /// + /// This example shows how to build a new empty `GroupInfo` and query it + /// for information. + /// + /// ``` + /// use regex_automata::util::captures::GroupInfo; + /// + /// let info = GroupInfo::empty(); + /// // Everything is zero. + /// assert_eq!(0, info.pattern_len()); + /// assert_eq!(0, info.all_group_len()); + /// assert_eq!(0, info.slot_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn empty() -> GroupInfo { + GroupInfo::new(core::iter::empty::<[Option<&str>; 0]>()) + .expect("empty group info is always valid") + } + + /// Return the capture group index corresponding to the given name in the + /// given pattern. If no such capture group name exists in the given + /// pattern, then this returns `None`. + /// + /// If the given pattern ID is invalid, then this returns `None`. + /// + /// This also returns `None` for all inputs if these captures are empty + /// (e.g., built from an empty [`GroupInfo`]). To check whether captures + /// are present for a specific pattern, use [`GroupInfo::group_len`]. + /// + /// # Example + /// + /// This example shows how to find the capture index for the given pattern + /// and group name. + /// + /// Remember that capture indices are relative to the pattern, such that + /// the same capture index value may refer to different capturing groups + /// for distinct patterns. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let (pid0, pid1) = (PatternID::must(0), PatternID::must(1)); + /// + /// let nfa = NFA::new_many(&[ + /// r"a(?P<quux>\w+)z(?P<foo>\s+)", + /// r"a(?P<foo>\d+)z", + /// ])?; + /// let groups = nfa.group_info(); + /// assert_eq!(Some(2), groups.to_index(pid0, "foo")); + /// // Recall that capture index 0 is always unnamed and refers to the + /// // entire pattern. So the first capturing group present in the pattern + /// // itself always starts at index 1. + /// assert_eq!(Some(1), groups.to_index(pid1, "foo")); + /// + /// // And if a name does not exist for a particular pattern, None is + /// // returned. + /// assert!(groups.to_index(pid0, "quux").is_some()); + /// assert!(groups.to_index(pid1, "quux").is_none()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn to_index(&self, pid: PatternID, name: &str) -> Option<usize> { + let indices = self.0.name_to_index.get(pid.as_usize())?; + indices.get(name).cloned().map(|i| i.as_usize()) + } + + /// Return the capture name for the given index and given pattern. If the + /// corresponding group does not have a name, then this returns `None`. + /// + /// If the pattern ID is invalid, then this returns `None`. + /// + /// If the group index is invalid for the given pattern, then this returns + /// `None`. A group `index` is valid for a pattern `pid` in an `nfa` if and + /// only if `index < nfa.pattern_capture_len(pid)`. + /// + /// This also returns `None` for all inputs if these captures are empty + /// (e.g., built from an empty [`GroupInfo`]). To check whether captures + /// are present for a specific pattern, use [`GroupInfo::group_len`]. + /// + /// # Example + /// + /// This example shows how to find the capture group name for the given + /// pattern and group index. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let (pid0, pid1) = (PatternID::must(0), PatternID::must(1)); + /// + /// let nfa = NFA::new_many(&[ + /// r"a(?P<foo>\w+)z(\s+)x(\d+)", + /// r"a(\d+)z(?P<foo>\s+)", + /// ])?; + /// let groups = nfa.group_info(); + /// assert_eq!(None, groups.to_name(pid0, 0)); + /// assert_eq!(Some("foo"), groups.to_name(pid0, 1)); + /// assert_eq!(None, groups.to_name(pid0, 2)); + /// assert_eq!(None, groups.to_name(pid0, 3)); + /// + /// assert_eq!(None, groups.to_name(pid1, 0)); + /// assert_eq!(None, groups.to_name(pid1, 1)); + /// assert_eq!(Some("foo"), groups.to_name(pid1, 2)); + /// // '3' is not a valid capture index for the second pattern. + /// assert_eq!(None, groups.to_name(pid1, 3)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn to_name(&self, pid: PatternID, group_index: usize) -> Option<&str> { + let pattern_names = self.0.index_to_name.get(pid.as_usize())?; + pattern_names.get(group_index)?.as_deref() + } + + /// Return an iterator of all capture groups and their names (if present) + /// for a particular pattern. + /// + /// If the given pattern ID is invalid or if this `GroupInfo` is empty, + /// then the iterator yields no elements. + /// + /// The number of elements yielded by this iterator is always equal to + /// the result of calling [`GroupInfo::group_len`] with the same + /// `PatternID`. + /// + /// # Example + /// + /// This example shows how to get a list of all capture group names for + /// a particular pattern. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let nfa = NFA::new(r"(a)(?P<foo>b)(c)(d)(?P<bar>e)")?; + /// // The first is the implicit group that is always unnamed. The next + /// // 5 groups are the explicit groups found in the concrete syntax above. + /// let expected = vec![None, None, Some("foo"), None, None, Some("bar")]; + /// let got: Vec<Option<&str>> = + /// nfa.group_info().pattern_names(PatternID::ZERO).collect(); + /// assert_eq!(expected, got); + /// + /// // Using an invalid pattern ID will result in nothing yielded. + /// let got = nfa.group_info().pattern_names(PatternID::must(999)).count(); + /// assert_eq!(0, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn pattern_names(&self, pid: PatternID) -> GroupInfoPatternNames<'_> { + GroupInfoPatternNames { + it: self + .0 + .index_to_name + .get(pid.as_usize()) + .map(|indices| indices.iter()) + .unwrap_or([].iter()), + } + } + + /// Return an iterator of all capture groups for all patterns supported by + /// this `GroupInfo`. Each item yielded is a triple of the group's pattern + /// ID, index in the pattern and the group's name, if present. + /// + /// # Example + /// + /// This example shows how to get a list of all capture groups found in + /// one NFA, potentially spanning multiple patterns. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let nfa = NFA::new_many(&[ + /// r"(?P<foo>a)", + /// r"a", + /// r"(a)", + /// ])?; + /// let expected = vec![ + /// (PatternID::must(0), 0, None), + /// (PatternID::must(0), 1, Some("foo")), + /// (PatternID::must(1), 0, None), + /// (PatternID::must(2), 0, None), + /// (PatternID::must(2), 1, None), + /// ]; + /// let got: Vec<(PatternID, usize, Option<&str>)> = + /// nfa.group_info().all_names().collect(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// Unlike other capturing group related routines, this routine doesn't + /// panic even if captures aren't enabled on this NFA: + /// + /// ``` + /// use regex_automata::nfa::thompson::{NFA, WhichCaptures}; + /// + /// let nfa = NFA::compiler() + /// .configure(NFA::config().which_captures(WhichCaptures::None)) + /// .build_many(&[ + /// r"(?P<foo>a)", + /// r"a", + /// r"(a)", + /// ])?; + /// // When captures aren't enabled, there's nothing to return. + /// assert_eq!(0, nfa.group_info().all_names().count()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn all_names(&self) -> GroupInfoAllNames<'_> { + GroupInfoAllNames { + group_info: self, + pids: PatternID::iter(self.pattern_len()), + current_pid: None, + names: None, + } + } + + /// Returns the starting and ending slot corresponding to the given + /// capturing group for the given pattern. The ending slot is always one + /// more than the starting slot returned. + /// + /// Note that this is like [`GroupInfo::slot`], except that it also returns + /// the ending slot value for convenience. + /// + /// If either the pattern ID or the capture index is invalid, then this + /// returns None. + /// + /// # Example + /// + /// This example shows that the starting slots for the first capturing + /// group of each pattern are distinct. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let nfa = NFA::new_many(&["a", "b"])?; + /// assert_ne!( + /// nfa.group_info().slots(PatternID::must(0), 0), + /// nfa.group_info().slots(PatternID::must(1), 0), + /// ); + /// + /// // Also, the start and end slot values are never equivalent. + /// let (start, end) = nfa.group_info().slots(PatternID::ZERO, 0).unwrap(); + /// assert_ne!(start, end); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn slots( + &self, + pid: PatternID, + group_index: usize, + ) -> Option<(usize, usize)> { + // Since 'slot' only even returns valid starting slots, we know that + // there must also be an end slot and that end slot is always one more + // than the start slot. + self.slot(pid, group_index).map(|start| (start, start + 1)) + } + + /// Returns the starting slot corresponding to the given capturing group + /// for the given pattern. The ending slot is always one more than the + /// value returned. + /// + /// If either the pattern ID or the capture index is invalid, then this + /// returns None. + /// + /// # Example + /// + /// This example shows that the starting slots for the first capturing + /// group of each pattern are distinct. + /// + /// ``` + /// use regex_automata::{nfa::thompson::NFA, PatternID}; + /// + /// let nfa = NFA::new_many(&["a", "b"])?; + /// assert_ne!( + /// nfa.group_info().slot(PatternID::must(0), 0), + /// nfa.group_info().slot(PatternID::must(1), 0), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn slot(&self, pid: PatternID, group_index: usize) -> Option<usize> { + if group_index >= self.group_len(pid) { + return None; + } + // At this point, we know that 'pid' refers to a real pattern and that + // 'group_index' refers to a real group. We therefore also know that + // the pattern and group can be combined to return a correct slot. + // That's why we don't need to use checked arithmetic below. + if group_index == 0 { + Some(pid.as_usize() * 2) + } else { + // As above, we don't need to check that our slot is less than the + // end of our range since we already know the group index is a + // valid index for the given pattern. + let (start, _) = self.0.slot_ranges[pid]; + Some(start.as_usize() + ((group_index - 1) * 2)) + } + } + + /// Returns the total number of patterns in this `GroupInfo`. + /// + /// This may return zero if the `GroupInfo` was constructed with no + /// patterns. + /// + /// This is guaranteed to be no bigger than [`PatternID::LIMIT`] because + /// `GroupInfo` construction will fail if too many patterns are added. + /// + /// # Example + /// + /// ``` + /// use regex_automata::nfa::thompson::NFA; + /// + /// let nfa = NFA::new_many(&["[0-9]+", "[a-z]+", "[A-Z]+"])?; + /// assert_eq!(3, nfa.group_info().pattern_len()); + /// + /// let nfa = NFA::never_match(); + /// assert_eq!(0, nfa.group_info().pattern_len()); + /// + /// let nfa = NFA::always_match(); + /// assert_eq!(1, nfa.group_info().pattern_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn pattern_len(&self) -> usize { + self.0.pattern_len() + } + + /// Return the number of capture groups in a pattern. + /// + /// If the pattern ID is invalid, then this returns `0`. + /// + /// # Example + /// + /// This example shows how the values returned by this routine may vary + /// for different patterns and NFA configurations. + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, WhichCaptures}, PatternID}; + /// + /// let nfa = NFA::new(r"(a)(b)(c)")?; + /// // There are 3 explicit groups in the pattern's concrete syntax and + /// // 1 unnamed and implicit group spanning the entire pattern. + /// assert_eq!(4, nfa.group_info().group_len(PatternID::ZERO)); + /// + /// let nfa = NFA::new(r"abc")?; + /// // There is just the unnamed implicit group. + /// assert_eq!(1, nfa.group_info().group_len(PatternID::ZERO)); + /// + /// let nfa = NFA::compiler() + /// .configure(NFA::config().which_captures(WhichCaptures::None)) + /// .build(r"abc")?; + /// // We disabled capturing groups, so there are none. + /// assert_eq!(0, nfa.group_info().group_len(PatternID::ZERO)); + /// + /// let nfa = NFA::compiler() + /// .configure(NFA::config().which_captures(WhichCaptures::None)) + /// .build(r"(a)(b)(c)")?; + /// // We disabled capturing groups, so there are none, even if there are + /// // explicit groups in the concrete syntax. + /// assert_eq!(0, nfa.group_info().group_len(PatternID::ZERO)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn group_len(&self, pid: PatternID) -> usize { + self.0.group_len(pid) + } + + /// Return the total number of capture groups across all patterns. + /// + /// This includes implicit groups that represent the entire match of a + /// pattern. + /// + /// # Example + /// + /// This example shows how the values returned by this routine may vary + /// for different patterns and NFA configurations. + /// + /// ``` + /// use regex_automata::{nfa::thompson::{NFA, WhichCaptures}, PatternID}; + /// + /// let nfa = NFA::new(r"(a)(b)(c)")?; + /// // There are 3 explicit groups in the pattern's concrete syntax and + /// // 1 unnamed and implicit group spanning the entire pattern. + /// assert_eq!(4, nfa.group_info().all_group_len()); + /// + /// let nfa = NFA::new(r"abc")?; + /// // There is just the unnamed implicit group. + /// assert_eq!(1, nfa.group_info().all_group_len()); + /// + /// let nfa = NFA::new_many(&["(a)", "b", "(c)"])?; + /// // Each pattern has one implicit groups, and two + /// // patterns have one explicit group each. + /// assert_eq!(5, nfa.group_info().all_group_len()); + /// + /// let nfa = NFA::compiler() + /// .configure(NFA::config().which_captures(WhichCaptures::None)) + /// .build(r"abc")?; + /// // We disabled capturing groups, so there are none. + /// assert_eq!(0, nfa.group_info().all_group_len()); + /// + /// let nfa = NFA::compiler() + /// .configure(NFA::config().which_captures(WhichCaptures::None)) + /// .build(r"(a)(b)(c)")?; + /// // We disabled capturing groups, so there are none, even if there are + /// // explicit groups in the concrete syntax. + /// assert_eq!(0, nfa.group_info().group_len(PatternID::ZERO)); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn all_group_len(&self) -> usize { + self.slot_len() / 2 + } + + /// Returns the total number of slots in this `GroupInfo` across all + /// patterns. + /// + /// The total number of slots is always twice the total number of capturing + /// groups, including both implicit and explicit groups. + /// + /// # Example + /// + /// This example shows the relationship between the number of capturing + /// groups and slots. + /// + /// ``` + /// use regex_automata::util::captures::GroupInfo; + /// + /// // There are 11 total groups here. + /// let info = GroupInfo::new(vec![ + /// vec![None, Some("foo")], + /// vec![None], + /// vec![None, None, None, Some("bar"), None], + /// vec![None, None, Some("foo")], + /// ])?; + /// // 2 slots per group gives us 11*2=22 slots. + /// assert_eq!(22, info.slot_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn slot_len(&self) -> usize { + self.0.small_slot_len().as_usize() + } + + /// Returns the total number of slots for implicit capturing groups. + /// + /// This is like [`GroupInfo::slot_len`], except it doesn't include the + /// explicit slots for each pattern. Since there are always exactly 2 + /// implicit slots for each pattern, the number of implicit slots is always + /// equal to twice the number of patterns. + /// + /// # Example + /// + /// This example shows the relationship between the number of capturing + /// groups, implicit slots and explicit slots. + /// + /// ``` + /// use regex_automata::util::captures::GroupInfo; + /// + /// // There are 11 total groups here. + /// let info = GroupInfo::new(vec![vec![None, Some("foo"), Some("bar")]])?; + /// // 2 slots per group gives us 11*2=22 slots. + /// assert_eq!(6, info.slot_len()); + /// // 2 implicit slots per pattern gives us 2 implicit slots since there + /// // is 1 pattern. + /// assert_eq!(2, info.implicit_slot_len()); + /// // 2 explicit capturing groups gives us 2*2=4 explicit slots. + /// assert_eq!(4, info.explicit_slot_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn implicit_slot_len(&self) -> usize { + self.pattern_len() * 2 + } + + /// Returns the total number of slots for explicit capturing groups. + /// + /// This is like [`GroupInfo::slot_len`], except it doesn't include the + /// implicit slots for each pattern. (There are always 2 implicit slots for + /// each pattern.) + /// + /// For a non-empty `GroupInfo`, it is always the case that `slot_len` is + /// strictly greater than `explicit_slot_len`. For an empty `GroupInfo`, + /// both the total number of slots and the number of explicit slots is + /// `0`. + /// + /// # Example + /// + /// This example shows the relationship between the number of capturing + /// groups, implicit slots and explicit slots. + /// + /// ``` + /// use regex_automata::util::captures::GroupInfo; + /// + /// // There are 11 total groups here. + /// let info = GroupInfo::new(vec![vec![None, Some("foo"), Some("bar")]])?; + /// // 2 slots per group gives us 11*2=22 slots. + /// assert_eq!(6, info.slot_len()); + /// // 2 implicit slots per pattern gives us 2 implicit slots since there + /// // is 1 pattern. + /// assert_eq!(2, info.implicit_slot_len()); + /// // 2 explicit capturing groups gives us 2*2=4 explicit slots. + /// assert_eq!(4, info.explicit_slot_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn explicit_slot_len(&self) -> usize { + self.slot_len().saturating_sub(self.implicit_slot_len()) + } + + /// Returns the memory usage, in bytes, of this `GroupInfo`. + /// + /// This does **not** include the stack size used up by this `GroupInfo`. + /// To compute that, use `std::mem::size_of::<GroupInfo>()`. + #[inline] + pub fn memory_usage(&self) -> usize { + use core::mem::size_of as s; + + s::<GroupInfoInner>() + + self.0.slot_ranges.len() * s::<(SmallIndex, SmallIndex)>() + + self.0.name_to_index.len() * s::<CaptureNameMap>() + + self.0.index_to_name.len() * s::<Vec<Option<Arc<str>>>>() + + self.0.memory_extra + } +} + +/// A map from capture group name to its corresponding capture group index. +/// +/// This type is actually wrapped inside a Vec indexed by pattern ID on a +/// `GroupInfo`, since multiple patterns may have the same capture group name. +/// That is, each pattern gets its own namespace of capture group names. +/// +/// Perhaps a more memory efficient representation would be +/// HashMap<(PatternID, Arc<str>), usize>, but this makes it difficult to look +/// up a capture index by name without producing a `Arc<str>`, which requires +/// an allocation. To fix this, I think we'd need to define our own unsized +/// type or something? Anyway, I didn't give this much thought since it +/// probably doesn't matter much in the grand scheme of things. But it did +/// stand out to me as mildly wasteful. +#[cfg(feature = "std")] +type CaptureNameMap = std::collections::HashMap<Arc<str>, SmallIndex>; +#[cfg(not(feature = "std"))] +type CaptureNameMap = alloc::collections::BTreeMap<Arc<str>, SmallIndex>; + +/// The inner guts of `GroupInfo`. This type only exists so that it can +/// be wrapped in an `Arc` to make `GroupInfo` reference counted. +#[derive(Debug, Default)] +struct GroupInfoInner { + slot_ranges: Vec<(SmallIndex, SmallIndex)>, + name_to_index: Vec<CaptureNameMap>, + index_to_name: Vec<Vec<Option<Arc<str>>>>, + memory_extra: usize, +} + +impl GroupInfoInner { + /// This adds the first unnamed group for the given pattern ID. The given + /// pattern ID must be zero if this is the first time this method is + /// called, or must be exactly one more than the pattern ID supplied to the + /// previous call to this method. (This method panics if this rule is + /// violated.) + /// + /// This can be thought of as initializing the GroupInfo state for the + /// given pattern and closing off the state for any previous pattern. + fn add_first_group(&mut self, pid: PatternID) { + assert_eq!(pid.as_usize(), self.slot_ranges.len()); + assert_eq!(pid.as_usize(), self.name_to_index.len()); + assert_eq!(pid.as_usize(), self.index_to_name.len()); + // This is the start of our slots for the explicit capturing groups. + // Note that since the slots for the 0th group for every pattern appear + // before any slots for the nth group (where n > 0) in any pattern, we + // will have to fix up the slot ranges once we know how many patterns + // we've added capture groups for. + let slot_start = self.small_slot_len(); + self.slot_ranges.push((slot_start, slot_start)); + self.name_to_index.push(CaptureNameMap::new()); + self.index_to_name.push(vec![None]); + self.memory_extra += core::mem::size_of::<Option<Arc<str>>>(); + } + + /// Add an explicit capturing group for the given pattern with the given + /// index. If the group has a name, then that must be given as well. + /// + /// Note that every capturing group except for the first or zeroth group is + /// explicit. + /// + /// This returns an error if adding this group would result in overflowing + /// slot indices or if a capturing group with the same name for this + /// pattern has already been added. + fn add_explicit_group<N: AsRef<str>>( + &mut self, + pid: PatternID, + group: SmallIndex, + maybe_name: Option<N>, + ) -> Result<(), GroupInfoError> { + // We also need to check that the slot index generated for + // this group is also valid. Although, this is a little weird + // because we offset these indices below, at which point, we'll + // have to recheck them. Gosh this is annoying. Note that + // the '+2' below is OK because 'end' is guaranteed to be less + // than isize::MAX. + let end = &mut self.slot_ranges[pid].1; + *end = SmallIndex::new(end.as_usize() + 2).map_err(|_| { + GroupInfoError::too_many_groups(pid, group.as_usize()) + })?; + if let Some(name) = maybe_name { + let name = Arc::<str>::from(name.as_ref()); + if self.name_to_index[pid].contains_key(&*name) { + return Err(GroupInfoError::duplicate(pid, &name)); + } + let len = name.len(); + self.name_to_index[pid].insert(Arc::clone(&name), group); + self.index_to_name[pid].push(Some(name)); + // Adds the memory used by the Arc<str> in both maps. + self.memory_extra += + 2 * (len + core::mem::size_of::<Option<Arc<str>>>()); + // And also the value entry for the 'name_to_index' map. + // This is probably an underestimate for 'name_to_index' since + // hashmaps/btrees likely have some non-zero overhead, but we + // assume here that they have zero overhead. + self.memory_extra += core::mem::size_of::<SmallIndex>(); + } else { + self.index_to_name[pid].push(None); + self.memory_extra += core::mem::size_of::<Option<Arc<str>>>(); + } + // This is a sanity assert that checks that our group index + // is in line with the number of groups added so far for this + // pattern. + assert_eq!(group.one_more(), self.group_len(pid)); + // And is also in line with the 'index_to_name' map. + assert_eq!(group.one_more(), self.index_to_name[pid].len()); + Ok(()) + } + + /// This corrects the slot ranges to account for the slots corresponding + /// to the zeroth group of each pattern. That is, every slot range is + /// offset by 'pattern_len() * 2', since each pattern uses two slots to + /// represent the zeroth group. + fn fixup_slot_ranges(&mut self) -> Result<(), GroupInfoError> { + use crate::util::primitives::IteratorIndexExt; + // Since we know number of patterns fits in PatternID and + // PatternID::MAX < isize::MAX, it follows that multiplying by 2 will + // never overflow usize. + let offset = self.pattern_len().checked_mul(2).unwrap(); + for (pid, &mut (ref mut start, ref mut end)) in + self.slot_ranges.iter_mut().with_pattern_ids() + { + let group_len = 1 + ((end.as_usize() - start.as_usize()) / 2); + let new_end = match end.as_usize().checked_add(offset) { + Some(new_end) => new_end, + None => { + return Err(GroupInfoError::too_many_groups( + pid, group_len, + )) + } + }; + *end = SmallIndex::new(new_end).map_err(|_| { + GroupInfoError::too_many_groups(pid, group_len) + })?; + // Since start <= end, if end is valid then start must be too. + *start = SmallIndex::new(start.as_usize() + offset).unwrap(); + } + Ok(()) + } + + /// Return the total number of patterns represented by this capture slot + /// info. + fn pattern_len(&self) -> usize { + self.slot_ranges.len() + } + + /// Return the total number of capturing groups for the given pattern. If + /// the given pattern isn't valid for this capture slot info, then 0 is + /// returned. + fn group_len(&self, pid: PatternID) -> usize { + let (start, end) = match self.slot_ranges.get(pid.as_usize()) { + None => return 0, + Some(range) => range, + }; + // The difference between any two SmallIndex values always fits in a + // usize since we know that SmallIndex::MAX <= isize::MAX-1. We also + // know that start<=end by construction and that the number of groups + // never exceeds SmallIndex and thus never overflows usize. + 1 + ((end.as_usize() - start.as_usize()) / 2) + } + + /// Return the total number of slots in this capture slot info as a + /// "small index." + fn small_slot_len(&self) -> SmallIndex { + // Since slots are allocated in order of pattern (starting at 0) and + // then in order of capture group, it follows that the number of slots + // is the end of the range of slots for the last pattern. This is + // true even when the last pattern has no capturing groups, since + // 'slot_ranges' will still represent it explicitly with an empty + // range. + self.slot_ranges.last().map_or(SmallIndex::ZERO, |&(_, end)| end) + } +} + +/// An error that may occur when building a `GroupInfo`. +/// +/// Building a `GroupInfo` does a variety of checks to make sure the +/// capturing groups satisfy a number of invariants. This includes, but is not +/// limited to, ensuring that the first capturing group is unnamed and that +/// there are no duplicate capture groups for a specific pattern. +#[derive(Clone, Debug)] +pub struct GroupInfoError { + kind: GroupInfoErrorKind, +} + +/// The kind of error that occurs when building a `GroupInfo` fails. +/// +/// We keep this un-exported because it's not clear how useful it is to +/// export it. +#[derive(Clone, Debug)] +enum GroupInfoErrorKind { + /// This occurs when too many patterns have been added. i.e., It would + /// otherwise overflow a `PatternID`. + TooManyPatterns { err: PatternIDError }, + /// This occurs when too many capturing groups have been added for a + /// particular pattern. + TooManyGroups { + /// The ID of the pattern that had too many groups. + pattern: PatternID, + /// The minimum number of groups that the caller has tried to add for + /// a pattern. + minimum: usize, + }, + /// An error that occurs when a pattern has no capture groups. Either the + /// group info must be empty, or all patterns must have at least one group + /// (corresponding to the unnamed group for the entire pattern). + MissingGroups { + /// The ID of the pattern that had no capturing groups. + pattern: PatternID, + }, + /// An error that occurs when one tries to provide a name for the capture + /// group at index 0. This capturing group must currently always be + /// unnamed. + FirstMustBeUnnamed { + /// The ID of the pattern that was found to have a named first + /// capturing group. + pattern: PatternID, + }, + /// An error that occurs when duplicate capture group names for the same + /// pattern are added. + /// + /// NOTE: At time of writing, this error can never occur if you're using + /// regex-syntax, since the parser itself will reject patterns with + /// duplicate capture group names. This error can only occur when the + /// builder is used to hand construct NFAs. + Duplicate { + /// The pattern in which the duplicate capture group name was found. + pattern: PatternID, + /// The duplicate name. + name: String, + }, +} + +impl GroupInfoError { + fn too_many_patterns(err: PatternIDError) -> GroupInfoError { + GroupInfoError { kind: GroupInfoErrorKind::TooManyPatterns { err } } + } + + fn too_many_groups(pattern: PatternID, minimum: usize) -> GroupInfoError { + GroupInfoError { + kind: GroupInfoErrorKind::TooManyGroups { pattern, minimum }, + } + } + + fn missing_groups(pattern: PatternID) -> GroupInfoError { + GroupInfoError { kind: GroupInfoErrorKind::MissingGroups { pattern } } + } + + fn first_must_be_unnamed(pattern: PatternID) -> GroupInfoError { + GroupInfoError { + kind: GroupInfoErrorKind::FirstMustBeUnnamed { pattern }, + } + } + + fn duplicate(pattern: PatternID, name: &str) -> GroupInfoError { + GroupInfoError { + kind: GroupInfoErrorKind::Duplicate { + pattern, + name: String::from(name), + }, + } + } +} + +#[cfg(feature = "std")] +impl std::error::Error for GroupInfoError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self.kind { + GroupInfoErrorKind::TooManyPatterns { .. } + | GroupInfoErrorKind::TooManyGroups { .. } + | GroupInfoErrorKind::MissingGroups { .. } + | GroupInfoErrorKind::FirstMustBeUnnamed { .. } + | GroupInfoErrorKind::Duplicate { .. } => None, + } + } +} + +impl core::fmt::Display for GroupInfoError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + use self::GroupInfoErrorKind::*; + + match self.kind { + TooManyPatterns { ref err } => { + write!(f, "too many patterns to build capture info: {err}") + } + TooManyGroups { pattern, minimum } => { + write!( + f, + "too many capture groups (at least {}) were \ + found for pattern {}", + minimum, + pattern.as_usize() + ) + } + MissingGroups { pattern } => write!( + f, + "no capturing groups found for pattern {} \ + (either all patterns have zero groups or all patterns have \ + at least one group)", + pattern.as_usize(), + ), + FirstMustBeUnnamed { pattern } => write!( + f, + "first capture group (at index 0) for pattern {} has a name \ + (it must be unnamed)", + pattern.as_usize(), + ), + Duplicate { pattern, ref name } => write!( + f, + "duplicate capture group name '{}' found for pattern {}", + name, + pattern.as_usize(), + ), + } + } +} + +/// An iterator over capturing groups and their names for a specific pattern. +/// +/// This iterator is created by [`GroupInfo::pattern_names`]. +/// +/// The lifetime parameter `'a` refers to the lifetime of the `GroupInfo` +/// from which this iterator was created. +#[derive(Clone, Debug)] +pub struct GroupInfoPatternNames<'a> { + it: core::slice::Iter<'a, Option<Arc<str>>>, +} + +impl GroupInfoPatternNames<'static> { + fn empty() -> GroupInfoPatternNames<'static> { + GroupInfoPatternNames { it: [].iter() } + } +} + +impl<'a> Iterator for GroupInfoPatternNames<'a> { + type Item = Option<&'a str>; + + fn next(&mut self) -> Option<Option<&'a str>> { + self.it.next().map(|x| x.as_deref()) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + self.it.size_hint() + } + + fn count(self) -> usize { + self.it.count() + } +} + +impl<'a> ExactSizeIterator for GroupInfoPatternNames<'a> {} +impl<'a> core::iter::FusedIterator for GroupInfoPatternNames<'a> {} + +/// An iterator over capturing groups and their names for a `GroupInfo`. +/// +/// This iterator is created by [`GroupInfo::all_names`]. +/// +/// The lifetime parameter `'a` refers to the lifetime of the `GroupInfo` +/// from which this iterator was created. +#[derive(Debug)] +pub struct GroupInfoAllNames<'a> { + group_info: &'a GroupInfo, + pids: PatternIDIter, + current_pid: Option<PatternID>, + names: Option<core::iter::Enumerate<GroupInfoPatternNames<'a>>>, +} + +impl<'a> Iterator for GroupInfoAllNames<'a> { + type Item = (PatternID, usize, Option<&'a str>); + + fn next(&mut self) -> Option<(PatternID, usize, Option<&'a str>)> { + // If the group info has no captures, then we never have anything + // to yield. We need to consider this case explicitly (at time of + // writing) because 'pattern_capture_names' will panic if captures + // aren't enabled. + if self.group_info.0.index_to_name.is_empty() { + return None; + } + if self.current_pid.is_none() { + self.current_pid = Some(self.pids.next()?); + } + let pid = self.current_pid.unwrap(); + if self.names.is_none() { + self.names = Some(self.group_info.pattern_names(pid).enumerate()); + } + let (group_index, name) = match self.names.as_mut().unwrap().next() { + Some((group_index, name)) => (group_index, name), + None => { + self.current_pid = None; + self.names = None; + return self.next(); + } + }; + Some((pid, group_index, name)) + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/determinize/mod.rs b/third_party/crates/regex-automata-0.4.18/src/util/determinize/mod.rs new file mode 100644 index 0000000..22e38c9 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/determinize/mod.rs @@ -0,0 +1,682 @@ +/*! +This module contains types and routines for implementing determinization. + +In this crate, there are at least two places where we implement +determinization: fully ahead-of-time compiled DFAs in the `dfa` module and +lazily compiled DFAs in the `hybrid` module. The stuff in this module +corresponds to the things that are in common between these implementations. + +There are three broad things that our implementations of determinization have +in common, as defined by this module: + +* The classification of start states. That is, whether we're dealing with +word boundaries, line boundaries, etc., is all the same. This also includes +the look-behind assertions that are satisfied by each starting state +classification. +* The representation of DFA states as sets of NFA states, including +convenience types for building these DFA states that are amenable to reusing +allocations. +* Routines for the "classical" parts of determinization: computing the +epsilon closure, tracking match states (with corresponding pattern IDs, since +we support multi-pattern finite automata) and, of course, computing the +transition function between states for units of input. + +I did consider a couple of alternatives to this particular form of code reuse: + +1. Don't do any code reuse. The problem here is that we *really* want both +forms of determinization to do exactly identical things when it comes to +their handling of NFA states. While our tests generally ensure this, the code +is tricky and large enough where not reusing code is a pretty big bummer. + +2. Implement all of determinization once and make it generic over fully +compiled DFAs and lazily compiled DFAs. While I didn't actually try this +approach, my instinct is that it would be more complex than is needed here. +And the interface required would be pretty hairy. Instead, I think splitting +it into logical sub-components works better. +*/ + +use alloc::vec::Vec; + +pub(crate) use self::state::{ + State, StateBuilderEmpty, StateBuilderMatches, StateBuilderNFA, +}; + +use crate::{ + nfa::thompson, + util::{ + alphabet, + look::{Look, LookSet}, + primitives::StateID, + search::MatchKind, + sparse_set::{SparseSet, SparseSets}, + start::Start, + utf8, + }, +}; + +mod state; + +/// Compute the set of all reachable NFA states, including the full epsilon +/// closure, from a DFA state for a single unit of input. The set of reachable +/// states is returned as a `StateBuilderNFA`. The `StateBuilderNFA` returned +/// also includes any look-behind assertions satisfied by `unit`, in addition +/// to whether it is a match state. For multi-pattern DFAs, the builder will +/// also include the pattern IDs that match (in the order seen). +/// +/// `nfa` must be able to resolve any NFA state in `state` and any NFA state +/// reachable via the epsilon closure of any NFA state in `state`. `sparses` +/// must have capacity equivalent to `nfa.len()`. +/// +/// `match_kind` should correspond to the match semantics implemented by the +/// DFA being built. Generally speaking, for leftmost-first match semantics, +/// states that appear after the first NFA match state will not be included in +/// the `StateBuilderNFA` returned since they are impossible to visit. +/// +/// `sparses` is used as scratch space for NFA traversal. Other than their +/// capacity requirements (detailed above), there are no requirements on what's +/// contained within them (if anything). Similarly, what's inside of them once +/// this routine returns is unspecified. +/// +/// `stack` must have length 0. It is used as scratch space for depth first +/// traversal. After returning, it is guaranteed that `stack` will have length +/// 0. +/// +/// `state` corresponds to the current DFA state on which one wants to compute +/// the transition for the input `unit`. +/// +/// `empty_builder` corresponds to the builder allocation to use to produce a +/// complete `StateBuilderNFA` state. If the state is not needed (or is already +/// cached), then it can be cleared and reused without needing to create a new +/// `State`. The `StateBuilderNFA` state returned is final and ready to be +/// turned into a `State` if necessary. +pub(crate) fn next( + nfa: &thompson::NFA, + match_kind: MatchKind, + sparses: &mut SparseSets, + stack: &mut Vec<StateID>, + state: &State, + unit: alphabet::Unit, + empty_builder: StateBuilderEmpty, +) -> StateBuilderNFA { + sparses.clear(); + + // Whether the NFA is matched in reverse or not. We use this in some + // conditional logic for dealing with the exceptionally annoying CRLF-aware + // line anchors. + let rev = nfa.is_reverse(); + // The look-around matcher that our NFA is configured with. We don't + // actually use it to match look-around assertions, but we do need its + // configuration for constructing states consistent with how it matches. + let lookm = nfa.look_matcher(); + + // Put the NFA state IDs into a sparse set in case we need to + // re-compute their epsilon closure. + // + // Doing this state shuffling is technically not necessary unless some + // kind of look-around is used in the DFA. Some ad hoc experiments + // suggested that avoiding this didn't lead to much of an improvement, + // but perhaps more rigorous experimentation should be done. And in + // particular, avoiding this check requires some light refactoring of + // the code below. + state.iter_nfa_state_ids(|nfa_id| { + sparses.set1.insert(nfa_id); + }); + + // Compute look-ahead assertions originating from the current state. Based + // on the input unit we're transitioning over, some additional set of + // assertions may be true. Thus, we re-compute this state's epsilon closure + // (but only if necessary). Notably, when we build a DFA state initially, + // we don't enable any look-ahead assertions because we don't know whether + // they're true or not at that point. + if !state.look_need().is_empty() { + // Add look-ahead assertions that are now true based on the current + // input unit. + let mut look_have = state.look_have(); + match unit.as_u8() { + Some(b'\r') => { + if !rev || !state.is_half_crlf() { + look_have = look_have.insert(Look::EndCRLF); + } + } + Some(b'\n') => { + if rev || !state.is_half_crlf() { + look_have = look_have.insert(Look::EndCRLF); + } + } + Some(_) => {} + None => { + look_have = look_have + .insert(Look::End) + .insert(Look::EndLF) + .insert(Look::EndCRLF); + } + } + if unit.is_byte(lookm.get_line_terminator()) { + look_have = look_have.insert(Look::EndLF); + } + if state.is_half_crlf() + && ((rev && !unit.is_byte(b'\r')) + || (!rev && !unit.is_byte(b'\n'))) + { + look_have = look_have.insert(Look::StartCRLF); + } + if state.is_from_word() == unit.is_word_byte() { + look_have = look_have + .insert(Look::WordAsciiNegate) + .insert(Look::WordUnicodeNegate); + } else { + look_have = + look_have.insert(Look::WordAscii).insert(Look::WordUnicode); + } + if !unit.is_word_byte() { + look_have = look_have + .insert(Look::WordEndHalfAscii) + .insert(Look::WordEndHalfUnicode); + } + if state.is_from_word() && !unit.is_word_byte() { + look_have = look_have + .insert(Look::WordEndAscii) + .insert(Look::WordEndUnicode); + } else if !state.is_from_word() && unit.is_word_byte() { + look_have = look_have + .insert(Look::WordStartAscii) + .insert(Look::WordStartUnicode); + } + // If we have new assertions satisfied that are among the set of + // assertions that exist in this state (that is, just because we added + // an EndLF assertion above doesn't mean there is an EndLF conditional + // epsilon transition in this state), then we re-compute this state's + // epsilon closure using the updated set of assertions. + // + // Note that since our DFA states omit unconditional epsilon + // transitions, this check is necessary for correctness. If we re-did + // the epsilon closure below needlessly, it could change based on the + // fact that we omitted epsilon states originally. + if !look_have + .subtract(state.look_have()) + .intersect(state.look_need()) + .is_empty() + { + for nfa_id in sparses.set1.iter() { + epsilon_closure( + nfa, + nfa_id, + look_have, + stack, + &mut sparses.set2, + ); + } + sparses.swap(); + sparses.set2.clear(); + } + } + + // Convert our empty builder into one that can record assertions and match + // pattern IDs. + let mut builder = empty_builder.into_matches(); + // Set whether the StartLF look-behind assertion is true for this + // transition or not. The look-behind assertion for ASCII word boundaries + // is handled below. + if nfa.look_set_any().contains_anchor_line() + && unit.is_byte(lookm.get_line_terminator()) + { + // Why only handle StartLF here and not Start? That's because Start + // can only impact the starting state, which is special cased in + // start state handling. + builder.set_look_have(|have| have.insert(Look::StartLF)); + } + // We also need to add StartCRLF to our assertions too, if we can. This + // is unfortunately a bit more complicated, because it depends on the + // direction of the search. In the forward direction, ^ matches after a + // \n, but in the reverse direction, ^ only matches after a \r. (This is + // further complicated by the fact that reverse a regex means changing a ^ + // to a $ and vice versa.) + if nfa.look_set_any().contains_anchor_crlf() + && ((rev && unit.is_byte(b'\r')) || (!rev && unit.is_byte(b'\n'))) + { + builder.set_look_have(|have| have.insert(Look::StartCRLF)); + } + // And also for the start-half word boundary assertions. As long as the + // look-behind byte is not a word char, then the assertions are satisfied. + if nfa.look_set_any().contains_word() && !unit.is_word_byte() { + builder.set_look_have(|have| { + have.insert(Look::WordStartHalfAscii) + .insert(Look::WordStartHalfUnicode) + }); + } + for nfa_id in sparses.set1.iter() { + match *nfa.state(nfa_id) { + thompson::State::Union { .. } + | thompson::State::BinaryUnion { .. } + | thompson::State::Fail + | thompson::State::Look { .. } + | thompson::State::Capture { .. } => {} + thompson::State::Match { pattern_id } => { + // Notice here that we are calling the NEW state a match + // state if the OLD state we are transitioning from + // contains an NFA match state. This is precisely how we + // delay all matches by one byte and also what therefore + // guarantees that starting states cannot be match states. + // + // If we didn't delay matches by one byte, then whether + // a DFA is a matching state or not would be determined + // by whether one of its own constituent NFA states + // was a match state. (And that would be done in + // 'add_nfa_states'.) + // + // Also, 'add_match_pattern_id' requires that callers never + // pass duplicative pattern IDs. We do in fact uphold that + // guarantee here, but it's subtle. In particular, a Thompson + // NFA guarantees that each pattern has exactly one match + // state. Moreover, since we're iterating over the NFA state + // IDs in a set, we are guaranteed not to have any duplicative + // match states. Thus, it is impossible to add the same pattern + // ID more than once. + // + // N.B. We delay matches by 1 byte as a way to hack 1-byte + // look-around into DFA searches. This lets us support ^, $ + // and ASCII-only \b. The delay is also why we need a special + // "end-of-input" (EOI) sentinel and why we need to follow the + // EOI sentinel at the end of every search. This final EOI + // transition is necessary to report matches found at the end + // of a haystack. + builder.add_match_pattern_id(pattern_id); + if !match_kind.continue_past_first_match() { + break; + } + } + thompson::State::ByteRange { ref trans } => { + if trans.matches_unit(unit) { + epsilon_closure( + nfa, + trans.next, + builder.look_have(), + stack, + &mut sparses.set2, + ); + } + } + thompson::State::Sparse(ref sparse) => { + if let Some(next) = sparse.matches_unit(unit) { + epsilon_closure( + nfa, + next, + builder.look_have(), + stack, + &mut sparses.set2, + ); + } + } + thompson::State::Dense(ref dense) => { + if let Some(next) = dense.matches_unit(unit) { + epsilon_closure( + nfa, + next, + builder.look_have(), + stack, + &mut sparses.set2, + ); + } + } + } + } + // We only set the word byte if there's a word boundary look-around + // anywhere in this regex. Otherwise, there's no point in bloating the + // number of states if we don't have one. + // + // We also only set it when the state has a non-zero number of NFA states. + // Otherwise, we could wind up with states that *should* be DEAD states + // but are otherwise distinct from DEAD states because of this look-behind + // assertion being set. While this can't technically impact correctness *in + // theory*, it can create pathological DFAs that consume input until EOI or + // a quit byte is seen. Consuming until EOI isn't a correctness problem, + // but a (serious) perf problem. Hitting a quit byte, however, could be a + // correctness problem since it could cause search routines to report an + // error instead of a detected match once the quit state is entered. (The + // search routine could be made to be a bit smarter by reporting a match + // if one was detected once it enters a quit state (and indeed, the search + // routines in this crate do just that), but it seems better to prevent + // these things by construction if possible.) + if !sparses.set2.is_empty() { + if nfa.look_set_any().contains_word() && unit.is_word_byte() { + builder.set_is_from_word(); + } + if nfa.look_set_any().contains_anchor_crlf() + && ((rev && unit.is_byte(b'\n')) || (!rev && unit.is_byte(b'\r'))) + { + builder.set_is_half_crlf(); + } + } + let mut builder_nfa = builder.into_nfa(); + add_nfa_states(nfa, &sparses.set2, &mut builder_nfa); + builder_nfa +} + +/// Compute the epsilon closure for the given NFA state. The epsilon closure +/// consists of all NFA state IDs, including `start_nfa_id`, that can be +/// reached from `start_nfa_id` without consuming any input. These state IDs +/// are written to `set` in the order they are visited, but only if they are +/// not already in `set`. `start_nfa_id` must be a valid state ID for the NFA +/// given. +/// +/// `look_have` consists of the satisfied assertions at the current +/// position. For conditional look-around epsilon transitions, these are +/// only followed if they are satisfied by `look_have`. +/// +/// `stack` must have length 0. It is used as scratch space for depth first +/// traversal. After returning, it is guaranteed that `stack` will have length +/// 0. +pub(crate) fn epsilon_closure( + nfa: &thompson::NFA, + start_nfa_id: StateID, + look_have: LookSet, + stack: &mut Vec<StateID>, + set: &mut SparseSet, +) { + assert!(stack.is_empty()); + // If this isn't an epsilon state, then the epsilon closure is always just + // itself, so there's no need to spin up the machinery below to handle it. + if !nfa.state(start_nfa_id).is_epsilon() { + set.insert(start_nfa_id); + return; + } + + stack.push(start_nfa_id); + while let Some(mut id) = stack.pop() { + // In many cases, we can avoid stack operations when an NFA state only + // adds one new state to visit. In that case, we just set our ID to + // that state and mush on. We only use the stack when an NFA state + // introduces multiple new states to visit. + loop { + // Insert this NFA state, and if it's already in the set and thus + // already visited, then we can move on to the next one. + if !set.insert(id) { + break; + } + match *nfa.state(id) { + thompson::State::ByteRange { .. } + | thompson::State::Sparse { .. } + | thompson::State::Dense { .. } + | thompson::State::Fail + | thompson::State::Match { .. } => break, + thompson::State::Look { look, next } => { + if !look_have.contains(look) { + break; + } + id = next; + } + thompson::State::Union { ref alternates } => { + id = match alternates.get(0) { + None => break, + Some(&id) => id, + }; + // We need to process our alternates in order to preserve + // match preferences, so put the earliest alternates closer + // to the top of the stack. + stack.extend(alternates[1..].iter().rev()); + } + thompson::State::BinaryUnion { alt1, alt2 } => { + id = alt1; + stack.push(alt2); + } + thompson::State::Capture { next, .. } => { + id = next; + } + } + } + } +} + +/// Add the NFA state IDs in the given `set` to the given DFA builder state. +/// The order in which states are added corresponds to the order in which they +/// were added to `set`. +/// +/// The DFA builder state given should already have its complete set of match +/// pattern IDs added (if any) and any look-behind assertions (StartLF, Start +/// and whether this state is being generated for a transition over a word byte +/// when applicable) that are true immediately prior to transitioning into this +/// state (via `builder.look_have()`). The match pattern IDs should correspond +/// to matches that occurred on the previous transition, since all matches are +/// delayed by one byte. The things that should _not_ be set are look-ahead +/// assertions (EndLF, End and whether the next byte is a word byte or not). +/// The builder state should also not have anything in `look_need` set, as this +/// routine will compute that for you. +/// +/// The given NFA should be able to resolve all identifiers in `set` to a +/// particular NFA state. Additionally, `set` must have capacity equivalent +/// to `nfa.len()`. +pub(crate) fn add_nfa_states( + nfa: &thompson::NFA, + set: &SparseSet, + builder: &mut StateBuilderNFA, +) { + for nfa_id in set.iter() { + match *nfa.state(nfa_id) { + thompson::State::ByteRange { .. } => { + builder.add_nfa_state_id(nfa_id); + } + thompson::State::Sparse { .. } => { + builder.add_nfa_state_id(nfa_id); + } + thompson::State::Dense { .. } => { + builder.add_nfa_state_id(nfa_id); + } + thompson::State::Look { look, .. } => { + builder.add_nfa_state_id(nfa_id); + builder.set_look_need(|need| need.insert(look)); + } + thompson::State::Union { .. } + | thompson::State::BinaryUnion { .. } => { + // Pure epsilon transitions don't need to be tracked as part + // of the DFA state. Tracking them is actually superfluous; + // they won't cause any harm other than making determinization + // slower. + // + // Why aren't these needed? Well, in an NFA, epsilon + // transitions are really just jumping points to other states. + // So once you hit an epsilon transition, the same set of + // resulting states always appears. Therefore, putting them in + // a DFA's set of ordered NFA states is strictly redundant. + // + // Look-around states are also epsilon transitions, but + // they are *conditional*. So their presence could be + // discriminatory, and thus, they are tracked above. + // + // But wait... why are epsilon states in our `set` in the first + // place? Why not just leave them out? They're in our `set` + // because it was generated by computing an epsilon closure, + // and we want to keep track of all states we visited to avoid + // re-visiting them. In exchange, we have to do this second + // iteration over our collected states to finalize our DFA + // state. In theory, we could avoid this second iteration if + // we maintained two sets during epsilon closure: the set of + // visited states (to avoid cycles) and the set of states that + // will actually be used to construct the next DFA state. + // + // Note that this optimization requires that we re-compute the + // epsilon closure to account for look-ahead in 'next' *only + // when necessary*. Namely, only when the set of look-around + // assertions changes and only when those changes are within + // the set of assertions that are needed in order to step + // through the closure correctly. Otherwise, if we re-do the + // epsilon closure needlessly, it could change based on the + // fact that we are omitting epsilon states here. + // + // ----- + // + // Welp, scratch the above. It turns out that recording these + // is in fact necessary to seemingly handle one particularly + // annoying case: when a conditional epsilon transition is + // put inside of a repetition operator. One specific case I + // ran into was the regex `(?:\b|%)+` on the haystack `z%`. + // The correct leftmost first matches are: [0, 0] and [1, 1]. + // But the DFA was reporting [0, 0] and [1, 2]. To understand + // why this happens, consider the NFA for the aforementioned + // regex: + // + // >000000: binary-union(4, 1) + // 000001: \x00-\xFF => 0 + // 000002: WordAscii => 5 + // 000003: % => 5 + // ^000004: binary-union(2, 3) + // 000005: binary-union(4, 6) + // 000006: MATCH(0) + // + // The problem here is that one of the DFA start states is + // going to consist of the NFA states [2, 3] by computing the + // epsilon closure of state 4. State 4 isn't included because + // we previously were not keeping track of union states. But + // only a subset of transitions out of this state will be able + // to follow WordAscii, and in those cases, the epsilon closure + // is redone. The only problem is that computing the epsilon + // closure from [2, 3] is different than computing the epsilon + // closure from [4]. In the former case, assuming the WordAscii + // assertion is satisfied, you get: [2, 3, 6]. In the latter + // case, you get: [2, 6, 3]. Notice that '6' is the match state + // and appears AFTER '3' in the former case. This leads to a + // preferential but incorrect match of '%' before returning + // a match. In the latter case, the match is preferred over + // continuing to accept the '%'. + // + // It almost feels like we might be able to fix the NFA states + // to avoid this, or to at least only keep track of union + // states where this actually matters, since in the vast + // majority of cases, this doesn't matter. + // + // Another alternative would be to define a new HIR property + // called "assertion is repeated anywhere" and compute it + // inductively over the entire pattern. If it happens anywhere, + // which is probably pretty rare, then we record union states. + // Otherwise we don't. + builder.add_nfa_state_id(nfa_id); + } + // Capture states we definitely do not need to record, since they + // are unconditional epsilon transitions with no branching. + thompson::State::Capture { .. } => {} + // It's not totally clear whether we need to record fail states or + // not, but we do so out of an abundance of caution. Since they are + // quite rare in practice, there isn't much cost to recording them. + thompson::State::Fail => { + builder.add_nfa_state_id(nfa_id); + } + thompson::State::Match { .. } => { + // Normally, the NFA match state doesn't actually need to + // be inside the DFA state. But since we delay matches by + // one byte, the matching DFA state corresponds to states + // that transition from the one we're building here. And + // the way we detect those cases is by looking for an NFA + // match state. See 'next' for how this is handled. + builder.add_nfa_state_id(nfa_id); + } + } + } + // If we know this state contains no look-around assertions, then + // there's no reason to track which look-around assertions were + // satisfied when this state was created. + if builder.look_need().is_empty() { + builder.set_look_have(|_| LookSet::empty()); + } +} + +/// Sets the appropriate look-behind assertions on the given state based on +/// this starting configuration. +pub(crate) fn set_lookbehind_from_start( + nfa: &thompson::NFA, + start: &Start, + builder: &mut StateBuilderMatches, +) { + let rev = nfa.is_reverse(); + let lineterm = nfa.look_matcher().get_line_terminator(); + let lookset = nfa.look_set_any(); + match *start { + Start::NonWordByte => { + if lookset.contains_word() { + builder.set_look_have(|have| { + have.insert(Look::WordStartHalfAscii) + .insert(Look::WordStartHalfUnicode) + }); + } + } + Start::WordByte => { + if lookset.contains_word() { + builder.set_is_from_word(); + } + } + Start::Text => { + if lookset.contains_anchor_haystack() { + builder.set_look_have(|have| have.insert(Look::Start)); + } + if lookset.contains_anchor_line() { + builder.set_look_have(|have| { + have.insert(Look::StartLF).insert(Look::StartCRLF) + }); + } + if lookset.contains_word() { + builder.set_look_have(|have| { + have.insert(Look::WordStartHalfAscii) + .insert(Look::WordStartHalfUnicode) + }); + } + } + Start::LineLF => { + if rev { + if lookset.contains_anchor_crlf() { + builder.set_is_half_crlf(); + } + if lookset.contains_anchor_line() { + builder.set_look_have(|have| have.insert(Look::StartLF)); + } + } else { + if lookset.contains_anchor_line() { + builder.set_look_have(|have| have.insert(Look::StartCRLF)); + } + } + if lookset.contains_anchor_line() && lineterm == b'\n' { + builder.set_look_have(|have| have.insert(Look::StartLF)); + } + if lookset.contains_word() { + builder.set_look_have(|have| { + have.insert(Look::WordStartHalfAscii) + .insert(Look::WordStartHalfUnicode) + }); + } + } + Start::LineCR => { + if lookset.contains_anchor_crlf() { + if rev { + builder.set_look_have(|have| have.insert(Look::StartCRLF)); + } else { + builder.set_is_half_crlf(); + } + } + if lookset.contains_anchor_line() && lineterm == b'\r' { + builder.set_look_have(|have| have.insert(Look::StartLF)); + } + if lookset.contains_word() { + builder.set_look_have(|have| { + have.insert(Look::WordStartHalfAscii) + .insert(Look::WordStartHalfUnicode) + }); + } + } + Start::CustomLineTerminator => { + if lookset.contains_anchor_line() { + builder.set_look_have(|have| have.insert(Look::StartLF)); + } + // This is a bit of a tricky case, but if the line terminator was + // set to a word byte, then we also need to behave as if the start + // configuration is Start::WordByte. That is, we need to mark our + // state as having come from a word byte. + if lookset.contains_word() { + if utf8::is_word_byte(lineterm) { + builder.set_is_from_word(); + } else { + builder.set_look_have(|have| { + have.insert(Look::WordStartHalfAscii) + .insert(Look::WordStartHalfUnicode) + }); + } + } + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/determinize/state.rs b/third_party/crates/regex-automata-0.4.18/src/util/determinize/state.rs new file mode 100644 index 0000000..f410f9a --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/determinize/state.rs @@ -0,0 +1,907 @@ +/*! +This module defines a DFA state representation and builders for constructing +DFA states. + +This representation is specifically for use in implementations of NFA-to-DFA +conversion via powerset construction. (Also called "determinization" in this +crate.) + +The term "DFA state" is somewhat overloaded in this crate. In some cases, it +refers to the set of transitions over an alphabet for a particular state. In +other cases, it refers to a set of NFA states. The former is really about the +final representation of a state in a DFA's transition table, where as the +latter---what this module is focused on---is closer to an intermediate form +that is used to help eventually build the transition table. + +This module exports four types. All four types represent the same idea: an +ordered set of NFA states. This ordered set represents the epsilon closure of a +particular NFA state, where the "epsilon closure" is the set of NFA states that +can be transitioned to without consuming any input. i.e., Follow all of the NFA +state's epsilon transitions. In addition, this implementation of DFA states +cares about two other things: the ordered set of pattern IDs corresponding +to the patterns that match if the state is a match state, and the set of +look-behind assertions that were true when the state was created. + +The first, `State`, is a frozen representation of a state that cannot be +modified. It may be cheaply cloned without copying the state itself and can be +accessed safely from multiple threads simultaneously. This type is useful for +when one knows that the DFA state being constructed is distinct from any other +previously constructed states. Namely, powerset construction, in practice, +requires one to keep a cache of previously created DFA states. Otherwise, +the number of DFA states created in memory balloons to an impractically +large number. For this reason, equivalent states should endeavor to have an +equivalent byte-level representation. (In general, "equivalency" here means, +"equivalent assertions, pattern IDs and NFA state IDs." We do not require that +full DFA minimization be implemented here. This form of equivalency is only +surface deep and is more-or-less a practical necessity.) + +The other three types represent different phases in the construction of a +DFA state. Internally, these three types (and `State`) all use the same +byte-oriented representation. That means one can use any of the builder types +to check whether the state it represents already exists or not. If it does, +then there is no need to freeze it into a `State` (which requires an alloc and +a copy). Here are the three types described succinctly: + +* `StateBuilderEmpty` represents a state with no pattern IDs, no assertions +and no NFA states. Creating a `StateBuilderEmpty` performs no allocs. A +`StateBuilderEmpty` can only be used to query its underlying memory capacity, +or to convert into a builder for recording pattern IDs and/or assertions. + +* `StateBuilderMatches` represents a state with zero or more pattern IDs, zero +or more satisfied assertions and zero NFA state IDs. A `StateBuilderMatches` +can only be used for adding pattern IDs and recording assertions. + +* `StateBuilderNFA` represents a state with zero or more pattern IDs, zero or +more satisfied assertions and zero or more NFA state IDs. A `StateBuilderNFA` +can only be used for adding NFA state IDs and recording some assertions. + +The expected flow here is to use the above builders to construct a candidate +DFA state to check if it already exists. If it does, then there's no need to +freeze it into a `State`. If it doesn't exist, then `StateBuilderNFA::to_state` +can be called to freeze the builder into an immutable `State`. In either +case, `clear` should be called on the builder to turn it back into a +`StateBuilderEmpty` that reuses the underlying memory. + +The main purpose for splitting the builder into these distinct types is to +make it impossible to do things like adding a pattern ID after adding an NFA +state ID. Namely, this makes it simpler to use a space-and-time efficient +binary representation for the state. (The format is documented on the `Repr` +type below.) If we just used one type for everything, it would be possible for +callers to use an incorrect interleaving of calls and thus result in a corrupt +representation. I chose to use more type machinery to make this impossible to +do because 1) determinization is itself pretty complex and it wouldn't be too +hard to foul this up and 2) there isn't too much machinery involved and it's +well contained. + +As an optimization, sometimes states won't have certain things set. For +example, if the underlying NFA has no word boundary assertions, then there is +no reason to set a state's look-behind assertion as to whether it was generated +from a word byte or not. Similarly, if a state has no NFA states corresponding +to look-around assertions, then there is no reason to set `look_have` to a +non-empty set. Finally, callers usually omit unconditional epsilon transitions +when adding NFA state IDs since they aren't discriminatory. + +Finally, the binary representation used by these states is, thankfully, not +serialized anywhere. So any kind of change can be made with reckless abandon, +as long as everything in this module agrees. +*/ + +use core::mem; + +use alloc::{sync::Arc, vec::Vec}; + +use crate::util::{ + int::{I32, U32}, + look::LookSet, + primitives::{PatternID, StateID}, + wire::{self, Endian}, +}; + +/// A DFA state that, at its core, is represented by an ordered set of NFA +/// states. +/// +/// This type is intended to be used only in NFA-to-DFA conversion via powerset +/// construction. +/// +/// It may be cheaply cloned and accessed safely from multiple threads +/// simultaneously. +#[derive(Clone, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub(crate) struct State(Arc<[u8]>); + +/// This Borrow impl permits us to lookup any state in a map by its byte +/// representation. This is particularly convenient when one has a StateBuilder +/// and we want to see if a correspondingly equivalent state already exists. If +/// one does exist, then we can reuse the allocation required by StateBuilder +/// without having to convert it into a State first. +impl core::borrow::Borrow<[u8]> for State { + fn borrow(&self) -> &[u8] { + &self.0 + } +} + +impl core::fmt::Debug for State { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_tuple("State").field(&self.repr()).finish() + } +} + +/// For docs on these routines, see the internal Repr and ReprVec types below. +impl State { + pub(crate) fn dead() -> State { + StateBuilderEmpty::new().into_matches().into_nfa().to_state() + } + + pub(crate) fn is_match(&self) -> bool { + self.repr().is_match() + } + + pub(crate) fn is_from_word(&self) -> bool { + self.repr().is_from_word() + } + + pub(crate) fn is_half_crlf(&self) -> bool { + self.repr().is_half_crlf() + } + + pub(crate) fn look_have(&self) -> LookSet { + self.repr().look_have() + } + + pub(crate) fn look_need(&self) -> LookSet { + self.repr().look_need() + } + + pub(crate) fn match_len(&self) -> usize { + self.repr().match_len() + } + + pub(crate) fn match_pattern(&self, index: usize) -> PatternID { + self.repr().match_pattern(index) + } + + pub(crate) fn match_pattern_ids(&self) -> Option<Vec<PatternID>> { + self.repr().match_pattern_ids() + } + + #[cfg(all(test, not(miri)))] + pub(crate) fn iter_match_pattern_ids<F: FnMut(PatternID)>(&self, f: F) { + self.repr().iter_match_pattern_ids(f) + } + + pub(crate) fn iter_nfa_state_ids<F: FnMut(StateID)>(&self, f: F) { + self.repr().iter_nfa_state_ids(f) + } + + pub(crate) fn memory_usage(&self) -> usize { + self.0.len() + } + + fn repr(&self) -> Repr<'_> { + Repr(&self.0) + } +} + +/// A state builder that represents an empty state. +/// +/// This is a useful "initial condition" for state construction. It has no +/// NFA state IDs, no assertions set and no pattern IDs. No allocations are +/// made when new() is called. Its main use is for being converted into a +/// builder that can capture assertions and pattern IDs. +#[derive(Clone, Debug)] +pub(crate) struct StateBuilderEmpty(Vec<u8>); + +/// For docs on these routines, see the internal Repr and ReprVec types below. +impl StateBuilderEmpty { + pub(crate) fn new() -> StateBuilderEmpty { + StateBuilderEmpty(alloc::vec![]) + } + + pub(crate) fn into_matches(mut self) -> StateBuilderMatches { + self.0.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0]); + StateBuilderMatches(self.0) + } + + fn clear(&mut self) { + self.0.clear(); + } + + pub(crate) fn capacity(&self) -> usize { + self.0.capacity() + } +} + +/// A state builder that collects assertions and pattern IDs. +/// +/// When collecting pattern IDs is finished, this can be converted into a +/// builder that collects NFA state IDs. +#[derive(Clone)] +pub(crate) struct StateBuilderMatches(Vec<u8>); + +impl core::fmt::Debug for StateBuilderMatches { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_tuple("StateBuilderMatches").field(&self.repr()).finish() + } +} + +/// For docs on these routines, see the internal Repr and ReprVec types below. +impl StateBuilderMatches { + pub(crate) fn into_nfa(mut self) -> StateBuilderNFA { + self.repr_vec().close_match_pattern_ids(); + StateBuilderNFA { repr: self.0, prev_nfa_state_id: StateID::ZERO } + } + + pub(crate) fn set_is_from_word(&mut self) { + self.repr_vec().set_is_from_word() + } + + pub(crate) fn set_is_half_crlf(&mut self) { + self.repr_vec().set_is_half_crlf() + } + + pub(crate) fn look_have(&self) -> LookSet { + LookSet::read_repr(&self.0[1..]) + } + + pub(crate) fn set_look_have( + &mut self, + set: impl FnMut(LookSet) -> LookSet, + ) { + self.repr_vec().set_look_have(set) + } + + pub(crate) fn add_match_pattern_id(&mut self, pid: PatternID) { + self.repr_vec().add_match_pattern_id(pid) + } + + fn repr(&self) -> Repr<'_> { + Repr(&self.0) + } + + fn repr_vec(&mut self) -> ReprVec<'_> { + ReprVec(&mut self.0) + } +} + +/// A state builder that collects some assertions and NFA state IDs. +/// +/// When collecting NFA state IDs is finished, this can be used to build a +/// `State` if necessary. +/// +/// When dont with building a state (regardless of whether it got kept or not), +/// it's usually a good idea to call `clear` to get an empty builder back so +/// that it can be reused to build the next state. +#[derive(Clone)] +pub(crate) struct StateBuilderNFA { + repr: Vec<u8>, + prev_nfa_state_id: StateID, +} + +impl core::fmt::Debug for StateBuilderNFA { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_tuple("StateBuilderNFA").field(&self.repr()).finish() + } +} + +/// For docs on these routines, see the internal Repr and ReprVec types below. +impl StateBuilderNFA { + pub(crate) fn to_state(&self) -> State { + State(Arc::from(&*self.repr)) + } + + pub(crate) fn clear(self) -> StateBuilderEmpty { + let mut builder = StateBuilderEmpty(self.repr); + builder.clear(); + builder + } + + pub(crate) fn look_need(&self) -> LookSet { + self.repr().look_need() + } + + pub(crate) fn set_look_have( + &mut self, + set: impl FnMut(LookSet) -> LookSet, + ) { + self.repr_vec().set_look_have(set) + } + + pub(crate) fn set_look_need( + &mut self, + set: impl FnMut(LookSet) -> LookSet, + ) { + self.repr_vec().set_look_need(set) + } + + pub(crate) fn add_nfa_state_id(&mut self, sid: StateID) { + ReprVec(&mut self.repr) + .add_nfa_state_id(&mut self.prev_nfa_state_id, sid) + } + + pub(crate) fn as_bytes(&self) -> &[u8] { + &self.repr + } + + fn repr(&self) -> Repr<'_> { + Repr(&self.repr) + } + + fn repr_vec(&mut self) -> ReprVec<'_> { + ReprVec(&mut self.repr) + } +} + +/// Repr is a read-only view into the representation of a DFA state. +/// +/// Primarily, a Repr is how we achieve DRY: we implement decoding the format +/// in one place, and then use a Repr to implement the various methods on the +/// public state types. +/// +/// The format is as follows: +/// +/// The first three bytes correspond to bitsets. +/// +/// Byte 0 is a bitset corresponding to miscellaneous flags associated with the +/// state. Bit 0 is set to 1 if the state is a match state. Bit 1 is set to 1 +/// if the state has pattern IDs explicitly written to it. (This is a flag that +/// is not meant to be set by determinization, but rather, is used as part of +/// an internal space-saving optimization.) Bit 2 is set to 1 if the state was +/// generated by a transition over a "word" byte. (Callers may not always set +/// this. For example, if the NFA has no word boundary assertion, then needing +/// to track whether a state came from a word byte or not is superfluous and +/// wasteful.) Bit 3 is set to 1 if the state was generated by a transition +/// from a `\r` (forward search) or a `\n` (reverse search) when CRLF mode is +/// enabled. +/// +/// Bytes 1..5 correspond to the look-behind assertions that were satisfied +/// by the transition that created this state. (Look-ahead assertions are not +/// tracked as part of states. Instead, these are applied by re-computing the +/// epsilon closure of a state when computing the transition function. See +/// `next` in the parent module.) +/// +/// Bytes 5..9 correspond to the set of look-around assertions (including both +/// look-behind and look-ahead) that appear somewhere in this state's set of +/// NFA state IDs. This is used to determine whether this state's epsilon +/// closure should be re-computed when computing the transition function. +/// Namely, look-around assertions are "just" conditional epsilon transitions, +/// so if there are new assertions available when computing the transition +/// function, we should only re-compute the epsilon closure if those new +/// assertions are relevant to this particular state. +/// +/// Bytes 9..13 correspond to a 32-bit native-endian encoded integer +/// corresponding to the number of patterns encoded in this state. If the state +/// is not a match state (byte 0 bit 0 is 0) or if it's only pattern ID is +/// PatternID::ZERO, then no integer is encoded at this position. Instead, byte +/// offset 3 is the position at which the first NFA state ID is encoded. +/// +/// For a match state with at least one non-ZERO pattern ID, the next bytes +/// correspond to a sequence of 32-bit native endian encoded integers that +/// represent each pattern ID, in order, that this match state represents. +/// +/// After the pattern IDs (if any), NFA state IDs are delta encoded as +/// varints.[1] The first NFA state ID is encoded as itself, and each +/// subsequent NFA state ID is encoded as the difference between itself and the +/// previous NFA state ID. +/// +/// [1] - https://developers.google.com/protocol-buffers/docs/encoding#varints +struct Repr<'a>(&'a [u8]); + +impl<'a> Repr<'a> { + /// Returns true if and only if this is a match state. + /// + /// If callers have added pattern IDs to this state, then callers MUST set + /// this state as a match state explicitly. However, as a special case, + /// states that are marked as match states but with no pattern IDs, then + /// the state is treated as if it had a single pattern ID equivalent to + /// PatternID::ZERO. + fn is_match(&self) -> bool { + self.0[0] & (1 << 0) > 0 + } + + /// Returns true if and only if this state has had at least one pattern + /// ID added to it. + /// + /// This is an internal-only flag that permits the representation to save + /// space in the common case of an NFA with one pattern in it. In that + /// case, a match state can only ever have exactly one pattern ID: + /// PatternID::ZERO. So there's no need to represent it. + fn has_pattern_ids(&self) -> bool { + self.0[0] & (1 << 1) > 0 + } + + /// Returns true if and only if this state is marked as having been created + /// from a transition over a word byte. This is useful for checking whether + /// a word boundary assertion is true or not, which requires look-behind + /// (whether the current state came from a word byte or not) and look-ahead + /// (whether the transition byte is a word byte or not). + /// + /// Since states with this set are distinct from states that don't have + /// this set (even if they are otherwise equivalent), callers should not + /// set this assertion unless the underlying NFA has at least one word + /// boundary assertion somewhere. Otherwise, a superfluous number of states + /// may be created. + fn is_from_word(&self) -> bool { + self.0[0] & (1 << 2) > 0 + } + + /// Returns true if and only if this state is marked as being inside of a + /// CRLF terminator. In the forward direction, this means the state was + /// created after seeing a `\r`. In the reverse direction, this means the + /// state was created after seeing a `\n`. + fn is_half_crlf(&self) -> bool { + self.0[0] & (1 << 3) > 0 + } + + /// The set of look-behind assertions that were true in the transition that + /// created this state. + /// + /// Generally, this should be empty if 'look_need' is empty, since there is + /// no reason to track which look-behind assertions are true if the state + /// has no conditional epsilon transitions. + /// + /// Satisfied look-ahead assertions are not tracked in states. Instead, + /// these are re-computed on demand via epsilon closure when computing the + /// transition function. + fn look_have(&self) -> LookSet { + LookSet::read_repr(&self.0[1..]) + } + + /// The set of look-around (both behind and ahead) assertions that appear + /// at least once in this state's set of NFA states. + /// + /// This is used to determine whether the epsilon closure needs to be + /// re-computed when computing the transition function. Namely, if the + /// state has no conditional epsilon transitions, then there is no need + /// to re-compute the epsilon closure. + fn look_need(&self) -> LookSet { + LookSet::read_repr(&self.0[5..]) + } + + /// Returns the total number of match pattern IDs in this state. + /// + /// If this state is not a match state, then this always returns 0. + fn match_len(&self) -> usize { + if !self.is_match() { + 0 + } else if !self.has_pattern_ids() { + 1 + } else { + self.encoded_pattern_len() + } + } + + /// Returns the pattern ID for this match state at the given index. + /// + /// If the given index is greater than or equal to `match_len()` for this + /// state, then this could panic or return incorrect results. + fn match_pattern(&self, index: usize) -> PatternID { + if !self.has_pattern_ids() { + PatternID::ZERO + } else { + let offset = 13 + index * PatternID::SIZE; + // This is OK since we only ever serialize valid PatternIDs to + // states. + wire::read_pattern_id_unchecked(&self.0[offset..]).0 + } + } + + /// Returns a copy of all match pattern IDs in this state. If this state + /// is not a match state, then this returns None. + fn match_pattern_ids(&self) -> Option<Vec<PatternID>> { + if !self.is_match() { + return None; + } + let mut pids = alloc::vec![]; + self.iter_match_pattern_ids(|pid| pids.push(pid)); + Some(pids) + } + + /// Calls the given function on every pattern ID in this state. + fn iter_match_pattern_ids<F: FnMut(PatternID)>(&self, mut f: F) { + if !self.is_match() { + return; + } + // As an optimization for a very common case, when this is a match + // state for an NFA with only one pattern, we don't actually write the + // pattern ID to the state representation. Instead, we know it must + // be there since it is the only possible choice. + if !self.has_pattern_ids() { + f(PatternID::ZERO); + return; + } + let mut pids = &self.0[13..self.pattern_offset_end()]; + while !pids.is_empty() { + let pid = wire::read_u32(pids); + pids = &pids[PatternID::SIZE..]; + // This is OK since we only ever serialize valid PatternIDs to + // states. And since pattern IDs can never exceed a usize, the + // unwrap is OK. + f(PatternID::new_unchecked(usize::try_from(pid).unwrap())); + } + } + + /// Calls the given function on every NFA state ID in this state. + fn iter_nfa_state_ids<F: FnMut(StateID)>(&self, mut f: F) { + let mut sids = &self.0[self.pattern_offset_end()..]; + let mut prev = 0i32; + while !sids.is_empty() { + let (delta, nr) = read_vari32(sids); + sids = &sids[nr..]; + let sid = prev + delta; + prev = sid; + // This is OK since we only ever serialize valid StateIDs to + // states. And since state IDs can never exceed an isize, they must + // always be able to fit into a usize, and thus cast is OK. + f(StateID::new_unchecked(sid.as_usize())) + } + } + + /// Returns the offset into this state's representation where the pattern + /// IDs end and the NFA state IDs begin. + fn pattern_offset_end(&self) -> usize { + let encoded = self.encoded_pattern_len(); + if encoded == 0 { + return 9; + } + // This arithmetic is OK since we were able to address this many bytes + // when writing to the state, thus, it must fit into a usize. + encoded.checked_mul(4).unwrap().checked_add(13).unwrap() + } + + /// Returns the total number of *encoded* pattern IDs in this state. + /// + /// This may return 0 even when this is a match state, since the pattern + /// ID `PatternID::ZERO` is not encoded when it's the only pattern ID in + /// the match state (the overwhelming common case). + fn encoded_pattern_len(&self) -> usize { + if !self.has_pattern_ids() { + return 0; + } + // This unwrap is OK since the total number of patterns is always + // guaranteed to fit into a usize. + usize::try_from(wire::read_u32(&self.0[9..13])).unwrap() + } +} + +impl<'a> core::fmt::Debug for Repr<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let mut nfa_ids = alloc::vec![]; + self.iter_nfa_state_ids(|sid| nfa_ids.push(sid)); + f.debug_struct("Repr") + .field("is_match", &self.is_match()) + .field("is_from_word", &self.is_from_word()) + .field("is_half_crlf", &self.is_half_crlf()) + .field("look_have", &self.look_have()) + .field("look_need", &self.look_need()) + .field("match_pattern_ids", &self.match_pattern_ids()) + .field("nfa_state_ids", &nfa_ids) + .finish() + } +} + +/// ReprVec is a write-only view into the representation of a DFA state. +/// +/// See Repr for more details on the purpose of this type and also the format. +/// +/// Note that not all possible combinations of methods may be called. This is +/// precisely what the various StateBuilder types encapsulate: they only +/// permit valid combinations via Rust's linear typing. +struct ReprVec<'a>(&'a mut Vec<u8>); + +impl<'a> ReprVec<'a> { + /// Set this state as a match state. + /// + /// This should not be exposed explicitly outside of this module. It is + /// set automatically when a pattern ID is added. + fn set_is_match(&mut self) { + self.0[0] |= 1 << 0; + } + + /// Set that this state has pattern IDs explicitly written to it. + /// + /// This should not be exposed explicitly outside of this module. This is + /// used internally as a space saving optimization. Namely, if the state + /// is a match state but does not have any pattern IDs written to it, + /// then it is automatically inferred to have a pattern ID of ZERO. + fn set_has_pattern_ids(&mut self) { + self.0[0] |= 1 << 1; + } + + /// Set this state as being built from a transition over a word byte. + /// + /// Setting this is only necessary when one needs to deal with word + /// boundary assertions. Therefore, if the underlying NFA has no word + /// boundary assertions, callers should not set this. + fn set_is_from_word(&mut self) { + self.0[0] |= 1 << 2; + } + + /// Set this state as having seen half of a CRLF terminator. + /// + /// In the forward direction, this should be set when a `\r` has been seen. + /// In the reverse direction, this should be set when a `\n` has been seen. + fn set_is_half_crlf(&mut self) { + self.0[0] |= 1 << 3; + } + + /// The set of look-behind assertions that were true in the transition that + /// created this state. + fn look_have(&self) -> LookSet { + self.repr().look_have() + } + + /// The set of look-around (both behind and ahead) assertions that appear + /// at least once in this state's set of NFA states. + fn look_need(&self) -> LookSet { + self.repr().look_need() + } + + /// Mutate the set of look-behind assertions that were true in the + /// transition that created this state. + fn set_look_have(&mut self, mut set: impl FnMut(LookSet) -> LookSet) { + set(self.look_have()).write_repr(&mut self.0[1..]); + } + + /// Mutate the set of look-around (both behind and ahead) assertions that + /// appear at least once in this state's set of NFA states. + fn set_look_need(&mut self, mut set: impl FnMut(LookSet) -> LookSet) { + set(self.look_need()).write_repr(&mut self.0[5..]); + } + + /// Add a pattern ID to this state. All match states must have at least + /// one pattern ID associated with it. + /// + /// Callers must never add duplicative pattern IDs. + /// + /// The order in which patterns are added must correspond to the order + /// in which patterns are reported as matches. + fn add_match_pattern_id(&mut self, pid: PatternID) { + // As a (somewhat small) space saving optimization, in the case where + // a matching state has exactly one pattern ID, PatternID::ZERO, we do + // not write either the pattern ID or the number of patterns encoded. + // Instead, all we do is set the 'is_match' bit on this state. Overall, + // this saves 8 bytes per match state for the overwhelming majority of + // match states. + // + // In order to know whether pattern IDs need to be explicitly read or + // not, we use another internal-only bit, 'has_pattern_ids', to + // indicate whether they have been explicitly written or not. + if !self.repr().has_pattern_ids() { + if pid == PatternID::ZERO { + self.set_is_match(); + return; + } + // Make room for 'close_match_pattern_ids' to write the total + // number of pattern IDs written. + self.0.extend(core::iter::repeat(0).take(PatternID::SIZE)); + self.set_has_pattern_ids(); + // If this was already a match state, then the only way that's + // possible when the state doesn't have pattern IDs is if + // PatternID::ZERO was added by the caller previously. In this + // case, we are now adding a non-ZERO pattern ID after it, in + // which case, we want to make sure to represent ZERO explicitly + // now. + if self.repr().is_match() { + write_u32(self.0, 0) + } else { + // Otherwise, just make sure the 'is_match' bit is set. + self.set_is_match(); + } + } + write_u32(self.0, pid.as_u32()); + } + + /// Indicate that no more pattern IDs will be added to this state. + /// + /// Once this is called, callers must not call it or 'add_match_pattern_id' + /// again. + /// + /// This should not be exposed explicitly outside of this module. It + /// should be called only when converting a StateBuilderMatches into a + /// StateBuilderNFA. + fn close_match_pattern_ids(&mut self) { + // If we never wrote any pattern IDs, then there's nothing to do here. + if !self.repr().has_pattern_ids() { + return; + } + let patsize = PatternID::SIZE; + let pattern_bytes = self.0.len() - 13; + // Every pattern ID uses 4 bytes, so number of bytes should be + // divisible by 4. + assert_eq!(pattern_bytes % patsize, 0); + // This unwrap is OK since we are guaranteed that the maximum number + // of possible patterns fits into a u32. + let count32 = u32::try_from(pattern_bytes / patsize).unwrap(); + wire::NE::write_u32(count32, &mut self.0[9..13]); + } + + /// Add an NFA state ID to this state. The order in which NFA states are + /// added matters. It is the caller's responsibility to ensure that + /// duplicate NFA state IDs are not added. + fn add_nfa_state_id(&mut self, prev: &mut StateID, sid: StateID) { + let delta = sid.as_i32() - prev.as_i32(); + write_vari32(self.0, delta); + *prev = sid; + } + + /// Return a read-only view of this state's representation. + fn repr(&self) -> Repr<'_> { + Repr(self.0.as_slice()) + } +} + +/// Write a signed 32-bit integer using zig-zag encoding. +/// +/// https://developers.google.com/protocol-buffers/docs/encoding#varints +fn write_vari32(data: &mut Vec<u8>, n: i32) { + let mut un = n.to_bits() << 1; + if n < 0 { + un = !un; + } + write_varu32(data, un) +} + +/// Read a signed 32-bit integer using zig-zag encoding. Also, return the +/// number of bytes read. +/// +/// https://developers.google.com/protocol-buffers/docs/encoding#varints +fn read_vari32(data: &[u8]) -> (i32, usize) { + let (un, i) = read_varu32(data); + let mut n = i32::from_bits(un >> 1); + if un & 1 != 0 { + n = !n; + } + (n, i) +} + +/// Write an unsigned 32-bit integer as a varint. In essence, `n` is written +/// as a sequence of bytes where all bytes except for the last one have the +/// most significant bit set. The least significant 7 bits correspond to the +/// actual bits of `n`. So in the worst case, a varint uses 5 bytes, but in +/// very common cases, it uses fewer than 4. +/// +/// https://developers.google.com/protocol-buffers/docs/encoding#varints +fn write_varu32(data: &mut Vec<u8>, mut n: u32) { + while n >= 0b1000_0000 { + data.push(n.low_u8() | 0b1000_0000); + n >>= 7; + } + data.push(n.low_u8()); +} + +/// Read an unsigned 32-bit varint. Also, return the number of bytes read. +/// +/// https://developers.google.com/protocol-buffers/docs/encoding#varints +fn read_varu32(data: &[u8]) -> (u32, usize) { + // N.B. We can assume correctness here since we know that all var-u32 are + // written with write_varu32. Hence, the 'as' uses and unchecked arithmetic + // is all okay. + let mut n: u32 = 0; + let mut shift: u32 = 0; + for (i, &b) in data.iter().enumerate() { + if b < 0b1000_0000 { + return (n | (u32::from(b) << shift), i + 1); + } + n |= (u32::from(b) & 0b0111_1111) << shift; + shift += 7; + } + (0, 0) +} + +/// Push a native-endian encoded `n` on to `dst`. +fn write_u32(dst: &mut Vec<u8>, n: u32) { + use crate::util::wire::NE; + + let start = dst.len(); + dst.extend(core::iter::repeat(0).take(mem::size_of::<u32>())); + NE::write_u32(n, &mut dst[start..]); +} + +#[cfg(test)] +mod tests { + use alloc::vec; + + use quickcheck::quickcheck; + + use super::*; + + #[cfg(not(miri))] + quickcheck! { + fn prop_state_read_write_nfa_state_ids(sids: Vec<StateID>) -> bool { + // Builders states do not permit duplicate IDs. + let sids = dedup_state_ids(sids); + + let mut b = StateBuilderEmpty::new().into_matches().into_nfa(); + for &sid in &sids { + b.add_nfa_state_id(sid); + } + let s = b.to_state(); + let mut got = vec![]; + s.iter_nfa_state_ids(|sid| got.push(sid)); + got == sids + } + + fn prop_state_read_write_pattern_ids(pids: Vec<PatternID>) -> bool { + // Builders states do not permit duplicate IDs. + let pids = dedup_pattern_ids(pids); + + let mut b = StateBuilderEmpty::new().into_matches(); + for &pid in &pids { + b.add_match_pattern_id(pid); + } + let s = b.into_nfa().to_state(); + let mut got = vec![]; + s.iter_match_pattern_ids(|pid| got.push(pid)); + got == pids + } + + fn prop_state_read_write_nfa_state_and_pattern_ids( + sids: Vec<StateID>, + pids: Vec<PatternID> + ) -> bool { + // Builders states do not permit duplicate IDs. + let sids = dedup_state_ids(sids); + let pids = dedup_pattern_ids(pids); + + let mut b = StateBuilderEmpty::new().into_matches(); + for &pid in &pids { + b.add_match_pattern_id(pid); + } + + let mut b = b.into_nfa(); + for &sid in &sids { + b.add_nfa_state_id(sid); + } + + let s = b.to_state(); + let mut got_pids = vec![]; + s.iter_match_pattern_ids(|pid| got_pids.push(pid)); + let mut got_sids = vec![]; + s.iter_nfa_state_ids(|sid| got_sids.push(sid)); + got_pids == pids && got_sids == sids + } + } + + quickcheck! { + fn prop_read_write_varu32(n: u32) -> bool { + let mut buf = vec![]; + write_varu32(&mut buf, n); + let (got, nread) = read_varu32(&buf); + nread == buf.len() && got == n + } + + fn prop_read_write_vari32(n: i32) -> bool { + let mut buf = vec![]; + write_vari32(&mut buf, n); + let (got, nread) = read_vari32(&buf); + nread == buf.len() && got == n + } + } + + #[cfg(not(miri))] + fn dedup_state_ids(sids: Vec<StateID>) -> Vec<StateID> { + let mut set = alloc::collections::BTreeSet::new(); + let mut deduped = vec![]; + for sid in sids { + if set.contains(&sid) { + continue; + } + set.insert(sid); + deduped.push(sid); + } + deduped + } + + #[cfg(not(miri))] + fn dedup_pattern_ids(pids: Vec<PatternID>) -> Vec<PatternID> { + let mut set = alloc::collections::BTreeSet::new(); + let mut deduped = vec![]; + for pid in pids { + if set.contains(&pid) { + continue; + } + set.insert(pid); + deduped.push(pid); + } + deduped + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/empty.rs b/third_party/crates/regex-automata-0.4.18/src/util/empty.rs new file mode 100644 index 0000000..e16af3b --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/empty.rs @@ -0,0 +1,265 @@ +/*! +This module provides helper routines for dealing with zero-width matches. + +The main problem being solved here is this: + +1. The caller wants to search something that they know is valid UTF-8, such +as a Rust `&str`. +2. The regex used by the caller can match the empty string. For example, `a*`. +3. The caller should never get match offsets returned that occur within the +encoding of a UTF-8 codepoint. It is logically incorrect, and also means that, +e.g., slicing the `&str` at those offsets will lead to a panic. + +So the question here is, how do we prevent the caller from getting match +offsets that split a codepoint? For example, strictly speaking, the regex `a*` +matches `☃` at the positions `[0, 0]`, `[1, 1]`, `[2, 2]` and `[3, 3]` since +the UTF-8 encoding of `☃` is `\xE2\x98\x83`. In particular, the `NFA` that +underlies all of the matching engines in this crate doesn't have anything in +its state graph that prevents matching between UTF-8 code units. Indeed, any +engine derived from the `NFA` will match at those positions by virtue of the +fact that the `NFA` is byte oriented. That is, its transitions are defined over +bytes and the matching engines work by proceeding one byte at a time. + +(An alternative architecture would be to define the transitions in an `NFA` +over codepoints, or `char`. And then make the matching engines proceed by +decoding one codepoint at a time. This is a viable strategy, but it doesn't +work for DFA matching engines because designing a fast and memory efficient +transition table for an alphabet as large as Unicode is quite difficult. More +to the point, the top-level `regex` crate supports matching on arbitrary bytes +when Unicode mode is disabled and one is searching a `&[u8]`. So in that case, +you can't just limit yourself to decoding codepoints and matching those. You +really do need to be able to follow byte oriented transitions on the `NFA`.) + +In an older version of the regex crate, we handled this case not in the regex +engine, but in the iterators over matches. Namely, since this case only arises +when the match is empty, we "just" incremented the next starting position +of the search by `N`, where `N` is the length of the codepoint encoded at +the current position. The alternative or more "natural" solution of just +incrementing by `1` would result in executing a search of `a*` on `☃` like +this: + +* Start search at `0`. +* Found match at `[0, 0]`. +* Next start position is `0`. +* To avoid an infinite loop, since it's an empty match, increment by `1`. +* Start search at `1`. +* Found match at `[1, 1]`. Oops. + +But if we instead incremented by `3` (the length in bytes of `☃`), then we get +the following: + +* Start search at `0`. +* Found match at `[0, 0]`. +* Next start position is `0`. +* To avoid an infinite loop, since it's an empty match, increment by `3`. +* Start search at `3`. +* Found match at `[3, 3]`. + +And we get the correct result. But does this technique work in all cases? +Crucially, it requires that a zero-width match that splits a codepoint never +occurs beyond the starting position of the search. Because if it did, merely +incrementing the start position by the number of bytes in the codepoint at +the current position wouldn't be enough. A zero-width match could just occur +anywhere. It turns out that it is _almost_ true. We can convince ourselves by +looking at all possible patterns that can match the empty string: + +* Patterns like `a*`, `a{0}`, `(?:)`, `a|` and `|a` all unconditionally match +the empty string. That is, assuming there isn't an `a` at the current position, +they will all match the empty string at the start of a search. There is no way +to move past it because any other match would not be "leftmost." +* `^` only matches at the beginning of the haystack, where the start position +is `0`. Since we know we're searching valid UTF-8 (if it isn't valid UTF-8, +then this entire problem goes away because it implies your string type supports +invalid UTF-8 and thus must deal with offsets that not only split a codepoint +but occur in entirely invalid UTF-8 somehow), it follows that `^` never matches +between the code units of a codepoint because the start of a valid UTF-8 string +is never within the encoding of a codepoint. +* `$` basically the same logic as `^`, but for the end of a string. A valid +UTF-8 string can't have an incomplete codepoint at the end of it. +* `(?m:^)` follows similarly to `^`, but it can match immediately following +a `\n`. However, since a `\n` is always a codepoint itself and can never +appear within a codepoint, it follows that the position immediately following +a `\n` in a string that is valid UTF-8 is guaranteed to not be between the +code units of another codepoint. (One caveat here is that the line terminator +for multi-line anchors can now be changed to any arbitrary byte, including +things like `\x98` which might occur within a codepoint. However, this wasn't +supported by the old regex crate. If it was, it pose the same problems as +`(?-u:\B)`, as we'll discuss below.) +* `(?m:$)` a similar argument as for `(?m:^)`. The only difference is that a +`(?m:$)` matches just before a `\n`. But the same argument applies. +* `(?Rm:^)` and `(?Rm:$)` weren't supported by the old regex crate, but the +CRLF aware line anchors follow a similar argument as for `(?m:^)` and `(?m:$)`. +Namely, since they only ever match at a boundary where one side is either a +`\r` or a `\n`, neither of which can occur within a codepoint. +* `\b` only matches at positions where both sides are valid codepoints, so +this cannot split a codepoint. +* `\B`, like `\b`, also only matches at positions where both sides are valid +codepoints. So this cannot split a codepoint either. +* `(?-u:\b)` matches only at positions where at least one side of it is an ASCII +word byte. Since ASCII bytes cannot appear as code units in non-ASCII codepoints +(one of the many amazing qualities of UTF-8), it follows that this too cannot +split a codepoint. +* `(?-u:\B)` finally represents a problem. It can matches between *any* two +bytes that are either both word bytes or non-word bytes. Since code units like +`\xE2` and `\x98` (from the UTF-8 encoding of `☃`) are both non-word bytes, +`(?-u:\B)` will match at the position between them. + +Thus, our approach of incrementing one codepoint at a time after seeing an +empty match is flawed because `(?-u:\B)` can result in an empty match that +splits a codepoint at a position past the starting point of a search. For +example, searching `(?-u:\B)` on `a☃` would produce the following matches: `[2, +2]`, `[3, 3]` and `[4, 4]`. The positions at `0` and `1` don't match because +they correspond to word boundaries since `a` is an ASCII word byte. + +So what did the old regex crate do to avoid this? It banned `(?-u:\B)` from +regexes that could match `&str`. That might sound extreme, but a lot of other +things were banned too. For example, all of `(?-u:.)`, `(?-u:[^a])` and +`(?-u:\W)` can match invalid UTF-8 too, including individual code units with a +codepoint. The key difference is that those expressions could never produce an +empty match. That ban happens when translating an `Ast` to an `Hir`, because +that process that reason about whether an `Hir` can produce *non-empty* matches +at invalid UTF-8 boundaries. Bottom line though is that we side-stepped the +`(?-u:\B)` issue by banning it. + +If banning `(?-u:\B)` were the only issue with the old regex crate's approach, +then I probably would have kept it. `\B` is rarely used, so it's not such a big +deal to have to work-around it. However, the problem with the above approach +is that it doesn't compose. The logic for avoiding splitting a codepoint only +lived in the iterator, which means if anyone wants to implement their own +iterator over regex matches, they have to deal with this extremely subtle edge +case to get full correctness. + +Instead, in this crate, we take the approach of pushing this complexity down +to the lowest layers of each regex engine. The approach is pretty simple: + +* If this corner case doesn't apply, don't do anything. (For example, if UTF-8 +mode isn't enabled or if the regex cannot match the empty string.) +* If an empty match is reported, explicitly check if it splits a codepoint. +* If it doesn't, we're done, return the match. +* If it does, then ignore the match and re-run the search. +* Repeat the above process until the end of the haystack is reached or a match +is found that doesn't split a codepoint or isn't zero width. + +And that's pretty much what this module provides. Every regex engine uses these +methods in their lowest level public APIs, but just above the layer where +their internal engine is used. That way, all regex engines can be arbitrarily +composed without worrying about handling this case, and iterators don't need to +handle it explicitly. + +(It turns out that a new feature I added, support for changing the line +terminator in a regex to any arbitrary byte, also provokes the above problem. +Namely, the byte could be invalid UTF-8 or a UTF-8 continuation byte. So that +support would need to be limited or banned when UTF-8 mode is enabled, just +like we did for `(?-u:\B)`. But thankfully our more robust approach in this +crate handles that case just fine too.) +*/ + +use crate::util::search::{Input, MatchError}; + +#[cold] +#[inline(never)] +pub(crate) fn skip_splits_fwd<T, F>( + input: &Input<'_>, + init_value: T, + match_offset: usize, + find: F, +) -> Result<Option<T>, MatchError> +where + F: FnMut(&Input<'_>) -> Result<Option<(T, usize)>, MatchError>, +{ + skip_splits(true, input, init_value, match_offset, find) +} + +#[cold] +#[inline(never)] +pub(crate) fn skip_splits_rev<T, F>( + input: &Input<'_>, + init_value: T, + match_offset: usize, + find: F, +) -> Result<Option<T>, MatchError> +where + F: FnMut(&Input<'_>) -> Result<Option<(T, usize)>, MatchError>, +{ + skip_splits(false, input, init_value, match_offset, find) +} + +fn skip_splits<T, F>( + forward: bool, + input: &Input<'_>, + init_value: T, + mut match_offset: usize, + mut find: F, +) -> Result<Option<T>, MatchError> +where + F: FnMut(&Input<'_>) -> Result<Option<(T, usize)>, MatchError>, +{ + // If our config says to do an anchored search, then we're definitely + // done. We just need to determine whether we have a valid match or + // not. If we don't, then we're not allowed to continue, so we report + // no match. + // + // This is actually quite a subtle correctness thing. The key here is + // that if we got an empty match that splits a codepoint after doing an + // anchored search in UTF-8 mode, then that implies that we must have + // *started* the search at a location that splits a codepoint. This + // follows from the fact that if a match is reported from an anchored + // search, then the start offset of the match *must* match the start + // offset of the search. + // + // It also follows that no other non-empty match is possible. For + // example, you might write a regex like '(?:)|SOMETHING' and start its + // search in the middle of a codepoint. The first branch is an empty + // regex that will bubble up a match at the first position, and then + // get rejected here and report no match. But what if 'SOMETHING' could + // have matched? We reason that such a thing is impossible, because + // if it does, it must report a match that starts in the middle of a + // codepoint. This in turn implies that a match is reported whose span + // does not correspond to valid UTF-8, and this breaks the promise + // made when UTF-8 mode is enabled. (That promise *can* be broken, for + // example, by enabling UTF-8 mode but building an by hand NFA that + // produces non-empty matches that span invalid UTF-8. This is an unchecked + // but documented precondition violation of UTF-8 mode, and is documented + // to have unspecified behavior.) + // + // I believe this actually means that if an anchored search is run, and + // UTF-8 mode is enabled and the start position splits a codepoint, + // then it is correct to immediately report no match without even + // executing the regex engine. But it doesn't really seem worth writing + // out that case in every regex engine to save a tiny bit of work in an + // extremely pathological case, so we just handle it here. + if input.get_anchored().is_anchored() { + return Ok(if input.is_char_boundary(match_offset) { + Some(init_value) + } else { + None + }); + } + // Otherwise, we have an unanchored search, so just keep looking for + // matches until we have one that does not split a codepoint or we hit + // EOI. + let mut value = init_value; + let mut input = input.clone(); + while !input.is_char_boundary(match_offset) { + if forward { + // The unwrap is OK here because overflowing usize while + // iterating over a slice is impossible, at it would require + // a slice of length greater than isize::MAX, which is itself + // impossible. + input.set_start(input.start().checked_add(1).unwrap()); + } else { + input.set_end(match input.end().checked_sub(1) { + None => return Ok(None), + Some(end) => end, + }); + } + match find(&input)? { + None => return Ok(None), + Some((new_value, new_match_end)) => { + value = new_value; + match_offset = new_match_end; + } + } + } + Ok(Some(value)) +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/escape.rs b/third_party/crates/regex-automata-0.4.18/src/util/escape.rs new file mode 100644 index 0000000..9c5b72e --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/escape.rs @@ -0,0 +1,84 @@ +/*! +Provides convenience routines for escaping raw bytes. + +Since this crate tends to deal with `&[u8]` everywhere and the default +`Debug` implementation just shows decimal integers, it makes debugging those +representations quite difficult. This module provides types that show `&[u8]` +as if it were a string, with invalid UTF-8 escaped into its byte-by-byte hex +representation. +*/ + +use crate::util::utf8; + +/// Provides a convenient `Debug` implementation for a `u8`. +/// +/// The `Debug` impl treats the byte as an ASCII, and emits a human readable +/// representation of it. If the byte isn't ASCII, then it's emitted as a hex +/// escape sequence. +#[derive(Clone, Copy)] +pub struct DebugByte(pub u8); + +impl core::fmt::Debug for DebugByte { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + // Special case ASCII space. It's too hard to read otherwise, so + // put quotes around it. I sometimes wonder whether just '\x20' would + // be better... + if self.0 == b' ' { + return write!(f, "' '"); + } + // 10 bytes is enough to cover any output from ascii::escape_default. + let mut bytes = [0u8; 10]; + let mut len = 0; + for (i, mut b) in core::ascii::escape_default(self.0).enumerate() { + // capitalize \xab to \xAB + if i >= 2 && b'a' <= b && b <= b'f' { + b -= 32; + } + bytes[len] = b; + len += 1; + } + write!(f, "{}", core::str::from_utf8(&bytes[..len]).unwrap()) + } +} + +/// Provides a convenient `Debug` implementation for `&[u8]`. +/// +/// This generally works best when the bytes are presumed to be mostly UTF-8, +/// but will work for anything. For any bytes that aren't UTF-8, they are +/// emitted as hex escape sequences. +pub struct DebugHaystack<'a>(pub &'a [u8]); + +impl<'a> core::fmt::Debug for DebugHaystack<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "\"")?; + // This is a sad re-implementation of a similar impl found in bstr. + let mut bytes = self.0; + while let Some(result) = utf8::decode(bytes) { + let ch = match result { + Ok(ch) => ch, + Err(byte) => { + write!(f, r"\x{byte:02x}")?; + bytes = &bytes[1..]; + continue; + } + }; + bytes = &bytes[ch.len_utf8()..]; + match ch { + '\0' => write!(f, "\\0")?, + // ASCII control characters except \0, \n, \r, \t + '\x01'..='\x08' + | '\x0b' + | '\x0c' + | '\x0e'..='\x19' + | '\x7f' => { + write!(f, "\\x{:02x}", u32::from(ch))?; + } + '\n' | '\r' | '\t' | _ => { + write!(f, "{}", ch.escape_debug())?; + } + } + } + write!(f, "\"")?; + Ok(()) + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/int.rs b/third_party/crates/regex-automata-0.4.18/src/util/int.rs new file mode 100644 index 0000000..b726e93 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/int.rs @@ -0,0 +1,246 @@ +/*! +This module provides several integer oriented traits for converting between +both fixed size integers and integers whose size varies based on the target +(like `usize`). + +The driving design principle of this module is to attempt to centralize as many +`as` casts as possible here. And in particular, we separate casts into two +buckets: + +* Casts that we use for their truncating behavior. In this case, we use more +descriptive names, like `low_u32` and `high_u32`. +* Casts that we use for converting back-and-forth between `usize`. These +conversions are generally necessary because we often store indices in different +formats to save on memory, which requires converting to and from `usize`. In +this case, we very specifically do not want to overflow, and so the methods +defined here will panic if the `as` cast would be lossy in debug mode. (A +normal `as` cast will never panic!) + +For `as` casts between raw pointers, we use `cast`, so `as` isn't needed there. + +For regex engines, floating point is just never used, so we don't have to worry +about `as` casts for those. + +Otherwise, this module pretty much covers all of our `as` needs except for one +thing: const contexts. There are a select few places in this crate where we +still need to use `as` because const functions on traits aren't stable yet. +If we wind up significantly expanding our const footprint in this crate, it +might be worth defining free functions to handle those cases. But at the time +of writing, that just seemed like too much ceremony. Instead, I comment each +such use of `as` in a const context with a "fixme" notice. + +NOTE: for simplicity, we don't take target pointer width into account here for +`usize` conversions. Since we currently only panic in debug mode, skipping the +check when it can be proven it isn't needed at compile time doesn't really +matter. Now, if we wind up wanting to do as many checks as possible in release +mode, then we would want to skip those when we know the conversions are always +non-lossy. + +NOTE: this module isn't an exhaustive API. For example, we still use things +like `u64::from` where possible, or even `usize::try_from()` for when we do +explicitly want to panic or when we want to return an error for overflow. +*/ + +// We define a little more than what we need, but I'd rather just have +// everything via a consistent and uniform API then have holes. +#![allow(dead_code)] + +pub(crate) trait U8 { + fn as_usize(self) -> usize; +} + +impl U8 for u8 { + fn as_usize(self) -> usize { + usize::from(self) + } +} + +pub(crate) trait U16 { + fn as_usize(self) -> usize; + fn low_u8(self) -> u8; + fn high_u8(self) -> u8; +} + +impl U16 for u16 { + fn as_usize(self) -> usize { + usize::from(self) + } + + fn low_u8(self) -> u8 { + self as u8 + } + + fn high_u8(self) -> u8 { + (self >> 8) as u8 + } +} + +pub(crate) trait U32 { + fn as_usize(self) -> usize; + fn low_u8(self) -> u8; + fn low_u16(self) -> u16; + fn high_u16(self) -> u16; +} + +impl U32 for u32 { + fn as_usize(self) -> usize { + #[cfg(debug_assertions)] + { + usize::try_from(self).expect("u32 overflowed usize") + } + #[cfg(not(debug_assertions))] + { + self as usize + } + } + + fn low_u8(self) -> u8 { + self as u8 + } + + fn low_u16(self) -> u16 { + self as u16 + } + + fn high_u16(self) -> u16 { + (self >> 16) as u16 + } +} + +pub(crate) trait U64 { + fn as_usize(self) -> usize; + fn low_u8(self) -> u8; + fn low_u16(self) -> u16; + fn low_u32(self) -> u32; + fn high_u32(self) -> u32; +} + +impl U64 for u64 { + fn as_usize(self) -> usize { + #[cfg(debug_assertions)] + { + usize::try_from(self).expect("u64 overflowed usize") + } + #[cfg(not(debug_assertions))] + { + self as usize + } + } + + fn low_u8(self) -> u8 { + self as u8 + } + + fn low_u16(self) -> u16 { + self as u16 + } + + fn low_u32(self) -> u32 { + self as u32 + } + + fn high_u32(self) -> u32 { + (self >> 32) as u32 + } +} + +pub(crate) trait I32 { + fn as_usize(self) -> usize; + fn to_bits(self) -> u32; + fn from_bits(n: u32) -> i32; +} + +impl I32 for i32 { + fn as_usize(self) -> usize { + #[cfg(debug_assertions)] + { + usize::try_from(self).expect("i32 overflowed usize") + } + #[cfg(not(debug_assertions))] + { + self as usize + } + } + + fn to_bits(self) -> u32 { + self as u32 + } + + fn from_bits(n: u32) -> i32 { + n as i32 + } +} + +pub(crate) trait Usize { + fn as_u8(self) -> u8; + fn as_u16(self) -> u16; + fn as_u32(self) -> u32; + fn as_u64(self) -> u64; +} + +impl Usize for usize { + fn as_u8(self) -> u8 { + #[cfg(debug_assertions)] + { + u8::try_from(self).expect("usize overflowed u8") + } + #[cfg(not(debug_assertions))] + { + self as u8 + } + } + + fn as_u16(self) -> u16 { + #[cfg(debug_assertions)] + { + u16::try_from(self).expect("usize overflowed u16") + } + #[cfg(not(debug_assertions))] + { + self as u16 + } + } + + fn as_u32(self) -> u32 { + #[cfg(debug_assertions)] + { + u32::try_from(self).expect("usize overflowed u32") + } + #[cfg(not(debug_assertions))] + { + self as u32 + } + } + + fn as_u64(self) -> u64 { + #[cfg(debug_assertions)] + { + u64::try_from(self).expect("usize overflowed u64") + } + #[cfg(not(debug_assertions))] + { + self as u64 + } + } +} + +// Pointers aren't integers, but we convert pointers to integers to perform +// offset arithmetic in some places. (And no, we don't convert the integers +// back to pointers.) So add 'as_usize' conversions here too for completeness. +// +// These 'as' casts are actually okay because they're always non-lossy. But the +// idea here is to just try and remove as much 'as' as possible, particularly +// in this crate where we are being really paranoid about offsets and making +// sure we don't panic on inputs that might be untrusted. This way, the 'as' +// casts become easier to audit if they're all in one place, even when some of +// them are actually okay 100% of the time. + +pub(crate) trait Pointer { + fn as_usize(self) -> usize; +} + +impl<T> Pointer for *const T { + fn as_usize(self) -> usize { + self as usize + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/interpolate.rs b/third_party/crates/regex-automata-0.4.18/src/util/interpolate.rs new file mode 100644 index 0000000..2b851aa --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/interpolate.rs @@ -0,0 +1,576 @@ +/*! +Provides routines for interpolating capture group references. + +That is, if a replacement string contains references like `$foo` or `${foo1}`, +then they are replaced with the corresponding capture values for the groups +named `foo` and `foo1`, respectively. Similarly, syntax like `$1` and `${1}` +is supported as well, with `1` corresponding to a capture group index and not +a name. + +This module provides the free functions [`string`] and [`bytes`], which +interpolate Rust Unicode strings and byte strings, respectively. + +# Format + +These routines support two different kinds of capture references: unbraced and +braced. + +For the unbraced format, the format supported is `$ref` where `name` can be +any character in the class `[0-9A-Za-z_]`. `ref` is always the longest +possible parse. So for example, `$1a` corresponds to the capture group named +`1a` and not the capture group at index `1`. If `ref` matches `^[0-9]+$`, then +it is treated as a capture group index itself and not a name. + +For the braced format, the format supported is `${ref}` where `ref` can be any +sequence of bytes except for `}`. If no closing brace occurs, then it is not +considered a capture reference. As with the unbraced format, if `ref` matches +`^[0-9]+$`, then it is treated as a capture group index and not a name. + +The braced format is useful for exerting precise control over the name of the +capture reference. For example, `${1}a` corresponds to the capture group +reference `1` followed by the letter `a`, where as `$1a` (as mentioned above) +corresponds to the capture group reference `1a`. The braced format is also +useful for expressing capture group names that use characters not supported by +the unbraced format. For example, `${foo[bar].baz}` refers to the capture group +named `foo[bar].baz`. + +If a capture group reference is found and it does not refer to a valid capture +group, then it will be replaced with the empty string. + +To write a literal `$`, use `$$`. + +To be clear, and as exhibited via the type signatures in the routines in this +module, it is impossible for a replacement string to be invalid. A replacement +string may not have the intended semantics, but the interpolation procedure +itself can never fail. +*/ + +use alloc::{string::String, vec::Vec}; + +use crate::util::memchr::memchr; + +/// Accepts a replacement string and interpolates capture references with their +/// corresponding values. +/// +/// `append` should be a function that appends the string value of a capture +/// group at a particular index to the string given. If the capture group +/// index is invalid, then nothing should be appended. +/// +/// `name_to_index` should be a function that maps a capture group name to a +/// capture group index. If the given name doesn't exist, then `None` should +/// be returned. +/// +/// Finally, `dst` is where the final interpolated contents should be written. +/// If `replacement` contains no capture group references, then `dst` will be +/// equivalent to `replacement`. +/// +/// See the [module documentation](self) for details about the format +/// supported. +/// +/// # Example +/// +/// ``` +/// use regex_automata::util::interpolate; +/// +/// let mut dst = String::new(); +/// interpolate::string( +/// "foo $bar baz", +/// |index, dst| { +/// if index == 0 { +/// dst.push_str("BAR"); +/// } +/// }, +/// |name| { +/// if name == "bar" { +/// Some(0) +/// } else { +/// None +/// } +/// }, +/// &mut dst, +/// ); +/// assert_eq!("foo BAR baz", dst); +/// ``` +pub fn string( + mut replacement: &str, + mut append: impl FnMut(usize, &mut String), + mut name_to_index: impl FnMut(&str) -> Option<usize>, + dst: &mut String, +) { + while !replacement.is_empty() { + match memchr(b'$', replacement.as_bytes()) { + None => break, + Some(i) => { + dst.push_str(&replacement[..i]); + replacement = &replacement[i..]; + } + } + // Handle escaping of '$'. + if replacement.as_bytes().get(1).map_or(false, |&b| b == b'$') { + dst.push_str("$"); + replacement = &replacement[2..]; + continue; + } + debug_assert!(!replacement.is_empty()); + let cap_ref = match find_cap_ref(replacement.as_bytes()) { + Some(cap_ref) => cap_ref, + None => { + dst.push_str("$"); + replacement = &replacement[1..]; + continue; + } + }; + replacement = &replacement[cap_ref.end..]; + match cap_ref.cap { + Ref::Number(i) => append(i, dst), + Ref::Named(name) => { + if let Some(i) = name_to_index(name) { + append(i, dst); + } + } + } + } + dst.push_str(replacement); +} + +/// Accepts a replacement byte string and interpolates capture references with +/// their corresponding values. +/// +/// `append` should be a function that appends the byte string value of a +/// capture group at a particular index to the byte string given. If the +/// capture group index is invalid, then nothing should be appended. +/// +/// `name_to_index` should be a function that maps a capture group name to a +/// capture group index. If the given name doesn't exist, then `None` should +/// be returned. +/// +/// Finally, `dst` is where the final interpolated contents should be written. +/// If `replacement` contains no capture group references, then `dst` will be +/// equivalent to `replacement`. +/// +/// See the [module documentation](self) for details about the format +/// supported. +/// +/// # Example +/// +/// ``` +/// use regex_automata::util::interpolate; +/// +/// let mut dst = vec![]; +/// interpolate::bytes( +/// b"foo $bar baz", +/// |index, dst| { +/// if index == 0 { +/// dst.extend_from_slice(b"BAR"); +/// } +/// }, +/// |name| { +/// if name == "bar" { +/// Some(0) +/// } else { +/// None +/// } +/// }, +/// &mut dst, +/// ); +/// assert_eq!(&b"foo BAR baz"[..], dst); +/// ``` +pub fn bytes( + mut replacement: &[u8], + mut append: impl FnMut(usize, &mut Vec<u8>), + mut name_to_index: impl FnMut(&str) -> Option<usize>, + dst: &mut Vec<u8>, +) { + while !replacement.is_empty() { + match memchr(b'$', replacement) { + None => break, + Some(i) => { + dst.extend_from_slice(&replacement[..i]); + replacement = &replacement[i..]; + } + } + // Handle escaping of '$'. + if replacement.get(1).map_or(false, |&b| b == b'$') { + dst.push(b'$'); + replacement = &replacement[2..]; + continue; + } + debug_assert!(!replacement.is_empty()); + let cap_ref = match find_cap_ref(replacement) { + Some(cap_ref) => cap_ref, + None => { + dst.push(b'$'); + replacement = &replacement[1..]; + continue; + } + }; + replacement = &replacement[cap_ref.end..]; + match cap_ref.cap { + Ref::Number(i) => append(i, dst), + Ref::Named(name) => { + if let Some(i) = name_to_index(name) { + append(i, dst); + } + } + } + } + dst.extend_from_slice(replacement); +} + +/// `CaptureRef` represents a reference to a capture group inside some text. +/// The reference is either a capture group name or a number. +/// +/// It is also tagged with the position in the text following the +/// capture reference. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct CaptureRef<'a> { + cap: Ref<'a>, + end: usize, +} + +/// A reference to a capture group in some text. +/// +/// e.g., `$2`, `$foo`, `${foo}`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Ref<'a> { + Named(&'a str), + Number(usize), +} + +impl<'a> From<&'a str> for Ref<'a> { + fn from(x: &'a str) -> Ref<'a> { + Ref::Named(x) + } +} + +impl From<usize> for Ref<'static> { + fn from(x: usize) -> Ref<'static> { + Ref::Number(x) + } +} + +/// Parses a possible reference to a capture group name in the given text, +/// starting at the beginning of `replacement`. +/// +/// If no such valid reference could be found, None is returned. +/// +/// Note that this returns a "possible" reference because this routine doesn't +/// know whether the reference is to a valid group or not. If it winds up not +/// being a valid reference, then it should be replaced with the empty string. +fn find_cap_ref(replacement: &[u8]) -> Option<CaptureRef<'_>> { + let mut i = 0; + let rep: &[u8] = replacement; + if rep.len() <= 1 || rep[0] != b'$' { + return None; + } + i += 1; + if rep[i] == b'{' { + return find_cap_ref_braced(rep, i + 1); + } + let mut cap_end = i; + while rep.get(cap_end).copied().map_or(false, is_valid_cap_letter) { + cap_end += 1; + } + if cap_end == i { + return None; + } + // We just verified that the range 0..cap_end is valid ASCII, so it must + // therefore be valid UTF-8. If we really cared, we could avoid this UTF-8 + // check via an unchecked conversion or by parsing the number straight from + // &[u8]. + let cap = core::str::from_utf8(&rep[i..cap_end]) + .expect("valid UTF-8 capture name"); + Some(CaptureRef { + cap: match cap.parse::<usize>() { + Ok(i) => Ref::Number(i), + Err(_) => Ref::Named(cap), + }, + end: cap_end, + }) +} + +/// Looks for a braced reference, e.g., `${foo1}`. This assumes that an opening +/// brace has been found at `i-1` in `rep`. This then looks for a closing +/// brace and returns the capture reference within the brace. +fn find_cap_ref_braced(rep: &[u8], mut i: usize) -> Option<CaptureRef<'_>> { + assert_eq!(b'{', rep[i.checked_sub(1).unwrap()]); + let start = i; + while rep.get(i).map_or(false, |&b| b != b'}') { + i += 1; + } + if !rep.get(i).map_or(false, |&b| b == b'}') { + return None; + } + // When looking at braced names, we don't put any restrictions on the name, + // so it's possible it could be invalid UTF-8. But a capture group name + // can never be invalid UTF-8, so if we have invalid UTF-8, then we can + // safely return None. + let cap = match core::str::from_utf8(&rep[start..i]) { + Err(_) => return None, + Ok(cap) => cap, + }; + Some(CaptureRef { + cap: match cap.parse::<usize>() { + Ok(i) => Ref::Number(i), + Err(_) => Ref::Named(cap), + }, + end: i + 1, + }) +} + +/// Returns true if and only if the given byte is allowed in a capture name +/// written in non-brace form. +fn is_valid_cap_letter(b: u8) -> bool { + matches!(b, b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' | b'_') +} + +#[cfg(test)] +mod tests { + use alloc::{string::String, vec, vec::Vec}; + + use super::{find_cap_ref, CaptureRef}; + + macro_rules! find { + ($name:ident, $text:expr) => { + #[test] + fn $name() { + assert_eq!(None, find_cap_ref($text.as_bytes())); + } + }; + ($name:ident, $text:expr, $capref:expr) => { + #[test] + fn $name() { + assert_eq!(Some($capref), find_cap_ref($text.as_bytes())); + } + }; + } + + macro_rules! c { + ($name_or_number:expr, $pos:expr) => { + CaptureRef { cap: $name_or_number.into(), end: $pos } + }; + } + + find!(find_cap_ref1, "$foo", c!("foo", 4)); + find!(find_cap_ref2, "${foo}", c!("foo", 6)); + find!(find_cap_ref3, "$0", c!(0, 2)); + find!(find_cap_ref4, "$5", c!(5, 2)); + find!(find_cap_ref5, "$10", c!(10, 3)); + // See https://github.com/rust-lang/regex/pull/585 + // for more on characters following numbers + find!(find_cap_ref6, "$42a", c!("42a", 4)); + find!(find_cap_ref7, "${42}a", c!(42, 5)); + find!(find_cap_ref8, "${42"); + find!(find_cap_ref9, "${42 "); + find!(find_cap_ref10, " $0 "); + find!(find_cap_ref11, "$"); + find!(find_cap_ref12, " "); + find!(find_cap_ref13, ""); + find!(find_cap_ref14, "$1-$2", c!(1, 2)); + find!(find_cap_ref15, "$1_$2", c!("1_", 3)); + find!(find_cap_ref16, "$x-$y", c!("x", 2)); + find!(find_cap_ref17, "$x_$y", c!("x_", 3)); + find!(find_cap_ref18, "${#}", c!("#", 4)); + find!(find_cap_ref19, "${Z[}", c!("Z[", 5)); + find!(find_cap_ref20, "${¾}", c!("¾", 5)); + find!(find_cap_ref21, "${¾a}", c!("¾a", 6)); + find!(find_cap_ref22, "${a¾}", c!("a¾", 6)); + find!(find_cap_ref23, "${☃}", c!("☃", 6)); + find!(find_cap_ref24, "${a☃}", c!("a☃", 7)); + find!(find_cap_ref25, "${☃a}", c!("☃a", 7)); + find!(find_cap_ref26, "${名字}", c!("名字", 9)); + + fn interpolate_string( + mut name_to_index: Vec<(&'static str, usize)>, + caps: Vec<&'static str>, + replacement: &str, + ) -> String { + name_to_index.sort_by_key(|x| x.0); + + let mut dst = String::new(); + super::string( + replacement, + |i, dst| { + if let Some(&s) = caps.get(i) { + dst.push_str(s); + } + }, + |name| -> Option<usize> { + name_to_index + .binary_search_by_key(&name, |x| x.0) + .ok() + .map(|i| name_to_index[i].1) + }, + &mut dst, + ); + dst + } + + fn interpolate_bytes( + mut name_to_index: Vec<(&'static str, usize)>, + caps: Vec<&'static str>, + replacement: &str, + ) -> String { + name_to_index.sort_by_key(|x| x.0); + + let mut dst = vec![]; + super::bytes( + replacement.as_bytes(), + |i, dst| { + if let Some(&s) = caps.get(i) { + dst.extend_from_slice(s.as_bytes()); + } + }, + |name| -> Option<usize> { + name_to_index + .binary_search_by_key(&name, |x| x.0) + .ok() + .map(|i| name_to_index[i].1) + }, + &mut dst, + ); + String::from_utf8(dst).unwrap() + } + + macro_rules! interp { + ($name:ident, $map:expr, $caps:expr, $hay:expr, $expected:expr $(,)*) => { + #[test] + fn $name() { + assert_eq!( + $expected, + interpolate_string($map, $caps, $hay), + "interpolate::string failed", + ); + assert_eq!( + $expected, + interpolate_bytes($map, $caps, $hay), + "interpolate::bytes failed", + ); + } + }; + } + + interp!( + interp1, + vec![("foo", 2)], + vec!["", "", "xxx"], + "test $foo test", + "test xxx test", + ); + + interp!( + interp2, + vec![("foo", 2)], + vec!["", "", "xxx"], + "test$footest", + "test", + ); + + interp!( + interp3, + vec![("foo", 2)], + vec!["", "", "xxx"], + "test${foo}test", + "testxxxtest", + ); + + interp!( + interp4, + vec![("foo", 2)], + vec!["", "", "xxx"], + "test$2test", + "test", + ); + + interp!( + interp5, + vec![("foo", 2)], + vec!["", "", "xxx"], + "test${2}test", + "testxxxtest", + ); + + interp!( + interp6, + vec![("foo", 2)], + vec!["", "", "xxx"], + "test $$foo test", + "test $foo test", + ); + + interp!( + interp7, + vec![("foo", 2)], + vec!["", "", "xxx"], + "test $foo", + "test xxx", + ); + + interp!( + interp8, + vec![("foo", 2)], + vec!["", "", "xxx"], + "$foo test", + "xxx test", + ); + + interp!( + interp9, + vec![("bar", 1), ("foo", 2)], + vec!["", "yyy", "xxx"], + "test $bar$foo", + "test yyyxxx", + ); + + interp!( + interp10, + vec![("bar", 1), ("foo", 2)], + vec!["", "yyy", "xxx"], + "test $ test", + "test $ test", + ); + + interp!( + interp11, + vec![("bar", 1), ("foo", 2)], + vec!["", "yyy", "xxx"], + "test ${} test", + "test test", + ); + + interp!( + interp12, + vec![("bar", 1), ("foo", 2)], + vec!["", "yyy", "xxx"], + "test ${ } test", + "test test", + ); + + interp!( + interp13, + vec![("bar", 1), ("foo", 2)], + vec!["", "yyy", "xxx"], + "test ${a b} test", + "test test", + ); + + interp!( + interp14, + vec![("bar", 1), ("foo", 2)], + vec!["", "yyy", "xxx"], + "test ${a} test", + "test test", + ); + + // This is a funny case where a braced reference is never closed, but + // within the unclosed braced reference, there is an unbraced reference. + // In this case, the braced reference is just treated literally and the + // unbraced reference is found. + interp!( + interp15, + vec![("bar", 1), ("foo", 2)], + vec!["", "yyy", "xxx"], + "test ${wat $bar ok", + "test ${wat yyy ok", + ); +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/iter.rs b/third_party/crates/regex-automata-0.4.18/src/util/iter.rs new file mode 100644 index 0000000..dcfa4a4 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/iter.rs @@ -0,0 +1,1022 @@ +/*! +Generic helpers for iteration of matches from a regex engine in a haystack. + +The principle type in this module is a [`Searcher`]. A `Searcher` provides +its own lower level iterator-like API in addition to methods for constructing +types that implement `Iterator`. The documentation for `Searcher` explains a +bit more about why these different APIs exist. + +Currently, this module supports iteration over any regex engine that works +with the [`HalfMatch`], [`Match`] or [`Captures`] types. +*/ + +#[cfg(feature = "alloc")] +use crate::util::captures::Captures; +use crate::util::search::{HalfMatch, Input, Match, MatchError}; + +/// A searcher for creating iterators and performing lower level iteration. +/// +/// This searcher encapsulates the logic required for finding all successive +/// non-overlapping matches in a haystack. In theory, iteration would look +/// something like this: +/// +/// 1. Setting the start position to `0`. +/// 2. Execute a regex search. If no match, end iteration. +/// 3. Report the match and set the start position to the end of the match. +/// 4. Go back to (2). +/// +/// And if this were indeed the case, it's likely that `Searcher` wouldn't +/// exist. Unfortunately, because a regex may match the empty string, the above +/// logic won't work for all possible regexes. Namely, if an empty match is +/// found, then step (3) would set the start position of the search to the +/// position it was at. Thus, iteration would never end. +/// +/// Instead, a `Searcher` knows how to detect these cases and forcefully +/// advance iteration in the case of an empty match that overlaps with a +/// previous match. +/// +/// If you know that your regex cannot match any empty string, then the simple +/// algorithm described above will work correctly. +/// +/// When possible, prefer the iterators defined on the regex engine you're +/// using. This tries to abstract over the regex engine and is thus a bit more +/// unwieldy to use. +/// +/// In particular, a `Searcher` is not itself an iterator. Instead, it provides +/// `advance` routines that permit moving the search along explicitly. It also +/// provides various routines, like [`Searcher::into_matches_iter`], that +/// accept a closure (representing how a regex engine executes a search) and +/// returns a conventional iterator. +/// +/// The lifetime parameters come from the [`Input`] type passed to +/// [`Searcher::new`]: +/// +/// * `'h` is the lifetime of the underlying haystack. +/// +/// # Searcher vs Iterator +/// +/// Why does a search type with "advance" APIs exist at all when we also have +/// iterators? Unfortunately, the reasoning behind this split is a complex +/// combination of the following things: +/// +/// 1. While many of the regex engines expose their own iterators, it is also +/// nice to expose this lower level iteration helper because it permits callers +/// to provide their own `Input` configuration. Moreover, a `Searcher` can work +/// with _any_ regex engine instead of only the ones defined in this crate. +/// This way, everyone benefits from a shared iteration implementation. +/// 2. There are many different regex engines that, while they have the same +/// match semantics, they have slightly different APIs. Iteration is just +/// complex enough to want to share code, and so we need a way of abstracting +/// over those different regex engines. While we could define a new trait that +/// describes any regex engine search API, it would wind up looking very close +/// to a closure. While there may still be reasons for the more generic trait +/// to exist, for now and for the purposes of iteration, we use a closure. +/// Closures also provide a lot of easy flexibility at the call site, in that +/// they permit the caller to borrow any kind of state they want for use during +/// each search call. +/// 3. As a result of using closures, and because closures are anonymous types +/// that cannot be named, it is difficult to encapsulate them without both +/// costs to speed and added complexity to the public API. For example, in +/// defining an iterator type like +/// [`dfa::regex::FindMatches`](crate::dfa::regex::FindMatches), +/// if we use a closure internally, it's not possible to name this type in the +/// return type of the iterator constructor. Thus, the only way around it is +/// to erase the type by boxing it and turning it into a `Box<dyn FnMut ...>`. +/// This boxed closure is unlikely to be inlined _and_ it infects the public +/// API in subtle ways. Namely, unless you declare the closure as implementing +/// `Send` and `Sync`, then the resulting iterator type won't implement it +/// either. But there are practical issues with requiring the closure to +/// implement `Send` and `Sync` that result in other API complexities that +/// are beyond the scope of this already long exposition. +/// 4. Some regex engines expose more complex match information than just +/// "which pattern matched" and "at what offsets." For example, the PikeVM +/// exposes match spans for each capturing group that participated in the +/// match. In such cases, it can be quite beneficial to reuse the capturing +/// group allocation on subsequent searches. A proper iterator doesn't permit +/// this API due to its interface, so it's useful to have something a bit lower +/// level that permits callers to amortize allocations while also reusing a +/// shared implementation of iteration. (See the documentation for +/// [`Searcher::advance`] for an example of using the "advance" API with the +/// PikeVM.) +/// +/// What this boils down to is that there are "advance" APIs which require +/// handing a closure to it for every call, and there are also APIs to create +/// iterators from a closure. The former are useful for _implementing_ +/// iterators or when you need more flexibility, while the latter are useful +/// for conveniently writing custom iterators on-the-fly. +/// +/// # Example: iterating with captures +/// +/// Several regex engines in this crate over convenient iterator APIs over +/// [`Captures`] values. To do so, this requires allocating a new `Captures` +/// value for each iteration step. This can perhaps be more costly than you +/// might want. Instead of implementing your own iterator to avoid that +/// cost (which can be a little subtle if you want to handle empty matches +/// correctly), you can use this `Searcher` to do it for you: +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::pikevm::PikeVM, +/// util::iter::Searcher, +/// Input, Span, +/// }; +/// +/// let re = PikeVM::new("foo(?P<numbers>[0-9]+)")?; +/// let haystack = "foo1 foo12 foo123"; +/// +/// let mut caps = re.create_captures(); +/// let mut cache = re.create_cache(); +/// let mut matches = vec![]; +/// let mut searcher = Searcher::new(Input::new(haystack)); +/// while let Some(_) = searcher.advance(|input| { +/// re.search(&mut cache, input, &mut caps); +/// Ok(caps.get_match()) +/// }) { +/// // The unwrap is OK since 'numbers' matches if the pattern matches. +/// matches.push(caps.get_group_by_name("numbers").unwrap()); +/// } +/// assert_eq!(matches, vec![ +/// Span::from(3..4), +/// Span::from(8..10), +/// Span::from(14..17), +/// ]); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Searcher<'h> { + /// The input parameters to give to each regex engine call. + /// + /// The start position of the search is mutated during iteration. + input: Input<'h>, + /// Records the end offset of the most recent match. This is necessary to + /// handle a corner case for preventing empty matches from overlapping with + /// the ending bounds of a prior match. + last_match_end: Option<usize>, +} + +impl<'h> Searcher<'h> { + /// Create a new fallible non-overlapping matches iterator. + /// + /// The given `input` provides the parameters (including the haystack), + /// while the `finder` represents a closure that calls the underlying regex + /// engine. The closure may borrow any additional state that is needed, + /// such as a prefilter scanner. + pub fn new(input: Input<'h>) -> Searcher<'h> { + Searcher { input, last_match_end: None } + } + + /// Returns the current `Input` used by this searcher. + /// + /// The `Input` returned is generally equivalent to the one given to + /// [`Searcher::new`], but its start position may be different to reflect + /// the start of the next search to be executed. + pub fn input<'s>(&'s self) -> &'s Input<'h> { + &self.input + } + + /// Return the next half match for an infallible search if one exists, and + /// advance to the next position. + /// + /// This is like `try_advance_half`, except errors are converted into + /// panics. + /// + /// # Panics + /// + /// If the given closure returns an error, then this panics. This is useful + /// when you know your underlying regex engine has been configured to not + /// return an error. + /// + /// # Example + /// + /// This example shows how to use a `Searcher` to iterate over all matches + /// when using a DFA, which only provides "half" matches. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// util::iter::Searcher, + /// HalfMatch, Input, + /// }; + /// + /// let re = DFA::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")?; + /// let mut cache = re.create_cache(); + /// + /// let input = Input::new("2010-03-14 2016-10-08 2020-10-22"); + /// let mut it = Searcher::new(input); + /// + /// let expected = Some(HalfMatch::must(0, 10)); + /// let got = it.advance_half(|input| re.try_search_fwd(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = Some(HalfMatch::must(0, 21)); + /// let got = it.advance_half(|input| re.try_search_fwd(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = Some(HalfMatch::must(0, 32)); + /// let got = it.advance_half(|input| re.try_search_fwd(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = None; + /// let got = it.advance_half(|input| re.try_search_fwd(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// This correctly moves iteration forward even when an empty match occurs: + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// util::iter::Searcher, + /// HalfMatch, Input, + /// }; + /// + /// let re = DFA::new(r"a|")?; + /// let mut cache = re.create_cache(); + /// + /// let input = Input::new("abba"); + /// let mut it = Searcher::new(input); + /// + /// let expected = Some(HalfMatch::must(0, 1)); + /// let got = it.advance_half(|input| re.try_search_fwd(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = Some(HalfMatch::must(0, 2)); + /// let got = it.advance_half(|input| re.try_search_fwd(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = Some(HalfMatch::must(0, 4)); + /// let got = it.advance_half(|input| re.try_search_fwd(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = None; + /// let got = it.advance_half(|input| re.try_search_fwd(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn advance_half<F>(&mut self, finder: F) -> Option<HalfMatch> + where + F: FnMut(&Input<'_>) -> Result<Option<HalfMatch>, MatchError>, + { + match self.try_advance_half(finder) { + Ok(m) => m, + Err(err) => panic!( + "unexpected regex half find error: {err}\n\ + to handle find errors, use 'try' or 'search' methods", + ), + } + } + + /// Return the next match for an infallible search if one exists, and + /// advance to the next position. + /// + /// The search is advanced even in the presence of empty matches by + /// forbidding empty matches from overlapping with any other match. + /// + /// This is like `try_advance`, except errors are converted into panics. + /// + /// # Panics + /// + /// If the given closure returns an error, then this panics. This is useful + /// when you know your underlying regex engine has been configured to not + /// return an error. + /// + /// # Example + /// + /// This example shows how to use a `Searcher` to iterate over all matches + /// when using a regex based on lazy DFAs: + /// + /// ``` + /// use regex_automata::{ + /// hybrid::regex::Regex, + /// util::iter::Searcher, + /// Match, Input, + /// }; + /// + /// let re = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")?; + /// let mut cache = re.create_cache(); + /// + /// let input = Input::new("2010-03-14 2016-10-08 2020-10-22"); + /// let mut it = Searcher::new(input); + /// + /// let expected = Some(Match::must(0, 0..10)); + /// let got = it.advance(|input| re.try_search(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = Some(Match::must(0, 11..21)); + /// let got = it.advance(|input| re.try_search(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = Some(Match::must(0, 22..32)); + /// let got = it.advance(|input| re.try_search(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// let expected = None; + /// let got = it.advance(|input| re.try_search(&mut cache, input)); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// This example shows the same as above, but with the PikeVM. This example + /// is useful because it shows how to use this API even when the regex + /// engine doesn't directly return a `Match`. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::iter::Searcher, + /// Match, Input, + /// }; + /// + /// let re = PikeVM::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// + /// let input = Input::new("2010-03-14 2016-10-08 2020-10-22"); + /// let mut it = Searcher::new(input); + /// + /// let expected = Some(Match::must(0, 0..10)); + /// let got = it.advance(|input| { + /// re.search(&mut cache, input, &mut caps); + /// Ok(caps.get_match()) + /// }); + /// // Note that if we wanted to extract capturing group spans, we could + /// // do that here with 'caps'. + /// assert_eq!(expected, got); + /// + /// let expected = Some(Match::must(0, 11..21)); + /// let got = it.advance(|input| { + /// re.search(&mut cache, input, &mut caps); + /// Ok(caps.get_match()) + /// }); + /// assert_eq!(expected, got); + /// + /// let expected = Some(Match::must(0, 22..32)); + /// let got = it.advance(|input| { + /// re.search(&mut cache, input, &mut caps); + /// Ok(caps.get_match()) + /// }); + /// assert_eq!(expected, got); + /// + /// let expected = None; + /// let got = it.advance(|input| { + /// re.search(&mut cache, input, &mut caps); + /// Ok(caps.get_match()) + /// }); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn advance<F>(&mut self, finder: F) -> Option<Match> + where + F: FnMut(&Input<'_>) -> Result<Option<Match>, MatchError>, + { + match self.try_advance(finder) { + Ok(m) => m, + Err(err) => panic!( + "unexpected regex find error: {err}\n\ + to handle find errors, use 'try' or 'search' methods", + ), + } + } + + /// Return the next half match for a fallible search if one exists, and + /// advance to the next position. + /// + /// This is like `advance_half`, except it permits callers to handle errors + /// during iteration. + #[inline] + pub fn try_advance_half<F>( + &mut self, + mut finder: F, + ) -> Result<Option<HalfMatch>, MatchError> + where + F: FnMut(&Input<'_>) -> Result<Option<HalfMatch>, MatchError>, + { + let mut m = match finder(&self.input)? { + None => return Ok(None), + Some(m) => m, + }; + if Some(m.offset()) == self.last_match_end { + m = match self.handle_overlapping_empty_half_match(m, finder)? { + None => return Ok(None), + Some(m) => m, + }; + } + self.input.set_start(m.offset()); + self.last_match_end = Some(m.offset()); + Ok(Some(m)) + } + + /// Return the next match for a fallible search if one exists, and advance + /// to the next position. + /// + /// This is like `advance`, except it permits callers to handle errors + /// during iteration. + #[inline] + pub fn try_advance<F>( + &mut self, + mut finder: F, + ) -> Result<Option<Match>, MatchError> + where + F: FnMut(&Input<'_>) -> Result<Option<Match>, MatchError>, + { + let mut m = match finder(&self.input)? { + None => return Ok(None), + Some(m) => m, + }; + if m.is_empty() && Some(m.end()) == self.last_match_end { + m = match self.handle_overlapping_empty_match(m, finder)? { + None => return Ok(None), + Some(m) => m, + }; + } + self.input.set_start(m.end()); + self.last_match_end = Some(m.end()); + Ok(Some(m)) + } + + /// Given a closure that executes a single search, return an iterator over + /// all successive non-overlapping half matches. + /// + /// The iterator returned yields result values. If the underlying regex + /// engine is configured to never return an error, consider calling + /// [`TryHalfMatchesIter::infallible`] to convert errors into panics. + /// + /// # Example + /// + /// This example shows how to use a `Searcher` to create a proper + /// iterator over half matches. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::dfa::DFA, + /// util::iter::Searcher, + /// HalfMatch, Input, + /// }; + /// + /// let re = DFA::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")?; + /// let mut cache = re.create_cache(); + /// + /// let input = Input::new("2010-03-14 2016-10-08 2020-10-22"); + /// let mut it = Searcher::new(input).into_half_matches_iter(|input| { + /// re.try_search_fwd(&mut cache, input) + /// }); + /// + /// let expected = Some(Ok(HalfMatch::must(0, 10))); + /// assert_eq!(expected, it.next()); + /// + /// let expected = Some(Ok(HalfMatch::must(0, 21))); + /// assert_eq!(expected, it.next()); + /// + /// let expected = Some(Ok(HalfMatch::must(0, 32))); + /// assert_eq!(expected, it.next()); + /// + /// let expected = None; + /// assert_eq!(expected, it.next()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn into_half_matches_iter<F>( + self, + finder: F, + ) -> TryHalfMatchesIter<'h, F> + where + F: FnMut(&Input<'_>) -> Result<Option<HalfMatch>, MatchError>, + { + TryHalfMatchesIter { it: self, finder } + } + + /// Given a closure that executes a single search, return an iterator over + /// all successive non-overlapping matches. + /// + /// The iterator returned yields result values. If the underlying regex + /// engine is configured to never return an error, consider calling + /// [`TryMatchesIter::infallible`] to convert errors into panics. + /// + /// # Example + /// + /// This example shows how to use a `Searcher` to create a proper + /// iterator over matches. + /// + /// ``` + /// use regex_automata::{ + /// hybrid::regex::Regex, + /// util::iter::Searcher, + /// Match, Input, + /// }; + /// + /// let re = Regex::new(r"[0-9]{4}-[0-9]{2}-[0-9]{2}")?; + /// let mut cache = re.create_cache(); + /// + /// let input = Input::new("2010-03-14 2016-10-08 2020-10-22"); + /// let mut it = Searcher::new(input).into_matches_iter(|input| { + /// re.try_search(&mut cache, input) + /// }); + /// + /// let expected = Some(Ok(Match::must(0, 0..10))); + /// assert_eq!(expected, it.next()); + /// + /// let expected = Some(Ok(Match::must(0, 11..21))); + /// assert_eq!(expected, it.next()); + /// + /// let expected = Some(Ok(Match::must(0, 22..32))); + /// assert_eq!(expected, it.next()); + /// + /// let expected = None; + /// assert_eq!(expected, it.next()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn into_matches_iter<F>(self, finder: F) -> TryMatchesIter<'h, F> + where + F: FnMut(&Input<'_>) -> Result<Option<Match>, MatchError>, + { + TryMatchesIter { it: self, finder } + } + + /// Given a closure that executes a single search, return an iterator over + /// all successive non-overlapping `Captures` values. + /// + /// The iterator returned yields result values. If the underlying regex + /// engine is configured to never return an error, consider calling + /// [`TryCapturesIter::infallible`] to convert errors into panics. + /// + /// Unlike the other iterator constructors, this accepts an initial + /// `Captures` value. This `Captures` value is reused for each search, and + /// the iterator implementation clones it before returning it. The caller + /// must provide this value because the iterator is purposely ignorant + /// of the underlying regex engine and thus doesn't know how to create + /// one itself. More to the point, a `Captures` value itself has a few + /// different constructors, which change which kind of information is + /// available to query in exchange for search performance. + /// + /// # Example + /// + /// This example shows how to use a `Searcher` to create a proper iterator + /// over `Captures` values, which provides access to all capturing group + /// spans for each match. + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// util::iter::Searcher, + /// Input, + /// }; + /// + /// let re = PikeVM::new( + /// r"(?P<y>[0-9]{4})-(?P<m>[0-9]{2})-(?P<d>[0-9]{2})", + /// )?; + /// let (mut cache, caps) = (re.create_cache(), re.create_captures()); + /// + /// let haystack = "2010-03-14 2016-10-08 2020-10-22"; + /// let input = Input::new(haystack); + /// let mut it = Searcher::new(input) + /// .into_captures_iter(caps, |input, caps| { + /// re.search(&mut cache, input, caps); + /// Ok(()) + /// }); + /// + /// let got = it.next().expect("first date")?; + /// let year = got.get_group_by_name("y").expect("must match"); + /// assert_eq!("2010", &haystack[year]); + /// + /// let got = it.next().expect("second date")?; + /// let month = got.get_group_by_name("m").expect("must match"); + /// assert_eq!("10", &haystack[month]); + /// + /// let got = it.next().expect("third date")?; + /// let day = got.get_group_by_name("d").expect("must match"); + /// assert_eq!("22", &haystack[day]); + /// + /// assert!(it.next().is_none()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "alloc")] + #[inline] + pub fn into_captures_iter<F>( + self, + caps: Captures, + finder: F, + ) -> TryCapturesIter<'h, F> + where + F: FnMut(&Input<'_>, &mut Captures) -> Result<(), MatchError>, + { + TryCapturesIter { it: self, caps, finder } + } + + /// Handles the special case of a match that begins where the previous + /// match ended. Without this special handling, it'd be possible to get + /// stuck where an empty match never results in forward progress. This + /// also makes it more consistent with how presiding general purpose regex + /// engines work. + #[cold] + #[inline(never)] + fn handle_overlapping_empty_half_match<F>( + &mut self, + _: HalfMatch, + mut finder: F, + ) -> Result<Option<HalfMatch>, MatchError> + where + F: FnMut(&Input<'_>) -> Result<Option<HalfMatch>, MatchError>, + { + // Since we are only here when 'm.offset()' matches the offset of the + // last match, it follows that this must have been an empty match. + // Since we both need to make progress *and* prevent overlapping + // matches, we discard this match and advance the search by 1. + // + // Note that this may start a search in the middle of a codepoint. The + // regex engines themselves are expected to deal with that and not + // report any matches within a codepoint if they are configured in + // UTF-8 mode. + self.input.set_start(self.input.start().checked_add(1).unwrap()); + finder(&self.input) + } + + /// Handles the special case of an empty match by ensuring that 1) the + /// iterator always advances and 2) empty matches never overlap with other + /// matches. + /// + /// (1) is necessary because we principally make progress by setting the + /// starting location of the next search to the ending location of the last + /// match. But if a match is empty, then this results in a search that does + /// not advance and thus does not terminate. + /// + /// (2) is not strictly necessary, but makes intuitive sense and matches + /// the presiding behavior of most general purpose regex engines. The + /// "intuitive sense" here is that we want to report NON-overlapping + /// matches. So for example, given the regex 'a|(?:)' against the haystack + /// 'a', without the special handling, you'd get the matches [0, 1) and [1, + /// 1), where the latter overlaps with the end bounds of the former. + /// + /// Note that we mark this cold and forcefully prevent inlining because + /// handling empty matches like this is extremely rare and does require + /// quite a bit of code, comparatively. Keeping this code out of the main + /// iterator function keeps it smaller and more amenable to inlining + /// itself. + #[cold] + #[inline(never)] + fn handle_overlapping_empty_match<F>( + &mut self, + m: Match, + mut finder: F, + ) -> Result<Option<Match>, MatchError> + where + F: FnMut(&Input<'_>) -> Result<Option<Match>, MatchError>, + { + assert!(m.is_empty()); + self.input.set_start(self.input.start().checked_add(1).unwrap()); + finder(&self.input) + } +} + +/// An iterator over all non-overlapping half matches for a fallible search. +/// +/// The iterator yields a `Result<HalfMatch, MatchError>` value until no more +/// matches could be found. +/// +/// The type parameters are as follows: +/// +/// * `F` represents the type of a closure that executes the search. +/// +/// The lifetime parameters come from the [`Input`] type: +/// +/// * `'h` is the lifetime of the underlying haystack. +/// +/// When possible, prefer the iterators defined on the regex engine you're +/// using. This tries to abstract over the regex engine and is thus a bit more +/// unwieldy to use. +/// +/// This iterator is created by [`Searcher::into_half_matches_iter`]. +pub struct TryHalfMatchesIter<'h, F> { + it: Searcher<'h>, + finder: F, +} + +impl<'h, F> TryHalfMatchesIter<'h, F> { + /// Return an infallible version of this iterator. + /// + /// Any item yielded that corresponds to an error results in a panic. This + /// is useful if your underlying regex engine is configured in a way that + /// it is guaranteed to never return an error. + pub fn infallible(self) -> HalfMatchesIter<'h, F> { + HalfMatchesIter(self) + } + + /// Returns the current `Input` used by this iterator. + /// + /// The `Input` returned is generally equivalent to the one used to + /// construct this iterator, but its start position may be different to + /// reflect the start of the next search to be executed. + pub fn input<'i>(&'i self) -> &'i Input<'h> { + self.it.input() + } +} + +impl<'h, F> Iterator for TryHalfMatchesIter<'h, F> +where + F: FnMut(&Input<'_>) -> Result<Option<HalfMatch>, MatchError>, +{ + type Item = Result<HalfMatch, MatchError>; + + #[inline] + fn next(&mut self) -> Option<Result<HalfMatch, MatchError>> { + self.it.try_advance_half(&mut self.finder).transpose() + } +} + +impl<'h, F> core::fmt::Debug for TryHalfMatchesIter<'h, F> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TryHalfMatchesIter") + .field("it", &self.it) + .field("finder", &"<closure>") + .finish() + } +} + +/// An iterator over all non-overlapping half matches for an infallible search. +/// +/// The iterator yields a [`HalfMatch`] value until no more matches could be +/// found. +/// +/// The type parameters are as follows: +/// +/// * `F` represents the type of a closure that executes the search. +/// +/// The lifetime parameters come from the [`Input`] type: +/// +/// * `'h` is the lifetime of the underlying haystack. +/// +/// When possible, prefer the iterators defined on the regex engine you're +/// using. This tries to abstract over the regex engine and is thus a bit more +/// unwieldy to use. +/// +/// This iterator is created by [`Searcher::into_half_matches_iter`] and +/// then calling [`TryHalfMatchesIter::infallible`]. +#[derive(Debug)] +pub struct HalfMatchesIter<'h, F>(TryHalfMatchesIter<'h, F>); + +impl<'h, F> HalfMatchesIter<'h, F> { + /// Returns the current `Input` used by this iterator. + /// + /// The `Input` returned is generally equivalent to the one used to + /// construct this iterator, but its start position may be different to + /// reflect the start of the next search to be executed. + pub fn input<'i>(&'i self) -> &'i Input<'h> { + self.0.it.input() + } +} + +impl<'h, F> Iterator for HalfMatchesIter<'h, F> +where + F: FnMut(&Input<'_>) -> Result<Option<HalfMatch>, MatchError>, +{ + type Item = HalfMatch; + + #[inline] + fn next(&mut self) -> Option<HalfMatch> { + match self.0.next()? { + Ok(m) => Some(m), + Err(err) => panic!( + "unexpected regex half find error: {err}\n\ + to handle find errors, use 'try' or 'search' methods", + ), + } + } +} + +/// An iterator over all non-overlapping matches for a fallible search. +/// +/// The iterator yields a `Result<Match, MatchError>` value until no more +/// matches could be found. +/// +/// The type parameters are as follows: +/// +/// * `F` represents the type of a closure that executes the search. +/// +/// The lifetime parameters come from the [`Input`] type: +/// +/// * `'h` is the lifetime of the underlying haystack. +/// +/// When possible, prefer the iterators defined on the regex engine you're +/// using. This tries to abstract over the regex engine and is thus a bit more +/// unwieldy to use. +/// +/// This iterator is created by [`Searcher::into_matches_iter`]. +pub struct TryMatchesIter<'h, F> { + it: Searcher<'h>, + finder: F, +} + +impl<'h, F> TryMatchesIter<'h, F> { + /// Return an infallible version of this iterator. + /// + /// Any item yielded that corresponds to an error results in a panic. This + /// is useful if your underlying regex engine is configured in a way that + /// it is guaranteed to never return an error. + pub fn infallible(self) -> MatchesIter<'h, F> { + MatchesIter(self) + } + + /// Returns the current `Input` used by this iterator. + /// + /// The `Input` returned is generally equivalent to the one used to + /// construct this iterator, but its start position may be different to + /// reflect the start of the next search to be executed. + pub fn input<'i>(&'i self) -> &'i Input<'h> { + self.it.input() + } +} + +impl<'h, F> Iterator for TryMatchesIter<'h, F> +where + F: FnMut(&Input<'_>) -> Result<Option<Match>, MatchError>, +{ + type Item = Result<Match, MatchError>; + + #[inline] + fn next(&mut self) -> Option<Result<Match, MatchError>> { + self.it.try_advance(&mut self.finder).transpose() + } +} + +impl<'h, F> core::fmt::Debug for TryMatchesIter<'h, F> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TryMatchesIter") + .field("it", &self.it) + .field("finder", &"<closure>") + .finish() + } +} + +/// An iterator over all non-overlapping matches for an infallible search. +/// +/// The iterator yields a [`Match`] value until no more matches could be found. +/// +/// The type parameters are as follows: +/// +/// * `F` represents the type of a closure that executes the search. +/// +/// The lifetime parameters come from the [`Input`] type: +/// +/// * `'h` is the lifetime of the underlying haystack. +/// +/// When possible, prefer the iterators defined on the regex engine you're +/// using. This tries to abstract over the regex engine and is thus a bit more +/// unwieldy to use. +/// +/// This iterator is created by [`Searcher::into_matches_iter`] and +/// then calling [`TryMatchesIter::infallible`]. +#[derive(Debug)] +pub struct MatchesIter<'h, F>(TryMatchesIter<'h, F>); + +impl<'h, F> MatchesIter<'h, F> { + /// Returns the current `Input` used by this iterator. + /// + /// The `Input` returned is generally equivalent to the one used to + /// construct this iterator, but its start position may be different to + /// reflect the start of the next search to be executed. + pub fn input<'i>(&'i self) -> &'i Input<'h> { + self.0.it.input() + } +} + +impl<'h, F> Iterator for MatchesIter<'h, F> +where + F: FnMut(&Input<'_>) -> Result<Option<Match>, MatchError>, +{ + type Item = Match; + + #[inline] + fn next(&mut self) -> Option<Match> { + match self.0.next()? { + Ok(m) => Some(m), + Err(err) => panic!( + "unexpected regex find error: {err}\n\ + to handle find errors, use 'try' or 'search' methods", + ), + } + } +} + +/// An iterator over all non-overlapping captures for a fallible search. +/// +/// The iterator yields a `Result<Captures, MatchError>` value until no more +/// matches could be found. +/// +/// The type parameters are as follows: +/// +/// * `F` represents the type of a closure that executes the search. +/// +/// The lifetime parameters come from the [`Input`] type: +/// +/// * `'h` is the lifetime of the underlying haystack. +/// +/// When possible, prefer the iterators defined on the regex engine you're +/// using. This tries to abstract over the regex engine and is thus a bit more +/// unwieldy to use. +/// +/// This iterator is created by [`Searcher::into_captures_iter`]. +#[cfg(feature = "alloc")] +pub struct TryCapturesIter<'h, F> { + it: Searcher<'h>, + caps: Captures, + finder: F, +} + +#[cfg(feature = "alloc")] +impl<'h, F> TryCapturesIter<'h, F> { + /// Return an infallible version of this iterator. + /// + /// Any item yielded that corresponds to an error results in a panic. This + /// is useful if your underlying regex engine is configured in a way that + /// it is guaranteed to never return an error. + pub fn infallible(self) -> CapturesIter<'h, F> { + CapturesIter(self) + } +} + +#[cfg(feature = "alloc")] +impl<'h, F> Iterator for TryCapturesIter<'h, F> +where + F: FnMut(&Input<'_>, &mut Captures) -> Result<(), MatchError>, +{ + type Item = Result<Captures, MatchError>; + + #[inline] + fn next(&mut self) -> Option<Result<Captures, MatchError>> { + let TryCapturesIter { ref mut it, ref mut caps, ref mut finder } = + *self; + let result = it + .try_advance(|input| { + (finder)(input, caps)?; + Ok(caps.get_match()) + }) + .transpose()?; + match result { + Ok(_) => Some(Ok(caps.clone())), + Err(err) => Some(Err(err)), + } + } +} + +#[cfg(feature = "alloc")] +impl<'h, F> core::fmt::Debug for TryCapturesIter<'h, F> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TryCapturesIter") + .field("it", &self.it) + .field("caps", &self.caps) + .field("finder", &"<closure>") + .finish() + } +} + +/// An iterator over all non-overlapping captures for an infallible search. +/// +/// The iterator yields a [`Captures`] value until no more matches could be +/// found. +/// +/// The type parameters are as follows: +/// +/// * `F` represents the type of a closure that executes the search. +/// +/// The lifetime parameters come from the [`Input`] type: +/// +/// * `'h` is the lifetime of the underlying haystack. +/// +/// When possible, prefer the iterators defined on the regex engine you're +/// using. This tries to abstract over the regex engine and is thus a bit more +/// unwieldy to use. +/// +/// This iterator is created by [`Searcher::into_captures_iter`] and then +/// calling [`TryCapturesIter::infallible`]. +#[cfg(feature = "alloc")] +#[derive(Debug)] +pub struct CapturesIter<'h, F>(TryCapturesIter<'h, F>); + +#[cfg(feature = "alloc")] +impl<'h, F> Iterator for CapturesIter<'h, F> +where + F: FnMut(&Input<'_>, &mut Captures) -> Result<(), MatchError>, +{ + type Item = Captures; + + #[inline] + fn next(&mut self) -> Option<Captures> { + match self.0.next()? { + Ok(m) => Some(m), + Err(err) => panic!( + "unexpected regex captures error: {err}\n\ + to handle find errors, use 'try' or 'search' methods", + ), + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/lazy.rs b/third_party/crates/regex-automata-0.4.18/src/util/lazy.rs new file mode 100644 index 0000000..c590338 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/lazy.rs @@ -0,0 +1,461 @@ +/*! +A lazily initialized value for safe sharing between threads. + +The principal type in this module is `Lazy`, which makes it easy to construct +values that are shared safely across multiple threads simultaneously. +*/ + +use core::fmt; + +/// A lazily initialized value that implements `Deref` for `T`. +/// +/// A `Lazy` takes an initialization function and permits callers from any +/// thread to access the result of that initialization function in a safe +/// manner. In effect, this permits one-time initialization of global resources +/// in a (possibly) multi-threaded program. +/// +/// This type and its functionality are available even when neither the `alloc` +/// nor the `std` features are enabled. In exchange, a `Lazy` does **not** +/// guarantee that the given `create` function is called at most once. It +/// might be called multiple times. Moreover, a call to `Lazy::get` (either +/// explicitly or implicitly via `Lazy`'s `Deref` impl) may block until a `T` +/// is available. +/// +/// This is very similar to `lazy_static` or `once_cell`, except it doesn't +/// guarantee that the initialization function will be run once and it works +/// in no-alloc no-std environments. With that said, if you need stronger +/// guarantees or a more flexible API, then it is recommended to use either +/// `lazy_static` or `once_cell`. +/// +/// # Warning: may use a spin lock +/// +/// When this crate is compiled _without_ the `alloc` feature, then this type +/// may used a spin lock internally. This can have subtle effects that may +/// be undesirable. See [Spinlocks Considered Harmful][spinharm] for a more +/// thorough treatment of this topic. +/// +/// [spinharm]: https://matklad.github.io/2020/01/02/spinlocks-considered-harmful.html +/// +/// # Example +/// +/// This type is useful for creating regexes once, and then using them from +/// multiple threads simultaneously without worrying about synchronization. +/// +/// ``` +/// use regex_automata::{dfa::regex::Regex, util::lazy::Lazy, Match}; +/// +/// static RE: Lazy<Regex> = Lazy::new(|| Regex::new("foo[0-9]+bar").unwrap()); +/// +/// let expected = Some(Match::must(0, 3..14)); +/// assert_eq!(expected, RE.find(b"zzzfoo12345barzzz")); +/// ``` +pub struct Lazy<T, F = fn() -> T>(lazy::Lazy<T, F>); + +impl<T, F> Lazy<T, F> { + /// Create a new `Lazy` value that is initialized via the given function. + /// + /// The `T` type is automatically inferred from the return type of the + /// `create` function given. + pub const fn new(create: F) -> Lazy<T, F> { + Lazy(lazy::Lazy::new(create)) + } +} + +impl<T, F: Fn() -> T> Lazy<T, F> { + /// Return a reference to the lazily initialized value. + /// + /// This routine may block if another thread is initializing a `T`. + /// + /// Note that given a `x` which has type `Lazy`, this must be called via + /// `Lazy::get(x)` and not `x.get()`. This routine is defined this way + /// because `Lazy` impls `Deref` with a target of `T`. + /// + /// # Panics + /// + /// This panics if the `create` function inside this lazy value panics. + /// If the panic occurred in another thread, then this routine _may_ also + /// panic (but is not guaranteed to do so). + pub fn get(this: &Lazy<T, F>) -> &T { + this.0.get() + } +} + +impl<T, F: Fn() -> T> core::ops::Deref for Lazy<T, F> { + type Target = T; + + fn deref(&self) -> &T { + Lazy::get(self) + } +} + +impl<T: fmt::Debug, F: Fn() -> T> fmt::Debug for Lazy<T, F> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +#[cfg(feature = "alloc")] +mod lazy { + use core::{ + fmt, + marker::PhantomData, + sync::atomic::{AtomicPtr, Ordering}, + }; + + use alloc::boxed::Box; + + /// A non-std lazy initialized value. + /// + /// This might run the initialization function more than once, but will + /// never block. + /// + /// I wish I could get these semantics into the non-alloc non-std Lazy + /// type below, but I'm not sure how to do it. If you can do an alloc, + /// then the implementation becomes very simple if you don't care about + /// redundant work precisely because a pointer can be atomically swapped. + /// + /// Perhaps making this approach work in the non-alloc non-std case + /// requires asking the caller for a pointer? It would make the API less + /// convenient I think. + pub(super) struct Lazy<T, F> { + data: AtomicPtr<T>, + create: F, + // This indicates to the compiler that this type can drop T. It's not + // totally clear how the absence of this marker could lead to trouble, + // but putting here doesn't have any downsides so we hedge until someone + // can from the Unsafe Working Group can tell us definitively that we + // don't need it. + // + // See: https://github.com/BurntSushi/regex-automata/issues/30 + owned: PhantomData<Box<T>>, + } + + // SAFETY: So long as T and &T (and F and &F) can themselves be safely + // shared among threads, so to can a Lazy<T, _>. Namely, the Lazy API only + // permits accessing a &T and initialization is free of data races. So if T + // is thread safe, then so to is Lazy<T, _>. + // + // We specifically require that T: Send in order for Lazy<T> to be Sync. + // Without that requirement, it's possible to send a T from one thread to + // another via Lazy's destructor. + // + // It's not clear whether we need F: Send+Sync for Lazy to be Sync. But + // we're conservative for now and keep both. + unsafe impl<T: Send + Sync, F: Send + Sync> Sync for Lazy<T, F> {} + + impl<T, F> Lazy<T, F> { + /// Create a new alloc but non-std lazy value that is racily + /// initialized. That is, the 'create' function may be called more than + /// once. + pub(super) const fn new(create: F) -> Lazy<T, F> { + Lazy { + data: AtomicPtr::new(core::ptr::null_mut()), + create, + owned: PhantomData, + } + } + } + + impl<T, F: Fn() -> T> Lazy<T, F> { + /// Get the underlying lazy value. If it hasn't been initialized + /// yet, then always attempt to initialize it (even if some other + /// thread is initializing it) and atomically attach it to this lazy + /// value before returning it. + pub(super) fn get(&self) -> &T { + if let Some(data) = self.poll() { + return data; + } + let data = (self.create)(); + let mut ptr = Box::into_raw(Box::new(data)); + // We attempt to stuff our initialized value into our atomic + // pointer. Upon success, we don't need to do anything. But if + // someone else beat us to the punch, then we need to make sure + // our newly created value is dropped. + let result = self.data.compare_exchange( + core::ptr::null_mut(), + ptr, + Ordering::AcqRel, + Ordering::Acquire, + ); + if let Err(old) = result { + // SAFETY: We created 'ptr' via Box::into_raw above, so turning + // it back into a Box via from_raw is safe. + drop(unsafe { Box::from_raw(ptr) }); + ptr = old; + } + // SAFETY: We just set the pointer above to a non-null value, even + // in the error case, and set it to a fully initialized value + // returned by 'create'. + unsafe { &*ptr } + } + + /// If this lazy value has been initialized successfully, then return + /// that value. Otherwise return None immediately. This never attempts + /// to run initialization itself. + fn poll(&self) -> Option<&T> { + let ptr = self.data.load(Ordering::Acquire); + if ptr.is_null() { + return None; + } + // SAFETY: We just checked that the pointer is not null. Since it's + // not null, it must have been fully initialized by 'get' at some + // point. + Some(unsafe { &*ptr }) + } + } + + impl<T: fmt::Debug, F: Fn() -> T> fmt::Debug for Lazy<T, F> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Lazy").field("data", &self.poll()).finish() + } + } + + impl<T, F> Drop for Lazy<T, F> { + fn drop(&mut self) { + let ptr = *self.data.get_mut(); + if !ptr.is_null() { + // SAFETY: We just checked that 'ptr' is not null. And since + // we have exclusive access, there are no races to worry about. + drop(unsafe { Box::from_raw(ptr) }); + } + } + } +} + +#[cfg(not(feature = "alloc"))] +mod lazy { + use core::{ + cell::Cell, + fmt, + mem::MaybeUninit, + panic::{RefUnwindSafe, UnwindSafe}, + sync::atomic::{AtomicU8, Ordering}, + }; + + /// Our 'Lazy' value can be in one of three states: + /// + /// * INIT is where it starts, and also ends up back here if the + /// 'create' routine panics. + /// * BUSY is where it sits while initialization is running in exactly + /// one thread. + /// * DONE is where it sits after 'create' has completed and 'data' has + /// been fully initialized. + const LAZY_STATE_INIT: u8 = 0; + const LAZY_STATE_BUSY: u8 = 1; + const LAZY_STATE_DONE: u8 = 2; + + /// A non-alloc non-std lazy initialized value. + /// + /// This guarantees initialization only happens once, but uses a spinlock + /// to block in the case of simultaneous access. Blocking occurs so that + /// one thread waits while another thread initializes the value. + /// + /// I would much rather have the semantics of the 'alloc' Lazy type above. + /// Namely, that we might run the initialization function more than once, + /// but we never otherwise block. However, I don't know how to do that in + /// a non-alloc non-std context. + pub(super) struct Lazy<T, F> { + state: AtomicU8, + create: Cell<Option<F>>, + data: Cell<MaybeUninit<T>>, + } + + // SAFETY: So long as T and &T (and F and &F) can themselves be safely + // shared among threads, so to can a Lazy<T, _>. Namely, the Lazy API only + // permits accessing a &T and initialization is free of data races. So if T + // is thread safe, then so to is Lazy<T, _>. + unsafe impl<T: Send + Sync, F: Send + Sync> Sync for Lazy<T, F> {} + // A reference to a Lazy is unwind safe because we specifically take + // precautions to poison all accesses to a Lazy if the caller-provided + // 'create' function panics. + impl<T: UnwindSafe, F: UnwindSafe + RefUnwindSafe> RefUnwindSafe + for Lazy<T, F> + { + } + + impl<T, F> Lazy<T, F> { + /// Create a new non-alloc non-std lazy value that is initialized + /// exactly once on first use using the given function. + pub(super) const fn new(create: F) -> Lazy<T, F> { + Lazy { + state: AtomicU8::new(LAZY_STATE_INIT), + create: Cell::new(Some(create)), + data: Cell::new(MaybeUninit::uninit()), + } + } + } + + impl<T, F: FnOnce() -> T> Lazy<T, F> { + /// Get the underlying lazy value. If it isn't been initialized + /// yet, then either initialize it or block until some other thread + /// initializes it. If the 'create' function given to Lazy::new panics + /// (even in another thread), then this panics too. + pub(super) fn get(&self) -> &T { + // This is effectively a spinlock. We loop until we enter a DONE + // state, and if possible, initialize it ourselves. The only way + // we exit the loop is if 'create' panics, we initialize 'data' or + // some other thread initializes 'data'. + // + // Yes, I have read spinlocks considered harmful[1]. And that + // article is why this spinlock is only active when 'alloc' isn't + // enabled. I did this because I don't think there is really + // another choice without 'alloc', other than not providing this at + // all. But I think that's a big bummer. + // + // [1]: https://matklad.github.io/2020/01/02/spinlocks-considered-harmful.html + while self.state.load(Ordering::Acquire) != LAZY_STATE_DONE { + // Check if we're the first ones to get here. If so, we'll be + // the ones who initialize. + let result = self.state.compare_exchange( + LAZY_STATE_INIT, + LAZY_STATE_BUSY, + Ordering::AcqRel, + Ordering::Acquire, + ); + // This means we saw the INIT state and nobody else can. So we + // must take responsibility for initializing. And by virtue of + // observing INIT, we have also told anyone else trying to + // get here that we are BUSY. If someone else sees BUSY, then + // they will spin until we finish initialization. + if let Ok(_) = result { + // Since we are guaranteed to be the only ones here, we + // know that 'create' is there... Unless someone else got + // here before us and 'create' panicked. In which case, + // 'self.create' is now 'None' and we forward the panic + // to the caller. (i.e., We implement poisoning.) + // + // SAFETY: Our use of 'self.state' guarantees that we are + // the only thread executing this line, and thus there are + // no races. + let create = unsafe { + (*self.create.as_ptr()).take().expect( + "Lazy's create function panicked, \ + preventing initialization, + poisoning current thread", + ) + }; + let guard = Guard { state: &self.state }; + // SAFETY: Our use of 'self.state' guarantees that we are + // the only thread executing this line, and thus there are + // no races. + unsafe { + (*self.data.as_ptr()).as_mut_ptr().write(create()); + } + // All is well. 'self.create' ran successfully, so we + // forget the guard. + core::mem::forget(guard); + // Everything is initialized, so we can declare success. + self.state.store(LAZY_STATE_DONE, Ordering::Release); + break; + } + core::hint::spin_loop(); + } + // We only get here if data is fully initialized, and thus poll + // will always return something. + self.poll().unwrap() + } + + /// If this lazy value has been initialized successfully, then return + /// that value. Otherwise return None immediately. This never blocks. + fn poll(&self) -> Option<&T> { + if self.state.load(Ordering::Acquire) == LAZY_STATE_DONE { + // SAFETY: The DONE state only occurs when data has been fully + // initialized. + Some(unsafe { &*(*self.data.as_ptr()).as_ptr() }) + } else { + None + } + } + } + + impl<T: fmt::Debug, F: FnMut() -> T> fmt::Debug for Lazy<T, F> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Lazy") + .field("state", &self.state.load(Ordering::Acquire)) + .field("create", &"<closure>") + .field("data", &self.poll()) + .finish() + } + } + + impl<T, F> Drop for Lazy<T, F> { + fn drop(&mut self) { + if *self.state.get_mut() == LAZY_STATE_DONE { + // SAFETY: state is DONE if and only if data has been fully + // initialized. At which point, it is safe to drop. + unsafe { + self.data.get_mut().assume_init_drop(); + } + } + } + } + + /// A guard that will reset a Lazy's state back to INIT when dropped. The + /// idea here is to 'forget' this guard on success. On failure (when a + /// panic occurs), the Drop impl runs and causes all in-progress and future + /// 'get' calls to panic. Without this guard, all in-progress and future + /// 'get' calls would spin forever. Crashing is much better than getting + /// stuck in an infinite loop. + struct Guard<'a> { + state: &'a AtomicU8, + } + + impl<'a> Drop for Guard<'a> { + fn drop(&mut self) { + // We force ourselves back into an INIT state. This will in turn + // cause any future 'get' calls to attempt calling 'self.create' + // again which will in turn panic because 'self.create' will now + // be 'None'. + self.state.store(LAZY_STATE_INIT, Ordering::Release); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_send<T: Send>() {} + fn assert_sync<T: Sync>() {} + fn assert_unwind<T: core::panic::UnwindSafe>() {} + fn assert_refunwind<T: core::panic::RefUnwindSafe>() {} + + #[test] + fn oibits() { + assert_send::<Lazy<u64>>(); + assert_sync::<Lazy<u64>>(); + assert_unwind::<Lazy<u64>>(); + assert_refunwind::<Lazy<u64>>(); + } + + // This is a regression test because we used to rely on the inferred Sync + // impl for the Lazy type defined above (for 'alloc' mode). In the + // inferred impl, it only requires that T: Sync for Lazy<T>: Sync. But + // if we have that, we can actually make use of the fact that Lazy<T> drops + // T to create a value on one thread and drop it on another. This *should* + // require T: Send, but our missing bounds before let it sneak by. + // + // Basically, this test should not compile, so we... comment it out. We + // don't have a great way of testing compile-fail tests right now. + // + // See: https://github.com/BurntSushi/regex-automata/issues/30 + /* + #[test] + fn sync_not_send() { + #[allow(dead_code)] + fn inner<T: Sync + Default>() { + let lazy = Lazy::new(move || T::default()); + std::thread::scope(|scope| { + scope.spawn(|| { + Lazy::get(&lazy); // We create T in this thread + }); + }); + // And drop in this thread. + drop(lazy); + // So we have send a !Send type over threads. (with some more + // legwork, its possible to even sneak the value out of drop + // through thread local) + } + } + */ +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/look.rs b/third_party/crates/regex-automata-0.4.18/src/util/look.rs new file mode 100644 index 0000000..63bc97c --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/look.rs @@ -0,0 +1,2546 @@ +/*! +Types and routines for working with look-around assertions. + +This module principally defines two types: + +* [`Look`] enumerates all of the assertions supported by this crate. +* [`LookSet`] provides a way to efficiently store a set of [`Look`] values. +* [`LookMatcher`] provides routines for checking whether a `Look` or a +`LookSet` matches at a particular position in a haystack. +*/ + +// LAMENTATION: Sadly, a lot of the API of `Look` and `LookSet` were basically +// copied verbatim from the regex-syntax crate. I would have no problems using +// the regex-syntax types and defining the matching routines (only found +// in this crate) as free functions, except the `Look` and `LookSet` types +// are used in lots of places. Including in places we expect to work when +// regex-syntax is *not* enabled, such as in the definition of the NFA itself. +// +// Thankfully the code we copy is pretty simple and there isn't much of it. +// Otherwise, the rest of this module deals with *matching* the assertions, +// which is not something that regex-syntax handles. + +use crate::util::{escape::DebugByte, utf8}; + +/// A look-around assertion. +/// +/// An assertion matches at a position between characters in a haystack. +/// Namely, it does not actually "consume" any input as most parts of a regular +/// expression do. Assertions are a way of stating that some property must be +/// true at a particular point during matching. +/// +/// For example, `(?m)^[a-z]+$` is a pattern that: +/// +/// * Scans the haystack for a position at which `(?m:^)` is satisfied. That +/// occurs at either the beginning of the haystack, or immediately following +/// a `\n` character. +/// * Looks for one or more occurrences of `[a-z]`. +/// * Once `[a-z]+` has matched as much as it can, an overall match is only +/// reported when `[a-z]+` stops just before a `\n`. +/// +/// So in this case, `abc` and `\nabc\n` match, but `\nabc1\n` does not. +/// +/// Assertions are also called "look-around," "look-behind" and "look-ahead." +/// Specifically, some assertions are look-behind (like `^`), other assertions +/// are look-ahead (like `$`) and yet other assertions are both look-ahead and +/// look-behind (like `\b`). +/// +/// # Assertions in an NFA +/// +/// An assertion in a [`thompson::NFA`](crate::nfa::thompson::NFA) can be +/// thought of as a conditional epsilon transition. That is, a matching engine +/// like the [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM) only permits +/// moving through conditional epsilon transitions when their condition +/// is satisfied at whatever position the `PikeVM` is currently at in the +/// haystack. +/// +/// How assertions are handled in a `DFA` is trickier, since a DFA does not +/// have epsilon transitions at all. In this case, they are compiled into the +/// automaton itself, at the expense of more states than what would be required +/// without an assertion. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Look { + /// Match the beginning of text. Specifically, this matches at the starting + /// position of the input. + Start = 1 << 0, + /// Match the end of text. Specifically, this matches at the ending + /// position of the input. + End = 1 << 1, + /// Match the beginning of a line or the beginning of text. Specifically, + /// this matches at the starting position of the input, or at the position + /// immediately following a `\n` character. + StartLF = 1 << 2, + /// Match the end of a line or the end of text. Specifically, this matches + /// at the end position of the input, or at the position immediately + /// preceding a `\n` character. + EndLF = 1 << 3, + /// Match the beginning of a line or the beginning of text. Specifically, + /// this matches at the starting position of the input, or at the position + /// immediately following either a `\r` or `\n` character, but never after + /// a `\r` when a `\n` follows. + StartCRLF = 1 << 4, + /// Match the end of a line or the end of text. Specifically, this matches + /// at the end position of the input, or at the position immediately + /// preceding a `\r` or `\n` character, but never before a `\n` when a `\r` + /// precedes it. + EndCRLF = 1 << 5, + /// Match an ASCII-only word boundary. That is, this matches a position + /// where the left adjacent character and right adjacent character + /// correspond to a word and non-word or a non-word and word character. + WordAscii = 1 << 6, + /// Match an ASCII-only negation of a word boundary. + WordAsciiNegate = 1 << 7, + /// Match a Unicode-aware word boundary. That is, this matches a position + /// where the left adjacent character and right adjacent character + /// correspond to a word and non-word or a non-word and word character. + WordUnicode = 1 << 8, + /// Match a Unicode-aware negation of a word boundary. + WordUnicodeNegate = 1 << 9, + /// Match the start of an ASCII-only word boundary. That is, this matches a + /// position at either the beginning of the haystack or where the previous + /// character is not a word character and the following character is a word + /// character. + WordStartAscii = 1 << 10, + /// Match the end of an ASCII-only word boundary. That is, this matches + /// a position at either the end of the haystack or where the previous + /// character is a word character and the following character is not a word + /// character. + WordEndAscii = 1 << 11, + /// Match the start of a Unicode word boundary. That is, this matches a + /// position at either the beginning of the haystack or where the previous + /// character is not a word character and the following character is a word + /// character. + WordStartUnicode = 1 << 12, + /// Match the end of a Unicode word boundary. That is, this matches a + /// position at either the end of the haystack or where the previous + /// character is a word character and the following character is not a word + /// character. + WordEndUnicode = 1 << 13, + /// Match the start half of an ASCII-only word boundary. That is, this + /// matches a position at either the beginning of the haystack or where the + /// previous character is not a word character. + WordStartHalfAscii = 1 << 14, + /// Match the end half of an ASCII-only word boundary. That is, this + /// matches a position at either the end of the haystack or where the + /// following character is not a word character. + WordEndHalfAscii = 1 << 15, + /// Match the start half of a Unicode word boundary. That is, this matches + /// a position at either the beginning of the haystack or where the + /// previous character is not a word character. + WordStartHalfUnicode = 1 << 16, + /// Match the end half of a Unicode word boundary. That is, this matches + /// a position at either the end of the haystack or where the following + /// character is not a word character. + WordEndHalfUnicode = 1 << 17, +} + +impl Look { + /// Flip the look-around assertion to its equivalent for reverse searches. + /// For example, `StartLF` gets translated to `EndLF`. + /// + /// Some assertions, such as `WordUnicode`, remain the same since they + /// match the same positions regardless of the direction of the search. + #[inline] + pub const fn reversed(self) -> Look { + match self { + Look::Start => Look::End, + Look::End => Look::Start, + Look::StartLF => Look::EndLF, + Look::EndLF => Look::StartLF, + Look::StartCRLF => Look::EndCRLF, + Look::EndCRLF => Look::StartCRLF, + Look::WordAscii => Look::WordAscii, + Look::WordAsciiNegate => Look::WordAsciiNegate, + Look::WordUnicode => Look::WordUnicode, + Look::WordUnicodeNegate => Look::WordUnicodeNegate, + Look::WordStartAscii => Look::WordEndAscii, + Look::WordEndAscii => Look::WordStartAscii, + Look::WordStartUnicode => Look::WordEndUnicode, + Look::WordEndUnicode => Look::WordStartUnicode, + Look::WordStartHalfAscii => Look::WordEndHalfAscii, + Look::WordEndHalfAscii => Look::WordStartHalfAscii, + Look::WordStartHalfUnicode => Look::WordEndHalfUnicode, + Look::WordEndHalfUnicode => Look::WordStartHalfUnicode, + } + } + + /// Return the underlying representation of this look-around enumeration + /// as an integer. Giving the return value to the [`Look::from_repr`] + /// constructor is guaranteed to return the same look-around variant that + /// one started with within a semver compatible release of this crate. + #[inline] + pub const fn as_repr(self) -> u32 { + // AFAIK, 'as' is the only way to zero-cost convert an int enum to an + // actual int. + self as u32 + } + + /// Given the underlying representation of a `Look` value, return the + /// corresponding `Look` value if the representation is valid. Otherwise + /// `None` is returned. + #[inline] + pub const fn from_repr(repr: u32) -> Option<Look> { + match repr { + 0b00_0000_0000_0000_0001 => Some(Look::Start), + 0b00_0000_0000_0000_0010 => Some(Look::End), + 0b00_0000_0000_0000_0100 => Some(Look::StartLF), + 0b00_0000_0000_0000_1000 => Some(Look::EndLF), + 0b00_0000_0000_0001_0000 => Some(Look::StartCRLF), + 0b00_0000_0000_0010_0000 => Some(Look::EndCRLF), + 0b00_0000_0000_0100_0000 => Some(Look::WordAscii), + 0b00_0000_0000_1000_0000 => Some(Look::WordAsciiNegate), + 0b00_0000_0001_0000_0000 => Some(Look::WordUnicode), + 0b00_0000_0010_0000_0000 => Some(Look::WordUnicodeNegate), + 0b00_0000_0100_0000_0000 => Some(Look::WordStartAscii), + 0b00_0000_1000_0000_0000 => Some(Look::WordEndAscii), + 0b00_0001_0000_0000_0000 => Some(Look::WordStartUnicode), + 0b00_0010_0000_0000_0000 => Some(Look::WordEndUnicode), + 0b00_0100_0000_0000_0000 => Some(Look::WordStartHalfAscii), + 0b00_1000_0000_0000_0000 => Some(Look::WordEndHalfAscii), + 0b01_0000_0000_0000_0000 => Some(Look::WordStartHalfUnicode), + 0b10_0000_0000_0000_0000 => Some(Look::WordEndHalfUnicode), + _ => None, + } + } + + /// Returns a convenient single codepoint representation of this + /// look-around assertion. Each assertion is guaranteed to be represented + /// by a distinct character. + /// + /// This is useful for succinctly representing a look-around assertion in + /// human friendly but succinct output intended for a programmer working on + /// regex internals. + #[inline] + pub const fn as_char(self) -> char { + match self { + Look::Start => 'A', + Look::End => 'z', + Look::StartLF => '^', + Look::EndLF => '$', + Look::StartCRLF => 'r', + Look::EndCRLF => 'R', + Look::WordAscii => 'b', + Look::WordAsciiNegate => 'B', + Look::WordUnicode => '𝛃', + Look::WordUnicodeNegate => '𝚩', + Look::WordStartAscii => '<', + Look::WordEndAscii => '>', + Look::WordStartUnicode => '〈', + Look::WordEndUnicode => '〉', + Look::WordStartHalfAscii => '◁', + Look::WordEndHalfAscii => '▷', + Look::WordStartHalfUnicode => '◀', + Look::WordEndHalfUnicode => '▶', + } + } +} + +/// LookSet is a memory-efficient set of look-around assertions. +/// +/// This is useful for efficiently tracking look-around assertions. For +/// example, a [`thompson::NFA`](crate::nfa::thompson::NFA) provides properties +/// that return `LookSet`s. +#[derive(Clone, Copy, Default, Eq, PartialEq)] +pub struct LookSet { + /// The underlying representation this set is exposed to make it possible + /// to store it somewhere efficiently. The representation is that + /// of a bitset, where each assertion occupies bit `i` where + /// `i = Look::as_repr()`. + /// + /// Note that users of this internal representation must permit the full + /// range of `u16` values to be represented. For example, even if the + /// current implementation only makes use of the 10 least significant bits, + /// it may use more bits in a future semver compatible release. + pub bits: u32, +} + +impl LookSet { + /// Create an empty set of look-around assertions. + #[inline] + pub fn empty() -> LookSet { + LookSet { bits: 0 } + } + + /// Create a full set of look-around assertions. + /// + /// This set contains all possible look-around assertions. + #[inline] + pub fn full() -> LookSet { + LookSet { bits: !0 } + } + + /// Create a look-around set containing the look-around assertion given. + /// + /// This is a convenience routine for creating an empty set and inserting + /// one look-around assertions. + #[inline] + pub fn singleton(look: Look) -> LookSet { + LookSet::empty().insert(look) + } + + /// Returns the total number of look-around assertions in this set. + #[inline] + pub fn len(self) -> usize { + // OK because max value always fits in a u8, which in turn always + // fits in a usize, regardless of target. + usize::try_from(self.bits.count_ones()).unwrap() + } + + /// Returns true if and only if this set is empty. + #[inline] + pub fn is_empty(self) -> bool { + self.len() == 0 + } + + /// Returns true if and only if the given look-around assertion is in this + /// set. + #[inline] + pub fn contains(self, look: Look) -> bool { + self.bits & look.as_repr() != 0 + } + + /// Returns true if and only if this set contains any anchor assertions. + /// This includes both "start/end of haystack" and "start/end of line." + #[inline] + pub fn contains_anchor(&self) -> bool { + self.contains_anchor_haystack() || self.contains_anchor_line() + } + + /// Returns true if and only if this set contains any "start/end of + /// haystack" anchors. This doesn't include "start/end of line" anchors. + #[inline] + pub fn contains_anchor_haystack(&self) -> bool { + self.contains(Look::Start) || self.contains(Look::End) + } + + /// Returns true if and only if this set contains any "start/end of line" + /// anchors. This doesn't include "start/end of haystack" anchors. This + /// includes both `\n` line anchors and CRLF (`\r\n`) aware line anchors. + #[inline] + pub fn contains_anchor_line(&self) -> bool { + self.contains(Look::StartLF) + || self.contains(Look::EndLF) + || self.contains(Look::StartCRLF) + || self.contains(Look::EndCRLF) + } + + /// Returns true if and only if this set contains any "start/end of line" + /// anchors that only treat `\n` as line terminators. This does not include + /// haystack anchors or CRLF aware line anchors. + #[inline] + pub fn contains_anchor_lf(&self) -> bool { + self.contains(Look::StartLF) || self.contains(Look::EndLF) + } + + /// Returns true if and only if this set contains any "start/end of line" + /// anchors that are CRLF-aware. This doesn't include "start/end of + /// haystack" or "start/end of line-feed" anchors. + #[inline] + pub fn contains_anchor_crlf(&self) -> bool { + self.contains(Look::StartCRLF) || self.contains(Look::EndCRLF) + } + + /// Returns true if and only if this set contains any word boundary or + /// negated word boundary assertions. This include both Unicode and ASCII + /// word boundaries. + #[inline] + pub fn contains_word(self) -> bool { + self.contains_word_unicode() || self.contains_word_ascii() + } + + /// Returns true if and only if this set contains any Unicode word boundary + /// or negated Unicode word boundary assertions. + #[inline] + pub fn contains_word_unicode(self) -> bool { + self.contains(Look::WordUnicode) + || self.contains(Look::WordUnicodeNegate) + || self.contains(Look::WordStartUnicode) + || self.contains(Look::WordEndUnicode) + || self.contains(Look::WordStartHalfUnicode) + || self.contains(Look::WordEndHalfUnicode) + } + + /// Returns true if and only if this set contains any ASCII word boundary + /// or negated ASCII word boundary assertions. + #[inline] + pub fn contains_word_ascii(self) -> bool { + self.contains(Look::WordAscii) + || self.contains(Look::WordAsciiNegate) + || self.contains(Look::WordStartAscii) + || self.contains(Look::WordEndAscii) + || self.contains(Look::WordStartHalfAscii) + || self.contains(Look::WordEndHalfAscii) + } + + /// Returns an iterator over all of the look-around assertions in this set. + #[inline] + pub fn iter(self) -> LookSetIter { + LookSetIter { set: self } + } + + /// Return a new set that is equivalent to the original, but with the given + /// assertion added to it. If the assertion is already in the set, then the + /// returned set is equivalent to the original. + #[inline] + pub fn insert(self, look: Look) -> LookSet { + LookSet { bits: self.bits | look.as_repr() } + } + + /// Updates this set in place with the result of inserting the given + /// assertion into this set. + #[inline] + pub fn set_insert(&mut self, look: Look) { + *self = self.insert(look); + } + + /// Return a new set that is equivalent to the original, but with the given + /// assertion removed from it. If the assertion is not in the set, then the + /// returned set is equivalent to the original. + #[inline] + pub fn remove(self, look: Look) -> LookSet { + LookSet { bits: self.bits & !look.as_repr() } + } + + /// Updates this set in place with the result of removing the given + /// assertion from this set. + #[inline] + pub fn set_remove(&mut self, look: Look) { + *self = self.remove(look); + } + + /// Returns a new set that is the result of subtracting the given set from + /// this set. + #[inline] + pub fn subtract(self, other: LookSet) -> LookSet { + LookSet { bits: self.bits & !other.bits } + } + + /// Updates this set in place with the result of subtracting the given set + /// from this set. + #[inline] + pub fn set_subtract(&mut self, other: LookSet) { + *self = self.subtract(other); + } + + /// Returns a new set that is the union of this and the one given. + #[inline] + pub fn union(self, other: LookSet) -> LookSet { + LookSet { bits: self.bits | other.bits } + } + + /// Updates this set in place with the result of unioning it with the one + /// given. + #[inline] + pub fn set_union(&mut self, other: LookSet) { + *self = self.union(other); + } + + /// Returns a new set that is the intersection of this and the one given. + #[inline] + pub fn intersect(self, other: LookSet) -> LookSet { + LookSet { bits: self.bits & other.bits } + } + + /// Updates this set in place with the result of intersecting it with the + /// one given. + #[inline] + pub fn set_intersect(&mut self, other: LookSet) { + *self = self.intersect(other); + } + + /// Return a `LookSet` from the slice given as a native endian 32-bit + /// integer. + /// + /// # Panics + /// + /// This panics if `slice.len() < 4`. + #[inline] + pub fn read_repr(slice: &[u8]) -> LookSet { + let bits = u32::from_ne_bytes(slice[..4].try_into().unwrap()); + LookSet { bits } + } + + /// Write a `LookSet` as a native endian 32-bit integer to the beginning + /// of the slice given. + /// + /// # Panics + /// + /// This panics if `slice.len() < 4`. + #[inline] + pub fn write_repr(self, slice: &mut [u8]) { + let raw = self.bits.to_ne_bytes(); + slice[0] = raw[0]; + slice[1] = raw[1]; + slice[2] = raw[2]; + slice[3] = raw[3]; + } + + /// Checks that all assertions in this set can be matched. + /// + /// Some assertions, such as Unicode word boundaries, require optional (but + /// enabled by default) tables that may not be available. If there are + /// assertions in this set that require tables that are not available, then + /// this will return an error. + /// + /// Specifically, this returns an error when the + /// `unicode-word-boundary` feature is _not_ enabled _and_ this set + /// contains a Unicode word boundary assertion. + /// + /// It can be useful to use this on the result of + /// [`NFA::look_set_any`](crate::nfa::thompson::NFA::look_set_any) + /// when building a matcher engine to ensure methods like + /// [`LookMatcher::matches_set`] do not panic at search time. + pub fn available(self) -> Result<(), UnicodeWordBoundaryError> { + if self.contains_word_unicode() { + UnicodeWordBoundaryError::check()?; + } + Ok(()) + } +} + +impl core::fmt::Debug for LookSet { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + if self.is_empty() { + return write!(f, "∅"); + } + for look in self.iter() { + write!(f, "{}", look.as_char())?; + } + Ok(()) + } +} + +/// An iterator over all look-around assertions in a [`LookSet`]. +/// +/// This iterator is created by [`LookSet::iter`]. +#[derive(Clone, Debug)] +pub struct LookSetIter { + set: LookSet, +} + +impl Iterator for LookSetIter { + type Item = Look; + + #[inline] + fn next(&mut self) -> Option<Look> { + if self.set.is_empty() { + return None; + } + // We'll never have more than u8::MAX distinct look-around assertions, + // so 'bit' will always fit into a u16. + let bit = u16::try_from(self.set.bits.trailing_zeros()).unwrap(); + let look = Look::from_repr(1 << bit)?; + self.set = self.set.remove(look); + Some(look) + } +} + +/// A matcher for look-around assertions. +/// +/// This matcher permits configuring aspects of how look-around assertions are +/// matched. +/// +/// # Example +/// +/// A `LookMatcher` can change the line terminator used for matching multi-line +/// anchors such as `(?m:^)` and `(?m:$)`. +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::{self, pikevm::PikeVM}, +/// util::look::LookMatcher, +/// Match, Input, +/// }; +/// +/// let mut lookm = LookMatcher::new(); +/// lookm.set_line_terminator(b'\x00'); +/// +/// let re = PikeVM::builder() +/// .thompson(thompson::Config::new().look_matcher(lookm)) +/// .build(r"(?m)^[a-z]+$")?; +/// let mut cache = re.create_cache(); +/// +/// // Multi-line assertions now use NUL as a terminator. +/// assert_eq!( +/// Some(Match::must(0, 1..4)), +/// re.find(&mut cache, b"\x00abc\x00"), +/// ); +/// // ... and \n is no longer recognized as a terminator. +/// assert_eq!( +/// None, +/// re.find(&mut cache, b"\nabc\n"), +/// ); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct LookMatcher { + lineterm: DebugByte, +} + +impl LookMatcher { + /// Creates a new default matcher for look-around assertions. + pub fn new() -> LookMatcher { + LookMatcher { lineterm: DebugByte(b'\n') } + } + + /// Sets the line terminator for use with `(?m:^)` and `(?m:$)`. + /// + /// Namely, instead of `^` matching after `\n` and `$` matching immediately + /// before a `\n`, this will cause it to match after and before the byte + /// given. + /// + /// It can occasionally be useful to use this to configure the line + /// terminator to the NUL byte when searching binary data. + /// + /// Note that this does not apply to CRLF-aware line anchors such as + /// `(?Rm:^)` and `(?Rm:$)`. CRLF-aware line anchors are hard-coded to + /// use `\r` and `\n`. + pub fn set_line_terminator(&mut self, byte: u8) -> &mut LookMatcher { + self.lineterm.0 = byte; + self + } + + /// Returns the line terminator that was configured for this matcher. + /// + /// If no line terminator was configured, then this returns `\n`. + /// + /// Note that the line terminator should only be used for matching `(?m:^)` + /// and `(?m:$)` assertions. It specifically should _not_ be used for + /// matching the CRLF aware assertions `(?Rm:^)` and `(?Rm:$)`. + pub fn get_line_terminator(&self) -> u8 { + self.lineterm.0 + } + + /// Returns true when the position `at` in `haystack` satisfies the given + /// look-around assertion. + /// + /// # Panics + /// + /// This panics when testing any Unicode word boundary assertion in this + /// set and when the Unicode word data is not available. Specifically, this + /// only occurs when the `unicode-word-boundary` feature is not enabled. + /// + /// Since it's generally expected that this routine is called inside of + /// a matching engine, callers should check the error condition when + /// building the matching engine. If there is a Unicode word boundary + /// in the matcher and the data isn't available, then the matcher should + /// fail to build. + /// + /// Callers can check the error condition with [`LookSet::available`]. + /// + /// This also may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn matches(&self, look: Look, haystack: &[u8], at: usize) -> bool { + self.matches_inline(look, haystack, at) + } + + /// Like `matches`, but forcefully inlined. + /// + /// # Panics + /// + /// This panics when testing any Unicode word boundary assertion in this + /// set and when the Unicode word data is not available. Specifically, this + /// only occurs when the `unicode-word-boundary` feature is not enabled. + /// + /// Since it's generally expected that this routine is called inside of + /// a matching engine, callers should check the error condition when + /// building the matching engine. If there is a Unicode word boundary + /// in the matcher and the data isn't available, then the matcher should + /// fail to build. + /// + /// Callers can check the error condition with [`LookSet::available`]. + /// + /// This also may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn matches_inline( + &self, + look: Look, + haystack: &[u8], + at: usize, + ) -> bool { + match look { + Look::Start => self.is_start(haystack, at), + Look::End => self.is_end(haystack, at), + Look::StartLF => self.is_start_lf(haystack, at), + Look::EndLF => self.is_end_lf(haystack, at), + Look::StartCRLF => self.is_start_crlf(haystack, at), + Look::EndCRLF => self.is_end_crlf(haystack, at), + Look::WordAscii => self.is_word_ascii(haystack, at), + Look::WordAsciiNegate => self.is_word_ascii_negate(haystack, at), + Look::WordUnicode => self.is_word_unicode(haystack, at).unwrap(), + Look::WordUnicodeNegate => { + self.is_word_unicode_negate(haystack, at).unwrap() + } + Look::WordStartAscii => self.is_word_start_ascii(haystack, at), + Look::WordEndAscii => self.is_word_end_ascii(haystack, at), + Look::WordStartUnicode => { + self.is_word_start_unicode(haystack, at).unwrap() + } + Look::WordEndUnicode => { + self.is_word_end_unicode(haystack, at).unwrap() + } + Look::WordStartHalfAscii => { + self.is_word_start_half_ascii(haystack, at) + } + Look::WordEndHalfAscii => { + self.is_word_end_half_ascii(haystack, at) + } + Look::WordStartHalfUnicode => { + self.is_word_start_half_unicode(haystack, at).unwrap() + } + Look::WordEndHalfUnicode => { + self.is_word_end_half_unicode(haystack, at).unwrap() + } + } + } + + /// Returns true when _all_ of the assertions in the given set match at the + /// given position in the haystack. + /// + /// # Panics + /// + /// This panics when testing any Unicode word boundary assertion in this + /// set and when the Unicode word data is not available. Specifically, this + /// only occurs when the `unicode-word-boundary` feature is not enabled. + /// + /// Since it's generally expected that this routine is called inside of + /// a matching engine, callers should check the error condition when + /// building the matching engine. If there is a Unicode word boundary + /// in the matcher and the data isn't available, then the matcher should + /// fail to build. + /// + /// Callers can check the error condition with [`LookSet::available`]. + /// + /// This also may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn matches_set( + &self, + set: LookSet, + haystack: &[u8], + at: usize, + ) -> bool { + self.matches_set_inline(set, haystack, at) + } + + /// Like `LookSet::matches`, but forcefully inlined for perf. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn matches_set_inline( + &self, + set: LookSet, + haystack: &[u8], + at: usize, + ) -> bool { + // This used to use LookSet::iter with Look::matches on each element, + // but that proved to be quite disastrous for perf. The manual "if + // the set has this assertion, check it" turns out to be quite a bit + // faster. + if set.contains(Look::Start) { + if !self.is_start(haystack, at) { + return false; + } + } + if set.contains(Look::End) { + if !self.is_end(haystack, at) { + return false; + } + } + if set.contains(Look::StartLF) { + if !self.is_start_lf(haystack, at) { + return false; + } + } + if set.contains(Look::EndLF) { + if !self.is_end_lf(haystack, at) { + return false; + } + } + if set.contains(Look::StartCRLF) { + if !self.is_start_crlf(haystack, at) { + return false; + } + } + if set.contains(Look::EndCRLF) { + if !self.is_end_crlf(haystack, at) { + return false; + } + } + if set.contains(Look::WordAscii) { + if !self.is_word_ascii(haystack, at) { + return false; + } + } + if set.contains(Look::WordAsciiNegate) { + if !self.is_word_ascii_negate(haystack, at) { + return false; + } + } + if set.contains(Look::WordUnicode) { + if !self.is_word_unicode(haystack, at).unwrap() { + return false; + } + } + if set.contains(Look::WordUnicodeNegate) { + if !self.is_word_unicode_negate(haystack, at).unwrap() { + return false; + } + } + if set.contains(Look::WordStartAscii) { + if !self.is_word_start_ascii(haystack, at) { + return false; + } + } + if set.contains(Look::WordEndAscii) { + if !self.is_word_end_ascii(haystack, at) { + return false; + } + } + if set.contains(Look::WordStartUnicode) { + if !self.is_word_start_unicode(haystack, at).unwrap() { + return false; + } + } + if set.contains(Look::WordEndUnicode) { + if !self.is_word_end_unicode(haystack, at).unwrap() { + return false; + } + } + if set.contains(Look::WordStartHalfAscii) { + if !self.is_word_start_half_ascii(haystack, at) { + return false; + } + } + if set.contains(Look::WordEndHalfAscii) { + if !self.is_word_end_half_ascii(haystack, at) { + return false; + } + } + if set.contains(Look::WordStartHalfUnicode) { + if !self.is_word_start_half_unicode(haystack, at).unwrap() { + return false; + } + } + if set.contains(Look::WordEndHalfUnicode) { + if !self.is_word_end_half_unicode(haystack, at).unwrap() { + return false; + } + } + true + } + + /// Split up the given byte classes into equivalence classes in a way that + /// is consistent with this look-around assertion. + #[cfg(feature = "alloc")] + pub(crate) fn add_to_byteset( + &self, + look: Look, + set: &mut crate::util::alphabet::ByteClassSet, + ) { + match look { + Look::Start | Look::End => {} + Look::StartLF | Look::EndLF => { + set.set_range(self.lineterm.0, self.lineterm.0); + } + Look::StartCRLF | Look::EndCRLF => { + set.set_range(b'\r', b'\r'); + set.set_range(b'\n', b'\n'); + } + Look::WordAscii + | Look::WordAsciiNegate + | Look::WordUnicode + | Look::WordUnicodeNegate + | Look::WordStartAscii + | Look::WordEndAscii + | Look::WordStartUnicode + | Look::WordEndUnicode + | Look::WordStartHalfAscii + | Look::WordEndHalfAscii + | Look::WordStartHalfUnicode + | Look::WordEndHalfUnicode => { + // We need to mark all ranges of bytes whose pairs result in + // evaluating \b differently. This isn't technically correct + // for Unicode word boundaries, but DFAs can't handle those + // anyway, and thus, the byte classes don't need to either + // since they are themselves only used in DFAs. + // + // FIXME: It seems like the calls to 'set_range' here are + // completely invariant, which means we could just hard-code + // them here without needing to write a loop. And we only need + // to do this dance at most once per regex. + // + // FIXME: Is this correct for \B? + let iswb = utf8::is_word_byte; + // This unwrap is OK because we guard every use of 'asu8' with + // a check that the input is <= 255. + let asu8 = |b: u16| u8::try_from(b).unwrap(); + let mut b1: u16 = 0; + let mut b2: u16; + while b1 <= 255 { + b2 = b1 + 1; + while b2 <= 255 && iswb(asu8(b1)) == iswb(asu8(b2)) { + b2 += 1; + } + // The guards above guarantee that b2 can never get any + // bigger. + assert!(b2 <= 256); + // Subtracting 1 from b2 is always OK because it is always + // at least 1 greater than b1, and the assert above + // guarantees that the asu8 conversion will succeed. + set.set_range(asu8(b1), asu8(b2.checked_sub(1).unwrap())); + b1 = b2; + } + } + } + } + + /// Returns true when [`Look::Start`] is satisfied `at` the given position + /// in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_start(&self, _haystack: &[u8], at: usize) -> bool { + at == 0 + } + + /// Returns true when [`Look::End`] is satisfied `at` the given position in + /// `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_end(&self, haystack: &[u8], at: usize) -> bool { + at == haystack.len() + } + + /// Returns true when [`Look::StartLF`] is satisfied `at` the given + /// position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_start_lf(&self, haystack: &[u8], at: usize) -> bool { + self.is_start(haystack, at) || haystack[at - 1] == self.lineterm.0 + } + + /// Returns true when [`Look::EndLF`] is satisfied `at` the given position + /// in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_end_lf(&self, haystack: &[u8], at: usize) -> bool { + self.is_end(haystack, at) || haystack[at] == self.lineterm.0 + } + + /// Returns true when [`Look::StartCRLF`] is satisfied `at` the given + /// position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_start_crlf(&self, haystack: &[u8], at: usize) -> bool { + self.is_start(haystack, at) + || haystack[at - 1] == b'\n' + || (haystack[at - 1] == b'\r' + && (at >= haystack.len() || haystack[at] != b'\n')) + } + + /// Returns true when [`Look::EndCRLF`] is satisfied `at` the given + /// position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_end_crlf(&self, haystack: &[u8], at: usize) -> bool { + self.is_end(haystack, at) + || haystack[at] == b'\r' + || (haystack[at] == b'\n' + && (at == 0 || haystack[at - 1] != b'\r')) + } + + /// Returns true when [`Look::WordAscii`] is satisfied `at` the given + /// position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_word_ascii(&self, haystack: &[u8], at: usize) -> bool { + let word_before = at > 0 && utf8::is_word_byte(haystack[at - 1]); + let word_after = + at < haystack.len() && utf8::is_word_byte(haystack[at]); + word_before != word_after + } + + /// Returns true when [`Look::WordAsciiNegate`] is satisfied `at` the given + /// position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_word_ascii_negate(&self, haystack: &[u8], at: usize) -> bool { + !self.is_word_ascii(haystack, at) + } + + /// Returns true when [`Look::WordUnicode`] is satisfied `at` the given + /// position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + /// + /// # Errors + /// + /// This returns an error when Unicode word boundary tables + /// are not available. Specifically, this only occurs when the + /// `unicode-word-boundary` feature is not enabled. + #[inline] + pub fn is_word_unicode( + &self, + haystack: &[u8], + at: usize, + ) -> Result<bool, UnicodeWordBoundaryError> { + let word_before = is_word_char::rev(haystack, at)?; + let word_after = is_word_char::fwd(haystack, at)?; + Ok(word_before != word_after) + } + + /// Returns true when [`Look::WordUnicodeNegate`] is satisfied `at` the + /// given position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + /// + /// # Errors + /// + /// This returns an error when Unicode word boundary tables + /// are not available. Specifically, this only occurs when the + /// `unicode-word-boundary` feature is not enabled. + #[inline] + pub fn is_word_unicode_negate( + &self, + haystack: &[u8], + at: usize, + ) -> Result<bool, UnicodeWordBoundaryError> { + // This is pretty subtle. Why do we need to do UTF-8 decoding here? + // Well... at time of writing, the is_word_char_{fwd,rev} routines will + // only return true if there is a valid UTF-8 encoding of a "word" + // codepoint, and false in every other case (including invalid UTF-8). + // This means that in regions of invalid UTF-8 (which might be a + // subset of valid UTF-8!), it would result in \B matching. While this + // would be questionable in the context of truly invalid UTF-8, it is + // *certainly* wrong to report match boundaries that split the encoding + // of a codepoint. So to work around this, we ensure that we can decode + // a codepoint on either side of `at`. If either direction fails, then + // we don't permit \B to match at all. + // + // Now, this isn't exactly optimal from a perf perspective. We could + // try and detect this in is_word_char::{fwd,rev}, but it's not clear + // if it's worth it. \B is, after all, rarely used. Even worse, + // is_word_char::{fwd,rev} could do its own UTF-8 decoding, and so this + // will wind up doing UTF-8 decoding twice. Ouch. We could fix this + // with more code complexity, but it just doesn't feel worth it for \B. + // + // And in particular, we do *not* have to do this with \b, because \b + // *requires* that at least one side of `at` be a "word" codepoint, + // which in turn implies one side of `at` must be valid UTF-8. This in + // turn implies that \b can never split a valid UTF-8 encoding of a + // codepoint. In the case where one side of `at` is truly invalid UTF-8 + // and the other side IS a word codepoint, then we want \b to match + // since it represents a valid UTF-8 boundary. It also makes sense. For + // example, you'd want \b\w+\b to match 'abc' in '\xFFabc\xFF'. + // + // Note also that this is not just '!is_word_unicode(..)' like it is + // for the ASCII case. For example, neither \b nor \B is satisfied + // within invalid UTF-8 sequences. + let word_before = at > 0 + && match utf8::decode_last(&haystack[..at]) { + None | Some(Err(_)) => return Ok(false), + Some(Ok(_)) => is_word_char::rev(haystack, at)?, + }; + let word_after = at < haystack.len() + && match utf8::decode(&haystack[at..]) { + None | Some(Err(_)) => return Ok(false), + Some(Ok(_)) => is_word_char::fwd(haystack, at)?, + }; + Ok(word_before == word_after) + } + + /// Returns true when [`Look::WordStartAscii`] is satisfied `at` the given + /// position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_word_start_ascii(&self, haystack: &[u8], at: usize) -> bool { + let word_before = at > 0 && utf8::is_word_byte(haystack[at - 1]); + let word_after = + at < haystack.len() && utf8::is_word_byte(haystack[at]); + !word_before && word_after + } + + /// Returns true when [`Look::WordEndAscii`] is satisfied `at` the given + /// position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_word_end_ascii(&self, haystack: &[u8], at: usize) -> bool { + let word_before = at > 0 && utf8::is_word_byte(haystack[at - 1]); + let word_after = + at < haystack.len() && utf8::is_word_byte(haystack[at]); + word_before && !word_after + } + + /// Returns true when [`Look::WordStartUnicode`] is satisfied `at` the + /// given position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + /// + /// # Errors + /// + /// This returns an error when Unicode word boundary tables + /// are not available. Specifically, this only occurs when the + /// `unicode-word-boundary` feature is not enabled. + #[inline] + pub fn is_word_start_unicode( + &self, + haystack: &[u8], + at: usize, + ) -> Result<bool, UnicodeWordBoundaryError> { + let word_before = is_word_char::rev(haystack, at)?; + let word_after = is_word_char::fwd(haystack, at)?; + Ok(!word_before && word_after) + } + + /// Returns true when [`Look::WordEndUnicode`] is satisfied `at` the + /// given position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + /// + /// # Errors + /// + /// This returns an error when Unicode word boundary tables + /// are not available. Specifically, this only occurs when the + /// `unicode-word-boundary` feature is not enabled. + #[inline] + pub fn is_word_end_unicode( + &self, + haystack: &[u8], + at: usize, + ) -> Result<bool, UnicodeWordBoundaryError> { + let word_before = is_word_char::rev(haystack, at)?; + let word_after = is_word_char::fwd(haystack, at)?; + Ok(word_before && !word_after) + } + + /// Returns true when [`Look::WordStartHalfAscii`] is satisfied `at` the + /// given position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_word_start_half_ascii( + &self, + haystack: &[u8], + at: usize, + ) -> bool { + let word_before = at > 0 && utf8::is_word_byte(haystack[at - 1]); + !word_before + } + + /// Returns true when [`Look::WordEndHalfAscii`] is satisfied `at` the + /// given position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + #[inline] + pub fn is_word_end_half_ascii(&self, haystack: &[u8], at: usize) -> bool { + let word_after = + at < haystack.len() && utf8::is_word_byte(haystack[at]); + !word_after + } + + /// Returns true when [`Look::WordStartHalfUnicode`] is satisfied `at` the + /// given position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + /// + /// # Errors + /// + /// This returns an error when Unicode word boundary tables + /// are not available. Specifically, this only occurs when the + /// `unicode-word-boundary` feature is not enabled. + #[inline] + pub fn is_word_start_half_unicode( + &self, + haystack: &[u8], + at: usize, + ) -> Result<bool, UnicodeWordBoundaryError> { + // See `is_word_unicode_negate` for why we need to do this. We don't + // need to do it for `is_word_start_unicode` because that guarantees + // that the position matched falls on a valid UTF-8 boundary given + // that the right side must be in \w. + let word_before = at > 0 + && match utf8::decode_last(&haystack[..at]) { + None | Some(Err(_)) => return Ok(false), + Some(Ok(_)) => is_word_char::rev(haystack, at)?, + }; + Ok(!word_before) + } + + /// Returns true when [`Look::WordEndHalfUnicode`] is satisfied `at` the + /// given position in `haystack`. + /// + /// # Panics + /// + /// This may panic when `at > haystack.len()`. Note that `at == + /// haystack.len()` is legal and guaranteed not to panic. + /// + /// # Errors + /// + /// This returns an error when Unicode word boundary tables + /// are not available. Specifically, this only occurs when the + /// `unicode-word-boundary` feature is not enabled. + #[inline] + pub fn is_word_end_half_unicode( + &self, + haystack: &[u8], + at: usize, + ) -> Result<bool, UnicodeWordBoundaryError> { + // See `is_word_unicode_negate` for why we need to do this. We don't + // need to do it for `is_word_end_unicode` because that guarantees + // that the position matched falls on a valid UTF-8 boundary given + // that the left side must be in \w. + let word_after = at < haystack.len() + && match utf8::decode(&haystack[at..]) { + None | Some(Err(_)) => return Ok(false), + Some(Ok(_)) => is_word_char::fwd(haystack, at)?, + }; + Ok(!word_after) + } +} + +impl Default for LookMatcher { + fn default() -> LookMatcher { + LookMatcher::new() + } +} + +/// An error that occurs when the Unicode-aware `\w` class is unavailable. +/// +/// This error can occur when the data tables necessary for the Unicode aware +/// Perl character class `\w` are unavailable. The `\w` class is used to +/// determine whether a codepoint is considered a word character or not when +/// determining whether a Unicode aware `\b` (or `\B`) matches at a particular +/// position. +/// +/// This error can only occur when the `unicode-word-boundary` feature is +/// disabled. +#[derive(Clone, Debug)] +pub struct UnicodeWordBoundaryError(()); + +impl UnicodeWordBoundaryError { + #[cfg(not(feature = "unicode-word-boundary"))] + pub(crate) fn new() -> UnicodeWordBoundaryError { + UnicodeWordBoundaryError(()) + } + + /// Returns an error if and only if Unicode word boundary data is + /// unavailable. + pub fn check() -> Result<(), UnicodeWordBoundaryError> { + is_word_char::check() + } +} + +#[cfg(feature = "std")] +impl std::error::Error for UnicodeWordBoundaryError {} + +impl core::fmt::Display for UnicodeWordBoundaryError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "Unicode-aware \\b and \\B are unavailable because the \ + requisite data tables are missing, please enable the \ + unicode-word-boundary feature" + ) + } +} + +// Below are FOUR different ways for checking whether whether a "word" +// codepoint exists at a particular position in the haystack. The four +// different approaches are, in order of preference: +// +// 1. Parse '\w', convert to an NFA, convert to a fully compiled DFA on the +// first call, and then use that DFA for all subsequent calls. +// 2. Do UTF-8 decoding and use regex_syntax::is_word_character if available. +// 3. Do UTF-8 decoding and use our own 'perl_word' table. +// 4. Return an error. +// +// The reason for all of these approaches is a combination of perf and +// permitting one to build regex-automata without the Unicode data necessary +// for handling Unicode-aware word boundaries. (In which case, '(?-u:\b)' would +// still work.) +// +// The DFA approach is the fastest, but it requires the regex parser, the +// NFA compiler, the DFA builder and the DFA search runtime. That's a lot to +// bring in, but if it's available, it's (probably) the best we can do. +// +// Approaches (2) and (3) are effectively equivalent, but (2) reuses the +// data in regex-syntax and avoids duplicating it in regex-automata. +// +// Finally, (4) unconditionally returns an error since the requisite data isn't +// available anywhere. +// +// There are actually more approaches possible that we didn't implement. For +// example, if the DFA builder is available but the syntax parser is not, we +// could technically hand construct our own NFA from the 'perl_word' data +// table. But to avoid some pretty hairy code duplication, we would in turn +// need to pull the UTF-8 compiler out of the NFA compiler. Yikes. +// +// A possibly more sensible alternative is to use a lazy DFA when the full +// DFA builder isn't available... +// +// Yet another choice would be to build the full DFA and then embed it into the +// source. Then we'd only need to bring in the DFA search runtime, which is +// considerably smaller than the DFA builder code. The problem here is that the +// Debian people have spooked me[1] into avoiding cyclic dependencies. Namely, +// we'd need to build regex-cli, which depends on regex-automata in order to +// build some part of regex-automata. But to be honest, something like this has +// to be allowed somehow? I just don't know what the right process is. +// +// There are perhaps other choices as well. Why did I stop at these 4? Because +// I wanted to preserve my sanity. I suspect I'll wind up adding the lazy DFA +// approach eventually, as the benefits of the DFA approach are somewhat +// compelling. The 'boundary-words-holmes' benchmark tests this. (Note that +// the commands below no longer work. If necessary, we should re-capitulate +// the benchmark from whole cloth in rebar.) +// +// $ regex-cli bench measure -f boundary-words-holmes -e pikevm > dfa.csv +// +// Then I changed the code below so that the util/unicode_data/perl_word table +// was used and re-ran the benchmark: +// +// $ regex-cli bench measure -f boundary-words-holmes -e pikevm > table.csv +// +// And compared them: +// +// $ regex-cli bench diff dfa.csv table.csv +// benchmark engine dfa table +// --------- ------ --- ----- +// internal/count/boundary-words-holmes regex/automata/pikevm 18.6 MB/s 12.9 MB/s +// +// Which is a nice improvement. +// +// UPDATE: It turns out that it takes approximately 22ms to build the reverse +// DFA for \w. (And about 3ms for the forward DFA.) It's probably not much in +// the grand scheme things, but that is a significant latency cost. So I'm not +// sure that's a good idea. I then tried using a lazy DFA instead, and that +// eliminated the overhead, but since the lazy DFA requires mutable working +// memory, that requires introducing a 'Cache' for every simultaneous call. +// +// I ended up deciding for now to just keep the "UTF-8 decode and check the +// table." The DFA and lazy DFA approaches are still below, but commented out. +// +// [1]: https://github.com/BurntSushi/ucd-generate/issues/11 + +/* +/// A module that looks for word codepoints using lazy DFAs. +#[cfg(all( + feature = "unicode-word-boundary", + feature = "syntax", + feature = "unicode-perl", + feature = "hybrid" +))] +mod is_word_char { + use alloc::vec::Vec; + + use crate::{ + hybrid::dfa::{Cache, DFA}, + nfa::thompson::NFA, + util::{lazy::Lazy, pool::Pool, primitives::StateID}, + Anchored, Input, + }; + + pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { + Ok(()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn fwd( + haystack: &[u8], + mut at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + static WORD: Lazy<DFA> = Lazy::new(|| DFA::new(r"\w").unwrap()); + static CACHE: Lazy<Pool<Cache>> = + Lazy::new(|| Pool::new(|| WORD.create_cache())); + let dfa = Lazy::get(&WORD); + let mut cache = Lazy::get(&CACHE).get(); + let mut sid = dfa + .start_state_forward( + &mut cache, + &Input::new("").anchored(Anchored::Yes), + ) + .unwrap(); + while at < haystack.len() { + let byte = haystack[at]; + sid = dfa.next_state(&mut cache, sid, byte).unwrap(); + at += 1; + if sid.is_tagged() { + if sid.is_match() { + return Ok(true); + } else if sid.is_dead() { + return Ok(false); + } + } + } + Ok(dfa.next_eoi_state(&mut cache, sid).unwrap().is_match()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn rev( + haystack: &[u8], + mut at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + static WORD: Lazy<DFA> = Lazy::new(|| { + DFA::builder() + .thompson(NFA::config().reverse(true)) + .build(r"\w") + .unwrap() + }); + static CACHE: Lazy<Pool<Cache>> = + Lazy::new(|| Pool::new(|| WORD.create_cache())); + let dfa = Lazy::get(&WORD); + let mut cache = Lazy::get(&CACHE).get(); + let mut sid = dfa + .start_state_reverse( + &mut cache, + &Input::new("").anchored(Anchored::Yes), + ) + .unwrap(); + while at > 0 { + at -= 1; + let byte = haystack[at]; + sid = dfa.next_state(&mut cache, sid, byte).unwrap(); + if sid.is_tagged() { + if sid.is_match() { + return Ok(true); + } else if sid.is_dead() { + return Ok(false); + } + } + } + Ok(dfa.next_eoi_state(&mut cache, sid).unwrap().is_match()) + } +} +*/ + +/* +/// A module that looks for word codepoints using fully compiled DFAs. +#[cfg(all( + feature = "unicode-word-boundary", + feature = "syntax", + feature = "unicode-perl", + feature = "dfa-build" +))] +mod is_word_char { + use alloc::vec::Vec; + + use crate::{ + dfa::{dense::DFA, Automaton, StartKind}, + nfa::thompson::NFA, + util::{lazy::Lazy, primitives::StateID}, + Anchored, Input, + }; + + pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { + Ok(()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn fwd( + haystack: &[u8], + mut at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + static WORD: Lazy<(DFA<Vec<u32>>, StateID)> = Lazy::new(|| { + let dfa = DFA::builder() + .configure(DFA::config().start_kind(StartKind::Anchored)) + .build(r"\w") + .unwrap(); + // OK because our regex has no look-around. + let start_id = dfa.universal_start_state(Anchored::Yes).unwrap(); + (dfa, start_id) + }); + let &(ref dfa, mut sid) = Lazy::get(&WORD); + while at < haystack.len() { + let byte = haystack[at]; + sid = dfa.next_state(sid, byte); + at += 1; + if dfa.is_special_state(sid) { + if dfa.is_match_state(sid) { + return Ok(true); + } else if dfa.is_dead_state(sid) { + return Ok(false); + } + } + } + Ok(dfa.is_match_state(dfa.next_eoi_state(sid))) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn rev( + haystack: &[u8], + mut at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + static WORD: Lazy<(DFA<Vec<u32>>, StateID)> = Lazy::new(|| { + let dfa = DFA::builder() + .configure(DFA::config().start_kind(StartKind::Anchored)) + // From ad hoc measurements, it looks like setting + // shrink==false is slightly faster than shrink==true. I kind + // of feel like this indicates that shrinking is probably a + // failure, although it can help in some cases. Sigh. + .thompson(NFA::config().reverse(true).shrink(false)) + .build(r"\w") + .unwrap(); + // OK because our regex has no look-around. + let start_id = dfa.universal_start_state(Anchored::Yes).unwrap(); + (dfa, start_id) + }); + let &(ref dfa, mut sid) = Lazy::get(&WORD); + while at > 0 { + at -= 1; + let byte = haystack[at]; + sid = dfa.next_state(sid, byte); + if dfa.is_special_state(sid) { + if dfa.is_match_state(sid) { + return Ok(true); + } else if dfa.is_dead_state(sid) { + return Ok(false); + } + } + } + Ok(dfa.is_match_state(dfa.next_eoi_state(sid))) + } +} +*/ + +/// A module that looks for word codepoints using regex-syntax's data tables. +#[cfg(all( + feature = "unicode-word-boundary", + feature = "syntax", + feature = "unicode-perl", +))] +mod is_word_char { + use crate::util::utf8; + + pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { + Ok(()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn fwd( + haystack: &[u8], + at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + Ok(match utf8::decode(&haystack[at..]) { + None | Some(Err(_)) => false, + Some(Ok(ch)) => is_word_character(ch), + }) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn rev( + haystack: &[u8], + at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + Ok(match utf8::decode_last(&haystack[..at]) { + None | Some(Err(_)) => false, + Some(Ok(ch)) => is_word_character(ch), + }) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_word_character(ch: char) -> bool { + regex_syntax::try_is_word_character(ch).expect( + "since unicode-word-boundary, syntax and unicode-perl \ + are all enabled, it is expected that \ + try_is_word_character succeeds", + ) + } +} + +/// A module that looks for word codepoints using regex-automata's data tables +/// (which are only compiled when regex-syntax's tables aren't available). +/// +/// Note that the cfg should match the one in src/util/unicode_data/mod.rs for +/// perl_word. +#[cfg(all( + feature = "unicode-word-boundary", + not(all(feature = "syntax", feature = "unicode-perl")), +))] +mod is_word_char { + use crate::util::utf8; + + pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { + Ok(()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn fwd( + haystack: &[u8], + at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + Ok(match utf8::decode(&haystack[at..]) { + None | Some(Err(_)) => false, + Some(Ok(ch)) => is_word_character(ch), + }) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn rev( + haystack: &[u8], + at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + Ok(match utf8::decode_last(&haystack[..at]) { + None | Some(Err(_)) => false, + Some(Ok(ch)) => is_word_character(ch), + }) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_word_character(c: char) -> bool { + use crate::util::{unicode_data::perl_word::PERL_WORD, utf8}; + + if u8::try_from(c).map_or(false, utf8::is_word_byte) { + return true; + } + PERL_WORD + .binary_search_by(|&(start, end)| { + use core::cmp::Ordering; + + if start <= c && c <= end { + Ordering::Equal + } else if start > c { + Ordering::Greater + } else { + Ordering::Less + } + }) + .is_ok() + } +} + +/// A module that always returns an error if Unicode word boundaries are +/// disabled. When this feature is disabled, then regex-automata will not +/// include its own data tables even if regex-syntax is disabled. +#[cfg(not(feature = "unicode-word-boundary"))] +mod is_word_char { + pub(super) fn check() -> Result<(), super::UnicodeWordBoundaryError> { + Err(super::UnicodeWordBoundaryError::new()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn fwd( + _bytes: &[u8], + _at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + Err(super::UnicodeWordBoundaryError::new()) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(super) fn rev( + _bytes: &[u8], + _at: usize, + ) -> Result<bool, super::UnicodeWordBoundaryError> { + Err(super::UnicodeWordBoundaryError::new()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! testlook { + ($look:expr, $haystack:expr, $at:expr) => { + LookMatcher::default().matches($look, $haystack.as_bytes(), $at) + }; + } + + #[test] + fn look_matches_start_line() { + let look = Look::StartLF; + + assert!(testlook!(look, "", 0)); + assert!(testlook!(look, "\n", 0)); + assert!(testlook!(look, "\n", 1)); + assert!(testlook!(look, "a", 0)); + assert!(testlook!(look, "\na", 1)); + + assert!(!testlook!(look, "a", 1)); + assert!(!testlook!(look, "a\na", 1)); + } + + #[test] + fn look_matches_end_line() { + let look = Look::EndLF; + + assert!(testlook!(look, "", 0)); + assert!(testlook!(look, "\n", 1)); + assert!(testlook!(look, "\na", 0)); + assert!(testlook!(look, "\na", 2)); + assert!(testlook!(look, "a\na", 1)); + + assert!(!testlook!(look, "a", 0)); + assert!(!testlook!(look, "\na", 1)); + assert!(!testlook!(look, "a\na", 0)); + assert!(!testlook!(look, "a\na", 2)); + } + + #[test] + fn look_matches_start_text() { + let look = Look::Start; + + assert!(testlook!(look, "", 0)); + assert!(testlook!(look, "\n", 0)); + assert!(testlook!(look, "a", 0)); + + assert!(!testlook!(look, "\n", 1)); + assert!(!testlook!(look, "\na", 1)); + assert!(!testlook!(look, "a", 1)); + assert!(!testlook!(look, "a\na", 1)); + } + + #[test] + fn look_matches_end_text() { + let look = Look::End; + + assert!(testlook!(look, "", 0)); + assert!(testlook!(look, "\n", 1)); + assert!(testlook!(look, "\na", 2)); + + assert!(!testlook!(look, "\na", 0)); + assert!(!testlook!(look, "a\na", 1)); + assert!(!testlook!(look, "a", 0)); + assert!(!testlook!(look, "\na", 1)); + assert!(!testlook!(look, "a\na", 0)); + assert!(!testlook!(look, "a\na", 2)); + } + + #[test] + #[cfg(all(not(miri), feature = "unicode-word-boundary"))] + fn look_matches_word_unicode() { + let look = Look::WordUnicode; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(testlook!(look, "a", 0)); + assert!(testlook!(look, "a", 1)); + assert!(testlook!(look, "a ", 1)); + assert!(testlook!(look, " a ", 1)); + assert!(testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. + assert!(testlook!(look, "𝛃", 0)); + assert!(testlook!(look, "𝛃", 4)); + assert!(testlook!(look, "𝛃 ", 4)); + assert!(testlook!(look, " 𝛃 ", 1)); + assert!(testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. + assert!(testlook!(look, "𝛃𐆀", 0)); + assert!(testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(!testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(!testlook!(look, "a ", 2)); + assert!(!testlook!(look, " a ", 0)); + assert!(!testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃b", 4)); + assert!(!testlook!(look, "𝛃 ", 5)); + assert!(!testlook!(look, " 𝛃 ", 0)); + assert!(!testlook!(look, " 𝛃 ", 6)); + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + assert!(!testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + fn look_matches_word_ascii() { + let look = Look::WordAscii; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(testlook!(look, "a", 0)); + assert!(testlook!(look, "a", 1)); + assert!(testlook!(look, "a ", 1)); + assert!(testlook!(look, " a ", 1)); + assert!(testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. Since this is + // an ASCII word boundary, none of these match. + assert!(!testlook!(look, "𝛃", 0)); + assert!(!testlook!(look, "𝛃", 4)); + assert!(!testlook!(look, "𝛃 ", 4)); + assert!(!testlook!(look, " 𝛃 ", 1)); + assert!(!testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. Again, since + // this is an ASCII word boundary, none of these match. + assert!(!testlook!(look, "𝛃𐆀", 0)); + assert!(!testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(!testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(!testlook!(look, "a ", 2)); + assert!(!testlook!(look, " a ", 0)); + assert!(!testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(testlook!(look, "𝛃b", 4)); + assert!(!testlook!(look, "𝛃 ", 5)); + assert!(!testlook!(look, " 𝛃 ", 0)); + assert!(!testlook!(look, " 𝛃 ", 6)); + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + assert!(!testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + #[cfg(all(not(miri), feature = "unicode-word-boundary"))] + fn look_matches_word_unicode_negate() { + let look = Look::WordUnicodeNegate; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(!testlook!(look, "a", 0)); + assert!(!testlook!(look, "a", 1)); + assert!(!testlook!(look, "a ", 1)); + assert!(!testlook!(look, " a ", 1)); + assert!(!testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃", 0)); + assert!(!testlook!(look, "𝛃", 4)); + assert!(!testlook!(look, "𝛃 ", 4)); + assert!(!testlook!(look, " 𝛃 ", 1)); + assert!(!testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 0)); + assert!(!testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(testlook!(look, "", 0)); + assert!(testlook!(look, "ab", 1)); + assert!(testlook!(look, "a ", 2)); + assert!(testlook!(look, " a ", 0)); + assert!(testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(testlook!(look, "𝛃b", 4)); + assert!(testlook!(look, "𝛃 ", 5)); + assert!(testlook!(look, " 𝛃 ", 0)); + assert!(testlook!(look, " 𝛃 ", 6)); + // These don't match because they could otherwise return an offset that + // splits the UTF-8 encoding of a codepoint. + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. These also don't + // match because they could otherwise return an offset that splits the + // UTF-8 encoding of a codepoint. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + // But this one does, since 𐆀 isn't a word codepoint, and 8 is the end + // of the haystack. So the "end" of the haystack isn't a word and 𐆀 + // isn't a word, thus, \B matches. + assert!(testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + fn look_matches_word_ascii_negate() { + let look = Look::WordAsciiNegate; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(!testlook!(look, "a", 0)); + assert!(!testlook!(look, "a", 1)); + assert!(!testlook!(look, "a ", 1)); + assert!(!testlook!(look, " a ", 1)); + assert!(!testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. Since this is + // an ASCII word boundary, none of these match. + assert!(testlook!(look, "𝛃", 0)); + assert!(testlook!(look, "𝛃", 4)); + assert!(testlook!(look, "𝛃 ", 4)); + assert!(testlook!(look, " 𝛃 ", 1)); + assert!(testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. Again, since + // this is an ASCII word boundary, none of these match. + assert!(testlook!(look, "𝛃𐆀", 0)); + assert!(testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(testlook!(look, "", 0)); + assert!(testlook!(look, "ab", 1)); + assert!(testlook!(look, "a ", 2)); + assert!(testlook!(look, " a ", 0)); + assert!(testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃b", 4)); + assert!(testlook!(look, "𝛃 ", 5)); + assert!(testlook!(look, " 𝛃 ", 0)); + assert!(testlook!(look, " 𝛃 ", 6)); + assert!(testlook!(look, "𝛃", 1)); + assert!(testlook!(look, "𝛃", 2)); + assert!(testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(testlook!(look, "𝛃𐆀", 1)); + assert!(testlook!(look, "𝛃𐆀", 2)); + assert!(testlook!(look, "𝛃𐆀", 3)); + assert!(testlook!(look, "𝛃𐆀", 5)); + assert!(testlook!(look, "𝛃𐆀", 6)); + assert!(testlook!(look, "𝛃𐆀", 7)); + assert!(testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + fn look_matches_word_start_ascii() { + let look = Look::WordStartAscii; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(testlook!(look, "a", 0)); + assert!(!testlook!(look, "a", 1)); + assert!(!testlook!(look, "a ", 1)); + assert!(testlook!(look, " a ", 1)); + assert!(!testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. Since this is + // an ASCII word boundary, none of these match. + assert!(!testlook!(look, "𝛃", 0)); + assert!(!testlook!(look, "𝛃", 4)); + assert!(!testlook!(look, "𝛃 ", 4)); + assert!(!testlook!(look, " 𝛃 ", 1)); + assert!(!testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. Again, since + // this is an ASCII word boundary, none of these match. + assert!(!testlook!(look, "𝛃𐆀", 0)); + assert!(!testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(!testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(!testlook!(look, "a ", 2)); + assert!(!testlook!(look, " a ", 0)); + assert!(!testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(testlook!(look, "𝛃b", 4)); + assert!(!testlook!(look, "b𝛃", 1)); + assert!(!testlook!(look, "𝛃 ", 5)); + assert!(!testlook!(look, " 𝛃 ", 0)); + assert!(!testlook!(look, " 𝛃 ", 6)); + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + assert!(!testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + fn look_matches_word_end_ascii() { + let look = Look::WordEndAscii; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(!testlook!(look, "a", 0)); + assert!(testlook!(look, "a", 1)); + assert!(testlook!(look, "a ", 1)); + assert!(!testlook!(look, " a ", 1)); + assert!(testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. Since this is + // an ASCII word boundary, none of these match. + assert!(!testlook!(look, "𝛃", 0)); + assert!(!testlook!(look, "𝛃", 4)); + assert!(!testlook!(look, "𝛃 ", 4)); + assert!(!testlook!(look, " 𝛃 ", 1)); + assert!(!testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. Again, since + // this is an ASCII word boundary, none of these match. + assert!(!testlook!(look, "𝛃𐆀", 0)); + assert!(!testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(!testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(!testlook!(look, "a ", 2)); + assert!(!testlook!(look, " a ", 0)); + assert!(!testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃b", 4)); + assert!(testlook!(look, "b𝛃", 1)); + assert!(!testlook!(look, "𝛃 ", 5)); + assert!(!testlook!(look, " 𝛃 ", 0)); + assert!(!testlook!(look, " 𝛃 ", 6)); + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + assert!(!testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + #[cfg(all(not(miri), feature = "unicode-word-boundary"))] + fn look_matches_word_start_unicode() { + let look = Look::WordStartUnicode; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(testlook!(look, "a", 0)); + assert!(!testlook!(look, "a", 1)); + assert!(!testlook!(look, "a ", 1)); + assert!(testlook!(look, " a ", 1)); + assert!(!testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. + assert!(testlook!(look, "𝛃", 0)); + assert!(!testlook!(look, "𝛃", 4)); + assert!(!testlook!(look, "𝛃 ", 4)); + assert!(testlook!(look, " 𝛃 ", 1)); + assert!(!testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. + assert!(testlook!(look, "𝛃𐆀", 0)); + assert!(!testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(!testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(!testlook!(look, "a ", 2)); + assert!(!testlook!(look, " a ", 0)); + assert!(!testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃b", 4)); + assert!(!testlook!(look, "b𝛃", 1)); + assert!(!testlook!(look, "𝛃 ", 5)); + assert!(!testlook!(look, " 𝛃 ", 0)); + assert!(!testlook!(look, " 𝛃 ", 6)); + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + assert!(!testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + #[cfg(all(not(miri), feature = "unicode-word-boundary"))] + fn look_matches_word_end_unicode() { + let look = Look::WordEndUnicode; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(!testlook!(look, "a", 0)); + assert!(testlook!(look, "a", 1)); + assert!(testlook!(look, "a ", 1)); + assert!(!testlook!(look, " a ", 1)); + assert!(testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃", 0)); + assert!(testlook!(look, "𝛃", 4)); + assert!(testlook!(look, "𝛃 ", 4)); + assert!(!testlook!(look, " 𝛃 ", 1)); + assert!(testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 0)); + assert!(testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(!testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(!testlook!(look, "a ", 2)); + assert!(!testlook!(look, " a ", 0)); + assert!(!testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃b", 4)); + assert!(!testlook!(look, "b𝛃", 1)); + assert!(!testlook!(look, "𝛃 ", 5)); + assert!(!testlook!(look, " 𝛃 ", 0)); + assert!(!testlook!(look, " 𝛃 ", 6)); + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + assert!(!testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + fn look_matches_word_start_half_ascii() { + let look = Look::WordStartHalfAscii; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(testlook!(look, "a", 0)); + assert!(!testlook!(look, "a", 1)); + assert!(!testlook!(look, "a ", 1)); + assert!(testlook!(look, " a ", 1)); + assert!(!testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. Since this is + // an ASCII word boundary, none of these match. + assert!(testlook!(look, "𝛃", 0)); + assert!(testlook!(look, "𝛃", 4)); + assert!(testlook!(look, "𝛃 ", 4)); + assert!(testlook!(look, " 𝛃 ", 1)); + assert!(testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. Again, since + // this is an ASCII word boundary, none of these match. + assert!(testlook!(look, "𝛃𐆀", 0)); + assert!(testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(testlook!(look, "a ", 2)); + assert!(testlook!(look, " a ", 0)); + assert!(testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(testlook!(look, "𝛃b", 4)); + assert!(!testlook!(look, "b𝛃", 1)); + assert!(testlook!(look, "𝛃 ", 5)); + assert!(testlook!(look, " 𝛃 ", 0)); + assert!(testlook!(look, " 𝛃 ", 6)); + assert!(testlook!(look, "𝛃", 1)); + assert!(testlook!(look, "𝛃", 2)); + assert!(testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(testlook!(look, "𝛃𐆀", 1)); + assert!(testlook!(look, "𝛃𐆀", 2)); + assert!(testlook!(look, "𝛃𐆀", 3)); + assert!(testlook!(look, "𝛃𐆀", 5)); + assert!(testlook!(look, "𝛃𐆀", 6)); + assert!(testlook!(look, "𝛃𐆀", 7)); + assert!(testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + fn look_matches_word_end_half_ascii() { + let look = Look::WordEndHalfAscii; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(!testlook!(look, "a", 0)); + assert!(testlook!(look, "a", 1)); + assert!(testlook!(look, "a ", 1)); + assert!(!testlook!(look, " a ", 1)); + assert!(testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. Since this is + // an ASCII word boundary, none of these match. + assert!(testlook!(look, "𝛃", 0)); + assert!(testlook!(look, "𝛃", 4)); + assert!(testlook!(look, "𝛃 ", 4)); + assert!(testlook!(look, " 𝛃 ", 1)); + assert!(testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. Again, since + // this is an ASCII word boundary, none of these match. + assert!(testlook!(look, "𝛃𐆀", 0)); + assert!(testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(testlook!(look, "a ", 2)); + assert!(testlook!(look, " a ", 0)); + assert!(testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃b", 4)); + assert!(testlook!(look, "b𝛃", 1)); + assert!(testlook!(look, "𝛃 ", 5)); + assert!(testlook!(look, " 𝛃 ", 0)); + assert!(testlook!(look, " 𝛃 ", 6)); + assert!(testlook!(look, "𝛃", 1)); + assert!(testlook!(look, "𝛃", 2)); + assert!(testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(testlook!(look, "𝛃𐆀", 1)); + assert!(testlook!(look, "𝛃𐆀", 2)); + assert!(testlook!(look, "𝛃𐆀", 3)); + assert!(testlook!(look, "𝛃𐆀", 5)); + assert!(testlook!(look, "𝛃𐆀", 6)); + assert!(testlook!(look, "𝛃𐆀", 7)); + assert!(testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + #[cfg(all(not(miri), feature = "unicode-word-boundary"))] + fn look_matches_word_start_half_unicode() { + let look = Look::WordStartHalfUnicode; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(testlook!(look, "a", 0)); + assert!(!testlook!(look, "a", 1)); + assert!(!testlook!(look, "a ", 1)); + assert!(testlook!(look, " a ", 1)); + assert!(!testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. + assert!(testlook!(look, "𝛃", 0)); + assert!(!testlook!(look, "𝛃", 4)); + assert!(!testlook!(look, "𝛃 ", 4)); + assert!(testlook!(look, " 𝛃 ", 1)); + assert!(!testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. + assert!(testlook!(look, "𝛃𐆀", 0)); + assert!(!testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(testlook!(look, "a ", 2)); + assert!(testlook!(look, " a ", 0)); + assert!(testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃b", 4)); + assert!(!testlook!(look, "b𝛃", 1)); + assert!(testlook!(look, "𝛃 ", 5)); + assert!(testlook!(look, " 𝛃 ", 0)); + assert!(testlook!(look, " 𝛃 ", 6)); + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + assert!(testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + #[cfg(all(not(miri), feature = "unicode-word-boundary"))] + fn look_matches_word_end_half_unicode() { + let look = Look::WordEndHalfUnicode; + + // \xF0\x9D\x9B\x83 = 𝛃 (in \w) + // \xF0\x90\x86\x80 = 𐆀 (not in \w) + + // Simple ASCII word boundaries. + assert!(!testlook!(look, "a", 0)); + assert!(testlook!(look, "a", 1)); + assert!(testlook!(look, "a ", 1)); + assert!(!testlook!(look, " a ", 1)); + assert!(testlook!(look, " a ", 2)); + + // Unicode word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃", 0)); + assert!(testlook!(look, "𝛃", 4)); + assert!(testlook!(look, "𝛃 ", 4)); + assert!(!testlook!(look, " 𝛃 ", 1)); + assert!(testlook!(look, " 𝛃 ", 5)); + + // Unicode word boundaries between non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 0)); + assert!(testlook!(look, "𝛃𐆀", 4)); + + // Non word boundaries for ASCII. + assert!(testlook!(look, "", 0)); + assert!(!testlook!(look, "ab", 1)); + assert!(testlook!(look, "a ", 2)); + assert!(testlook!(look, " a ", 0)); + assert!(testlook!(look, " a ", 3)); + + // Non word boundaries with a non-ASCII codepoint. + assert!(!testlook!(look, "𝛃b", 4)); + assert!(!testlook!(look, "b𝛃", 1)); + assert!(testlook!(look, "𝛃 ", 5)); + assert!(testlook!(look, " 𝛃 ", 0)); + assert!(testlook!(look, " 𝛃 ", 6)); + assert!(!testlook!(look, "𝛃", 1)); + assert!(!testlook!(look, "𝛃", 2)); + assert!(!testlook!(look, "𝛃", 3)); + + // Non word boundaries with non-ASCII codepoints. + assert!(!testlook!(look, "𝛃𐆀", 1)); + assert!(!testlook!(look, "𝛃𐆀", 2)); + assert!(!testlook!(look, "𝛃𐆀", 3)); + assert!(!testlook!(look, "𝛃𐆀", 5)); + assert!(!testlook!(look, "𝛃𐆀", 6)); + assert!(!testlook!(look, "𝛃𐆀", 7)); + assert!(testlook!(look, "𝛃𐆀", 8)); + } + + #[test] + fn look_set() { + let mut f = LookSet::default(); + assert!(!f.contains(Look::Start)); + assert!(!f.contains(Look::End)); + assert!(!f.contains(Look::StartLF)); + assert!(!f.contains(Look::EndLF)); + assert!(!f.contains(Look::WordUnicode)); + assert!(!f.contains(Look::WordUnicodeNegate)); + assert!(!f.contains(Look::WordAscii)); + assert!(!f.contains(Look::WordAsciiNegate)); + + f = f.insert(Look::Start); + assert!(f.contains(Look::Start)); + f = f.remove(Look::Start); + assert!(!f.contains(Look::Start)); + + f = f.insert(Look::End); + assert!(f.contains(Look::End)); + f = f.remove(Look::End); + assert!(!f.contains(Look::End)); + + f = f.insert(Look::StartLF); + assert!(f.contains(Look::StartLF)); + f = f.remove(Look::StartLF); + assert!(!f.contains(Look::StartLF)); + + f = f.insert(Look::EndLF); + assert!(f.contains(Look::EndLF)); + f = f.remove(Look::EndLF); + assert!(!f.contains(Look::EndLF)); + + f = f.insert(Look::StartCRLF); + assert!(f.contains(Look::StartCRLF)); + f = f.remove(Look::StartCRLF); + assert!(!f.contains(Look::StartCRLF)); + + f = f.insert(Look::EndCRLF); + assert!(f.contains(Look::EndCRLF)); + f = f.remove(Look::EndCRLF); + assert!(!f.contains(Look::EndCRLF)); + + f = f.insert(Look::WordUnicode); + assert!(f.contains(Look::WordUnicode)); + f = f.remove(Look::WordUnicode); + assert!(!f.contains(Look::WordUnicode)); + + f = f.insert(Look::WordUnicodeNegate); + assert!(f.contains(Look::WordUnicodeNegate)); + f = f.remove(Look::WordUnicodeNegate); + assert!(!f.contains(Look::WordUnicodeNegate)); + + f = f.insert(Look::WordAscii); + assert!(f.contains(Look::WordAscii)); + f = f.remove(Look::WordAscii); + assert!(!f.contains(Look::WordAscii)); + + f = f.insert(Look::WordAsciiNegate); + assert!(f.contains(Look::WordAsciiNegate)); + f = f.remove(Look::WordAsciiNegate); + assert!(!f.contains(Look::WordAsciiNegate)); + + f = f.insert(Look::WordStartAscii); + assert!(f.contains(Look::WordStartAscii)); + f = f.remove(Look::WordStartAscii); + assert!(!f.contains(Look::WordStartAscii)); + + f = f.insert(Look::WordEndAscii); + assert!(f.contains(Look::WordEndAscii)); + f = f.remove(Look::WordEndAscii); + assert!(!f.contains(Look::WordEndAscii)); + + f = f.insert(Look::WordStartUnicode); + assert!(f.contains(Look::WordStartUnicode)); + f = f.remove(Look::WordStartUnicode); + assert!(!f.contains(Look::WordStartUnicode)); + + f = f.insert(Look::WordEndUnicode); + assert!(f.contains(Look::WordEndUnicode)); + f = f.remove(Look::WordEndUnicode); + assert!(!f.contains(Look::WordEndUnicode)); + + f = f.insert(Look::WordStartHalfAscii); + assert!(f.contains(Look::WordStartHalfAscii)); + f = f.remove(Look::WordStartHalfAscii); + assert!(!f.contains(Look::WordStartHalfAscii)); + + f = f.insert(Look::WordEndHalfAscii); + assert!(f.contains(Look::WordEndHalfAscii)); + f = f.remove(Look::WordEndHalfAscii); + assert!(!f.contains(Look::WordEndHalfAscii)); + + f = f.insert(Look::WordStartHalfUnicode); + assert!(f.contains(Look::WordStartHalfUnicode)); + f = f.remove(Look::WordStartHalfUnicode); + assert!(!f.contains(Look::WordStartHalfUnicode)); + + f = f.insert(Look::WordEndHalfUnicode); + assert!(f.contains(Look::WordEndHalfUnicode)); + f = f.remove(Look::WordEndHalfUnicode); + assert!(!f.contains(Look::WordEndHalfUnicode)); + } + + #[test] + fn look_set_iter() { + let set = LookSet::empty(); + assert_eq!(0, set.iter().count()); + + let set = LookSet::full(); + assert_eq!(18, set.iter().count()); + + let set = + LookSet::empty().insert(Look::StartLF).insert(Look::WordUnicode); + assert_eq!(2, set.iter().count()); + + let set = LookSet::empty().insert(Look::StartLF); + assert_eq!(1, set.iter().count()); + + let set = LookSet::empty().insert(Look::WordAsciiNegate); + assert_eq!(1, set.iter().count()); + + let set = LookSet::empty().insert(Look::WordEndHalfUnicode); + assert_eq!(1, set.iter().count()); + } + + #[test] + #[cfg(feature = "alloc")] + fn look_set_debug() { + let res = alloc::format!("{:?}", LookSet::empty()); + assert_eq!("∅", res); + let res = alloc::format!("{:?}", LookSet::full()); + assert_eq!("Az^$rRbB𝛃𝚩<>〈〉◁▷◀▶", res); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/memchr.rs b/third_party/crates/regex-automata-0.4.18/src/util/memchr.rs new file mode 100644 index 0000000..a2cbb07 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/memchr.rs @@ -0,0 +1,93 @@ +/*! +This module defines simple wrapper routines for the memchr functions from the +`memchr` crate. Basically, when the `memchr` crate is available, we use it, +otherwise we use a naive implementation which is still pretty fast. +*/ + +pub(crate) use self::inner::*; + +#[cfg(feature = "perf-literal-substring")] +pub(super) mod inner { + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memchr(n1: u8, haystack: &[u8]) -> Option<usize> { + memchr::memchr(n1, haystack) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option<usize> { + memchr::memchr2(n1, n2, haystack) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], + ) -> Option<usize> { + memchr::memchr3(n1, n2, n3, haystack) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memrchr(n1: u8, haystack: &[u8]) -> Option<usize> { + memchr::memrchr(n1, haystack) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memrchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option<usize> { + memchr::memrchr2(n1, n2, haystack) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memrchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], + ) -> Option<usize> { + memchr::memrchr3(n1, n2, n3, haystack) + } +} + +#[cfg(not(feature = "perf-literal-substring"))] +pub(super) mod inner { + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memchr(n1: u8, haystack: &[u8]) -> Option<usize> { + haystack.iter().position(|&b| b == n1) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option<usize> { + haystack.iter().position(|&b| b == n1 || b == n2) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], + ) -> Option<usize> { + haystack.iter().position(|&b| b == n1 || b == n2 || b == n3) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memrchr(n1: u8, haystack: &[u8]) -> Option<usize> { + haystack.iter().rposition(|&b| b == n1) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memrchr2(n1: u8, n2: u8, haystack: &[u8]) -> Option<usize> { + haystack.iter().rposition(|&b| b == n1 || b == n2) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn memrchr3( + n1: u8, + n2: u8, + n3: u8, + haystack: &[u8], + ) -> Option<usize> { + haystack.iter().rposition(|&b| b == n1 || b == n2 || b == n3) + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/mod.rs b/third_party/crates/regex-automata-0.4.18/src/util/mod.rs new file mode 100644 index 0000000..b3eef64 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/mod.rs @@ -0,0 +1,57 @@ +/*! +A collection of modules that provide APIs that are useful across many regex +engines. + +While one should explore the sub-modules directly to get a sense of what's +there, here are some highlights that tie the sub-modules to higher level +use cases: + +* `alphabet` contains APIs that are useful if you're doing low level things +with the DFAs in this crate. For example, implementing determinization or +walking its state graph directly. +* `captures` contains APIs for dealing with capture group matches and their +mapping to "slots" used inside an NFA graph. This is also where you can find +iterators over capture group names. +* `escape` contains types for pretty-printing raw byte slices as strings. +* `iter` contains API helpers for writing regex iterators. +* `lazy` contains a no-std and no-alloc variant of `lazy_static!` and +`once_cell`. +* `look` contains APIs for matching and configuring look-around assertions. +* `pool` provides a way to reuse mutable memory allocated in a thread safe +manner. +* `prefilter` provides APIs for building prefilters and using them in searches. +* `primitives` are what you might use if you're doing lower level work on +automata, such as walking an NFA state graph. +* `syntax` provides some higher level convenience functions for interacting +with the `regex-syntax` crate. +* `wire` is useful if you're working with DFA serialization. +*/ + +pub mod alphabet; +#[cfg(feature = "alloc")] +pub mod captures; +pub mod escape; +#[cfg(feature = "alloc")] +pub mod interpolate; +pub mod iter; +pub mod lazy; +pub mod look; +#[cfg(feature = "alloc")] +pub mod pool; +pub mod prefilter; +pub mod primitives; +pub mod start; +#[cfg(feature = "syntax")] +pub mod syntax; +pub mod wire; + +#[cfg(any(feature = "dfa-build", feature = "hybrid"))] +pub(crate) mod determinize; +pub(crate) mod empty; +pub(crate) mod int; +pub(crate) mod memchr; +pub(crate) mod search; +#[cfg(feature = "alloc")] +pub(crate) mod sparse_set; +pub(crate) mod unicode_data; +pub(crate) mod utf8; diff --git a/third_party/crates/regex-automata-0.4.18/src/util/pool.rs b/third_party/crates/regex-automata-0.4.18/src/util/pool.rs new file mode 100644 index 0000000..9290293 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/pool.rs @@ -0,0 +1,1269 @@ +// This module provides a relatively simple thread-safe pool of reusable +// objects. For the most part, it's implemented by a stack represented by a +// Mutex<Vec<T>>. It has one small trick: because unlocking a mutex is somewhat +// costly, in the case where a pool is accessed by the first thread that tried +// to get a value, we bypass the mutex. Here are some benchmarks showing the +// difference. +// +// 2022-10-15: These benchmarks are from the old regex crate and they aren't +// easy to reproduce because some rely on older implementations of Pool that +// are no longer around. I've left the results here for posterity, but any +// enterprising individual should feel encouraged to re-litigate the way Pool +// works. I am not at all certain it is the best approach. +// +// 1) misc::anchored_literal_long_non_match 21 (18571 MB/s) +// 2) misc::anchored_literal_long_non_match 107 (3644 MB/s) +// 3) misc::anchored_literal_long_non_match 45 (8666 MB/s) +// 4) misc::anchored_literal_long_non_match 19 (20526 MB/s) +// +// (1) represents our baseline: the master branch at the time of writing when +// using the 'thread_local' crate to implement the pool below. +// +// (2) represents a naive pool implemented completely via Mutex<Vec<T>>. There +// is no special trick for bypassing the mutex. +// +// (3) is the same as (2), except it uses Mutex<Vec<Box<T>>>. It is twice as +// fast because a Box<T> is much smaller than the T we use with a Pool in this +// crate. So pushing and popping a Box<T> from a Vec is quite a bit faster +// than for T. +// +// (4) is the same as (3), but with the trick for bypassing the mutex in the +// case of the first-to-get thread. +// +// Why move off of thread_local? Even though (4) is a hair faster than (1) +// above, this was not the main goal. The main goal was to move off of +// thread_local and find a way to *simply* re-capture some of its speed for +// regex's specific case. So again, why move off of it? The *primary* reason is +// because of memory leaks. See https://github.com/rust-lang/regex/issues/362 +// for example. (Why do I want it to be simple? Well, I suppose what I mean is, +// "use as much safe code as possible to minimize risk and be as sure as I can +// be that it is correct.") +// +// My guess is that the thread_local design is probably not appropriate for +// regex since its memory usage scales to the number of active threads that +// have used a regex, where as the pool below scales to the number of threads +// that simultaneously use a regex. While neither case permits contraction, +// since we own the pool data structure below, we can add contraction if a +// clear use case pops up in the wild. More pressingly though, it seems that +// there are at least some use case patterns where one might have many threads +// sitting around that might have used a regex at one point. While thread_local +// does try to reuse space previously used by a thread that has since stopped, +// its maximal memory usage still scales with the total number of active +// threads. In contrast, the pool below scales with the total number of threads +// *simultaneously* using the pool. The hope is that this uses less memory +// overall. And if it doesn't, we can hopefully tune it somehow. +// +// It seems that these sort of conditions happen frequently +// in FFI inside of other more "managed" languages. This was +// mentioned in the issue linked above, and also mentioned here: +// https://github.com/BurntSushi/rure-go/issues/3. And in particular, users +// confirm that disabling the use of thread_local resolves the leak. +// +// There were other weaker reasons for moving off of thread_local as well. +// Namely, at the time, I was looking to reduce dependencies. And for something +// like regex, maintenance can be simpler when we own the full dependency tree. +// +// Note that I am not entirely happy with this pool. It has some subtle +// implementation details and is overall still observable (even with the +// thread owner optimization) in benchmarks. If someone wants to take a crack +// at building something better, please file an issue. Even if it means a +// different API. The API exposed by this pool is not the minimal thing that +// something like a 'Regex' actually needs. It could adapt to, for example, +// an API more like what is found in the 'thread_local' crate. However, we do +// really need to support the no-std alloc-only context, or else the regex +// crate wouldn't be able to support no-std alloc-only. However, I'm generally +// okay with making the alloc-only context slower (as it is here), although I +// do find it unfortunate. + +/*! +A thread safe memory pool. + +The principal type in this module is a [`Pool`]. It main use case is for +holding a thread safe collection of mutable scratch spaces (usually called +`Cache` in this crate) that regex engines need to execute a search. This then +permits sharing the same read-only regex object across multiple threads while +having a quick way of reusing scratch space in a thread safe way. This avoids +needing to re-create the scratch space for every search, which could wind up +being quite expensive. +*/ + +/// A thread safe pool that works in an `alloc`-only context. +/// +/// Getting a value out comes with a guard. When that guard is dropped, the +/// value is automatically put back in the pool. The guard provides both a +/// `Deref` and a `DerefMut` implementation for easy access to an underlying +/// `T`. +/// +/// A `Pool` impls `Sync` when `T` is `Send` (even if `T` is not `Sync`). This +/// is possible because a pool is guaranteed to provide a value to exactly one +/// thread at any time. +/// +/// Currently, a pool never contracts in size. Its size is proportional to the +/// maximum number of simultaneous uses. This may change in the future. +/// +/// A `Pool` is a particularly useful data structure for this crate because +/// many of the regex engines require a mutable "cache" in order to execute +/// a search. Since regexes themselves tend to be global, the problem is then: +/// how do you get a mutable cache to execute a search? You could: +/// +/// 1. Use a `thread_local!`, which requires the standard library and requires +/// that the regex pattern be statically known. +/// 2. Use a `Pool`. +/// 3. Make the cache an explicit dependency in your code and pass it around. +/// 4. Put the cache state in a `Mutex`, but this means only one search can +/// execute at a time. +/// 5. Create a new cache for every search. +/// +/// A `thread_local!` is perhaps the best choice if it works for your use case. +/// Putting the cache in a mutex or creating a new cache for every search are +/// perhaps the worst choices. Of the remaining two choices, whether you use +/// this `Pool` or thread through a cache explicitly in your code is a matter +/// of taste and depends on your code architecture. +/// +/// # Warning: may use a spin lock +/// +/// When this crate is compiled _without_ the `std` feature, then this type +/// may used a spin lock internally. This can have subtle effects that may +/// be undesirable. See [Spinlocks Considered Harmful][spinharm] for a more +/// thorough treatment of this topic. +/// +/// [spinharm]: https://matklad.github.io/2020/01/02/spinlocks-considered-harmful.html +/// +/// # Example +/// +/// This example shows how to share a single hybrid regex among multiple +/// threads, while also safely getting exclusive access to a hybrid's +/// [`Cache`](crate::hybrid::regex::Cache) without preventing other searches +/// from running while your thread uses the `Cache`. +/// +/// ``` +/// use regex_automata::{ +/// hybrid::regex::{Cache, Regex}, +/// util::{lazy::Lazy, pool::Pool}, +/// Match, +/// }; +/// +/// static RE: Lazy<Regex> = +/// Lazy::new(|| Regex::new("foo[0-9]+bar").unwrap()); +/// static CACHE: Lazy<Pool<Cache>> = +/// Lazy::new(|| Pool::new(|| RE.create_cache())); +/// +/// let expected = Some(Match::must(0, 3..14)); +/// assert_eq!(expected, RE.find(&mut CACHE.get(), b"zzzfoo12345barzzz")); +/// ``` +pub struct Pool<T, F = fn() -> T>(alloc::boxed::Box<inner::Pool<T, F>>); + +impl<T, F> Pool<T, F> { + /// Create a new pool. The given closure is used to create values in + /// the pool when necessary. + pub fn new(create: F) -> Pool<T, F> { + Pool(alloc::boxed::Box::new(inner::Pool::new(create))) + } + + /// Create a new pool. The given closure is used to create values in + /// the pool when necessary. + /// + /// When the `std` feature is enabled, a `Pool` is thread-aware and spreads + /// its memory out across multiple cache lines. The number of cache lines + /// is determined by the `capacity` parameter passed here. By default, a + /// fixed reasonable number is used. A smaller number means less memory is + /// used, but a higher number means there may be less contention on this + /// pool in highly threaded environments doing a lot of searches using the + /// same `Regex` value. + /// + /// When `std` is not enabled, then the capacity parameter is ignored + /// because the underlying pool implementation is not thread-aware. + /// + /// The capacity must be at least 1. If it's less than 1, then it is + /// forced to be 1. + pub fn with_capacity(capacity: usize, create: F) -> Pool<T, F> { + Pool(alloc::boxed::Box::new(inner::Pool::with_capacity( + capacity, create, + ))) + } + + /// Create a new pool. The given closure is used to create values in + /// the pool when necessary. + /// + /// This is a convenience routine for calling `Pool::with_capacity` with + /// a number equivalent to the available parallelism for this environment. + /// + /// If `std` is not enabled or if the query for available parallelism + /// failed, then this is equivalent to calling `Pool::new`. + pub fn with_available_parallelism_capacity(create: F) -> Pool<T, F> { + #[cfg(feature = "std")] + { + use crate::util::lazy::Lazy; + + static AVAILABLE_PARALLELISM: Lazy<Option<usize>> = + Lazy::new(|| { + std::thread::available_parallelism().map(|n| n.get()).ok() + }); + let &Some(n) = Lazy::get(&AVAILABLE_PARALLELISM) else { + return Pool::new(create); + }; + Pool::with_capacity(n, create) + } + #[cfg(not(feature = "std"))] + { + Pool::new(create) + } + } +} + +impl<T: Send, F: Fn() -> T> Pool<T, F> { + /// Get a value from the pool. The caller is guaranteed to have + /// exclusive access to the given value. Namely, it is guaranteed that + /// this will never return a value that was returned by another call to + /// `get` but was not put back into the pool. + /// + /// When the guard goes out of scope and its destructor is called, then + /// it will automatically be put back into the pool. Alternatively, + /// [`PoolGuard::put`] may be used to explicitly put it back in the pool + /// without relying on its destructor. + /// + /// Note that there is no guarantee provided about which value in the + /// pool is returned. That is, calling get, dropping the guard (causing + /// the value to go back into the pool) and then calling get again is + /// *not* guaranteed to return the same value received in the first `get` + /// call. + #[inline] + pub fn get(&self) -> PoolGuard<'_, T, F> { + PoolGuard(self.0.get()) + } +} + +impl<T: core::fmt::Debug, F> core::fmt::Debug for Pool<T, F> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_tuple("Pool").field(&self.0).finish() + } +} + +/// A guard that is returned when a caller requests a value from the pool. +/// +/// The purpose of the guard is to use RAII to automatically put the value +/// back in the pool once it's dropped. +pub struct PoolGuard<'a, T: Send, F: Fn() -> T>(inner::PoolGuard<'a, T, F>); + +impl<'a, T: Send, F: Fn() -> T> PoolGuard<'a, T, F> { + /// Consumes this guard and puts it back into the pool. + /// + /// This circumvents the guard's `Drop` implementation. This can be useful + /// in circumstances where the automatic `Drop` results in poorer codegen, + /// such as calling non-inlined functions. + #[inline] + pub fn put(this: PoolGuard<'_, T, F>) { + inner::PoolGuard::put(this.0); + } +} + +impl<'a, T: Send, F: Fn() -> T> core::ops::Deref for PoolGuard<'a, T, F> { + type Target = T; + + #[inline] + fn deref(&self) -> &T { + self.0.value() + } +} + +impl<'a, T: Send, F: Fn() -> T> core::ops::DerefMut for PoolGuard<'a, T, F> { + #[inline] + fn deref_mut(&mut self) -> &mut T { + self.0.value_mut() + } +} + +impl<'a, T: Send + core::fmt::Debug, F: Fn() -> T> core::fmt::Debug + for PoolGuard<'a, T, F> +{ + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_tuple("PoolGuard").field(&self.0).finish() + } +} + +#[cfg(feature = "std")] +mod inner { + use core::{ + cell::UnsafeCell, + panic::{RefUnwindSafe, UnwindSafe}, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use alloc::{boxed::Box, vec, vec::Vec}; + + use std::{sync::Mutex, thread_local}; + + /// An atomic counter used to allocate thread IDs. + /// + /// We specifically start our counter at 3 so that we can use the values + /// less than it as sentinels. + static COUNTER: AtomicUsize = AtomicUsize::new(3); + + /// A thread ID indicating that there is no owner. This is the initial + /// state of a pool. Once a pool has an owner, there is no way to change + /// it. + static THREAD_ID_UNOWNED: usize = 0; + + /// A thread ID indicating that the special owner value is in use and not + /// available. This state is useful for avoiding a case where the owner + /// of a pool calls `get` before putting the result of a previous `get` + /// call back into the pool. + static THREAD_ID_INUSE: usize = 1; + + /// This sentinel is used to indicate that a guard has already been dropped + /// and should not be re-dropped. We use this because our drop code can be + /// called outside of Drop and thus there could be a bug in the internal + /// implementation that results in trying to put the same guard back into + /// the same pool multiple times, and *that* could result in UB if we + /// didn't mark the guard as already having been put back in the pool. + /// + /// So this isn't strictly necessary, but this let's us define some + /// routines as safe (like PoolGuard::put_imp) that we couldn't otherwise + /// do. + static THREAD_ID_DROPPED: usize = 2; + + /// The number of stacks we use inside of the pool. These are only used for + /// non-owners. That is, these represent the "slow" path. + /// + /// In the original implementation of this pool, we only used a single + /// stack. While this might be okay for a couple threads, the prevalence of + /// 32, 64 and even 128 core CPUs has made it untenable. The contention + /// such an environment introduces when threads are doing a lot of searches + /// on short haystacks (a not uncommon use case) is palpable and leads to + /// huge slowdowns. + /// + /// This constant reflects a change from using one stack to the number of + /// stacks that this constant is set to. The stack for a particular thread + /// is simply chosen by `thread_id % MAX_POOL_STACKS`. The idea behind + /// this setup is that there should be a good chance that accesses to the + /// pool will be distributed over several stacks instead of all of them + /// converging to one. + /// + /// This is not a particularly smart or dynamic strategy. Fixing this to a + /// specific number has at least two downsides. First is that it will help, + /// say, an 8 core CPU more than it will a 128 core CPU. (But, crucially, + /// it will still help the 128 core case.) Second is that this may wind + /// up being a little wasteful with respect to memory usage. Namely, if a + /// regex is used on one thread and then moved to another thread, then it + /// could result in creating a new copy of the data in the pool even though + /// only one is actually needed. + /// + /// And that memory usage bit is why this is set to 8 and not, say, 64. + /// Keeping it at 8 limits, to an extent, how much unnecessary memory can + /// be allocated. + /// + /// In an ideal world, we'd be able to have something like this: + /// + /// * Grow the number of stacks as the number of concurrent callers + /// increases. I spent a little time trying this, but even just adding an + /// atomic addition/subtraction for each pop/push for tracking concurrent + /// callers led to a big perf hit. Since even more work would seemingly be + /// required than just an addition/subtraction, I abandoned this approach. + /// * The maximum amount of memory used should scale with respect to the + /// number of concurrent callers and *not* the total number of existing + /// threads. This is primarily why the `thread_local` crate isn't used, as + /// as some environments spin up a lot of threads. This led to multiple + /// reports of extremely high memory usage (often described as memory + /// leaks). + /// * Even more ideally, the pool should contract in size. That is, it + /// should grow with bursts and then shrink. But this is a pretty thorny + /// issue to tackle and it might be better to just not. + /// * It would be nice to explore the use of, say, a lock-free stack + /// instead of using a mutex to guard a `Vec` that is ultimately just + /// treated as a stack. The main thing preventing me from exploring this + /// is the ABA problem. The `crossbeam` crate has tools for dealing with + /// this sort of problem (via its epoch based memory reclamation strategy), + /// but I can't justify bringing in all of `crossbeam` as a dependency of + /// `regex` for this. + /// + /// See this issue for more context and discussion: + /// https://github.com/rust-lang/regex/issues/934 + const MAX_POOL_STACKS: usize = 8; + + thread_local!( + /// A thread local used to assign an ID to a thread. + static THREAD_ID: usize = { + let next = COUNTER.fetch_add(1, Ordering::Relaxed); + // SAFETY: We cannot permit the reuse of thread IDs since reusing a + // thread ID might result in more than one thread "owning" a pool, + // and thus, permit accessing a mutable value from multiple threads + // simultaneously without synchronization. The intent of this panic + // is to be a sanity check. It is not expected that the thread ID + // space will actually be exhausted in practice. Even on a 32-bit + // system, it would require spawning 2^32 threads (although they + // wouldn't all need to run simultaneously, so it is in theory + // possible). + // + // This checks that the counter never wraps around, since atomic + // addition wraps around on overflow. + if next == 0 { + panic!("regex: thread ID allocation space exhausted"); + } + next + }; + ); + + /// This puts each stack in the pool below into its own cache line. This is + /// an absolutely critical optimization that tends to have the most impact + /// in high contention workloads. Without forcing each mutex protected + /// into its own cache line, high contention exacerbates the performance + /// problem by causing "false sharing." By putting each mutex in its own + /// cache-line, we avoid the false sharing problem and the affects of + /// contention are greatly reduced. + #[derive(Debug)] + #[repr(C, align(64))] + struct CacheLine<T>(T); + + /// A thread safe pool utilizing std-only features. + /// + /// The main difference between this and the simplistic alloc-only pool is + /// the use of std::sync::Mutex and an "owner thread" optimization that + /// makes accesses by the owner of a pool faster than all other threads. + /// This makes the common case of running a regex within a single thread + /// faster by avoiding mutex unlocking. + pub(super) struct Pool<T, F> { + /// A function to create more T values when stack is empty and a caller + /// has requested a T. + create: F, + /// Multiple stacks of T values to hand out. These are used when a Pool + /// is accessed by a thread that didn't create it. + /// + /// Conceptually this is `Mutex<Vec<Box<T>>>`, but sharded out to make + /// it scale better under high contention work-loads. We index into + /// this sequence via `thread_id % stacks.len()`. + stacks: Vec<CacheLine<Mutex<Vec<Box<T>>>>>, + /// The ID of the thread that owns this pool. The owner is the thread + /// that makes the first call to 'get'. When the owner calls 'get', it + /// gets 'owner_val' directly instead of returning a T from 'stack'. + /// See comments elsewhere for details, but this is intended to be an + /// optimization for the common case that makes getting a T faster. + /// + /// It is initialized to a value of zero (an impossible thread ID) as a + /// sentinel to indicate that it is unowned. + owner: AtomicUsize, + /// A value to return when the caller is in the same thread that + /// first called `Pool::get`. + /// + /// This is set to None when a Pool is first created, and set to Some + /// once the first thread calls Pool::get. + owner_val: UnsafeCell<Option<T>>, + } + + // SAFETY: Since we want to use a Pool from multiple threads simultaneously + // behind an Arc, we need for it to be Sync. In cases where T is sync, + // Pool<T> would be Sync. However, since we use a Pool to store mutable + // scratch space, we wind up using a T that has interior mutability and is + // thus itself not Sync. So what we *really* want is for our Pool<T> to by + // Sync even when T is not Sync (but is at least Send). + // + // The only non-sync aspect of a Pool is its 'owner_val' field, which is + // used to implement faster access to a pool value in the common case of + // a pool being accessed in the same thread in which it was created. The + // 'stack' field is also shared, but a Mutex<T> where T: Send is already + // Sync. So we only need to worry about 'owner_val'. + // + // The key is to guarantee that 'owner_val' can only ever be accessed from + // one thread. In our implementation below, we guarantee this by only + // returning the 'owner_val' when the ID of the current thread matches the + // ID of the thread that first called 'Pool::get'. Since this can only ever + // be one thread, it follows that only one thread can access 'owner_val' at + // any point in time. Thus, it is safe to declare that Pool<T> is Sync when + // T is Send. + // + // If there is a way to achieve our performance goals using safe code, then + // I would very much welcome a patch. As it stands, the implementation + // below tries to balance safety with performance. The case where a Regex + // is used from multiple threads simultaneously will suffer a bit since + // getting a value out of the pool will require unlocking a mutex. + // + // We require `F: Send + Sync` because we call `F` at any point on demand, + // potentially from multiple threads simultaneously. + unsafe impl<T: Send, F: Send + Sync> Sync for Pool<T, F> {} + + // If T is UnwindSafe, then since we provide exclusive access to any + // particular value in the pool, the pool should therefore also be + // considered UnwindSafe. + // + // We require `F: UnwindSafe + RefUnwindSafe` because we call `F` at any + // point on demand, so it needs to be unwind safe on both dimensions for + // the entire Pool to be unwind safe. + impl<T: UnwindSafe, F: UnwindSafe + RefUnwindSafe> UnwindSafe for Pool<T, F> {} + + // If T is UnwindSafe, then since we provide exclusive access to any + // particular value in the pool, the pool should therefore also be + // considered RefUnwindSafe. + // + // We require `F: UnwindSafe + RefUnwindSafe` because we call `F` at any + // point on demand, so it needs to be unwind safe on both dimensions for + // the entire Pool to be unwind safe. + impl<T: UnwindSafe, F: UnwindSafe + RefUnwindSafe> RefUnwindSafe + for Pool<T, F> + { + } + + impl<T, F> Pool<T, F> { + /// Create a new pool. The given closure is used to create values in + /// the pool when necessary. + pub(super) fn new(create: F) -> Pool<T, F> { + Pool::with_capacity(MAX_POOL_STACKS, create) + } + + /// Create a new pool. The given closure is used to create values in + /// the pool when necessary. + /// + /// The given capacity is used to determine how many cache lines to + /// maintain. Each cache line contains a stack of cached entries. + /// + /// The capacity must be at least 1. If it's less than 1, then it is + /// forced to be 1. + pub(super) fn with_capacity(capacity: usize, create: F) -> Pool<T, F> { + // FIXME: Now that we require 1.65+, Mutex::new is available as + // const... So we can almost mark this function as const. But of + // course, we're creating a Vec of stacks below (we didn't when I + // originally wrote this code). It seems like the best way to work + // around this would be to use a `[Stack; MAX_POOL_STACKS]` instead + // of a `Vec<Stack>`. I refrained from making this change at time + // of writing (2023/10/08) because I was making a lot of other + // changes at the same time and wanted to do this more carefully. + // Namely, because of the cache line optimization, that `[Stack; + // MAX_POOL_STACKS]` would be quite big. It's unclear how bad (if + // at all) that would be. + // + // Another choice would be to lazily allocate the stacks, but... + // I'm not so sure about that. Seems like a fair bit of complexity? + // + // Maybe there's a simple solution I'm missing. + // + // ... OK, I tried to fix this. First, I did it by putting `stacks` + // in an `UnsafeCell` and using a `Once` to lazily initialize it. + // I benchmarked it and everything looked okay. I then made this + // function `const` and thought I was just about done. But the + // public pool type wraps its inner pool in a `Box` to keep its + // size down. Blech. + // + // So then I thought that I could push the box down into this + // type (and leave the non-std version unboxed) and use the same + // `UnsafeCell` technique to lazily initialize it. This has the + // downside of the `Once` now needing to get hit in the owner fast + // path, but maybe that's OK? However, I then realized that we can + // only lazily initialize `stacks`, `owner` and `owner_val`. The + // `create` function needs to be put somewhere outside of the box. + // So now the pool is a `Box`, `Once` and a function. Now we're + // starting to defeat the point of boxing in the first place. So I + // backed out that change too. + // + // Back to square one. I maybe we just don't make a pool's + // constructor const and live with it. It's probably not a huge + // deal. + let mut stacks = Vec::with_capacity(capacity.max(1)); + for _ in 0..stacks.capacity() { + stacks.push(CacheLine(Mutex::new(vec![]))); + } + let owner = AtomicUsize::new(THREAD_ID_UNOWNED); + let owner_val = UnsafeCell::new(None); // init'd on first access + Pool { create, stacks, owner, owner_val } + } + } + + impl<T: Send, F: Fn() -> T> Pool<T, F> { + /// Get a value from the pool. This may block if another thread is also + /// attempting to retrieve a value from the pool. + #[inline] + pub(super) fn get(&self) -> PoolGuard<'_, T, F> { + // Our fast path checks if the caller is the thread that "owns" + // this pool. Or stated differently, whether it is the first thread + // that tried to extract a value from the pool. If it is, then we + // can return a T to the caller without going through a mutex. + // + // SAFETY: We must guarantee that only one thread gets access + // to this value. Since a thread is uniquely identified by the + // THREAD_ID thread local, it follows that if the caller's thread + // ID is equal to the owner, then only one thread may receive this + // value. This is also why we can get away with what looks like a + // racy load and a store. We know that if 'owner == caller', then + // only one thread can be here, so we don't need to worry about any + // other thread setting the owner to something else. + let caller = THREAD_ID.with(|id| *id); + let owner = self.owner.load(Ordering::Acquire); + if caller == owner { + // N.B. We could also do a CAS here instead of a load/store, + // but ad hoc benchmarking suggests it is slower. And a lot + // slower in the case where `get_slow` is common. + self.owner.store(THREAD_ID_INUSE, Ordering::Release); + return self.guard_owned(caller); + } + self.get_slow(caller, owner) + } + + /// This is the "slow" version that goes through a mutex to pop an + /// allocated value off a stack to return to the caller. (Or, if the + /// stack is empty, a new value is created.) + /// + /// If the pool has no owner, then this will set the owner. + #[cold] + fn get_slow( + &self, + caller: usize, + owner: usize, + ) -> PoolGuard<'_, T, F> { + if owner == THREAD_ID_UNOWNED { + // This sentinel means this pool is not yet owned. We try to + // atomically set the owner. If we do, then this thread becomes + // the owner and we can return a guard that represents the + // special T for the owner. + // + // Note that we set the owner to a different sentinel that + // indicates that the owned value is in use. The owner ID will + // get updated to the actual ID of this thread once the guard + // returned by this function is put back into the pool. + let res = self.owner.compare_exchange( + THREAD_ID_UNOWNED, + THREAD_ID_INUSE, + Ordering::AcqRel, + Ordering::Acquire, + ); + if res.is_ok() { + // SAFETY: A successful CAS above implies this thread is + // the owner and that this is the only such thread that + // can reach here. Thus, there is no data race. + unsafe { + *self.owner_val.get() = Some((self.create)()); + } + return self.guard_owned(caller); + } + } + let stack_id = caller % self.stacks.len(); + // We try to acquire exclusive access to this thread's stack, and + // if so, grab a value from it if we can. We put this in a loop so + // that it's easy to tweak and experiment with a different number + // of tries. In the end, I couldn't see anything obviously better + // than one attempt in ad hoc testing. + for _ in 0..1 { + let mut stack = match self.stacks[stack_id].0.try_lock() { + Err(_) => continue, + Ok(stack) => stack, + }; + if let Some(value) = stack.pop() { + return self.guard_stack(value); + } + // Unlock the mutex guarding the stack before creating a fresh + // value since we no longer need the stack. + drop(stack); + let value = Box::new((self.create)()); + return self.guard_stack(value); + } + // We're only here if we could get access to our stack, so just + // create a new value. This seems like it could be wasteful, but + // waiting for exclusive access to a stack when there's high + // contention is brutal for perf. + self.guard_stack_transient(Box::new((self.create)())) + } + + /// Puts a value back into the pool. Callers don't need to call this. + /// Once the guard that's returned by 'get' is dropped, it is put back + /// into the pool automatically. + #[inline] + fn put_value(&self, value: Box<T>) { + let caller = THREAD_ID.with(|id| *id); + let stack_id = caller % self.stacks.len(); + // As with trying to pop a value from this thread's stack, we + // merely attempt to get access to push this value back on the + // stack. If there's too much contention, we just give up and throw + // the value away. + // + // Interestingly, in ad hoc benchmarking, it is beneficial to + // attempt to push the value back more than once, unlike when + // popping the value. I don't have a good theory for why this is. + // I guess if we drop too many values then that winds up forcing + // the pop operation to create new fresh values and thus leads to + // less reuse. There's definitely a balancing act here. + for _ in 0..10 { + let mut stack = match self.stacks[stack_id].0.try_lock() { + Err(_) => continue, + Ok(stack) => stack, + }; + stack.push(value); + return; + } + } + + /// Create a guard that represents the special owned T. + #[inline] + fn guard_owned(&self, caller: usize) -> PoolGuard<'_, T, F> { + PoolGuard { pool: self, value: Err(caller), discard: false } + } + + /// Create a guard that contains a value from the pool's stack. + #[inline] + fn guard_stack(&self, value: Box<T>) -> PoolGuard<'_, T, F> { + PoolGuard { pool: self, value: Ok(value), discard: false } + } + + /// Create a guard that contains a value from the pool's stack with an + /// instruction to throw away the value instead of putting it back + /// into the pool. + #[inline] + fn guard_stack_transient(&self, value: Box<T>) -> PoolGuard<'_, T, F> { + PoolGuard { pool: self, value: Ok(value), discard: true } + } + } + + impl<T: core::fmt::Debug, F> core::fmt::Debug for Pool<T, F> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Pool") + .field("stacks", &self.stacks) + .field("owner", &self.owner) + .field("owner_val", &self.owner_val) + .finish() + } + } + + /// A guard that is returned when a caller requests a value from the pool. + pub(super) struct PoolGuard<'a, T: Send, F: Fn() -> T> { + /// The pool that this guard is attached to. + pool: &'a Pool<T, F>, + /// This is Err when the guard represents the special "owned" value. + /// In which case, the value is retrieved from 'pool.owner_val'. And + /// in the special case of `Err(THREAD_ID_DROPPED)`, it means the + /// guard has been put back into the pool and should no longer be used. + value: Result<Box<T>, usize>, + /// When true, the value should be discarded instead of being pushed + /// back into the pool. We tend to use this under high contention, and + /// this allows us to avoid inflating the size of the pool. (Because + /// under contention, we tend to create more values instead of waiting + /// for access to a stack of existing values.) + discard: bool, + } + + impl<'a, T: Send, F: Fn() -> T> PoolGuard<'a, T, F> { + /// Return the underlying value. + #[inline] + pub(super) fn value(&self) -> &T { + match self.value { + Ok(ref v) => v, + // SAFETY: This is safe because the only way a PoolGuard gets + // created for self.value=Err is when the current thread + // corresponds to the owning thread, of which there can only + // be one. Thus, we are guaranteed to be providing exclusive + // access here which makes this safe. + // + // Also, since 'owner_val' is guaranteed to be initialized + // before an owned PoolGuard is created, the unchecked unwrap + // is safe. + Err(id) => unsafe { + // This assert is *not* necessary for safety, since we + // should never be here if the guard had been put back into + // the pool. This is a sanity check to make sure we didn't + // break an internal invariant. + debug_assert_ne!(THREAD_ID_DROPPED, id); + (*self.pool.owner_val.get()).as_ref().unwrap_unchecked() + }, + } + } + + /// Return the underlying value as a mutable borrow. + #[inline] + pub(super) fn value_mut(&mut self) -> &mut T { + match self.value { + Ok(ref mut v) => v, + // SAFETY: This is safe because the only way a PoolGuard gets + // created for self.value=None is when the current thread + // corresponds to the owning thread, of which there can only + // be one. Thus, we are guaranteed to be providing exclusive + // access here which makes this safe. + // + // Also, since 'owner_val' is guaranteed to be initialized + // before an owned PoolGuard is created, the unwrap_unchecked + // is safe. + Err(id) => unsafe { + // This assert is *not* necessary for safety, since we + // should never be here if the guard had been put back into + // the pool. This is a sanity check to make sure we didn't + // break an internal invariant. + debug_assert_ne!(THREAD_ID_DROPPED, id); + (*self.pool.owner_val.get()).as_mut().unwrap_unchecked() + }, + } + } + + /// Consumes this guard and puts it back into the pool. + #[inline] + pub(super) fn put(this: PoolGuard<'_, T, F>) { + // Since this is effectively consuming the guard and putting the + // value back into the pool, there's no reason to run its Drop + // impl after doing this. I don't believe there is a correctness + // problem with doing so, but there's definitely a perf problem + // by redoing this work. So we avoid it. + let mut this = core::mem::ManuallyDrop::new(this); + this.put_imp(); + } + + /// Puts this guard back into the pool by only borrowing the guard as + /// mutable. This should be called at most once. + #[inline(always)] + fn put_imp(&mut self) { + match core::mem::replace(&mut self.value, Err(THREAD_ID_DROPPED)) { + Ok(value) => { + // If we were told to discard this value then don't bother + // trying to put it back into the pool. This occurs when + // the pop operation failed to acquire a lock and we + // decided to create a new value in lieu of contending for + // the lock. + if self.discard { + return; + } + self.pool.put_value(value); + } + // If this guard has a value "owned" by the thread, then + // the Pool guarantees that this is the ONLY such guard. + // Therefore, in order to place it back into the pool and make + // it available, we need to change the owner back to the owning + // thread's ID. But note that we use the ID that was stored in + // the guard, since a guard can be moved to another thread and + // dropped. (A previous iteration of this code read from the + // THREAD_ID thread local, which uses the ID of the current + // thread which may not be the ID of the owning thread! This + // also avoids the TLS access, which is likely a hair faster.) + Err(owner) => { + // If we hit this point, it implies 'put_imp' has been + // called multiple times for the same guard which in turn + // corresponds to a bug in this implementation. + assert_ne!(THREAD_ID_DROPPED, owner); + self.pool.owner.store(owner, Ordering::Release); + } + } + } + } + + impl<'a, T: Send, F: Fn() -> T> Drop for PoolGuard<'a, T, F> { + #[inline] + fn drop(&mut self) { + self.put_imp(); + } + } + + impl<'a, T: Send + core::fmt::Debug, F: Fn() -> T> core::fmt::Debug + for PoolGuard<'a, T, F> + { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_struct("PoolGuard") + .field("pool", &self.pool) + .field("value", &self.value) + .finish() + } + } +} + +// FUTURE: We should consider using Mara Bos's nearly-lock-free version of this +// here: https://gist.github.com/m-ou-se/5fdcbdf7dcf4585199ce2de697f367a4. +// +// One reason why I did things with a "mutex" below is that it isolates the +// safety concerns to just the Mutex, where as the safety of Mara's pool is a +// bit more sprawling. I also expect this code to not be used that much, and +// so is unlikely to get as much real world usage with which to test it. That +// means the "obviously correct" lever is an important one. +// +// The specific reason to use Mara's pool is that it is likely faster and also +// less likely to hit problems with spin-locks, although it is not completely +// impervious to them. +// +// The best solution to this problem, probably, is a truly lock free pool. That +// could be done with a lock free linked list. The issue is the ABA problem. It +// is difficult to avoid, and doing so is complex. BUT, the upshot of that is +// that if we had a truly lock free pool, then we could also use it above in +// the 'std' pool instead of a Mutex because it should be completely free the +// problems that come from spin-locks. +#[cfg(not(feature = "std"))] +mod inner { + use core::{ + cell::UnsafeCell, + panic::{RefUnwindSafe, UnwindSafe}, + sync::atomic::{AtomicBool, Ordering}, + }; + + use alloc::{boxed::Box, vec, vec::Vec}; + + /// A thread safe pool utilizing alloc-only features. + /// + /// Unlike the std version, it doesn't seem possible(?) to implement the + /// "thread owner" optimization because alloc-only doesn't have any concept + /// of threads. So the best we can do is just a normal stack. This will + /// increase latency in alloc-only environments. + pub(super) struct Pool<T, F> { + /// A stack of T values to hand out. These are used when a Pool is + /// accessed by a thread that didn't create it. + stack: Mutex<Vec<Box<T>>>, + /// A function to create more T values when stack is empty and a caller + /// has requested a T. + create: F, + } + + // If T is UnwindSafe, then since we provide exclusive access to any + // particular value in the pool, it should therefore also be considered + // RefUnwindSafe. + impl<T: UnwindSafe, F: UnwindSafe> RefUnwindSafe for Pool<T, F> {} + + impl<T, F> Pool<T, F> { + /// Create a new pool. The given closure is used to create values in + /// the pool when necessary. + pub(super) const fn new(create: F) -> Pool<T, F> { + Pool { stack: Mutex::new(vec![]), create } + } + + /// This is a no-op since this pool implementation isn't thread-aware. + pub(super) const fn with_capacity( + _capacity: usize, + create: F, + ) -> Pool<T, F> { + Pool::new(create) + } + } + + impl<T: Send, F: Fn() -> T> Pool<T, F> { + /// Get a value from the pool. This may block if another thread is also + /// attempting to retrieve a value from the pool. + #[inline] + pub(super) fn get(&self) -> PoolGuard<'_, T, F> { + let mut stack = self.stack.lock(); + let value = match stack.pop() { + None => Box::new((self.create)()), + Some(value) => value, + }; + PoolGuard { pool: self, value: Some(value) } + } + + #[inline] + fn put(&self, guard: PoolGuard<'_, T, F>) { + let mut guard = core::mem::ManuallyDrop::new(guard); + if let Some(value) = guard.value.take() { + self.put_value(value); + } + } + + /// Puts a value back into the pool. Callers don't need to call this. + /// Once the guard that's returned by 'get' is dropped, it is put back + /// into the pool automatically. + #[inline] + fn put_value(&self, value: Box<T>) { + let mut stack = self.stack.lock(); + stack.push(value); + } + } + + impl<T: core::fmt::Debug, F> core::fmt::Debug for Pool<T, F> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Pool").field("stack", &self.stack).finish() + } + } + + /// A guard that is returned when a caller requests a value from the pool. + pub(super) struct PoolGuard<'a, T: Send, F: Fn() -> T> { + /// The pool that this guard is attached to. + pool: &'a Pool<T, F>, + /// This is None after the guard has been put back into the pool. + value: Option<Box<T>>, + } + + impl<'a, T: Send, F: Fn() -> T> PoolGuard<'a, T, F> { + /// Return the underlying value. + #[inline] + pub(super) fn value(&self) -> &T { + self.value.as_deref().unwrap() + } + + /// Return the underlying value as a mutable borrow. + #[inline] + pub(super) fn value_mut(&mut self) -> &mut T { + self.value.as_deref_mut().unwrap() + } + + /// Consumes this guard and puts it back into the pool. + #[inline] + pub(super) fn put(this: PoolGuard<'_, T, F>) { + // Since this is effectively consuming the guard and putting the + // value back into the pool, there's no reason to run its Drop + // impl after doing this. I don't believe there is a correctness + // problem with doing so, but there's definitely a perf problem + // by redoing this work. So we avoid it. + let mut this = core::mem::ManuallyDrop::new(this); + this.put_imp(); + } + + /// Puts this guard back into the pool by only borrowing the guard as + /// mutable. This should be called at most once. + #[inline(always)] + fn put_imp(&mut self) { + if let Some(value) = self.value.take() { + self.pool.put_value(value); + } + } + } + + impl<'a, T: Send, F: Fn() -> T> Drop for PoolGuard<'a, T, F> { + #[inline] + fn drop(&mut self) { + self.put_imp(); + } + } + + impl<'a, T: Send + core::fmt::Debug, F: Fn() -> T> core::fmt::Debug + for PoolGuard<'a, T, F> + { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_struct("PoolGuard") + .field("pool", &self.pool) + .field("value", &self.value) + .finish() + } + } + + /// A spin-lock based mutex. Yes, I have read spinlocks considered + /// harmful[1], and if there's a reasonable alternative choice, I'll + /// happily take it. + /// + /// I suspect the most likely alternative here is a Treiber stack, but + /// implementing one correctly in a way that avoids the ABA problem looks + /// subtle enough that I'm not sure I want to attempt that. But otherwise, + /// we only need a mutex in order to implement our pool, so if there's + /// something simpler we can use that works for our `Pool` use case, then + /// that would be great. + /// + /// Note that this mutex does not do poisoning. + /// + /// [1]: https://matklad.github.io/2020/01/02/spinlocks-considered-harmful.html + #[derive(Debug)] + struct Mutex<T> { + locked: AtomicBool, + data: UnsafeCell<T>, + } + + // SAFETY: Since a Mutex guarantees exclusive access, as long as we can + // send it across threads, it must also be Sync. + unsafe impl<T: Send> Sync for Mutex<T> {} + + impl<T> Mutex<T> { + /// Create a new mutex for protecting access to the given value across + /// multiple threads simultaneously. + const fn new(value: T) -> Mutex<T> { + Mutex { + locked: AtomicBool::new(false), + data: UnsafeCell::new(value), + } + } + + /// Lock this mutex and return a guard providing exclusive access to + /// `T`. This blocks if some other thread has already locked this + /// mutex. + #[inline] + fn lock(&self) -> MutexGuard<'_, T> { + while self + .locked + .compare_exchange( + false, + true, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_err() + { + core::hint::spin_loop(); + } + // SAFETY: The only way we're here is if we successfully set + // 'locked' to true, which implies we must be the only thread here + // and thus have exclusive access to 'data'. + let data = unsafe { &mut *self.data.get() }; + MutexGuard { locked: &self.locked, data } + } + } + + /// A guard that derefs to &T and &mut T. When it's dropped, the lock is + /// released. + #[derive(Debug)] + struct MutexGuard<'a, T> { + locked: &'a AtomicBool, + data: &'a mut T, + } + + impl<'a, T> core::ops::Deref for MutexGuard<'a, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &T { + self.data + } + } + + impl<'a, T> core::ops::DerefMut for MutexGuard<'a, T> { + #[inline] + fn deref_mut(&mut self) -> &mut T { + self.data + } + } + + impl<'a, T> Drop for MutexGuard<'a, T> { + #[inline] + fn drop(&mut self) { + // Drop means 'data' is no longer accessible, so we can unlock + // the mutex. + self.locked.store(false, Ordering::Release); + } + } +} + +#[cfg(test)] +mod tests { + use core::panic::{RefUnwindSafe, UnwindSafe}; + + use alloc::{boxed::Box, vec, vec::Vec}; + + use super::*; + + #[test] + fn oibits() { + fn assert_oitbits<T: Send + Sync + UnwindSafe + RefUnwindSafe>() {} + assert_oitbits::<Pool<Vec<u32>>>(); + assert_oitbits::<Pool<core::cell::RefCell<Vec<u32>>>>(); + assert_oitbits::< + Pool< + Vec<u32>, + Box< + dyn Fn() -> Vec<u32> + + Send + + Sync + + UnwindSafe + + RefUnwindSafe, + >, + >, + >(); + } + + // Tests that Pool implements the "single owner" optimization. That is, the + // thread that first accesses the pool gets its own copy, while all other + // threads get distinct copies. + #[cfg(feature = "std")] + #[test] + fn thread_owner_optimization() { + use std::{cell::RefCell, sync::Arc, vec}; + + let pool: Arc<Pool<RefCell<Vec<char>>>> = + Arc::new(Pool::new(|| RefCell::new(vec!['a']))); + pool.get().borrow_mut().push('x'); + + let pool1 = pool.clone(); + let t1 = std::thread::spawn(move || { + let guard = pool1.get(); + guard.borrow_mut().push('y'); + }); + + let pool2 = pool.clone(); + let t2 = std::thread::spawn(move || { + let guard = pool2.get(); + guard.borrow_mut().push('z'); + }); + + t1.join().unwrap(); + t2.join().unwrap(); + + // If we didn't implement the single owner optimization, then one of + // the threads above is likely to have mutated the [a, x] vec that + // we stuffed in the pool before spawning the threads. But since + // neither thread was first to access the pool, and because of the + // optimization, we should be guaranteed that neither thread mutates + // the special owned pool value. + // + // (Technically this is an implementation detail and not a contract of + // Pool's API.) + assert_eq!(vec!['a', 'x'], *pool.get().borrow()); + } + + // This tests that if the "owner" of a pool asks for two values, then it + // gets two distinct values and not the same one. This test failed in the + // course of developing the pool, which in turn resulted in UB because it + // permitted getting aliasing &mut borrows to the same place in memory. + #[test] + fn thread_owner_distinct() { + let pool = Pool::new(|| vec!['a']); + + { + let mut g1 = pool.get(); + let v1 = &mut *g1; + let mut g2 = pool.get(); + let v2 = &mut *g2; + v1.push('b'); + v2.push('c'); + assert_eq!(&mut vec!['a', 'b'], v1); + assert_eq!(&mut vec!['a', 'c'], v2); + } + // This isn't technically guaranteed, but we + // expect to now get the "owned" value (the first + // call to 'get()' above) now that it's back in + // the pool. + assert_eq!(&mut vec!['a', 'b'], &mut *pool.get()); + } + + // This tests that we can share a guard with another thread, mutate the + // underlying value and everything works. This failed in the course of + // developing a pool since the pool permitted 'get()' to return the same + // value to the owner thread, even before the previous value was put back + // into the pool. This in turn resulted in this test producing a data race. + #[cfg(feature = "std")] + #[test] + fn thread_owner_sync() { + let pool = Pool::new(|| vec!['a']); + { + let mut g1 = pool.get(); + let mut g2 = pool.get(); + std::thread::scope(|s| { + s.spawn(|| { + g1.push('b'); + }); + s.spawn(|| { + g2.push('c'); + }); + }); + + let v1 = &mut *g1; + let v2 = &mut *g2; + assert_eq!(&mut vec!['a', 'b'], v1); + assert_eq!(&mut vec!['a', 'c'], v2); + } + + // This isn't technically guaranteed, but we + // expect to now get the "owned" value (the first + // call to 'get()' above) now that it's back in + // the pool. + assert_eq!(&mut vec!['a', 'b'], &mut *pool.get()); + } + + // This tests that if we move a PoolGuard that is owned by the current + // thread to another thread and drop it, then the thread owner doesn't + // change. During development of the pool, this test failed because the + // PoolGuard assumed it was dropped in the same thread from which it was + // created, and thus used the current thread's ID as the owner, which could + // be different than the actual owner of the pool. + #[cfg(feature = "std")] + #[test] + fn thread_owner_send_drop() { + let pool = Pool::new(|| vec!['a']); + // Establishes this thread as the owner. + { + pool.get().push('b'); + } + std::thread::scope(|s| { + // Sanity check that we get the same value back. + // (Not technically guaranteed.) + let mut g = pool.get(); + assert_eq!(&vec!['a', 'b'], &*g); + // Now push it to another thread and drop it. + s.spawn(move || { + g.push('c'); + }) + .join() + .unwrap(); + }); + // Now check that we're still the owner. This is not technically + // guaranteed by the API, but is true in practice given the thread + // owner optimization. + assert_eq!(&vec!['a', 'b', 'c'], &*pool.get()); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/prefilter/aho_corasick.rs b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/aho_corasick.rs new file mode 100644 index 0000000..4126bb6 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/aho_corasick.rs @@ -0,0 +1,153 @@ +use crate::util::{ + prefilter::PrefilterI, + search::{MatchKind, Span}, +}; + +#[derive(Clone, Debug)] +pub(crate) struct AhoCorasick { + #[cfg(not(feature = "perf-literal-multisubstring"))] + _unused: (), + #[cfg(feature = "perf-literal-multisubstring")] + ac: aho_corasick::AhoCorasick, +} + +impl AhoCorasick { + pub(crate) fn new<B: AsRef<[u8]>>( + kind: MatchKind, + needles: &[B], + ) -> Option<AhoCorasick> { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + None + } + #[cfg(feature = "perf-literal-multisubstring")] + { + // We used to use `aho_corasick::MatchKind::Standard` here when + // `kind` was `MatchKind::All`, but this is not correct. The + // "standard" Aho-Corasick match semantics are to report a match + // immediately as soon as it is seen, but `All` isn't like that. + // In particular, with "standard" semantics, given the needles + // "abc" and "b" and the haystack "abc," it would report a match + // at offset 1 before a match at offset 0. This is never what we + // want in the context of the regex engine, regardless of whether + // we have leftmost-first or 'all' semantics. Namely, we always + // want the leftmost match. + let ac_match_kind = match kind { + MatchKind::LeftmostFirst | MatchKind::All => { + aho_corasick::MatchKind::LeftmostFirst + } + }; + // This is kind of just an arbitrary number, but basically, if we + // have a small enough set of literals, then we try to use the VERY + // memory hungry DFA. Otherwise, we wimp out and use an NFA. The + // upshot is that the NFA is quite lean and decently fast. Faster + // than a naive Aho-Corasick NFA anyway. + let ac_kind = if needles.len() <= 500 { + aho_corasick::AhoCorasickKind::DFA + } else { + aho_corasick::AhoCorasickKind::ContiguousNFA + }; + let result = aho_corasick::AhoCorasick::builder() + .kind(Some(ac_kind)) + .match_kind(ac_match_kind) + .start_kind(aho_corasick::StartKind::Both) + // We try to handle all of the prefilter cases in the super + // module, and only use Aho-Corasick for the actual automaton. + // The aho-corasick crate does have some extra prefilters, + // namely, looking for rare bytes to feed to memchr{,2,3} + // instead of just the first byte. If we end up wanting + // those---and they are somewhat tricky to implement---then + // we could port them to this crate. + // + // The main reason for doing things this way is so we have a + // complete and easy to understand picture of which prefilters + // are available and how they work. Otherwise it seems too + // easy to get into a situation where we have a prefilter + // layered on top of prefilter, and that might have unintended + // consequences. + .prefilter(false) + .build(needles); + let ac = match result { + Ok(ac) => ac, + Err(_err) => { + debug!("aho-corasick prefilter failed to build: {_err}"); + return None; + } + }; + Some(AhoCorasick { ac }) + } + } +} + +impl PrefilterI for AhoCorasick { + fn name(&self) -> &'static str { + "aho-corasick" + } + + fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-multisubstring")] + { + let input = + aho_corasick::Input::new(haystack).span(span.start..span.end); + self.ac + .find(input) + .map(|m| Span { start: m.start(), end: m.end() }) + } + } + + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-multisubstring")] + { + let input = aho_corasick::Input::new(haystack) + .anchored(aho_corasick::Anchored::Yes) + .span(span.start..span.end); + self.ac + .find(input) + .map(|m| Span { start: m.start(), end: m.end() }) + } + } + + fn memory_usage(&self) -> usize { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-multisubstring")] + { + self.ac.memory_usage() + } + } + + fn is_fast(&self) -> bool { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-multisubstring")] + { + // Aho-Corasick is never considered "fast" because it's never + // going to be even close to an order of magnitude faster than the + // regex engine itself (assuming a DFA is used). In fact, it is + // usually slower. The magic of Aho-Corasick is that it can search + // a *large* number of literals with a relatively small amount of + // memory. The regex engines are far more wasteful. + // + // Aho-Corasick may be "fast" when the regex engine corresponds + // to, say, the PikeVM. That happens when the lazy DFA couldn't be + // built or used for some reason. But in these cases, the regex + // itself is likely quite big and we're probably hosed no matter + // what we do. (In this case, the best bet is for the caller to + // increase some of the memory limits on the hybrid cache capacity + // and hope that's enough.) + false + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/prefilter/byteset.rs b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/byteset.rs new file mode 100644 index 0000000..31599a3 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/byteset.rs @@ -0,0 +1,62 @@ +use crate::util::{ + prefilter::PrefilterI, + search::{MatchKind, Span}, +}; + +#[derive(Clone, Debug)] +pub(crate) struct ByteSet([bool; 256]); + +impl ByteSet { + pub(crate) fn new<B: AsRef<[u8]>>( + _kind: MatchKind, + needles: &[B], + ) -> Option<ByteSet> { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + None + } + #[cfg(feature = "perf-literal-multisubstring")] + { + let mut set = [false; 256]; + for needle in needles.iter() { + let needle = needle.as_ref(); + if needle.len() != 1 { + return None; + } + set[usize::from(needle[0])] = true; + } + Some(ByteSet(set)) + } + } +} + +impl PrefilterI for ByteSet { + fn name(&self) -> &'static str { + "byteset" + } + + fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + haystack[span].iter().position(|&b| self.0[usize::from(b)]).map(|i| { + let start = span.start + i; + let end = start + 1; + Span { start, end } + }) + } + + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + let b = *haystack.get(span.start)?; + if self.0[usize::from(b)] { + Some(Span { start: span.start, end: span.start + 1 }) + } else { + None + } + } + + fn memory_usage(&self) -> usize { + 0 + } + + fn is_fast(&self) -> bool { + false + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/prefilter/memchr.rs b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/memchr.rs new file mode 100644 index 0000000..5b74671 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/memchr.rs @@ -0,0 +1,198 @@ +use crate::util::{ + prefilter::PrefilterI, + search::{MatchKind, Span}, +}; + +#[derive(Clone, Debug)] +pub(crate) struct Memchr(u8); + +impl Memchr { + pub(crate) fn new<B: AsRef<[u8]>>( + _kind: MatchKind, + needles: &[B], + ) -> Option<Memchr> { + #[cfg(not(feature = "perf-literal-substring"))] + { + None + } + #[cfg(feature = "perf-literal-substring")] + { + if needles.len() != 1 { + return None; + } + if needles[0].as_ref().len() != 1 { + return None; + } + Some(Memchr(needles[0].as_ref()[0])) + } + } +} + +impl PrefilterI for Memchr { + fn name(&self) -> &'static str { + "memchr" + } + + fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "perf-literal-substring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-substring")] + { + memchr::memchr(self.0, &haystack[span]).map(|i| { + let start = span.start + i; + let end = start + 1; + Span { start, end } + }) + } + } + + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + let b = *haystack.get(span.start)?; + if self.0 == b { + Some(Span { start: span.start, end: span.start + 1 }) + } else { + None + } + } + + fn memory_usage(&self) -> usize { + 0 + } + + fn is_fast(&self) -> bool { + true + } +} + +#[derive(Clone, Debug)] +pub(crate) struct Memchr2(u8, u8); + +impl Memchr2 { + pub(crate) fn new<B: AsRef<[u8]>>( + _kind: MatchKind, + needles: &[B], + ) -> Option<Memchr2> { + #[cfg(not(feature = "perf-literal-substring"))] + { + None + } + #[cfg(feature = "perf-literal-substring")] + { + if needles.len() != 2 { + return None; + } + if !needles.iter().all(|n| n.as_ref().len() == 1) { + return None; + } + let b1 = needles[0].as_ref()[0]; + let b2 = needles[1].as_ref()[0]; + Some(Memchr2(b1, b2)) + } + } +} + +impl PrefilterI for Memchr2 { + fn name(&self) -> &'static str { + "memchr2" + } + + fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "perf-literal-substring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-substring")] + { + memchr::memchr2(self.0, self.1, &haystack[span]).map(|i| { + let start = span.start + i; + let end = start + 1; + Span { start, end } + }) + } + } + + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + let b = *haystack.get(span.start)?; + if self.0 == b || self.1 == b { + Some(Span { start: span.start, end: span.start + 1 }) + } else { + None + } + } + + fn memory_usage(&self) -> usize { + 0 + } + + fn is_fast(&self) -> bool { + true + } +} + +#[derive(Clone, Debug)] +pub(crate) struct Memchr3(u8, u8, u8); + +impl Memchr3 { + pub(crate) fn new<B: AsRef<[u8]>>( + _kind: MatchKind, + needles: &[B], + ) -> Option<Memchr3> { + #[cfg(not(feature = "perf-literal-substring"))] + { + None + } + #[cfg(feature = "perf-literal-substring")] + { + if needles.len() != 3 { + return None; + } + if !needles.iter().all(|n| n.as_ref().len() == 1) { + return None; + } + let b1 = needles[0].as_ref()[0]; + let b2 = needles[1].as_ref()[0]; + let b3 = needles[2].as_ref()[0]; + Some(Memchr3(b1, b2, b3)) + } + } +} + +impl PrefilterI for Memchr3 { + fn name(&self) -> &'static str { + "memchr3" + } + + fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "perf-literal-substring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-substring")] + { + memchr::memchr3(self.0, self.1, self.2, &haystack[span]).map(|i| { + let start = span.start + i; + let end = start + 1; + Span { start, end } + }) + } + } + + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + let b = *haystack.get(span.start)?; + if self.0 == b || self.1 == b || self.2 == b { + Some(Span { start: span.start, end: span.start + 1 }) + } else { + None + } + } + + fn memory_usage(&self) -> usize { + 0 + } + + fn is_fast(&self) -> bool { + true + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/prefilter/memmem.rs b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/memmem.rs new file mode 100644 index 0000000..63490c7 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/memmem.rs @@ -0,0 +1,92 @@ +use crate::util::{ + prefilter::PrefilterI, + search::{MatchKind, Span}, +}; + +#[derive(Clone, Debug)] +pub(crate) struct Memmem { + #[cfg(not(all(feature = "std", feature = "perf-literal-substring")))] + _unused: (), + #[cfg(all(feature = "std", feature = "perf-literal-substring"))] + finder: memchr::memmem::Finder<'static>, +} + +impl Memmem { + pub(crate) fn new<B: AsRef<[u8]>>( + _kind: MatchKind, + needles: &[B], + ) -> Option<Memmem> { + #[cfg(not(all(feature = "std", feature = "perf-literal-substring")))] + { + None + } + #[cfg(all(feature = "std", feature = "perf-literal-substring"))] + { + if needles.len() != 1 { + return None; + } + let needle = needles[0].as_ref(); + let finder = memchr::memmem::Finder::new(needle).into_owned(); + Some(Memmem { finder }) + } + } +} + +impl PrefilterI for Memmem { + fn name(&self) -> &'static str { + "memmem" + } + + fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(all(feature = "std", feature = "perf-literal-substring")))] + { + unreachable!() + } + #[cfg(all(feature = "std", feature = "perf-literal-substring"))] + { + self.finder.find(&haystack[span]).map(|i| { + let start = span.start + i; + let end = start + self.finder.needle().len(); + Span { start, end } + }) + } + } + + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(all(feature = "std", feature = "perf-literal-substring")))] + { + unreachable!() + } + #[cfg(all(feature = "std", feature = "perf-literal-substring"))] + { + let needle = self.finder.needle(); + if haystack[span].starts_with(needle) { + Some(Span { end: span.start + needle.len(), ..span }) + } else { + None + } + } + } + + fn memory_usage(&self) -> usize { + #[cfg(not(all(feature = "std", feature = "perf-literal-substring")))] + { + unreachable!() + } + #[cfg(all(feature = "std", feature = "perf-literal-substring"))] + { + self.finder.needle().len() + } + } + + fn is_fast(&self) -> bool { + #[cfg(not(all(feature = "std", feature = "perf-literal-substring")))] + { + unreachable!() + } + #[cfg(all(feature = "std", feature = "perf-literal-substring"))] + { + true + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/prefilter/mod.rs b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/mod.rs new file mode 100644 index 0000000..3f50df2 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/mod.rs @@ -0,0 +1,726 @@ +/*! +Defines a prefilter for accelerating regex searches. + +A prefilter can be created by building a [`Prefilter`] value. + +A prefilter represents one of the most important optimizations available for +accelerating regex searches. The idea of a prefilter is to very quickly find +candidate locations in a haystack where a regex _could_ match. Once a candidate +is found, it is then intended for the regex engine to run at that position to +determine whether the candidate is a match or a false positive. + +In the aforementioned description of the prefilter optimization also lay its +demise. Namely, if a prefilter has a high false positive rate and it produces +lots of candidates, then a prefilter can overall make a regex search slower. +It can run more slowly because more time is spent ping-ponging between the +prefilter search and the regex engine attempting to confirm each candidate as +a match. This ping-ponging has overhead that adds up, and is exacerbated by +a high false positive rate. + +Nevertheless, the optimization is still generally worth performing in most +cases. Particularly given just how much throughput can be improved. (It is not +uncommon for prefilter optimizations to improve throughput by one or two orders +of magnitude.) + +Typically a prefilter is used to find occurrences of literal prefixes from a +regex pattern, but this isn't required. A prefilter can be used to look for +suffixes or even inner literals. + +Note that as of now, prefilters throw away information about which pattern +each literal comes from. In other words, when a prefilter finds a match, +there's no way to know which pattern (or patterns) it came from. Therefore, +in order to confirm a match, you'll have to check all of the patterns by +running the full regex engine. +*/ + +mod aho_corasick; +mod byteset; +mod memchr; +mod memmem; +mod teddy; + +use core::{ + borrow::Borrow, + fmt::Debug, + panic::{RefUnwindSafe, UnwindSafe}, +}; + +#[cfg(feature = "alloc")] +use alloc::sync::Arc; + +#[cfg(feature = "syntax")] +use regex_syntax::hir::{literal, Hir}; + +use crate::util::search::{MatchKind, Span}; + +pub(crate) use crate::util::prefilter::{ + aho_corasick::AhoCorasick, + byteset::ByteSet, + memchr::{Memchr, Memchr2, Memchr3}, + memmem::Memmem, + teddy::Teddy, +}; + +/// A prefilter for accelerating regex searches. +/// +/// If you already have your literals that you want to search with, +/// then the vanilla [`Prefilter::new`] constructor is for you. But +/// if you have an [`Hir`] value from the `regex-syntax` crate, then +/// [`Prefilter::from_hir_prefix`] might be more convenient. Namely, it uses +/// the [`regex-syntax::hir::literal`](regex_syntax::hir::literal) module to +/// extract literal prefixes for you, optimize them and then select and build a +/// prefilter matcher. +/// +/// A prefilter must have **zero false negatives**. However, by its very +/// nature, it may produce false positives. That is, a prefilter will never +/// skip over a position in the haystack that corresponds to a match of the +/// original regex pattern, but it *may* produce a match for a position +/// in the haystack that does *not* correspond to a match of the original +/// regex pattern. If you use either the [`Prefilter::from_hir_prefix`] or +/// [`Prefilter::from_hirs_prefix`] constructors, then this guarantee is +/// upheld for you automatically. This guarantee is not preserved if you use +/// [`Prefilter::new`] though, since it is up to the caller to provide correct +/// literal strings with respect to the original regex pattern. +/// +/// # Cloning +/// +/// It is an API guarantee that cloning a prefilter is cheap. That is, cloning +/// it will not duplicate whatever heap memory is used to represent the +/// underlying matcher. +/// +/// # Example +/// +/// This example shows how to attach a `Prefilter` to the +/// [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM) in order to accelerate +/// searches. +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::pikevm::PikeVM, +/// util::prefilter::Prefilter, +/// Match, MatchKind, +/// }; +/// +/// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["Bruce "]) +/// .expect("a prefilter"); +/// let re = PikeVM::builder() +/// .configure(PikeVM::config().prefilter(Some(pre))) +/// .build(r"Bruce \w+")?; +/// let mut cache = re.create_cache(); +/// assert_eq!( +/// Some(Match::must(0, 6..23)), +/// re.find(&mut cache, "Hello Bruce Springsteen!"), +/// ); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// But note that if you get your prefilter incorrect, it could lead to an +/// incorrect result! +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::pikevm::PikeVM, +/// util::prefilter::Prefilter, +/// Match, MatchKind, +/// }; +/// +/// // This prefilter is wrong! +/// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["Patti "]) +/// .expect("a prefilter"); +/// let re = PikeVM::builder() +/// .configure(PikeVM::config().prefilter(Some(pre))) +/// .build(r"Bruce \w+")?; +/// let mut cache = re.create_cache(); +/// // We find no match even though the regex does match. +/// assert_eq!( +/// None, +/// re.find(&mut cache, "Hello Bruce Springsteen!"), +/// ); +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Prefilter { + #[cfg(not(feature = "alloc"))] + _unused: (), + #[cfg(feature = "alloc")] + pre: Arc<dyn PrefilterI>, + #[cfg(feature = "alloc")] + is_fast: bool, + #[cfg(feature = "alloc")] + max_needle_len: usize, +} + +impl Prefilter { + /// Create a new prefilter from a sequence of needles and a corresponding + /// match semantics. + /// + /// This may return `None` for a variety of reasons, for example, if + /// a suitable prefilter could not be constructed. That might occur + /// if they are unavailable (e.g., the `perf-literal-substring` and + /// `perf-literal-multisubstring` features aren't enabled), or it might + /// occur because of heuristics or other artifacts of how the prefilter + /// works. + /// + /// Note that if you have an [`Hir`] expression, it may be more convenient + /// to use [`Prefilter::from_hir_prefix`]. It will automatically handle the + /// task of extracting prefix literals for you. + /// + /// # Example + /// + /// This example shows how match semantics can impact the matching + /// algorithm used by the prefilter. For this reason, it is important to + /// ensure that the match semantics given here are consistent with the + /// match semantics intended for the regular expression that the literals + /// were extracted from. + /// + /// ``` + /// use regex_automata::{ + /// util::{prefilter::Prefilter, syntax}, + /// MatchKind, Span, + /// }; + /// + /// let hay = "Hello samwise"; + /// + /// // With leftmost-first, we find 'samwise' here because it comes + /// // before 'sam' in the sequence we give it.. + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["samwise", "sam"]) + /// .expect("a prefilter"); + /// assert_eq!( + /// Some(Span::from(6..13)), + /// pre.find(hay.as_bytes(), Span::from(0..hay.len())), + /// ); + /// // Still with leftmost-first but with the literals reverse, now 'sam' + /// // will match instead! + /// let pre = Prefilter::new(MatchKind::LeftmostFirst, &["sam", "samwise"]) + /// .expect("a prefilter"); + /// assert_eq!( + /// Some(Span::from(6..9)), + /// pre.find(hay.as_bytes(), Span::from(0..hay.len())), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn new<B: AsRef<[u8]>>( + kind: MatchKind, + needles: &[B], + ) -> Option<Prefilter> { + Choice::new(kind, needles).and_then(|choice| { + let max_needle_len = + needles.iter().map(|b| b.as_ref().len()).max().unwrap_or(0); + Prefilter::from_choice(choice, max_needle_len) + }) + } + + /// This turns a prefilter selection into a `Prefilter`. That is, in turns + /// the enum given into a trait object. + fn from_choice( + choice: Choice, + max_needle_len: usize, + ) -> Option<Prefilter> { + #[cfg(not(feature = "alloc"))] + { + None + } + #[cfg(feature = "alloc")] + { + let pre: Arc<dyn PrefilterI> = match choice { + Choice::Memchr(p) => Arc::new(p), + Choice::Memchr2(p) => Arc::new(p), + Choice::Memchr3(p) => Arc::new(p), + Choice::Memmem(p) => Arc::new(p), + Choice::Teddy(p) => Arc::new(p), + Choice::ByteSet(p) => Arc::new(p), + Choice::AhoCorasick(p) => Arc::new(p), + }; + let is_fast = pre.is_fast(); + Some(Prefilter { pre, is_fast, max_needle_len }) + } + } + + /// This attempts to extract prefixes from the given `Hir` expression for + /// the given match semantics, and if possible, builds a prefilter for + /// them. + /// + /// # Example + /// + /// This example shows how to build a prefilter directly from an [`Hir`] + /// expression, and use to find an occurrence of a prefix from the regex + /// pattern. + /// + /// ``` + /// use regex_automata::{ + /// util::{prefilter::Prefilter, syntax}, + /// MatchKind, Span, + /// }; + /// + /// let hir = syntax::parse(r"(Bruce|Patti) \w+")?; + /// let pre = Prefilter::from_hir_prefix(MatchKind::LeftmostFirst, &hir) + /// .expect("a prefilter"); + /// let hay = "Hello Patti Scialfa!"; + /// assert_eq!( + /// Some(Span::from(6..12)), + /// pre.find(hay.as_bytes(), Span::from(0..hay.len())), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn from_hir_prefix(kind: MatchKind, hir: &Hir) -> Option<Prefilter> { + Prefilter::from_hirs_prefix(kind, &[hir]) + } + + /// This attempts to extract prefixes from the given `Hir` expressions for + /// the given match semantics, and if possible, builds a prefilter for + /// them. + /// + /// Note that as of now, prefilters throw away information about which + /// pattern each literal comes from. In other words, when a prefilter finds + /// a match, there's no way to know which pattern (or patterns) it came + /// from. Therefore, in order to confirm a match, you'll have to check all + /// of the patterns by running the full regex engine. + /// + /// # Example + /// + /// This example shows how to build a prefilter directly from multiple + /// `Hir` expressions expression, and use it to find an occurrence of a + /// prefix from the regex patterns. + /// + /// ``` + /// use regex_automata::{ + /// util::{prefilter::Prefilter, syntax}, + /// MatchKind, Span, + /// }; + /// + /// let hirs = syntax::parse_many(&[ + /// r"(Bruce|Patti) \w+", + /// r"Mrs?\. Doubtfire", + /// ])?; + /// let pre = Prefilter::from_hirs_prefix(MatchKind::LeftmostFirst, &hirs) + /// .expect("a prefilter"); + /// let hay = "Hello Mrs. Doubtfire"; + /// assert_eq!( + /// Some(Span::from(6..20)), + /// pre.find(hay.as_bytes(), Span::from(0..hay.len())), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[cfg(feature = "syntax")] + pub fn from_hirs_prefix<H: Borrow<Hir>>( + kind: MatchKind, + hirs: &[H], + ) -> Option<Prefilter> { + prefixes(kind, hirs) + .literals() + .and_then(|lits| Prefilter::new(kind, lits)) + } + + /// Run this prefilter on `haystack[span.start..end]` and return a matching + /// span if one exists. + /// + /// The span returned is guaranteed to have a start position greater than + /// or equal to the one given, and an end position less than or equal to + /// the one given. + /// + /// # Example + /// + /// This example shows how to build a prefilter directly from an [`Hir`] + /// expression, and use it to find an occurrence of a prefix from the regex + /// pattern. + /// + /// ``` + /// use regex_automata::{ + /// util::{prefilter::Prefilter, syntax}, + /// MatchKind, Span, + /// }; + /// + /// let hir = syntax::parse(r"Bruce \w+")?; + /// let pre = Prefilter::from_hir_prefix(MatchKind::LeftmostFirst, &hir) + /// .expect("a prefilter"); + /// let hay = "Hello Bruce Springsteen!"; + /// assert_eq!( + /// Some(Span::from(6..12)), + /// pre.find(hay.as_bytes(), Span::from(0..hay.len())), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "alloc"))] + { + unreachable!() + } + #[cfg(feature = "alloc")] + { + self.pre.find(haystack, span) + } + } + + /// Returns the span of a prefix of `haystack[span.start..span.end]` if + /// the prefilter matches. + /// + /// The span returned is guaranteed to have a start position equivalent to + /// the one given, and an end position less than or equal to the one given. + /// + /// # Example + /// + /// This example shows how to build a prefilter directly from an [`Hir`] + /// expression, and use it to find an occurrence of a prefix from the regex + /// pattern that begins at the start of a haystack only. + /// + /// ``` + /// use regex_automata::{ + /// util::{prefilter::Prefilter, syntax}, + /// MatchKind, Span, + /// }; + /// + /// let hir = syntax::parse(r"Bruce \w+")?; + /// let pre = Prefilter::from_hir_prefix(MatchKind::LeftmostFirst, &hir) + /// .expect("a prefilter"); + /// let hay = "Hello Bruce Springsteen!"; + /// // Nothing is found here because 'Bruce' does + /// // not occur at the beginning of our search. + /// assert_eq!( + /// None, + /// pre.prefix(hay.as_bytes(), Span::from(0..hay.len())), + /// ); + /// // But if we change where we start the search + /// // to begin where 'Bruce ' begins, then a + /// // match will be found. + /// assert_eq!( + /// Some(Span::from(6..12)), + /// pre.prefix(hay.as_bytes(), Span::from(6..hay.len())), + /// ); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "alloc"))] + { + unreachable!() + } + #[cfg(feature = "alloc")] + { + self.pre.prefix(haystack, span) + } + } + + /// Returns the heap memory, in bytes, used by the underlying prefilter. + #[inline] + pub fn memory_usage(&self) -> usize { + #[cfg(not(feature = "alloc"))] + { + unreachable!() + } + #[cfg(feature = "alloc")] + { + self.pre.memory_usage() + } + } + + /// Return the length of the longest needle + /// in this Prefilter + #[inline] + pub fn max_needle_len(&self) -> usize { + #[cfg(not(feature = "alloc"))] + { + unreachable!() + } + #[cfg(feature = "alloc")] + { + self.max_needle_len + } + } + + /// Implementations might return true here if they believe themselves to + /// be "fast." The concept of "fast" is deliberately left vague, but in + /// practice this usually corresponds to whether it's believed that SIMD + /// will be used. + /// + /// Why do we care about this? Well, some prefilter tricks tend to come + /// with their own bits of overhead, and so might only make sense if we + /// know that a scan will be *much* faster than the regex engine itself. + /// Otherwise, the trick may not be worth doing. Whether something is + /// "much" faster than the regex engine generally boils down to whether + /// SIMD is used. (But not always. Even a SIMD matcher with a high false + /// positive rate can become quite slow.) + /// + /// Even if this returns true, it is still possible for the prefilter to + /// be "slow." Remember, prefilters are just heuristics. We can't really + /// *know* a prefilter will be fast without actually trying the prefilter. + /// (Which of course we cannot afford to do.) + #[inline] + pub fn is_fast(&self) -> bool { + #[cfg(not(feature = "alloc"))] + { + unreachable!() + } + #[cfg(feature = "alloc")] + { + self.is_fast + } + } +} + +/// A trait for abstracting over prefilters. Basically, a prefilter is +/// something that do an unanchored *and* an anchored search in a haystack +/// within a given span. +/// +/// This exists pretty much only so that we can use prefilters as a trait +/// object (which is what `Prefilter` is). If we ever move off of trait objects +/// and to an enum, then it's likely this trait could be removed. +pub(crate) trait PrefilterI: + Debug + Send + Sync + RefUnwindSafe + UnwindSafe + 'static +{ + /// Return the name of this prefilter. + fn name(&self) -> &'static str; + + /// Run this prefilter on `haystack[span.start..end]` and return a matching + /// span if one exists. + /// + /// The span returned is guaranteed to have a start position greater than + /// or equal to the one given, and an end position less than or equal to + /// the one given. + fn find(&self, haystack: &[u8], span: Span) -> Option<Span>; + + /// Returns the span of a prefix of `haystack[span.start..span.end]` if + /// the prefilter matches. + /// + /// The span returned is guaranteed to have a start position equivalent to + /// the one given, and an end position less than or equal to the one given. + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span>; + + /// Returns the heap memory, in bytes, used by the underlying prefilter. + fn memory_usage(&self) -> usize; + + /// Implementations might return true here if they believe themselves to + /// be "fast." See [`Prefilter::is_fast`] for more details. + fn is_fast(&self) -> bool; +} + +#[cfg(feature = "alloc")] +impl<P: PrefilterI + ?Sized> PrefilterI for Arc<P> { + fn name(&self) -> &'static str { + (**self).name() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + (**self).find(haystack, span) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + (**self).prefix(haystack, span) + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn memory_usage(&self) -> usize { + (**self).memory_usage() + } + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn is_fast(&self) -> bool { + (&**self).is_fast() + } +} + +/// A type that encapsulates the selection of a prefilter algorithm from a +/// sequence of needles. +/// +/// The existence of this type is a little tricky, because we don't (currently) +/// use it for performing a search. Instead, we really only consume it by +/// converting the underlying prefilter into a trait object, whether that be +/// `dyn PrefilterI` or `dyn Strategy` (for the meta regex engine). In order +/// to avoid re-copying the prefilter selection logic, we isolate it here, and +/// then force anything downstream that wants to convert it to a trait object +/// to do trivial case analysis on it. +/// +/// One wonders whether we *should* use an enum instead of a trait object. +/// At time of writing, I chose trait objects based on instinct because 1) I +/// knew I wasn't going to inline anything and 2) there would potentially be +/// many different choices. However, as of time of writing, I haven't actually +/// compared the trait object approach to the enum approach. That probably +/// should be litigated, but I ran out of steam. +/// +/// Note that if the `alloc` feature is disabled, then values of this type +/// are (and should) never be constructed. Also, in practice, for any of the +/// prefilters to be selected, you'll need at least one of the `perf-literal-*` +/// features enabled. +#[derive(Clone, Debug)] +pub(crate) enum Choice { + Memchr(Memchr), + Memchr2(Memchr2), + Memchr3(Memchr3), + Memmem(Memmem), + Teddy(Teddy), + ByteSet(ByteSet), + AhoCorasick(AhoCorasick), +} + +impl Choice { + /// Select what is believed to be the best prefilter algorithm for the + /// match semantics and sequence of needles given. + /// + /// This selection algorithm uses the needles as given without any + /// modification. For example, if `[bar]` is given, then this doesn't + /// try to select `memchr` for `b`. Instead, it would select `memmem` + /// for `bar`. If callers would want `memchr` selected for `[bar]`, then + /// callers should massages the literals themselves. That is, callers are + /// responsible for heuristics surrounding which sequence of literals is + /// best. + /// + /// What this selection algorithm does is attempt to use the fastest + /// prefilter that works for the literals given. So if `[a, b]`, is given, + /// then `memchr2` is selected. + /// + /// Of course, which prefilter is selected is also subject to what + /// is available. For example, if `alloc` isn't enabled, then + /// that limits which prefilters can be selected. Similarly, if + /// `perf-literal-substring` isn't enabled, then nothing from the `memchr` + /// crate can be returned. + pub(crate) fn new<B: AsRef<[u8]>>( + kind: MatchKind, + needles: &[B], + ) -> Option<Choice> { + // An empty set means the regex matches nothing, so no sense in + // building a prefilter. + if needles.len() == 0 { + debug!("prefilter building failed: found empty set of literals"); + return None; + } + // If the regex can match the empty string, then the prefilter + // will by definition match at every position. This is obviously + // completely ineffective. + if needles.iter().any(|n| n.as_ref().is_empty()) { + debug!("prefilter building failed: literals match empty string"); + return None; + } + // BREADCRUMBS: Perhaps the literal optimizer should special case + // sequences of length two or three if the leading bytes of each are + // "rare"? Or perhaps, if there are two or three total possible leading + // bytes, regardless of the number of literals, and all are rare... + // Then well, perhaps we should use memchr2 or memchr3 in those cases? + if let Some(pre) = Memchr::new(kind, needles) { + debug!("prefilter built: memchr"); + return Some(Choice::Memchr(pre)); + } + if let Some(pre) = Memchr2::new(kind, needles) { + debug!("prefilter built: memchr2"); + return Some(Choice::Memchr2(pre)); + } + if let Some(pre) = Memchr3::new(kind, needles) { + debug!("prefilter built: memchr3"); + return Some(Choice::Memchr3(pre)); + } + if let Some(pre) = Memmem::new(kind, needles) { + debug!("prefilter built: memmem"); + return Some(Choice::Memmem(pre)); + } + if let Some(pre) = Teddy::new(kind, needles) { + debug!("prefilter built: teddy"); + return Some(Choice::Teddy(pre)); + } + if let Some(pre) = ByteSet::new(kind, needles) { + debug!("prefilter built: byteset"); + return Some(Choice::ByteSet(pre)); + } + if let Some(pre) = AhoCorasick::new(kind, needles) { + debug!("prefilter built: aho-corasick"); + return Some(Choice::AhoCorasick(pre)); + } + debug!("prefilter building failed: no strategy could be found"); + None + } +} + +/// Extracts all of the prefix literals from the given HIR expressions into a +/// single `Seq`. The literals in the sequence are ordered with respect to the +/// order of the given HIR expressions and consistent with the match semantics +/// given. +/// +/// The sequence returned is "optimized." That is, they may be shrunk or even +/// truncated according to heuristics with the intent of making them more +/// useful as a prefilter. (Which translates to both using faster algorithms +/// and minimizing the false positive rate.) +/// +/// Note that this erases any connection between the literals and which pattern +/// (or patterns) they came from. +/// +/// The match kind given must correspond to the match semantics of the regex +/// that is represented by the HIRs given. The match semantics may change the +/// literal sequence returned. +#[cfg(feature = "syntax")] +pub(crate) fn prefixes<H>(kind: MatchKind, hirs: &[H]) -> literal::Seq +where + H: core::borrow::Borrow<Hir>, +{ + let mut extractor = literal::Extractor::new(); + extractor.kind(literal::ExtractKind::Prefix); + + let mut prefixes = literal::Seq::empty(); + for hir in hirs { + prefixes.union(&mut extractor.extract(hir.borrow())); + } + debug!( + "prefixes (len={:?}, exact={:?}) extracted before optimization: {:?}", + prefixes.len(), + prefixes.is_exact(), + prefixes + ); + match kind { + MatchKind::All => { + prefixes.sort(); + prefixes.dedup(); + } + MatchKind::LeftmostFirst => { + prefixes.optimize_for_prefix_by_preference(); + } + } + debug!( + "prefixes (len={:?}, exact={:?}) extracted after optimization: {:?}", + prefixes.len(), + prefixes.is_exact(), + prefixes + ); + prefixes +} + +/// Like `prefixes`, but for all suffixes of all matches for the given HIRs. +#[cfg(feature = "syntax")] +pub(crate) fn suffixes<H>(kind: MatchKind, hirs: &[H]) -> literal::Seq +where + H: core::borrow::Borrow<Hir>, +{ + let mut extractor = literal::Extractor::new(); + extractor.kind(literal::ExtractKind::Suffix); + + let mut suffixes = literal::Seq::empty(); + for hir in hirs { + suffixes.union(&mut extractor.extract(hir.borrow())); + } + debug!( + "suffixes (len={:?}, exact={:?}) extracted before optimization: {:?}", + suffixes.len(), + suffixes.is_exact(), + suffixes + ); + match kind { + MatchKind::All => { + suffixes.sort(); + suffixes.dedup(); + } + MatchKind::LeftmostFirst => { + suffixes.optimize_for_suffix_by_preference(); + } + } + debug!( + "suffixes (len={:?}, exact={:?}) extracted after optimization: {:?}", + suffixes.len(), + suffixes.is_exact(), + suffixes + ); + suffixes +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/prefilter/teddy.rs b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/teddy.rs new file mode 100644 index 0000000..e0028cd --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/prefilter/teddy.rs @@ -0,0 +1,164 @@ +use crate::util::{ + prefilter::PrefilterI, + search::{MatchKind, Span}, +}; + +#[derive(Clone, Debug)] +pub(crate) struct Teddy { + #[cfg(not(feature = "perf-literal-multisubstring"))] + _unused: (), + /// The actual Teddy searcher. + /// + /// Technically, it's possible that Teddy doesn't actually get used, since + /// Teddy does require its haystack to at least be of a certain size + /// (usually around the size of whatever vector is being used, so ~16 + /// or ~32 bytes). For haystacks shorter than that, the implementation + /// currently uses Rabin-Karp. + #[cfg(feature = "perf-literal-multisubstring")] + searcher: aho_corasick::packed::Searcher, + /// When running an anchored search, the packed searcher can't handle it so + /// we defer to Aho-Corasick itself. Kind of sad, but changing the packed + /// searchers to support anchored search would be difficult at worst and + /// annoying at best. Since packed searchers only apply to small numbers of + /// literals, we content ourselves that this is not much of an added cost. + /// (That packed searchers only work with a small number of literals is + /// also why we use a DFA here. Otherwise, the memory usage of a DFA would + /// likely be unacceptable.) + #[cfg(feature = "perf-literal-multisubstring")] + anchored_ac: aho_corasick::dfa::DFA, + /// The length of the smallest literal we look for. + /// + /// We use this as a heuristic to figure out whether this will be "fast" or + /// not. Generally, the longer the better, because longer needles are more + /// discriminating and thus reduce false positive rate. + #[cfg(feature = "perf-literal-multisubstring")] + minimum_len: usize, +} + +impl Teddy { + pub(crate) fn new<B: AsRef<[u8]>>( + kind: MatchKind, + needles: &[B], + ) -> Option<Teddy> { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + None + } + #[cfg(feature = "perf-literal-multisubstring")] + { + // We only really support leftmost-first semantics. In + // theory we could at least support leftmost-longest, as the + // aho-corasick crate does, but regex-automata doesn't know about + // leftmost-longest currently. + // + // And like the aho-corasick prefilter, if we're using `All` + // semantics, then we can still use leftmost semantics for a + // prefilter. (This might be a suspicious choice for the literal + // engine, which uses a prefilter as a regex engine directly, but + // that only happens when using leftmost-first semantics.) + let (packed_match_kind, ac_match_kind) = match kind { + MatchKind::LeftmostFirst | MatchKind::All => ( + aho_corasick::packed::MatchKind::LeftmostFirst, + aho_corasick::MatchKind::LeftmostFirst, + ), + }; + let minimum_len = + needles.iter().map(|n| n.as_ref().len()).min().unwrap_or(0); + let packed = aho_corasick::packed::Config::new() + .match_kind(packed_match_kind) + .builder() + .extend(needles) + .build()?; + let anchored_ac = aho_corasick::dfa::DFA::builder() + .match_kind(ac_match_kind) + .start_kind(aho_corasick::StartKind::Anchored) + .prefilter(false) + .build(needles) + .ok()?; + Some(Teddy { searcher: packed, anchored_ac, minimum_len }) + } + } +} + +impl PrefilterI for Teddy { + fn name(&self) -> &'static str { + "teddy" + } + + fn find(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-multisubstring")] + { + let ac_span = + aho_corasick::Span { start: span.start, end: span.end }; + self.searcher + .find_in(haystack, ac_span) + .map(|m| Span { start: m.start(), end: m.end() }) + } + } + + fn prefix(&self, haystack: &[u8], span: Span) -> Option<Span> { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-multisubstring")] + { + use aho_corasick::automaton::Automaton; + let input = aho_corasick::Input::new(haystack) + .anchored(aho_corasick::Anchored::Yes) + .span(span.start..span.end); + self.anchored_ac + .try_find(&input) + // OK because we build the DFA with anchored support. + .expect("aho-corasick DFA should never fail") + .map(|m| Span { start: m.start(), end: m.end() }) + } + } + + fn memory_usage(&self) -> usize { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-multisubstring")] + { + use aho_corasick::automaton::Automaton; + self.searcher.memory_usage() + self.anchored_ac.memory_usage() + } + } + + fn is_fast(&self) -> bool { + #[cfg(not(feature = "perf-literal-multisubstring"))] + { + unreachable!() + } + #[cfg(feature = "perf-literal-multisubstring")] + { + // Teddy is usually quite fast, but I have seen some cases where + // a large number of literals can overwhelm it and make it not so + // fast. We make an educated but conservative guess at a limit, at + // which point, we're not so comfortable thinking Teddy is "fast." + // + // Well... this used to incorporate a "limit" on the *number* + // of literals, but I have since changed it to a minimum on the + // *smallest* literal. Namely, when there is a very small literal + // (1 or 2 bytes), it is far more likely that it leads to a higher + // false positive rate. (Although, of course, not always. For + // example, 'zq' is likely to have a very low false positive rate.) + // But when we have 3 bytes, we have a really good chance of being + // quite discriminatory and thus fast. + // + // We may still want to add some kind of limit on the number of + // literals here, but keep in mind that Teddy already has its own + // somewhat small limit (64 at time of writing). The main issue + // here is that if 'is_fast' is false, it opens the door for the + // reverse inner optimization to kick in. We really only want to + // resort to the reverse inner optimization if we absolutely must. + self.minimum_len >= 3 + } + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/primitives.rs b/third_party/crates/regex-automata-0.4.18/src/util/primitives.rs new file mode 100644 index 0000000..5c5d187 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/primitives.rs @@ -0,0 +1,776 @@ +/*! +Lower level primitive types that are useful in a variety of circumstances. + +# Overview + +This list represents the principle types in this module and briefly describes +when you might want to use them. + +* [`PatternID`] - A type that represents the identifier of a regex pattern. +This is probably the most widely used type in this module (which is why it's +also re-exported in the crate root). +* [`StateID`] - A type the represents the identifier of a finite automaton +state. This is used for both NFAs and DFAs, with the notable exception of +the hybrid NFA/DFA. (The hybrid NFA/DFA uses a special purpose "lazy" state +identifier.) +* [`SmallIndex`] - The internal representation of both a `PatternID` and a +`StateID`. Its purpose is to serve as a type that can index memory without +being as big as a `usize` on 64-bit targets. The main idea behind this type +is that there are many things in regex engines that will, in practice, never +overflow a 32-bit integer. (For example, like the number of patterns in a regex +or the number of states in an NFA.) Thus, a `SmallIndex` can be used to index +memory without peppering `as` casts everywhere. Moreover, it forces callers +to handle errors in the case where, somehow, the value would otherwise overflow +either a 32-bit integer or a `usize` (e.g., on 16-bit targets). +* [`NonMaxUsize`] - Represents a `usize` that cannot be `usize::MAX`. As a +result, `Option<NonMaxUsize>` has the same size in memory as a `usize`. This +useful, for example, when representing the offsets of submatches since it +reduces memory usage by a factor of 2. It is a legal optimization since Rust +guarantees that slices never have a length that exceeds `isize::MAX`. +*/ + +use core::num::NonZeroUsize; + +#[cfg(feature = "alloc")] +use alloc::vec::Vec; + +use crate::util::int::{Usize, U16, U32, U64}; + +/// A `usize` that can never be `usize::MAX`. +/// +/// This is similar to `core::num::NonZeroUsize`, but instead of not permitting +/// a zero value, this does not permit a max value. +/// +/// This is useful in certain contexts where one wants to optimize the memory +/// usage of things that contain match offsets. Namely, since Rust slices +/// are guaranteed to never have a length exceeding `isize::MAX`, we can use +/// `usize::MAX` as a sentinel to indicate that no match was found. Indeed, +/// types like `Option<NonMaxUsize>` have exactly the same size in memory as a +/// `usize`. +/// +/// This type is defined to be `repr(transparent)` for +/// `core::num::NonZeroUsize`, which is in turn defined to be +/// `repr(transparent)` for `usize`. +#[derive(Clone, Copy, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct NonMaxUsize(NonZeroUsize); + +impl NonMaxUsize { + /// Create a new `NonMaxUsize` from the given value. + /// + /// This returns `None` only when the given value is equal to `usize::MAX`. + #[inline] + pub fn new(value: usize) -> Option<NonMaxUsize> { + NonZeroUsize::new(value.wrapping_add(1)).map(NonMaxUsize) + } + + /// Return the underlying `usize` value. The returned value is guaranteed + /// to not equal `usize::MAX`. + #[inline] + pub fn get(self) -> usize { + self.0.get().wrapping_sub(1) + } +} + +// We provide our own Debug impl because seeing the internal repr can be quite +// surprising if you aren't expecting it. e.g., 'NonMaxUsize(5)' vs just '5'. +impl core::fmt::Debug for NonMaxUsize { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?}", self.get()) + } +} + +/// A type that represents a "small" index. +/// +/// The main idea of this type is to provide something that can index memory, +/// but uses less memory than `usize` on 64-bit systems. Specifically, its +/// representation is always a `u32` and has `repr(transparent)` enabled. (So +/// it is safe to transmute between a `u32` and a `SmallIndex`.) +/// +/// A small index is typically useful in cases where there is no practical way +/// that the index will overflow a 32-bit integer. A good example of this is +/// an NFA state. If you could somehow build an NFA with `2^30` states, its +/// memory usage would be exorbitant and its runtime execution would be so +/// slow as to be completely worthless. Therefore, this crate generally deems +/// it acceptable to return an error if it would otherwise build an NFA that +/// requires a slice longer than what a 32-bit integer can index. In exchange, +/// we can use 32-bit indices instead of 64-bit indices in various places. +/// +/// This type ensures this by providing a constructor that will return an error +/// if its argument cannot fit into the type. This makes it much easier to +/// handle these sorts of boundary cases that are otherwise extremely subtle. +/// +/// On all targets, this type guarantees that its value will fit in a `u32`, +/// `i32`, `usize` and an `isize`. This means that on 16-bit targets, for +/// example, this type's maximum value will never overflow an `isize`, +/// which means it will never overflow a `i16` even though its internal +/// representation is still a `u32`. +/// +/// The purpose for making the type fit into even signed integer types like +/// `isize` is to guarantee that the difference between any two small indices +/// is itself also a small index. This is useful in certain contexts, e.g., +/// for delta encoding. +/// +/// # Other types +/// +/// The following types wrap `SmallIndex` to provide a more focused use case: +/// +/// * [`PatternID`] is for representing the identifiers of patterns. +/// * [`StateID`] is for representing the identifiers of states in finite +/// automata. It is used for both NFAs and DFAs. +/// +/// # Representation +/// +/// This type is always represented internally by a `u32` and is marked as +/// `repr(transparent)`. Thus, this type always has the same representation as +/// a `u32`. It is thus safe to transmute between a `u32` and a `SmallIndex`. +/// +/// # Indexing +/// +/// For convenience, callers may use a `SmallIndex` to index slices. +/// +/// # Safety +/// +/// While a `SmallIndex` is meant to guarantee that its value fits into `usize` +/// without using as much space as a `usize` on all targets, callers must +/// not rely on this property for safety. Callers may choose to rely on this +/// property for correctness however. For example, creating a `SmallIndex` with +/// an invalid value can be done in entirely safe code. This may in turn result +/// in panics or silent logical errors. +#[derive( + Clone, Copy, Debug, Default, Eq, Hash, PartialEq, PartialOrd, Ord, +)] +#[repr(transparent)] +pub struct SmallIndex(u32); + +impl SmallIndex { + /// The maximum index value. + #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))] + pub const MAX: SmallIndex = + // FIXME: Use as_usize() once const functions in traits are stable. + SmallIndex::new_unchecked(core::i32::MAX as usize - 1); + + /// The maximum index value. + #[cfg(target_pointer_width = "16")] + pub const MAX: SmallIndex = + SmallIndex::new_unchecked(core::isize::MAX - 1); + + /// The total number of values that can be represented as a small index. + pub const LIMIT: usize = SmallIndex::MAX.as_usize() + 1; + + /// The zero index value. + pub const ZERO: SmallIndex = SmallIndex::new_unchecked(0); + + /// The number of bytes that a single small index uses in memory. + pub const SIZE: usize = core::mem::size_of::<SmallIndex>(); + + /// Create a new small index. + /// + /// If the given index exceeds [`SmallIndex::MAX`], then this returns + /// an error. + #[inline] + pub fn new(index: usize) -> Result<SmallIndex, SmallIndexError> { + SmallIndex::try_from(index) + } + + /// Create a new small index without checking whether the given value + /// exceeds [`SmallIndex::MAX`]. + /// + /// Using this routine with an invalid index value will result in + /// unspecified behavior, but *not* undefined behavior. In particular, an + /// invalid index value is likely to cause panics or possibly even silent + /// logical errors. + /// + /// Callers must never rely on a `SmallIndex` to be within a certain range + /// for memory safety. + #[inline] + pub const fn new_unchecked(index: usize) -> SmallIndex { + // FIXME: Use as_u32() once const functions in traits are stable. + SmallIndex(index as u32) + } + + /// Like [`SmallIndex::new`], but panics if the given index is not valid. + #[inline] + pub fn must(index: usize) -> SmallIndex { + SmallIndex::new(index).expect("invalid small index") + } + + /// Return this small index as a `usize`. This is guaranteed to never + /// overflow `usize`. + #[inline] + pub const fn as_usize(&self) -> usize { + // FIXME: Use as_usize() once const functions in traits are stable. + self.0 as usize + } + + /// Return this small index as a `u64`. This is guaranteed to never + /// overflow. + #[inline] + pub const fn as_u64(&self) -> u64 { + // FIXME: Use u64::from() once const functions in traits are stable. + self.0 as u64 + } + + /// Return the internal `u32` of this small index. This is guaranteed to + /// never overflow `u32`. + #[inline] + pub const fn as_u32(&self) -> u32 { + self.0 + } + + /// Return the internal `u32` of this small index represented as an `i32`. + /// This is guaranteed to never overflow an `i32`. + #[inline] + pub const fn as_i32(&self) -> i32 { + // This is OK because we guarantee that our max value is <= i32::MAX. + self.0 as i32 + } + + /// Returns one more than this small index as a usize. + /// + /// Since a small index has constraints on its maximum value, adding `1` to + /// it will always fit in a `usize`, `u32` and a `i32`. + #[inline] + pub fn one_more(&self) -> usize { + self.as_usize() + 1 + } + + /// Decode this small index from the bytes given using the native endian + /// byte order for the current target. + /// + /// If the decoded integer is not representable as a small index for the + /// current target, then this returns an error. + #[inline] + pub fn from_ne_bytes( + bytes: [u8; 4], + ) -> Result<SmallIndex, SmallIndexError> { + let id = u32::from_ne_bytes(bytes); + if id > SmallIndex::MAX.as_u32() { + return Err(SmallIndexError { attempted: u64::from(id) }); + } + Ok(SmallIndex::new_unchecked(id.as_usize())) + } + + /// Decode this small index from the bytes given using the native endian + /// byte order for the current target. + /// + /// This is analogous to [`SmallIndex::new_unchecked`] in that is does not + /// check whether the decoded integer is representable as a small index. + #[inline] + pub fn from_ne_bytes_unchecked(bytes: [u8; 4]) -> SmallIndex { + SmallIndex::new_unchecked(u32::from_ne_bytes(bytes).as_usize()) + } + + /// Return the underlying small index integer as raw bytes in native endian + /// format. + #[inline] + pub fn to_ne_bytes(&self) -> [u8; 4] { + self.0.to_ne_bytes() + } +} + +impl<T> core::ops::Index<SmallIndex> for [T] { + type Output = T; + + #[inline] + fn index(&self, index: SmallIndex) -> &T { + &self[index.as_usize()] + } +} + +impl<T> core::ops::IndexMut<SmallIndex> for [T] { + #[inline] + fn index_mut(&mut self, index: SmallIndex) -> &mut T { + &mut self[index.as_usize()] + } +} + +#[cfg(feature = "alloc")] +impl<T> core::ops::Index<SmallIndex> for Vec<T> { + type Output = T; + + #[inline] + fn index(&self, index: SmallIndex) -> &T { + &self[index.as_usize()] + } +} + +#[cfg(feature = "alloc")] +impl<T> core::ops::IndexMut<SmallIndex> for Vec<T> { + #[inline] + fn index_mut(&mut self, index: SmallIndex) -> &mut T { + &mut self[index.as_usize()] + } +} + +impl From<u8> for SmallIndex { + fn from(index: u8) -> SmallIndex { + SmallIndex::new_unchecked(usize::from(index)) + } +} + +impl TryFrom<u16> for SmallIndex { + type Error = SmallIndexError; + + fn try_from(index: u16) -> Result<SmallIndex, SmallIndexError> { + if u32::from(index) > SmallIndex::MAX.as_u32() { + return Err(SmallIndexError { attempted: u64::from(index) }); + } + Ok(SmallIndex::new_unchecked(index.as_usize())) + } +} + +impl TryFrom<u32> for SmallIndex { + type Error = SmallIndexError; + + fn try_from(index: u32) -> Result<SmallIndex, SmallIndexError> { + if index > SmallIndex::MAX.as_u32() { + return Err(SmallIndexError { attempted: u64::from(index) }); + } + Ok(SmallIndex::new_unchecked(index.as_usize())) + } +} + +impl TryFrom<u64> for SmallIndex { + type Error = SmallIndexError; + + fn try_from(index: u64) -> Result<SmallIndex, SmallIndexError> { + if index > SmallIndex::MAX.as_u64() { + return Err(SmallIndexError { attempted: index }); + } + Ok(SmallIndex::new_unchecked(index.as_usize())) + } +} + +impl TryFrom<usize> for SmallIndex { + type Error = SmallIndexError; + + fn try_from(index: usize) -> Result<SmallIndex, SmallIndexError> { + if index > SmallIndex::MAX.as_usize() { + return Err(SmallIndexError { attempted: index.as_u64() }); + } + Ok(SmallIndex::new_unchecked(index)) + } +} + +#[cfg(test)] +impl quickcheck::Arbitrary for SmallIndex { + fn arbitrary(gen: &mut quickcheck::Gen) -> SmallIndex { + use core::cmp::max; + + let id = max(i32::MIN + 1, i32::arbitrary(gen)).abs(); + if id > SmallIndex::MAX.as_i32() { + SmallIndex::MAX + } else { + SmallIndex::new(usize::try_from(id).unwrap()).unwrap() + } + } +} + +/// This error occurs when a small index could not be constructed. +/// +/// This occurs when given an integer exceeding the maximum small index value. +/// +/// When the `std` feature is enabled, this implements the `Error` trait. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SmallIndexError { + attempted: u64, +} + +impl SmallIndexError { + /// Returns the value that could not be converted to a small index. + pub fn attempted(&self) -> u64 { + self.attempted + } +} + +#[cfg(feature = "std")] +impl std::error::Error for SmallIndexError {} + +impl core::fmt::Display for SmallIndexError { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "failed to create small index from {:?}, which exceeds {:?}", + self.attempted(), + SmallIndex::MAX, + ) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct SmallIndexIter { + rng: core::ops::Range<usize>, +} + +impl Iterator for SmallIndexIter { + type Item = SmallIndex; + + fn next(&mut self) -> Option<SmallIndex> { + if self.rng.start >= self.rng.end { + return None; + } + let next_id = self.rng.start + 1; + let id = core::mem::replace(&mut self.rng.start, next_id); + // new_unchecked is OK since we asserted that the number of + // elements in this iterator will fit in an ID at construction. + Some(SmallIndex::new_unchecked(id)) + } +} + +macro_rules! index_type_impls { + ($name:ident, $err:ident, $iter:ident, $withiter:ident) => { + impl $name { + /// The maximum value. + pub const MAX: $name = $name(SmallIndex::MAX); + + /// The total number of values that can be represented. + pub const LIMIT: usize = SmallIndex::LIMIT; + + /// The zero value. + pub const ZERO: $name = $name(SmallIndex::ZERO); + + /// The number of bytes that a single value uses in memory. + pub const SIZE: usize = SmallIndex::SIZE; + + /// Create a new value that is represented by a "small index." + /// + /// If the given index exceeds the maximum allowed value, then this + /// returns an error. + #[inline] + pub fn new(value: usize) -> Result<$name, $err> { + SmallIndex::new(value).map($name).map_err($err) + } + + /// Create a new value without checking whether the given argument + /// exceeds the maximum. + /// + /// Using this routine with an invalid value will result in + /// unspecified behavior, but *not* undefined behavior. In + /// particular, an invalid ID value is likely to cause panics or + /// possibly even silent logical errors. + /// + /// Callers must never rely on this type to be within a certain + /// range for memory safety. + #[inline] + pub const fn new_unchecked(value: usize) -> $name { + $name(SmallIndex::new_unchecked(value)) + } + + /// Like `new`, but panics if the given value is not valid. + #[inline] + pub fn must(value: usize) -> $name { + $name::new(value).expect(concat!( + "invalid ", + stringify!($name), + " value" + )) + } + + /// Return the internal value as a `usize`. This is guaranteed to + /// never overflow `usize`. + #[inline] + pub const fn as_usize(&self) -> usize { + self.0.as_usize() + } + + /// Return the internal value as a `u64`. This is guaranteed to + /// never overflow. + #[inline] + pub const fn as_u64(&self) -> u64 { + self.0.as_u64() + } + + /// Return the internal value as a `u32`. This is guaranteed to + /// never overflow `u32`. + #[inline] + pub const fn as_u32(&self) -> u32 { + self.0.as_u32() + } + + /// Return the internal value as a i32`. This is guaranteed to + /// never overflow an `i32`. + #[inline] + pub const fn as_i32(&self) -> i32 { + self.0.as_i32() + } + + /// Returns one more than this value as a usize. + /// + /// Since values represented by a "small index" have constraints + /// on their maximum value, adding `1` to it will always fit in a + /// `usize`, `u32` and a `i32`. + #[inline] + pub fn one_more(&self) -> usize { + self.0.one_more() + } + + /// Decode this value from the bytes given using the native endian + /// byte order for the current target. + /// + /// If the decoded integer is not representable as a small index + /// for the current target, then this returns an error. + #[inline] + pub fn from_ne_bytes(bytes: [u8; 4]) -> Result<$name, $err> { + SmallIndex::from_ne_bytes(bytes).map($name).map_err($err) + } + + /// Decode this value from the bytes given using the native endian + /// byte order for the current target. + /// + /// This is analogous to `new_unchecked` in that is does not check + /// whether the decoded integer is representable as a small index. + #[inline] + pub fn from_ne_bytes_unchecked(bytes: [u8; 4]) -> $name { + $name(SmallIndex::from_ne_bytes_unchecked(bytes)) + } + + /// Return the underlying integer as raw bytes in native endian + /// format. + #[inline] + pub fn to_ne_bytes(&self) -> [u8; 4] { + self.0.to_ne_bytes() + } + + /// Returns an iterator over all values from 0 up to and not + /// including the given length. + /// + /// If the given length exceeds this type's limit, then this + /// panics. + pub(crate) fn iter(len: usize) -> $iter { + $iter::new(len) + } + } + + // We write our own Debug impl so that we get things like PatternID(5) + // instead of PatternID(SmallIndex(5)). + impl core::fmt::Debug for $name { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_tuple(stringify!($name)).field(&self.as_u32()).finish() + } + } + + impl<T> core::ops::Index<$name> for [T] { + type Output = T; + + #[inline] + fn index(&self, index: $name) -> &T { + &self[index.as_usize()] + } + } + + impl<T> core::ops::IndexMut<$name> for [T] { + #[inline] + fn index_mut(&mut self, index: $name) -> &mut T { + &mut self[index.as_usize()] + } + } + + #[cfg(feature = "alloc")] + impl<T> core::ops::Index<$name> for Vec<T> { + type Output = T; + + #[inline] + fn index(&self, index: $name) -> &T { + &self[index.as_usize()] + } + } + + #[cfg(feature = "alloc")] + impl<T> core::ops::IndexMut<$name> for Vec<T> { + #[inline] + fn index_mut(&mut self, index: $name) -> &mut T { + &mut self[index.as_usize()] + } + } + + impl From<u8> for $name { + fn from(value: u8) -> $name { + $name(SmallIndex::from(value)) + } + } + + impl TryFrom<u16> for $name { + type Error = $err; + + fn try_from(value: u16) -> Result<$name, $err> { + SmallIndex::try_from(value).map($name).map_err($err) + } + } + + impl TryFrom<u32> for $name { + type Error = $err; + + fn try_from(value: u32) -> Result<$name, $err> { + SmallIndex::try_from(value).map($name).map_err($err) + } + } + + impl TryFrom<u64> for $name { + type Error = $err; + + fn try_from(value: u64) -> Result<$name, $err> { + SmallIndex::try_from(value).map($name).map_err($err) + } + } + + impl TryFrom<usize> for $name { + type Error = $err; + + fn try_from(value: usize) -> Result<$name, $err> { + SmallIndex::try_from(value).map($name).map_err($err) + } + } + + #[cfg(test)] + impl quickcheck::Arbitrary for $name { + fn arbitrary(gen: &mut quickcheck::Gen) -> $name { + $name(SmallIndex::arbitrary(gen)) + } + } + + /// This error occurs when a value could not be constructed. + /// + /// This occurs when given an integer exceeding the maximum allowed + /// value. + /// + /// When the `std` feature is enabled, this implements the `Error` + /// trait. + #[derive(Clone, Debug, Eq, PartialEq)] + pub struct $err(SmallIndexError); + + impl $err { + /// Returns the value that could not be converted to an ID. + pub fn attempted(&self) -> u64 { + self.0.attempted() + } + } + + #[cfg(feature = "std")] + impl std::error::Error for $err {} + + impl core::fmt::Display for $err { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "failed to create {} from {:?}, which exceeds {:?}", + stringify!($name), + self.attempted(), + $name::MAX, + ) + } + } + + #[derive(Clone, Debug)] + pub(crate) struct $iter(SmallIndexIter); + + impl $iter { + fn new(len: usize) -> $iter { + assert!( + len <= $name::LIMIT, + "cannot create iterator for {} when number of \ + elements exceed {:?}", + stringify!($name), + $name::LIMIT, + ); + $iter(SmallIndexIter { rng: 0..len }) + } + } + + impl Iterator for $iter { + type Item = $name; + + fn next(&mut self) -> Option<$name> { + self.0.next().map($name) + } + } + + /// An iterator adapter that is like std::iter::Enumerate, but attaches + /// small index values instead. It requires `ExactSizeIterator`. At + /// construction, it ensures that the index of each element in the + /// iterator is representable in the corresponding small index type. + #[derive(Clone, Debug)] + pub(crate) struct $withiter<I> { + it: I, + ids: $iter, + } + + impl<I: Iterator + ExactSizeIterator> $withiter<I> { + fn new(it: I) -> $withiter<I> { + let ids = $name::iter(it.len()); + $withiter { it, ids } + } + } + + impl<I: Iterator + ExactSizeIterator> Iterator for $withiter<I> { + type Item = ($name, I::Item); + + fn next(&mut self) -> Option<($name, I::Item)> { + let item = self.it.next()?; + // Number of elements in this iterator must match, according + // to contract of ExactSizeIterator. + let id = self.ids.next().unwrap(); + Some((id, item)) + } + } + }; +} + +/// The identifier of a regex pattern, represented by a [`SmallIndex`]. +/// +/// The identifier for a pattern corresponds to its relative position among +/// other patterns in a single finite state machine. Namely, when building +/// a multi-pattern regex engine, one must supply a sequence of patterns to +/// match. The position (starting at 0) of each pattern in that sequence +/// represents its identifier. This identifier is in turn used to identify and +/// report matches of that pattern in various APIs. +/// +/// See the [`SmallIndex`] type for more information about what it means for +/// a pattern ID to be a "small index." +/// +/// Note that this type is defined in the +/// [`util::primitives`](crate::util::primitives) module, but it is also +/// re-exported at the crate root due to how common it is. +#[derive(Clone, Copy, Default, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct PatternID(SmallIndex); + +/// The identifier of a finite automaton state, represented by a +/// [`SmallIndex`]. +/// +/// Most regex engines in this crate are built on top of finite automata. Each +/// state in a finite automaton defines transitions from its state to another. +/// Those transitions point to other states via their identifiers, i.e., a +/// `StateID`. Since finite automata tend to contain many transitions, it is +/// much more memory efficient to define state IDs as small indices. +/// +/// See the [`SmallIndex`] type for more information about what it means for +/// a state ID to be a "small index." +#[derive(Clone, Copy, Default, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[repr(transparent)] +pub struct StateID(SmallIndex); + +index_type_impls!(PatternID, PatternIDError, PatternIDIter, WithPatternIDIter); +index_type_impls!(StateID, StateIDError, StateIDIter, WithStateIDIter); + +/// A utility trait that defines a couple of adapters for making it convenient +/// to access indices as "small index" types. We require ExactSizeIterator so +/// that iterator construction can do a single check to make sure the index of +/// each element is representable by its small index type. +pub(crate) trait IteratorIndexExt: Iterator { + fn with_pattern_ids(self) -> WithPatternIDIter<Self> + where + Self: Sized + ExactSizeIterator, + { + WithPatternIDIter::new(self) + } + + fn with_state_ids(self) -> WithStateIDIter<Self> + where + Self: Sized + ExactSizeIterator, + { + WithStateIDIter::new(self) + } +} + +impl<I: Iterator> IteratorIndexExt for I {} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/search.rs b/third_party/crates/regex-automata-0.4.18/src/util/search.rs new file mode 100644 index 0000000..37999f4 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/search.rs @@ -0,0 +1,1997 @@ +/*! +Types and routines that support the search APIs of most regex engines. + +This sub-module isn't exposed directly, but rather, its contents are exported +at the crate root due to the universality of most of the types and routines in +this module. +*/ + +use core::ops::{Range, RangeBounds}; + +use crate::util::{escape::DebugByte, primitives::PatternID, utf8}; + +/// The parameters for a regex search including the haystack to search. +/// +/// It turns out that regex searches have a few parameters, and in most cases, +/// those parameters have defaults that work in the vast majority of cases. +/// This `Input` type exists to make that common case seamless while also +/// providing an avenue for changing the parameters of a search. In particular, +/// this type enables doing so without a combinatorial explosion of different +/// methods and/or superfluous parameters in the common cases. +/// +/// An `Input` permits configuring the following things: +/// +/// * Search only a substring of a haystack, while taking the broader context +/// into account for resolving look-around assertions. +/// * Indicating whether to search for all patterns in a regex, or to +/// only search for one pattern in particular. +/// * Whether to perform an anchored on unanchored search. +/// * Whether to report a match as early as possible. +/// +/// All of these parameters, except for the haystack, have sensible default +/// values. This means that the minimal search configuration is simply a call +/// to [`Input::new`] with your haystack. Setting any other parameter is +/// optional. +/// +/// Moreover, for any `H` that implements `AsRef<[u8]>`, there exists a +/// `From<H> for Input` implementation. This is useful because many of the +/// search APIs in this crate accept an `Into<Input>`. This means you can +/// provide string or byte strings to these routines directly, and they'll +/// automatically get converted into an `Input` for you. +/// +/// The lifetime parameter `'h` refers to the lifetime of the haystack. +/// +/// # Organization +/// +/// The API of `Input` is split into a few different parts: +/// +/// * A builder-like API that transforms a `Input` by value. Examples: +/// [`Input::span`] and [`Input::anchored`]. +/// * A setter API that permits mutating parameters in place. Examples: +/// [`Input::set_span`] and [`Input::set_anchored`]. +/// * A getter API that permits retrieving any of the search parameters. +/// Examples: [`Input::get_span`] and [`Input::get_anchored`]. +/// * A few convenience getter routines that don't conform to the above naming +/// pattern due to how common they are. Examples: [`Input::haystack`], +/// [`Input::start`] and [`Input::end`]. +/// * Miscellaneous predicates and other helper routines that are useful +/// in some contexts. Examples: [`Input::is_char_boundary`]. +/// +/// A `Input` exposes so much because it is meant to be used by both callers of +/// regex engines _and_ implementors of regex engines. A constraining factor is +/// that regex engines should accept a `&Input` as its lowest level API, which +/// means that implementors should only use the "getter" APIs of a `Input`. +/// +/// # Valid bounds and search termination +/// +/// An `Input` permits setting the bounds of a search via either +/// [`Input::span`] or [`Input::range`]. The bounds set must be valid, or +/// else a panic will occur. Bounds are valid if and only if: +/// +/// * The bounds represent a valid range into the input's haystack. +/// * **or** the end bound is a valid ending bound for the haystack *and* +/// the start bound is exactly one greater than the start bound. +/// +/// In the latter case, [`Input::is_done`] will return true and indicates any +/// search receiving such an input should immediately return with no match. +/// +/// Note that while `Input` is used for reverse searches in this crate, the +/// `Input::is_done` predicate assumes a forward search. Because unsigned +/// offsets are used internally, there is no way to tell from only the offsets +/// whether a reverse search is done or not. +/// +/// # Regex engine support +/// +/// Any regex engine accepting an `Input` must support at least the following +/// things: +/// +/// * Searching a `&[u8]` for matches. +/// * Searching a substring of `&[u8]` for a match, such that any match +/// reported must appear entirely within that substring. +/// * For a forwards search, a match should never be reported when +/// [`Input::is_done`] returns true. (For reverse searches, termination should +/// be handled outside of `Input`.) +/// +/// Supporting other aspects of an `Input` are optional, but regex engines +/// should handle aspects they don't support gracefully. How this is done is +/// generally up to the regex engine. This crate generally treats unsupported +/// anchored modes as an error to report for example, but for simplicity, in +/// the meta regex engine, trying to search with an invalid pattern ID just +/// results in no match being reported. +#[derive(Clone)] +pub struct Input<'h> { + haystack: &'h [u8], + span: Span, + anchored: Anchored, + earliest: bool, +} + +impl<'h> Input<'h> { + /// Create a new search configuration for the given haystack. + #[inline] + pub fn new<H: ?Sized + AsRef<[u8]>>(haystack: &'h H) -> Input<'h> { + // Perform only one call to `haystack.as_ref()` to protect from incorrect + // implementations that return different values from multiple calls. + // This is important because there's code that relies on `span` not being + // out of bounds with respect to the stored `haystack`. + let haystack = haystack.as_ref(); + Input { + haystack, + span: Span { start: 0, end: haystack.len() }, + anchored: Anchored::No, + earliest: false, + } + } + + /// Set the span for this search. + /// + /// This routine does not panic if the span given is not a valid range for + /// this search's haystack. If this search is run with an invalid range, + /// then the most likely outcome is that the actual search execution will + /// panic. + /// + /// This routine is generic over how a span is provided. While + /// a [`Span`] may be given directly, one may also provide a + /// `std::ops::Range<usize>`. To provide anything supported by range + /// syntax, use the [`Input::range`] method. + /// + /// The default span is the entire haystack. + /// + /// Note that [`Input::range`] overrides this method and vice versa. + /// + /// # Panics + /// + /// This panics if the given span does not correspond to valid bounds in + /// the haystack or the termination of a search. + /// + /// # Example + /// + /// This example shows how the span of the search can impact whether a + /// match is reported or not. This is particularly relevant for look-around + /// operators, which might take things outside of the span into account + /// when determining whether they match. + /// + /// ``` + /// # if cfg!(miri) { return Ok(()); } // miri takes too long + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// Match, Input, + /// }; + /// + /// // Look for 'at', but as a distinct word. + /// let re = PikeVM::new(r"\bat\b")?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// // Our haystack contains 'at', but not as a distinct word. + /// let haystack = "batter"; + /// + /// // A standard search finds nothing, as expected. + /// let input = Input::new(haystack); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(None, caps.get_match()); + /// + /// // But if we wanted to search starting at position '1', we might + /// // slice the haystack. If we do this, it's impossible for the \b + /// // anchors to take the surrounding context into account! And thus, + /// // a match is produced. + /// let input = Input::new(&haystack[1..3]); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 0..2)), caps.get_match()); + /// + /// // But if we specify the span of the search instead of slicing the + /// // haystack, then the regex engine can "see" outside of the span + /// // and resolve the anchors correctly. + /// let input = Input::new(haystack).span(1..3); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(None, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// This may seem a little ham-fisted, but this scenario tends to come up + /// if some other regex engine found the match span and now you need to + /// re-process that span to look for capturing groups. (e.g., Run a faster + /// DFA first, find a match, then run the PikeVM on just the match span to + /// resolve capturing groups.) In order to implement that sort of logic + /// correctly, you need to set the span on the search instead of slicing + /// the haystack directly. + /// + /// The other advantage of using this routine to specify the bounds of the + /// search is that the match offsets are still reported in terms of the + /// original haystack. For example, the second search in the example above + /// reported a match at position `0`, even though `at` starts at offset + /// `1` because we sliced the haystack. + #[inline] + pub fn span<S: Into<Span>>(mut self, span: S) -> Input<'h> { + self.set_span(span); + self + } + + /// Like `Input::span`, but accepts any range instead. + /// + /// This routine does not panic if the range given is not a valid range for + /// this search's haystack. If this search is run with an invalid range, + /// then the most likely outcome is that the actual search execution will + /// panic. + /// + /// The default range is the entire haystack. + /// + /// Note that [`Input::span`] overrides this method and vice versa. + /// + /// # Panics + /// + /// This routine will panic if the given range could not be converted + /// to a valid [`Range`]. For example, this would panic when given + /// `0..=usize::MAX` since it cannot be represented using a half-open + /// interval in terms of `usize`. + /// + /// This also panics if the given range does not correspond to valid bounds + /// in the haystack or the termination of a search. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// + /// let input = Input::new("foobar").range(2..=4); + /// assert_eq!(2..5, input.get_range()); + /// ``` + #[inline] + pub fn range<R: RangeBounds<usize>>(mut self, range: R) -> Input<'h> { + self.set_range(range); + self + } + + /// Sets the anchor mode of a search. + /// + /// When a search is anchored (so that's [`Anchored::Yes`] or + /// [`Anchored::Pattern`]), a match must begin at the start of a search. + /// When a search is not anchored (that's [`Anchored::No`]), regex engines + /// will behave as if the pattern started with a `(?s-u:.)*?`. This prefix + /// permits a match to appear anywhere. + /// + /// By default, the anchored mode is [`Anchored::No`]. + /// + /// **WARNING:** this is subtly different than using a `^` at the start of + /// your regex. A `^` forces a regex to match exclusively at the start of + /// a haystack, regardless of where you begin your search. In contrast, + /// anchoring a search will allow your regex to match anywhere in your + /// haystack, but the match must start at the beginning of a search. + /// + /// For example, consider the haystack `aba` and the following searches: + /// + /// 1. The regex `^a` is compiled with `Anchored::No` and searches `aba` + /// starting at position `2`. Since `^` requires the match to start at + /// the beginning of the haystack and `2 > 0`, no match is found. + /// 2. The regex `a` is compiled with `Anchored::Yes` and searches `aba` + /// starting at position `2`. This reports a match at `[2, 3]` since + /// the match starts where the search started. Since there is no `^`, + /// there is no requirement for the match to start at the beginning of + /// the haystack. + /// 3. The regex `a` is compiled with `Anchored::Yes` and searches `aba` + /// starting at position `1`. Since `b` corresponds to position `1` and + /// since the search is anchored, it finds no match. While the regex + /// matches at other positions, configuring the search to be anchored + /// requires that it only report a match that begins at the same offset + /// as the beginning of the search. + /// 4. The regex `a` is compiled with `Anchored::No` and searches `aba` + /// starting at position `1`. Since the search is not anchored and + /// the regex does not start with `^`, the search executes as if there + /// is a `(?s:.)*?` prefix that permits it to match anywhere. Thus, it + /// reports a match at `[2, 3]`. + /// + /// Note that the [`Anchored::Pattern`] mode is like `Anchored::Yes`, + /// except it only reports matches for a particular pattern. + /// + /// # Example + /// + /// This demonstrates the differences between an anchored search and + /// a pattern that begins with `^` (as described in the above warning + /// message). + /// + /// ``` + /// use regex_automata::{ + /// nfa::thompson::pikevm::PikeVM, + /// Anchored, Match, Input, + /// }; + /// + /// let haystack = "aba"; + /// + /// let re = PikeVM::new(r"^a")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let input = Input::new(haystack).span(2..3).anchored(Anchored::No); + /// re.search(&mut cache, &input, &mut caps); + /// // No match is found because 2 is not the beginning of the haystack, + /// // which is what ^ requires. + /// assert_eq!(None, caps.get_match()); + /// + /// let re = PikeVM::new(r"a")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let input = Input::new(haystack).span(2..3).anchored(Anchored::Yes); + /// re.search(&mut cache, &input, &mut caps); + /// // An anchored search can still match anywhere in the haystack, it just + /// // must begin at the start of the search which is '2' in this case. + /// assert_eq!(Some(Match::must(0, 2..3)), caps.get_match()); + /// + /// let re = PikeVM::new(r"a")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let input = Input::new(haystack).span(1..3).anchored(Anchored::Yes); + /// re.search(&mut cache, &input, &mut caps); + /// // No match is found since we start searching at offset 1 which + /// // corresponds to 'b'. Since there is no '(?s:.)*?' prefix, no match + /// // is found. + /// assert_eq!(None, caps.get_match()); + /// + /// let re = PikeVM::new(r"a")?; + /// let (mut cache, mut caps) = (re.create_cache(), re.create_captures()); + /// let input = Input::new(haystack).span(1..3).anchored(Anchored::No); + /// re.search(&mut cache, &input, &mut caps); + /// // Since anchored=no, an implicit '(?s:.)*?' prefix was added to the + /// // pattern. Even though the search starts at 'b', the 'match anything' + /// // prefix allows the search to match 'a'. + /// let expected = Some(Match::must(0, 2..3)); + /// assert_eq!(expected, caps.get_match()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn anchored(mut self, mode: Anchored) -> Input<'h> { + self.set_anchored(mode); + self + } + + /// Whether to execute an "earliest" search or not. + /// + /// When running a non-overlapping search, an "earliest" search will return + /// the match location as early as possible. For example, given a pattern + /// of `foo[0-9]+` and a haystack of `foo12345`, a normal leftmost search + /// will return `foo12345` as a match. But an "earliest" search for regex + /// engines that support "earliest" semantics will return `foo1` as a + /// match, since as soon as the first digit following `foo` is seen, it is + /// known to have found a match. + /// + /// Note that "earliest" semantics generally depend on the regex engine. + /// Different regex engines may determine there is a match at different + /// points. So there is no guarantee that "earliest" matches will always + /// return the same offsets for all regex engines. The "earliest" notion + /// is really about when the particular regex engine determines there is + /// a match rather than a consistent semantic unto itself. This is often + /// useful for implementing "did a match occur or not" predicates, but + /// sometimes the offset is useful as well. + /// + /// This is disabled by default. + /// + /// # Example + /// + /// This example shows the difference between "earliest" searching and + /// normal searching. + /// + /// ``` + /// use regex_automata::{nfa::thompson::pikevm::PikeVM, Match, Input}; + /// + /// let re = PikeVM::new(r"foo[0-9]+")?; + /// let mut cache = re.create_cache(); + /// let mut caps = re.create_captures(); + /// + /// // A normal search implements greediness like you expect. + /// let input = Input::new("foo12345"); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 0..8)), caps.get_match()); + /// + /// // When 'earliest' is enabled and the regex engine supports + /// // it, the search will bail once it knows a match has been + /// // found. + /// let input = Input::new("foo12345").earliest(true); + /// re.search(&mut cache, &input, &mut caps); + /// assert_eq!(Some(Match::must(0, 0..4)), caps.get_match()); + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn earliest(mut self, yes: bool) -> Input<'h> { + self.set_earliest(yes); + self + } + + /// Set the span for this search configuration. + /// + /// This is like the [`Input::span`] method, except this mutates the + /// span in place. + /// + /// This routine is generic over how a span is provided. While + /// a [`Span`] may be given directly, one may also provide a + /// `std::ops::Range<usize>`. + /// + /// # Panics + /// + /// This panics if the given span does not correspond to valid bounds in + /// the haystack or the termination of a search. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// input.set_span(2..4); + /// assert_eq!(2..4, input.get_range()); + /// ``` + #[inline] + pub fn set_span<S: Into<Span>>(&mut self, span: S) { + let span = span.into(); + assert!( + span.end <= self.haystack.len() + && span.start <= span.end.wrapping_add(1), + "invalid span {:?} for haystack of length {}", + span, + self.haystack.len(), + ); + self.span = span; + } + + /// Set the span for this search configuration given any range. + /// + /// This is like the [`Input::range`] method, except this mutates the + /// span in place. + /// + /// This routine does not panic if the range given is not a valid range for + /// this search's haystack. If this search is run with an invalid range, + /// then the most likely outcome is that the actual search execution will + /// panic. + /// + /// # Panics + /// + /// This routine will panic if the given range could not be converted + /// to a valid [`Range`]. For example, this would panic when given + /// `0..=usize::MAX` since it cannot be represented using a half-open + /// interval in terms of `usize`. + /// + /// This also panics if the given span does not correspond to valid bounds + /// in the haystack or the termination of a search. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// input.set_range(2..=4); + /// assert_eq!(2..5, input.get_range()); + /// ``` + #[inline] + pub fn set_range<R: RangeBounds<usize>>(&mut self, range: R) { + use core::ops::Bound; + + // It's a little weird to convert ranges into spans, and then spans + // back into ranges when we actually slice the haystack. Because + // of that process, we always represent everything as a half-open + // internal. Therefore, handling things like m..=n is a little awkward. + let start = match range.start_bound() { + Bound::Included(&i) => i, + // Can this case ever happen? Range syntax doesn't support it... + Bound::Excluded(&i) => i.checked_add(1).unwrap(), + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(&i) => i.checked_add(1).unwrap(), + Bound::Excluded(&i) => i, + Bound::Unbounded => self.haystack().len(), + }; + self.set_span(Span { start, end }); + } + + /// Set the starting offset for the span for this search configuration. + /// + /// This is a convenience routine for only mutating the start of a span + /// without having to set the entire span. + /// + /// # Panics + /// + /// This panics if the span resulting from the new start position does not + /// correspond to valid bounds in the haystack or the termination of a + /// search. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// input.set_start(5); + /// assert_eq!(5..6, input.get_range()); + /// ``` + #[inline] + pub fn set_start(&mut self, start: usize) { + self.set_span(Span { start, ..self.get_span() }); + } + + /// Set the ending offset for the span for this search configuration. + /// + /// This is a convenience routine for only mutating the end of a span + /// without having to set the entire span. + /// + /// # Panics + /// + /// This panics if the span resulting from the new end position does not + /// correspond to valid bounds in the haystack or the termination of a + /// search. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// input.set_end(5); + /// assert_eq!(0..5, input.get_range()); + /// ``` + #[inline] + pub fn set_end(&mut self, end: usize) { + self.set_span(Span { end, ..self.get_span() }); + } + + /// Set the anchor mode of a search. + /// + /// This is like [`Input::anchored`], except it mutates the search + /// configuration in place. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Anchored, Input, PatternID}; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(Anchored::No, input.get_anchored()); + /// + /// let pid = PatternID::must(5); + /// input.set_anchored(Anchored::Pattern(pid)); + /// assert_eq!(Anchored::Pattern(pid), input.get_anchored()); + /// ``` + #[inline] + pub fn set_anchored(&mut self, mode: Anchored) { + self.anchored = mode; + } + + /// Set whether the search should execute in "earliest" mode or not. + /// + /// This is like [`Input::earliest`], except it mutates the search + /// configuration in place. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert!(!input.get_earliest()); + /// input.set_earliest(true); + /// assert!(input.get_earliest()); + /// ``` + #[inline] + pub fn set_earliest(&mut self, yes: bool) { + self.earliest = yes; + } + + /// Return a borrow of the underlying haystack as a slice of bytes. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(b"foobar", input.haystack()); + /// ``` + #[inline] + pub fn haystack(&self) -> &'h [u8] { + self.haystack + } + + /// Return the start position of this search. + /// + /// This is a convenience routine for `search.get_span().start()`. + /// + /// When [`Input::is_done`] is `false`, this is guaranteed to return + /// an offset that is less than or equal to [`Input::end`]. Otherwise, + /// the offset is one greater than [`Input::end`]. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(0, input.start()); + /// + /// let input = Input::new("foobar").span(2..4); + /// assert_eq!(2, input.start()); + /// ``` + #[inline] + pub fn start(&self) -> usize { + self.get_span().start + } + + /// Return the end position of this search. + /// + /// This is a convenience routine for `search.get_span().end()`. + /// + /// This is guaranteed to return an offset that is a valid exclusive end + /// bound for this input's haystack. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(6, input.end()); + /// + /// let input = Input::new("foobar").span(2..4); + /// assert_eq!(4, input.end()); + /// ``` + #[inline] + pub fn end(&self) -> usize { + self.get_span().end + } + + /// Return the span for this search configuration. + /// + /// If one was not explicitly set, then the span corresponds to the entire + /// range of the haystack. + /// + /// When [`Input::is_done`] is `false`, the span returned is guaranteed + /// to correspond to valid bounds for this input's haystack. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Input, Span}; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(Span { start: 0, end: 6 }, input.get_span()); + /// ``` + #[inline] + pub fn get_span(&self) -> Span { + self.span + } + + /// Return the span as a range for this search configuration. + /// + /// If one was not explicitly set, then the span corresponds to the entire + /// range of the haystack. + /// + /// When [`Input::is_done`] is `false`, the range returned is guaranteed + /// to correspond to valid bounds for this input's haystack. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let input = Input::new("foobar"); + /// assert_eq!(0..6, input.get_range()); + /// ``` + #[inline] + pub fn get_range(&self) -> Range<usize> { + self.get_span().range() + } + + /// Return the anchored mode for this search configuration. + /// + /// If no anchored mode was set, then it defaults to [`Anchored::No`]. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Anchored, Input, PatternID}; + /// + /// let mut input = Input::new("foobar"); + /// assert_eq!(Anchored::No, input.get_anchored()); + /// + /// let pid = PatternID::must(5); + /// input.set_anchored(Anchored::Pattern(pid)); + /// assert_eq!(Anchored::Pattern(pid), input.get_anchored()); + /// ``` + #[inline] + pub fn get_anchored(&self) -> Anchored { + self.anchored + } + + /// Return whether this search should execute in "earliest" mode. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let input = Input::new("foobar"); + /// assert!(!input.get_earliest()); + /// ``` + #[inline] + pub fn get_earliest(&self) -> bool { + self.earliest + } + + /// Return true if and only if this search can never return any other + /// matches. + /// + /// This occurs when the start position of this search is greater than the + /// end position of the search. + /// + /// # Example + /// + /// ``` + /// use regex_automata::Input; + /// + /// let mut input = Input::new("foobar"); + /// assert!(!input.is_done()); + /// input.set_start(6); + /// assert!(!input.is_done()); + /// input.set_start(7); + /// assert!(input.is_done()); + /// ``` + #[inline] + pub fn is_done(&self) -> bool { + self.get_span().start > self.get_span().end + } + + /// Returns true if and only if the given offset in this search's haystack + /// falls on a valid UTF-8 encoded codepoint boundary. + /// + /// If the haystack is not valid UTF-8, then the behavior of this routine + /// is unspecified. + /// + /// # Example + /// + /// This shows where codepoint boundaries do and don't exist in valid + /// UTF-8. + /// + /// ``` + /// use regex_automata::Input; + /// + /// let input = Input::new("☃"); + /// assert!(input.is_char_boundary(0)); + /// assert!(!input.is_char_boundary(1)); + /// assert!(!input.is_char_boundary(2)); + /// assert!(input.is_char_boundary(3)); + /// assert!(!input.is_char_boundary(4)); + /// ``` + #[inline] + pub fn is_char_boundary(&self, offset: usize) -> bool { + utf8::is_boundary(self.haystack(), offset) + } +} + +impl<'h> core::fmt::Debug for Input<'h> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + use crate::util::escape::DebugHaystack; + + f.debug_struct("Input") + .field("haystack", &DebugHaystack(self.haystack())) + .field("span", &self.span) + .field("anchored", &self.anchored) + .field("earliest", &self.earliest) + .finish() + } +} + +impl<'h, H: ?Sized + AsRef<[u8]>> From<&'h H> for Input<'h> { + fn from(haystack: &'h H) -> Input<'h> { + Input::new(haystack) + } +} + +/// A representation of a span reported by a regex engine. +/// +/// A span corresponds to the starting and ending _byte offsets_ of a +/// contiguous region of bytes. The starting offset is inclusive while the +/// ending offset is exclusive. That is, a span is a half-open interval. +/// +/// A span is used to report the offsets of a match, but it is also used to +/// convey which region of a haystack should be searched via routines like +/// [`Input::span`]. +/// +/// This is basically equivalent to a `std::ops::Range<usize>`, except this +/// type implements `Copy` which makes it more ergonomic to use in the context +/// of this crate. Like a range, this implements `Index` for `[u8]` and `str`, +/// and `IndexMut` for `[u8]`. For convenience, this also impls `From<Range>`, +/// which means things like `Span::from(5..10)` work. +#[derive(Clone, Copy, Eq, Hash, PartialEq)] +pub struct Span { + /// The start offset of the span, inclusive. + pub start: usize, + /// The end offset of the span, exclusive. + pub end: usize, +} + +impl Span { + /// Returns this span as a range. + #[inline] + pub fn range(&self) -> Range<usize> { + Range::from(*self) + } + + /// Returns true when this span is empty. That is, when `start >= end`. + #[inline] + pub fn is_empty(&self) -> bool { + self.start >= self.end + } + + /// Returns the length of this span. + /// + /// This returns `0` in precisely the cases that `is_empty` returns `true`. + #[inline] + pub fn len(&self) -> usize { + self.end.saturating_sub(self.start) + } + + /// Returns true when the given offset is contained within this span. + /// + /// Note that an empty span contains no offsets and will always return + /// false. + #[inline] + pub fn contains(&self, offset: usize) -> bool { + !self.is_empty() && self.start <= offset && offset <= self.end + } + + /// Returns a new span with `offset` added to this span's `start` and `end` + /// values. + #[inline] + pub fn offset(&self, offset: usize) -> Span { + Span { start: self.start + offset, end: self.end + offset } + } +} + +impl core::fmt::Debug for Span { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{}..{}", self.start, self.end) + } +} + +impl core::ops::Index<Span> for [u8] { + type Output = [u8]; + + #[inline] + fn index(&self, index: Span) -> &[u8] { + &self[index.range()] + } +} + +impl core::ops::IndexMut<Span> for [u8] { + #[inline] + fn index_mut(&mut self, index: Span) -> &mut [u8] { + &mut self[index.range()] + } +} + +impl core::ops::Index<Span> for str { + type Output = str; + + #[inline] + fn index(&self, index: Span) -> &str { + &self[index.range()] + } +} + +impl From<Range<usize>> for Span { + #[inline] + fn from(range: Range<usize>) -> Span { + Span { start: range.start, end: range.end } + } +} + +impl From<Span> for Range<usize> { + #[inline] + fn from(span: Span) -> Range<usize> { + Range { start: span.start, end: span.end } + } +} + +impl PartialEq<Range<usize>> for Span { + #[inline] + fn eq(&self, range: &Range<usize>) -> bool { + self.start == range.start && self.end == range.end + } +} + +impl PartialEq<Span> for Range<usize> { + #[inline] + fn eq(&self, span: &Span) -> bool { + self.start == span.start && self.end == span.end + } +} + +/// A representation of "half" of a match reported by a DFA. +/// +/// This is called a "half" match because it only includes the end location (or +/// start location for a reverse search) of a match. This corresponds to the +/// information that a single DFA scan can report. Getting the other half of +/// the match requires a second scan with a reversed DFA. +/// +/// A half match also includes the pattern that matched. The pattern is +/// identified by an ID, which corresponds to its position (starting from `0`) +/// relative to other patterns used to construct the corresponding DFA. If only +/// a single pattern is provided to the DFA, then all matches are guaranteed to +/// have a pattern ID of `0`. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct HalfMatch { + /// The pattern ID. + pattern: PatternID, + /// The offset of the match. + /// + /// For forward searches, the offset is exclusive. For reverse searches, + /// the offset is inclusive. + offset: usize, +} + +impl HalfMatch { + /// Create a new half match from a pattern ID and a byte offset. + #[inline] + pub fn new(pattern: PatternID, offset: usize) -> HalfMatch { + HalfMatch { pattern, offset } + } + + /// Create a new half match from a pattern ID and a byte offset. + /// + /// This is like [`HalfMatch::new`], but accepts a `usize` instead of a + /// [`PatternID`]. This panics if the given `usize` is not representable + /// as a `PatternID`. + #[inline] + pub fn must(pattern: usize, offset: usize) -> HalfMatch { + HalfMatch::new(PatternID::new(pattern).unwrap(), offset) + } + + /// Returns the ID of the pattern that matched. + /// + /// The ID of a pattern is derived from the position in which it was + /// originally inserted into the corresponding DFA. The first pattern has + /// identifier `0`, and each subsequent pattern is `1`, `2` and so on. + #[inline] + pub fn pattern(&self) -> PatternID { + self.pattern + } + + /// The position of the match. + /// + /// If this match was produced by a forward search, then the offset is + /// exclusive. If this match was produced by a reverse search, then the + /// offset is inclusive. + #[inline] + pub fn offset(&self) -> usize { + self.offset + } +} + +/// A representation of a match reported by a regex engine. +/// +/// A match has two essential pieces of information: the [`PatternID`] that +/// matches, and the [`Span`] of the match in a haystack. +/// +/// The pattern is identified by an ID, which corresponds to its position +/// (starting from `0`) relative to other patterns used to construct the +/// corresponding regex engine. If only a single pattern is provided, then all +/// matches are guaranteed to have a pattern ID of `0`. +/// +/// Every match reported by a regex engine guarantees that its span has its +/// start offset as less than or equal to its end offset. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct Match { + /// The pattern ID. + pattern: PatternID, + /// The underlying match span. + span: Span, +} + +impl Match { + /// Create a new match from a pattern ID and a span. + /// + /// This constructor is generic over how a span is provided. While + /// a [`Span`] may be given directly, one may also provide a + /// `std::ops::Range<usize>`. + /// + /// # Panics + /// + /// This panics if `end < start`. + /// + /// # Example + /// + /// This shows how to create a match for the first pattern in a regex + /// object using convenient range syntax. + /// + /// ``` + /// use regex_automata::{Match, PatternID}; + /// + /// let m = Match::new(PatternID::ZERO, 5..10); + /// assert_eq!(0, m.pattern().as_usize()); + /// assert_eq!(5, m.start()); + /// assert_eq!(10, m.end()); + /// ``` + #[inline] + pub fn new<S: Into<Span>>(pattern: PatternID, span: S) -> Match { + let span: Span = span.into(); + assert!(span.start <= span.end, "invalid match span"); + Match { pattern, span } + } + + /// Create a new match from a pattern ID and a byte offset span. + /// + /// This constructor is generic over how a span is provided. While + /// a [`Span`] may be given directly, one may also provide a + /// `std::ops::Range<usize>`. + /// + /// This is like [`Match::new`], but accepts a `usize` instead of a + /// [`PatternID`]. This panics if the given `usize` is not representable + /// as a `PatternID`. + /// + /// # Panics + /// + /// This panics if `end < start` or if `pattern > PatternID::MAX`. + /// + /// # Example + /// + /// This shows how to create a match for the third pattern in a regex + /// object using convenient range syntax. + /// + /// ``` + /// use regex_automata::Match; + /// + /// let m = Match::must(3, 5..10); + /// assert_eq!(3, m.pattern().as_usize()); + /// assert_eq!(5, m.start()); + /// assert_eq!(10, m.end()); + /// ``` + #[inline] + pub fn must<S: Into<Span>>(pattern: usize, span: S) -> Match { + Match::new(PatternID::must(pattern), span) + } + + /// Returns the ID of the pattern that matched. + /// + /// The ID of a pattern is derived from the position in which it was + /// originally inserted into the corresponding regex engine. The first + /// pattern has identifier `0`, and each subsequent pattern is `1`, `2` and + /// so on. + #[inline] + pub fn pattern(&self) -> PatternID { + self.pattern + } + + /// The starting position of the match. + /// + /// This is a convenience routine for `Match::span().start`. + #[inline] + pub fn start(&self) -> usize { + self.span().start + } + + /// The ending position of the match. + /// + /// This is a convenience routine for `Match::span().end`. + #[inline] + pub fn end(&self) -> usize { + self.span().end + } + + /// Returns the match span as a range. + /// + /// This is a convenience routine for `Match::span().range()`. + #[inline] + pub fn range(&self) -> core::ops::Range<usize> { + self.span().range() + } + + /// Returns the span for this match. + #[inline] + pub fn span(&self) -> Span { + self.span + } + + /// Returns true when the span in this match is empty. + /// + /// An empty match can only be returned when the regex itself can match + /// the empty string. + #[inline] + pub fn is_empty(&self) -> bool { + self.span().is_empty() + } + + /// Returns the length of this match. + /// + /// This returns `0` in precisely the cases that `is_empty` returns `true`. + #[inline] + pub fn len(&self) -> usize { + self.span().len() + } +} + +/// A set of `PatternID`s. +/// +/// A set of pattern identifiers is useful for recording which patterns have +/// matched a particular haystack. A pattern set _only_ includes pattern +/// identifiers. It does not include offset information. +/// +/// # Example +/// +/// This shows basic usage of a set. +/// +/// ``` +/// use regex_automata::{PatternID, PatternSet}; +/// +/// let pid1 = PatternID::must(5); +/// let pid2 = PatternID::must(8); +/// // Create a new empty set. +/// let mut set = PatternSet::new(10); +/// // Insert pattern IDs. +/// set.insert(pid1); +/// set.insert(pid2); +/// // Test membership. +/// assert!(set.contains(pid1)); +/// assert!(set.contains(pid2)); +/// // Get all members. +/// assert_eq!( +/// vec![5, 8], +/// set.iter().map(|p| p.as_usize()).collect::<Vec<usize>>(), +/// ); +/// // Clear the set. +/// set.clear(); +/// // Test that it is indeed empty. +/// assert!(set.is_empty()); +/// ``` +#[cfg(feature = "alloc")] +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PatternSet { + /// The number of patterns set to 'true' in this set. + len: usize, + /// A map from PatternID to boolean of whether a pattern matches or not. + /// + /// This should probably be a bitset, but it's probably unlikely to matter + /// much in practice. + /// + /// The main downside of this representation (and similarly for a bitset) + /// is that iteration scales with the capacity of the set instead of + /// the length of the set. This doesn't seem likely to be a problem in + /// practice. + /// + /// Another alternative is to just use a 'SparseSet' for this. It does use + /// more memory (quite a bit more), but that seems fine I think compared + /// to the memory being used by the regex engine. The real hiccup with + /// it is that it yields pattern IDs in the order they were inserted. + /// Which is actually kind of nice, but at the time of writing, pattern + /// IDs are yielded in ascending order in the regex crate RegexSet API. + /// If we did change to 'SparseSet', we could provide an additional + /// 'iter_match_order' iterator, but keep the ascending order one for + /// compatibility. + which: alloc::boxed::Box<[bool]>, +} + +#[cfg(feature = "alloc")] +impl PatternSet { + /// Create a new set of pattern identifiers with the given capacity. + /// + /// The given capacity typically corresponds to (at least) the number of + /// patterns in a compiled regex object. + /// + /// # Panics + /// + /// This panics if the given capacity exceeds [`PatternID::LIMIT`]. This is + /// impossible if you use the `pattern_len()` method as defined on any of + /// the regex engines in this crate. Namely, a regex will fail to build by + /// returning an error if the number of patterns given to it exceeds the + /// limit. Therefore, the number of patterns in a valid regex is always + /// a correct capacity to provide here. + pub fn new(capacity: usize) -> PatternSet { + assert!( + capacity <= PatternID::LIMIT, + "pattern set capacity exceeds limit of {}", + PatternID::LIMIT, + ); + PatternSet { + len: 0, + which: alloc::vec![false; capacity].into_boxed_slice(), + } + } + + /// Clear this set such that it contains no pattern IDs. + pub fn clear(&mut self) { + self.len = 0; + for matched in self.which.iter_mut() { + *matched = false; + } + } + + /// Return true if and only if the given pattern identifier is in this set. + pub fn contains(&self, pid: PatternID) -> bool { + pid.as_usize() < self.capacity() && self.which[pid] + } + + /// Insert the given pattern identifier into this set and return `true` if + /// the given pattern ID was not previously in this set. + /// + /// If the pattern identifier is already in this set, then this is a no-op. + /// + /// Use [`PatternSet::try_insert`] for a fallible version of this routine. + /// + /// # Panics + /// + /// This panics if this pattern set has insufficient capacity to + /// store the given pattern ID. + pub fn insert(&mut self, pid: PatternID) -> bool { + self.try_insert(pid) + .expect("PatternSet should have sufficient capacity") + } + + /// Insert the given pattern identifier into this set and return `true` if + /// the given pattern ID was not previously in this set. + /// + /// If the pattern identifier is already in this set, then this is a no-op. + /// + /// # Errors + /// + /// This returns an error if this pattern set has insufficient capacity to + /// store the given pattern ID. + pub fn try_insert( + &mut self, + pid: PatternID, + ) -> Result<bool, PatternSetInsertError> { + if pid.as_usize() >= self.capacity() { + return Err(PatternSetInsertError { + attempted: pid, + capacity: self.capacity(), + }); + } + if self.which[pid] { + return Ok(false); + } + self.len += 1; + self.which[pid] = true; + Ok(true) + } + + /* + // This is currently commented out because it is unused and it is unclear + // whether it's useful or not. What's the harm in having it? When, if + // we ever wanted to change our representation to a 'SparseSet', then + // supporting this method would be a bit tricky. So in order to keep some + // API evolution flexibility, we leave it out for now. + + /// Remove the given pattern identifier from this set. + /// + /// If the pattern identifier was not previously in this set, then this + /// does not change the set and returns `false`. + /// + /// # Panics + /// + /// This panics if `pid` exceeds the capacity of this set. + pub fn remove(&mut self, pid: PatternID) -> bool { + if !self.which[pid] { + return false; + } + self.len -= 1; + self.which[pid] = false; + true + } + */ + + /// Return true if and only if this set has no pattern identifiers in it. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Return true if and only if this set has the maximum number of pattern + /// identifiers in the set. This occurs precisely when `PatternSet::len() + /// == PatternSet::capacity()`. + /// + /// This particular property is useful to test because it may allow one to + /// stop a search earlier than you might otherwise. Namely, if a search is + /// only reporting which patterns match a haystack and if you know all of + /// the patterns match at a given point, then there's no new information + /// that can be learned by continuing the search. (Because a pattern set + /// does not keep track of offset information.) + pub fn is_full(&self) -> bool { + self.len() == self.capacity() + } + + /// Returns the total number of pattern identifiers in this set. + pub fn len(&self) -> usize { + self.len + } + + /// Returns the total number of pattern identifiers that may be stored + /// in this set. + /// + /// This is guaranteed to be less than or equal to [`PatternID::LIMIT`]. + /// + /// Typically, the capacity of a pattern set matches the number of patterns + /// in a regex object with which you are searching. + pub fn capacity(&self) -> usize { + self.which.len() + } + + /// Returns an iterator over all pattern identifiers in this set. + /// + /// The iterator yields pattern identifiers in ascending order, starting + /// at zero. + pub fn iter(&self) -> PatternSetIter<'_> { + PatternSetIter { it: self.which.iter().enumerate() } + } +} + +/// An error that occurs when a `PatternID` failed to insert into a +/// `PatternSet`. +/// +/// An insert fails when the given `PatternID` exceeds the configured capacity +/// of the `PatternSet`. +/// +/// This error is created by the [`PatternSet::try_insert`] routine. +#[cfg(feature = "alloc")] +#[derive(Clone, Debug)] +pub struct PatternSetInsertError { + attempted: PatternID, + capacity: usize, +} + +#[cfg(feature = "std")] +impl std::error::Error for PatternSetInsertError {} + +#[cfg(feature = "alloc")] +impl core::fmt::Display for PatternSetInsertError { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!( + f, + "failed to insert pattern ID {} into pattern set \ + with insufficient capacity of {}", + self.attempted.as_usize(), + self.capacity, + ) + } +} + +/// An iterator over all pattern identifiers in a [`PatternSet`]. +/// +/// The lifetime parameter `'a` refers to the lifetime of the pattern set being +/// iterated over. +/// +/// This iterator is created by the [`PatternSet::iter`] method. +#[cfg(feature = "alloc")] +#[derive(Clone, Debug)] +pub struct PatternSetIter<'a> { + it: core::iter::Enumerate<core::slice::Iter<'a, bool>>, +} + +#[cfg(feature = "alloc")] +impl<'a> Iterator for PatternSetIter<'a> { + type Item = PatternID; + + fn next(&mut self) -> Option<PatternID> { + while let Some((index, &yes)) = self.it.next() { + if yes { + // Only valid 'PatternID' values can be inserted into the set + // and construction of the set panics if the capacity would + // permit storing invalid pattern IDs. Thus, 'yes' is only true + // precisely when 'index' corresponds to a valid 'PatternID'. + return Some(PatternID::new_unchecked(index)); + } + } + None + } + + fn size_hint(&self) -> (usize, Option<usize>) { + self.it.size_hint() + } +} + +#[cfg(feature = "alloc")] +impl<'a> DoubleEndedIterator for PatternSetIter<'a> { + fn next_back(&mut self) -> Option<PatternID> { + while let Some((index, &yes)) = self.it.next_back() { + if yes { + // Only valid 'PatternID' values can be inserted into the set + // and construction of the set panics if the capacity would + // permit storing invalid pattern IDs. Thus, 'yes' is only true + // precisely when 'index' corresponds to a valid 'PatternID'. + return Some(PatternID::new_unchecked(index)); + } + } + None + } +} + +/// The type of anchored search to perform. +/// +/// This is *almost* a boolean option. That is, you can either do an unanchored +/// search for any pattern in a regex, or you can do an anchored search for any +/// pattern in a regex. +/// +/// A third option exists that, assuming the regex engine supports it, permits +/// you to do an anchored search for a specific pattern. +/// +/// Note that there is no way to run an unanchored search for a specific +/// pattern. If you need that, you'll need to build separate regexes for each +/// pattern. +/// +/// # Errors +/// +/// If a regex engine does not support the anchored mode selected, then the +/// regex engine will return an error. While any non-trivial regex engine +/// should support at least one of the available anchored modes, there is no +/// singular mode that is guaranteed to be universally supported. Some regex +/// engines might only support unanchored searches (DFAs compiled without +/// anchored starting states) and some regex engines might only support +/// anchored searches (like the one-pass DFA). +/// +/// The specific error returned is a [`MatchError`] with a +/// [`MatchErrorKind::UnsupportedAnchored`] kind. The kind includes the +/// `Anchored` value given that is unsupported. +/// +/// Note that regex engines should report "no match" if, for example, an +/// `Anchored::Pattern` is provided with an invalid pattern ID _but_ where +/// anchored searches for a specific pattern are supported. This is smooths out +/// behavior such that it's possible to guarantee that an error never occurs +/// based on how the regex engine is configured. All regex engines in this +/// crate report "no match" when searching for an invalid pattern ID, but where +/// searching for a valid pattern ID is otherwise supported. +/// +/// # Example +/// +/// This example shows how to use the various `Anchored` modes to run a +/// search. We use the [`PikeVM`](crate::nfa::thompson::pikevm::PikeVM) +/// because it supports all modes unconditionally. Some regex engines, like +/// the [`onepass::DFA`](crate::dfa::onepass::DFA) cannot support unanchored +/// searches. +/// +/// ``` +/// # if cfg!(miri) { return Ok(()); } // miri takes too long +/// use regex_automata::{ +/// nfa::thompson::pikevm::PikeVM, +/// Anchored, Input, Match, PatternID, +/// }; +/// +/// let re = PikeVM::new_many(&[ +/// r"Mrs. \w+", +/// r"Miss \w+", +/// r"Mr. \w+", +/// r"Ms. \w+", +/// ])?; +/// let mut cache = re.create_cache(); +/// let hay = "Hello Mr. Springsteen!"; +/// +/// // The default is to do an unanchored search. +/// assert_eq!(Some(Match::must(2, 6..21)), re.find(&mut cache, hay)); +/// // Explicitly ask for an unanchored search. Same as above. +/// let input = Input::new(hay).anchored(Anchored::No); +/// assert_eq!(Some(Match::must(2, 6..21)), re.find(&mut cache, hay)); +/// +/// // Now try an anchored search. Since the match doesn't start at the +/// // beginning of the haystack, no match is found! +/// let input = Input::new(hay).anchored(Anchored::Yes); +/// assert_eq!(None, re.find(&mut cache, input)); +/// +/// // We can try an anchored search again, but move the location of where +/// // we start the search. Note that the offsets reported are still in +/// // terms of the overall haystack and not relative to where we started +/// // the search. +/// let input = Input::new(hay).anchored(Anchored::Yes).range(6..); +/// assert_eq!(Some(Match::must(2, 6..21)), re.find(&mut cache, input)); +/// +/// // Now try an anchored search for a specific pattern. We specifically +/// // choose a pattern that we know doesn't match to prove that the search +/// // only looks for the pattern we provide. +/// let input = Input::new(hay) +/// .anchored(Anchored::Pattern(PatternID::must(1))) +/// .range(6..); +/// assert_eq!(None, re.find(&mut cache, input)); +/// +/// // But if we switch it to the pattern that we know matches, then we find +/// // the match. +/// let input = Input::new(hay) +/// .anchored(Anchored::Pattern(PatternID::must(2))) +/// .range(6..); +/// assert_eq!(Some(Match::must(2, 6..21)), re.find(&mut cache, input)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Anchored { + /// Run an unanchored search. This means a match may occur anywhere at or + /// after the start position of the search. + /// + /// This search can return a match for any pattern in the regex. + No, + /// Run an anchored search. This means that a match must begin at the + /// start position of the search. + /// + /// This search can return a match for any pattern in the regex. + Yes, + /// Run an anchored search for a specific pattern. This means that a match + /// must be for the given pattern and must begin at the start position of + /// the search. + Pattern(PatternID), +} + +impl Anchored { + /// Returns true if and only if this anchor mode corresponds to any kind of + /// anchored search. + /// + /// # Example + /// + /// This examples shows that both `Anchored::Yes` and `Anchored::Pattern` + /// are considered anchored searches. + /// + /// ``` + /// use regex_automata::{Anchored, PatternID}; + /// + /// assert!(!Anchored::No.is_anchored()); + /// assert!(Anchored::Yes.is_anchored()); + /// assert!(Anchored::Pattern(PatternID::ZERO).is_anchored()); + /// ``` + #[inline] + pub fn is_anchored(&self) -> bool { + matches!(*self, Anchored::Yes | Anchored::Pattern(_)) + } + + /// Returns the pattern ID associated with this configuration if it is an + /// anchored search for a specific pattern. Otherwise `None` is returned. + /// + /// # Example + /// + /// ``` + /// use regex_automata::{Anchored, PatternID}; + /// + /// assert_eq!(None, Anchored::No.pattern()); + /// assert_eq!(None, Anchored::Yes.pattern()); + /// + /// let pid = PatternID::must(5); + /// assert_eq!(Some(pid), Anchored::Pattern(pid).pattern()); + /// ``` + #[inline] + pub fn pattern(&self) -> Option<PatternID> { + match *self { + Anchored::Pattern(pid) => Some(pid), + _ => None, + } + } +} + +/// The kind of match semantics to use for a regex pattern. +/// +/// The default match kind is `LeftmostFirst`, and this corresponds to the +/// match semantics used by most backtracking engines, such as Perl. +/// +/// # Leftmost first or "preference order" match semantics +/// +/// Leftmost-first semantics determine which match to report when there are +/// multiple paths through a regex that match at the same position. The tie is +/// essentially broken by how a backtracker would behave. For example, consider +/// running the regex `foofoofoo|foofoo|foo` on the haystack `foofoo`. In this +/// case, both the `foofoo` and `foo` branches match at position `0`. So should +/// the end of the match be `3` or `6`? +/// +/// A backtracker will conceptually work by trying `foofoofoo` and failing. +/// Then it will try `foofoo`, find the match and stop there. Thus, the +/// leftmost-first match position is `6`. This is called "leftmost-first" or +/// "preference order" because the order of the branches as written in the +/// regex pattern is what determines how to break the tie. +/// +/// (Note that leftmost-longest match semantics, which break ties by always +/// taking the longest matching string, are not currently supported by this +/// crate. These match semantics tend to be found in POSIX regex engines.) +/// +/// This example shows how leftmost-first semantics work, and how it even +/// applies to multi-pattern regexes: +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::pikevm::PikeVM, +/// Match, +/// }; +/// +/// let re = PikeVM::new_many(&[ +/// r"foofoofoo", +/// r"foofoo", +/// r"foo", +/// ])?; +/// let mut cache = re.create_cache(); +/// let got: Vec<Match> = re.find_iter(&mut cache, "foofoo").collect(); +/// let expected = vec![Match::must(1, 0..6)]; +/// assert_eq!(expected, got); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// # All matches +/// +/// The `All` match semantics report any and all matches, and generally will +/// attempt to match as much as possible. It doesn't respect any sort of match +/// priority at all, so things like non-greedy matching don't work in this +/// mode. +/// +/// The fact that non-greedy matching doesn't work generally makes most forms +/// of unanchored non-overlapping searches have unintuitive behavior. Namely, +/// unanchored searches behave as if there is a `(?s-u:.)*?` prefix at the +/// beginning of the pattern, which is specifically non-greedy. Since it will +/// be treated as greedy in `All` match semantics, this generally means that +/// it will first attempt to consume all of the haystack and is likely to wind +/// up skipping matches. +/// +/// Generally speaking, `All` should only be used in two circumstances: +/// +/// * When running an anchored search and there is a desire to match as much as +/// possible. For example, when building a reverse regex matcher to find the +/// start of a match after finding the end. In this case, the reverse search +/// is anchored to the end of the match found by the forward search. +/// * When running overlapping searches. Since `All` encodes all possible +/// matches, this is generally what you want for an overlapping search. If you +/// try to use leftmost-first in an overlapping search, it is likely to produce +/// counter-intuitive results since leftmost-first specifically excludes some +/// matches from its underlying finite state machine. +/// +/// This example demonstrates the counter-intuitive behavior of `All` semantics +/// when using a standard leftmost unanchored search: +/// +/// ``` +/// use regex_automata::{ +/// nfa::thompson::pikevm::PikeVM, +/// Match, MatchKind, +/// }; +/// +/// let re = PikeVM::builder() +/// .configure(PikeVM::config().match_kind(MatchKind::All)) +/// .build("foo")?; +/// let hay = "first foo second foo wat"; +/// let mut cache = re.create_cache(); +/// let got: Vec<Match> = re.find_iter(&mut cache, hay).collect(); +/// // Notice that it completely skips the first 'foo'! +/// let expected = vec![Match::must(0, 17..20)]; +/// assert_eq!(expected, got); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// This second example shows how `All` semantics are useful for an overlapping +/// search. Note that we use lower level lazy DFA APIs here since the NFA +/// engines only currently support a very limited form of overlapping search. +/// +/// ``` +/// use regex_automata::{ +/// hybrid::dfa::{DFA, OverlappingState}, +/// HalfMatch, Input, MatchKind, +/// }; +/// +/// let re = DFA::builder() +/// // If we didn't set 'All' semantics here, then the regex would only +/// // match 'foo' at offset 3 and nothing else. Why? Because the state +/// // machine implements preference order and knows that the 'foofoo' and +/// // 'foofoofoo' branches can never match since 'foo' will always match +/// // when they match and take priority. +/// .configure(DFA::config().match_kind(MatchKind::All)) +/// .build(r"foo|foofoo|foofoofoo")?; +/// let mut cache = re.create_cache(); +/// let mut state = OverlappingState::start(); +/// let input = Input::new("foofoofoo"); +/// let mut got = vec![]; +/// loop { +/// re.try_search_overlapping_fwd(&mut cache, &input, &mut state)?; +/// let m = match state.get_match() { +/// None => break, +/// Some(m) => m, +/// }; +/// got.push(m); +/// } +/// let expected = vec![ +/// HalfMatch::must(0, 3), +/// HalfMatch::must(0, 6), +/// HalfMatch::must(0, 9), +/// ]; +/// assert_eq!(expected, got); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[non_exhaustive] +#[derive(Clone, Copy, Default, Debug, Eq, PartialEq)] +pub enum MatchKind { + /// Report all possible matches. + All, + /// Report only the leftmost matches. When multiple leftmost matches exist, + /// report the match corresponding to the part of the regex that appears + /// first in the syntax. + #[default] + LeftmostFirst, + // There is prior art in RE2 that shows that we should be able to add + // LeftmostLongest too. The tricky part of it is supporting ungreedy + // repetitions. Instead of treating all NFA states as having equivalent + // priority (as in 'All') or treating all NFA states as having distinct + // priority based on order (as in 'LeftmostFirst'), we instead group NFA + // states into sets, and treat members of each set as having equivalent + // priority, but having greater priority than all following members + // of different sets. + // + // However, it's not clear whether it's really worth adding this. After + // all, leftmost-longest can be emulated when using literals by using + // leftmost-first and sorting the literals by length in descending order. + // However, this won't work for arbitrary regexes. e.g., `\w|\w\w` will + // always match `a` in `ab` when using leftmost-first, but leftmost-longest + // would match `ab`. +} + +impl MatchKind { + #[cfg(feature = "alloc")] + pub(crate) fn continue_past_first_match(&self) -> bool { + *self == MatchKind::All + } +} + +/// An error indicating that a search stopped before reporting whether a +/// match exists or not. +/// +/// To be very clear, this error type implies that one cannot assume that no +/// matches occur, since the search stopped before completing. That is, if +/// you're looking for information about where a search determined that no +/// match can occur, then this error type does *not* give you that. (Indeed, at +/// the time of writing, if you need such a thing, you have to write your own +/// search routine.) +/// +/// Normally, when one searches for something, the response is either an +/// affirmative "it was found at this location" or a negative "not found at +/// all." However, in some cases, a regex engine can be configured to stop its +/// search before concluding whether a match exists or not. When this happens, +/// it may be important for the caller to know why the regex engine gave up and +/// where in the input it gave up at. This error type exposes the 'why' and the +/// 'where.' +/// +/// For example, the DFAs provided by this library generally cannot correctly +/// implement Unicode word boundaries. Instead, they provide an option to +/// eagerly support them on ASCII text (since Unicode word boundaries are +/// equivalent to ASCII word boundaries when searching ASCII text), but will +/// "give up" if a non-ASCII byte is seen. In such cases, one is usually +/// required to either report the failure to the caller (unergonomic) or +/// otherwise fall back to some other regex engine (ergonomic, but potentially +/// costly). +/// +/// More generally, some regex engines offer the ability for callers to specify +/// certain bytes that will trigger the regex engine to automatically quit if +/// they are seen. +/// +/// Still yet, there may be other reasons for a failed match. For example, +/// the hybrid DFA provided by this crate can be configured to give up if it +/// believes that it is not efficient. This in turn permits callers to choose a +/// different regex engine. +/// +/// (Note that DFAs are configured by default to never quit or give up in this +/// fashion. For example, by default, a DFA will fail to build if the regex +/// pattern contains a Unicode word boundary. One needs to opt into the "quit" +/// behavior via options, like +/// [`hybrid::dfa::Config::unicode_word_boundary`](crate::hybrid::dfa::Config::unicode_word_boundary).) +/// +/// There are a couple other ways a search +/// can fail. For example, when using the +/// [`BoundedBacktracker`](crate::nfa::thompson::backtrack::BoundedBacktracker) +/// with a haystack that is too long, or trying to run an unanchored search +/// with a [one-pass DFA](crate::dfa::onepass). +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct MatchError( + #[cfg(feature = "alloc")] alloc::boxed::Box<MatchErrorKind>, + #[cfg(not(feature = "alloc"))] MatchErrorKind, +); + +impl MatchError { + /// Create a new error value with the given kind. + /// + /// This is a more verbose version of the kind-specific constructors, + /// e.g., `MatchError::quit`. + pub fn new(kind: MatchErrorKind) -> MatchError { + #[cfg(feature = "alloc")] + { + MatchError(alloc::boxed::Box::new(kind)) + } + #[cfg(not(feature = "alloc"))] + { + MatchError(kind) + } + } + + /// Returns a reference to the underlying error kind. + pub fn kind(&self) -> &MatchErrorKind { + &self.0 + } + + /// Create a new "quit" error. The given `byte` corresponds to the value + /// that tripped a search's quit condition, and `offset` corresponds to the + /// location in the haystack at which the search quit. + /// + /// This is the same as calling `MatchError::new` with a + /// [`MatchErrorKind::Quit`] kind. + pub fn quit(byte: u8, offset: usize) -> MatchError { + MatchError::new(MatchErrorKind::Quit { byte, offset }) + } + + /// Create a new "gave up" error. The given `offset` corresponds to the + /// location in the haystack at which the search gave up. + /// + /// This is the same as calling `MatchError::new` with a + /// [`MatchErrorKind::GaveUp`] kind. + pub fn gave_up(offset: usize) -> MatchError { + MatchError::new(MatchErrorKind::GaveUp { offset }) + } + + /// Create a new "haystack too long" error. The given `len` corresponds to + /// the length of the haystack that was problematic. + /// + /// This is the same as calling `MatchError::new` with a + /// [`MatchErrorKind::HaystackTooLong`] kind. + pub fn haystack_too_long(len: usize) -> MatchError { + MatchError::new(MatchErrorKind::HaystackTooLong { len }) + } + + /// Create a new "unsupported anchored" error. This occurs when the caller + /// requests a search with an anchor mode that is not supported by the + /// regex engine. + /// + /// This is the same as calling `MatchError::new` with a + /// [`MatchErrorKind::UnsupportedAnchored`] kind. + pub fn unsupported_anchored(mode: Anchored) -> MatchError { + MatchError::new(MatchErrorKind::UnsupportedAnchored { mode }) + } +} + +/// The underlying kind of a [`MatchError`]. +/// +/// This is a **non-exhaustive** enum. That means new variants may be added in +/// a semver-compatible release. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum MatchErrorKind { + /// The search saw a "quit" byte at which it was instructed to stop + /// searching. + Quit { + /// The "quit" byte that was observed that caused the search to stop. + byte: u8, + /// The offset at which the quit byte was observed. + offset: usize, + }, + /// The search, based on heuristics, determined that it would be better + /// to stop, typically to provide the caller an opportunity to use an + /// alternative regex engine. + /// + /// Currently, the only way for this to occur is via the lazy DFA and + /// only when it is configured to do so (it will not return this error by + /// default). + GaveUp { + /// The offset at which the search stopped. This corresponds to the + /// position immediately following the last byte scanned. + offset: usize, + }, + /// This error occurs if the haystack given to the regex engine was too + /// long to be searched. This occurs, for example, with regex engines + /// like the bounded backtracker that have a configurable fixed amount of + /// capacity that is tied to the length of the haystack. Anything beyond + /// that configured limit will result in an error at search time. + HaystackTooLong { + /// The length of the haystack that exceeded the limit. + len: usize, + }, + /// An error indicating that a particular type of anchored search was + /// requested, but that the regex engine does not support it. + /// + /// When this error occurs with [`Anchored::Pattern`], then one common + /// cause is that the underlying regex engine was built without support + /// for anchored starting states for each pattern. For DFA engines, this + /// typically means enabling the + /// [`hybrid::dfa::Config::starts_for_each_pattern`](crate::hybrid::dfa::Config::starts_for_each_pattern) + /// or + /// [`dfa::dense::Config::starts_for_each_pattern`](crate::dfa::dense::Config::starts_for_each_pattern) + /// configuration option, depending on which regex engine is being built. + /// + /// Note that this error should not be returned by a regex engine simply + /// because the pattern ID is invalid (i.e., equal to or exceeds the number + /// of patterns in the regex). In that case, the regex engine should report + /// a non-match. + UnsupportedAnchored { + /// The anchored mode given that is unsupported. + mode: Anchored, + }, +} + +#[cfg(feature = "std")] +impl std::error::Error for MatchError {} + +impl core::fmt::Display for MatchError { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + match *self.kind() { + MatchErrorKind::Quit { byte, offset } => write!( + f, + "quit search after observing byte {:?} at offset {}", + DebugByte(byte), + offset, + ), + MatchErrorKind::GaveUp { offset } => { + write!(f, "gave up searching at offset {offset}") + } + MatchErrorKind::HaystackTooLong { len } => { + write!(f, "haystack of length {len} is too long") + } + MatchErrorKind::UnsupportedAnchored { mode: Anchored::Yes } => { + write!(f, "anchored searches are not supported or enabled") + } + MatchErrorKind::UnsupportedAnchored { mode: Anchored::No } => { + write!(f, "unanchored searches are not supported or enabled") + } + MatchErrorKind::UnsupportedAnchored { + mode: Anchored::Pattern(pid), + } => { + write!( + f, + "anchored searches for a specific pattern ({}) are \ + not supported or enabled", + pid.as_usize(), + ) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // We test that our 'MatchError' type is the size we expect. This isn't an + // API guarantee, but if the size increases, we really want to make sure we + // decide to do that intentionally. So this should be a speed bump. And in + // general, we should not increase the size without a very good reason. + // + // Why? Because low level search APIs return Result<.., MatchError>. When + // MatchError gets bigger, so to does the Result type. + // + // Now, when 'alloc' is enabled, we do box the error, which de-emphasizes + // the importance of keeping a small error type. But without 'alloc', we + // still want things to be small. + #[test] + fn match_error_size() { + let expected_size = if cfg!(feature = "alloc") { + core::mem::size_of::<usize>() + } else { + 2 * core::mem::size_of::<usize>() + }; + assert_eq!(expected_size, core::mem::size_of::<MatchError>()); + } + + // Same as above, but for the underlying match error kind. + #[cfg(target_pointer_width = "64")] + #[test] + fn match_error_kind_size() { + let expected_size = 2 * core::mem::size_of::<usize>(); + assert_eq!(expected_size, core::mem::size_of::<MatchErrorKind>()); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn match_error_kind_size() { + let expected_size = 3 * core::mem::size_of::<usize>(); + assert_eq!(expected_size, core::mem::size_of::<MatchErrorKind>()); + } + + #[test] + fn incorrect_asref_guard() { + struct Bad(std::cell::Cell<bool>); + + impl AsRef<[u8]> for Bad { + fn as_ref(&self) -> &[u8] { + if self.0.replace(false) { + &[] + } else { + &[0; 1000] + } + } + } + + let bad = Bad(std::cell::Cell::new(true)); + let input = Input::new(&bad); + assert!(input.end() <= input.haystack().len()); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/sparse_set.rs b/third_party/crates/regex-automata-0.4.18/src/util/sparse_set.rs new file mode 100644 index 0000000..e30d5b9 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/sparse_set.rs @@ -0,0 +1,239 @@ +/*! +This module defines a sparse set data structure. Its most interesting +properties are: + +* They preserve insertion order. +* Set membership testing is done in constant time. +* Set insertion is done in constant time. +* Clearing the set is done in constant time. + +The cost for doing this is that the capacity of the set needs to be known up +front, and the elements in the set are limited to state identifiers. + +These sets are principally used when traversing an NFA state graph. This +happens at search time, for example, in the PikeVM. It also happens during DFA +determinization. +*/ + +use alloc::{vec, vec::Vec}; + +use crate::util::primitives::StateID; + +/// A pair of sparse sets. +/// +/// This is useful when one needs to compute NFA epsilon closures from a +/// previous set of states derived from an epsilon closure. One set can be the +/// starting states where as the other set can be the destination states after +/// following the transitions for a particular byte of input. +/// +/// There is no significance to 'set1' or 'set2'. They are both sparse sets of +/// the same size. +/// +/// The members of this struct are exposed so that callers may borrow 'set1' +/// and 'set2' individually without being force to borrow both at the same +/// time. +#[derive(Clone, Debug)] +pub(crate) struct SparseSets { + pub(crate) set1: SparseSet, + pub(crate) set2: SparseSet, +} + +impl SparseSets { + /// Create a new pair of sparse sets where each set has the given capacity. + /// + /// This panics if the capacity given is bigger than `StateID::LIMIT`. + pub(crate) fn new(capacity: usize) -> SparseSets { + SparseSets { + set1: SparseSet::new(capacity), + set2: SparseSet::new(capacity), + } + } + + /// Resizes these sparse sets to have the new capacity given. + /// + /// The sets are automatically cleared. + /// + /// This panics if the capacity given is bigger than `StateID::LIMIT`. + #[inline] + pub(crate) fn resize(&mut self, new_capacity: usize) { + self.set1.resize(new_capacity); + self.set2.resize(new_capacity); + } + + /// Clear both sparse sets. + pub(crate) fn clear(&mut self) { + self.set1.clear(); + self.set2.clear(); + } + + /// Swap set1 with set2. + pub(crate) fn swap(&mut self) { + core::mem::swap(&mut self.set1, &mut self.set2); + } + + /// Returns the memory usage, in bytes, used by this pair of sparse sets. + pub(crate) fn memory_usage(&self) -> usize { + self.set1.memory_usage() + self.set2.memory_usage() + } +} + +/// A sparse set used for representing ordered NFA states. +/// +/// This supports constant time addition and membership testing. Clearing an +/// entire set can also be done in constant time. Iteration yields elements +/// in the order in which they were inserted. +/// +/// The data structure is based on: https://research.swtch.com/sparse +/// Note though that we don't actually use uninitialized memory. We generally +/// reuse sparse sets, so the initial allocation cost is bearable. However, its +/// other properties listed above are extremely useful. +#[derive(Clone)] +pub(crate) struct SparseSet { + /// The number of elements currently in this set. + len: usize, + /// Dense contains the ids in the order in which they were inserted. + dense: Vec<StateID>, + /// Sparse maps ids to their location in dense. + /// + /// A state ID is in the set if and only if + /// sparse[id] < len && id == dense[sparse[id]]. + /// + /// Note that these are indices into 'dense'. It's a little weird to use + /// StateID here, but we know our length can never exceed the bounds of + /// StateID (enforced by 'resize') and StateID will be at most 4 bytes + /// where as a usize is likely double that in most cases. + sparse: Vec<StateID>, +} + +impl SparseSet { + /// Create a new sparse set with the given capacity. + /// + /// Sparse sets have a fixed size and they cannot grow. Attempting to + /// insert more distinct elements than the total capacity of the set will + /// result in a panic. + /// + /// This panics if the capacity given is bigger than `StateID::LIMIT`. + #[inline] + pub(crate) fn new(capacity: usize) -> SparseSet { + let mut set = SparseSet { len: 0, dense: vec![], sparse: vec![] }; + set.resize(capacity); + set + } + + /// Resizes this sparse set to have the new capacity given. + /// + /// This set is automatically cleared. + /// + /// This panics if the capacity given is bigger than `StateID::LIMIT`. + #[inline] + pub(crate) fn resize(&mut self, new_capacity: usize) { + assert!( + new_capacity <= StateID::LIMIT, + "sparse set capacity cannot exceed {:?}", + StateID::LIMIT + ); + self.clear(); + self.dense.resize(new_capacity, StateID::ZERO); + self.sparse.resize(new_capacity, StateID::ZERO); + } + + /// Returns the capacity of this set. + /// + /// The capacity represents a fixed limit on the number of distinct + /// elements that are allowed in this set. The capacity cannot be changed. + #[inline] + pub(crate) fn capacity(&self) -> usize { + self.dense.len() + } + + /// Returns the number of elements in this set. + #[inline] + pub(crate) fn len(&self) -> usize { + self.len + } + + /// Returns true if and only if this set is empty. + #[inline] + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Insert the state ID value into this set and return true if the given + /// state ID was not previously in this set. + /// + /// This operation is idempotent. If the given value is already in this + /// set, then this is a no-op. + /// + /// If more than `capacity` ids are inserted, then this panics. + /// + /// This is marked as inline(always) since the compiler won't inline it + /// otherwise, and it's a fairly hot piece of code in DFA determinization. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn insert(&mut self, id: StateID) -> bool { + if self.contains(id) { + return false; + } + + let i = self.len(); + assert!( + i < self.capacity(), + "{:?} exceeds capacity of {:?} when inserting {:?}", + i, + self.capacity(), + id, + ); + // OK since i < self.capacity() and self.capacity() is guaranteed to + // be <= StateID::LIMIT. + let index = StateID::new_unchecked(i); + self.dense[index] = id; + self.sparse[id] = index; + self.len += 1; + true + } + + /// Returns true if and only if this set contains the given value. + #[inline] + pub(crate) fn contains(&self, id: StateID) -> bool { + let index = self.sparse[id]; + index.as_usize() < self.len() && self.dense[index] == id + } + + /// Clear this set such that it has no members. + #[inline] + pub(crate) fn clear(&mut self) { + self.len = 0; + } + + #[inline] + pub(crate) fn iter(&self) -> SparseSetIter<'_> { + SparseSetIter(self.dense[..self.len()].iter()) + } + + /// Returns the heap memory usage, in bytes, used by this sparse set. + #[inline] + pub(crate) fn memory_usage(&self) -> usize { + self.dense.len() * StateID::SIZE + self.sparse.len() * StateID::SIZE + } +} + +impl core::fmt::Debug for SparseSet { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let elements: Vec<StateID> = self.iter().collect(); + f.debug_tuple("SparseSet").field(&elements).finish() + } +} + +/// An iterator over all elements in a sparse set. +/// +/// The lifetime `'a` refers to the lifetime of the set being iterated over. +#[derive(Debug)] +pub(crate) struct SparseSetIter<'a>(core::slice::Iter<'a, StateID>); + +impl<'a> Iterator for SparseSetIter<'a> { + type Item = StateID; + + #[cfg_attr(feature = "perf-inline", inline(always))] + fn next(&mut self) -> Option<StateID> { + self.0.next().copied() + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/start.rs b/third_party/crates/regex-automata-0.4.18/src/util/start.rs new file mode 100644 index 0000000..c34b376 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/start.rs @@ -0,0 +1,479 @@ +/*! +Provides helpers for dealing with start state configurations in DFAs. +*/ + +use crate::util::{ + look::LookMatcher, + search::{Anchored, Input}, + wire::{self, DeserializeError, SerializeError}, +}; + +/// The configuration used to determine a DFA's start state for a search. +/// +/// A DFA has a single starting state in the typical textbook description. That +/// is, it corresponds to the set of all starting states for the NFA that built +/// it, along with their epsilon closures. In this crate, however, DFAs have +/// many possible start states due to a few factors: +/// +/// * DFAs support the ability to run either anchored or unanchored searches. +/// Each type of search needs its own start state. For example, an unanchored +/// search requires starting at a state corresponding to a regex with a +/// `(?s-u:.)*?` prefix, which will match through anything. +/// * DFAs also optionally support starting an anchored search for any one +/// specific pattern. Each such pattern requires its own start state. +/// * If a look-behind assertion like `^` or `\b` is used in the regex, then +/// the DFA will need to inspect a single byte immediately before the start of +/// the search to choose the correct start state. +/// +/// Indeed, this configuration precisely encapsulates all of the above factors. +/// The [`Config::anchored`] method sets which kind of anchored search to +/// perform while the [`Config::look_behind`] method provides a way to set +/// the byte that occurs immediately before the start of the search. +/// +/// Generally speaking, this type is only useful when you want to run searches +/// without using an [`Input`]. In particular, an `Input` wants a haystack +/// slice, but callers may not have a contiguous sequence of bytes as a +/// haystack in all cases. This type provides a lower level of control such +/// that callers can provide their own anchored configuration and look-behind +/// byte explicitly. +/// +/// # Example +/// +/// This shows basic usage that permits running a search with a DFA without +/// using the `Input` abstraction. +/// +/// ``` +/// use regex_automata::{ +/// dfa::{Automaton, dense}, +/// util::start, +/// Anchored, +/// }; +/// +/// let dfa = dense::DFA::new(r"(?-u)\b\w+\b")?; +/// let haystack = "quartz"; +/// +/// let config = start::Config::new().anchored(Anchored::Yes); +/// let mut state = dfa.start_state(&config)?; +/// for &b in haystack.as_bytes().iter() { +/// state = dfa.next_state(state, b); +/// } +/// state = dfa.next_eoi_state(state); +/// assert!(dfa.is_match_state(state)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// This example shows how to correctly run a search that doesn't begin at +/// the start of a haystack. Notice how we set the look-behind byte, and as +/// a result, the `\b` assertion does not match. +/// +/// ``` +/// use regex_automata::{ +/// dfa::{Automaton, dense}, +/// util::start, +/// Anchored, +/// }; +/// +/// let dfa = dense::DFA::new(r"(?-u)\b\w+\b")?; +/// let haystack = "quartz"; +/// +/// let config = start::Config::new() +/// .anchored(Anchored::Yes) +/// .look_behind(Some(b'q')); +/// let mut state = dfa.start_state(&config)?; +/// for &b in haystack.as_bytes().iter().skip(1) { +/// state = dfa.next_state(state, b); +/// } +/// state = dfa.next_eoi_state(state); +/// // No match! +/// assert!(!dfa.is_match_state(state)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// If we had instead not set a look-behind byte, then the DFA would assume +/// that it was starting at the beginning of the haystack, and thus `\b` should +/// match. This in turn would result in erroneously reporting a match: +/// +/// ``` +/// use regex_automata::{ +/// dfa::{Automaton, dense}, +/// util::start, +/// Anchored, +/// }; +/// +/// let dfa = dense::DFA::new(r"(?-u)\b\w+\b")?; +/// let haystack = "quartz"; +/// +/// // Whoops, forgot the look-behind byte... +/// let config = start::Config::new().anchored(Anchored::Yes); +/// let mut state = dfa.start_state(&config)?; +/// for &b in haystack.as_bytes().iter().skip(1) { +/// state = dfa.next_state(state, b); +/// } +/// state = dfa.next_eoi_state(state); +/// // And now we get a match unexpectedly. +/// assert!(dfa.is_match_state(state)); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Config { + look_behind: Option<u8>, + anchored: Anchored, +} + +impl Config { + /// Create a new default start configuration. + /// + /// The default is an unanchored search that starts at the beginning of the + /// haystack. + pub fn new() -> Config { + Config { anchored: Anchored::No, look_behind: None } + } + + /// A convenience routine for building a start configuration from an + /// [`Input`] for a forward search. + /// + /// This automatically sets the look-behind byte to the byte immediately + /// preceding the start of the search. If the start of the search is at + /// offset `0`, then no look-behind byte is set. + pub fn from_input_forward(input: &Input<'_>) -> Config { + let look_behind = input + .start() + .checked_sub(1) + .and_then(|i| input.haystack().get(i).copied()); + Config { look_behind, anchored: input.get_anchored() } + } + + /// A convenience routine for building a start configuration from an + /// [`Input`] for a reverse search. + /// + /// This automatically sets the look-behind byte to the byte immediately + /// following the end of the search. If the end of the search is at + /// offset `haystack.len()`, then no look-behind byte is set. + pub fn from_input_reverse(input: &Input<'_>) -> Config { + let look_behind = input.haystack().get(input.end()).copied(); + Config { look_behind, anchored: input.get_anchored() } + } + + /// Set the look-behind byte at the start of a search. + /// + /// Unless the search is intended to logically start at the beginning of a + /// haystack, this should _always_ be set to the byte immediately preceding + /// the start of the search. If no look-behind byte is set, then the start + /// configuration will assume it is at the beginning of the haystack. For + /// example, the anchor `^` will match. + /// + /// The default is that no look-behind byte is set. + pub fn look_behind(mut self, byte: Option<u8>) -> Config { + self.look_behind = byte; + self + } + + /// Set the anchored mode of a search. + /// + /// The default is an unanchored search. + pub fn anchored(mut self, mode: Anchored) -> Config { + self.anchored = mode; + self + } + + /// Return the look-behind byte in this configuration, if one exists. + pub fn get_look_behind(&self) -> Option<u8> { + self.look_behind + } + + /// Return the anchored mode in this configuration. + pub fn get_anchored(&self) -> Anchored { + self.anchored + } +} + +/// A map from every possible byte value to its corresponding starting +/// configuration. +/// +/// This map is used in order to lookup the start configuration for a particular +/// position in a haystack. This start configuration is then used in +/// combination with things like the anchored mode and pattern ID to fully +/// determine the start state. +/// +/// Generally speaking, this map is only used for fully compiled DFAs and lazy +/// DFAs. For NFAs (including the one-pass DFA), the start state is generally +/// selected by virtue of traversing the NFA state graph. DFAs do the same +/// thing, but at build time and not search time. (Well, technically the lazy +/// DFA does it at search time, but it does enough work to cache the full +/// result of the epsilon closure that the NFA engines tend to need to do.) +#[derive(Clone)] +pub(crate) struct StartByteMap { + map: [Start; 256], +} + +impl StartByteMap { + /// Create a new map from byte values to their corresponding starting + /// configurations. The map is determined, in part, by how look-around + /// assertions are matched via the matcher given. + pub(crate) fn new(lookm: &LookMatcher) -> StartByteMap { + let mut map = [Start::NonWordByte; 256]; + map[usize::from(b'\n')] = Start::LineLF; + map[usize::from(b'\r')] = Start::LineCR; + map[usize::from(b'_')] = Start::WordByte; + + let mut byte = b'0'; + while byte <= b'9' { + map[usize::from(byte)] = Start::WordByte; + byte += 1; + } + byte = b'A'; + while byte <= b'Z' { + map[usize::from(byte)] = Start::WordByte; + byte += 1; + } + byte = b'a'; + while byte <= b'z' { + map[usize::from(byte)] = Start::WordByte; + byte += 1; + } + + let lineterm = lookm.get_line_terminator(); + // If our line terminator is normal, then it is already handled by + // the LineLF and LineCR configurations. But if it's weird, then we + // overwrite whatever was there before for that terminator with a + // special configuration. The trick here is that if the terminator + // is, say, a word byte like `a`, then callers seeing this start + // configuration need to account for that and build their DFA state as + // if it *also* came from a word byte. + if lineterm != b'\r' && lineterm != b'\n' { + map[usize::from(lineterm)] = Start::CustomLineTerminator; + } + StartByteMap { map } + } + + /// Return the starting configuration for the given look-behind byte. + /// + /// If no look-behind exists, callers should use `Start::Text`. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn get(&self, byte: u8) -> Start { + self.map[usize::from(byte)] + } + + /// Deserializes a byte class map from the given slice. If the slice is of + /// insufficient length or otherwise contains an impossible mapping, then + /// an error is returned. Upon success, the number of bytes read along with + /// the map are returned. The number of bytes read is always a multiple of + /// 8. + pub(crate) fn from_bytes( + slice: &[u8], + ) -> Result<(StartByteMap, usize), DeserializeError> { + wire::check_slice_len(slice, 256, "start byte map")?; + let mut map = [Start::NonWordByte; 256]; + for (i, &repr) in slice[..256].iter().enumerate() { + map[i] = match Start::from_usize(usize::from(repr)) { + Some(start) => start, + None => { + return Err(DeserializeError::generic( + "found invalid starting configuration", + )) + } + }; + } + Ok((StartByteMap { map }, 256)) + } + + /// Writes this map to the given byte buffer. if the given buffer is too + /// small, then an error is returned. Upon success, the total number of + /// bytes written is returned. The number of bytes written is guaranteed to + /// be a multiple of 8. + pub(crate) fn write_to( + &self, + dst: &mut [u8], + ) -> Result<usize, SerializeError> { + let nwrite = self.write_to_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("start byte map")); + } + for (i, &start) in self.map.iter().enumerate() { + dst[i] = start.as_u8(); + } + Ok(nwrite) + } + + /// Returns the total number of bytes written by `write_to`. + pub(crate) fn write_to_len(&self) -> usize { + 256 + } +} + +impl core::fmt::Debug for StartByteMap { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + use crate::util::escape::DebugByte; + + write!(f, "StartByteMap{{")?; + for byte in 0..=255 { + if byte > 0 { + write!(f, ", ")?; + } + let start = self.map[usize::from(byte)]; + write!(f, "{:?} => {:?}", DebugByte(byte), start)?; + } + write!(f, "}}")?; + Ok(()) + } +} + +/// Represents the six possible starting configurations of a DFA search. +/// +/// The starting configuration is determined by inspecting the beginning +/// of the haystack (up to 1 byte). Ultimately, this along with a pattern ID +/// (if specified) and the type of search (anchored or not) is what selects the +/// start state to use in a DFA. +/// +/// As one example, if a DFA only supports unanchored searches and does not +/// support anchored searches for each pattern, then it will have at most 6 +/// distinct start states. (Some start states may be reused if determinization +/// can determine that they will be equivalent.) If the DFA supports both +/// anchored and unanchored searches, then it will have a maximum of 12 +/// distinct start states. Finally, if the DFA also supports anchored searches +/// for each pattern, then it can have up to `12 + (N * 6)` start states, where +/// `N` is the number of patterns. +/// +/// Handling each of these starting configurations in the context of DFA +/// determinization can be *quite* tricky and subtle. But the code is small +/// and can be found at `crate::util::determinize::set_lookbehind_from_start`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Start { + /// This occurs when the starting position is not any of the ones below. + NonWordByte = 0, + /// This occurs when the byte immediately preceding the start of the search + /// is an ASCII word byte. + WordByte = 1, + /// This occurs when the starting position of the search corresponds to the + /// beginning of the haystack. + Text = 2, + /// This occurs when the byte immediately preceding the start of the search + /// is a line terminator. Specifically, `\n`. + LineLF = 3, + /// This occurs when the byte immediately preceding the start of the search + /// is a line terminator. Specifically, `\r`. + LineCR = 4, + /// This occurs when a custom line terminator has been set via a + /// `LookMatcher`, and when that line terminator is neither a `\r` or a + /// `\n`. + /// + /// If the custom line terminator is a word byte, then this start + /// configuration is still selected. DFAs that implement word boundary + /// assertions will likely need to check whether the custom line terminator + /// is a word byte, in which case, it should behave as if the byte + /// satisfies `\b` in addition to multi-line anchors. + CustomLineTerminator = 5, +} + +impl Start { + /// Return the starting state corresponding to the given integer. If no + /// starting state exists for the given integer, then None is returned. + pub(crate) fn from_usize(n: usize) -> Option<Start> { + match n { + 0 => Some(Start::NonWordByte), + 1 => Some(Start::WordByte), + 2 => Some(Start::Text), + 3 => Some(Start::LineLF), + 4 => Some(Start::LineCR), + 5 => Some(Start::CustomLineTerminator), + _ => None, + } + } + + /// Returns the total number of starting state configurations. + pub(crate) fn len() -> usize { + 6 + } + + /// Return this starting configuration as `u8` integer. It is guaranteed to + /// be less than `Start::len()`. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn as_u8(&self) -> u8 { + // AFAIK, 'as' is the only way to zero-cost convert an int enum to an + // actual int. + *self as u8 + } + + /// Return this starting configuration as a `usize` integer. It is + /// guaranteed to be less than `Start::len()`. + #[cfg_attr(feature = "perf-inline", inline(always))] + pub(crate) fn as_usize(&self) -> usize { + usize::from(self.as_u8()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_fwd_done_range() { + let smap = StartByteMap::new(&LookMatcher::default()); + let input = Input::new("").range(1..0); + let config = Config::from_input_forward(&input); + let start = + config.get_look_behind().map_or(Start::Text, |b| smap.get(b)); + assert_eq!(Start::Text, start); + } + + #[test] + fn start_rev_done_range() { + let smap = StartByteMap::new(&LookMatcher::default()); + let input = Input::new("").range(1..0); + let config = Config::from_input_reverse(&input); + let start = + config.get_look_behind().map_or(Start::Text, |b| smap.get(b)); + assert_eq!(Start::Text, start); + } + + #[test] + fn start_fwd() { + let f = |haystack, start, end| { + let smap = StartByteMap::new(&LookMatcher::default()); + let input = Input::new(haystack).range(start..end); + let config = Config::from_input_forward(&input); + let start = + config.get_look_behind().map_or(Start::Text, |b| smap.get(b)); + start + }; + + assert_eq!(Start::Text, f("", 0, 0)); + assert_eq!(Start::Text, f("abc", 0, 3)); + assert_eq!(Start::Text, f("\nabc", 0, 3)); + + assert_eq!(Start::LineLF, f("\nabc", 1, 3)); + + assert_eq!(Start::LineCR, f("\rabc", 1, 3)); + + assert_eq!(Start::WordByte, f("abc", 1, 3)); + + assert_eq!(Start::NonWordByte, f(" abc", 1, 3)); + } + + #[test] + fn start_rev() { + let f = |haystack, start, end| { + let smap = StartByteMap::new(&LookMatcher::default()); + let input = Input::new(haystack).range(start..end); + let config = Config::from_input_reverse(&input); + let start = + config.get_look_behind().map_or(Start::Text, |b| smap.get(b)); + start + }; + + assert_eq!(Start::Text, f("", 0, 0)); + assert_eq!(Start::Text, f("abc", 0, 3)); + assert_eq!(Start::Text, f("abc\n", 0, 4)); + + assert_eq!(Start::LineLF, f("abc\nz", 0, 3)); + + assert_eq!(Start::LineCR, f("abc\rz", 0, 3)); + + assert_eq!(Start::WordByte, f("abc", 0, 2)); + + assert_eq!(Start::NonWordByte, f("abc ", 0, 3)); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/syntax.rs b/third_party/crates/regex-automata-0.4.18/src/util/syntax.rs new file mode 100644 index 0000000..3be07bc --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/syntax.rs @@ -0,0 +1,482 @@ +/*! +Utilities for dealing with the syntax of a regular expression. + +This module currently only exposes a [`Config`] type that +itself represents a wrapper around the configuration for a +[`regex-syntax::ParserBuilder`](regex_syntax::ParserBuilder). The purpose of +this wrapper is to make configuring syntax options very similar to how other +configuration is done throughout this crate. Namely, instead of duplicating +syntax options across every builder (of which there are many), we instead +create small config objects like this one that can be passed around and +composed. +*/ + +use alloc::{vec, vec::Vec}; + +use regex_syntax::{ + ast, + hir::{self, Hir}, + Error, ParserBuilder, +}; + +/// A convenience routine for parsing a pattern into an HIR value with the +/// default configuration. +/// +/// # Example +/// +/// This shows how to parse a pattern into an HIR value: +/// +/// ``` +/// use regex_automata::util::syntax; +/// +/// let hir = syntax::parse(r"([a-z]+)|([0-9]+)")?; +/// assert_eq!(Some(1), hir.properties().static_explicit_captures_len()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +pub fn parse(pattern: &str) -> Result<Hir, Error> { + parse_with(pattern, &Config::default()) +} + +/// A convenience routine for parsing many patterns into HIR value with the +/// default configuration. +/// +/// # Example +/// +/// This shows how to parse many patterns into an corresponding HIR values: +/// +/// ``` +/// use { +/// regex_automata::util::syntax, +/// regex_syntax::hir::Properties, +/// }; +/// +/// let hirs = syntax::parse_many(&[ +/// r"([a-z]+)|([0-9]+)", +/// r"foo(A-Z]+)bar", +/// ])?; +/// let props = Properties::union(hirs.iter().map(|h| h.properties())); +/// assert_eq!(Some(1), props.static_explicit_captures_len()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +pub fn parse_many<P: AsRef<str>>(patterns: &[P]) -> Result<Vec<Hir>, Error> { + parse_many_with(patterns, &Config::default()) +} + +/// A convenience routine for parsing a pattern into an HIR value using a +/// `Config`. +/// +/// # Example +/// +/// This shows how to parse a pattern into an HIR value with a non-default +/// configuration: +/// +/// ``` +/// use regex_automata::util::syntax; +/// +/// let hir = syntax::parse_with( +/// r"^[a-z]+$", +/// &syntax::Config::new().multi_line(true).crlf(true), +/// )?; +/// assert!(hir.properties().look_set().contains_anchor_crlf()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +pub fn parse_with(pattern: &str, config: &Config) -> Result<Hir, Error> { + let mut builder = ParserBuilder::new(); + config.apply(&mut builder); + builder.build().parse(pattern) +} + +/// A convenience routine for parsing many patterns into HIR values using a +/// `Config`. +/// +/// # Example +/// +/// This shows how to parse many patterns into an corresponding HIR values +/// with a non-default configuration: +/// +/// ``` +/// use { +/// regex_automata::util::syntax, +/// regex_syntax::hir::Properties, +/// }; +/// +/// let patterns = &[ +/// r"([a-z]+)|([0-9]+)", +/// r"\W", +/// r"foo(A-Z]+)bar", +/// ]; +/// let config = syntax::Config::new().unicode(false).utf8(false); +/// let hirs = syntax::parse_many_with(patterns, &config)?; +/// let props = Properties::union(hirs.iter().map(|h| h.properties())); +/// assert!(!props.is_utf8()); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +pub fn parse_many_with<P: AsRef<str>>( + patterns: &[P], + config: &Config, +) -> Result<Vec<Hir>, Error> { + let mut builder = ParserBuilder::new(); + config.apply(&mut builder); + let mut hirs = vec![]; + for p in patterns.iter() { + hirs.push(builder.build().parse(p.as_ref())?); + } + Ok(hirs) +} + +/// A common set of configuration options that apply to the syntax of a regex. +/// +/// This represents a group of configuration options that specifically apply +/// to how the concrete syntax of a regular expression is interpreted. In +/// particular, they are generally forwarded to the +/// [`ParserBuilder`](https://docs.rs/regex-syntax/*/regex_syntax/struct.ParserBuilder.html) +/// in the +/// [`regex-syntax`](https://docs.rs/regex-syntax) +/// crate when building a regex from its concrete syntax directly. +/// +/// These options are defined as a group since they apply to every regex engine +/// in this crate. Instead of re-defining them on every engine's builder, they +/// are instead provided here as one cohesive unit. +#[derive(Clone, Copy, Debug)] +pub struct Config { + case_insensitive: bool, + multi_line: bool, + dot_matches_new_line: bool, + crlf: bool, + line_terminator: u8, + swap_greed: bool, + ignore_whitespace: bool, + unicode: bool, + utf8: bool, + nest_limit: u32, + octal: bool, +} + +impl Config { + /// Return a new default syntax configuration. + pub fn new() -> Config { + // These defaults match the ones used in regex-syntax. + Config { + case_insensitive: false, + multi_line: false, + dot_matches_new_line: false, + crlf: false, + line_terminator: b'\n', + swap_greed: false, + ignore_whitespace: false, + unicode: true, + utf8: true, + nest_limit: 250, + octal: false, + } + } + + /// Enable or disable the case insensitive flag by default. + /// + /// When Unicode mode is enabled, case insensitivity is Unicode-aware. + /// Specifically, it will apply the "simple" case folding rules as + /// specified by Unicode. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `i` flag. + pub fn case_insensitive(mut self, yes: bool) -> Config { + self.case_insensitive = yes; + self + } + + /// Enable or disable the multi-line matching flag by default. + /// + /// When this is enabled, the `^` and `$` look-around assertions will + /// match immediately after and immediately before a new line character, + /// respectively. Note that the `\A` and `\z` look-around assertions are + /// unaffected by this setting and always correspond to matching at the + /// beginning and end of the input. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `m` flag. + pub fn multi_line(mut self, yes: bool) -> Config { + self.multi_line = yes; + self + } + + /// Enable or disable the "dot matches any character" flag by default. + /// + /// When this is enabled, `.` will match any character. When it's disabled, + /// then `.` will match any character except for a new line character. + /// + /// Note that `.` is impacted by whether the "unicode" setting is enabled + /// or not. When Unicode is enabled (the default), `.` will match any UTF-8 + /// encoding of any Unicode scalar value (sans a new line, depending on + /// whether this "dot matches new line" option is enabled). When Unicode + /// mode is disabled, `.` will match any byte instead. Because of this, + /// when Unicode mode is disabled, `.` can only be used when the "allow + /// invalid UTF-8" option is enabled, since `.` could otherwise match + /// invalid UTF-8. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `s` flag. + pub fn dot_matches_new_line(mut self, yes: bool) -> Config { + self.dot_matches_new_line = yes; + self + } + + /// Enable or disable the "CRLF mode" flag by default. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `R` flag. + /// + /// When CRLF mode is enabled, the following happens: + /// + /// * Unless `dot_matches_new_line` is enabled, `.` will match any character + /// except for `\r` and `\n`. + /// * When `multi_line` mode is enabled, `^` and `$` will treat `\r\n`, + /// `\r` and `\n` as line terminators. And in particular, neither will + /// match between a `\r` and a `\n`. + pub fn crlf(mut self, yes: bool) -> Config { + self.crlf = yes; + self + } + + /// Sets the line terminator for use with `(?u-s:.)` and `(?-us:.)`. + /// + /// Namely, instead of `.` (by default) matching everything except for `\n`, + /// this will cause `.` to match everything except for the byte given. + /// + /// If `.` is used in a context where Unicode mode is enabled and this byte + /// isn't ASCII, then an error will be returned. When Unicode mode is + /// disabled, then any byte is permitted, but will return an error if UTF-8 + /// mode is enabled and it is a non-ASCII byte. + /// + /// In short, any ASCII value for a line terminator is always okay. But a + /// non-ASCII byte might result in an error depending on whether Unicode + /// mode or UTF-8 mode are enabled. + /// + /// Note that if `R` mode is enabled then it always takes precedence and + /// the line terminator will be treated as `\r` and `\n` simultaneously. + /// + /// Note also that this *doesn't* impact the look-around assertions + /// `(?m:^)` and `(?m:$)`. That's usually controlled by additional + /// configuration in the regex engine itself. + pub fn line_terminator(mut self, byte: u8) -> Config { + self.line_terminator = byte; + self + } + + /// Enable or disable the "swap greed" flag by default. + /// + /// When this is enabled, `.*` (for example) will become ungreedy and `.*?` + /// will become greedy. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `U` flag. + pub fn swap_greed(mut self, yes: bool) -> Config { + self.swap_greed = yes; + self + } + + /// Enable verbose mode in the regular expression. + /// + /// When enabled, verbose mode permits insignificant whitespace in many + /// places in the regular expression, as well as comments. Comments are + /// started using `#` and continue until the end of the line. + /// + /// By default, this is disabled. It may be selectively enabled in the + /// regular expression by using the `x` flag regardless of this setting. + pub fn ignore_whitespace(mut self, yes: bool) -> Config { + self.ignore_whitespace = yes; + self + } + + /// Enable or disable the Unicode flag (`u`) by default. + /// + /// By default this is **enabled**. It may alternatively be selectively + /// disabled in the regular expression itself via the `u` flag. + /// + /// Note that unless "allow invalid UTF-8" is enabled (it's disabled by + /// default), a regular expression will fail to parse if Unicode mode is + /// disabled and a sub-expression could possibly match invalid UTF-8. + /// + /// **WARNING**: Unicode mode can greatly increase the size of the compiled + /// DFA, which can noticeably impact both memory usage and compilation + /// time. This is especially noticeable if your regex contains character + /// classes like `\w` that are impacted by whether Unicode is enabled or + /// not. If Unicode is not necessary, you are encouraged to disable it. + pub fn unicode(mut self, yes: bool) -> Config { + self.unicode = yes; + self + } + + /// When disabled, the builder will permit the construction of a regular + /// expression that may match invalid UTF-8. + /// + /// For example, when [`Config::unicode`] is disabled, then + /// expressions like `[^a]` may match invalid UTF-8 since they can match + /// any single byte that is not `a`. By default, these sub-expressions + /// are disallowed to avoid returning offsets that split a UTF-8 + /// encoded codepoint. However, in cases where matching at arbitrary + /// locations is desired, this option can be disabled to permit all such + /// sub-expressions. + /// + /// When enabled (the default), the builder is guaranteed to produce a + /// regex that will only ever match valid UTF-8 (otherwise, the builder + /// will return an error). + pub fn utf8(mut self, yes: bool) -> Config { + self.utf8 = yes; + self + } + + /// Set the nesting limit used for the regular expression parser. + /// + /// The nesting limit controls how deep the abstract syntax tree is allowed + /// to be. If the AST exceeds the given limit (e.g., with too many nested + /// groups), then an error is returned by the parser. + /// + /// The purpose of this limit is to act as a heuristic to prevent stack + /// overflow when building a finite automaton from a regular expression's + /// abstract syntax tree. In particular, construction currently uses + /// recursion. In the future, the implementation may stop using recursion + /// and this option will no longer be necessary. + /// + /// This limit is not checked until the entire AST is parsed. Therefore, + /// if callers want to put a limit on the amount of heap space used, then + /// they should impose a limit on the length, in bytes, of the concrete + /// pattern string. In particular, this is viable since the parser will + /// limit itself to heap space proportional to the length of the pattern + /// string. + /// + /// Note that a nest limit of `0` will return a nest limit error for most + /// patterns but not all. For example, a nest limit of `0` permits `a` but + /// not `ab`, since `ab` requires a concatenation AST item, which results + /// in a nest depth of `1`. In general, a nest limit is not something that + /// manifests in an obvious way in the concrete syntax, therefore, it + /// should not be used in a granular way. + pub fn nest_limit(mut self, limit: u32) -> Config { + self.nest_limit = limit; + self + } + + /// Whether to support octal syntax or not. + /// + /// Octal syntax is a little-known way of uttering Unicode codepoints in + /// a regular expression. For example, `a`, `\x61`, `\u0061` and + /// `\141` are all equivalent regular expressions, where the last example + /// shows octal syntax. + /// + /// While supporting octal syntax isn't in and of itself a problem, it does + /// make good error messages harder. That is, in PCRE based regex engines, + /// syntax like `\1` invokes a backreference, which is explicitly + /// unsupported in Rust's regex engine. However, many users expect it to + /// be supported. Therefore, when octal support is disabled, the error + /// message will explicitly mention that backreferences aren't supported. + /// + /// Octal syntax is disabled by default. + pub fn octal(mut self, yes: bool) -> Config { + self.octal = yes; + self + } + + /// Returns whether "unicode" mode is enabled. + pub fn get_unicode(&self) -> bool { + self.unicode + } + + /// Returns whether "case insensitive" mode is enabled. + pub fn get_case_insensitive(&self) -> bool { + self.case_insensitive + } + + /// Returns whether "multi line" mode is enabled. + pub fn get_multi_line(&self) -> bool { + self.multi_line + } + + /// Returns whether "dot matches new line" mode is enabled. + pub fn get_dot_matches_new_line(&self) -> bool { + self.dot_matches_new_line + } + + /// Returns whether "CRLF" mode is enabled. + pub fn get_crlf(&self) -> bool { + self.crlf + } + + /// Returns the line terminator in this syntax configuration. + pub fn get_line_terminator(&self) -> u8 { + self.line_terminator + } + + /// Returns whether "swap greed" mode is enabled. + pub fn get_swap_greed(&self) -> bool { + self.swap_greed + } + + /// Returns whether "ignore whitespace" mode is enabled. + pub fn get_ignore_whitespace(&self) -> bool { + self.ignore_whitespace + } + + /// Returns whether UTF-8 mode is enabled. + pub fn get_utf8(&self) -> bool { + self.utf8 + } + + /// Returns the "nest limit" setting. + pub fn get_nest_limit(&self) -> u32 { + self.nest_limit + } + + /// Returns whether "octal" mode is enabled. + pub fn get_octal(&self) -> bool { + self.octal + } + + /// Applies this configuration to the given parser. + pub(crate) fn apply(&self, builder: &mut ParserBuilder) { + builder + .unicode(self.unicode) + .case_insensitive(self.case_insensitive) + .multi_line(self.multi_line) + .dot_matches_new_line(self.dot_matches_new_line) + .crlf(self.crlf) + .line_terminator(self.line_terminator) + .swap_greed(self.swap_greed) + .ignore_whitespace(self.ignore_whitespace) + .utf8(self.utf8) + .nest_limit(self.nest_limit) + .octal(self.octal); + } + + /// Applies this configuration to the given AST parser. + pub(crate) fn apply_ast(&self, builder: &mut ast::parse::ParserBuilder) { + builder + .ignore_whitespace(self.ignore_whitespace) + .nest_limit(self.nest_limit) + .octal(self.octal); + } + + /// Applies this configuration to the given AST-to-HIR translator. + pub(crate) fn apply_hir( + &self, + builder: &mut hir::translate::TranslatorBuilder, + ) { + builder + .unicode(self.unicode) + .case_insensitive(self.case_insensitive) + .multi_line(self.multi_line) + .crlf(self.crlf) + .dot_matches_new_line(self.dot_matches_new_line) + .line_terminator(self.line_terminator) + .swap_greed(self.swap_greed) + .utf8(self.utf8); + } +} + +impl Default for Config { + fn default() -> Config { + Config::new() + } +} diff --git a/third_party/crates/regex-automata-0.4.18/src/util/unicode_data/mod.rs b/third_party/crates/regex-automata-0.4.18/src/util/unicode_data/mod.rs new file mode 100644 index 0000000..fc7b1c7 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/unicode_data/mod.rs @@ -0,0 +1,17 @@ +// This cfg should match the one in src/util/look.rs that uses perl_word. +#[cfg(all( + // We have to explicitly want to support Unicode word boundaries. + feature = "unicode-word-boundary", + not(all( + // If we don't have regex-syntax at all, then we definitely need to + // bring our own \w data table. + feature = "syntax", + // If unicode-perl is enabled, then regex-syntax/unicode-perl is + // also enabled, which in turn means we can use regex-syntax's + // is_word_character routine (and thus use its data tables). But if + // unicode-perl is not enabled, even if syntax is, then we need to + // bring our own. + feature = "unicode-perl", + )), +))] +pub(crate) mod perl_word; diff --git a/third_party/crates/regex-automata-0.4.18/src/util/unicode_data/perl_word.rs b/third_party/crates/regex-automata-0.4.18/src/util/unicode_data/perl_word.rs new file mode 100644 index 0000000..21c8c0f --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/unicode_data/perl_word.rs @@ -0,0 +1,806 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate perl-word ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const PERL_WORD: &'static [(char, char)] = &[ + ('0', '9'), + ('A', 'Z'), + ('_', '_'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('\u{300}', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('\u{483}', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', 'ՙ'), + ('ՠ', 'ֈ'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('\u{610}', '\u{61a}'), + ('ؠ', '٩'), + ('ٮ', 'ۓ'), + ('ە', '\u{6dc}'), + ('\u{6df}', '\u{6e8}'), + ('\u{6ea}', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', '\u{74a}'), + ('ݍ', 'ޱ'), + ('߀', 'ߵ'), + ('ߺ', 'ߺ'), + ('\u{7fd}', '\u{7fd}'), + ('ࠀ', '\u{82d}'), + ('ࡀ', '\u{85b}'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('\u{897}', '\u{8e1}'), + ('\u{8e3}', '\u{963}'), + ('०', '९'), + ('ॱ', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('\u{9bc}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', 'ৎ'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', '\u{9e3}'), + ('০', 'ৱ'), + ('ৼ', 'ৼ'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', 'ਃ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('੦', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('\u{abc}', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('ૐ', 'ૐ'), + ('ૠ', '\u{ae3}'), + ('૦', '૯'), + ('ૹ', '\u{aff}'), + ('\u{b01}', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('\u{b3c}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', '\u{b63}'), + ('୦', '୯'), + ('ୱ', 'ୱ'), + ('\u{b82}', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('ௐ', 'ௐ'), + ('\u{bd7}', '\u{bd7}'), + ('௦', '௯'), + ('\u{c00}', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('\u{c3c}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', '\u{c63}'), + ('౦', '౯'), + ('ಀ', 'ಃ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('\u{cbc}', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('ೝ', 'ೞ'), + ('ೠ', '\u{ce3}'), + ('೦', '೯'), + ('ೱ', 'ೳ'), + ('\u{d00}', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', 'ൎ'), + ('ൔ', '\u{d57}'), + ('ൟ', '\u{d63}'), + ('൦', '൯'), + ('ൺ', 'ൿ'), + ('\u{d81}', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('෦', '෯'), + ('ෲ', 'ෳ'), + ('ก', '\u{e3a}'), + ('เ', '\u{e4e}'), + ('๐', '๙'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('\u{ec8}', '\u{ece}'), + ('໐', '໙'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('\u{f18}', '\u{f19}'), + ('༠', '༩'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('༾', 'ཇ'), + ('ཉ', 'ཬ'), + ('\u{f71}', '\u{f84}'), + ('\u{f86}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('က', '၉'), + ('ၐ', '\u{109d}'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('\u{135d}', '\u{135f}'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', '\u{1715}'), + ('ᜟ', '\u{1734}'), + ('ᝀ', '\u{1753}'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('\u{1772}', '\u{1773}'), + ('ក', '\u{17d3}'), + ('ៗ', 'ៗ'), + ('ៜ', '\u{17dd}'), + ('០', '៩'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '᠙'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('᥆', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('᧐', '᧙'), + ('ᨀ', '\u{1a1b}'), + ('ᨠ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '᪉'), + ('᪐', '᪙'), + ('ᪧ', 'ᪧ'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', 'ᭌ'), + ('᭐', '᭙'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1bf3}'), + ('ᰀ', '\u{1c37}'), + ('᱀', '᱉'), + ('ᱍ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', 'ᳺ'), + ('ᴀ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('\u{200c}', '\u{200d}'), + ('‿', '⁀'), + ('⁔', '⁔'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('\u{20d0}', '\u{20f0}'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℯ', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⓐ', 'ⓩ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('\u{2d7f}', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('\u{2de0}', '\u{2dff}'), + ('ⸯ', 'ⸯ'), + ('々', '〇'), + ('〡', '\u{302f}'), + ('〱', '〵'), + ('〸', '〼'), + ('ぁ', 'ゖ'), + ('\u{3099}', '\u{309a}'), + ('ゝ', 'ゟ'), + ('ァ', 'ヺ'), + ('ー', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘫ'), + ('Ꙁ', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('ꙿ', '\u{a6f1}'), + ('ꜗ', 'ꜟ'), + ('Ꜣ', 'ꞈ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠧ'), + ('\u{a82c}', '\u{a82c}'), + ('ꡀ', 'ꡳ'), + ('ꢀ', '\u{a8c5}'), + ('꣐', '꣙'), + ('\u{a8e0}', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', '\u{a92d}'), + ('ꤰ', '\u{a953}'), + ('ꥠ', 'ꥼ'), + ('\u{a980}', '\u{a9c0}'), + ('ꧏ', '꧙'), + ('ꧠ', 'ꧾ'), + ('ꨀ', '\u{aa36}'), + ('ꩀ', 'ꩍ'), + ('꩐', '꩙'), + ('ꩠ', 'ꩶ'), + ('ꩺ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫯ'), + ('ꫲ', '\u{aaf6}'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꯪ'), + ('꯬', '\u{abed}'), + ('꯰', '꯹'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('︳', '︴'), + ('﹍', '﹏'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('０', '９'), + ('Ａ', 'Ｚ'), + ('＿', '＿'), + ('ａ', 'ｚ'), + ('ｦ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('\u{101fd}', '\u{101fd}'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('\u{102e0}', '\u{102e0}'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '\u{1037a}'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐐀', '𐒝'), + ('𐒠', '𐒩'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '\u{10ae6}'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '\u{10d27}'), + ('𐴰', '𐴹'), + ('𐵀', '𐵥'), + ('\u{10d69}', '\u{10d6d}'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('\u{10eab}', '\u{10eac}'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('\u{10efc}', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '\u{10f50}'), + ('𐽰', '\u{10f85}'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀀', '\u{11046}'), + ('𑁦', '𑁵'), + ('\u{1107f}', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('𑃐', '𑃨'), + ('𑃰', '𑃹'), + ('\u{11100}', '\u{11134}'), + ('𑄶', '𑄿'), + ('𑅄', '𑅇'), + ('𑅐', '\u{11173}'), + ('𑅶', '𑅶'), + ('\u{11180}', '𑇄'), + ('\u{111c9}', '\u{111cc}'), + ('𑇎', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '\u{11237}'), + ('\u{1123e}', '\u{11241}'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '\u{112ea}'), + ('𑋰', '𑋹'), + ('\u{11300}', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('\u{1133b}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('𑍐', '𑍐'), + ('\u{11357}', '\u{11357}'), + ('𑍝', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏓'), + ('\u{113e1}', '\u{113e2}'), + ('𑐀', '𑑊'), + ('𑑐', '𑑙'), + ('\u{1145e}', '𑑡'), + ('𑒀', '𑓅'), + ('𑓇', '𑓇'), + ('𑓐', '𑓙'), + ('𑖀', '\u{115b5}'), + ('𑖸', '\u{115c0}'), + ('𑗘', '\u{115dd}'), + ('𑘀', '\u{11640}'), + ('𑙄', '𑙄'), + ('𑙐', '𑙙'), + ('𑚀', '𑚸'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜀', '𑜚'), + ('\u{1171d}', '\u{1172b}'), + ('𑜰', '𑜹'), + ('𑝀', '𑝆'), + ('𑠀', '\u{1183a}'), + ('𑢠', '𑣩'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{11943}'), + ('𑥐', '𑥙'), + ('𑦠', '𑦧'), + ('𑦪', '\u{119d7}'), + ('\u{119da}', '𑧡'), + ('𑧣', '𑧤'), + ('𑨀', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('𑩐', '\u{11a99}'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑯰', '𑯹'), + ('𑰀', '𑰈'), + ('𑰊', '\u{11c36}'), + ('\u{11c38}', '𑱀'), + ('𑱐', '𑱙'), + ('𑱲', '𑲏'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d47}'), + ('𑵐', '𑵙'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶘'), + ('𑶠', '𑶩'), + ('𑻠', '𑻶'), + ('\u{11f00}', '𑼐'), + ('𑼒', '\u{11f3a}'), + ('𑼾', '\u{11f42}'), + ('𑽐', '\u{11f5a}'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('\u{13440}', '\u{13455}'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄹'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩠', '𖩩'), + ('𖩰', '𖪾'), + ('𖫀', '𖫉'), + ('𖫐', '𖫭'), + ('\u{16af0}', '\u{16af4}'), + ('𖬀', '\u{16b36}'), + ('𖭀', '𖭃'), + ('𖭐', '𖭙'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖵰', '𖵹'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('\u{16f4f}', '𖾇'), + ('\u{16f8f}', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('𜳰', '𜳹'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝟎', '𝟿'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), + ('𞄀', '𞄬'), + ('\u{1e130}', '𞄽'), + ('𞅀', '𞅉'), + ('𞅎', '𞅎'), + ('𞊐', '\u{1e2ae}'), + ('𞋀', '𞋹'), + ('𞓐', '𞓹'), + ('𞗐', '𞗺'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('𞤀', '𞥋'), + ('𞥐', '𞥙'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('🄰', '🅉'), + ('🅐', '🅩'), + ('🅰', '🆉'), + ('🯰', '🯹'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), + ('\u{e0100}', '\u{e01ef}'), +]; diff --git a/third_party/crates/regex-automata-0.4.18/src/util/utf8.rs b/third_party/crates/regex-automata-0.4.18/src/util/utf8.rs new file mode 100644 index 0000000..6c86e8d --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/utf8.rs @@ -0,0 +1,191 @@ +/*! +Utilities for dealing with UTF-8. + +This module provides some UTF-8 related helper routines, including an +incremental decoder. +*/ + +/// Returns true if and only if the given byte is considered a word character. +/// This only applies to ASCII. +/// +/// This was copied from regex-syntax so that we can use it to determine the +/// starting DFA state while searching without depending on regex-syntax. The +/// definition is never going to change, so there's no maintenance/bit-rot +/// hazard here. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn is_word_byte(b: u8) -> bool { + const fn mkwordset() -> [bool; 256] { + // FIXME: Use as_usize() once const functions in traits are stable. + let mut set = [false; 256]; + set[b'_' as usize] = true; + + let mut byte = b'0'; + while byte <= b'9' { + set[byte as usize] = true; + byte += 1; + } + byte = b'A'; + while byte <= b'Z' { + set[byte as usize] = true; + byte += 1; + } + byte = b'a'; + while byte <= b'z' { + set[byte as usize] = true; + byte += 1; + } + set + } + const WORD: [bool; 256] = mkwordset(); + WORD[b as usize] +} + +/// Decodes the next UTF-8 encoded codepoint from the given byte slice. +/// +/// If no valid encoding of a codepoint exists at the beginning of the given +/// byte slice, then the first byte is returned instead. +/// +/// This returns `None` if and only if `bytes` is empty. +/// +/// This never panics. +/// +/// *WARNING*: This is not designed for performance. If you're looking for a +/// fast UTF-8 decoder, this is not it. If you feel like you need one in this +/// crate, then please file an issue and discuss your use case. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn decode(bytes: &[u8]) -> Option<Result<char, u8>> { + if bytes.is_empty() { + return None; + } + let len = match len(bytes[0]) { + None => return Some(Err(bytes[0])), + Some(len) if len > bytes.len() => return Some(Err(bytes[0])), + Some(1) => return Some(Ok(char::from(bytes[0]))), + Some(len) => len, + }; + match core::str::from_utf8(&bytes[..len]) { + Ok(s) => Some(Ok(s.chars().next().unwrap())), + Err(_) => Some(Err(bytes[0])), + } +} + +/// Decodes the last UTF-8 encoded codepoint from the given byte slice. +/// +/// If no valid encoding of a codepoint exists at the end of the given byte +/// slice, then the last byte is returned instead. +/// +/// This returns `None` if and only if `bytes` is empty. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn decode_last(bytes: &[u8]) -> Option<Result<char, u8>> { + if bytes.is_empty() { + return None; + } + let mut start = bytes.len() - 1; + let limit = bytes.len().saturating_sub(4); + while start > limit && !is_leading_or_invalid_byte(bytes[start]) { + start -= 1; + } + match decode(&bytes[start..]) { + None => None, + Some(Ok(ch)) => Some(Ok(ch)), + Some(Err(_)) => Some(Err(bytes[bytes.len() - 1])), + } +} + +/// Given a UTF-8 leading byte, this returns the total number of code units +/// in the following encoded codepoint. +/// +/// If the given byte is not a valid UTF-8 leading byte, then this returns +/// `None`. +#[cfg_attr(feature = "perf-inline", inline(always))] +fn len(byte: u8) -> Option<usize> { + match byte { + 0b0000_0000..=0b0111_1111 => Some(1), + 0b1000_0000..=0b1011_1111 => None, + 0b1100_0000..=0b1101_1111 => Some(2), + 0b1110_0000..=0b1110_1111 => Some(3), + 0b1111_0000..=0b1111_0111 => Some(4), + _ => None, + } +} + +/// Returns true if and only if the given offset in the given bytes falls on a +/// valid UTF-8 encoded codepoint boundary. +/// +/// If `bytes` is not valid UTF-8, then the behavior of this routine is +/// unspecified. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn is_boundary(bytes: &[u8], i: usize) -> bool { + match bytes.get(i) { + // The position at the end of the bytes always represents an empty + // string, which is a valid boundary. But anything after that doesn't + // make much sense to call valid a boundary. + None => i == bytes.len(), + // Other than ASCII (where the most significant bit is never set), + // valid starting bytes always have their most significant two bits + // set, where as continuation bytes never have their second most + // significant bit set. Therefore, this only returns true when bytes[i] + // corresponds to a byte that begins a valid UTF-8 encoding of a + // Unicode scalar value. + Some(&b) => b <= 0b0111_1111 || b >= 0b1100_0000, + } +} + +/// Returns true if and only if the given byte is either a valid leading UTF-8 +/// byte, or is otherwise an invalid byte that can never appear anywhere in a +/// valid UTF-8 sequence. +#[cfg_attr(feature = "perf-inline", inline(always))] +fn is_leading_or_invalid_byte(b: u8) -> bool { + // In the ASCII case, the most significant bit is never set. The leading + // byte of a 2/3/4-byte sequence always has the top two most significant + // bits set. For bytes that can never appear anywhere in valid UTF-8, this + // also returns true, since every such byte has its two most significant + // bits set: + // + // \xC0 :: 11000000 + // \xC1 :: 11000001 + // \xF5 :: 11110101 + // \xF6 :: 11110110 + // \xF7 :: 11110111 + // \xF8 :: 11111000 + // \xF9 :: 11111001 + // \xFA :: 11111010 + // \xFB :: 11111011 + // \xFC :: 11111100 + // \xFD :: 11111101 + // \xFE :: 11111110 + // \xFF :: 11111111 + (b & 0b1100_0000) != 0b1000_0000 +} + +/* +/// Returns the smallest possible index of the next valid UTF-8 sequence +/// starting after `i`. +/// +/// For all inputs, including invalid UTF-8 and any value of `i`, the return +/// value is guaranteed to be greater than `i`. (If there is no value greater +/// than `i` that fits in `usize`, then this panics.) +/// +/// Generally speaking, this should only be called on `text` when it is +/// permitted to assume that it is valid UTF-8 and where either `i >= +/// text.len()` or where `text[i]` is a leading byte of a UTF-8 sequence. +/// +/// NOTE: This method was used in a previous conception of iterators where we +/// specifically tried to skip over empty matches that split a codepoint by +/// simply requiring that our next search begin at the beginning of codepoint. +/// But we ended up changing that technique to always advance by 1 byte and +/// then filter out matches that split a codepoint after-the-fact. Thus, we no +/// longer use this method. But I've kept it around in case we want to switch +/// back to this approach. Its guarantees are a little subtle, so I'd prefer +/// not to rebuild it from whole cloth. +pub(crate) fn next(text: &[u8], i: usize) -> usize { + let b = match text.get(i) { + None => return i.checked_add(1).unwrap(), + Some(&b) => b, + }; + // For cases where we see an invalid UTF-8 byte, there isn't much we can do + // other than just start at the next byte. + let inc = len(b).unwrap_or(1); + i.checked_add(inc).unwrap() +} +*/ diff --git a/third_party/crates/regex-automata-0.4.18/src/util/wire.rs b/third_party/crates/regex-automata-0.4.18/src/util/wire.rs new file mode 100644 index 0000000..210ab6f --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/src/util/wire.rs @@ -0,0 +1,947 @@ +/*! +Types and routines that support the wire format of finite automata. + +Currently, this module just exports a few error types and some small helpers +for deserializing [dense DFAs](crate::dfa::dense::DFA) using correct alignment. +*/ + +/* +A collection of helper functions, types and traits for serializing automata. + +This crate defines its own bespoke serialization mechanism for some structures +provided in the public API, namely, DFAs. A bespoke mechanism was developed +primarily because structures like automata demand a specific binary format. +Attempting to encode their rich structure in an existing serialization +format is just not feasible. Moreover, the format for each structure is +generally designed such that deserialization is cheap. More specifically, that +deserialization can be done in constant time. (The idea being that you can +embed it into your binary or mmap it, and then use it immediately.) + +In order to achieve this, the dense and sparse DFAs in this crate use an +in-memory representation that very closely corresponds to its binary serialized +form. This pervades and complicates everything, and in some cases, requires +dealing with alignment and reasoning about safety. + +This technique does have major advantages. In particular, it permits doing +the potentially costly work of compiling a finite state machine in an offline +manner, and then loading it at runtime not only without having to re-compile +the regex, but even without the code required to do the compilation. This, for +example, permits one to use a pre-compiled DFA not only in environments without +Rust's standard library, but also in environments without a heap. + +In the code below, whenever we insert some kind of padding, it's to enforce a +4-byte alignment, unless otherwise noted. Namely, u32 is the only state ID type +supported. (In a previous version of this library, DFAs were generic over the +state ID representation.) + +Also, serialization generally requires the caller to specify endianness, +where as deserialization always assumes native endianness (otherwise cheap +deserialization would be impossible). This implies that serializing a structure +generally requires serializing both its big-endian and little-endian variants, +and then loading the correct one based on the target's endianness. +*/ + +use core::{cmp, mem::size_of}; + +#[cfg(feature = "alloc")] +use alloc::{vec, vec::Vec}; + +use crate::util::{ + int::Pointer, + primitives::{PatternID, PatternIDError, StateID, StateIDError}, +}; + +/// A hack to align a smaller type `B` with a bigger type `T`. +/// +/// The usual use of this is with `B = [u8]` and `T = u32`. That is, +/// it permits aligning a sequence of bytes on a 4-byte boundary. This +/// is useful in contexts where one wants to embed a serialized [dense +/// DFA](crate::dfa::dense::DFA) into a Rust a program while guaranteeing the +/// alignment required for the DFA. +/// +/// See [`dense::DFA::from_bytes`](crate::dfa::dense::DFA::from_bytes) for an +/// example of how to use this type. +#[repr(C)] +#[derive(Debug)] +pub struct AlignAs<B: ?Sized, T> { + /// A zero-sized field indicating the alignment we want. + pub _align: [T; 0], + /// A possibly non-sized field containing a sequence of bytes. + pub bytes: B, +} + +/// An error that occurs when serializing an object from this crate. +/// +/// Serialization, as used in this crate, universally refers to the process +/// of transforming a structure (like a DFA) into a custom binary format +/// represented by `&[u8]`. To this end, serialization is generally infallible. +/// However, it can fail when caller provided buffer sizes are too small. When +/// that occurs, a serialization error is reported. +/// +/// A `SerializeError` provides no introspection capabilities. Its only +/// supported operation is conversion to a human readable error message. +/// +/// This error type implements the `std::error::Error` trait only when the +/// `std` feature is enabled. Otherwise, this type is defined in all +/// configurations. +#[derive(Debug)] +pub struct SerializeError { + /// The name of the thing that a buffer is too small for. + /// + /// Currently, the only kind of serialization error is one that is + /// committed by a caller: providing a destination buffer that is too + /// small to fit the serialized object. This makes sense conceptually, + /// since every valid inhabitant of a type should be serializable. + /// + /// This is somewhat exposed in the public API of this crate. For example, + /// the `to_bytes_{big,little}_endian` APIs return a `Vec<u8>` and are + /// guaranteed to never panic or error. This is only possible because the + /// implementation guarantees that it will allocate a `Vec<u8>` that is + /// big enough. + /// + /// In summary, if a new serialization error kind needs to be added, then + /// it will need careful consideration. + what: &'static str, +} + +impl SerializeError { + pub(crate) fn buffer_too_small(what: &'static str) -> SerializeError { + SerializeError { what } + } +} + +impl core::fmt::Display for SerializeError { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "destination buffer is too small to write {}", self.what) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for SerializeError {} + +/// An error that occurs when deserializing an object defined in this crate. +/// +/// Serialization, as used in this crate, universally refers to the process +/// of transforming a structure (like a DFA) into a custom binary format +/// represented by `&[u8]`. Deserialization, then, refers to the process of +/// cheaply converting this binary format back to the object's in-memory +/// representation as defined in this crate. To the extent possible, +/// deserialization will report this error whenever this process fails. +/// +/// A `DeserializeError` provides no introspection capabilities. Its only +/// supported operation is conversion to a human readable error message. +/// +/// This error type implements the `std::error::Error` trait only when the +/// `std` feature is enabled. Otherwise, this type is defined in all +/// configurations. +#[derive(Debug)] +pub struct DeserializeError(DeserializeErrorKind); + +#[derive(Debug)] +enum DeserializeErrorKind { + Generic { msg: &'static str }, + BufferTooSmall { what: &'static str }, + InvalidUsize { what: &'static str }, + VersionMismatch { expected: u32, found: u32 }, + EndianMismatch { expected: u32, found: u32 }, + AlignmentMismatch { alignment: usize, address: usize }, + LabelMismatch { expected: &'static str }, + ArithmeticOverflow { what: &'static str }, + PatternID { err: PatternIDError, what: &'static str }, + StateID { err: StateIDError, what: &'static str }, +} + +impl DeserializeError { + pub(crate) fn generic(msg: &'static str) -> DeserializeError { + DeserializeError(DeserializeErrorKind::Generic { msg }) + } + + pub(crate) fn buffer_too_small(what: &'static str) -> DeserializeError { + DeserializeError(DeserializeErrorKind::BufferTooSmall { what }) + } + + fn invalid_usize(what: &'static str) -> DeserializeError { + DeserializeError(DeserializeErrorKind::InvalidUsize { what }) + } + + fn version_mismatch(expected: u32, found: u32) -> DeserializeError { + DeserializeError(DeserializeErrorKind::VersionMismatch { + expected, + found, + }) + } + + fn endian_mismatch(expected: u32, found: u32) -> DeserializeError { + DeserializeError(DeserializeErrorKind::EndianMismatch { + expected, + found, + }) + } + + fn alignment_mismatch( + alignment: usize, + address: usize, + ) -> DeserializeError { + DeserializeError(DeserializeErrorKind::AlignmentMismatch { + alignment, + address, + }) + } + + fn label_mismatch(expected: &'static str) -> DeserializeError { + DeserializeError(DeserializeErrorKind::LabelMismatch { expected }) + } + + fn arithmetic_overflow(what: &'static str) -> DeserializeError { + DeserializeError(DeserializeErrorKind::ArithmeticOverflow { what }) + } + + fn pattern_id_error( + err: PatternIDError, + what: &'static str, + ) -> DeserializeError { + DeserializeError(DeserializeErrorKind::PatternID { err, what }) + } + + pub(crate) fn state_id_error( + err: StateIDError, + what: &'static str, + ) -> DeserializeError { + DeserializeError(DeserializeErrorKind::StateID { err, what }) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for DeserializeError {} + +impl core::fmt::Display for DeserializeError { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + use self::DeserializeErrorKind::*; + + match self.0 { + Generic { msg } => write!(f, "{msg}"), + BufferTooSmall { what } => { + write!(f, "buffer is too small to read {what}") + } + InvalidUsize { what } => { + write!(f, "{what} is too big to fit in a usize") + } + VersionMismatch { expected, found } => write!( + f, + "unsupported version: \ + expected version {expected} but found version {found}", + ), + EndianMismatch { expected, found } => write!( + f, + "endianness mismatch: expected 0x{expected:X} but \ + got 0x{found:X}. (Are you trying to load an object \ + serialized with a different endianness?)", + ), + AlignmentMismatch { alignment, address } => write!( + f, + "alignment mismatch: slice starts at address 0x{address:X}, \ + which is not aligned to a {alignment} byte boundary", + ), + LabelMismatch { expected } => write!( + f, + "label mismatch: start of serialized object should \ + contain a NUL terminated {expected:?} label, but a different \ + label was found", + ), + ArithmeticOverflow { what } => { + write!(f, "arithmetic overflow for {what}") + } + PatternID { ref err, what } => { + write!(f, "failed to read pattern ID for {what}: {err}") + } + StateID { ref err, what } => { + write!(f, "failed to read state ID for {what}: {err}") + } + } + } +} + +/// Safely converts a `&[u32]` to `&[StateID]` with zero cost. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn u32s_to_state_ids(slice: &[u32]) -> &[StateID] { + // SAFETY: This is safe because StateID is defined to have the same memory + // representation as a u32 (it is repr(transparent)). While not every u32 + // is a "valid" StateID, callers are not permitted to rely on the validity + // of StateIDs for memory safety. It can only lead to logical errors. (This + // is why StateID::new_unchecked is safe.) + unsafe { + core::slice::from_raw_parts( + slice.as_ptr().cast::<StateID>(), + slice.len(), + ) + } +} + +/// Safely converts a `&mut [u32]` to `&mut [StateID]` with zero cost. +pub(crate) fn u32s_to_state_ids_mut(slice: &mut [u32]) -> &mut [StateID] { + // SAFETY: This is safe because StateID is defined to have the same memory + // representation as a u32 (it is repr(transparent)). While not every u32 + // is a "valid" StateID, callers are not permitted to rely on the validity + // of StateIDs for memory safety. It can only lead to logical errors. (This + // is why StateID::new_unchecked is safe.) + unsafe { + core::slice::from_raw_parts_mut( + slice.as_mut_ptr().cast::<StateID>(), + slice.len(), + ) + } +} + +/// Safely converts a `&[u32]` to `&[PatternID]` with zero cost. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn u32s_to_pattern_ids(slice: &[u32]) -> &[PatternID] { + // SAFETY: This is safe because PatternID is defined to have the same + // memory representation as a u32 (it is repr(transparent)). While not + // every u32 is a "valid" PatternID, callers are not permitted to rely + // on the validity of PatternIDs for memory safety. It can only lead to + // logical errors. (This is why PatternID::new_unchecked is safe.) + unsafe { + core::slice::from_raw_parts( + slice.as_ptr().cast::<PatternID>(), + slice.len(), + ) + } +} + +/// Checks that the given slice has an alignment that matches `T`. +/// +/// This is useful for checking that a slice has an appropriate alignment +/// before casting it to a &[T]. Note though that alignment is not itself +/// sufficient to perform the cast for any `T`. +pub(crate) fn check_alignment<T>( + slice: &[u8], +) -> Result<(), DeserializeError> { + let alignment = core::mem::align_of::<T>(); + let address = slice.as_ptr().as_usize(); + if address % alignment == 0 { + return Ok(()); + } + Err(DeserializeError::alignment_mismatch(alignment, address)) +} + +/// Reads a possibly empty amount of padding, up to 7 bytes, from the beginning +/// of the given slice. All padding bytes must be NUL bytes. +/// +/// This is useful because it can be theoretically necessary to pad the +/// beginning of a serialized object with NUL bytes to ensure that it starts +/// at a correctly aligned address. These padding bytes should come immediately +/// before the label. +/// +/// This returns the number of bytes read from the given slice. +pub(crate) fn skip_initial_padding(slice: &[u8]) -> usize { + let mut nread = 0; + while nread < 7 && nread < slice.len() && slice[nread] == 0 { + nread += 1; + } + nread +} + +/// Allocate a byte buffer of the given size, along with some initial padding +/// such that `buf[padding..]` has the same alignment as `T`, where the +/// alignment of `T` must be at most `8`. In particular, callers should treat +/// the first N bytes (second return value) as padding bytes that must not be +/// overwritten. In all cases, the following identity holds: +/// +/// ```ignore +/// let (buf, padding) = alloc_aligned_buffer::<StateID>(SIZE); +/// assert_eq!(SIZE, buf[padding..].len()); +/// ``` +/// +/// In practice, padding is often zero. +/// +/// The requirement for `8` as a maximum here is somewhat arbitrary. In +/// practice, we never need anything bigger in this crate, and so this function +/// does some sanity asserts under the assumption of a max alignment of `8`. +#[cfg(feature = "alloc")] +pub(crate) fn alloc_aligned_buffer<T>(size: usize) -> (Vec<u8>, usize) { + // NOTE: This is a kludge because there's no easy way to allocate a Vec<u8> + // with an alignment guaranteed to be greater than 1. We could create a + // Vec<u32>, but this cannot be safely transmuted to a Vec<u8> without + // concern, since reallocing or dropping the Vec<u8> is UB (different + // alignment than the initial allocation). We could define a wrapper type + // to manage this for us, but it seems like more machinery than it's worth. + let buf = vec![0; size]; + let align = core::mem::align_of::<T>(); + let address = buf.as_ptr().as_usize(); + if address % align == 0 { + return (buf, 0); + } + // Let's try this again. We have to create a totally new alloc with + // the maximum amount of bytes we might need. We can't just extend our + // pre-existing 'buf' because that might create a new alloc with a + // different alignment. + let extra = align - 1; + let mut buf = vec![0; size + extra]; + let address = buf.as_ptr().as_usize(); + // The code below handles the case where 'address' is aligned to T, so if + // we got lucky and 'address' is now aligned to T (when it previously + // wasn't), then we're done. + if address % align == 0 { + buf.truncate(size); + return (buf, 0); + } + let padding = ((address & !(align - 1)).checked_add(align).unwrap()) + .checked_sub(address) + .unwrap(); + assert!(padding <= 7, "padding of {padding} is bigger than 7"); + assert!( + padding <= extra, + "padding of {padding} is bigger than extra {extra} bytes", + ); + buf.truncate(size + padding); + assert_eq!(size + padding, buf.len()); + assert_eq!( + 0, + buf[padding..].as_ptr().as_usize() % align, + "expected end of initial padding to be aligned to {align}", + ); + (buf, padding) +} + +/// Reads a NUL terminated label starting at the beginning of the given slice. +/// +/// If a NUL terminated label could not be found, then an error is returned. +/// Similarly, if a label is found but doesn't match the expected label, then +/// an error is returned. +/// +/// Upon success, the total number of bytes read (including padding bytes) is +/// returned. +pub(crate) fn read_label( + slice: &[u8], + expected_label: &'static str, +) -> Result<usize, DeserializeError> { + // Set an upper bound on how many bytes we scan for a NUL. Since no label + // in this crate is longer than 256 bytes, if we can't find one within that + // range, then we have corrupted data. + let first_nul = + slice[..cmp::min(slice.len(), 256)].iter().position(|&b| b == 0); + let first_nul = match first_nul { + Some(first_nul) => first_nul, + None => { + return Err(DeserializeError::generic( + "could not find NUL terminated label \ + at start of serialized object", + )); + } + }; + let len = first_nul + padding_len(first_nul); + if slice.len() < len { + return Err(DeserializeError::generic( + "could not find properly sized label at start of serialized object" + )); + } + if expected_label.as_bytes() != &slice[..first_nul] { + return Err(DeserializeError::label_mismatch(expected_label)); + } + Ok(len) +} + +/// Writes the given label to the buffer as a NUL terminated string. The label +/// given must not contain NUL, otherwise this will panic. Similarly, the label +/// must not be longer than 255 bytes, otherwise this will panic. +/// +/// Additional NUL bytes are written as necessary to ensure that the number of +/// bytes written is always a multiple of 4. +/// +/// Upon success, the total number of bytes written (including padding) is +/// returned. +pub(crate) fn write_label( + label: &str, + dst: &mut [u8], +) -> Result<usize, SerializeError> { + let nwrite = write_label_len(label); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("label")); + } + dst[..label.len()].copy_from_slice(label.as_bytes()); + for i in 0..(nwrite - label.len()) { + dst[label.len() + i] = 0; + } + assert_eq!(nwrite % 4, 0); + Ok(nwrite) +} + +/// Returns the total number of bytes (including padding) that would be written +/// for the given label. This panics if the given label contains a NUL byte or +/// is longer than 255 bytes. (The size restriction exists so that searching +/// for a label during deserialization can be done in small bounded space.) +pub(crate) fn write_label_len(label: &str) -> usize { + assert!(label.len() <= 255, "label must not be longer than 255 bytes"); + assert!(label.bytes().all(|b| b != 0), "label must not contain NUL bytes"); + let label_len = label.len() + 1; // +1 for the NUL terminator + label_len + padding_len(label_len) +} + +/// Reads the endianness check from the beginning of the given slice and +/// confirms that the endianness of the serialized object matches the expected +/// endianness. If the slice is too small or if the endianness check fails, +/// this returns an error. +/// +/// Upon success, the total number of bytes read is returned. +pub(crate) fn read_endianness_check( + slice: &[u8], +) -> Result<usize, DeserializeError> { + let (n, nr) = try_read_u32(slice, "endianness check")?; + assert_eq!(nr, write_endianness_check_len()); + if n != 0xFEFF { + return Err(DeserializeError::endian_mismatch(0xFEFF, n)); + } + Ok(nr) +} + +/// Writes 0xFEFF as an integer using the given endianness. +/// +/// This is useful for writing into the header of a serialized object. It can +/// be read during deserialization as a sanity check to ensure the proper +/// endianness is used. +/// +/// Upon success, the total number of bytes written is returned. +pub(crate) fn write_endianness_check<E: Endian>( + dst: &mut [u8], +) -> Result<usize, SerializeError> { + let nwrite = write_endianness_check_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("endianness check")); + } + E::write_u32(0xFEFF, dst); + Ok(nwrite) +} + +/// Returns the number of bytes written by the endianness check. +pub(crate) fn write_endianness_check_len() -> usize { + size_of::<u32>() +} + +/// Reads a version number from the beginning of the given slice and confirms +/// that is matches the expected version number given. If the slice is too +/// small or if the version numbers aren't equivalent, this returns an error. +/// +/// Upon success, the total number of bytes read is returned. +/// +/// N.B. Currently, we require that the version number is exactly equivalent. +/// In the future, if we bump the version number without a semver bump, then +/// we'll need to relax this a bit and support older versions. +pub(crate) fn read_version( + slice: &[u8], + expected_version: u32, +) -> Result<usize, DeserializeError> { + let (n, nr) = try_read_u32(slice, "version")?; + assert_eq!(nr, write_version_len()); + if n != expected_version { + return Err(DeserializeError::version_mismatch(expected_version, n)); + } + Ok(nr) +} + +/// Writes the given version number to the beginning of the given slice. +/// +/// This is useful for writing into the header of a serialized object. It can +/// be read during deserialization as a sanity check to ensure that the library +/// code supports the format of the serialized object. +/// +/// Upon success, the total number of bytes written is returned. +pub(crate) fn write_version<E: Endian>( + version: u32, + dst: &mut [u8], +) -> Result<usize, SerializeError> { + let nwrite = write_version_len(); + if dst.len() < nwrite { + return Err(SerializeError::buffer_too_small("version number")); + } + E::write_u32(version, dst); + Ok(nwrite) +} + +/// Returns the number of bytes written by writing the version number. +pub(crate) fn write_version_len() -> usize { + size_of::<u32>() +} + +/// Reads a pattern ID from the given slice. If the slice has insufficient +/// length, then this panics. If the deserialized integer exceeds the pattern +/// ID limit for the current target, then this returns an error. +/// +/// Upon success, this also returns the number of bytes read. +pub(crate) fn read_pattern_id( + slice: &[u8], + what: &'static str, +) -> Result<(PatternID, usize), DeserializeError> { + let bytes: [u8; PatternID::SIZE] = + slice[..PatternID::SIZE].try_into().unwrap(); + let pid = PatternID::from_ne_bytes(bytes) + .map_err(|err| DeserializeError::pattern_id_error(err, what))?; + Ok((pid, PatternID::SIZE)) +} + +/// Reads a pattern ID from the given slice. If the slice has insufficient +/// length, then this panics. Otherwise, the deserialized integer is assumed +/// to be a valid pattern ID. +/// +/// This also returns the number of bytes read. +pub(crate) fn read_pattern_id_unchecked(slice: &[u8]) -> (PatternID, usize) { + let pid = PatternID::from_ne_bytes_unchecked( + slice[..PatternID::SIZE].try_into().unwrap(), + ); + (pid, PatternID::SIZE) +} + +/// Write the given pattern ID to the beginning of the given slice of bytes +/// using the specified endianness. The given slice must have length at least +/// `PatternID::SIZE`, or else this panics. Upon success, the total number of +/// bytes written is returned. +pub(crate) fn write_pattern_id<E: Endian>( + pid: PatternID, + dst: &mut [u8], +) -> usize { + E::write_u32(pid.as_u32(), dst); + PatternID::SIZE +} + +/// Attempts to read a state ID from the given slice. If the slice has an +/// insufficient number of bytes or if the state ID exceeds the limit for +/// the current target, then this returns an error. +/// +/// Upon success, this also returns the number of bytes read. +pub(crate) fn try_read_state_id( + slice: &[u8], + what: &'static str, +) -> Result<(StateID, usize), DeserializeError> { + if slice.len() < StateID::SIZE { + return Err(DeserializeError::buffer_too_small(what)); + } + read_state_id(slice, what) +} + +/// Reads a state ID from the given slice. If the slice has insufficient +/// length, then this panics. If the deserialized integer exceeds the state ID +/// limit for the current target, then this returns an error. +/// +/// Upon success, this also returns the number of bytes read. +pub(crate) fn read_state_id( + slice: &[u8], + what: &'static str, +) -> Result<(StateID, usize), DeserializeError> { + let bytes: [u8; StateID::SIZE] = + slice[..StateID::SIZE].try_into().unwrap(); + let sid = StateID::from_ne_bytes(bytes) + .map_err(|err| DeserializeError::state_id_error(err, what))?; + Ok((sid, StateID::SIZE)) +} + +/// Reads a state ID from the given slice. If the slice has insufficient +/// length, then this panics. Otherwise, the deserialized integer is assumed +/// to be a valid state ID. +/// +/// This also returns the number of bytes read. +pub(crate) fn read_state_id_unchecked(slice: &[u8]) -> (StateID, usize) { + let sid = StateID::from_ne_bytes_unchecked( + slice[..StateID::SIZE].try_into().unwrap(), + ); + (sid, StateID::SIZE) +} + +/// Write the given state ID to the beginning of the given slice of bytes +/// using the specified endianness. The given slice must have length at least +/// `StateID::SIZE`, or else this panics. Upon success, the total number of +/// bytes written is returned. +pub(crate) fn write_state_id<E: Endian>( + sid: StateID, + dst: &mut [u8], +) -> usize { + E::write_u32(sid.as_u32(), dst); + StateID::SIZE +} + +/// Try to read a u16 as a usize from the beginning of the given slice in +/// native endian format. If the slice has fewer than 2 bytes or if the +/// deserialized number cannot be represented by usize, then this returns an +/// error. The error message will include the `what` description of what is +/// being deserialized, for better error messages. `what` should be a noun in +/// singular form. +/// +/// Upon success, this also returns the number of bytes read. +pub(crate) fn try_read_u16_as_usize( + slice: &[u8], + what: &'static str, +) -> Result<(usize, usize), DeserializeError> { + try_read_u16(slice, what).and_then(|(n, nr)| { + usize::try_from(n) + .map(|n| (n, nr)) + .map_err(|_| DeserializeError::invalid_usize(what)) + }) +} + +/// Try to read a u32 as a usize from the beginning of the given slice in +/// native endian format. If the slice has fewer than 4 bytes or if the +/// deserialized number cannot be represented by usize, then this returns an +/// error. The error message will include the `what` description of what is +/// being deserialized, for better error messages. `what` should be a noun in +/// singular form. +/// +/// Upon success, this also returns the number of bytes read. +pub(crate) fn try_read_u32_as_usize( + slice: &[u8], + what: &'static str, +) -> Result<(usize, usize), DeserializeError> { + try_read_u32(slice, what).and_then(|(n, nr)| { + usize::try_from(n) + .map(|n| (n, nr)) + .map_err(|_| DeserializeError::invalid_usize(what)) + }) +} + +/// Try to read a u16 from the beginning of the given slice in native endian +/// format. If the slice has fewer than 2 bytes, then this returns an error. +/// The error message will include the `what` description of what is being +/// deserialized, for better error messages. `what` should be a noun in +/// singular form. +/// +/// Upon success, this also returns the number of bytes read. +pub(crate) fn try_read_u16( + slice: &[u8], + what: &'static str, +) -> Result<(u16, usize), DeserializeError> { + check_slice_len(slice, size_of::<u16>(), what)?; + Ok((read_u16(slice), size_of::<u16>())) +} + +/// Try to read a u32 from the beginning of the given slice in native endian +/// format. If the slice has fewer than 4 bytes, then this returns an error. +/// The error message will include the `what` description of what is being +/// deserialized, for better error messages. `what` should be a noun in +/// singular form. +/// +/// Upon success, this also returns the number of bytes read. +pub(crate) fn try_read_u32( + slice: &[u8], + what: &'static str, +) -> Result<(u32, usize), DeserializeError> { + check_slice_len(slice, size_of::<u32>(), what)?; + Ok((read_u32(slice), size_of::<u32>())) +} + +/// Try to read a u128 from the beginning of the given slice in native endian +/// format. If the slice has fewer than 16 bytes, then this returns an error. +/// The error message will include the `what` description of what is being +/// deserialized, for better error messages. `what` should be a noun in +/// singular form. +/// +/// Upon success, this also returns the number of bytes read. +pub(crate) fn try_read_u128( + slice: &[u8], + what: &'static str, +) -> Result<(u128, usize), DeserializeError> { + check_slice_len(slice, size_of::<u128>(), what)?; + Ok((read_u128(slice), size_of::<u128>())) +} + +/// Read a u16 from the beginning of the given slice in native endian format. +/// If the slice has fewer than 2 bytes, then this panics. +/// +/// Marked as inline to speed up sparse searching which decodes integers from +/// its automaton at search time. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn read_u16(slice: &[u8]) -> u16 { + let bytes: [u8; 2] = slice[..size_of::<u16>()].try_into().unwrap(); + u16::from_ne_bytes(bytes) +} + +/// Read a u32 from the beginning of the given slice in native endian format. +/// If the slice has fewer than 4 bytes, then this panics. +/// +/// Marked as inline to speed up sparse searching which decodes integers from +/// its automaton at search time. +#[cfg_attr(feature = "perf-inline", inline(always))] +pub(crate) fn read_u32(slice: &[u8]) -> u32 { + let bytes: [u8; 4] = slice[..size_of::<u32>()].try_into().unwrap(); + u32::from_ne_bytes(bytes) +} + +/// Read a u128 from the beginning of the given slice in native endian format. +/// If the slice has fewer than 16 bytes, then this panics. +pub(crate) fn read_u128(slice: &[u8]) -> u128 { + let bytes: [u8; 16] = slice[..size_of::<u128>()].try_into().unwrap(); + u128::from_ne_bytes(bytes) +} + +/// Checks that the given slice has some minimal length. If it's smaller than +/// the bound given, then a "buffer too small" error is returned with `what` +/// describing what the buffer represents. +pub(crate) fn check_slice_len<T>( + slice: &[T], + at_least_len: usize, + what: &'static str, +) -> Result<(), DeserializeError> { + if slice.len() < at_least_len { + return Err(DeserializeError::buffer_too_small(what)); + } + Ok(()) +} + +/// Multiply the given numbers, and on overflow, return an error that includes +/// 'what' in the error message. +/// +/// This is useful when doing arithmetic with untrusted data. +pub(crate) fn mul( + a: usize, + b: usize, + what: &'static str, +) -> Result<usize, DeserializeError> { + match a.checked_mul(b) { + Some(c) => Ok(c), + None => Err(DeserializeError::arithmetic_overflow(what)), + } +} + +/// Add the given numbers, and on overflow, return an error that includes +/// 'what' in the error message. +/// +/// This is useful when doing arithmetic with untrusted data. +pub(crate) fn add( + a: usize, + b: usize, + what: &'static str, +) -> Result<usize, DeserializeError> { + match a.checked_add(b) { + Some(c) => Ok(c), + None => Err(DeserializeError::arithmetic_overflow(what)), + } +} + +/// Shift `a` left by `b`, and on overflow, return an error that includes +/// 'what' in the error message. +/// +/// This is useful when doing arithmetic with untrusted data. +pub(crate) fn shl( + a: usize, + b: usize, + what: &'static str, +) -> Result<usize, DeserializeError> { + let amount = u32::try_from(b) + .map_err(|_| DeserializeError::arithmetic_overflow(what))?; + match a.checked_shl(amount) { + Some(c) => Ok(c), + None => Err(DeserializeError::arithmetic_overflow(what)), + } +} + +/// Returns the number of additional bytes required to add to the given length +/// in order to make the total length a multiple of 4. The return value is +/// always less than 4. +pub(crate) fn padding_len(non_padding_len: usize) -> usize { + (4 - (non_padding_len & 0b11)) & 0b11 +} + +/// A simple trait for writing code generic over endianness. +/// +/// This is similar to what byteorder provides, but we only need a very small +/// subset. +pub(crate) trait Endian { + /// Writes a u16 to the given destination buffer in a particular + /// endianness. If the destination buffer has a length smaller than 2, then + /// this panics. + fn write_u16(n: u16, dst: &mut [u8]); + + /// Writes a u32 to the given destination buffer in a particular + /// endianness. If the destination buffer has a length smaller than 4, then + /// this panics. + fn write_u32(n: u32, dst: &mut [u8]); + + /// Writes a u128 to the given destination buffer in a particular + /// endianness. If the destination buffer has a length smaller than 16, + /// then this panics. + fn write_u128(n: u128, dst: &mut [u8]); +} + +/// Little endian writing. +pub(crate) enum LE {} +/// Big endian writing. +pub(crate) enum BE {} + +#[cfg(target_endian = "little")] +pub(crate) type NE = LE; +#[cfg(target_endian = "big")] +pub(crate) type NE = BE; + +impl Endian for LE { + fn write_u16(n: u16, dst: &mut [u8]) { + dst[..2].copy_from_slice(&n.to_le_bytes()); + } + + fn write_u32(n: u32, dst: &mut [u8]) { + dst[..4].copy_from_slice(&n.to_le_bytes()); + } + + fn write_u128(n: u128, dst: &mut [u8]) { + dst[..16].copy_from_slice(&n.to_le_bytes()); + } +} + +impl Endian for BE { + fn write_u16(n: u16, dst: &mut [u8]) { + dst[..2].copy_from_slice(&n.to_be_bytes()); + } + + fn write_u32(n: u32, dst: &mut [u8]) { + dst[..4].copy_from_slice(&n.to_be_bytes()); + } + + fn write_u128(n: u128, dst: &mut [u8]) { + dst[..16].copy_from_slice(&n.to_be_bytes()); + } +} + +#[cfg(all(test, feature = "alloc"))] +mod tests { + use super::*; + + #[test] + fn labels() { + let mut buf = [0; 1024]; + + let nwrite = write_label("fooba", &mut buf).unwrap(); + assert_eq!(nwrite, 8); + assert_eq!(&buf[..nwrite], b"fooba\x00\x00\x00"); + + let nread = read_label(&buf, "fooba").unwrap(); + assert_eq!(nread, 8); + } + + #[test] + #[should_panic] + fn bad_label_interior_nul() { + // interior NULs are not allowed + write_label("foo\x00bar", &mut [0; 1024]).unwrap(); + } + + #[test] + fn bad_label_almost_too_long() { + // ok + write_label(&"z".repeat(255), &mut [0; 1024]).unwrap(); + } + + #[test] + #[should_panic] + fn bad_label_too_long() { + // labels longer than 255 bytes are banned + write_label(&"z".repeat(256), &mut [0; 1024]).unwrap(); + } + + #[test] + fn padding() { + assert_eq!(0, padding_len(8)); + assert_eq!(3, padding_len(9)); + assert_eq!(2, padding_len(10)); + assert_eq!(1, padding_len(11)); + assert_eq!(0, padding_len(12)); + assert_eq!(3, padding_len(13)); + assert_eq!(2, padding_len(14)); + assert_eq!(1, padding_len(15)); + assert_eq!(0, padding_len(16)); + } +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/dfa/api.rs b/third_party/crates/regex-automata-0.4.18/tests/dfa/api.rs new file mode 100644 index 0000000..8a015ad --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/dfa/api.rs @@ -0,0 +1,162 @@ +use std::error::Error; + +use regex_automata::{ + dfa::{dense, Automaton, OverlappingState}, + nfa::thompson, + Anchored, HalfMatch, Input, MatchError, +}; + +// Tests that quit bytes in the forward direction work correctly. +#[test] +fn quit_fwd() -> Result<(), Box<dyn Error>> { + let dfa = dense::Builder::new() + .configure(dense::Config::new().quit(b'x', true)) + .build("[[:word:]]+$")?; + + assert_eq!( + Err(MatchError::quit(b'x', 3)), + dfa.try_search_fwd(&Input::new(b"abcxyz")) + ); + assert_eq!( + dfa.try_search_overlapping_fwd( + &Input::new(b"abcxyz"), + &mut OverlappingState::start() + ), + Err(MatchError::quit(b'x', 3)), + ); + + Ok(()) +} + +// Tests that quit bytes in the reverse direction work correctly. +#[test] +fn quit_rev() -> Result<(), Box<dyn Error>> { + let dfa = dense::Builder::new() + .configure(dense::Config::new().quit(b'x', true)) + .thompson(thompson::Config::new().reverse(true)) + .build("^[[:word:]]+")?; + + assert_eq!( + Err(MatchError::quit(b'x', 3)), + dfa.try_search_rev(&Input::new(b"abcxyz")) + ); + + Ok(()) +} + +// Tests that if we heuristically enable Unicode word boundaries but then +// instruct that a non-ASCII byte should NOT be a quit byte, then the builder +// will panic. +#[test] +#[should_panic] +fn quit_panics() { + dense::Config::new().unicode_word_boundary(true).quit(b'\xFF', false); +} + +// This tests an intesting case where even if the Unicode word boundary option +// is disabled, setting all non-ASCII bytes to be quit bytes will cause Unicode +// word boundaries to be enabled. +#[test] +fn unicode_word_implicitly_works() -> Result<(), Box<dyn Error>> { + let mut config = dense::Config::new(); + for b in 0x80..=0xFF { + config = config.quit(b, true); + } + let dfa = dense::Builder::new().configure(config).build(r"\b")?; + let expected = HalfMatch::must(0, 1); + assert_eq!(Ok(Some(expected)), dfa.try_search_fwd(&Input::new(b" a"))); + Ok(()) +} + +// A variant of [`Automaton::is_special_state`]'s doctest, but with universal +// start states. +// +// See: https://github.com/rust-lang/regex/pull/1195 +#[test] +fn universal_start_search() -> Result<(), Box<dyn Error>> { + fn find<A: Automaton>( + dfa: &A, + haystack: &[u8], + ) -> Result<Option<HalfMatch>, MatchError> { + let mut state = dfa + .universal_start_state(Anchored::No) + .expect("regex should not require lookbehind"); + let mut last_match = None; + // Walk all the bytes in the haystack. We can quit early if we see + // a dead or a quit state. The former means the automaton will + // never transition to any other state. The latter means that the + // automaton entered a condition in which its search failed. + for (i, &b) in haystack.iter().enumerate() { + state = dfa.next_state(state, b); + if dfa.is_special_state(state) { + if dfa.is_match_state(state) { + last_match = + Some(HalfMatch::new(dfa.match_pattern(state, 0), i)); + } else if dfa.is_dead_state(state) { + return Ok(last_match); + } else if dfa.is_quit_state(state) { + // It is possible to enter into a quit state after + // observing a match has occurred. In that case, we + // should return the match instead of an error. + if last_match.is_some() { + return Ok(last_match); + } + return Err(MatchError::quit(b, i)); + } + // Implementors may also want to check for start or accel + // states and handle them differently for performance + // reasons. But it is not necessary for correctness. + } + } + // Matches are always delayed by 1 byte, so we must explicitly walk + // the special "EOI" transition at the end of the search. + state = dfa.next_eoi_state(state); + if dfa.is_match_state(state) { + last_match = Some(HalfMatch::new( + dfa.match_pattern(state, 0), + haystack.len(), + )); + } + Ok(last_match) + } + + fn check_impl( + dfa: impl Automaton, + haystack: &str, + pat: usize, + offset: usize, + ) -> Result<(), Box<dyn Error>> { + let haystack = haystack.as_bytes(); + let mat = find(&dfa, haystack)?.unwrap(); + assert_eq!(mat.pattern().as_usize(), pat); + assert_eq!(mat.offset(), offset); + Ok(()) + } + + fn check( + dfa: &dense::DFA<Vec<u32>>, + haystack: &str, + pat: usize, + offset: usize, + ) -> Result<(), Box<dyn Error>> { + check_impl(dfa, haystack, pat, offset)?; + check_impl(dfa.to_sparse()?, haystack, pat, offset)?; + Ok(()) + } + + let dfa = dense::DFA::new(r"[a-z]+")?; + let haystack = "123 foobar 4567"; + check(&dfa, haystack, 0, 10)?; + + let dfa = dense::DFA::new(r"[0-9]{4}")?; + let haystack = "123 foobar 4567"; + check(&dfa, haystack, 0, 15)?; + + let dfa = dense::DFA::new_many(&[r"[a-z]+", r"[0-9]+"])?; + let haystack = "123 foobar 4567"; + check(&dfa, haystack, 1, 3)?; + check(&dfa, &haystack[3..], 0, 7)?; + check(&dfa, &haystack[10..], 1, 5)?; + + Ok(()) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/dfa/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/dfa/mod.rs new file mode 100644 index 0000000..0d8f539 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/dfa/mod.rs @@ -0,0 +1,8 @@ +#[cfg(all(feature = "dfa-build", feature = "dfa-search"))] +mod api; +#[cfg(feature = "dfa-onepass")] +mod onepass; +#[cfg(all(feature = "dfa-build", feature = "dfa-search"))] +mod regression; +#[cfg(all(not(miri), feature = "dfa-build", feature = "dfa-search"))] +mod suite; diff --git a/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/mod.rs new file mode 100644 index 0000000..5a71144 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/mod.rs @@ -0,0 +1,3 @@ +mod regression; +#[cfg(not(miri))] +mod suite; diff --git a/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/regression.rs b/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/regression.rs new file mode 100644 index 0000000..7c72db3 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/regression.rs @@ -0,0 +1,61 @@ +// Regression test for zero-repetition capture groups, +// which caused a panic when the Vec passed into search_slots +// contained space for the capture group which would never +// have any results. +// +// See: https://github.com/rust-lang/regex/issues/1327 +#[test] +fn zero_repetition_capture_group() { + use regex_automata::{ + dfa::onepass::DFA, util::primitives::NonMaxUsize, Anchored, Input, + }; + + let expr = DFA::new(r"(abc)(ABC){0}").unwrap(); + let s = "abcABC"; + let input = Input::new(s).span(0..s.len()).anchored(Anchored::Yes); + + // Test with 4 slots, so the whole match plus the first capture group. + let mut cache = expr.create_cache(); + let mut slots: Vec<Option<NonMaxUsize>> = vec![None; 4]; + let pid = expr.try_search_slots(&mut cache, &input, &mut slots).unwrap(); + assert_eq!(pid, Some(regex_automata::PatternID::must(0))); + assert_eq!(slots[0], Some(NonMaxUsize::new(0).unwrap())); + assert_eq!(slots[1], Some(NonMaxUsize::new(3).unwrap())); + assert_eq!(slots[2], Some(NonMaxUsize::new(0).unwrap())); + assert_eq!(slots[3], Some(NonMaxUsize::new(3).unwrap())); + + // Test with larger slot array, which would fit the + // zero-repetition capture group. + slots.resize(6, None); + let pid = expr.try_search_slots(&mut cache, &input, &mut slots).unwrap(); + assert_eq!(pid, Some(regex_automata::PatternID::must(0))); + // First capture group should match + assert_eq!(slots[2], Some(NonMaxUsize::new(0).unwrap())); + assert_eq!(slots[3], Some(NonMaxUsize::new(3).unwrap())); + // Second capture group with {0} should be None. + assert_eq!(slots[4], None); + assert_eq!(slots[5], None); +} + +// Another regression test for the same case as +// `zero_repetition_capture_group`, but uses a simpler pattern. That +// is, a zero-repetition capture group is a red herring. The actual bug +// is simpler: it happens whenever too many slots are provided by the +// caller. +#[test] +fn too_many_slots_normal_pattern() { + use regex_automata::{ + dfa::onepass::DFA, util::primitives::NonMaxUsize, Anchored, Input, + }; + + let expr = DFA::new(r"abc").unwrap(); + let s = "abc"; + let input = Input::new(s).span(0..s.len()).anchored(Anchored::Yes); + + let mut cache = expr.create_cache(); + let mut slots: Vec<Option<NonMaxUsize>> = vec![None; 4]; + let pid = expr.try_search_slots(&mut cache, &input, &mut slots).unwrap(); + assert_eq!(pid, Some(regex_automata::PatternID::must(0))); + assert_eq!(slots[0], Some(NonMaxUsize::new(0).unwrap())); + assert_eq!(slots[1], Some(NonMaxUsize::new(3).unwrap())); +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/suite.rs b/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/suite.rs new file mode 100644 index 0000000..aba46c8 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/dfa/onepass/suite.rs @@ -0,0 +1,197 @@ +use { + anyhow::Result, + regex_automata::{ + dfa::onepass::{self, DFA}, + nfa::thompson, + util::{iter, syntax}, + }, + regex_test::{ + CompiledRegex, Match, RegexTest, SearchKind, Span, TestResult, + TestRunner, + }, +}; + +use crate::{create_input, suite, testify_captures, untestify_kind}; + +const EXPANSIONS: &[&str] = &["is_match", "find", "captures"]; + +/// Tests the default configuration of the hybrid NFA/DFA. +#[test] +fn default() -> Result<()> { + let builder = DFA::builder(); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the hybrid NFA/DFA when 'starts_for_each_pattern' is enabled for all +/// tests. +#[test] +fn starts_for_each_pattern() -> Result<()> { + let mut builder = DFA::builder(); + builder.configure(DFA::config().starts_for_each_pattern(true)); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the hybrid NFA/DFA when byte classes are disabled. +/// +/// N.B. Disabling byte classes doesn't avoid any indirection at search time. +/// All it does is cause every byte value to be its own distinct equivalence +/// class. +#[test] +fn no_byte_classes() -> Result<()> { + let mut builder = DFA::builder(); + builder.configure(DFA::config().byte_classes(false)); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +fn compiler( + mut builder: onepass::Builder, +) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> { + move |test, regexes| { + // Check if our regex contains things that aren't supported by DFAs. + // That is, Unicode word boundaries when searching non-ASCII text. + if !configure_onepass_builder(test, &mut builder) { + return Ok(CompiledRegex::skip()); + } + let re = match builder.build_many(®exes) { + Ok(re) => re, + Err(err) => { + let msg = err.to_string(); + // This is pretty gross, but when a regex fails to compile as + // a one-pass regex, then we want to be OK with that and just + // skip the test. But we have to be careful to only skip it + // when the expected result is that the regex compiles. If + // the test is specifically checking that the regex does not + // compile, then we should bubble up that error and allow the + // test to pass. + // + // Since our error types are all generally opaque, we just + // look for an error string. Not great, but not the end of the + // world. + if test.compiles() && msg.contains("not one-pass") { + return Ok(CompiledRegex::skip()); + } + return Err(err.into()); + } + }; + let mut cache = re.create_cache(); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, &mut cache, test) + })) + } +} + +fn run_test( + re: &DFA, + cache: &mut onepass::Cache, + test: &RegexTest, +) -> TestResult { + let input = create_input(test); + match test.additional_name() { + "is_match" => { + TestResult::matched(re.is_match(cache, input.earliest(true))) + } + "find" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Leftmost => { + let input = + input.earliest(test.search_kind() == SearchKind::Earliest); + let mut caps = re.create_captures(); + let it = iter::Searcher::new(input) + .into_matches_iter(|input| { + re.try_search(cache, input, &mut caps)?; + Ok(caps.get_match()) + }) + .infallible() + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: m.pattern().as_usize(), + span: Span { start: m.start(), end: m.end() }, + }); + TestResult::matches(it) + } + SearchKind::Overlapping => { + // The one-pass DFA does not support any kind of overlapping + // search. This is not just a matter of not having the API. + // It's fundamentally incompatible with the one-pass concept. + // If overlapping matches were possible, then the one-pass DFA + // would fail to build. + TestResult::skip() + } + }, + "captures" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Leftmost => { + let input = + input.earliest(test.search_kind() == SearchKind::Earliest); + let it = iter::Searcher::new(input) + .into_captures_iter(re.create_captures(), |input, caps| { + re.try_search(cache, input, caps) + }) + .infallible() + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|caps| testify_captures(&caps)); + TestResult::captures(it) + } + SearchKind::Overlapping => { + // The one-pass DFA does not support any kind of overlapping + // search. This is not just a matter of not having the API. + // It's fundamentally incompatible with the one-pass concept. + // If overlapping matches were possible, then the one-pass DFA + // would fail to build. + TestResult::skip() + } + }, + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Configures the given regex builder with all relevant settings on the given +/// regex test. +/// +/// If the regex test has a setting that is unsupported, then this returns +/// false (implying the test should be skipped). +fn configure_onepass_builder( + test: &RegexTest, + builder: &mut onepass::Builder, +) -> bool { + if !test.anchored() { + return false; + } + let match_kind = match untestify_kind(test.match_kind()) { + None => return false, + Some(k) => k, + }; + + let config = DFA::config().match_kind(match_kind); + builder + .configure(config) + .syntax(config_syntax(test)) + .thompson(config_thompson(test)); + true +} + +/// Configuration of a Thompson NFA compiler from a regex test. +fn config_thompson(test: &RegexTest) -> thompson::Config { + let mut lookm = regex_automata::util::look::LookMatcher::new(); + lookm.set_line_terminator(test.line_terminator()); + thompson::Config::new().utf8(test.utf8()).look_matcher(lookm) +} + +/// Configuration of the regex parser from a regex test. +fn config_syntax(test: &RegexTest) -> syntax::Config { + syntax::Config::new() + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .utf8(test.utf8()) + .line_terminator(test.line_terminator()) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/dfa/regression.rs b/third_party/crates/regex-automata-0.4.18/tests/dfa/regression.rs new file mode 100644 index 0000000..2c54077 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/dfa/regression.rs @@ -0,0 +1,75 @@ +// A regression test for checking that minimization correctly translates +// whether a state is a match state or not. Previously, it was possible for +// minimization to mark a non-matching state as matching. +#[test] +#[cfg(not(miri))] +fn minimize_sets_correct_match_states() { + use regex_automata::{ + dfa::{dense::DFA, Automaton, StartKind}, + Anchored, Input, + }; + + let pattern = + // This is a subset of the grapheme matching regex. I couldn't seem + // to get a repro any smaller than this unfortunately. + r"(?x) + (?: + \p{gcb=Prepend}* + (?: + (?: + (?: + \p{gcb=L}* + (?:\p{gcb=V}+|\p{gcb=LV}\p{gcb=V}*|\p{gcb=LVT}) + \p{gcb=T}* + ) + | + \p{gcb=L}+ + | + \p{gcb=T}+ + ) + | + \p{Extended_Pictographic} + (?:\p{gcb=Extend}*\p{gcb=ZWJ}\p{Extended_Pictographic})* + | + [^\p{gcb=Control}\p{gcb=CR}\p{gcb=LF}] + ) + [\p{gcb=Extend}\p{gcb=ZWJ}\p{gcb=SpacingMark}]* + ) + "; + + let dfa = DFA::builder() + .configure( + DFA::config().start_kind(StartKind::Anchored).minimize(true), + ) + .build(pattern) + .unwrap(); + let input = Input::new(b"\xE2").anchored(Anchored::Yes); + assert_eq!(Ok(None), dfa.try_search_fwd(&input)); +} + +// A prefilter must not skip to a new candidate after the DFA has already +// recorded a match. The DFA is still resolving the priority of that match, +// and skipping can cause a later match to overwrite it. +#[test] +#[cfg(feature = "perf-literal-multisubstring")] +fn prefilter_does_not_skip_pending_match() { + use regex_automata::{ + dfa::{dense::DFA, Automaton}, + util::prefilter::Prefilter, + HalfMatch, Input, MatchKind, + }; + + let pre = Prefilter::new(MatchKind::LeftmostFirst, &["ab", "c"]) + .expect("prefilter"); + let dfa = DFA::builder() + .configure( + DFA::config().prefilter(Some(pre)).starts_for_each_pattern(true), + ) + .build(r"(?:ab|c)*c") + .unwrap(); + let expected = Ok(Some(HalfMatch::must(0, 1))); + let input = Input::new("cabac"); + + assert_eq!(expected, dfa.try_search_fwd(&input)); + assert_eq!(expected, dfa.to_sparse().unwrap().try_search_fwd(&input)); +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/dfa/suite.rs b/third_party/crates/regex-automata-0.4.18/tests/dfa/suite.rs new file mode 100644 index 0000000..8368ffe --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/dfa/suite.rs @@ -0,0 +1,443 @@ +use { + anyhow::Result, + regex_automata::{ + dfa::{ + self, dense, regex::Regex, sparse, Automaton, OverlappingState, + StartKind, + }, + nfa::thompson, + util::{prefilter::Prefilter, syntax}, + Anchored, Input, PatternSet, + }, + regex_test::{ + CompiledRegex, Match, RegexTest, SearchKind, Span, TestResult, + TestRunner, + }, +}; + +use crate::{create_input, suite, untestify_kind}; + +const EXPANSIONS: &[&str] = &["is_match", "find", "which"]; + +/// Runs the test suite with the default configuration. +#[test] +fn unminimized_default() -> Result<()> { + let builder = Regex::builder(); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), dense_compiler(builder)) + .assert(); + Ok(()) +} + +/// Runs the test suite with the default configuration and a prefilter enabled, +/// if one can be built. +#[test] +fn unminimized_prefilter() -> Result<()> { + let my_compiler = |test: &RegexTest, regexes: &[String]| { + // Parse regexes as HIRs so we can get literals to build a prefilter. + let mut hirs = vec![]; + for pattern in regexes.iter() { + hirs.push(syntax::parse_with(pattern, &config_syntax(test))?); + } + let kind = match untestify_kind(test.match_kind()) { + None => return Ok(CompiledRegex::skip()), + Some(kind) => kind, + }; + let pre = Prefilter::from_hirs_prefix(kind, &hirs); + let mut builder = Regex::builder(); + builder.dense(dense::DFA::config().prefilter(pre)); + compiler(builder, |_, _, re| { + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, test) + })) + })(test, regexes) + }; + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), my_compiler) + .assert(); + Ok(()) +} + +/// Runs the test suite with start states specialized. +#[test] +fn unminimized_specialized_start_states() -> Result<()> { + let mut builder = Regex::builder(); + builder.dense(dense::Config::new().specialize_start_states(true)); + + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), dense_compiler(builder)) + .assert(); + Ok(()) +} + +/// Runs the test suite with byte classes disabled. +#[test] +fn unminimized_no_byte_class() -> Result<()> { + let mut builder = Regex::builder(); + builder.dense(dense::Config::new().byte_classes(false)); + + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), dense_compiler(builder)) + .assert(); + Ok(()) +} + +/// Runs the test suite with NFA shrinking enabled. +#[test] +fn unminimized_nfa_shrink() -> Result<()> { + let mut builder = Regex::builder(); + builder.thompson(thompson::Config::new().shrink(true)); + + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), dense_compiler(builder)) + .assert(); + Ok(()) +} + +/// Runs the test suite on a minimized DFA with an otherwise default +/// configuration. +#[test] +fn minimized_default() -> Result<()> { + let mut builder = Regex::builder(); + builder.dense(dense::Config::new().minimize(true)); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), dense_compiler(builder)) + .assert(); + Ok(()) +} + +/// Runs the test suite on a minimized DFA with byte classes disabled. +#[test] +fn minimized_no_byte_class() -> Result<()> { + let mut builder = Regex::builder(); + builder.dense(dense::Config::new().minimize(true).byte_classes(false)); + + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), dense_compiler(builder)) + .assert(); + Ok(()) +} + +/// Runs the test suite on a sparse unminimized DFA. +#[test] +fn sparse_unminimized_default() -> Result<()> { + let builder = Regex::builder(); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), sparse_compiler(builder)) + .assert(); + Ok(()) +} + +/// Runs the test suite on a sparse unminimized DFA with prefilters enabled. +#[test] +fn sparse_unminimized_prefilter() -> Result<()> { + let my_compiler = |test: &RegexTest, regexes: &[String]| { + // Parse regexes as HIRs so we can get literals to build a prefilter. + let mut hirs = vec![]; + for pattern in regexes.iter() { + hirs.push(syntax::parse_with(pattern, &config_syntax(test))?); + } + let kind = match untestify_kind(test.match_kind()) { + None => return Ok(CompiledRegex::skip()), + Some(kind) => kind, + }; + let pre = Prefilter::from_hirs_prefix(kind, &hirs); + let mut builder = Regex::builder(); + builder.dense(dense::DFA::config().prefilter(pre)); + compiler(builder, |builder, _, re| { + let fwd = re.forward().to_sparse()?; + let rev = re.reverse().to_sparse()?; + let re = builder.build_from_dfas(fwd, rev); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, test) + })) + })(test, regexes) + }; + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), my_compiler) + .assert(); + Ok(()) +} + +/// Another basic sanity test that checks we can serialize and then deserialize +/// a regex, and that the resulting regex can be used for searching correctly. +#[test] +fn serialization_unminimized_default() -> Result<()> { + let builder = Regex::builder(); + let my_compiler = |builder| { + compiler(builder, |builder, _, re| { + let builder = builder.clone(); + let (fwd_bytes, _) = re.forward().to_bytes_native_endian(); + let (rev_bytes, _) = re.reverse().to_bytes_native_endian(); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + let fwd: dense::DFA<&[u32]> = + dense::DFA::from_bytes(&fwd_bytes).unwrap().0; + let rev: dense::DFA<&[u32]> = + dense::DFA::from_bytes(&rev_bytes).unwrap().0; + let re = builder.build_from_dfas(fwd, rev); + + run_test(&re, test) + })) + }) + }; + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), my_compiler(builder)) + .assert(); + Ok(()) +} + +/// A basic sanity test that checks we can serialize and then deserialize a +/// regex using sparse DFAs, and that the resulting regex can be used for +/// searching correctly. +#[test] +fn sparse_serialization_unminimized_default() -> Result<()> { + let builder = Regex::builder(); + let my_compiler = |builder| { + compiler(builder, |builder, _, re| { + let builder = builder.clone(); + let fwd_bytes = re.forward().to_sparse()?.to_bytes_native_endian(); + let rev_bytes = re.reverse().to_sparse()?.to_bytes_native_endian(); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + let fwd: sparse::DFA<&[u8]> = + sparse::DFA::from_bytes(&fwd_bytes).unwrap().0; + let rev: sparse::DFA<&[u8]> = + sparse::DFA::from_bytes(&rev_bytes).unwrap().0; + let re = builder.build_from_dfas(fwd, rev); + run_test(&re, test) + })) + }) + }; + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .blacklist("expensive") + .test_iter(suite()?.iter(), my_compiler(builder)) + .assert(); + Ok(()) +} + +fn dense_compiler( + builder: dfa::regex::Builder, +) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> { + compiler(builder, |_, _, re| { + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, test) + })) + }) +} + +fn sparse_compiler( + builder: dfa::regex::Builder, +) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> { + compiler(builder, |builder, _, re| { + let fwd = re.forward().to_sparse()?; + let rev = re.reverse().to_sparse()?; + let re = builder.build_from_dfas(fwd, rev); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, test) + })) + }) +} + +fn compiler( + mut builder: dfa::regex::Builder, + mut create_matcher: impl FnMut( + &dfa::regex::Builder, + Option<Prefilter>, + Regex, + ) -> Result<CompiledRegex>, +) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> { + move |test, regexes| { + // Parse regexes as HIRs for some analysis below. + let mut hirs = vec![]; + for pattern in regexes.iter() { + hirs.push(syntax::parse_with(pattern, &config_syntax(test))?); + } + + // Get a prefilter in case the test wants it. + let kind = match untestify_kind(test.match_kind()) { + None => return Ok(CompiledRegex::skip()), + Some(kind) => kind, + }; + let pre = Prefilter::from_hirs_prefix(kind, &hirs); + + // Check if our regex contains things that aren't supported by DFAs. + // That is, Unicode word boundaries when searching non-ASCII text. + if !test.haystack().is_ascii() { + for hir in hirs.iter() { + if hir.properties().look_set().contains_word_unicode() { + return Ok(CompiledRegex::skip()); + } + } + } + if !configure_regex_builder(test, &mut builder) { + return Ok(CompiledRegex::skip()); + } + create_matcher(&builder, pre, builder.build_many(®exes)?) + } +} + +fn run_test<A: Automaton>(re: &Regex<A>, test: &RegexTest) -> TestResult { + let input = create_input(test); + match test.additional_name() { + "is_match" => TestResult::matched(re.is_match(input.earliest(true))), + "find" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Leftmost => { + let input = + input.earliest(test.search_kind() == SearchKind::Earliest); + TestResult::matches( + re.find_iter(input) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: m.pattern().as_usize(), + span: Span { start: m.start(), end: m.end() }, + }), + ) + } + SearchKind::Overlapping => { + try_search_overlapping(re, &input).unwrap() + } + }, + "which" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Leftmost => { + // There are no "which" APIs for standard searches. + TestResult::skip() + } + SearchKind::Overlapping => { + let dfa = re.forward(); + let mut patset = PatternSet::new(dfa.pattern_len()); + dfa.try_which_overlapping_matches(&input, &mut patset) + .unwrap(); + TestResult::which(patset.iter().map(|p| p.as_usize())) + } + }, + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Configures the given regex builder with all relevant settings on the given +/// regex test. +/// +/// If the regex test has a setting that is unsupported, then this returns +/// false (implying the test should be skipped). +fn configure_regex_builder( + test: &RegexTest, + builder: &mut dfa::regex::Builder, +) -> bool { + let match_kind = match untestify_kind(test.match_kind()) { + None => return false, + Some(k) => k, + }; + + let starts = if test.anchored() { + StartKind::Anchored + } else { + StartKind::Unanchored + }; + let mut dense_config = dense::Config::new() + .start_kind(starts) + .match_kind(match_kind) + .unicode_word_boundary(true); + // When doing an overlapping search, we might try to find the start of each + // match with a custom search routine. In that case, we need to tell the + // reverse search (for the start offset) which pattern to look for. The + // only way that API works is when anchored starting states are compiled + // for each pattern. This does technically also enable it for the forward + // DFA, but we're okay with that. + if test.search_kind() == SearchKind::Overlapping { + dense_config = dense_config.starts_for_each_pattern(true); + } + + builder + .syntax(config_syntax(test)) + .thompson(config_thompson(test)) + .dense(dense_config); + true +} + +/// Configuration of a Thompson NFA compiler from a regex test. +fn config_thompson(test: &RegexTest) -> thompson::Config { + let mut lookm = regex_automata::util::look::LookMatcher::new(); + lookm.set_line_terminator(test.line_terminator()); + thompson::Config::new().utf8(test.utf8()).look_matcher(lookm) +} + +/// Configuration of the regex syntax from a regex test. +fn config_syntax(test: &RegexTest) -> syntax::Config { + syntax::Config::new() + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .utf8(test.utf8()) + .line_terminator(test.line_terminator()) +} + +/// Execute an overlapping search, and for each match found, also find its +/// overlapping starting positions. +/// +/// N.B. This routine used to be part of the crate API, but 1) it wasn't clear +/// to me how useful it was and 2) it wasn't clear to me what its semantics +/// should be. In particular, a potentially surprising footgun of this routine +/// that it is worst case *quadratic* in the size of the haystack. Namely, it's +/// possible to report a match at every position, and for every such position, +/// scan all the way to the beginning of the haystack to find the starting +/// position. Typical leftmost non-overlapping searches don't suffer from this +/// because, well, matches can't overlap. So subsequent searches after a match +/// is found don't revisit previously scanned parts of the haystack. +/// +/// Its semantics can be strange for other reasons too. For example, given +/// the regex '.*' and the haystack 'zz', the full set of overlapping matches +/// is: [0, 0], [1, 1], [0, 1], [2, 2], [1, 2], [0, 2]. The ordering of +/// those matches is quite strange, but makes sense when you think about the +/// implementation: an end offset is found left-to-right, and then one or more +/// starting offsets are found right-to-left. +/// +/// Nevertheless, we provide this routine in our test suite because it's +/// useful to test the low level DFA overlapping search and our test suite +/// is written in a way that requires starting offsets. +fn try_search_overlapping<A: Automaton>( + re: &Regex<A>, + input: &Input<'_>, +) -> Result<TestResult> { + let mut matches = vec![]; + let mut fwd_state = OverlappingState::start(); + let (fwd_dfa, rev_dfa) = (re.forward(), re.reverse()); + while let Some(end) = { + fwd_dfa.try_search_overlapping_fwd(input, &mut fwd_state)?; + fwd_state.get_match() + } { + let revsearch = input + .clone() + .range(input.start()..end.offset()) + .anchored(Anchored::Pattern(end.pattern())) + .earliest(false); + let mut rev_state = OverlappingState::start(); + while let Some(start) = { + rev_dfa.try_search_overlapping_rev(&revsearch, &mut rev_state)?; + rev_state.get_match() + } { + let span = Span { start: start.offset(), end: end.offset() }; + let mat = Match { id: end.pattern().as_usize(), span }; + matches.push(mat); + } + } + Ok(TestResult::matches(matches)) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/fuzz/dense.rs b/third_party/crates/regex-automata-0.4.18/tests/fuzz/dense.rs new file mode 100644 index 0000000..213891b --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/fuzz/dense.rs @@ -0,0 +1,52 @@ +// This test was found by a fuzzer input that crafted a way to provide +// an invalid serialization of ByteClasses that passed our verification. +// Specifically, the verification step in the deserialization of ByteClasses +// used an iterator that depends on part of the serialized bytes being correct. +// (Specifically, the encoding of the number of classes.) +#[test] +fn invalid_byte_classes() { + let data = include_bytes!( + "testdata/deserialize_dense_crash-9486fb7c8a93b12c12a62166b43d31640c0208a9", + ); + let _ = fuzz_run(data); +} + +#[test] +fn invalid_byte_classes_min() { + let data = include_bytes!( + "testdata/deserialize_dense_minimized-from-9486fb7c8a93b12c12a62166b43d31640c0208a9", + ); + let _ = fuzz_run(data); +} + +// This is the code from the fuzz target. Kind of sucks to duplicate it here, +// but this is fundamentally how we interpret the date. +fn fuzz_run(given_data: &[u8]) -> Option<()> { + use regex_automata::dfa::Automaton; + + if given_data.len() < 2 { + return None; + } + let haystack_len = usize::from(given_data[0]); + let haystack = given_data.get(1..1 + haystack_len)?; + let given_dfa_bytes = given_data.get(1 + haystack_len..)?; + + // We help the fuzzer along by adding a preamble to the bytes that should + // at least make these first parts valid. The preamble expects a very + // specific sequence of bytes, so it makes sense to just force this. + let label = "rust-regex-automata-dfa-dense\x00\x00\x00"; + assert_eq!(0, label.len() % 4); + let endianness_check = 0xFEFFu32.to_ne_bytes().to_vec(); + let version_check = 2u32.to_ne_bytes().to_vec(); + let mut dfa_bytes: Vec<u8> = vec![]; + dfa_bytes.extend(label.as_bytes()); + dfa_bytes.extend(&endianness_check); + dfa_bytes.extend(&version_check); + dfa_bytes.extend(given_dfa_bytes); + // This is the real test: checking that any input we give to + // DFA::from_bytes will never result in a panic. + let (dfa, _) = + regex_automata::dfa::dense::DFA::from_bytes(&dfa_bytes).ok()?; + let _ = dfa.try_search_fwd(®ex_automata::Input::new(haystack)); + Some(()) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/fuzz/sparse.rs b/third_party/crates/regex-automata-0.4.18/tests/fuzz/sparse.rs new file mode 100644 index 0000000..837ad10 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/fuzz/sparse.rs @@ -0,0 +1,132 @@ +// This is a regression test for a bug in how special states are handled. The +// fuzzer found a case where a state returned true for 'is_special_state' but +// *didn't* return true for 'is_dead_state', 'is_quit_state', 'is_match_state', +// 'is_start_state' or 'is_accel_state'. This in turn tripped a debug assertion +// in the core matching loop that requires 'is_special_state' being true to +// imply that one of the other routines returns true. +// +// We fixed this by adding some validation to both dense and sparse DFAs that +// checks that this property is true for every state ID in the DFA. +#[test] +fn invalid_special_state() { + let data = include_bytes!( + "testdata/deserialize_sparse_crash-a1b839d899ced76d5d7d0f78f9edb7a421505838", + ); + let _ = fuzz_run(data); +} + +// This is an interesting case where a fuzzer generated a DFA with +// a transition to a state ID that decoded as a valid state, but +// where the ID itself did not point to one of the two existing +// states for this particular DFA. This combined with marking this +// transition's state ID as special but without actually making one of the +// 'is_{dead,quit,match,start,accel}_state' predicates return true ended up +// tripping the 'debug_assert(dfa.is_quit_state(sid))' code in the search +// routine. +// +// We fixed this in alloc mode by checking that every transition points to a +// valid state ID. Technically this bug still exists in core-only mode, but +// it's not clear how to fix it. And it's worth pointing out that the search +// routine won't panic in production. It will just provide invalid results. And +// that's acceptable within the contract of DFA::from_bytes. +#[test] +fn transition_to_invalid_but_valid_state() { + let data = include_bytes!( + "testdata/deserialize_sparse_crash-dbb8172d3984e7e7d03f4b5f8bb86ecd1460eff9", + ); + let _ = fuzz_run(data); +} + +// Another one caught by the fuzzer where it generated a DFA that reported a +// start state as a match state. Since matches are always delayed by one byte, +// start states specifically cannot be match states. And indeed, the search +// code relies on this. +#[test] +fn start_state_is_not_match_state() { + let data = include_bytes!( + "testdata/deserialize_sparse_crash-0da59c0434eaf35e5a6b470fa9244bb79c72b000", + ); + let _ = fuzz_run(data); +} + +// This is variation on 'transition_to_invalid_but_valid_state', but happens +// to a start state. Namely, the fuzz data here builds a DFA with a start +// state ID that is incorrect but points to a sequence of bytes that satisfies +// state decoding validation. This errant state in turn has a non-zero number +// of transitions, and its those transitions that point to a state that does +// *not* satisfy state decoding validation. But we never checked those. So the +// fix here was to add validation of the transitions off of the start state. +#[test] +fn start_state_has_valid_transitions() { + let data = include_bytes!( + "testdata/deserialize_sparse_crash-61fd8e3003bf9d99f6c1e5a8488727eefd234b98", + ); + let _ = fuzz_run(data); +} + +// This fuzz input generated a DFA with a state whose ID was in the match state +// ID range, but where the state itself was encoded with zero pattern IDs. We +// added validation code to check this case. +#[test] +fn match_state_inconsistency() { + let data = include_bytes!( + "testdata/deserialize_sparse_crash-c383ae07ec5e191422eadc492117439011816570", + ); + let _ = fuzz_run(data); +} + +// This fuzz input generated a DFA with a state whose ID was in the accelerator +// range, but who didn't have any accelerators. This violated an invariant that +// assumes that if 'dfa.is_accel_state(sid)' returns true, then the state must +// have some accelerators. +#[test] +fn invalid_accelerators() { + let data = include_bytes!( + "testdata/deserialize_sparse_crash-d07703ceb94b10dcd9e4acb809f2051420449e2b", + ); + let _ = fuzz_run(data); +} + +// This fuzz input generated a DFA with a state whose EOI transition led to +// a quit state, which is generally considered illegal. Why? Because the EOI +// transition is defined over a special sentinel alphabet element and one +// cannot configure a DFA to "quit" on that sentinel. +#[test] +fn eoi_transition_to_quit_state() { + let data = include_bytes!( + "testdata/deserialize_sparse_crash-18cfc246f2ddfc3dfc92b0c7893178c7cf65efa9", + ); + let _ = fuzz_run(data); +} + +// This is the code from the fuzz target. Kind of sucks to duplicate it here, +// but this is fundamentally how we interpret the date. +fn fuzz_run(given_data: &[u8]) -> Option<()> { + use regex_automata::dfa::Automaton; + + if given_data.len() < 2 { + return None; + } + let haystack_len = usize::from(given_data[0]); + let haystack = given_data.get(1..1 + haystack_len)?; + let given_dfa_bytes = given_data.get(1 + haystack_len..)?; + + // We help the fuzzer along by adding a preamble to the bytes that should + // at least make these first parts valid. The preamble expects a very + // specific sequence of bytes, so it makes sense to just force this. + let label = "rust-regex-automata-dfa-sparse\x00\x00"; + assert_eq!(0, label.len() % 4); + let endianness_check = 0xFEFFu32.to_ne_bytes().to_vec(); + let version_check = 2u32.to_ne_bytes().to_vec(); + let mut dfa_bytes: Vec<u8> = vec![]; + dfa_bytes.extend(label.as_bytes()); + dfa_bytes.extend(&endianness_check); + dfa_bytes.extend(&version_check); + dfa_bytes.extend(given_dfa_bytes); + // This is the real test: checking that any input we give to + // DFA::from_bytes will never result in a panic. + let (dfa, _) = + regex_automata::dfa::sparse::DFA::from_bytes(&dfa_bytes).ok()?; + let _ = dfa.try_search_fwd(®ex_automata::Input::new(haystack)); + Some(()) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/gen/dense/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/gen/dense/mod.rs new file mode 100644 index 0000000..b4365d4 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/gen/dense/mod.rs @@ -0,0 +1,22 @@ +use regex_automata::{Input, Match}; + +mod multi_pattern_v2; + +#[test] +fn multi_pattern_v2() { + use multi_pattern_v2::MULTI_PATTERN_V2 as RE; + + assert_eq!(Some(Match::must(0, 0..4)), RE.find("abcd")); + assert_eq!(Some(Match::must(0, 2..6)), RE.find("@ abcd @")); + assert_eq!(Some(Match::must(1, 0..6)), RE.find("@abcd@")); + assert_eq!(Some(Match::must(0, 1..5)), RE.find("\nabcd\n")); + assert_eq!(Some(Match::must(0, 1..5)), RE.find("\nabcd wxyz\n")); + assert_eq!(Some(Match::must(1, 1..7)), RE.find("\n@abcd@\n")); + assert_eq!(Some(Match::must(2, 0..6)), RE.find("@abcd@\r\n")); + assert_eq!(Some(Match::must(1, 2..8)), RE.find("\r\n@abcd@")); + assert_eq!(Some(Match::must(2, 2..8)), RE.find("\r\n@abcd@\r\n")); + + // Fails because we have heuristic support for Unicode word boundaries + // enabled. + assert!(RE.try_search(&Input::new(b"\xFF@abcd@\xFF")).is_err()); +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/gen/dense/multi_pattern_v2.rs b/third_party/crates/regex-automata-0.4.18/tests/gen/dense/multi_pattern_v2.rs new file mode 100644 index 0000000..a95fd20 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/gen/dense/multi_pattern_v2.rs @@ -0,0 +1,43 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// regex-cli generate serialize dense regex MULTI_PATTERN_V2 tests/gen/dense/ --rustfmt --safe --starts-for-each-pattern --specialize-start-states --start-kind both --unicode-word-boundary --minimize \b[a-zA-Z]+\b (?m)^\S+$ (?Rm)^\S+$ +// +// regex-cli 0.0.1 is available on crates.io. + +use regex_automata::{ + dfa::{dense::DFA, regex::Regex}, + util::{lazy::Lazy, wire::AlignAs}, +}; + +pub static MULTI_PATTERN_V2: Lazy<Regex<DFA<&'static [u32]>>> = + Lazy::new(|| { + let dfafwd = { + static ALIGNED: &AlignAs<[u8], u32> = &AlignAs { + _align: [], + #[cfg(target_endian = "big")] + bytes: *include_bytes!("multi_pattern_v2_fwd.bigendian.dfa"), + #[cfg(target_endian = "little")] + bytes: *include_bytes!( + "multi_pattern_v2_fwd.littleendian.dfa" + ), + }; + DFA::from_bytes(&ALIGNED.bytes) + .expect("serialized forward DFA should be valid") + .0 + }; + let dfarev = { + static ALIGNED: &AlignAs<[u8], u32> = &AlignAs { + _align: [], + #[cfg(target_endian = "big")] + bytes: *include_bytes!("multi_pattern_v2_rev.bigendian.dfa"), + #[cfg(target_endian = "little")] + bytes: *include_bytes!( + "multi_pattern_v2_rev.littleendian.dfa" + ), + }; + DFA::from_bytes(&ALIGNED.bytes) + .expect("serialized reverse DFA should be valid") + .0 + }; + Regex::builder().build_from_dfas(dfafwd, dfarev) + }); diff --git a/third_party/crates/regex-automata-0.4.18/tests/gen/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/gen/mod.rs new file mode 100644 index 0000000..960cb42 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/gen/mod.rs @@ -0,0 +1,2 @@ +mod dense; +mod sparse; diff --git a/third_party/crates/regex-automata-0.4.18/tests/gen/sparse/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/gen/sparse/mod.rs new file mode 100644 index 0000000..b4365d4 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/gen/sparse/mod.rs @@ -0,0 +1,22 @@ +use regex_automata::{Input, Match}; + +mod multi_pattern_v2; + +#[test] +fn multi_pattern_v2() { + use multi_pattern_v2::MULTI_PATTERN_V2 as RE; + + assert_eq!(Some(Match::must(0, 0..4)), RE.find("abcd")); + assert_eq!(Some(Match::must(0, 2..6)), RE.find("@ abcd @")); + assert_eq!(Some(Match::must(1, 0..6)), RE.find("@abcd@")); + assert_eq!(Some(Match::must(0, 1..5)), RE.find("\nabcd\n")); + assert_eq!(Some(Match::must(0, 1..5)), RE.find("\nabcd wxyz\n")); + assert_eq!(Some(Match::must(1, 1..7)), RE.find("\n@abcd@\n")); + assert_eq!(Some(Match::must(2, 0..6)), RE.find("@abcd@\r\n")); + assert_eq!(Some(Match::must(1, 2..8)), RE.find("\r\n@abcd@")); + assert_eq!(Some(Match::must(2, 2..8)), RE.find("\r\n@abcd@\r\n")); + + // Fails because we have heuristic support for Unicode word boundaries + // enabled. + assert!(RE.try_search(&Input::new(b"\xFF@abcd@\xFF")).is_err()); +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/gen/sparse/multi_pattern_v2.rs b/third_party/crates/regex-automata-0.4.18/tests/gen/sparse/multi_pattern_v2.rs new file mode 100644 index 0000000..911e3f5 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/gen/sparse/multi_pattern_v2.rs @@ -0,0 +1,37 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// regex-cli generate serialize sparse regex MULTI_PATTERN_V2 regex-automata/tests/gen/sparse/ --rustfmt --safe --starts-for-each-pattern --specialize-start-states --start-kind both --unicode-word-boundary --minimize \b[a-zA-Z]+\b (?m)^\S+$ (?Rm)^\S+$ +// +// regex-cli 0.0.1 is available on crates.io. + +use regex_automata::{ + dfa::{regex::Regex, sparse::DFA}, + util::lazy::Lazy, +}; + +pub static MULTI_PATTERN_V2: Lazy<Regex<DFA<&'static [u8]>>> = + Lazy::new(|| { + let dfafwd = { + #[cfg(target_endian = "big")] + static BYTES: &'static [u8] = + include_bytes!("multi_pattern_v2_fwd.bigendian.dfa"); + #[cfg(target_endian = "little")] + static BYTES: &'static [u8] = + include_bytes!("multi_pattern_v2_fwd.littleendian.dfa"); + DFA::from_bytes(BYTES) + .expect("serialized forward DFA should be valid") + .0 + }; + let dfarev = { + #[cfg(target_endian = "big")] + static BYTES: &'static [u8] = + include_bytes!("multi_pattern_v2_rev.bigendian.dfa"); + #[cfg(target_endian = "little")] + static BYTES: &'static [u8] = + include_bytes!("multi_pattern_v2_rev.littleendian.dfa"); + DFA::from_bytes(BYTES) + .expect("serialized reverse DFA should be valid") + .0 + }; + Regex::builder().build_from_dfas(dfafwd, dfarev) + }); diff --git a/third_party/crates/regex-automata-0.4.18/tests/hybrid/api.rs b/third_party/crates/regex-automata-0.4.18/tests/hybrid/api.rs new file mode 100644 index 0000000..07c947a --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/hybrid/api.rs @@ -0,0 +1,201 @@ +use std::error::Error; + +use regex_automata::{ + hybrid::dfa::{OverlappingState, DFA}, + nfa::thompson, + HalfMatch, Input, MatchError, +}; + +// A tagged start state can be reached after the DFA has recorded a match. A +// prefilter must not skip to a new candidate while that match is pending. +#[test] +#[cfg(feature = "perf-literal-multisubstring")] +fn prefilter_does_not_skip_pending_match() -> Result<(), Box<dyn Error>> { + use regex_automata::{util::prefilter::Prefilter, Anchored, MatchKind}; + + let pre = Prefilter::new(MatchKind::LeftmostFirst, &["ab", "c"]) + .expect("prefilter"); + let dfa = DFA::builder() + .configure( + DFA::config().prefilter(Some(pre)).starts_for_each_pattern(true), + ) + .build(r"(?:ab|c)*c")?; + let mut cache = dfa.create_cache(); + let haystack = "cabac"; + let expected = Some(HalfMatch::must(0, 1)); + + // Cache the anchored start state before the equivalent state is reached + // via an ordinary transition. Otherwise, that state is left untagged and + // the prefilter bug remains hidden. + let input = Input::new(haystack).anchored(Anchored::Yes); + assert_eq!(expected, dfa.try_search_fwd(&mut cache, &input)?); + assert_eq!( + expected, + dfa.try_search_fwd(&mut cache, &Input::new(haystack))?, + ); + Ok(()) +} + +// Tests that too many cache resets cause the lazy DFA to quit. +// +// We only test this on 64-bit because the test is gingerly crafted based on +// implementation details of cache sizes. It's not a great test because of +// that, but it does check some interesting properties around how positions are +// reported when a search "gives up." +// +// NOTE: If you change something in lazy DFA implementation that causes this +// test to fail by reporting different "gave up" positions, then it's generally +// okay to update the positions in the test below as long as you're sure your +// changes are correct. Namely, it is expected that if there are changes in the +// cache size (or changes in how big things are inside the cache), then its +// utilization may change slightly and thus impact where a search gives up. +// Precisely where a search gives up is not an API guarantee, so changing the +// offsets here is OK. +#[test] +#[cfg(target_pointer_width = "64")] +#[cfg(not(miri))] +fn too_many_cache_resets_cause_quit() -> Result<(), Box<dyn Error>> { + // This is a carefully chosen regex. The idea is to pick one that requires + // some decent number of states (hence the bounded repetition). But we + // specifically choose to create a class with an ASCII letter and a + // non-ASCII letter so that we can check that no new states are created + // once the cache is full. Namely, if we fill up the cache on a haystack + // of 'a's, then in order to match one 'β', a new state will need to be + // created since a 'β' is encoded with multiple bytes. + // + // So we proceed by "filling" up the cache by searching a haystack of just + // 'a's. The cache won't have enough room to add enough states to find the + // match (because of the bounded repetition), which should result in it + // giving up before it finds a match. + // + // Since there's now no more room to create states, we search a haystack + // of 'β' and confirm that it gives up immediately. + let pattern = r"[aβ]{99}"; + let dfa = DFA::builder() + .configure( + // Configure it so that we have the minimum cache capacity + // possible. And that if any resets occur, the search quits. + DFA::config() + .skip_cache_capacity_check(true) + .cache_capacity(0) + .minimum_cache_clear_count(Some(0)), + ) + .thompson(thompson::NFA::config()) + .build(pattern)?; + let mut cache = dfa.create_cache(); + + let haystack = "a".repeat(101).into_bytes(); + let err = MatchError::gave_up(24); + // Notice that we make the same amount of progress in each search! That's + // because the cache is reused and already has states to handle the first + // N bytes. + assert_eq!( + Err(err.clone()), + dfa.try_search_fwd(&mut cache, &Input::new(&haystack)) + ); + assert_eq!( + Err(err.clone()), + dfa.try_search_overlapping_fwd( + &mut cache, + &Input::new(&haystack), + &mut OverlappingState::start() + ), + ); + + let haystack = "β".repeat(101).into_bytes(); + let err = MatchError::gave_up(2); + assert_eq!( + Err(err), + dfa.try_search_fwd(&mut cache, &Input::new(&haystack)) + ); + // no need to test that other find routines quit, since we did that above + + // OK, if we reset the cache, then we should be able to create more states + // and make more progress with searching for betas. + cache.reset(&dfa); + let err = MatchError::gave_up(26); + assert_eq!( + Err(err), + dfa.try_search_fwd(&mut cache, &Input::new(&haystack)) + ); + + // ... switching back to ASCII still makes progress since it just needs to + // set transitions on existing states! + let haystack = "a".repeat(101).into_bytes(); + let err = MatchError::gave_up(13); + assert_eq!( + Err(err), + dfa.try_search_fwd(&mut cache, &Input::new(&haystack)) + ); + + Ok(()) +} + +// Tests that quit bytes in the forward direction work correctly. +#[test] +fn quit_fwd() -> Result<(), Box<dyn Error>> { + let dfa = DFA::builder() + .configure(DFA::config().quit(b'x', true)) + .build("[[:word:]]+$")?; + let mut cache = dfa.create_cache(); + + assert_eq!( + dfa.try_search_fwd(&mut cache, &Input::new("abcxyz")), + Err(MatchError::quit(b'x', 3)), + ); + assert_eq!( + dfa.try_search_overlapping_fwd( + &mut cache, + &Input::new(b"abcxyz"), + &mut OverlappingState::start() + ), + Err(MatchError::quit(b'x', 3)), + ); + + Ok(()) +} + +// Tests that quit bytes in the reverse direction work correctly. +#[test] +fn quit_rev() -> Result<(), Box<dyn Error>> { + let dfa = DFA::builder() + .configure(DFA::config().quit(b'x', true)) + .thompson(thompson::Config::new().reverse(true)) + .build("^[[:word:]]+")?; + let mut cache = dfa.create_cache(); + + assert_eq!( + dfa.try_search_rev(&mut cache, &Input::new("abcxyz")), + Err(MatchError::quit(b'x', 3)), + ); + + Ok(()) +} + +// Tests that if we heuristically enable Unicode word boundaries but then +// instruct that a non-ASCII byte should NOT be a quit byte, then the builder +// will panic. +#[test] +#[should_panic] +fn quit_panics() { + DFA::config().unicode_word_boundary(true).quit(b'\xFF', false); +} + +// This tests an intesting case where even if the Unicode word boundary option +// is disabled, setting all non-ASCII bytes to be quit bytes will cause Unicode +// word boundaries to be enabled. +#[test] +fn unicode_word_implicitly_works() -> Result<(), Box<dyn Error>> { + let mut config = DFA::config(); + for b in 0x80..=0xFF { + config = config.quit(b, true); + } + let dfa = DFA::builder().configure(config).build(r"\b")?; + let mut cache = dfa.create_cache(); + let expected = HalfMatch::must(0, 1); + assert_eq!( + Ok(Some(expected)), + dfa.try_search_fwd(&mut cache, &Input::new(" a")), + ); + Ok(()) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/hybrid/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/hybrid/mod.rs new file mode 100644 index 0000000..36667d0 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/hybrid/mod.rs @@ -0,0 +1,3 @@ +mod api; +#[cfg(not(miri))] +mod suite; diff --git a/third_party/crates/regex-automata-0.4.18/tests/hybrid/suite.rs b/third_party/crates/regex-automata-0.4.18/tests/hybrid/suite.rs new file mode 100644 index 0000000..f0c3ebd --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/hybrid/suite.rs @@ -0,0 +1,347 @@ +use { + anyhow::Result, + regex_automata::{ + hybrid::{ + dfa::{OverlappingState, DFA}, + regex::{self, Regex}, + }, + nfa::thompson, + util::{prefilter::Prefilter, syntax}, + Anchored, Input, PatternSet, + }, + regex_test::{ + CompiledRegex, Match, RegexTest, SearchKind, Span, TestResult, + TestRunner, + }, +}; + +use crate::{create_input, suite, untestify_kind}; + +const EXPANSIONS: &[&str] = &["is_match", "find", "which"]; + +/// Tests the default configuration of the hybrid NFA/DFA. +#[test] +fn default() -> Result<()> { + let builder = Regex::builder(); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + // Without NFA shrinking, this test blows the default cache capacity. + .blacklist("expensive/regression-many-repeat-no-stack-overflow") + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the hybrid NFA/DFA with prefilters enabled. +#[test] +fn prefilter() -> Result<()> { + let my_compiler = |test: &RegexTest, regexes: &[String]| { + // Parse regexes as HIRs so we can get literals to build a prefilter. + let mut hirs = vec![]; + for pattern in regexes.iter() { + hirs.push(syntax::parse_with(pattern, &config_syntax(test))?); + } + let kind = match untestify_kind(test.match_kind()) { + None => return Ok(CompiledRegex::skip()), + Some(kind) => kind, + }; + let pre = Prefilter::from_hirs_prefix(kind, &hirs); + let mut builder = Regex::builder(); + builder.dfa(DFA::config().prefilter(pre)); + compiler(builder)(test, regexes) + }; + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + // Without NFA shrinking, this test blows the default cache capacity. + .blacklist("expensive/regression-many-repeat-no-stack-overflow") + .test_iter(suite()?.iter(), my_compiler) + .assert(); + Ok(()) +} + +/// Tests the hybrid NFA/DFA with NFA shrinking enabled. +/// +/// This is *usually* not the configuration one wants for a lazy DFA. NFA +/// shrinking is mostly only advantageous when building a full DFA since it +/// can sharply decrease the amount of time determinization takes. But NFA +/// shrinking is itself otherwise fairly expensive currently. Since a lazy DFA +/// has no compilation time (other than for building the NFA of course) before +/// executing a search, it's usually worth it to forgo NFA shrinking. +/// +/// Nevertheless, we test to make sure everything is OK with NFA shrinking. As +/// a bonus, there are some tests we don't need to skip because they now fit in +/// the default cache capacity. +#[test] +fn nfa_shrink() -> Result<()> { + let mut builder = Regex::builder(); + builder.thompson(thompson::Config::new().shrink(true)); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the hybrid NFA/DFA when 'starts_for_each_pattern' is enabled for all +/// tests. +#[test] +fn starts_for_each_pattern() -> Result<()> { + let mut builder = Regex::builder(); + builder.dfa(DFA::config().starts_for_each_pattern(true)); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + // Without NFA shrinking, this test blows the default cache capacity. + .blacklist("expensive/regression-many-repeat-no-stack-overflow") + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the hybrid NFA/DFA when 'specialize_start_states' is enabled. +#[test] +fn specialize_start_states() -> Result<()> { + let mut builder = Regex::builder(); + builder.dfa(DFA::config().specialize_start_states(true)); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + // Without NFA shrinking, this test blows the default cache capacity. + .blacklist("expensive/regression-many-repeat-no-stack-overflow") + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the hybrid NFA/DFA when byte classes are disabled. +/// +/// N.B. Disabling byte classes doesn't avoid any indirection at search time. +/// All it does is cause every byte value to be its own distinct equivalence +/// class. +#[test] +fn no_byte_classes() -> Result<()> { + let mut builder = Regex::builder(); + builder.dfa(DFA::config().byte_classes(false)); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + // Without NFA shrinking, this test blows the default cache capacity. + .blacklist("expensive/regression-many-repeat-no-stack-overflow") + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests that hybrid NFA/DFA never clears its cache for any test with the +/// default capacity. +/// +/// N.B. If a regex suite test is added that causes the cache to be cleared, +/// then this should just skip that test. (Which can be done by calling the +/// 'blacklist' method on 'TestRunner'.) +#[test] +fn no_cache_clearing() -> Result<()> { + let mut builder = Regex::builder(); + builder.dfa(DFA::config().minimum_cache_clear_count(Some(0))); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + // Without NFA shrinking, this test blows the default cache capacity. + .blacklist("expensive/regression-many-repeat-no-stack-overflow") + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the hybrid NFA/DFA when the minimum cache capacity is set. +#[test] +fn min_cache_capacity() -> Result<()> { + let mut builder = Regex::builder(); + builder + .dfa(DFA::config().cache_capacity(0).skip_cache_capacity_check(true)); + TestRunner::new()? + .expand(EXPANSIONS, |t| t.compiles()) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +fn compiler( + mut builder: regex::Builder, +) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> { + move |test, regexes| { + // Parse regexes as HIRs for some analysis below. + let mut hirs = vec![]; + for pattern in regexes.iter() { + hirs.push(syntax::parse_with(pattern, &config_syntax(test))?); + } + + // Check if our regex contains things that aren't supported by DFAs. + // That is, Unicode word boundaries when searching non-ASCII text. + if !test.haystack().is_ascii() { + for hir in hirs.iter() { + if hir.properties().look_set().contains_word_unicode() { + return Ok(CompiledRegex::skip()); + } + } + } + if !configure_regex_builder(test, &mut builder) { + return Ok(CompiledRegex::skip()); + } + let re = builder.build_many(®exes)?; + let mut cache = re.create_cache(); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, &mut cache, test) + })) + } +} + +fn run_test( + re: &Regex, + cache: &mut regex::Cache, + test: &RegexTest, +) -> TestResult { + let input = create_input(test); + match test.additional_name() { + "is_match" => { + TestResult::matched(re.is_match(cache, input.earliest(true))) + } + "find" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Leftmost => { + let input = + input.earliest(test.search_kind() == SearchKind::Earliest); + TestResult::matches( + re.find_iter(cache, input) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: m.pattern().as_usize(), + span: Span { start: m.start(), end: m.end() }, + }), + ) + } + SearchKind::Overlapping => { + try_search_overlapping(re, cache, &input).unwrap() + } + }, + "which" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Leftmost => { + // There are no "which" APIs for standard searches. + TestResult::skip() + } + SearchKind::Overlapping => { + let dfa = re.forward(); + let cache = cache.as_parts_mut().0; + let mut patset = PatternSet::new(dfa.pattern_len()); + dfa.try_which_overlapping_matches(cache, &input, &mut patset) + .unwrap(); + TestResult::which(patset.iter().map(|p| p.as_usize())) + } + }, + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Configures the given regex builder with all relevant settings on the given +/// regex test. +/// +/// If the regex test has a setting that is unsupported, then this returns +/// false (implying the test should be skipped). +fn configure_regex_builder( + test: &RegexTest, + builder: &mut regex::Builder, +) -> bool { + let match_kind = match untestify_kind(test.match_kind()) { + None => return false, + Some(k) => k, + }; + + let mut dfa_config = + DFA::config().match_kind(match_kind).unicode_word_boundary(true); + // When doing an overlapping search, we might try to find the start of each + // match with a custom search routine. In that case, we need to tell the + // reverse search (for the start offset) which pattern to look for. The + // only way that API works is when anchored starting states are compiled + // for each pattern. This does technically also enable it for the forward + // DFA, but we're okay with that. + if test.search_kind() == SearchKind::Overlapping { + dfa_config = dfa_config.starts_for_each_pattern(true); + } + builder + .syntax(config_syntax(test)) + .thompson(config_thompson(test)) + .dfa(dfa_config); + true +} + +/// Configuration of a Thompson NFA compiler from a regex test. +fn config_thompson(test: &RegexTest) -> thompson::Config { + let mut lookm = regex_automata::util::look::LookMatcher::new(); + lookm.set_line_terminator(test.line_terminator()); + thompson::Config::new().utf8(test.utf8()).look_matcher(lookm) +} + +/// Configuration of the regex parser from a regex test. +fn config_syntax(test: &RegexTest) -> syntax::Config { + syntax::Config::new() + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .utf8(test.utf8()) + .line_terminator(test.line_terminator()) +} + +/// Execute an overlapping search, and for each match found, also find its +/// overlapping starting positions. +/// +/// N.B. This routine used to be part of the crate API, but 1) it wasn't clear +/// to me how useful it was and 2) it wasn't clear to me what its semantics +/// should be. In particular, a potentially surprising footgun of this routine +/// that it is worst case *quadratic* in the size of the haystack. Namely, it's +/// possible to report a match at every position, and for every such position, +/// scan all the way to the beginning of the haystack to find the starting +/// position. Typical leftmost non-overlapping searches don't suffer from this +/// because, well, matches can't overlap. So subsequent searches after a match +/// is found don't revisit previously scanned parts of the haystack. +/// +/// Its semantics can be strange for other reasons too. For example, given +/// the regex '.*' and the haystack 'zz', the full set of overlapping matches +/// is: [0, 0], [1, 1], [0, 1], [2, 2], [1, 2], [0, 2]. The ordering of +/// those matches is quite strange, but makes sense when you think about the +/// implementation: an end offset is found left-to-right, and then one or more +/// starting offsets are found right-to-left. +/// +/// Nevertheless, we provide this routine in our test suite because it's +/// useful to test the low level DFA overlapping search and our test suite +/// is written in a way that requires starting offsets. +fn try_search_overlapping( + re: &Regex, + cache: &mut regex::Cache, + input: &Input<'_>, +) -> Result<TestResult> { + let mut matches = vec![]; + let mut fwd_state = OverlappingState::start(); + let (fwd_dfa, rev_dfa) = (re.forward(), re.reverse()); + let (fwd_cache, rev_cache) = cache.as_parts_mut(); + while let Some(end) = { + fwd_dfa.try_search_overlapping_fwd( + fwd_cache, + input, + &mut fwd_state, + )?; + fwd_state.get_match() + } { + let revsearch = input + .clone() + .range(input.start()..end.offset()) + .anchored(Anchored::Pattern(end.pattern())) + .earliest(false); + let mut rev_state = OverlappingState::start(); + while let Some(start) = { + rev_dfa.try_search_overlapping_rev( + rev_cache, + &revsearch, + &mut rev_state, + )?; + rev_state.get_match() + } { + let span = Span { start: start.offset(), end: end.offset() }; + let mat = Match { id: end.pattern().as_usize(), span }; + matches.push(mat); + } + } + Ok(TestResult::matches(matches)) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/lib.rs b/third_party/crates/regex-automata-0.4.18/tests/lib.rs new file mode 100644 index 0000000..67c979a --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/lib.rs @@ -0,0 +1,115 @@ +// We have a similar config in the regex-automata crate root. Basically, it is +// just too annoying to deal with dead code when a subset of features is +// enabled. +#![cfg_attr( + not(all( + feature = "std", + feature = "nfa", + feature = "dfa", + feature = "hybrid", + feature = "perf-literal-substring", + feature = "perf-literal-multisubstring", + )), + allow(dead_code, unused_imports, unused_variables) +)] +// Similar deal with Miri. Just let dead code warnings be. +#![cfg_attr(miri, allow(dead_code, unused_imports, unused_variables))] + +#[cfg(any(feature = "dfa-search", feature = "dfa-onepass"))] +mod dfa; +#[cfg(feature = "dfa-search")] +mod fuzz; +#[cfg(feature = "dfa-search")] +mod gen; +#[cfg(feature = "hybrid")] +mod hybrid; +#[cfg(feature = "meta")] +mod meta; +#[cfg(any(feature = "nfa-backtrack", feature = "nfa-pikevm"))] +mod nfa; + +fn suite() -> anyhow::Result<regex_test::RegexTests> { + let _ = env_logger::try_init(); + + let mut tests = regex_test::RegexTests::new(); + macro_rules! load { + ($name:expr) => {{ + const DATA: &[u8] = + include_bytes!(concat!("../../testdata/", $name, ".toml")); + tests.load_slice($name, DATA)?; + }}; + } + + load!("anchored"); + load!("bytes"); + load!("crazy"); + load!("crlf"); + load!("earliest"); + load!("empty"); + load!("expensive"); + load!("flags"); + load!("iter"); + load!("leftmost-all"); + load!("line-terminator"); + load!("misc"); + load!("multiline"); + load!("no-unicode"); + load!("overlapping"); + load!("regression"); + load!("set"); + load!("substring"); + load!("unicode"); + load!("utf8"); + load!("word-boundary"); + load!("word-boundary-special"); + load!("fowler/basic"); + load!("fowler/nullsubexpr"); + load!("fowler/repetition"); + + Ok(tests) +} + +/// Configure a regex_automata::Input with the given test configuration. +fn create_input<'h>( + test: &'h regex_test::RegexTest, +) -> regex_automata::Input<'h> { + use regex_automata::Anchored; + + let bounds = test.bounds(); + let anchored = if test.anchored() { Anchored::Yes } else { Anchored::No }; + regex_automata::Input::new(test.haystack()) + .range(bounds.start..bounds.end) + .anchored(anchored) +} + +/// Convert capture matches into the test suite's capture values. +/// +/// The given captures must represent a valid match, where the first capturing +/// group has a non-None span. Otherwise this panics. +fn testify_captures( + caps: ®ex_automata::util::captures::Captures, +) -> regex_test::Captures { + assert!(caps.is_match(), "expected captures to represent a match"); + let spans = caps.iter().map(|group| { + group.map(|m| regex_test::Span { start: m.start, end: m.end }) + }); + // These unwraps are OK because we assume our 'caps' represents a match, + // and a match always gives a non-zero number of groups with the first + // group being non-None. + regex_test::Captures::new(caps.pattern().unwrap().as_usize(), spans) + .unwrap() +} + +/// Convert a test harness match kind to a regex-automata match kind. If +/// regex-automata doesn't support the harness kind, then `None` is returned. +fn untestify_kind( + kind: regex_test::MatchKind, +) -> Option<regex_automata::MatchKind> { + match kind { + regex_test::MatchKind::All => Some(regex_automata::MatchKind::All), + regex_test::MatchKind::LeftmostFirst => { + Some(regex_automata::MatchKind::LeftmostFirst) + } + regex_test::MatchKind::LeftmostLongest => None, + } +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/meta/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/meta/mod.rs new file mode 100644 index 0000000..9d6ab47 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/meta/mod.rs @@ -0,0 +1,2 @@ +#[cfg(not(miri))] +mod suite; diff --git a/third_party/crates/regex-automata-0.4.18/tests/meta/suite.rs b/third_party/crates/regex-automata-0.4.18/tests/meta/suite.rs new file mode 100644 index 0000000..2c3de64 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/meta/suite.rs @@ -0,0 +1,200 @@ +use { + anyhow::Result, + regex_automata::{ + meta::{self, Regex}, + util::syntax, + MatchKind, PatternSet, + }, + regex_test::{ + CompiledRegex, Match, RegexTest, SearchKind, Span, TestResult, + TestRunner, + }, +}; + +use crate::{create_input, suite, testify_captures}; + +const BLACKLIST: &[&str] = &[ + // These 'earliest' tests are blacklisted because the meta searcher doesn't + // give the same offsets that the test expects. This is legal because the + // 'earliest' routines don't guarantee a particular match offset other + // than "the earliest the regex engine can report a match." Some regex + // engines will quit earlier than others. The backtracker, for example, + // can't really quit before finding the full leftmost-first match. Many of + // the literal searchers also don't have the ability to quit fully or it's + // otherwise not worth doing. (A literal searcher not quitting as early as + // possible usually means looking at a few more bytes. That's no biggie.) + "earliest/", +]; + +/// Tests the default configuration of the meta regex engine. +#[test] +fn default() -> Result<()> { + let builder = Regex::builder(); + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "find", "captures"], |test| test.compiles()) + .blacklist_iter(BLACKLIST) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the default configuration minus the full DFA. +#[test] +fn no_dfa() -> Result<()> { + let mut builder = Regex::builder(); + builder.configure(Regex::config().dfa(false)); + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "find", "captures"], |test| test.compiles()) + .blacklist_iter(BLACKLIST) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the default configuration minus the full DFA and lazy DFA. +#[test] +fn no_dfa_hybrid() -> Result<()> { + let mut builder = Regex::builder(); + builder.configure(Regex::config().dfa(false).hybrid(false)); + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "find", "captures"], |test| test.compiles()) + .blacklist_iter(BLACKLIST) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the default configuration minus the full DFA, lazy DFA and one-pass +/// DFA. +#[test] +fn no_dfa_hybrid_onepass() -> Result<()> { + let mut builder = Regex::builder(); + builder.configure(Regex::config().dfa(false).hybrid(false).onepass(false)); + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "find", "captures"], |test| test.compiles()) + .blacklist_iter(BLACKLIST) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +/// Tests the default configuration minus the full DFA, lazy DFA, one-pass +/// DFA and backtracker. +#[test] +fn no_dfa_hybrid_onepass_backtrack() -> Result<()> { + let mut builder = Regex::builder(); + builder.configure( + Regex::config() + .dfa(false) + .hybrid(false) + .onepass(false) + .backtrack(false), + ); + let mut runner = TestRunner::new()?; + runner + .expand(&["is_match", "find", "captures"], |test| test.compiles()) + .blacklist_iter(BLACKLIST) + .test_iter(suite()?.iter(), compiler(builder)) + .assert(); + Ok(()) +} + +fn compiler( + mut builder: meta::Builder, +) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> { + move |test, regexes| { + if !configure_meta_builder(test, &mut builder) { + return Ok(CompiledRegex::skip()); + } + let re = builder.build_many(®exes)?; + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, test) + })) + } +} + +fn run_test(re: &Regex, test: &RegexTest) -> TestResult { + let input = create_input(test); + match test.additional_name() { + "is_match" => TestResult::matched(re.is_match(input)), + "find" => match test.search_kind() { + SearchKind::Earliest => TestResult::matches( + re.find_iter(input.earliest(true)) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: m.pattern().as_usize(), + span: Span { start: m.start(), end: m.end() }, + }), + ), + SearchKind::Leftmost => TestResult::matches( + re.find_iter(input) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: m.pattern().as_usize(), + span: Span { start: m.start(), end: m.end() }, + }), + ), + SearchKind::Overlapping => { + let mut patset = PatternSet::new(re.pattern_len()); + re.which_overlapping_matches(&input, &mut patset); + TestResult::which(patset.iter().map(|p| p.as_usize())) + } + }, + "captures" => match test.search_kind() { + SearchKind::Earliest => { + let it = re + .captures_iter(input.earliest(true)) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|caps| testify_captures(&caps)); + TestResult::captures(it) + } + SearchKind::Leftmost => { + let it = re + .captures_iter(input) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|caps| testify_captures(&caps)); + TestResult::captures(it) + } + SearchKind::Overlapping => { + // There is no overlapping regex API that supports captures. + TestResult::skip() + } + }, + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Configures the given regex builder with all relevant settings on the given +/// regex test. +/// +/// If the regex test has a setting that is unsupported, then this returns +/// false (implying the test should be skipped). +fn configure_meta_builder( + test: &RegexTest, + builder: &mut meta::Builder, +) -> bool { + let match_kind = match test.match_kind() { + regex_test::MatchKind::All => MatchKind::All, + regex_test::MatchKind::LeftmostFirst => MatchKind::LeftmostFirst, + regex_test::MatchKind::LeftmostLongest => return false, + }; + let meta_config = Regex::config() + .match_kind(match_kind) + .utf8_empty(test.utf8()) + .line_terminator(test.line_terminator()); + builder.configure(meta_config).syntax(config_syntax(test)); + true +} + +/// Configuration of the regex parser from a regex test. +fn config_syntax(test: &RegexTest) -> syntax::Config { + syntax::Config::new() + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .utf8(test.utf8()) + .line_terminator(test.line_terminator()) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/nfa/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/nfa/mod.rs new file mode 100644 index 0000000..3268621 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/nfa/mod.rs @@ -0,0 +1 @@ +mod thompson; diff --git a/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/mod.rs new file mode 100644 index 0000000..5a71144 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/mod.rs @@ -0,0 +1,3 @@ +mod regression; +#[cfg(not(miri))] +mod suite; diff --git a/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/regression.rs b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/regression.rs new file mode 100644 index 0000000..b392f7a --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/regression.rs @@ -0,0 +1,20 @@ +// Tests that we can call the backtracker with more slots +// than is actually in the compiled regex. +#[test] +fn too_many_slots_normal_pattern() { + use regex_automata::{ + nfa::thompson::backtrack::BoundedBacktracker, + util::primitives::NonMaxUsize, Anchored, Input, + }; + + let expr = BoundedBacktracker::new(r"abc").unwrap(); + let s = "abc"; + let input = Input::new(s).span(0..s.len()).anchored(Anchored::Yes); + + let mut cache = expr.create_cache(); + let mut slots: Vec<Option<NonMaxUsize>> = vec![None; 4]; + let pid = expr.try_search_slots(&mut cache, &input, &mut slots).unwrap(); + assert_eq!(pid, Some(regex_automata::PatternID::must(0))); + assert_eq!(slots[0], Some(NonMaxUsize::new(0).unwrap())); + assert_eq!(slots[1], Some(NonMaxUsize::new(3).unwrap())); +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/suite.rs b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/suite.rs new file mode 100644 index 0000000..c6f3b9f --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/backtrack/suite.rs @@ -0,0 +1,213 @@ +use { + anyhow::Result, + regex_automata::{ + nfa::thompson::{ + self, + backtrack::{self, BoundedBacktracker}, + NFA, + }, + util::{prefilter::Prefilter, syntax}, + Input, + }, + regex_test::{ + CompiledRegex, Match, MatchKind, RegexTest, SearchKind, Span, + TestResult, TestRunner, + }, +}; + +use crate::{create_input, suite, testify_captures}; + +/// Tests the default configuration of the bounded backtracker. +#[test] +fn default() -> Result<()> { + let builder = BoundedBacktracker::builder(); + let mut runner = TestRunner::new()?; + runner.expand(&["is_match", "find", "captures"], |test| test.compiles()); + // At the time of writing, every regex search in the test suite fits + // into the backtracker's default visited capacity (except for the + // blacklisted tests below). If regexes are added that blow that capacity, + // then they should be blacklisted here. A tempting alternative is to + // automatically skip them by checking the haystack length against + // BoundedBacktracker::max_haystack_len, but that could wind up hiding + // interesting failure modes. e.g., If the visited capacity is somehow + // wrong or smaller than it should be. + runner.blacklist("expensive/backtrack-blow-visited-capacity"); + runner.test_iter(suite()?.iter(), compiler(builder)).assert(); + Ok(()) +} + +/// Tests the backtracker with prefilters enabled. +#[test] +fn prefilter() -> Result<()> { + let my_compiler = |test: &RegexTest, regexes: &[String]| { + // Parse regexes as HIRs so we can get literals to build a prefilter. + let mut hirs = vec![]; + for pattern in regexes.iter() { + hirs.push(syntax::parse_with(pattern, &config_syntax(test))?); + } + // We can always select leftmost-first here because the backtracker + // only supports leftmost-first matching. + let pre = Prefilter::from_hirs_prefix( + regex_automata::MatchKind::LeftmostFirst, + &hirs, + ); + let mut builder = BoundedBacktracker::builder(); + builder.configure(BoundedBacktracker::config().prefilter(pre)); + compiler(builder)(test, regexes) + }; + let mut runner = TestRunner::new()?; + runner.expand(&["is_match", "find", "captures"], |test| test.compiles()); + runner.blacklist("expensive/backtrack-blow-visited-capacity"); + runner.test_iter(suite()?.iter(), my_compiler).assert(); + Ok(()) +} + +/// Tests the bounded backtracker when its visited capacity is set to its +/// minimum amount. +#[test] +fn min_visited_capacity() -> Result<()> { + let mut runner = TestRunner::new()?; + runner.expand(&["is_match", "find", "captures"], |test| test.compiles()); + runner + .test_iter(suite()?.iter(), move |test, regexes| { + let nfa = NFA::compiler() + .configure(config_thompson(test)) + .syntax(config_syntax(test)) + .build_many(®exes)?; + let mut builder = BoundedBacktracker::builder(); + if !configure_backtrack_builder(test, &mut builder) { + return Ok(CompiledRegex::skip()); + } + // Setup the bounded backtracker so that its visited capacity is + // the absolute minimum required for the test's haystack. + builder.configure(BoundedBacktracker::config().visited_capacity( + backtrack::min_visited_capacity( + &nfa, + &Input::new(test.haystack()), + ), + )); + + let re = builder.build_from_nfa(nfa)?; + let mut cache = re.create_cache(); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, &mut cache, test) + })) + }) + .assert(); + Ok(()) +} + +fn compiler( + mut builder: backtrack::Builder, +) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> { + move |test, regexes| { + if !configure_backtrack_builder(test, &mut builder) { + return Ok(CompiledRegex::skip()); + } + let re = builder.build_many(®exes)?; + let mut cache = re.create_cache(); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, &mut cache, test) + })) + } +} + +fn run_test( + re: &BoundedBacktracker, + cache: &mut backtrack::Cache, + test: &RegexTest, +) -> TestResult { + let input = create_input(test); + match test.additional_name() { + "is_match" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Overlapping => { + TestResult::skip() + } + SearchKind::Leftmost => { + let input = input.earliest(true); + TestResult::matched(re.try_is_match(cache, input).unwrap()) + } + }, + "find" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Overlapping => { + TestResult::skip() + } + SearchKind::Leftmost => TestResult::matches( + re.try_find_iter(cache, input) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|result| result.unwrap()) + .map(|m| Match { + id: m.pattern().as_usize(), + span: Span { start: m.start(), end: m.end() }, + }), + ), + }, + "captures" => match test.search_kind() { + SearchKind::Earliest | SearchKind::Overlapping => { + TestResult::skip() + } + SearchKind::Leftmost => TestResult::captures( + re.try_captures_iter(cache, input) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|result| result.unwrap()) + .map(|caps| testify_captures(&caps)), + ), + }, + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Configures the given regex builder with all relevant settings on the given +/// regex test. +/// +/// If the regex test has a setting that is unsupported, then this returns +/// false (implying the test should be skipped). +fn configure_backtrack_builder( + test: &RegexTest, + builder: &mut backtrack::Builder, +) -> bool { + match (test.search_kind(), test.match_kind()) { + // For testing the standard search APIs. This is the only supported + // configuration for the backtracker. + (SearchKind::Leftmost, MatchKind::LeftmostFirst) => {} + // Overlapping APIs not supported at all for backtracker. + (SearchKind::Overlapping, _) => return false, + // Backtracking doesn't really support the notion of 'earliest'. + // Namely, backtracking already works by returning as soon as it knows + // it has found a match. It just so happens that this corresponds to + // the standard 'leftmost' formulation. + // + // The 'earliest' definition in this crate does indeed permit this + // behavior, so this is "fine," but our test suite specifically looks + // for the earliest position at which a match is known, which our + // finite automata based regex engines have no problem providing. So + // for backtracking, we just skip these tests. + (SearchKind::Earliest, _) => return false, + // For backtracking, 'all' semantics don't really make sense. + (_, MatchKind::All) => return false, + // Not supported at all in regex-automata. + (_, MatchKind::LeftmostLongest) => return false, + }; + let backtrack_config = BoundedBacktracker::config(); + builder + .configure(backtrack_config) + .syntax(config_syntax(test)) + .thompson(config_thompson(test)); + true +} + +/// Configuration of a Thompson NFA compiler from a regex test. +fn config_thompson(test: &RegexTest) -> thompson::Config { + let mut lookm = regex_automata::util::look::LookMatcher::new(); + lookm.set_line_terminator(test.line_terminator()); + thompson::Config::new().utf8(test.utf8()).look_matcher(lookm) +} + +/// Configuration of the regex parser from a regex test. +fn config_syntax(test: &RegexTest) -> syntax::Config { + syntax::Config::new() + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .utf8(test.utf8()) + .line_terminator(test.line_terminator()) +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/mod.rs new file mode 100644 index 0000000..b2558f7 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/mod.rs @@ -0,0 +1,4 @@ +#[cfg(feature = "nfa-backtrack")] +mod backtrack; +#[cfg(feature = "nfa-pikevm")] +mod pikevm; diff --git a/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/mod.rs b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/mod.rs new file mode 100644 index 0000000..5a71144 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/mod.rs @@ -0,0 +1,3 @@ +mod regression; +#[cfg(not(miri))] +mod suite; diff --git a/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/regression.rs b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/regression.rs new file mode 100644 index 0000000..b5db634 --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/regression.rs @@ -0,0 +1,20 @@ +// Tests that we can call the PikeVM with more slots +// than is actually in the compiled regex. +#[test] +fn too_many_slots_normal_pattern() { + use regex_automata::{ + nfa::thompson::pikevm::PikeVM, util::primitives::NonMaxUsize, + Anchored, Input, + }; + + let expr = PikeVM::new(r"abc").unwrap(); + let s = "abc"; + let input = Input::new(s).span(0..s.len()).anchored(Anchored::Yes); + + let mut cache = expr.create_cache(); + let mut slots: Vec<Option<NonMaxUsize>> = vec![None; 4]; + let pid = expr.search_slots(&mut cache, &input, &mut slots); + assert_eq!(pid, Some(regex_automata::PatternID::must(0))); + assert_eq!(slots[0], Some(NonMaxUsize::new(0).unwrap())); + assert_eq!(slots[1], Some(NonMaxUsize::new(3).unwrap())); +} diff --git a/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/suite.rs b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/suite.rs new file mode 100644 index 0000000..1fb3fec --- /dev/null +++ b/third_party/crates/regex-automata-0.4.18/tests/nfa/thompson/pikevm/suite.rs @@ -0,0 +1,162 @@ +use { + anyhow::Result, + regex_automata::{ + nfa::thompson::{ + self, + pikevm::{self, PikeVM}, + }, + util::{prefilter::Prefilter, syntax}, + PatternSet, + }, + regex_test::{ + CompiledRegex, Match, RegexTest, SearchKind, Span, TestResult, + TestRunner, + }, +}; + +use crate::{create_input, suite, testify_captures, untestify_kind}; + +/// Tests the default configuration of the hybrid NFA/DFA. +#[test] +fn default() -> Result<()> { + let builder = PikeVM::builder(); + let mut runner = TestRunner::new()?; + runner.expand(&["is_match", "find", "captures"], |test| test.compiles()); + runner.test_iter(suite()?.iter(), compiler(builder)).assert(); + Ok(()) +} + +/// Tests the PikeVM with prefilters enabled. +#[test] +fn prefilter() -> Result<()> { + let my_compiler = |test: &RegexTest, regexes: &[String]| { + // Parse regexes as HIRs so we can get literals to build a prefilter. + let mut hirs = vec![]; + for pattern in regexes.iter() { + hirs.push(syntax::parse_with(pattern, &config_syntax(test))?); + } + let kind = match untestify_kind(test.match_kind()) { + None => return Ok(CompiledRegex::skip()), + Some(kind) => kind, + }; + let pre = Prefilter::from_hirs_prefix(kind, &hirs); + let mut builder = PikeVM::builder(); + builder.configure(PikeVM::config().prefilter(pre)); + compiler(builder)(test, regexes) + }; + let mut runner = TestRunner::new()?; + runner.expand(&["is_match", "find", "captures"], |test| test.compiles()); + runner.test_iter(suite()?.iter(), my_compiler).assert(); + Ok(()) +} + +fn compiler( + mut builder: pikevm::Builder, +) -> impl FnMut(&RegexTest, &[String]) -> Result<CompiledRegex> { + move |test, regexes| { + if !configure_pikevm_builder(test, &mut builder) { + return Ok(CompiledRegex::skip()); + } + let re = builder.build_many(®exes)?; + let mut cache = re.create_cache(); + Ok(CompiledRegex::compiled(move |test| -> TestResult { + run_test(&re, &mut cache, test) + })) + } +} + +fn run_test( + re: &PikeVM, + cache: &mut pikevm::Cache, + test: &RegexTest, +) -> TestResult { + let input = create_input(test); + match test.additional_name() { + "is_match" => TestResult::matched(re.is_match(cache, input)), + "find" => match test.search_kind() { + SearchKind::Earliest => { + let it = re + .find_iter(cache, input.earliest(true)) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: m.pattern().as_usize(), + span: Span { start: m.start(), end: m.end() }, + }); + TestResult::matches(it) + } + SearchKind::Leftmost => { + let it = re + .find_iter(cache, input) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|m| Match { + id: m.pattern().as_usize(), + span: Span { start: m.start(), end: m.end() }, + }); + TestResult::matches(it) + } + SearchKind::Overlapping => { + let mut patset = PatternSet::new(re.get_nfa().pattern_len()); + re.which_overlapping_matches(cache, &input, &mut patset); + TestResult::which(patset.iter().map(|p| p.as_usize())) + } + }, + "captures" => match test.search_kind() { + SearchKind::Earliest => { + let it = re + .captures_iter(cache, input.earliest(true)) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|caps| testify_captures(&caps)); + TestResult::captures(it) + } + SearchKind::Leftmost => { + let it = re + .captures_iter(cache, input) + .take(test.match_limit().unwrap_or(std::usize::MAX)) + .map(|caps| testify_captures(&caps)); + TestResult::captures(it) + } + SearchKind::Overlapping => { + // There is no overlapping PikeVM API that supports captures. + TestResult::skip() + } + }, + name => TestResult::fail(&format!("unrecognized test name: {name}")), + } +} + +/// Configures the given regex builder with all relevant settings on the given +/// regex test. +/// +/// If the regex test has a setting that is unsupported, then this returns +/// false (implying the test should be skipped). +fn configure_pikevm_builder( + test: &RegexTest, + builder: &mut pikevm::Builder, +) -> bool { + let match_kind = match untestify_kind(test.match_kind()) { + None => return false, + Some(k) => k, + }; + let pikevm_config = PikeVM::config().match_kind(match_kind); + builder + .configure(pikevm_config) + .syntax(config_syntax(test)) + .thompson(config_thompson(test)); + true +} + +/// Configuration of a Thompson NFA compiler from a regex test. +fn config_thompson(test: &RegexTest) -> thompson::Config { + let mut lookm = regex_automata::util::look::LookMatcher::new(); + lookm.set_line_terminator(test.line_terminator()); + thompson::Config::new().utf8(test.utf8()).look_matcher(lookm) +} + +/// Configuration of the regex parser from a regex test. +fn config_syntax(test: &RegexTest) -> syntax::Config { + syntax::Config::new() + .case_insensitive(test.case_insensitive()) + .unicode(test.unicode()) + .utf8(test.utf8()) + .line_terminator(test.line_terminator()) +} diff --git a/third_party/crates/regex-syntax-0.8.11/.cargo-checksum.json b/third_party/crates/regex-syntax-0.8.11/.cargo-checksum.json new file mode 100644 index 0000000..0664a25 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"aa8fc3d365726a6e5bb590b3ba7e84de64e4faaa0adcc9c1039570daeafe9018","Cargo.lock":"44d2a00bcc4bbcfcd5443d80781c42a4b8bfedc7b8720d0b30442e569143aa2f","Cargo.toml":"10c3f14629c0900ac452dd6d55a62aab561bbabbdf5774a802c8d223933cfa3b","Cargo.toml.orig":"fdfdadd590cb4121437df8e9b72f3ca2e6eff812bb9436f2828c76b1810b1f6b","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb","README.md":"b2484aa7e66fb92d1378e9a7ce7605af18f77cb12c179866eaf92ba28cfec1d9","benches/bench.rs":"d2b6ae5b939abd6093064f144b981b7739d7f474ec0698a1268052fc92406635","src/ast/mod.rs":"38df06574a3816eae2796c757d9d268b7b42ce3fb1feee86750e7d04de7649c7","src/ast/parse.rs":"af6d82e62e97379a91840e99c10be1279c0ae1298a0d8e5da429e9e7bc3ec339","src/ast/print.rs":"6a681ce021a384c47dda04c71e2868b5dd61c633bb5b1a67628b9a8df16f0413","src/ast/visitor.rs":"2af2efd77727803b8d15a6244af92267e96edbfc1a211bcbea2b32e1b5483918","src/debug.rs":"91b2492394de05bb11ee75329115b33fb378c19e076370d1a1ae665ce1682777","src/either.rs":"1758e3edd056884eccadd995708d1e374ba9aa65846bd0e13b1aae852607c560","src/error.rs":"e308b3ccad0bea927f4e3957170302e9cfa743bfdf9376f3a5f4137b44ca6cfc","src/hir/interval.rs":"170949e6fd9e9641830415ec3ea813ff2431c8594d49c559cd41d4903f3d29d0","src/hir/literal.rs":"61e9f54103c671694dd017c23c5c9263e032735921ef77527940e83b29ced540","src/hir/mod.rs":"13ee5b65fac1f2c9780ce48a500b1e9d198cb0bc07c0d7f4a4391aab87424563","src/hir/print.rs":"e1e1dfa71983c8fea64f500a0b9dfcbd258b4060e12b95d432468015a247a5cb","src/hir/translate.rs":"8c01783e74261a8562c2694394d94ccb8cb173274932731fa77118041fab4507","src/hir/visitor.rs":"71ca9c93aa48a5ed445399659fa6455093a1bbd9ef44b66bc7095c1b08b2ec1f","src/lib.rs":"c51d1e55a8b6c4608e21a278ed0ef9480f73ab5b814b6ca6127f4a049c4d5007","src/parser.rs":"6b2f4f27e3331a01a25b87c89368dd2e54396bd425dac57941f9c1ebfd238ac8","src/rank.rs":"ff3d58b0cc5ffa69e2e8c56fc7d9ef41dd399d59a639a253a51551b858cb5bbd","src/unicode.rs":"b2084dcbd4331501b9a895fd7e7575d93ff96eb661c6e6adbc8c66bb72685cde","src/unicode_tables/LICENSE-UNICODE":"74db5baf44a41b1000312c673544b3374e4198af5605c7f9080a402cec42cfa3","src/unicode_tables/age.rs":"71b7cf52acdb4aa98b44145303b8efbfa94913235493521941ef1e0092a0ffe2","src/unicode_tables/case_folding_simple.rs":"7622c7f7f03ac0dc2f2bcd51c81a217d64de0cc912f62f1add5f676603a02456","src/unicode_tables/general_category.rs":"9488e3721f7c2ae20e1b77fcff9a59b4ed8f22954b8645ea6d8592eac1856423","src/unicode_tables/grapheme_cluster_break.rs":"0dd9d66bad598f4ec3451b6699f05c17c52079e37d463baf6385bbe51aa218f1","src/unicode_tables/mod.rs":"26c837099cd934c8062e24bc9a0aaecf15fe1de03f9c6da3f3e1e5ac3ca24bee","src/unicode_tables/perl_decimal.rs":"6a59143db81a0bcaf0e8d0af265e711d1a6472e1f091ee9ee4377da5d5d0cd1f","src/unicode_tables/perl_space.rs":"ec9bb22ed7e99feef292249c7e6f4673ee0af9635d4d158f93923494c14cd5ed","src/unicode_tables/perl_word.rs":"30f073baae28ea34c373c7778c00f20c1621c3e644404eff031f7d1cc8e9c9e2","src/unicode_tables/property_bool.rs":"66cf5bd2a1438bf9694152f077a285cf014fbd50b9dd63a97233b2ea61d64962","src/unicode_tables/property_names.rs":"8c93985d1bcb01735667a3c4cb92f7e260d267326bde9d7f048bc77cd7e07855","src/unicode_tables/property_values.rs":"ef9131ce0a575c7327ec6d466aafd8b7c25600d80c232b5a4110bbf0a5a59136","src/unicode_tables/script.rs":"41bd424f1e3a03290cf4995ced678dcf24c94b38c905c62f6819bf67e098a2ec","src/unicode_tables/script_extension.rs":"a314099ddbf50a07fe350bb0835bf2fe494ed5ad278b30e171e21506eb557906","src/unicode_tables/sentence_break.rs":"be84fbe8c5c67e761b16fe6c27f16664dbb145357835cd6b92bc2a4a4c52ee79","src/unicode_tables/word_break.rs":"c551681ad49ec28c7ae32bab1371945821c736ca8f0de410cb89f28066ec2ecf","src/utf8.rs":"193f280f3b48116ed6ca8b5fe80b9d6401b6e733dfa138caf64ec1b017b6f175"},"package":"d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"} \ No newline at end of file diff --git a/third_party/crates/regex-syntax-0.8.11/.cargo_vcs_info.json b/third_party/crates/regex-syntax-0.8.11/.cargo_vcs_info.json new file mode 100644 index 0000000..26fef5c --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "140167995737fa11dfe11b8af8b9aa143b790b4e" + }, + "path_in_vcs": "regex-syntax" +} \ No newline at end of file diff --git a/third_party/crates/regex-syntax-0.8.11/BUILD.bazel b/third_party/crates/regex-syntax-0.8.11/BUILD.bazel new file mode 100644 index 0000000..bd2eb35 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/BUILD.bazel @@ -0,0 +1,122 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_library( + name = "regex_syntax", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "default", + "std", + "unicode", + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", + ], + crate_root = "src/lib.rs", + edition = "2021", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=regex-syntax", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "0.8.11", +) diff --git a/third_party/crates/regex-syntax-0.8.11/Cargo.lock b/third_party/crates/regex-syntax-0.8.11/Cargo.lock new file mode 100644 index 0000000..fbe20f4 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/Cargo.lock @@ -0,0 +1,65 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" diff --git a/third_party/crates/regex-syntax-0.8.11/Cargo.toml b/third_party/crates/regex-syntax-0.8.11/Cargo.toml new file mode 100644 index 0000000..3d08b41 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/Cargo.toml @@ -0,0 +1,97 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.65" +name = "regex-syntax" +version = "0.8.11" +authors = [ + "The Rust Project Developers", + "Andrew Gallant <jamslam@gmail.com>", +] +build = false +include = [ + "/CHANGELOG.md", + "/Cargo.toml", + "/LICENSE-MIT", + "/LICENSE-APACHE", + "/README.md", + "/UNICODE.md", + "bench/README.md", + "src/**/*.rs", + "testdata/**.toml", + "tests/**/*.rs", + "bench/**/*.rs", + "benches/**/*.rs", + "LICENSE-UNICODE", + "!/tests/fuzz/mod.rs", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "A regular expression parser." +homepage = "https://github.com/rust-lang/regex/tree/master/regex-syntax" +documentation = "https://docs.rs/regex-syntax" +readme = "README.md" +license = "MIT OR Apache-2.0" +repository = "https://github.com/rust-lang/regex" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs_regex", +] + +[features] +arbitrary = ["dep:arbitrary"] +default = [ + "std", + "unicode", +] +std = [] +unicode = [ + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", +] +unicode-age = [] +unicode-bool = [] +unicode-case = [] +unicode-gencat = [] +unicode-perl = [] +unicode-script = [] +unicode-segment = [] + +[lib] +name = "regex_syntax" +path = "src/lib.rs" + +[[bench]] +name = "bench" +path = "benches/bench.rs" + +[dependencies.arbitrary] +version = "1.3.0" +features = ["derive"] +optional = true + +[lints.rust.unexpected_cfgs] +level = "allow" +priority = 0 +check-cfg = ["cfg(docsrs_regex)"] diff --git a/third_party/crates/regex-syntax-0.8.11/Cargo.toml.orig b/third_party/crates/regex-syntax-0.8.11/Cargo.toml.orig new file mode 100644 index 0000000..fa63da9 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/Cargo.toml.orig @@ -0,0 +1,63 @@ +[package] +name = "regex-syntax" +version = "0.8.11" #:version +authors = ["The Rust Project Developers", "Andrew Gallant <jamslam@gmail.com>"] +license = "MIT OR Apache-2.0" +homepage = "https://github.com/rust-lang/regex/tree/master/regex-syntax" +repository = "https://github.com/rust-lang/regex" +documentation = "https://docs.rs/regex-syntax" +description = "A regular expression parser." +workspace = ".." +edition = "2021" +rust-version = "1.65" +include.workspace = true + +# Features are documented in the "Crate features" section of the crate docs: +# https://docs.rs/regex-syntax/*/#crate-features +[features] +default = ["std", "unicode"] +std = [] +arbitrary = ["dep:arbitrary"] + +unicode = [ + "unicode-age", + "unicode-bool", + "unicode-case", + "unicode-gencat", + "unicode-perl", + "unicode-script", + "unicode-segment", +] +unicode-age = [] +unicode-bool = [] +unicode-case = [] +unicode-gencat = [] +unicode-perl = [] +unicode-script = [] +unicode-segment = [] + +[dependencies] +arbitrary = { version = "1.3.0", features = ["derive"], optional = true } + +[package.metadata.docs.rs] +# We want to document all features. +all-features = true +# Since this crate's feature setup is pretty complicated, it is worth opting +# into a nightly unstable option to show the features that need to be enabled +# for public API items. To do that, we set 'docsrs_regex', and when that's +# enabled, we enable the 'doc_cfg' feature. +# +# To test this locally, run: +# +# RUSTDOCFLAGS="--cfg docsrs_regex" cargo +nightly doc --all-features +# +# Note that we use `docsrs_regex` instead of the more standard `docsrs` because +# other crates use that same `cfg` knob. And since we are enabling a nightly +# feature, they sometimes break. By using our "own" `cfg` knob, we are closer +# to being masters of our own destiny. +rustdoc-args = ["--cfg", "docsrs_regex"] + +# This squashes the (AFAIK) erroneous warning that `docsrs_regex` is not a +# valid `cfg` knob. +[lints.rust] +unexpected_cfgs = { level = "allow", check-cfg = ['cfg(docsrs_regex)'] } diff --git a/third_party/crates/regex-syntax-0.8.11/LICENSE-APACHE b/third_party/crates/regex-syntax-0.8.11/LICENSE-APACHE new file mode 100644 index 0000000..16fe87b --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/crates/regex-syntax-0.8.11/LICENSE-MIT b/third_party/crates/regex-syntax-0.8.11/LICENSE-MIT new file mode 100644 index 0000000..39d4bdb --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2014 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third_party/crates/regex-syntax-0.8.11/README.md b/third_party/crates/regex-syntax-0.8.11/README.md new file mode 100644 index 0000000..529513b --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/README.md @@ -0,0 +1,96 @@ +regex-syntax +============ +This crate provides a robust regular expression parser. + +[![Build status](https://github.com/rust-lang/regex/workflows/ci/badge.svg)](https://github.com/rust-lang/regex/actions) +[![Crates.io](https://img.shields.io/crates/v/regex-syntax.svg)](https://crates.io/crates/regex-syntax) + + +### Documentation + +https://docs.rs/regex-syntax + + +### Overview + +There are two primary types exported by this crate: `Ast` and `Hir`. The former +is a faithful abstract syntax of a regular expression, and can convert regular +expressions back to their concrete syntax while mostly preserving its original +form. The latter type is a high level intermediate representation of a regular +expression that is amenable to analysis and compilation into byte codes or +automata. An `Hir` achieves this by drastically simplifying the syntactic +structure of the regular expression. While an `Hir` can be converted back to +its equivalent concrete syntax, the result is unlikely to resemble the original +concrete syntax that produced the `Hir`. + + +### Example + +This example shows how to parse a pattern string into its HIR: + +```rust +use regex_syntax::{hir::Hir, parse}; + +let hir = parse("a|b").unwrap(); +assert_eq!(hir, Hir::alternation(vec![ + Hir::literal("a".as_bytes()), + Hir::literal("b".as_bytes()), +])); +``` + + +### Safety + +This crate has no `unsafe` code and sets `forbid(unsafe_code)`. While it's +possible this crate could use `unsafe` code in the future, the standard +for doing so is extremely high. In general, most code in this crate is not +performance critical, since it tends to be dwarfed by the time it takes to +compile a regular expression into an automaton. Therefore, there is little need +for extreme optimization, and therefore, use of `unsafe`. + +The standard for using `unsafe` in this crate is extremely high because this +crate is intended to be reasonably safe to use with user supplied regular +expressions. Therefore, while there may be bugs in the regex parser itself, +they should _never_ result in memory unsafety unless there is either a bug +in the compiler or the standard library. (Since `regex-syntax` has zero +dependencies.) + + +### Crate features + +By default, this crate bundles a fairly large amount of Unicode data tables +(a source size of ~750KB). Because of their large size, one can disable some +or all of these data tables. If a regular expression attempts to use Unicode +data that is not available, then an error will occur when translating the `Ast` +to the `Hir`. + +The full set of features one can disable are +[in the "Crate features" section of the documentation](https://docs.rs/regex-syntax/*/#crate-features). + + +### Testing + +Simply running `cargo test` will give you very good coverage. However, because +of the large number of features exposed by this crate, a `test` script is +included in this directory which will test several feature combinations. This +is the same script that is run in CI. + + +### Motivation + +The primary purpose of this crate is to provide the parser used by `regex`. +Specifically, this crate is treated as an implementation detail of the `regex`, +and is primarily developed for the needs of `regex`. + +Since this crate is an implementation detail of `regex`, it may experience +breaking change releases at a different cadence from `regex`. This is only +possible because this crate is _not_ a public dependency of `regex`. + +Another consequence of this de-coupling is that there is no direct way to +compile a `regex::Regex` from a `regex_syntax::hir::Hir`. Instead, one must +first convert the `Hir` to a string (via its `std::fmt::Display`) and then +compile that via `Regex::new`. While this does repeat some work, compilation +typically takes much longer than parsing. + +Stated differently, the coupling between `regex` and `regex-syntax` exists only +at the level of the concrete syntax. diff --git a/third_party/crates/regex-syntax-0.8.11/benches/bench.rs b/third_party/crates/regex-syntax-0.8.11/benches/bench.rs new file mode 100644 index 0000000..d4703d4 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/benches/bench.rs @@ -0,0 +1,63 @@ +#![feature(test)] + +extern crate test; + +use regex_syntax::Parser; +use test::Bencher; + +#[bench] +fn parse_simple1(b: &mut Bencher) { + b.iter(|| { + let re = r"^bc(d|e)*$"; + Parser::new().parse(re).unwrap() + }); +} + +#[bench] +fn parse_simple2(b: &mut Bencher) { + b.iter(|| { + let re = r"'[a-zA-Z_][a-zA-Z0-9_]*(')\b"; + Parser::new().parse(re).unwrap() + }); +} + +#[bench] +fn parse_small1(b: &mut Bencher) { + b.iter(|| { + let re = r"\p{L}|\p{N}|\s|.|\d"; + Parser::new().parse(re).unwrap() + }); +} + +#[bench] +fn parse_medium1(b: &mut Bencher) { + b.iter(|| { + let re = r"\pL\p{Greek}\p{Hiragana}\p{Alphabetic}\p{Hebrew}\p{Arabic}"; + Parser::new().parse(re).unwrap() + }); +} + +#[bench] +fn parse_medium2(b: &mut Bencher) { + b.iter(|| { + let re = r"\s\S\w\W\d\D"; + Parser::new().parse(re).unwrap() + }); +} + +#[bench] +fn parse_medium3(b: &mut Bencher) { + b.iter(|| { + let re = + r"\p{age:3.2}\p{hira}\p{scx:hira}\p{alphabetic}\p{sc:Greek}\pL"; + Parser::new().parse(re).unwrap() + }); +} + +#[bench] +fn parse_huge(b: &mut Bencher) { + b.iter(|| { + let re = r"\p{L}{100}"; + Parser::new().parse(re).unwrap() + }); +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/ast/mod.rs b/third_party/crates/regex-syntax-0.8.11/src/ast/mod.rs new file mode 100644 index 0000000..7e2426d --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/ast/mod.rs @@ -0,0 +1,1807 @@ +/*! +Defines an abstract syntax for regular expressions. +*/ + +use core::cmp::Ordering; + +use alloc::{boxed::Box, string::String, vec, vec::Vec}; + +pub use crate::ast::visitor::{visit, Visitor}; + +pub mod parse; +pub mod print; +mod visitor; + +/// An error that occurred while parsing a regular expression into an abstract +/// syntax tree. +/// +/// Note that not all ASTs represents a valid regular expression. For example, +/// an AST is constructed without error for `\p{Quux}`, but `Quux` is not a +/// valid Unicode property name. That particular error is reported when +/// translating an AST to the high-level intermediate representation (`HIR`). +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Error { + /// The kind of error. + kind: ErrorKind, + /// The original pattern that the parser generated the error from. Every + /// span in an error is a valid range into this string. + pattern: String, + /// The span of this error. + span: Span, +} + +impl Error { + /// Return the type of this error. + pub fn kind(&self) -> &ErrorKind { + &self.kind + } + + /// The original pattern string in which this error occurred. + /// + /// Every span reported by this error is reported in terms of this string. + pub fn pattern(&self) -> &str { + &self.pattern + } + + /// Return the span at which this error occurred. + pub fn span(&self) -> &Span { + &self.span + } + + /// Return an auxiliary span. This span exists only for some errors that + /// benefit from being able to point to two locations in the original + /// regular expression. For example, "duplicate" errors will have the + /// main error position set to the duplicate occurrence while its + /// auxiliary span will be set to the initial occurrence. + pub fn auxiliary_span(&self) -> Option<&Span> { + use self::ErrorKind::*; + match self.kind { + FlagDuplicate { ref original } => Some(original), + FlagRepeatedNegation { ref original, .. } => Some(original), + GroupNameDuplicate { ref original, .. } => Some(original), + _ => None, + } + } +} + +/// The type of an error that occurred while building an AST. +/// +/// This error type is marked as `non_exhaustive`. This means that adding a +/// new variant is not considered a breaking change. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ErrorKind { + /// The capturing group limit was exceeded. + /// + /// Note that this represents a limit on the total number of capturing + /// groups in a regex and not necessarily the number of nested capturing + /// groups. That is, the nest limit can be low and it is still possible for + /// this error to occur. + CaptureLimitExceeded, + /// An invalid escape sequence was found in a character class set. + ClassEscapeInvalid, + /// An invalid character class range was found. An invalid range is any + /// range where the start is greater than the end. + ClassRangeInvalid, + /// An invalid range boundary was found in a character class. Range + /// boundaries must be a single literal codepoint, but this error indicates + /// that something else was found, such as a nested class. + ClassRangeLiteral, + /// An opening `[` was found with no corresponding closing `]`. + ClassUnclosed, + /// Note that this error variant is no longer used. Namely, a decimal + /// number can only appear as a repetition quantifier. When the number + /// in a repetition quantifier is empty, then it gets its own specialized + /// error, `RepetitionCountDecimalEmpty`. + DecimalEmpty, + /// An invalid decimal number was given where one was expected. + DecimalInvalid, + /// A bracketed hex literal was empty. + EscapeHexEmpty, + /// A bracketed hex literal did not correspond to a Unicode scalar value. + EscapeHexInvalid, + /// An invalid hexadecimal digit was found. + EscapeHexInvalidDigit, + /// EOF was found before an escape sequence was completed. + EscapeUnexpectedEof, + /// An unrecognized escape sequence. + EscapeUnrecognized, + /// A dangling negation was used when setting flags, e.g., `i-`. + FlagDanglingNegation, + /// A flag was used twice, e.g., `i-i`. + FlagDuplicate { + /// The position of the original flag. The error position + /// points to the duplicate flag. + original: Span, + }, + /// The negation operator was used twice, e.g., `-i-s`. + FlagRepeatedNegation { + /// The position of the original negation operator. The error position + /// points to the duplicate negation operator. + original: Span, + }, + /// Expected a flag but got EOF, e.g., `(?`. + FlagUnexpectedEof, + /// Unrecognized flag, e.g., `a`. + FlagUnrecognized, + /// A duplicate capture name was found. + GroupNameDuplicate { + /// The position of the initial occurrence of the capture name. The + /// error position itself points to the duplicate occurrence. + original: Span, + }, + /// A capture group name is empty, e.g., `(?P<>abc)`. + GroupNameEmpty, + /// An invalid character was seen for a capture group name. This includes + /// errors where the first character is a digit (even though subsequent + /// characters are allowed to be digits). + GroupNameInvalid, + /// A closing `>` could not be found for a capture group name. + GroupNameUnexpectedEof, + /// An unclosed group, e.g., `(ab`. + /// + /// The span of this error corresponds to the unclosed parenthesis. + GroupUnclosed, + /// An unopened group, e.g., `ab)`. + GroupUnopened, + /// The nest limit was exceeded. The limit stored here is the limit + /// configured in the parser. + NestLimitExceeded(u32), + /// The range provided in a counted repetition operator is invalid. The + /// range is invalid if the start is greater than the end. + RepetitionCountInvalid, + /// An opening `{` was not followed by a valid decimal value. + /// For example, `x{}` or `x{]}` would fail. + RepetitionCountDecimalEmpty, + /// An opening `{` was found with no corresponding closing `}`. + RepetitionCountUnclosed, + /// A repetition operator was applied to a missing sub-expression. This + /// occurs, for example, in the regex consisting of just a `*` or even + /// `(?i)*`. It is, however, possible to create a repetition operating on + /// an empty sub-expression. For example, `()*` is still considered valid. + RepetitionMissing, + /// The special word boundary syntax, `\b{something}`, was used, but + /// either EOF without `}` was seen, or an invalid character in the + /// braces was seen. + SpecialWordBoundaryUnclosed, + /// The special word boundary syntax, `\b{something}`, was used, but + /// `something` was not recognized as a valid word boundary kind. + SpecialWordBoundaryUnrecognized, + /// The syntax `\b{` was observed, but afterwards the end of the pattern + /// was observed without being able to tell whether it was meant to be a + /// bounded repetition on the `\b` or the beginning of a special word + /// boundary assertion. + SpecialWordOrRepetitionUnexpectedEof, + /// The Unicode class is not valid. This typically occurs when a `\p` is + /// followed by something other than a `{`. + UnicodeClassInvalid, + /// When octal support is disabled, this error is produced when an octal + /// escape is used. The octal escape is assumed to be an invocation of + /// a backreference, which is the common case. + UnsupportedBackreference, + /// When syntax similar to PCRE's look-around is used, this error is + /// returned. Some example syntaxes that are rejected include, but are + /// not necessarily limited to, `(?=re)`, `(?!re)`, `(?<=re)` and + /// `(?<!re)`. Note that all of these syntaxes are otherwise invalid; this + /// error is used to improve the user experience. + UnsupportedLookAround, +} + +#[cfg(feature = "std")] +impl std::error::Error for Error {} + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::error::Formatter::from(self).fmt(f) + } +} + +impl core::fmt::Display for ErrorKind { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + use self::ErrorKind::*; + match *self { + CaptureLimitExceeded => write!( + f, + "exceeded the maximum number of \ + capturing groups ({})", + u32::MAX + ), + ClassEscapeInvalid => { + write!(f, "invalid escape sequence found in character class") + } + ClassRangeInvalid => write!( + f, + "invalid character class range, \ + the start must be <= the end" + ), + ClassRangeLiteral => { + write!(f, "invalid range boundary, must be a literal") + } + ClassUnclosed => write!(f, "unclosed character class"), + DecimalEmpty => write!(f, "decimal literal empty"), + DecimalInvalid => write!(f, "decimal literal invalid"), + EscapeHexEmpty => write!(f, "hexadecimal literal empty"), + EscapeHexInvalid => { + write!(f, "hexadecimal literal is not a Unicode scalar value") + } + EscapeHexInvalidDigit => write!(f, "invalid hexadecimal digit"), + EscapeUnexpectedEof => write!( + f, + "incomplete escape sequence, \ + reached end of pattern prematurely" + ), + EscapeUnrecognized => write!(f, "unrecognized escape sequence"), + FlagDanglingNegation => { + write!(f, "dangling flag negation operator") + } + FlagDuplicate { .. } => write!(f, "duplicate flag"), + FlagRepeatedNegation { .. } => { + write!(f, "flag negation operator repeated") + } + FlagUnexpectedEof => { + write!(f, "expected flag but got end of regex") + } + FlagUnrecognized => write!(f, "unrecognized flag"), + GroupNameDuplicate { .. } => { + write!(f, "duplicate capture group name") + } + GroupNameEmpty => write!(f, "empty capture group name"), + GroupNameInvalid => write!(f, "invalid capture group character"), + GroupNameUnexpectedEof => write!(f, "unclosed capture group name"), + GroupUnclosed => write!(f, "unclosed group"), + GroupUnopened => write!(f, "unopened group"), + NestLimitExceeded(limit) => write!( + f, + "exceed the maximum number of \ + nested parentheses/brackets ({})", + limit + ), + RepetitionCountInvalid => write!( + f, + "invalid repetition count range, \ + the start must be <= the end" + ), + RepetitionCountDecimalEmpty => { + write!(f, "repetition quantifier expects a valid decimal") + } + RepetitionCountUnclosed => { + write!(f, "unclosed counted repetition") + } + RepetitionMissing => { + write!(f, "repetition operator missing expression") + } + SpecialWordBoundaryUnclosed => { + write!( + f, + "special word boundary assertion is either \ + unclosed or contains an invalid character", + ) + } + SpecialWordBoundaryUnrecognized => { + write!( + f, + "unrecognized special word boundary assertion, \ + valid choices are: start, end, start-half \ + or end-half", + ) + } + SpecialWordOrRepetitionUnexpectedEof => { + write!( + f, + "found either the beginning of a special word \ + boundary or a bounded repetition on a \\b with \ + an opening brace, but no closing brace", + ) + } + UnicodeClassInvalid => { + write!(f, "invalid Unicode character class") + } + UnsupportedBackreference => { + write!(f, "backreferences are not supported") + } + UnsupportedLookAround => write!( + f, + "look-around, including look-ahead and look-behind, \ + is not supported" + ), + } + } +} + +/// Span represents the position information of a single AST item. +/// +/// All span positions are absolute byte offsets that can be used on the +/// original regular expression that was parsed. +#[derive(Clone, Copy, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Span { + /// The start byte offset. + pub start: Position, + /// The end byte offset. + pub end: Position, +} + +impl core::fmt::Debug for Span { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "Span({:?}, {:?})", self.start, self.end) + } +} + +impl Ord for Span { + fn cmp(&self, other: &Span) -> Ordering { + (&self.start, &self.end).cmp(&(&other.start, &other.end)) + } +} + +impl PartialOrd for Span { + fn partial_cmp(&self, other: &Span) -> Option<Ordering> { + Some(self.cmp(other)) + } +} + +/// A single position in a regular expression. +/// +/// A position encodes one half of a span, and include the byte offset, line +/// number and column number. +#[derive(Clone, Copy, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Position { + /// The absolute offset of this position, starting at `0` from the + /// beginning of the regular expression pattern string. + pub offset: usize, + /// The line number, starting at `1`. + pub line: usize, + /// The approximate column number, starting at `1`. + pub column: usize, +} + +impl core::fmt::Debug for Position { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "Position(o: {:?}, l: {:?}, c: {:?})", + self.offset, self.line, self.column + ) + } +} + +impl Ord for Position { + fn cmp(&self, other: &Position) -> Ordering { + self.offset.cmp(&other.offset) + } +} + +impl PartialOrd for Position { + fn partial_cmp(&self, other: &Position) -> Option<Ordering> { + Some(self.cmp(other)) + } +} + +impl Span { + /// Create a new span with the given positions. + pub fn new(start: Position, end: Position) -> Span { + Span { start, end } + } + + /// Create a new span using the given position as the start and end. + pub fn splat(pos: Position) -> Span { + Span::new(pos, pos) + } + + /// Create a new span by replacing the starting the position with the one + /// given. + pub fn with_start(self, pos: Position) -> Span { + Span { start: pos, ..self } + } + + /// Create a new span by replacing the ending the position with the one + /// given. + pub fn with_end(self, pos: Position) -> Span { + Span { end: pos, ..self } + } + + /// Returns true if and only if this span occurs on a single line. + pub fn is_one_line(&self) -> bool { + self.start.line == self.end.line + } + + /// Returns true if and only if this span is empty. That is, it points to + /// a single position in the concrete syntax of a regular expression. + pub fn is_empty(&self) -> bool { + self.start.offset == self.end.offset + } +} + +impl Position { + /// Create a new position with the given information. + /// + /// `offset` is the absolute offset of the position, starting at `0` from + /// the beginning of the regular expression pattern string. + /// + /// `line` is the line number, starting at `1`. + /// + /// `column` is the approximate column number, starting at `1`. + pub fn new(offset: usize, line: usize, column: usize) -> Position { + Position { offset, line, column } + } +} + +/// An abstract syntax tree for a singular expression along with comments +/// found. +/// +/// Comments are not stored in the tree itself to avoid complexity. Each +/// comment contains a span of precisely where it occurred in the original +/// regular expression. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct WithComments { + /// The actual ast. + pub ast: Ast, + /// All comments found in the original regular expression. + pub comments: Vec<Comment>, +} + +/// A comment from a regular expression with an associated span. +/// +/// A regular expression can only contain comments when the `x` flag is +/// enabled. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Comment { + /// The span of this comment, including the beginning `#` and ending `\n`. + pub span: Span, + /// The comment text, starting with the first character following the `#` + /// and ending with the last character preceding the `\n`. + pub comment: String, +} + +/// An abstract syntax tree for a single regular expression. +/// +/// An `Ast`'s `fmt::Display` implementation uses constant stack space and heap +/// space proportional to the size of the `Ast`. +/// +/// This type defines its own destructor that uses constant stack space and +/// heap space proportional to the size of the `Ast`. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum Ast { + /// An empty regex that matches everything. + Empty(Box<Span>), + /// A set of flags, e.g., `(?is)`. + Flags(Box<SetFlags>), + /// A single character literal, which includes escape sequences. + Literal(Box<Literal>), + /// The "any character" class. + Dot(Box<Span>), + /// A single zero-width assertion. + Assertion(Box<Assertion>), + /// A single Unicode character class, e.g., `\pL` or `\p{Greek}`. + ClassUnicode(Box<ClassUnicode>), + /// A single perl character class, e.g., `\d` or `\W`. + ClassPerl(Box<ClassPerl>), + /// A single bracketed character class set, which may contain zero or more + /// character ranges and/or zero or more nested classes. e.g., + /// `[a-zA-Z\pL]`. + ClassBracketed(Box<ClassBracketed>), + /// A repetition operator applied to an arbitrary regular expression. + Repetition(Box<Repetition>), + /// A grouped regular expression. + Group(Box<Group>), + /// An alternation of regular expressions. + Alternation(Box<Alternation>), + /// A concatenation of regular expressions. + Concat(Box<Concat>), +} + +impl Ast { + /// Create an "empty" AST item. + pub fn empty(span: Span) -> Ast { + Ast::Empty(Box::new(span)) + } + + /// Create a "flags" AST item. + pub fn flags(e: SetFlags) -> Ast { + Ast::Flags(Box::new(e)) + } + + /// Create a "literal" AST item. + pub fn literal(e: Literal) -> Ast { + Ast::Literal(Box::new(e)) + } + + /// Create a "dot" AST item. + pub fn dot(span: Span) -> Ast { + Ast::Dot(Box::new(span)) + } + + /// Create a "assertion" AST item. + pub fn assertion(e: Assertion) -> Ast { + Ast::Assertion(Box::new(e)) + } + + /// Create a "Unicode class" AST item. + pub fn class_unicode(e: ClassUnicode) -> Ast { + Ast::ClassUnicode(Box::new(e)) + } + + /// Create a "Perl class" AST item. + pub fn class_perl(e: ClassPerl) -> Ast { + Ast::ClassPerl(Box::new(e)) + } + + /// Create a "bracketed class" AST item. + pub fn class_bracketed(e: ClassBracketed) -> Ast { + Ast::ClassBracketed(Box::new(e)) + } + + /// Create a "repetition" AST item. + pub fn repetition(e: Repetition) -> Ast { + Ast::Repetition(Box::new(e)) + } + + /// Create a "group" AST item. + pub fn group(e: Group) -> Ast { + Ast::Group(Box::new(e)) + } + + /// Create a "alternation" AST item. + pub fn alternation(e: Alternation) -> Ast { + Ast::Alternation(Box::new(e)) + } + + /// Create a "concat" AST item. + pub fn concat(e: Concat) -> Ast { + Ast::Concat(Box::new(e)) + } + + /// Return the span of this abstract syntax tree. + pub fn span(&self) -> &Span { + match *self { + Ast::Empty(ref span) => span, + Ast::Flags(ref x) => &x.span, + Ast::Literal(ref x) => &x.span, + Ast::Dot(ref span) => span, + Ast::Assertion(ref x) => &x.span, + Ast::ClassUnicode(ref x) => &x.span, + Ast::ClassPerl(ref x) => &x.span, + Ast::ClassBracketed(ref x) => &x.span, + Ast::Repetition(ref x) => &x.span, + Ast::Group(ref x) => &x.span, + Ast::Alternation(ref x) => &x.span, + Ast::Concat(ref x) => &x.span, + } + } + + /// Return true if and only if this Ast is empty. + pub fn is_empty(&self) -> bool { + match *self { + Ast::Empty(_) => true, + _ => false, + } + } + + /// Returns true if and only if this AST has any (including possibly empty) + /// subexpressions. + fn has_subexprs(&self) -> bool { + match *self { + Ast::Empty(_) + | Ast::Flags(_) + | Ast::Literal(_) + | Ast::Dot(_) + | Ast::Assertion(_) + | Ast::ClassUnicode(_) + | Ast::ClassPerl(_) => false, + Ast::ClassBracketed(_) + | Ast::Repetition(_) + | Ast::Group(_) + | Ast::Alternation(_) + | Ast::Concat(_) => true, + } + } +} + +/// Print a display representation of this Ast. +/// +/// This does not preserve any of the original whitespace formatting that may +/// have originally been present in the concrete syntax from which this Ast +/// was generated. +/// +/// This implementation uses constant stack space and heap space proportional +/// to the size of the `Ast`. +impl core::fmt::Display for Ast { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + use crate::ast::print::Printer; + Printer::new().print(self, f) + } +} + +/// An alternation of regular expressions. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Alternation { + /// The span of this alternation. + pub span: Span, + /// The alternate regular expressions. + pub asts: Vec<Ast>, +} + +impl Alternation { + /// Return this alternation as an AST. + /// + /// If this alternation contains zero ASTs, then `Ast::empty` is returned. + /// If this alternation contains exactly 1 AST, then the corresponding AST + /// is returned. Otherwise, `Ast::alternation` is returned. + pub fn into_ast(mut self) -> Ast { + match self.asts.len() { + 0 => Ast::empty(self.span), + 1 => self.asts.pop().unwrap(), + _ => Ast::alternation(self), + } + } +} + +/// A concatenation of regular expressions. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Concat { + /// The span of this concatenation. + pub span: Span, + /// The concatenation regular expressions. + pub asts: Vec<Ast>, +} + +impl Concat { + /// Return this concatenation as an AST. + /// + /// If this alternation contains zero ASTs, then `Ast::empty` is returned. + /// If this alternation contains exactly 1 AST, then the corresponding AST + /// is returned. Otherwise, `Ast::concat` is returned. + pub fn into_ast(mut self) -> Ast { + match self.asts.len() { + 0 => Ast::empty(self.span), + 1 => self.asts.pop().unwrap(), + _ => Ast::concat(self), + } + } +} + +/// A single literal expression. +/// +/// A literal corresponds to a single Unicode scalar value. Literals may be +/// represented in their literal form, e.g., `a` or in their escaped form, +/// e.g., `\x61`. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Literal { + /// The span of this literal. + pub span: Span, + /// The kind of this literal. + pub kind: LiteralKind, + /// The Unicode scalar value corresponding to this literal. + pub c: char, +} + +impl Literal { + /// If this literal was written as a `\x` hex escape, then this returns + /// the corresponding byte value. Otherwise, this returns `None`. + pub fn byte(&self) -> Option<u8> { + match self.kind { + LiteralKind::HexFixed(HexLiteralKind::X) => { + u8::try_from(self.c).ok() + } + _ => None, + } + } +} + +/// The kind of a single literal expression. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum LiteralKind { + /// The literal is written verbatim, e.g., `a` or `☃`. + Verbatim, + /// The literal is written as an escape because it is otherwise a special + /// regex meta character, e.g., `\*` or `\[`. + Meta, + /// The literal is written as an escape despite the fact that the escape is + /// unnecessary, e.g., `\%` or `\/`. + Superfluous, + /// The literal is written as an octal escape, e.g., `\141`. + Octal, + /// The literal is written as a hex code with a fixed number of digits + /// depending on the type of the escape, e.g., `\x61` or `\u0061` or + /// `\U00000061`. + HexFixed(HexLiteralKind), + /// The literal is written as a hex code with a bracketed number of + /// digits. The only restriction is that the bracketed hex code must refer + /// to a valid Unicode scalar value. + HexBrace(HexLiteralKind), + /// The literal is written as a specially recognized escape, e.g., `\f` + /// or `\n`. + Special(SpecialLiteralKind), +} + +/// The type of a special literal. +/// +/// A special literal is a special escape sequence recognized by the regex +/// parser, e.g., `\f` or `\n`. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum SpecialLiteralKind { + /// Bell, spelled `\a` (`\x07`). + Bell, + /// Form feed, spelled `\f` (`\x0C`). + FormFeed, + /// Tab, spelled `\t` (`\x09`). + Tab, + /// Line feed, spelled `\n` (`\x0A`). + LineFeed, + /// Carriage return, spelled `\r` (`\x0D`). + CarriageReturn, + /// Vertical tab, spelled `\v` (`\x0B`). + VerticalTab, + /// Space, spelled `\ ` (`\x20`). Note that this can only appear when + /// parsing in verbose mode. + Space, +} + +/// The type of a Unicode hex literal. +/// +/// Note that all variants behave the same when used with brackets. They only +/// differ when used without brackets in the number of hex digits that must +/// follow. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum HexLiteralKind { + /// A `\x` prefix. When used without brackets, this form is limited to + /// two digits. + X, + /// A `\u` prefix. When used without brackets, this form is limited to + /// four digits. + UnicodeShort, + /// A `\U` prefix. When used without brackets, this form is limited to + /// eight digits. + UnicodeLong, +} + +impl HexLiteralKind { + /// The number of digits that must be used with this literal form when + /// used without brackets. When used with brackets, there is no + /// restriction on the number of digits. + pub fn digits(&self) -> u32 { + match *self { + HexLiteralKind::X => 2, + HexLiteralKind::UnicodeShort => 4, + HexLiteralKind::UnicodeLong => 8, + } + } +} + +/// A Perl character class. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct ClassPerl { + /// The span of this class. + pub span: Span, + /// The kind of Perl class. + pub kind: ClassPerlKind, + /// Whether the class is negated or not. e.g., `\d` is not negated but + /// `\D` is. + pub negated: bool, +} + +/// The available Perl character classes. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ClassPerlKind { + /// Decimal numbers. + Digit, + /// Whitespace. + Space, + /// Word characters. + Word, +} + +/// An ASCII character class. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct ClassAscii { + /// The span of this class. + pub span: Span, + /// The kind of ASCII class. + pub kind: ClassAsciiKind, + /// Whether the class is negated or not. e.g., `[[:alpha:]]` is not negated + /// but `[[:^alpha:]]` is. + pub negated: bool, +} + +/// The available ASCII character classes. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ClassAsciiKind { + /// `[0-9A-Za-z]` + Alnum, + /// `[A-Za-z]` + Alpha, + /// `[\x00-\x7F]` + Ascii, + /// `[ \t]` + Blank, + /// `[\x00-\x1F\x7F]` + Cntrl, + /// `[0-9]` + Digit, + /// `[!-~]` + Graph, + /// `[a-z]` + Lower, + /// `[ -~]` + Print, + /// ``[!-/:-@\[-`{-~]`` + Punct, + /// `[\t\n\v\f\r ]` + Space, + /// `[A-Z]` + Upper, + /// `[0-9A-Za-z_]` + Word, + /// `[0-9A-Fa-f]` + Xdigit, +} + +impl ClassAsciiKind { + /// Return the corresponding ClassAsciiKind variant for the given name. + /// + /// The name given should correspond to the lowercase version of the + /// variant name. e.g., `cntrl` is the name for `ClassAsciiKind::Cntrl`. + /// + /// If no variant with the corresponding name exists, then `None` is + /// returned. + pub fn from_name(name: &str) -> Option<ClassAsciiKind> { + use self::ClassAsciiKind::*; + match name { + "alnum" => Some(Alnum), + "alpha" => Some(Alpha), + "ascii" => Some(Ascii), + "blank" => Some(Blank), + "cntrl" => Some(Cntrl), + "digit" => Some(Digit), + "graph" => Some(Graph), + "lower" => Some(Lower), + "print" => Some(Print), + "punct" => Some(Punct), + "space" => Some(Space), + "upper" => Some(Upper), + "word" => Some(Word), + "xdigit" => Some(Xdigit), + _ => None, + } + } +} + +/// A Unicode character class. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct ClassUnicode { + /// The span of this class. + pub span: Span, + /// Whether this class is negated or not. + /// + /// Note: be careful when using this attribute. This specifically refers + /// to whether the class is written as `\p` or `\P`, where the latter + /// is `negated = true`. However, it also possible to write something like + /// `\P{scx!=Katakana}` which is actually equivalent to + /// `\p{scx=Katakana}` and is therefore not actually negated even though + /// `negated = true` here. To test whether this class is truly negated + /// or not, use the `is_negated` method. + pub negated: bool, + /// The kind of Unicode class. + pub kind: ClassUnicodeKind, +} + +impl ClassUnicode { + /// Returns true if this class has been negated. + /// + /// Note that this takes the Unicode op into account, if it's present. + /// e.g., `is_negated` for `\P{scx!=Katakana}` will return `false`. + pub fn is_negated(&self) -> bool { + match self.kind { + ClassUnicodeKind::NamedValue { + op: ClassUnicodeOpKind::NotEqual, + .. + } => !self.negated, + _ => self.negated, + } + } +} + +/// The available forms of Unicode character classes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ClassUnicodeKind { + /// A one letter abbreviated class, e.g., `\pN`. + OneLetter(char), + /// A binary property, general category or script. The string may be + /// empty. + Named(String), + /// A property name and an associated value. + NamedValue { + /// The type of Unicode op used to associate `name` with `value`. + op: ClassUnicodeOpKind, + /// The property name (which may be empty). + name: String, + /// The property value (which may be empty). + value: String, + }, +} + +#[cfg(feature = "arbitrary")] +impl arbitrary::Arbitrary<'_> for ClassUnicodeKind { + fn arbitrary( + u: &mut arbitrary::Unstructured, + ) -> arbitrary::Result<ClassUnicodeKind> { + #[cfg(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", + ))] + { + use alloc::string::ToString; + + use super::unicode_tables::{ + property_names::PROPERTY_NAMES, + property_values::PROPERTY_VALUES, + }; + + match u.choose_index(3)? { + 0 => { + let all = PROPERTY_VALUES + .iter() + .flat_map(|e| e.1.iter()) + .filter(|(name, _)| name.len() == 1) + .count(); + let idx = u.choose_index(all)?; + let value = PROPERTY_VALUES + .iter() + .flat_map(|e| e.1.iter()) + .take(idx + 1) + .last() + .unwrap() + .0 + .chars() + .next() + .unwrap(); + Ok(ClassUnicodeKind::OneLetter(value)) + } + 1 => { + let all = PROPERTY_VALUES + .iter() + .map(|e| e.1.len()) + .sum::<usize>() + + PROPERTY_NAMES.len(); + let idx = u.choose_index(all)?; + let name = PROPERTY_VALUES + .iter() + .flat_map(|e| e.1.iter()) + .chain(PROPERTY_NAMES) + .map(|(_, e)| e) + .take(idx + 1) + .last() + .unwrap(); + Ok(ClassUnicodeKind::Named(name.to_string())) + } + 2 => { + let all = PROPERTY_VALUES + .iter() + .map(|e| e.1.len()) + .sum::<usize>(); + let idx = u.choose_index(all)?; + let (prop, value) = PROPERTY_VALUES + .iter() + .flat_map(|e| { + e.1.iter().map(|(_, value)| (e.0, value)) + }) + .take(idx + 1) + .last() + .unwrap(); + Ok(ClassUnicodeKind::NamedValue { + op: u.arbitrary()?, + name: prop.to_string(), + value: value.to_string(), + }) + } + _ => unreachable!("index chosen is impossible"), + } + } + #[cfg(not(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", + )))] + { + match u.choose_index(3)? { + 0 => Ok(ClassUnicodeKind::OneLetter(u.arbitrary()?)), + 1 => Ok(ClassUnicodeKind::Named(u.arbitrary()?)), + 2 => Ok(ClassUnicodeKind::NamedValue { + op: u.arbitrary()?, + name: u.arbitrary()?, + value: u.arbitrary()?, + }), + _ => unreachable!("index chosen is impossible"), + } + } + } + + fn size_hint(depth: usize) -> (usize, Option<usize>) { + #[cfg(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", + ))] + { + arbitrary::size_hint::and_all(&[ + usize::size_hint(depth), + usize::size_hint(depth), + arbitrary::size_hint::or( + (0, Some(0)), + ClassUnicodeOpKind::size_hint(depth), + ), + ]) + } + #[cfg(not(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", + )))] + { + arbitrary::size_hint::and( + usize::size_hint(depth), + arbitrary::size_hint::or_all(&[ + char::size_hint(depth), + String::size_hint(depth), + arbitrary::size_hint::and_all(&[ + String::size_hint(depth), + String::size_hint(depth), + ClassUnicodeOpKind::size_hint(depth), + ]), + ]), + ) + } + } +} + +/// The type of op used in a Unicode character class. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ClassUnicodeOpKind { + /// A property set to a specific value, e.g., `\p{scx=Katakana}`. + Equal, + /// A property set to a specific value using a colon, e.g., + /// `\p{scx:Katakana}`. + Colon, + /// A property that isn't a particular value, e.g., `\p{scx!=Katakana}`. + NotEqual, +} + +impl ClassUnicodeOpKind { + /// Whether the op is an equality op or not. + pub fn is_equal(&self) -> bool { + match *self { + ClassUnicodeOpKind::Equal | ClassUnicodeOpKind::Colon => true, + _ => false, + } + } +} + +/// A bracketed character class, e.g., `[a-z0-9]`. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct ClassBracketed { + /// The span of this class. + pub span: Span, + /// Whether this class is negated or not. e.g., `[a]` is not negated but + /// `[^a]` is. + pub negated: bool, + /// The type of this set. A set is either a normal union of things, e.g., + /// `[abc]` or a result of applying set operations, e.g., `[\pL--c]`. + pub kind: ClassSet, +} + +/// A character class set. +/// +/// This type corresponds to the internal structure of a bracketed character +/// class. That is, every bracketed character is one of two types: a union of +/// items (literals, ranges, other bracketed classes) or a tree of binary set +/// operations. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ClassSet { + /// An item, which can be a single literal, range, nested character class + /// or a union of items. + Item(ClassSetItem), + /// A single binary operation (i.e., &&, -- or ~~). + BinaryOp(ClassSetBinaryOp), +} + +impl ClassSet { + /// Build a set from a union. + pub fn union(ast: ClassSetUnion) -> ClassSet { + ClassSet::Item(ClassSetItem::Union(ast)) + } + + /// Return the span of this character class set. + pub fn span(&self) -> &Span { + match *self { + ClassSet::Item(ref x) => x.span(), + ClassSet::BinaryOp(ref x) => &x.span, + } + } + + /// Return true if and only if this class set is empty. + fn is_empty(&self) -> bool { + match *self { + ClassSet::Item(ClassSetItem::Empty(_)) => true, + _ => false, + } + } +} + +/// A single component of a character class set. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ClassSetItem { + /// An empty item. + /// + /// Note that a bracketed character class cannot contain a single empty + /// item. Empty items can appear when using one of the binary operators. + /// For example, `[&&]` is the intersection of two empty classes. + Empty(Span), + /// A single literal. + Literal(Literal), + /// A range between two literals. + Range(ClassSetRange), + /// An ASCII character class, e.g., `[:alnum:]` or `[:punct:]`. + Ascii(ClassAscii), + /// A Unicode character class, e.g., `\pL` or `\p{Greek}`. + Unicode(ClassUnicode), + /// A perl character class, e.g., `\d` or `\W`. + Perl(ClassPerl), + /// A bracketed character class set, which may contain zero or more + /// character ranges and/or zero or more nested classes. e.g., + /// `[a-zA-Z\pL]`. + Bracketed(Box<ClassBracketed>), + /// A union of items. + Union(ClassSetUnion), +} + +impl ClassSetItem { + /// Return the span of this character class set item. + pub fn span(&self) -> &Span { + match *self { + ClassSetItem::Empty(ref span) => span, + ClassSetItem::Literal(ref x) => &x.span, + ClassSetItem::Range(ref x) => &x.span, + ClassSetItem::Ascii(ref x) => &x.span, + ClassSetItem::Perl(ref x) => &x.span, + ClassSetItem::Unicode(ref x) => &x.span, + ClassSetItem::Bracketed(ref x) => &x.span, + ClassSetItem::Union(ref x) => &x.span, + } + } +} + +/// A single character class range in a set. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct ClassSetRange { + /// The span of this range. + pub span: Span, + /// The start of this range. + pub start: Literal, + /// The end of this range. + pub end: Literal, +} + +impl ClassSetRange { + /// Returns true if and only if this character class range is valid. + /// + /// The only case where a range is invalid is if its start is greater than + /// its end. + pub fn is_valid(&self) -> bool { + self.start.c <= self.end.c + } +} + +/// A union of items inside a character class set. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct ClassSetUnion { + /// The span of the items in this operation. e.g., the `a-z0-9` in + /// `[^a-z0-9]` + pub span: Span, + /// The sequence of items that make up this union. + pub items: Vec<ClassSetItem>, +} + +impl ClassSetUnion { + /// Push a new item in this union. + /// + /// The ending position of this union's span is updated to the ending + /// position of the span of the item given. If the union is empty, then + /// the starting position of this union is set to the starting position + /// of this item. + /// + /// In other words, if you only use this method to add items to a union + /// and you set the spans on each item correctly, then you should never + /// need to adjust the span of the union directly. + pub fn push(&mut self, item: ClassSetItem) { + if self.items.is_empty() { + self.span.start = item.span().start; + } + self.span.end = item.span().end; + self.items.push(item); + } + + /// Return this union as a character class set item. + /// + /// If this union contains zero items, then an empty union is + /// returned. If this concatenation contains exactly 1 item, then the + /// corresponding item is returned. Otherwise, ClassSetItem::Union is + /// returned. + pub fn into_item(mut self) -> ClassSetItem { + match self.items.len() { + 0 => ClassSetItem::Empty(self.span), + 1 => self.items.pop().unwrap(), + _ => ClassSetItem::Union(self), + } + } +} + +/// A Unicode character class set operation. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct ClassSetBinaryOp { + /// The span of this operation. e.g., the `a-z--[h-p]` in `[a-z--h-p]`. + pub span: Span, + /// The type of this set operation. + pub kind: ClassSetBinaryOpKind, + /// The left hand side of the operation. + pub lhs: Box<ClassSet>, + /// The right hand side of the operation. + pub rhs: Box<ClassSet>, +} + +/// The type of a Unicode character class set operation. +/// +/// Note that this doesn't explicitly represent union since there is no +/// explicit union operator. Concatenation inside a character class corresponds +/// to the union operation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum ClassSetBinaryOpKind { + /// The intersection of two sets, e.g., `\pN&&[a-z]`. + Intersection, + /// The difference of two sets, e.g., `\pN--[0-9]`. + Difference, + /// The symmetric difference of two sets. The symmetric difference is the + /// set of elements belonging to one but not both sets. + /// e.g., `[\pL~~[:ascii:]]`. + SymmetricDifference, +} + +/// A single zero-width assertion. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Assertion { + /// The span of this assertion. + pub span: Span, + /// The assertion kind, e.g., `\b` or `^`. + pub kind: AssertionKind, +} + +/// An assertion kind. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum AssertionKind { + /// `^` + StartLine, + /// `$` + EndLine, + /// `\A` + StartText, + /// `\z` + EndText, + /// `\b` + WordBoundary, + /// `\B` + NotWordBoundary, + /// `\b{start}` + WordBoundaryStart, + /// `\b{end}` + WordBoundaryEnd, + /// `\<` (alias for `\b{start}`) + WordBoundaryStartAngle, + /// `\>` (alias for `\b{end}`) + WordBoundaryEndAngle, + /// `\b{start-half}` + WordBoundaryStartHalf, + /// `\b{end-half}` + WordBoundaryEndHalf, +} + +/// A repetition operation applied to a regular expression. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Repetition { + /// The span of this operation. + pub span: Span, + /// The actual operation. + pub op: RepetitionOp, + /// Whether this operation was applied greedily or not. + pub greedy: bool, + /// The regular expression under repetition. + pub ast: Box<Ast>, +} + +/// The repetition operator itself. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct RepetitionOp { + /// The span of this operator. This includes things like `+`, `*?` and + /// `{m,n}`. + pub span: Span, + /// The type of operation. + pub kind: RepetitionKind, +} + +/// The kind of a repetition operator. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum RepetitionKind { + /// `?` + ZeroOrOne, + /// `*` + ZeroOrMore, + /// `+` + OneOrMore, + /// `{m,n}` + Range(RepetitionRange), +} + +/// A range repetition operator. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum RepetitionRange { + /// `{m}` + Exactly(u32), + /// `{m,}` + AtLeast(u32), + /// `{m,n}` + Bounded(u32, u32), +} + +impl RepetitionRange { + /// Returns true if and only if this repetition range is valid. + /// + /// The only case where a repetition range is invalid is if it is bounded + /// and its start is greater than its end. + pub fn is_valid(&self) -> bool { + match *self { + RepetitionRange::Bounded(s, e) if s > e => false, + _ => true, + } + } +} + +/// A grouped regular expression. +/// +/// This includes both capturing and non-capturing groups. This does **not** +/// include flag-only groups like `(?is)`, but does contain any group that +/// contains a sub-expression, e.g., `(a)`, `(?P<name>a)`, `(?:a)` and +/// `(?is:a)`. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Group { + /// The span of this group. + pub span: Span, + /// The kind of this group. + pub kind: GroupKind, + /// The regular expression in this group. + pub ast: Box<Ast>, +} + +impl Group { + /// If this group is non-capturing, then this returns the (possibly empty) + /// set of flags. Otherwise, `None` is returned. + pub fn flags(&self) -> Option<&Flags> { + match self.kind { + GroupKind::NonCapturing(ref flags) => Some(flags), + _ => None, + } + } + + /// Returns true if and only if this group is capturing. + pub fn is_capturing(&self) -> bool { + match self.kind { + GroupKind::CaptureIndex(_) | GroupKind::CaptureName { .. } => true, + GroupKind::NonCapturing(_) => false, + } + } + + /// Returns the capture index of this group, if this is a capturing group. + /// + /// This returns a capture index precisely when `is_capturing` is `true`. + pub fn capture_index(&self) -> Option<u32> { + match self.kind { + GroupKind::CaptureIndex(i) => Some(i), + GroupKind::CaptureName { ref name, .. } => Some(name.index), + GroupKind::NonCapturing(_) => None, + } + } +} + +/// The kind of a group. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum GroupKind { + /// `(a)` + CaptureIndex(u32), + /// `(?<name>a)` or `(?P<name>a)` + CaptureName { + /// True if the `?P<` syntax is used and false if the `?<` syntax is used. + starts_with_p: bool, + /// The capture name. + name: CaptureName, + }, + /// `(?:a)` and `(?i:a)` + NonCapturing(Flags), +} + +/// A capture name. +/// +/// This corresponds to the name itself between the angle brackets in, e.g., +/// `(?P<foo>expr)`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CaptureName { + /// The span of this capture name. + pub span: Span, + /// The capture name. + pub name: String, + /// The capture index. + pub index: u32, +} + +#[cfg(feature = "arbitrary")] +impl arbitrary::Arbitrary<'_> for CaptureName { + fn arbitrary( + u: &mut arbitrary::Unstructured, + ) -> arbitrary::Result<CaptureName> { + let len = u.arbitrary_len::<char>()?; + if len == 0 { + return Err(arbitrary::Error::NotEnoughData); + } + let mut name: String = String::new(); + for _ in 0..len { + let ch: char = u.arbitrary()?; + let cp = u32::from(ch); + let ascii_letter_offset = u8::try_from(cp % 26).unwrap(); + let ascii_letter = b'a' + ascii_letter_offset; + name.push(char::from(ascii_letter)); + } + Ok(CaptureName { span: u.arbitrary()?, name, index: u.arbitrary()? }) + } + + fn size_hint(depth: usize) -> (usize, Option<usize>) { + arbitrary::size_hint::and_all(&[ + Span::size_hint(depth), + usize::size_hint(depth), + u32::size_hint(depth), + ]) + } +} + +/// A group of flags that is not applied to a particular regular expression. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct SetFlags { + /// The span of these flags, including the grouping parentheses. + pub span: Span, + /// The actual sequence of flags. + pub flags: Flags, +} + +/// A group of flags. +/// +/// This corresponds only to the sequence of flags themselves, e.g., `is-u`. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct Flags { + /// The span of this group of flags. + pub span: Span, + /// A sequence of flag items. Each item is either a flag or a negation + /// operator. + pub items: Vec<FlagsItem>, +} + +impl Flags { + /// Add the given item to this sequence of flags. + /// + /// If the item was added successfully, then `None` is returned. If the + /// given item is a duplicate, then `Some(i)` is returned, where + /// `items[i].kind == item.kind`. + pub fn add_item(&mut self, item: FlagsItem) -> Option<usize> { + for (i, x) in self.items.iter().enumerate() { + if x.kind == item.kind { + return Some(i); + } + } + self.items.push(item); + None + } + + /// Returns the state of the given flag in this set. + /// + /// If the given flag is in the set but is negated, then `Some(false)` is + /// returned. + /// + /// If the given flag is in the set and is not negated, then `Some(true)` + /// is returned. + /// + /// Otherwise, `None` is returned. + pub fn flag_state(&self, flag: Flag) -> Option<bool> { + let mut negated = false; + for x in &self.items { + match x.kind { + FlagsItemKind::Negation => { + negated = true; + } + FlagsItemKind::Flag(ref xflag) if xflag == &flag => { + return Some(!negated); + } + _ => {} + } + } + None + } +} + +/// A single item in a group of flags. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub struct FlagsItem { + /// The span of this item. + pub span: Span, + /// The kind of this item. + pub kind: FlagsItemKind, +} + +/// The kind of an item in a group of flags. +#[derive(Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum FlagsItemKind { + /// A negation operator applied to all subsequent flags in the enclosing + /// group. + Negation, + /// A single flag in a group. + Flag(Flag), +} + +impl FlagsItemKind { + /// Returns true if and only if this item is a negation operator. + pub fn is_negation(&self) -> bool { + match *self { + FlagsItemKind::Negation => true, + _ => false, + } + } +} + +/// A single flag. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +pub enum Flag { + /// `i` + CaseInsensitive, + /// `m` + MultiLine, + /// `s` + DotMatchesNewLine, + /// `U` + SwapGreed, + /// `u` + Unicode, + /// `R` + CRLF, + /// `x` + IgnoreWhitespace, +} + +/// A custom `Drop` impl is used for `Ast` such that it uses constant stack +/// space but heap space proportional to the depth of the `Ast`. +impl Drop for Ast { + fn drop(&mut self) { + use core::mem; + + match *self { + Ast::Empty(_) + | Ast::Flags(_) + | Ast::Literal(_) + | Ast::Dot(_) + | Ast::Assertion(_) + | Ast::ClassUnicode(_) + | Ast::ClassPerl(_) + // Bracketed classes are recursive, they get their own Drop impl. + | Ast::ClassBracketed(_) => return, + Ast::Repetition(ref x) if !x.ast.has_subexprs() => return, + Ast::Group(ref x) if !x.ast.has_subexprs() => return, + Ast::Alternation(ref x) if x.asts.is_empty() => return, + Ast::Concat(ref x) if x.asts.is_empty() => return, + _ => {} + } + + let empty_span = || Span::splat(Position::new(0, 0, 0)); + let empty_ast = || Ast::empty(empty_span()); + let mut stack = vec![mem::replace(self, empty_ast())]; + while let Some(mut ast) = stack.pop() { + match ast { + Ast::Empty(_) + | Ast::Flags(_) + | Ast::Literal(_) + | Ast::Dot(_) + | Ast::Assertion(_) + | Ast::ClassUnicode(_) + | Ast::ClassPerl(_) + // Bracketed classes are recursive, so they get their own Drop + // impl. + | Ast::ClassBracketed(_) => {} + Ast::Repetition(ref mut x) => { + stack.push(mem::replace(&mut x.ast, empty_ast())); + } + Ast::Group(ref mut x) => { + stack.push(mem::replace(&mut x.ast, empty_ast())); + } + Ast::Alternation(ref mut x) => { + stack.extend(x.asts.drain(..)); + } + Ast::Concat(ref mut x) => { + stack.extend(x.asts.drain(..)); + } + } + } + } +} + +/// A custom `Drop` impl is used for `ClassSet` such that it uses constant +/// stack space but heap space proportional to the depth of the `ClassSet`. +impl Drop for ClassSet { + fn drop(&mut self) { + use core::mem; + + match *self { + ClassSet::Item(ref item) => match *item { + ClassSetItem::Empty(_) + | ClassSetItem::Literal(_) + | ClassSetItem::Range(_) + | ClassSetItem::Ascii(_) + | ClassSetItem::Unicode(_) + | ClassSetItem::Perl(_) => return, + ClassSetItem::Bracketed(ref x) => { + if x.kind.is_empty() { + return; + } + } + ClassSetItem::Union(ref x) => { + if x.items.is_empty() { + return; + } + } + }, + ClassSet::BinaryOp(ref op) => { + if op.lhs.is_empty() && op.rhs.is_empty() { + return; + } + } + } + + let empty_span = || Span::splat(Position::new(0, 0, 0)); + let empty_set = || ClassSet::Item(ClassSetItem::Empty(empty_span())); + let mut stack = vec![mem::replace(self, empty_set())]; + while let Some(mut set) = stack.pop() { + match set { + ClassSet::Item(ref mut item) => match *item { + ClassSetItem::Empty(_) + | ClassSetItem::Literal(_) + | ClassSetItem::Range(_) + | ClassSetItem::Ascii(_) + | ClassSetItem::Unicode(_) + | ClassSetItem::Perl(_) => {} + ClassSetItem::Bracketed(ref mut x) => { + stack.push(mem::replace(&mut x.kind, empty_set())); + } + ClassSetItem::Union(ref mut x) => { + stack.extend(x.items.drain(..).map(ClassSet::Item)); + } + }, + ClassSet::BinaryOp(ref mut op) => { + stack.push(mem::replace(&mut op.lhs, empty_set())); + stack.push(mem::replace(&mut op.rhs, empty_set())); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // We use a thread with an explicit stack size to test that our destructor + // for Ast can handle arbitrarily sized expressions in constant stack + // space. In case we run on a platform without threads (WASM?), we limit + // this test to Windows/Unix. + #[test] + #[cfg(any(unix, windows))] + fn no_stack_overflow_on_drop() { + use std::thread; + + let run = || { + let span = || Span::splat(Position::new(0, 0, 0)); + let mut ast = Ast::empty(span()); + for i in 0..200 { + ast = Ast::group(Group { + span: span(), + kind: GroupKind::CaptureIndex(i), + ast: Box::new(ast), + }); + } + assert!(!ast.is_empty()); + }; + + // We run our test on a thread with a small stack size so we can + // force the issue more easily. + // + // NOTE(2023-03-21): It turns out that some platforms (like FreeBSD) + // will just barf with very small stack sizes. So we bump this up a bit + // to give more room to breath. When I did this, I confirmed that if + // I remove the custom `Drop` impl for `Ast`, then this test does + // indeed still fail with a stack overflow. (At the time of writing, I + // had to bump it all the way up to 32K before the test would pass even + // without the custom `Drop` impl. So 16K seems like a safe number + // here.) + // + // See: https://github.com/rust-lang/regex/issues/967 + thread::Builder::new() + .stack_size(16 << 10) + .spawn(run) + .unwrap() + .join() + .unwrap(); + } + + // This tests that our `Ast` has a reasonable size. This isn't a hard rule + // and it can be increased if given a good enough reason. But this test + // exists because the size of `Ast` was at one point over 200 bytes on a + // 64-bit target. Wow. + #[test] + fn ast_size() { + let max = 2 * core::mem::size_of::<usize>(); + let size = core::mem::size_of::<Ast>(); + assert!( + size <= max, + "Ast size of {size} bytes is bigger than suggested max {max}", + ); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/ast/parse.rs b/third_party/crates/regex-syntax-0.8.11/src/ast/parse.rs new file mode 100644 index 0000000..bdaab72 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/ast/parse.rs @@ -0,0 +1,6377 @@ +/*! +This module provides a regular expression parser. +*/ + +use core::{ + borrow::Borrow, + cell::{Cell, RefCell}, + mem, +}; + +use alloc::{ + boxed::Box, + string::{String, ToString}, + vec, + vec::Vec, +}; + +use crate::{ + ast::{self, Ast, Position, Span}, + either::Either, + is_escapeable_character, is_meta_character, +}; + +type Result<T> = core::result::Result<T, ast::Error>; + +/// A primitive is an expression with no sub-expressions. This includes +/// literals, assertions and non-set character classes. This representation +/// is used as intermediate state in the parser. +/// +/// This does not include ASCII character classes, since they can only appear +/// within a set character class. +#[derive(Clone, Debug, Eq, PartialEq)] +enum Primitive { + Literal(ast::Literal), + Assertion(ast::Assertion), + Dot(Span), + Perl(ast::ClassPerl), + Unicode(ast::ClassUnicode), +} + +impl Primitive { + /// Return the span of this primitive. + fn span(&self) -> &Span { + match *self { + Primitive::Literal(ref x) => &x.span, + Primitive::Assertion(ref x) => &x.span, + Primitive::Dot(ref span) => span, + Primitive::Perl(ref x) => &x.span, + Primitive::Unicode(ref x) => &x.span, + } + } + + /// Convert this primitive into a proper AST. + fn into_ast(self) -> Ast { + match self { + Primitive::Literal(lit) => Ast::literal(lit), + Primitive::Assertion(assert) => Ast::assertion(assert), + Primitive::Dot(span) => Ast::dot(span), + Primitive::Perl(cls) => Ast::class_perl(cls), + Primitive::Unicode(cls) => Ast::class_unicode(cls), + } + } + + /// Convert this primitive into an item in a character class. + /// + /// If this primitive is not a legal item (i.e., an assertion or a dot), + /// then return an error. + fn into_class_set_item<P: Borrow<Parser>>( + self, + p: &ParserI<'_, P>, + ) -> Result<ast::ClassSetItem> { + use self::Primitive::*; + use crate::ast::ClassSetItem; + + match self { + Literal(lit) => Ok(ClassSetItem::Literal(lit)), + Perl(cls) => Ok(ClassSetItem::Perl(cls)), + Unicode(cls) => Ok(ClassSetItem::Unicode(cls)), + x => Err(p.error(*x.span(), ast::ErrorKind::ClassEscapeInvalid)), + } + } + + /// Convert this primitive into a literal in a character class. In + /// particular, literals are the only valid items that can appear in + /// ranges. + /// + /// If this primitive is not a legal item (i.e., a class, assertion or a + /// dot), then return an error. + fn into_class_literal<P: Borrow<Parser>>( + self, + p: &ParserI<'_, P>, + ) -> Result<ast::Literal> { + use self::Primitive::*; + + match self { + Literal(lit) => Ok(lit), + x => Err(p.error(*x.span(), ast::ErrorKind::ClassRangeLiteral)), + } + } +} + +/// Returns true if the given character is a hexadecimal digit. +fn is_hex(c: char) -> bool { + ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F') +} + +/// Returns true if the given character is a valid in a capture group name. +/// +/// If `first` is true, then `c` is treated as the first character in the +/// group name (which must be alphabetic or underscore). +fn is_capture_char(c: char, first: bool) -> bool { + if first { + c == '_' || c.is_alphabetic() + } else { + c == '_' || c == '.' || c == '[' || c == ']' || c.is_alphanumeric() + } +} + +/// A builder for a regular expression parser. +/// +/// This builder permits modifying configuration options for the parser. +#[derive(Clone, Debug)] +pub struct ParserBuilder { + ignore_whitespace: bool, + nest_limit: u32, + octal: bool, + empty_min_range: bool, +} + +impl Default for ParserBuilder { + fn default() -> ParserBuilder { + ParserBuilder::new() + } +} + +impl ParserBuilder { + /// Create a new parser builder with a default configuration. + pub fn new() -> ParserBuilder { + ParserBuilder { + ignore_whitespace: false, + nest_limit: 250, + octal: false, + empty_min_range: false, + } + } + + /// Build a parser from this configuration with the given pattern. + pub fn build(&self) -> Parser { + Parser { + pos: Cell::new(Position { offset: 0, line: 1, column: 1 }), + capture_index: Cell::new(0), + nest_limit: self.nest_limit, + octal: self.octal, + empty_min_range: self.empty_min_range, + initial_ignore_whitespace: self.ignore_whitespace, + ignore_whitespace: Cell::new(self.ignore_whitespace), + comments: RefCell::new(vec![]), + stack_group: RefCell::new(vec![]), + stack_class: RefCell::new(vec![]), + capture_names: RefCell::new(vec![]), + scratch: RefCell::new(String::new()), + } + } + + /// Set the nesting limit for this parser. + /// + /// The nesting limit controls how deep the abstract syntax tree is allowed + /// to be. If the AST exceeds the given limit (e.g., with too many nested + /// groups), then an error is returned by the parser. + /// + /// The purpose of this limit is to act as a heuristic to prevent stack + /// overflow for consumers that do structural induction on an `Ast` using + /// explicit recursion. While this crate never does this (instead using + /// constant stack space and moving the call stack to the heap), other + /// crates may. + /// + /// This limit is not checked until the entire AST is parsed. Therefore, + /// if callers want to put a limit on the amount of heap space used, then + /// they should impose a limit on the length, in bytes, of the concrete + /// pattern string. In particular, this is viable since this parser + /// implementation will limit itself to heap space proportional to the + /// length of the pattern string. + /// + /// Note that a nest limit of `0` will return a nest limit error for most + /// patterns but not all. For example, a nest limit of `0` permits `a` but + /// not `ab`, since `ab` requires a concatenation, which results in a nest + /// depth of `1`. In general, a nest limit is not something that manifests + /// in an obvious way in the concrete syntax, therefore, it should not be + /// used in a granular way. + pub fn nest_limit(&mut self, limit: u32) -> &mut ParserBuilder { + self.nest_limit = limit; + self + } + + /// Whether to support octal syntax or not. + /// + /// Octal syntax is a little-known way of uttering Unicode codepoints in + /// a regular expression. For example, `a`, `\x61`, `\u0061` and + /// `\141` are all equivalent regular expressions, where the last example + /// shows octal syntax. + /// + /// While supporting octal syntax isn't in and of itself a problem, it does + /// make good error messages harder. That is, in PCRE based regex engines, + /// syntax like `\0` invokes a backreference, which is explicitly + /// unsupported in Rust's regex engine. However, many users expect it to + /// be supported. Therefore, when octal support is disabled, the error + /// message will explicitly mention that backreferences aren't supported. + /// + /// Octal syntax is disabled by default. + pub fn octal(&mut self, yes: bool) -> &mut ParserBuilder { + self.octal = yes; + self + } + + /// Enable verbose mode in the regular expression. + /// + /// When enabled, verbose mode permits insignificant whitespace in many + /// places in the regular expression, as well as comments. Comments are + /// started using `#` and continue until the end of the line. + /// + /// By default, this is disabled. It may be selectively enabled in the + /// regular expression by using the `x` flag regardless of this setting. + pub fn ignore_whitespace(&mut self, yes: bool) -> &mut ParserBuilder { + self.ignore_whitespace = yes; + self + } + + /// Allow using `{,n}` as an equivalent to `{0,n}`. + /// + /// When enabled, the parser accepts `{,n}` as valid syntax for `{0,n}`. + /// Most regular expression engines don't support the `{,n}` syntax, but + /// some others do it, namely Python's `re` library. + /// + /// This is disabled by default. + pub fn empty_min_range(&mut self, yes: bool) -> &mut ParserBuilder { + self.empty_min_range = yes; + self + } +} + +/// A regular expression parser. +/// +/// This parses a string representation of a regular expression into an +/// abstract syntax tree. The size of the tree is proportional to the length +/// of the regular expression pattern. +/// +/// A `Parser` can be configured in more detail via a [`ParserBuilder`]. +#[derive(Clone, Debug)] +pub struct Parser { + /// The current position of the parser. + pos: Cell<Position>, + /// The current capture index. + capture_index: Cell<u32>, + /// The maximum number of open parens/brackets allowed. If the parser + /// exceeds this number, then an error is returned. + nest_limit: u32, + /// Whether to support octal syntax or not. When `false`, the parser will + /// return an error helpfully pointing out that backreferences are not + /// supported. + octal: bool, + /// The initial setting for `ignore_whitespace` as provided by + /// `ParserBuilder`. It is used when resetting the parser's state. + initial_ignore_whitespace: bool, + /// Whether the parser supports `{,n}` repetitions as an equivalent to + /// `{0,n}.` + empty_min_range: bool, + /// Whether whitespace should be ignored. When enabled, comments are + /// also permitted. + ignore_whitespace: Cell<bool>, + /// A list of comments, in order of appearance. + comments: RefCell<Vec<ast::Comment>>, + /// A stack of grouped sub-expressions, including alternations. + stack_group: RefCell<Vec<GroupState>>, + /// A stack of nested character classes. This is only non-empty when + /// parsing a class. + stack_class: RefCell<Vec<ClassState>>, + /// A sorted sequence of capture names. This is used to detect duplicate + /// capture names and report an error if one is detected. + capture_names: RefCell<Vec<ast::CaptureName>>, + /// A scratch buffer used in various places. Mostly this is used to + /// accumulate relevant characters from parts of a pattern. + scratch: RefCell<String>, +} + +/// ParserI is the internal parser implementation. +/// +/// We use this separate type so that we can carry the provided pattern string +/// along with us. In particular, a `Parser` internal state is not tied to any +/// one pattern, but `ParserI` is. +/// +/// This type also lets us use `ParserI<&Parser>` in production code while +/// retaining the convenience of `ParserI<Parser>` for tests, which sometimes +/// work against the internal interface of the parser. +#[derive(Clone, Debug)] +struct ParserI<'s, P> { + /// The parser state/configuration. + parser: P, + /// The full regular expression provided by the user. + pattern: &'s str, +} + +/// GroupState represents a single stack frame while parsing nested groups +/// and alternations. Each frame records the state up to an opening parenthesis +/// or a alternating bracket `|`. +#[derive(Clone, Debug)] +enum GroupState { + /// This state is pushed whenever an opening group is found. + Group { + /// The concatenation immediately preceding the opening group. + concat: ast::Concat, + /// The group that has been opened. Its sub-AST is always empty. + group: ast::Group, + /// Whether this group has the `x` flag enabled or not. + ignore_whitespace: bool, + }, + /// This state is pushed whenever a new alternation branch is found. If + /// an alternation branch is found and this state is at the top of the + /// stack, then this state should be modified to include the new + /// alternation. + Alternation(ast::Alternation), +} + +/// ClassState represents a single stack frame while parsing character classes. +/// Each frame records the state up to an intersection, difference, symmetric +/// difference or nested class. +/// +/// Note that a parser's character class stack is only non-empty when parsing +/// a character class. In all other cases, it is empty. +#[derive(Clone, Debug)] +enum ClassState { + /// This state is pushed whenever an opening bracket is found. + Open { + /// The union of class items immediately preceding this class. + union: ast::ClassSetUnion, + /// The class that has been opened. Typically this just corresponds + /// to the `[`, but it can also include `[^` since `^` indicates + /// negation of the class. + set: ast::ClassBracketed, + }, + /// This state is pushed when a operator is seen. When popped, the stored + /// set becomes the left hand side of the operator. + Op { + /// The type of the operation, i.e., &&, -- or ~~. + kind: ast::ClassSetBinaryOpKind, + /// The left-hand side of the operator. + lhs: ast::ClassSet, + }, +} + +impl Parser { + /// Create a new parser with a default configuration. + /// + /// The parser can be run with either the `parse` or `parse_with_comments` + /// methods. The parse methods return an abstract syntax tree. + /// + /// To set configuration options on the parser, use [`ParserBuilder`]. + pub fn new() -> Parser { + ParserBuilder::new().build() + } + + /// Parse the regular expression into an abstract syntax tree. + pub fn parse(&mut self, pattern: &str) -> Result<Ast> { + ParserI::new(self, pattern).parse() + } + + /// Parse the regular expression and return an abstract syntax tree with + /// all of the comments found in the pattern. + pub fn parse_with_comments( + &mut self, + pattern: &str, + ) -> Result<ast::WithComments> { + ParserI::new(self, pattern).parse_with_comments() + } + + /// Reset the internal state of a parser. + /// + /// This is called at the beginning of every parse. This prevents the + /// parser from running with inconsistent state (say, if a previous + /// invocation returned an error and the parser is reused). + fn reset(&self) { + // These settings should be in line with the construction + // in `ParserBuilder::build`. + self.pos.set(Position { offset: 0, line: 1, column: 1 }); + self.ignore_whitespace.set(self.initial_ignore_whitespace); + self.comments.borrow_mut().clear(); + self.stack_group.borrow_mut().clear(); + self.stack_class.borrow_mut().clear(); + } +} + +impl<'s, P: Borrow<Parser>> ParserI<'s, P> { + /// Build an internal parser from a parser configuration and a pattern. + fn new(parser: P, pattern: &'s str) -> ParserI<'s, P> { + ParserI { parser, pattern } + } + + /// Return a reference to the parser state. + fn parser(&self) -> &Parser { + self.parser.borrow() + } + + /// Return a reference to the pattern being parsed. + fn pattern(&self) -> &str { + self.pattern + } + + /// Create a new error with the given span and error type. + fn error(&self, span: Span, kind: ast::ErrorKind) -> ast::Error { + ast::Error { kind, pattern: self.pattern().to_string(), span } + } + + /// Return the current offset of the parser. + /// + /// The offset starts at `0` from the beginning of the regular expression + /// pattern string. + fn offset(&self) -> usize { + self.parser().pos.get().offset + } + + /// Return the current line number of the parser. + /// + /// The line number starts at `1`. + fn line(&self) -> usize { + self.parser().pos.get().line + } + + /// Return the current column of the parser. + /// + /// The column number starts at `1` and is reset whenever a `\n` is seen. + fn column(&self) -> usize { + self.parser().pos.get().column + } + + /// Return the next capturing index. Each subsequent call increments the + /// internal index. + /// + /// The span given should correspond to the location of the opening + /// parenthesis. + /// + /// If the capture limit is exceeded, then an error is returned. + fn next_capture_index(&self, span: Span) -> Result<u32> { + let current = self.parser().capture_index.get(); + let i = current.checked_add(1).ok_or_else(|| { + self.error(span, ast::ErrorKind::CaptureLimitExceeded) + })?; + self.parser().capture_index.set(i); + Ok(i) + } + + /// Adds the given capture name to this parser. If this capture name has + /// already been used, then an error is returned. + fn add_capture_name(&self, cap: &ast::CaptureName) -> Result<()> { + let mut names = self.parser().capture_names.borrow_mut(); + match names + .binary_search_by_key(&cap.name.as_str(), |c| c.name.as_str()) + { + Err(i) => { + names.insert(i, cap.clone()); + Ok(()) + } + Ok(i) => Err(self.error( + cap.span, + ast::ErrorKind::GroupNameDuplicate { original: names[i].span }, + )), + } + } + + /// Return whether the parser should ignore whitespace or not. + fn ignore_whitespace(&self) -> bool { + self.parser().ignore_whitespace.get() + } + + /// Return the character at the current position of the parser. + /// + /// This panics if the current position does not point to a valid char. + fn char(&self) -> char { + self.char_at(self.offset()) + } + + /// Return the character at the given position. + /// + /// This panics if the given position does not point to a valid char. + fn char_at(&self, i: usize) -> char { + self.pattern()[i..] + .chars() + .next() + .unwrap_or_else(|| panic!("expected char at offset {i}")) + } + + /// Bump the parser to the next Unicode scalar value. + /// + /// If the end of the input has been reached, then `false` is returned. + fn bump(&self) -> bool { + if self.is_eof() { + return false; + } + let Position { mut offset, mut line, mut column } = self.pos(); + if self.char() == '\n' { + line = line.checked_add(1).unwrap(); + column = 1; + } else { + column = column.checked_add(1).unwrap(); + } + offset += self.char().len_utf8(); + self.parser().pos.set(Position { offset, line, column }); + self.pattern()[self.offset()..].chars().next().is_some() + } + + /// If the substring starting at the current position of the parser has + /// the given prefix, then bump the parser to the character immediately + /// following the prefix and return true. Otherwise, don't bump the parser + /// and return false. + fn bump_if(&self, prefix: &str) -> bool { + if self.pattern()[self.offset()..].starts_with(prefix) { + for _ in 0..prefix.chars().count() { + self.bump(); + } + true + } else { + false + } + } + + /// Returns true if and only if the parser is positioned at a look-around + /// prefix. The conditions under which this returns true must always + /// correspond to a regular expression that would otherwise be consider + /// invalid. + /// + /// This should only be called immediately after parsing the opening of + /// a group or a set of flags. + fn is_lookaround_prefix(&self) -> bool { + self.bump_if("?=") + || self.bump_if("?!") + || self.bump_if("?<=") + || self.bump_if("?<!") + } + + /// Bump the parser, and if the `x` flag is enabled, bump through any + /// subsequent spaces. Return true if and only if the parser is not at + /// EOF. + fn bump_and_bump_space(&self) -> bool { + if !self.bump() { + return false; + } + self.bump_space(); + !self.is_eof() + } + + /// If the `x` flag is enabled (i.e., whitespace insensitivity with + /// comments), then this will advance the parser through all whitespace + /// and comments to the next non-whitespace non-comment byte. + /// + /// If the `x` flag is disabled, then this is a no-op. + /// + /// This should be used selectively throughout the parser where + /// arbitrary whitespace is permitted when the `x` flag is enabled. For + /// example, `{ 5 , 6}` is equivalent to `{5,6}`. + fn bump_space(&self) { + if !self.ignore_whitespace() { + return; + } + while !self.is_eof() { + if self.char().is_whitespace() { + self.bump(); + } else if self.char() == '#' { + let start = self.pos(); + let mut comment_text = String::new(); + self.bump(); + while !self.is_eof() { + let c = self.char(); + self.bump(); + if c == '\n' { + break; + } + comment_text.push(c); + } + let comment = ast::Comment { + span: Span::new(start, self.pos()), + comment: comment_text, + }; + self.parser().comments.borrow_mut().push(comment); + } else { + break; + } + } + } + + /// Peek at the next character in the input without advancing the parser. + /// + /// If the input has been exhausted, then this returns `None`. + fn peek(&self) -> Option<char> { + if self.is_eof() { + return None; + } + self.pattern()[self.offset() + self.char().len_utf8()..].chars().next() + } + + /// Like peek, but will ignore spaces when the parser is in whitespace + /// insensitive mode. + fn peek_space(&self) -> Option<char> { + if !self.ignore_whitespace() { + return self.peek(); + } + if self.is_eof() { + return None; + } + let mut start = self.offset() + self.char().len_utf8(); + let mut in_comment = false; + for (i, c) in self.pattern()[start..].char_indices() { + if c.is_whitespace() { + continue; + } else if !in_comment && c == '#' { + in_comment = true; + } else if in_comment && c == '\n' { + in_comment = false; + } else { + start += i; + break; + } + } + self.pattern()[start..].chars().next() + } + + /// Returns true if the next call to `bump` would return false. + fn is_eof(&self) -> bool { + self.offset() == self.pattern().len() + } + + /// Return the current position of the parser, which includes the offset, + /// line and column. + fn pos(&self) -> Position { + self.parser().pos.get() + } + + /// Create a span at the current position of the parser. Both the start + /// and end of the span are set. + fn span(&self) -> Span { + Span::splat(self.pos()) + } + + /// Create a span that covers the current character. + fn span_char(&self) -> Span { + let mut next = Position { + offset: self.offset().checked_add(self.char().len_utf8()).unwrap(), + line: self.line(), + column: self.column().checked_add(1).unwrap(), + }; + if self.char() == '\n' { + next.line += 1; + next.column = 1; + } + Span::new(self.pos(), next) + } + + /// Parse and push a single alternation on to the parser's internal stack. + /// If the top of the stack already has an alternation, then add to that + /// instead of pushing a new one. + /// + /// The concatenation given corresponds to a single alternation branch. + /// The concatenation returned starts the next branch and is empty. + /// + /// This assumes the parser is currently positioned at `|` and will advance + /// the parser to the character following `|`. + #[inline(never)] + fn push_alternate(&self, mut concat: ast::Concat) -> Result<ast::Concat> { + assert_eq!(self.char(), '|'); + concat.span.end = self.pos(); + self.push_or_add_alternation(concat); + self.bump(); + Ok(ast::Concat { span: self.span(), asts: vec![] }) + } + + /// Pushes or adds the given branch of an alternation to the parser's + /// internal stack of state. + fn push_or_add_alternation(&self, concat: ast::Concat) { + use self::GroupState::*; + + let mut stack = self.parser().stack_group.borrow_mut(); + if let Some(&mut Alternation(ref mut alts)) = stack.last_mut() { + alts.asts.push(concat.into_ast()); + return; + } + stack.push(Alternation(ast::Alternation { + span: Span::new(concat.span.start, self.pos()), + asts: vec![concat.into_ast()], + })); + } + + /// Parse and push a group AST (and its parent concatenation) on to the + /// parser's internal stack. Return a fresh concatenation corresponding + /// to the group's sub-AST. + /// + /// If a set of flags was found (with no group), then the concatenation + /// is returned with that set of flags added. + /// + /// This assumes that the parser is currently positioned on the opening + /// parenthesis. It advances the parser to the character at the start + /// of the sub-expression (or adjoining expression). + /// + /// If there was a problem parsing the start of the group, then an error + /// is returned. + #[inline(never)] + fn push_group(&self, mut concat: ast::Concat) -> Result<ast::Concat> { + assert_eq!(self.char(), '('); + match self.parse_group()? { + Either::Left(set) => { + let ignore = set.flags.flag_state(ast::Flag::IgnoreWhitespace); + if let Some(v) = ignore { + self.parser().ignore_whitespace.set(v); + } + + concat.asts.push(Ast::flags(set)); + Ok(concat) + } + Either::Right(group) => { + let old_ignore_whitespace = self.ignore_whitespace(); + let new_ignore_whitespace = group + .flags() + .and_then(|f| f.flag_state(ast::Flag::IgnoreWhitespace)) + .unwrap_or(old_ignore_whitespace); + self.parser().stack_group.borrow_mut().push( + GroupState::Group { + concat, + group, + ignore_whitespace: old_ignore_whitespace, + }, + ); + self.parser().ignore_whitespace.set(new_ignore_whitespace); + Ok(ast::Concat { span: self.span(), asts: vec![] }) + } + } + } + + /// Pop a group AST from the parser's internal stack and set the group's + /// AST to the given concatenation. Return the concatenation containing + /// the group. + /// + /// This assumes that the parser is currently positioned on the closing + /// parenthesis and advances the parser to the character following the `)`. + /// + /// If no such group could be popped, then an unopened group error is + /// returned. + #[inline(never)] + fn pop_group(&self, mut group_concat: ast::Concat) -> Result<ast::Concat> { + use self::GroupState::*; + + assert_eq!(self.char(), ')'); + let mut stack = self.parser().stack_group.borrow_mut(); + let (mut prior_concat, mut group, ignore_whitespace, alt) = match stack + .pop() + { + Some(Group { concat, group, ignore_whitespace }) => { + (concat, group, ignore_whitespace, None) + } + Some(Alternation(alt)) => match stack.pop() { + Some(Group { concat, group, ignore_whitespace }) => { + (concat, group, ignore_whitespace, Some(alt)) + } + None | Some(Alternation(_)) => { + return Err(self.error( + self.span_char(), + ast::ErrorKind::GroupUnopened, + )); + } + }, + None => { + return Err(self + .error(self.span_char(), ast::ErrorKind::GroupUnopened)); + } + }; + self.parser().ignore_whitespace.set(ignore_whitespace); + group_concat.span.end = self.pos(); + self.bump(); + group.span.end = self.pos(); + match alt { + Some(mut alt) => { + alt.span.end = group_concat.span.end; + alt.asts.push(group_concat.into_ast()); + group.ast = Box::new(alt.into_ast()); + } + None => { + group.ast = Box::new(group_concat.into_ast()); + } + } + prior_concat.asts.push(Ast::group(group)); + Ok(prior_concat) + } + + /// Pop the last state from the parser's internal stack, if it exists, and + /// add the given concatenation to it. There either must be no state or a + /// single alternation item on the stack. Any other scenario produces an + /// error. + /// + /// This assumes that the parser has advanced to the end. + #[inline(never)] + fn pop_group_end(&self, mut concat: ast::Concat) -> Result<Ast> { + concat.span.end = self.pos(); + let mut stack = self.parser().stack_group.borrow_mut(); + let ast = match stack.pop() { + None => Ok(concat.into_ast()), + Some(GroupState::Alternation(mut alt)) => { + alt.span.end = self.pos(); + alt.asts.push(concat.into_ast()); + Ok(Ast::alternation(alt)) + } + Some(GroupState::Group { group, .. }) => { + return Err( + self.error(group.span, ast::ErrorKind::GroupUnclosed) + ); + } + }; + // If we try to pop again, there should be nothing. + match stack.pop() { + None => ast, + Some(GroupState::Alternation(_)) => { + // This unreachable is unfortunate. This case can't happen + // because the only way we can be here is if there were two + // `GroupState::Alternation`s adjacent in the parser's stack, + // which we guarantee to never happen because we never push a + // `GroupState::Alternation` if one is already at the top of + // the stack. + unreachable!() + } + Some(GroupState::Group { group, .. }) => { + Err(self.error(group.span, ast::ErrorKind::GroupUnclosed)) + } + } + } + + /// Parse the opening of a character class and push the current class + /// parsing context onto the parser's stack. This assumes that the parser + /// is positioned at an opening `[`. The given union should correspond to + /// the union of set items built up before seeing the `[`. + /// + /// If there was a problem parsing the opening of the class, then an error + /// is returned. Otherwise, a new union of set items for the class is + /// returned (which may be populated with either a `]` or a `-`). + #[inline(never)] + fn push_class_open( + &self, + parent_union: ast::ClassSetUnion, + ) -> Result<ast::ClassSetUnion> { + assert_eq!(self.char(), '['); + + let (nested_set, nested_union) = self.parse_set_class_open()?; + self.parser() + .stack_class + .borrow_mut() + .push(ClassState::Open { union: parent_union, set: nested_set }); + Ok(nested_union) + } + + /// Parse the end of a character class set and pop the character class + /// parser stack. The union given corresponds to the last union built + /// before seeing the closing `]`. The union returned corresponds to the + /// parent character class set with the nested class added to it. + /// + /// This assumes that the parser is positioned at a `]` and will advance + /// the parser to the byte immediately following the `]`. + /// + /// If the stack is empty after popping, then this returns the final + /// "top-level" character class AST (where a "top-level" character class + /// is one that is not nested inside any other character class). + /// + /// If there is no corresponding opening bracket on the parser's stack, + /// then an error is returned. + #[inline(never)] + fn pop_class( + &self, + nested_union: ast::ClassSetUnion, + ) -> Result<Either<ast::ClassSetUnion, ast::ClassBracketed>> { + assert_eq!(self.char(), ']'); + + let item = ast::ClassSet::Item(nested_union.into_item()); + let prevset = self.pop_class_op(item); + let mut stack = self.parser().stack_class.borrow_mut(); + match stack.pop() { + None => { + // We can never observe an empty stack: + // + // 1) We are guaranteed to start with a non-empty stack since + // the character class parser is only initiated when it sees + // a `[`. + // 2) If we ever observe an empty stack while popping after + // seeing a `]`, then we signal the character class parser + // to terminate. + panic!("unexpected empty character class stack") + } + Some(ClassState::Op { .. }) => { + // This panic is unfortunate, but this case is impossible + // since we already popped the Op state if one exists above. + // Namely, every push to the class parser stack is guarded by + // whether an existing Op is already on the top of the stack. + // If it is, the existing Op is modified. That is, the stack + // can never have consecutive Op states. + panic!("unexpected ClassState::Op") + } + Some(ClassState::Open { mut union, mut set }) => { + self.bump(); + set.span.end = self.pos(); + set.kind = prevset; + if stack.is_empty() { + Ok(Either::Right(set)) + } else { + union.push(ast::ClassSetItem::Bracketed(Box::new(set))); + Ok(Either::Left(union)) + } + } + } + } + + /// Return an "unclosed class" error whose span points to the most + /// recently opened class. + /// + /// This should only be called while parsing a character class. + #[inline(never)] + fn unclosed_class_error(&self) -> ast::Error { + for state in self.parser().stack_class.borrow().iter().rev() { + if let ClassState::Open { ref set, .. } = *state { + return self.error(set.span, ast::ErrorKind::ClassUnclosed); + } + } + // We are guaranteed to have a non-empty stack with at least + // one open bracket, so we should never get here. + panic!("no open character class found") + } + + /// Push the current set of class items on to the class parser's stack as + /// the left hand side of the given operator. + /// + /// A fresh set union is returned, which should be used to build the right + /// hand side of this operator. + #[inline(never)] + fn push_class_op( + &self, + next_kind: ast::ClassSetBinaryOpKind, + next_union: ast::ClassSetUnion, + ) -> ast::ClassSetUnion { + let item = ast::ClassSet::Item(next_union.into_item()); + let new_lhs = self.pop_class_op(item); + self.parser() + .stack_class + .borrow_mut() + .push(ClassState::Op { kind: next_kind, lhs: new_lhs }); + ast::ClassSetUnion { span: self.span(), items: vec![] } + } + + /// Pop a character class set from the character class parser stack. If the + /// top of the stack is just an item (not an operation), then return the + /// given set unchanged. If the top of the stack is an operation, then the + /// given set will be used as the rhs of the operation on the top of the + /// stack. In that case, the binary operation is returned as a set. + #[inline(never)] + fn pop_class_op(&self, rhs: ast::ClassSet) -> ast::ClassSet { + let mut stack = self.parser().stack_class.borrow_mut(); + let (kind, lhs) = match stack.pop() { + Some(ClassState::Op { kind, lhs }) => (kind, lhs), + Some(state @ ClassState::Open { .. }) => { + stack.push(state); + return rhs; + } + None => unreachable!(), + }; + let span = Span::new(lhs.span().start, rhs.span().end); + ast::ClassSet::BinaryOp(ast::ClassSetBinaryOp { + span, + kind, + lhs: Box::new(lhs), + rhs: Box::new(rhs), + }) + } +} + +impl<'s, P: Borrow<Parser>> ParserI<'s, P> { + /// Parse the regular expression into an abstract syntax tree. + fn parse(&self) -> Result<Ast> { + self.parse_with_comments().map(|astc| astc.ast) + } + + /// Parse the regular expression and return an abstract syntax tree with + /// all of the comments found in the pattern. + fn parse_with_comments(&self) -> Result<ast::WithComments> { + assert_eq!(self.offset(), 0, "parser can only be used once"); + self.parser().reset(); + let mut concat = ast::Concat { span: self.span(), asts: vec![] }; + loop { + self.bump_space(); + if self.is_eof() { + break; + } + match self.char() { + '(' => concat = self.push_group(concat)?, + ')' => concat = self.pop_group(concat)?, + '|' => concat = self.push_alternate(concat)?, + '[' => { + let class = self.parse_set_class()?; + concat.asts.push(Ast::class_bracketed(class)); + } + '?' => { + concat = self.parse_uncounted_repetition( + concat, + ast::RepetitionKind::ZeroOrOne, + )?; + } + '*' => { + concat = self.parse_uncounted_repetition( + concat, + ast::RepetitionKind::ZeroOrMore, + )?; + } + '+' => { + concat = self.parse_uncounted_repetition( + concat, + ast::RepetitionKind::OneOrMore, + )?; + } + '{' => { + concat = self.parse_counted_repetition(concat)?; + } + _ => concat.asts.push(self.parse_primitive()?.into_ast()), + } + } + let ast = self.pop_group_end(concat)?; + NestLimiter::new(self).check(&ast)?; + Ok(ast::WithComments { + ast, + comments: mem::replace( + &mut *self.parser().comments.borrow_mut(), + vec![], + ), + }) + } + + /// Parses an uncounted repetition operation. An uncounted repetition + /// operator includes ?, * and +, but does not include the {m,n} syntax. + /// The given `kind` should correspond to the operator observed by the + /// caller. + /// + /// This assumes that the parser is currently positioned at the repetition + /// operator and advances the parser to the first character after the + /// operator. (Note that the operator may include a single additional `?`, + /// which makes the operator ungreedy.) + /// + /// The caller should include the concatenation that is being built. The + /// concatenation returned includes the repetition operator applied to the + /// last expression in the given concatenation. + #[inline(never)] + fn parse_uncounted_repetition( + &self, + mut concat: ast::Concat, + kind: ast::RepetitionKind, + ) -> Result<ast::Concat> { + assert!( + self.char() == '?' || self.char() == '*' || self.char() == '+' + ); + let op_start = self.pos(); + let ast = match concat.asts.pop() { + Some(ast) => ast, + None => { + return Err( + self.error(self.span(), ast::ErrorKind::RepetitionMissing) + ) + } + }; + match ast { + Ast::Empty(_) | Ast::Flags(_) => { + return Err( + self.error(self.span(), ast::ErrorKind::RepetitionMissing) + ) + } + _ => {} + } + let mut greedy = true; + if self.bump() && self.char() == '?' { + greedy = false; + self.bump(); + } + concat.asts.push(Ast::repetition(ast::Repetition { + span: ast.span().with_end(self.pos()), + op: ast::RepetitionOp { + span: Span::new(op_start, self.pos()), + kind, + }, + greedy, + ast: Box::new(ast), + })); + Ok(concat) + } + + /// Parses a counted repetition operation. A counted repetition operator + /// corresponds to the {m,n} syntax, and does not include the ?, * or + + /// operators. + /// + /// This assumes that the parser is currently positioned at the opening `{` + /// and advances the parser to the first character after the operator. + /// (Note that the operator may include a single additional `?`, which + /// makes the operator ungreedy.) + /// + /// The caller should include the concatenation that is being built. The + /// concatenation returned includes the repetition operator applied to the + /// last expression in the given concatenation. + #[inline(never)] + fn parse_counted_repetition( + &self, + mut concat: ast::Concat, + ) -> Result<ast::Concat> { + assert!(self.char() == '{'); + let start = self.pos(); + let ast = match concat.asts.pop() { + Some(ast) => ast, + None => { + return Err( + self.error(self.span(), ast::ErrorKind::RepetitionMissing) + ) + } + }; + match ast { + Ast::Empty(_) | Ast::Flags(_) => { + return Err( + self.error(self.span(), ast::ErrorKind::RepetitionMissing) + ) + } + _ => {} + } + if !self.bump_and_bump_space() { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::RepetitionCountUnclosed, + )); + } + let count_start = specialize_err( + self.parse_decimal(), + ast::ErrorKind::DecimalEmpty, + ast::ErrorKind::RepetitionCountDecimalEmpty, + ); + if self.is_eof() { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::RepetitionCountUnclosed, + )); + } + let range = if self.char() == ',' { + if !self.bump_and_bump_space() { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::RepetitionCountUnclosed, + )); + } + if self.char() != '}' { + let count_start = match count_start { + Ok(c) => c, + Err(err) + if err.kind + == ast::ErrorKind::RepetitionCountDecimalEmpty => + { + if self.parser().empty_min_range { + 0 + } else { + return Err(err); + } + } + err => err?, + }; + let count_end = specialize_err( + self.parse_decimal(), + ast::ErrorKind::DecimalEmpty, + ast::ErrorKind::RepetitionCountDecimalEmpty, + )?; + ast::RepetitionRange::Bounded(count_start, count_end) + } else { + ast::RepetitionRange::AtLeast(count_start?) + } + } else { + ast::RepetitionRange::Exactly(count_start?) + }; + + if self.is_eof() || self.char() != '}' { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::RepetitionCountUnclosed, + )); + } + + let mut greedy = true; + if self.bump_and_bump_space() && self.char() == '?' { + greedy = false; + self.bump(); + } + + let op_span = Span::new(start, self.pos()); + if !range.is_valid() { + return Err( + self.error(op_span, ast::ErrorKind::RepetitionCountInvalid) + ); + } + concat.asts.push(Ast::repetition(ast::Repetition { + span: ast.span().with_end(self.pos()), + op: ast::RepetitionOp { + span: op_span, + kind: ast::RepetitionKind::Range(range), + }, + greedy, + ast: Box::new(ast), + })); + Ok(concat) + } + + /// Parse a group (which contains a sub-expression) or a set of flags. + /// + /// If a group was found, then it is returned with an empty AST. If a set + /// of flags is found, then that set is returned. + /// + /// The parser should be positioned at the opening parenthesis. + /// + /// This advances the parser to the character before the start of the + /// sub-expression (in the case of a group) or to the closing parenthesis + /// immediately following the set of flags. + /// + /// # Errors + /// + /// If flags are given and incorrectly specified, then a corresponding + /// error is returned. + /// + /// If a capture name is given and it is incorrectly specified, then a + /// corresponding error is returned. + #[inline(never)] + fn parse_group(&self) -> Result<Either<ast::SetFlags, ast::Group>> { + assert_eq!(self.char(), '('); + let open_span = self.span_char(); + self.bump(); + self.bump_space(); + if self.is_lookaround_prefix() { + return Err(self.error( + Span::new(open_span.start, self.span().end), + ast::ErrorKind::UnsupportedLookAround, + )); + } + let inner_span = self.span(); + let mut starts_with_p = true; + if self.bump_if("?P<") || { + starts_with_p = false; + self.bump_if("?<") + } { + let capture_index = self.next_capture_index(open_span)?; + let name = self.parse_capture_name(capture_index)?; + Ok(Either::Right(ast::Group { + span: open_span, + kind: ast::GroupKind::CaptureName { starts_with_p, name }, + ast: Box::new(Ast::empty(self.span())), + })) + } else if self.bump_if("?") { + if self.is_eof() { + return Err( + self.error(open_span, ast::ErrorKind::GroupUnclosed) + ); + } + let flags = self.parse_flags()?; + let char_end = self.char(); + self.bump(); + if char_end == ')' { + // We don't allow empty flags, e.g., `(?)`. We instead + // interpret it as a repetition operator missing its argument. + if flags.items.is_empty() { + return Err(self.error( + inner_span, + ast::ErrorKind::RepetitionMissing, + )); + } + Ok(Either::Left(ast::SetFlags { + span: Span { end: self.pos(), ..open_span }, + flags, + })) + } else { + assert_eq!(char_end, ':'); + Ok(Either::Right(ast::Group { + span: open_span, + kind: ast::GroupKind::NonCapturing(flags), + ast: Box::new(Ast::empty(self.span())), + })) + } + } else { + let capture_index = self.next_capture_index(open_span)?; + Ok(Either::Right(ast::Group { + span: open_span, + kind: ast::GroupKind::CaptureIndex(capture_index), + ast: Box::new(Ast::empty(self.span())), + })) + } + } + + /// Parses a capture group name. Assumes that the parser is positioned at + /// the first character in the name following the opening `<` (and may + /// possibly be EOF). This advances the parser to the first character + /// following the closing `>`. + /// + /// The caller must provide the capture index of the group for this name. + #[inline(never)] + fn parse_capture_name( + &self, + capture_index: u32, + ) -> Result<ast::CaptureName> { + if self.is_eof() { + return Err(self + .error(self.span(), ast::ErrorKind::GroupNameUnexpectedEof)); + } + let start = self.pos(); + loop { + if self.char() == '>' { + break; + } + if !is_capture_char(self.char(), self.pos() == start) { + return Err(self.error( + self.span_char(), + ast::ErrorKind::GroupNameInvalid, + )); + } + if !self.bump() { + break; + } + } + let end = self.pos(); + if self.is_eof() { + return Err(self + .error(self.span(), ast::ErrorKind::GroupNameUnexpectedEof)); + } + assert_eq!(self.char(), '>'); + self.bump(); + let name = &self.pattern()[start.offset..end.offset]; + if name.is_empty() { + return Err(self.error( + Span::new(start, start), + ast::ErrorKind::GroupNameEmpty, + )); + } + let capname = ast::CaptureName { + span: Span::new(start, end), + name: name.to_string(), + index: capture_index, + }; + self.add_capture_name(&capname)?; + Ok(capname) + } + + /// Parse a sequence of flags starting at the current character. + /// + /// This advances the parser to the character immediately following the + /// flags, which is guaranteed to be either `:` or `)`. + /// + /// # Errors + /// + /// If any flags are duplicated, then an error is returned. + /// + /// If the negation operator is used more than once, then an error is + /// returned. + /// + /// If no flags could be found or if the negation operation is not followed + /// by any flags, then an error is returned. + #[inline(never)] + fn parse_flags(&self) -> Result<ast::Flags> { + let mut flags = ast::Flags { span: self.span(), items: vec![] }; + let mut last_was_negation = None; + while self.char() != ':' && self.char() != ')' { + if self.char() == '-' { + last_was_negation = Some(self.span_char()); + let item = ast::FlagsItem { + span: self.span_char(), + kind: ast::FlagsItemKind::Negation, + }; + if let Some(i) = flags.add_item(item) { + return Err(self.error( + self.span_char(), + ast::ErrorKind::FlagRepeatedNegation { + original: flags.items[i].span, + }, + )); + } + } else { + last_was_negation = None; + let item = ast::FlagsItem { + span: self.span_char(), + kind: ast::FlagsItemKind::Flag(self.parse_flag()?), + }; + if let Some(i) = flags.add_item(item) { + return Err(self.error( + self.span_char(), + ast::ErrorKind::FlagDuplicate { + original: flags.items[i].span, + }, + )); + } + } + if !self.bump() { + return Err( + self.error(self.span(), ast::ErrorKind::FlagUnexpectedEof) + ); + } + } + if let Some(span) = last_was_negation { + return Err(self.error(span, ast::ErrorKind::FlagDanglingNegation)); + } + flags.span.end = self.pos(); + Ok(flags) + } + + /// Parse the current character as a flag. Do not advance the parser. + /// + /// # Errors + /// + /// If the flag is not recognized, then an error is returned. + #[inline(never)] + fn parse_flag(&self) -> Result<ast::Flag> { + match self.char() { + 'i' => Ok(ast::Flag::CaseInsensitive), + 'm' => Ok(ast::Flag::MultiLine), + 's' => Ok(ast::Flag::DotMatchesNewLine), + 'U' => Ok(ast::Flag::SwapGreed), + 'u' => Ok(ast::Flag::Unicode), + 'R' => Ok(ast::Flag::CRLF), + 'x' => Ok(ast::Flag::IgnoreWhitespace), + _ => { + Err(self + .error(self.span_char(), ast::ErrorKind::FlagUnrecognized)) + } + } + } + + /// Parse a primitive AST. e.g., A literal, non-set character class or + /// assertion. + /// + /// This assumes that the parser expects a primitive at the current + /// location. i.e., All other non-primitive cases have been handled. + /// For example, if the parser's position is at `|`, then `|` will be + /// treated as a literal (e.g., inside a character class). + /// + /// This advances the parser to the first character immediately following + /// the primitive. + fn parse_primitive(&self) -> Result<Primitive> { + match self.char() { + '\\' => self.parse_escape(), + '.' => { + let ast = Primitive::Dot(self.span_char()); + self.bump(); + Ok(ast) + } + '^' => { + let ast = Primitive::Assertion(ast::Assertion { + span: self.span_char(), + kind: ast::AssertionKind::StartLine, + }); + self.bump(); + Ok(ast) + } + '$' => { + let ast = Primitive::Assertion(ast::Assertion { + span: self.span_char(), + kind: ast::AssertionKind::EndLine, + }); + self.bump(); + Ok(ast) + } + c => { + let ast = Primitive::Literal(ast::Literal { + span: self.span_char(), + kind: ast::LiteralKind::Verbatim, + c, + }); + self.bump(); + Ok(ast) + } + } + } + + /// Parse an escape sequence as a primitive AST. + /// + /// This assumes the parser is positioned at the start of the escape + /// sequence, i.e., `\`. It advances the parser to the first position + /// immediately following the escape sequence. + #[inline(never)] + fn parse_escape(&self) -> Result<Primitive> { + assert_eq!(self.char(), '\\'); + let start = self.pos(); + if !self.bump() { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::EscapeUnexpectedEof, + )); + } + let c = self.char(); + // Put some of the more complicated routines into helpers. + match c { + '0'..='7' => { + if !self.parser().octal { + return Err(self.error( + Span::new(start, self.span_char().end), + ast::ErrorKind::UnsupportedBackreference, + )); + } + let mut lit = self.parse_octal(); + lit.span.start = start; + return Ok(Primitive::Literal(lit)); + } + '8'..='9' if !self.parser().octal => { + return Err(self.error( + Span::new(start, self.span_char().end), + ast::ErrorKind::UnsupportedBackreference, + )); + } + 'x' | 'u' | 'U' => { + let mut lit = self.parse_hex()?; + lit.span.start = start; + return Ok(Primitive::Literal(lit)); + } + 'p' | 'P' => { + let mut cls = self.parse_unicode_class()?; + cls.span.start = start; + return Ok(Primitive::Unicode(cls)); + } + 'd' | 's' | 'w' | 'D' | 'S' | 'W' => { + let mut cls = self.parse_perl_class(); + cls.span.start = start; + return Ok(Primitive::Perl(cls)); + } + _ => {} + } + + // Handle all of the one letter sequences inline. + self.bump(); + let span = Span::new(start, self.pos()); + if is_meta_character(c) { + return Ok(Primitive::Literal(ast::Literal { + span, + kind: ast::LiteralKind::Meta, + c, + })); + } + if is_escapeable_character(c) { + return Ok(Primitive::Literal(ast::Literal { + span, + kind: ast::LiteralKind::Superfluous, + c, + })); + } + let special = |kind, c| { + Ok(Primitive::Literal(ast::Literal { + span, + kind: ast::LiteralKind::Special(kind), + c, + })) + }; + match c { + 'a' => special(ast::SpecialLiteralKind::Bell, '\x07'), + 'f' => special(ast::SpecialLiteralKind::FormFeed, '\x0C'), + 't' => special(ast::SpecialLiteralKind::Tab, '\t'), + 'n' => special(ast::SpecialLiteralKind::LineFeed, '\n'), + 'r' => special(ast::SpecialLiteralKind::CarriageReturn, '\r'), + 'v' => special(ast::SpecialLiteralKind::VerticalTab, '\x0B'), + 'A' => Ok(Primitive::Assertion(ast::Assertion { + span, + kind: ast::AssertionKind::StartText, + })), + 'z' => Ok(Primitive::Assertion(ast::Assertion { + span, + kind: ast::AssertionKind::EndText, + })), + 'b' => { + let mut wb = ast::Assertion { + span, + kind: ast::AssertionKind::WordBoundary, + }; + // After a \b, we "try" to parse things like \b{start} for + // special word boundary assertions. + if !self.is_eof() && self.char() == '{' { + if let Some(kind) = + self.maybe_parse_special_word_boundary(start)? + { + wb.kind = kind; + wb.span.end = self.pos(); + } + } + Ok(Primitive::Assertion(wb)) + } + 'B' => Ok(Primitive::Assertion(ast::Assertion { + span, + kind: ast::AssertionKind::NotWordBoundary, + })), + '<' => Ok(Primitive::Assertion(ast::Assertion { + span, + kind: ast::AssertionKind::WordBoundaryStartAngle, + })), + '>' => Ok(Primitive::Assertion(ast::Assertion { + span, + kind: ast::AssertionKind::WordBoundaryEndAngle, + })), + _ => Err(self.error(span, ast::ErrorKind::EscapeUnrecognized)), + } + } + + /// Attempt to parse a specialty word boundary. That is, `\b{start}`, + /// `\b{end}`, `\b{start-half}` or `\b{end-half}`. + /// + /// This is similar to `maybe_parse_ascii_class` in that, in most cases, + /// if it fails it will just return `None` with no error. This is done + /// because `\b{5}` is a valid expression and we want to let that be parsed + /// by the existing counted repetition parsing code. (I thought about just + /// invoking the counted repetition code from here, but it seemed a little + /// ham-fisted.) + /// + /// Unlike `maybe_parse_ascii_class` though, this can return an error. + /// Namely, if we definitely know it isn't a counted repetition, then we + /// return an error specific to the specialty word boundaries. + /// + /// This assumes the parser is positioned at a `{` immediately following + /// a `\b`. When `None` is returned, the parser is returned to the position + /// at which it started: pointing at a `{`. + /// + /// The position given should correspond to the start of the `\b`. + fn maybe_parse_special_word_boundary( + &self, + wb_start: Position, + ) -> Result<Option<ast::AssertionKind>> { + assert_eq!(self.char(), '{'); + + let is_valid_char = |c| match c { + 'A'..='Z' | 'a'..='z' | '-' => true, + _ => false, + }; + let start = self.pos(); + if !self.bump_and_bump_space() { + return Err(self.error( + Span::new(wb_start, self.pos()), + ast::ErrorKind::SpecialWordOrRepetitionUnexpectedEof, + )); + } + let start_contents = self.pos(); + // This is one of the critical bits: if the first non-whitespace + // character isn't in [-A-Za-z] (i.e., this can't be a special word + // boundary), then we bail and let the counted repetition parser deal + // with this. + if !is_valid_char(self.char()) { + self.parser().pos.set(start); + return Ok(None); + } + + // Now collect up our chars until we see a '}'. + let mut scratch = self.parser().scratch.borrow_mut(); + scratch.clear(); + while !self.is_eof() && is_valid_char(self.char()) { + scratch.push(self.char()); + self.bump_and_bump_space(); + } + if self.is_eof() || self.char() != '}' { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::SpecialWordBoundaryUnclosed, + )); + } + let end = self.pos(); + self.bump(); + let kind = match scratch.as_str() { + "start" => ast::AssertionKind::WordBoundaryStart, + "end" => ast::AssertionKind::WordBoundaryEnd, + "start-half" => ast::AssertionKind::WordBoundaryStartHalf, + "end-half" => ast::AssertionKind::WordBoundaryEndHalf, + _ => { + return Err(self.error( + Span::new(start_contents, end), + ast::ErrorKind::SpecialWordBoundaryUnrecognized, + )) + } + }; + Ok(Some(kind)) + } + + /// Parse an octal representation of a Unicode codepoint up to 3 digits + /// long. This expects the parser to be positioned at the first octal + /// digit and advances the parser to the first character immediately + /// following the octal number. This also assumes that parsing octal + /// escapes is enabled. + /// + /// Assuming the preconditions are met, this routine can never fail. + #[inline(never)] + fn parse_octal(&self) -> ast::Literal { + assert!(self.parser().octal); + assert!('0' <= self.char() && self.char() <= '7'); + let start = self.pos(); + // Parse up to two more digits. + while self.bump() + && '0' <= self.char() + && self.char() <= '7' + && self.pos().offset - start.offset <= 2 + {} + let end = self.pos(); + let octal = &self.pattern()[start.offset..end.offset]; + // Parsing the octal should never fail since the above guarantees a + // valid number. + let codepoint = + u32::from_str_radix(octal, 8).expect("valid octal number"); + // The max value for 3 digit octal is 0777 = 511 and [0, 511] has no + // invalid Unicode scalar values. + let c = char::from_u32(codepoint).expect("Unicode scalar value"); + ast::Literal { + span: Span::new(start, end), + kind: ast::LiteralKind::Octal, + c, + } + } + + /// Parse a hex representation of a Unicode codepoint. This handles both + /// hex notations, i.e., `\xFF` and `\x{FFFF}`. This expects the parser to + /// be positioned at the `x`, `u` or `U` prefix. The parser is advanced to + /// the first character immediately following the hexadecimal literal. + #[inline(never)] + fn parse_hex(&self) -> Result<ast::Literal> { + assert!( + self.char() == 'x' || self.char() == 'u' || self.char() == 'U' + ); + + let hex_kind = match self.char() { + 'x' => ast::HexLiteralKind::X, + 'u' => ast::HexLiteralKind::UnicodeShort, + _ => ast::HexLiteralKind::UnicodeLong, + }; + if !self.bump_and_bump_space() { + return Err( + self.error(self.span(), ast::ErrorKind::EscapeUnexpectedEof) + ); + } + if self.char() == '{' { + self.parse_hex_brace(hex_kind) + } else { + self.parse_hex_digits(hex_kind) + } + } + + /// Parse an N-digit hex representation of a Unicode codepoint. This + /// expects the parser to be positioned at the first digit and will advance + /// the parser to the first character immediately following the escape + /// sequence. + /// + /// The number of digits given must be 2 (for `\xNN`), 4 (for `\uNNNN`) + /// or 8 (for `\UNNNNNNNN`). + #[inline(never)] + fn parse_hex_digits( + &self, + kind: ast::HexLiteralKind, + ) -> Result<ast::Literal> { + let mut scratch = self.parser().scratch.borrow_mut(); + scratch.clear(); + + let start = self.pos(); + for i in 0..kind.digits() { + if i > 0 && !self.bump_and_bump_space() { + return Err(self + .error(self.span(), ast::ErrorKind::EscapeUnexpectedEof)); + } + if !is_hex(self.char()) { + return Err(self.error( + self.span_char(), + ast::ErrorKind::EscapeHexInvalidDigit, + )); + } + scratch.push(self.char()); + } + // The final bump just moves the parser past the literal, which may + // be EOF. + self.bump_and_bump_space(); + let end = self.pos(); + let hex = scratch.as_str(); + match u32::from_str_radix(hex, 16).ok().and_then(char::from_u32) { + None => Err(self.error( + Span::new(start, end), + ast::ErrorKind::EscapeHexInvalid, + )), + Some(c) => Ok(ast::Literal { + span: Span::new(start, end), + kind: ast::LiteralKind::HexFixed(kind), + c, + }), + } + } + + /// Parse a hex representation of any Unicode scalar value. This expects + /// the parser to be positioned at the opening brace `{` and will advance + /// the parser to the first character following the closing brace `}`. + #[inline(never)] + fn parse_hex_brace( + &self, + kind: ast::HexLiteralKind, + ) -> Result<ast::Literal> { + let mut scratch = self.parser().scratch.borrow_mut(); + scratch.clear(); + + let brace_pos = self.pos(); + let start = self.span_char().end; + while self.bump_and_bump_space() && self.char() != '}' { + if !is_hex(self.char()) { + return Err(self.error( + self.span_char(), + ast::ErrorKind::EscapeHexInvalidDigit, + )); + } + scratch.push(self.char()); + } + if self.is_eof() { + return Err(self.error( + Span::new(brace_pos, self.pos()), + ast::ErrorKind::EscapeUnexpectedEof, + )); + } + let end = self.pos(); + let hex = scratch.as_str(); + assert_eq!(self.char(), '}'); + self.bump_and_bump_space(); + + if hex.is_empty() { + return Err(self.error( + Span::new(brace_pos, self.pos()), + ast::ErrorKind::EscapeHexEmpty, + )); + } + match u32::from_str_radix(hex, 16).ok().and_then(char::from_u32) { + None => Err(self.error( + Span::new(start, end), + ast::ErrorKind::EscapeHexInvalid, + )), + Some(c) => Ok(ast::Literal { + span: Span::new(start, self.pos()), + kind: ast::LiteralKind::HexBrace(kind), + c, + }), + } + } + + /// Parse a decimal number into a u32 while trimming leading and trailing + /// whitespace. + /// + /// This expects the parser to be positioned at the first position where + /// a decimal digit could occur. This will advance the parser to the byte + /// immediately following the last contiguous decimal digit. + /// + /// If no decimal digit could be found or if there was a problem parsing + /// the complete set of digits into a u32, then an error is returned. + fn parse_decimal(&self) -> Result<u32> { + let mut scratch = self.parser().scratch.borrow_mut(); + scratch.clear(); + + while !self.is_eof() && self.char().is_whitespace() { + self.bump(); + } + let start = self.pos(); + while !self.is_eof() && '0' <= self.char() && self.char() <= '9' { + scratch.push(self.char()); + self.bump_and_bump_space(); + } + let span = Span::new(start, self.pos()); + while !self.is_eof() && self.char().is_whitespace() { + self.bump_and_bump_space(); + } + let digits = scratch.as_str(); + if digits.is_empty() { + return Err(self.error(span, ast::ErrorKind::DecimalEmpty)); + } + match u32::from_str_radix(digits, 10).ok() { + Some(n) => Ok(n), + None => Err(self.error(span, ast::ErrorKind::DecimalInvalid)), + } + } + + /// Parse a standard character class consisting primarily of characters or + /// character ranges, but can also contain nested character classes of + /// any type (sans `.`). + /// + /// This assumes the parser is positioned at the opening `[`. If parsing + /// is successful, then the parser is advanced to the position immediately + /// following the closing `]`. + #[inline(never)] + fn parse_set_class(&self) -> Result<ast::ClassBracketed> { + assert_eq!(self.char(), '['); + + let mut union = + ast::ClassSetUnion { span: self.span(), items: vec![] }; + loop { + self.bump_space(); + if self.is_eof() { + return Err(self.unclosed_class_error()); + } + match self.char() { + '[' => { + // If we've already parsed the opening bracket, then + // attempt to treat this as the beginning of an ASCII + // class. If ASCII class parsing fails, then the parser + // backs up to `[`. + if !self.parser().stack_class.borrow().is_empty() { + if let Some(cls) = self.maybe_parse_ascii_class() { + union.push(ast::ClassSetItem::Ascii(cls)); + continue; + } + } + union = self.push_class_open(union)?; + } + ']' => match self.pop_class(union)? { + Either::Left(nested_union) => { + union = nested_union; + } + Either::Right(class) => return Ok(class), + }, + '&' if self.peek() == Some('&') => { + assert!(self.bump_if("&&")); + union = self.push_class_op( + ast::ClassSetBinaryOpKind::Intersection, + union, + ); + } + '-' if self.peek() == Some('-') => { + assert!(self.bump_if("--")); + union = self.push_class_op( + ast::ClassSetBinaryOpKind::Difference, + union, + ); + } + '~' if self.peek() == Some('~') => { + assert!(self.bump_if("~~")); + union = self.push_class_op( + ast::ClassSetBinaryOpKind::SymmetricDifference, + union, + ); + } + _ => { + union.push(self.parse_set_class_range()?); + } + } + } + } + + /// Parse a single primitive item in a character class set. The item to + /// be parsed can either be one of a simple literal character, a range + /// between two simple literal characters or a "primitive" character + /// class like \w or \p{Greek}. + /// + /// If an invalid escape is found, or if a character class is found where + /// a simple literal is expected (e.g., in a range), then an error is + /// returned. + #[inline(never)] + fn parse_set_class_range(&self) -> Result<ast::ClassSetItem> { + let prim1 = self.parse_set_class_item()?; + self.bump_space(); + if self.is_eof() { + return Err(self.unclosed_class_error()); + } + // If the next char isn't a `-`, then we don't have a range. + // There are two exceptions. If the char after a `-` is a `]`, then + // `-` is interpreted as a literal `-`. Alternatively, if the char + // after a `-` is a `-`, then `--` corresponds to a "difference" + // operation. + if self.char() != '-' + || self.peek_space() == Some(']') + || self.peek_space() == Some('-') + { + return prim1.into_class_set_item(self); + } + // OK, now we're parsing a range, so bump past the `-` and parse the + // second half of the range. + if !self.bump_and_bump_space() { + return Err(self.unclosed_class_error()); + } + let prim2 = self.parse_set_class_item()?; + let range = ast::ClassSetRange { + span: Span::new(prim1.span().start, prim2.span().end), + start: prim1.into_class_literal(self)?, + end: prim2.into_class_literal(self)?, + }; + if !range.is_valid() { + return Err( + self.error(range.span, ast::ErrorKind::ClassRangeInvalid) + ); + } + Ok(ast::ClassSetItem::Range(range)) + } + + /// Parse a single item in a character class as a primitive, where the + /// primitive either consists of a verbatim literal or a single escape + /// sequence. + /// + /// This assumes the parser is positioned at the beginning of a primitive, + /// and advances the parser to the first position after the primitive if + /// successful. + /// + /// Note that it is the caller's responsibility to report an error if an + /// illegal primitive was parsed. + #[inline(never)] + fn parse_set_class_item(&self) -> Result<Primitive> { + if self.char() == '\\' { + self.parse_escape() + } else { + let x = Primitive::Literal(ast::Literal { + span: self.span_char(), + kind: ast::LiteralKind::Verbatim, + c: self.char(), + }); + self.bump(); + Ok(x) + } + } + + /// Parses the opening of a character class set. This includes the opening + /// bracket along with `^` if present to indicate negation. This also + /// starts parsing the opening set of unioned items if applicable, since + /// there are special rules applied to certain characters in the opening + /// of a character class. For example, `[^]]` is the class of all + /// characters not equal to `]`. (`]` would need to be escaped in any other + /// position.) Similarly for `-`. + /// + /// In all cases, the op inside the returned `ast::ClassBracketed` is an + /// empty union. This empty union should be replaced with the actual item + /// when it is popped from the parser's stack. + /// + /// This assumes the parser is positioned at the opening `[` and advances + /// the parser to the first non-special byte of the character class. + /// + /// An error is returned if EOF is found. + #[inline(never)] + fn parse_set_class_open( + &self, + ) -> Result<(ast::ClassBracketed, ast::ClassSetUnion)> { + assert_eq!(self.char(), '['); + let start = self.pos(); + if !self.bump_and_bump_space() { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::ClassUnclosed, + )); + } + + let negated = if self.char() != '^' { + false + } else { + if !self.bump_and_bump_space() { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::ClassUnclosed, + )); + } + true + }; + // Accept any number of `-` as literal `-`. + let mut union = + ast::ClassSetUnion { span: self.span(), items: vec![] }; + while self.char() == '-' { + union.push(ast::ClassSetItem::Literal(ast::Literal { + span: self.span_char(), + kind: ast::LiteralKind::Verbatim, + c: '-', + })); + if !self.bump_and_bump_space() { + return Err(self.error( + Span::new(start, start), + ast::ErrorKind::ClassUnclosed, + )); + } + } + // If `]` is the *first* char in a set, then interpret it as a literal + // `]`. That is, an empty class is impossible to write. + if union.items.is_empty() && self.char() == ']' { + union.push(ast::ClassSetItem::Literal(ast::Literal { + span: self.span_char(), + kind: ast::LiteralKind::Verbatim, + c: ']', + })); + if !self.bump_and_bump_space() { + return Err(self.error( + Span::new(start, self.pos()), + ast::ErrorKind::ClassUnclosed, + )); + } + } + let set = ast::ClassBracketed { + span: Span::new(start, self.pos()), + negated, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: Span::new(union.span.start, union.span.start), + items: vec![], + }), + }; + Ok((set, union)) + } + + /// Attempt to parse an ASCII character class, e.g., `[:alnum:]`. + /// + /// This assumes the parser is positioned at the opening `[`. + /// + /// If no valid ASCII character class could be found, then this does not + /// advance the parser and `None` is returned. Otherwise, the parser is + /// advanced to the first byte following the closing `]` and the + /// corresponding ASCII class is returned. + #[inline(never)] + fn maybe_parse_ascii_class(&self) -> Option<ast::ClassAscii> { + // ASCII character classes are interesting from a parsing perspective + // because parsing cannot fail with any interesting error. For example, + // in order to use an ASCII character class, it must be enclosed in + // double brackets, e.g., `[[:alnum:]]`. Alternatively, you might think + // of it as "ASCII character classes have the syntax `[:NAME:]` which + // can only appear within character brackets." This means that things + // like `[[:lower:]A]` are legal constructs. + // + // However, if one types an incorrect ASCII character class, e.g., + // `[[:loower:]]`, then we treat that as a normal nested character + // class containing the characters `:elorw`. One might argue that we + // should return an error instead since the repeated colons give away + // the intent to write an ASCII class. But what if the user typed + // `[[:lower]]` instead? How can we tell that was intended to be an + // ASCII class and not just a normal nested class? + // + // Reasonable people can probably disagree over this, but for better + // or worse, we implement semantics that never fails at the expense + // of better failure modes. + assert_eq!(self.char(), '['); + // If parsing fails, then we back up the parser to this starting point. + let start = self.pos(); + let mut negated = false; + if !self.bump() || self.char() != ':' { + self.parser().pos.set(start); + return None; + } + if !self.bump() { + self.parser().pos.set(start); + return None; + } + if self.char() == '^' { + negated = true; + if !self.bump() { + self.parser().pos.set(start); + return None; + } + } + let name_start = self.offset(); + while self.char() != ':' && self.bump() {} + if self.is_eof() { + self.parser().pos.set(start); + return None; + } + let name = &self.pattern()[name_start..self.offset()]; + if !self.bump_if(":]") { + self.parser().pos.set(start); + return None; + } + let kind = match ast::ClassAsciiKind::from_name(name) { + Some(kind) => kind, + None => { + self.parser().pos.set(start); + return None; + } + }; + Some(ast::ClassAscii { + span: Span::new(start, self.pos()), + kind, + negated, + }) + } + + /// Parse a Unicode class in either the single character notation, `\pN` + /// or the multi-character bracketed notation, `\p{Greek}`. This assumes + /// the parser is positioned at the `p` (or `P` for negation) and will + /// advance the parser to the character immediately following the class. + /// + /// Note that this does not check whether the class name is valid or not. + #[inline(never)] + fn parse_unicode_class(&self) -> Result<ast::ClassUnicode> { + assert!(self.char() == 'p' || self.char() == 'P'); + + let mut scratch = self.parser().scratch.borrow_mut(); + scratch.clear(); + + let negated = self.char() == 'P'; + if !self.bump_and_bump_space() { + return Err( + self.error(self.span(), ast::ErrorKind::EscapeUnexpectedEof) + ); + } + let (start, kind) = if self.char() == '{' { + let start = self.span_char().end; + while self.bump_and_bump_space() && self.char() != '}' { + scratch.push(self.char()); + } + if self.is_eof() { + return Err(self + .error(self.span(), ast::ErrorKind::EscapeUnexpectedEof)); + } + assert_eq!(self.char(), '}'); + self.bump(); + + let name = scratch.as_str(); + if let Some(i) = name.find("!=") { + ( + start, + ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::NotEqual, + name: name[..i].to_string(), + value: name[i + 2..].to_string(), + }, + ) + } else if let Some(i) = name.find(':') { + ( + start, + ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::Colon, + name: name[..i].to_string(), + value: name[i + 1..].to_string(), + }, + ) + } else if let Some(i) = name.find('=') { + ( + start, + ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::Equal, + name: name[..i].to_string(), + value: name[i + 1..].to_string(), + }, + ) + } else { + (start, ast::ClassUnicodeKind::Named(name.to_string())) + } + } else { + let start = self.pos(); + let c = self.char(); + if c == '\\' { + return Err(self.error( + self.span_char(), + ast::ErrorKind::UnicodeClassInvalid, + )); + } + self.bump_and_bump_space(); + let kind = ast::ClassUnicodeKind::OneLetter(c); + (start, kind) + }; + Ok(ast::ClassUnicode { + span: Span::new(start, self.pos()), + negated, + kind, + }) + } + + /// Parse a Perl character class, e.g., `\d` or `\W`. This assumes the + /// parser is currently at a valid character class name and will be + /// advanced to the character immediately following the class. + #[inline(never)] + fn parse_perl_class(&self) -> ast::ClassPerl { + let c = self.char(); + let span = self.span_char(); + self.bump(); + let (negated, kind) = match c { + 'd' => (false, ast::ClassPerlKind::Digit), + 'D' => (true, ast::ClassPerlKind::Digit), + 's' => (false, ast::ClassPerlKind::Space), + 'S' => (true, ast::ClassPerlKind::Space), + 'w' => (false, ast::ClassPerlKind::Word), + 'W' => (true, ast::ClassPerlKind::Word), + c => panic!("expected valid Perl class but got '{c}'"), + }; + ast::ClassPerl { span, kind, negated } + } +} + +/// A type that traverses a fully parsed Ast and checks whether its depth +/// exceeds the specified nesting limit. If it does, then an error is returned. +#[derive(Debug)] +struct NestLimiter<'p, 's, P> { + /// The parser that is checking the nest limit. + p: &'p ParserI<'s, P>, + /// The current depth while walking an Ast. + depth: u32, +} + +impl<'p, 's, P: Borrow<Parser>> NestLimiter<'p, 's, P> { + fn new(p: &'p ParserI<'s, P>) -> NestLimiter<'p, 's, P> { + NestLimiter { p, depth: 0 } + } + + #[inline(never)] + fn check(self, ast: &Ast) -> Result<()> { + ast::visit(ast, self) + } + + fn increment_depth(&mut self, span: &Span) -> Result<()> { + let new = self.depth.checked_add(1).ok_or_else(|| { + self.p.error( + span.clone(), + ast::ErrorKind::NestLimitExceeded(u32::MAX), + ) + })?; + let limit = self.p.parser().nest_limit; + if new > limit { + return Err(self.p.error( + span.clone(), + ast::ErrorKind::NestLimitExceeded(limit), + )); + } + self.depth = new; + Ok(()) + } + + fn decrement_depth(&mut self) { + // Assuming the correctness of the visitor, this should never drop + // below 0. + self.depth = self.depth.checked_sub(1).unwrap(); + } +} + +impl<'p, 's, P: Borrow<Parser>> ast::Visitor for NestLimiter<'p, 's, P> { + type Output = (); + type Err = ast::Error; + + fn finish(self) -> Result<()> { + Ok(()) + } + + fn visit_pre(&mut self, ast: &Ast) -> Result<()> { + let span = match *ast { + Ast::Empty(_) + | Ast::Flags(_) + | Ast::Literal(_) + | Ast::Dot(_) + | Ast::Assertion(_) + | Ast::ClassUnicode(_) + | Ast::ClassPerl(_) => { + // These are all base cases, so we don't increment depth. + return Ok(()); + } + Ast::ClassBracketed(ref x) => &x.span, + Ast::Repetition(ref x) => &x.span, + Ast::Group(ref x) => &x.span, + Ast::Alternation(ref x) => &x.span, + Ast::Concat(ref x) => &x.span, + }; + self.increment_depth(span) + } + + fn visit_post(&mut self, ast: &Ast) -> Result<()> { + match *ast { + Ast::Empty(_) + | Ast::Flags(_) + | Ast::Literal(_) + | Ast::Dot(_) + | Ast::Assertion(_) + | Ast::ClassUnicode(_) + | Ast::ClassPerl(_) => { + // These are all base cases, so we don't decrement depth. + Ok(()) + } + Ast::ClassBracketed(_) + | Ast::Repetition(_) + | Ast::Group(_) + | Ast::Alternation(_) + | Ast::Concat(_) => { + self.decrement_depth(); + Ok(()) + } + } + } + + fn visit_class_set_item_pre( + &mut self, + ast: &ast::ClassSetItem, + ) -> Result<()> { + let span = match *ast { + ast::ClassSetItem::Empty(_) + | ast::ClassSetItem::Literal(_) + | ast::ClassSetItem::Range(_) + | ast::ClassSetItem::Ascii(_) + | ast::ClassSetItem::Unicode(_) + | ast::ClassSetItem::Perl(_) => { + // These are all base cases, so we don't increment depth. + return Ok(()); + } + ast::ClassSetItem::Bracketed(ref x) => &x.span, + ast::ClassSetItem::Union(ref x) => &x.span, + }; + self.increment_depth(span) + } + + fn visit_class_set_item_post( + &mut self, + ast: &ast::ClassSetItem, + ) -> Result<()> { + match *ast { + ast::ClassSetItem::Empty(_) + | ast::ClassSetItem::Literal(_) + | ast::ClassSetItem::Range(_) + | ast::ClassSetItem::Ascii(_) + | ast::ClassSetItem::Unicode(_) + | ast::ClassSetItem::Perl(_) => { + // These are all base cases, so we don't decrement depth. + Ok(()) + } + ast::ClassSetItem::Bracketed(_) | ast::ClassSetItem::Union(_) => { + self.decrement_depth(); + Ok(()) + } + } + } + + fn visit_class_set_binary_op_pre( + &mut self, + ast: &ast::ClassSetBinaryOp, + ) -> Result<()> { + self.increment_depth(&ast.span) + } + + fn visit_class_set_binary_op_post( + &mut self, + _ast: &ast::ClassSetBinaryOp, + ) -> Result<()> { + self.decrement_depth(); + Ok(()) + } +} + +/// When the result is an error, transforms the ast::ErrorKind from the source +/// Result into another one. This function is used to return clearer error +/// messages when possible. +fn specialize_err<T>( + result: Result<T>, + from: ast::ErrorKind, + to: ast::ErrorKind, +) -> Result<T> { + if let Err(e) = result { + if e.kind == from { + Err(ast::Error { kind: to, pattern: e.pattern, span: e.span }) + } else { + Err(e) + } + } else { + result + } +} + +#[cfg(test)] +mod tests { + use core::ops::Range; + + use alloc::format; + + use super::*; + + // Our own assert_eq, which has slightly better formatting (but honestly + // still kind of crappy). + macro_rules! assert_eq { + ($left:expr, $right:expr) => {{ + match (&$left, &$right) { + (left_val, right_val) => { + if !(*left_val == *right_val) { + panic!( + "assertion failed: `(left == right)`\n\n\ + left: `{:?}`\nright: `{:?}`\n\n", + left_val, right_val + ) + } + } + } + }}; + } + + // We create these errors to compare with real ast::Errors in the tests. + // We define equality between TestError and ast::Error to disregard the + // pattern string in ast::Error, which is annoying to provide in tests. + #[derive(Clone, Debug)] + struct TestError { + span: Span, + kind: ast::ErrorKind, + } + + impl PartialEq<ast::Error> for TestError { + fn eq(&self, other: &ast::Error) -> bool { + self.span == other.span && self.kind == other.kind + } + } + + impl PartialEq<TestError> for ast::Error { + fn eq(&self, other: &TestError) -> bool { + self.span == other.span && self.kind == other.kind + } + } + + fn s(str: &str) -> String { + str.to_string() + } + + fn parser(pattern: &str) -> ParserI<'_, Parser> { + ParserI::new(Parser::new(), pattern) + } + + fn parser_octal(pattern: &str) -> ParserI<'_, Parser> { + let parser = ParserBuilder::new().octal(true).build(); + ParserI::new(parser, pattern) + } + + fn parser_empty_min_range(pattern: &str) -> ParserI<'_, Parser> { + let parser = ParserBuilder::new().empty_min_range(true).build(); + ParserI::new(parser, pattern) + } + + fn parser_nest_limit( + pattern: &str, + nest_limit: u32, + ) -> ParserI<'_, Parser> { + let p = ParserBuilder::new().nest_limit(nest_limit).build(); + ParserI::new(p, pattern) + } + + fn parser_ignore_whitespace(pattern: &str) -> ParserI<'_, Parser> { + let p = ParserBuilder::new().ignore_whitespace(true).build(); + ParserI::new(p, pattern) + } + + /// Short alias for creating a new span. + fn nspan(start: Position, end: Position) -> Span { + Span::new(start, end) + } + + /// Short alias for creating a new position. + fn npos(offset: usize, line: usize, column: usize) -> Position { + Position::new(offset, line, column) + } + + /// Create a new span from the given offset range. This assumes a single + /// line and sets the columns based on the offsets. i.e., This only works + /// out of the box for ASCII, which is fine for most tests. + fn span(range: Range<usize>) -> Span { + let start = Position::new(range.start, 1, range.start + 1); + let end = Position::new(range.end, 1, range.end + 1); + Span::new(start, end) + } + + /// Create a new span for the corresponding byte range in the given string. + fn span_range(subject: &str, range: Range<usize>) -> Span { + let start = Position { + offset: range.start, + line: 1 + subject[..range.start].matches('\n').count(), + column: 1 + subject[..range.start] + .chars() + .rev() + .position(|c| c == '\n') + .unwrap_or(subject[..range.start].chars().count()), + }; + let end = Position { + offset: range.end, + line: 1 + subject[..range.end].matches('\n').count(), + column: 1 + subject[..range.end] + .chars() + .rev() + .position(|c| c == '\n') + .unwrap_or(subject[..range.end].chars().count()), + }; + Span::new(start, end) + } + + /// Create a verbatim literal starting at the given position. + fn lit(c: char, start: usize) -> Ast { + lit_with(c, span(start..start + c.len_utf8())) + } + + /// Create a meta literal starting at the given position. + fn meta_lit(c: char, span: Span) -> Ast { + Ast::literal(ast::Literal { span, kind: ast::LiteralKind::Meta, c }) + } + + /// Create a verbatim literal with the given span. + fn lit_with(c: char, span: Span) -> Ast { + Ast::literal(ast::Literal { + span, + kind: ast::LiteralKind::Verbatim, + c, + }) + } + + /// Create a concatenation with the given range. + fn concat(range: Range<usize>, asts: Vec<Ast>) -> Ast { + concat_with(span(range), asts) + } + + /// Create a concatenation with the given span. + fn concat_with(span: Span, asts: Vec<Ast>) -> Ast { + Ast::concat(ast::Concat { span, asts }) + } + + /// Create an alternation with the given span. + fn alt(range: Range<usize>, asts: Vec<Ast>) -> Ast { + Ast::alternation(ast::Alternation { span: span(range), asts }) + } + + /// Create a capturing group with the given span. + fn group(range: Range<usize>, index: u32, ast: Ast) -> Ast { + Ast::group(ast::Group { + span: span(range), + kind: ast::GroupKind::CaptureIndex(index), + ast: Box::new(ast), + }) + } + + /// Create an ast::SetFlags. + /// + /// The given pattern should be the full pattern string. The range given + /// should correspond to the byte offsets where the flag set occurs. + /// + /// If negated is true, then the set is interpreted as beginning with a + /// negation. + fn flag_set( + pat: &str, + range: Range<usize>, + flag: ast::Flag, + negated: bool, + ) -> Ast { + let mut items = vec![ast::FlagsItem { + span: span_range(pat, (range.end - 2)..(range.end - 1)), + kind: ast::FlagsItemKind::Flag(flag), + }]; + if negated { + items.insert( + 0, + ast::FlagsItem { + span: span_range(pat, (range.start + 2)..(range.end - 2)), + kind: ast::FlagsItemKind::Negation, + }, + ); + } + Ast::flags(ast::SetFlags { + span: span_range(pat, range.clone()), + flags: ast::Flags { + span: span_range(pat, (range.start + 2)..(range.end - 1)), + items, + }, + }) + } + + #[test] + fn parse_nest_limit() { + // A nest limit of 0 still allows some types of regexes. + assert_eq!( + parser_nest_limit("", 0).parse(), + Ok(Ast::empty(span(0..0))) + ); + assert_eq!(parser_nest_limit("a", 0).parse(), Ok(lit('a', 0))); + + // Test repetition operations, which require one level of nesting. + assert_eq!( + parser_nest_limit("a+", 0).parse().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::NestLimitExceeded(0), + } + ); + assert_eq!( + parser_nest_limit("a+", 1).parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..2), + op: ast::RepetitionOp { + span: span(1..2), + kind: ast::RepetitionKind::OneOrMore, + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser_nest_limit("(a)+", 1).parse().unwrap_err(), + TestError { + span: span(0..3), + kind: ast::ErrorKind::NestLimitExceeded(1), + } + ); + assert_eq!( + parser_nest_limit("a+*", 1).parse().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::NestLimitExceeded(1), + } + ); + assert_eq!( + parser_nest_limit("a+*", 2).parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..3), + op: ast::RepetitionOp { + span: span(2..3), + kind: ast::RepetitionKind::ZeroOrMore, + }, + greedy: true, + ast: Box::new(Ast::repetition(ast::Repetition { + span: span(0..2), + op: ast::RepetitionOp { + span: span(1..2), + kind: ast::RepetitionKind::OneOrMore, + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })), + })) + ); + + // Test concatenations. A concatenation requires one level of nesting. + assert_eq!( + parser_nest_limit("ab", 0).parse().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::NestLimitExceeded(0), + } + ); + assert_eq!( + parser_nest_limit("ab", 1).parse(), + Ok(concat(0..2, vec![lit('a', 0), lit('b', 1)])) + ); + assert_eq!( + parser_nest_limit("abc", 1).parse(), + Ok(concat(0..3, vec![lit('a', 0), lit('b', 1), lit('c', 2)])) + ); + + // Test alternations. An alternation requires one level of nesting. + assert_eq!( + parser_nest_limit("a|b", 0).parse().unwrap_err(), + TestError { + span: span(0..3), + kind: ast::ErrorKind::NestLimitExceeded(0), + } + ); + assert_eq!( + parser_nest_limit("a|b", 1).parse(), + Ok(alt(0..3, vec![lit('a', 0), lit('b', 2)])) + ); + assert_eq!( + parser_nest_limit("a|b|c", 1).parse(), + Ok(alt(0..5, vec![lit('a', 0), lit('b', 2), lit('c', 4)])) + ); + + // Test character classes. Classes form their own mini-recursive + // syntax! + assert_eq!( + parser_nest_limit("[a]", 0).parse().unwrap_err(), + TestError { + span: span(0..3), + kind: ast::ErrorKind::NestLimitExceeded(0), + } + ); + assert_eq!( + parser_nest_limit("[a]", 1).parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..3), + negated: false, + kind: ast::ClassSet::Item(ast::ClassSetItem::Literal( + ast::Literal { + span: span(1..2), + kind: ast::LiteralKind::Verbatim, + c: 'a', + } + )), + })) + ); + assert_eq!( + parser_nest_limit("[ab]", 1).parse().unwrap_err(), + TestError { + span: span(1..3), + kind: ast::ErrorKind::NestLimitExceeded(1), + } + ); + assert_eq!( + parser_nest_limit("[ab[cd]]", 2).parse().unwrap_err(), + TestError { + span: span(3..7), + kind: ast::ErrorKind::NestLimitExceeded(2), + } + ); + assert_eq!( + parser_nest_limit("[ab[cd]]", 3).parse().unwrap_err(), + TestError { + span: span(4..6), + kind: ast::ErrorKind::NestLimitExceeded(3), + } + ); + assert_eq!( + parser_nest_limit("[a--b]", 1).parse().unwrap_err(), + TestError { + span: span(1..5), + kind: ast::ErrorKind::NestLimitExceeded(1), + } + ); + assert_eq!( + parser_nest_limit("[a--bc]", 2).parse().unwrap_err(), + TestError { + span: span(4..6), + kind: ast::ErrorKind::NestLimitExceeded(2), + } + ); + } + + #[test] + fn parse_comments() { + let pat = "(?x) +# This is comment 1. +foo # This is comment 2. + # This is comment 3. +bar +# This is comment 4."; + let astc = parser(pat).parse_with_comments().unwrap(); + assert_eq!( + astc.ast, + concat_with( + span_range(pat, 0..pat.len()), + vec![ + flag_set(pat, 0..4, ast::Flag::IgnoreWhitespace, false), + lit_with('f', span_range(pat, 26..27)), + lit_with('o', span_range(pat, 27..28)), + lit_with('o', span_range(pat, 28..29)), + lit_with('b', span_range(pat, 74..75)), + lit_with('a', span_range(pat, 75..76)), + lit_with('r', span_range(pat, 76..77)), + ] + ) + ); + assert_eq!( + astc.comments, + vec![ + ast::Comment { + span: span_range(pat, 5..26), + comment: s(" This is comment 1."), + }, + ast::Comment { + span: span_range(pat, 30..51), + comment: s(" This is comment 2."), + }, + ast::Comment { + span: span_range(pat, 53..74), + comment: s(" This is comment 3."), + }, + ast::Comment { + span: span_range(pat, 78..98), + comment: s(" This is comment 4."), + }, + ] + ); + } + + #[test] + fn parse_holistic() { + assert_eq!(parser("]").parse(), Ok(lit(']', 0))); + assert_eq!( + parser(r"\\\.\+\*\?\(\)\|\[\]\{\}\^\$\#\&\-\~").parse(), + Ok(concat( + 0..36, + vec![ + meta_lit('\\', span(0..2)), + meta_lit('.', span(2..4)), + meta_lit('+', span(4..6)), + meta_lit('*', span(6..8)), + meta_lit('?', span(8..10)), + meta_lit('(', span(10..12)), + meta_lit(')', span(12..14)), + meta_lit('|', span(14..16)), + meta_lit('[', span(16..18)), + meta_lit(']', span(18..20)), + meta_lit('{', span(20..22)), + meta_lit('}', span(22..24)), + meta_lit('^', span(24..26)), + meta_lit('$', span(26..28)), + meta_lit('#', span(28..30)), + meta_lit('&', span(30..32)), + meta_lit('-', span(32..34)), + meta_lit('~', span(34..36)), + ] + )) + ); + } + + #[test] + fn parse_ignore_whitespace() { + // Test that basic whitespace insensitivity works. + let pat = "(?x)a b"; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + nspan(npos(0, 1, 1), npos(7, 1, 8)), + vec![ + flag_set(pat, 0..4, ast::Flag::IgnoreWhitespace, false), + lit_with('a', nspan(npos(4, 1, 5), npos(5, 1, 6))), + lit_with('b', nspan(npos(6, 1, 7), npos(7, 1, 8))), + ] + )) + ); + + // Test that we can toggle whitespace insensitivity. + let pat = "(?x)a b(?-x)a b"; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + nspan(npos(0, 1, 1), npos(15, 1, 16)), + vec![ + flag_set(pat, 0..4, ast::Flag::IgnoreWhitespace, false), + lit_with('a', nspan(npos(4, 1, 5), npos(5, 1, 6))), + lit_with('b', nspan(npos(6, 1, 7), npos(7, 1, 8))), + flag_set(pat, 7..12, ast::Flag::IgnoreWhitespace, true), + lit_with('a', nspan(npos(12, 1, 13), npos(13, 1, 14))), + lit_with(' ', nspan(npos(13, 1, 14), npos(14, 1, 15))), + lit_with('b', nspan(npos(14, 1, 15), npos(15, 1, 16))), + ] + )) + ); + + // Test that nesting whitespace insensitive flags works. + let pat = "a (?x:a )a "; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + span_range(pat, 0..11), + vec![ + lit_with('a', span_range(pat, 0..1)), + lit_with(' ', span_range(pat, 1..2)), + Ast::group(ast::Group { + span: span_range(pat, 2..9), + kind: ast::GroupKind::NonCapturing(ast::Flags { + span: span_range(pat, 4..5), + items: vec![ast::FlagsItem { + span: span_range(pat, 4..5), + kind: ast::FlagsItemKind::Flag( + ast::Flag::IgnoreWhitespace + ), + },], + }), + ast: Box::new(lit_with('a', span_range(pat, 6..7))), + }), + lit_with('a', span_range(pat, 9..10)), + lit_with(' ', span_range(pat, 10..11)), + ] + )) + ); + + // Test that whitespace after an opening paren is insignificant. + let pat = "(?x)( ?P<foo> a )"; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + span_range(pat, 0..pat.len()), + vec![ + flag_set(pat, 0..4, ast::Flag::IgnoreWhitespace, false), + Ast::group(ast::Group { + span: span_range(pat, 4..pat.len()), + kind: ast::GroupKind::CaptureName { + starts_with_p: true, + name: ast::CaptureName { + span: span_range(pat, 9..12), + name: s("foo"), + index: 1, + } + }, + ast: Box::new(lit_with('a', span_range(pat, 14..15))), + }), + ] + )) + ); + let pat = "(?x)( a )"; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + span_range(pat, 0..pat.len()), + vec![ + flag_set(pat, 0..4, ast::Flag::IgnoreWhitespace, false), + Ast::group(ast::Group { + span: span_range(pat, 4..pat.len()), + kind: ast::GroupKind::CaptureIndex(1), + ast: Box::new(lit_with('a', span_range(pat, 7..8))), + }), + ] + )) + ); + let pat = "(?x)( ?: a )"; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + span_range(pat, 0..pat.len()), + vec![ + flag_set(pat, 0..4, ast::Flag::IgnoreWhitespace, false), + Ast::group(ast::Group { + span: span_range(pat, 4..pat.len()), + kind: ast::GroupKind::NonCapturing(ast::Flags { + span: span_range(pat, 8..8), + items: vec![], + }), + ast: Box::new(lit_with('a', span_range(pat, 11..12))), + }), + ] + )) + ); + let pat = r"(?x)\x { 53 }"; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + span_range(pat, 0..pat.len()), + vec![ + flag_set(pat, 0..4, ast::Flag::IgnoreWhitespace, false), + Ast::literal(ast::Literal { + span: span(4..13), + kind: ast::LiteralKind::HexBrace( + ast::HexLiteralKind::X + ), + c: 'S', + }), + ] + )) + ); + + // Test that whitespace after an escape is OK. + let pat = r"(?x)\ "; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + span_range(pat, 0..pat.len()), + vec![ + flag_set(pat, 0..4, ast::Flag::IgnoreWhitespace, false), + Ast::literal(ast::Literal { + span: span_range(pat, 4..6), + kind: ast::LiteralKind::Superfluous, + c: ' ', + }), + ] + )) + ); + } + + #[test] + fn parse_newlines() { + let pat = ".\n."; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + span_range(pat, 0..3), + vec![ + Ast::dot(span_range(pat, 0..1)), + lit_with('\n', span_range(pat, 1..2)), + Ast::dot(span_range(pat, 2..3)), + ] + )) + ); + + let pat = "foobar\nbaz\nquux\n"; + assert_eq!( + parser(pat).parse(), + Ok(concat_with( + span_range(pat, 0..pat.len()), + vec![ + lit_with('f', nspan(npos(0, 1, 1), npos(1, 1, 2))), + lit_with('o', nspan(npos(1, 1, 2), npos(2, 1, 3))), + lit_with('o', nspan(npos(2, 1, 3), npos(3, 1, 4))), + lit_with('b', nspan(npos(3, 1, 4), npos(4, 1, 5))), + lit_with('a', nspan(npos(4, 1, 5), npos(5, 1, 6))), + lit_with('r', nspan(npos(5, 1, 6), npos(6, 1, 7))), + lit_with('\n', nspan(npos(6, 1, 7), npos(7, 2, 1))), + lit_with('b', nspan(npos(7, 2, 1), npos(8, 2, 2))), + lit_with('a', nspan(npos(8, 2, 2), npos(9, 2, 3))), + lit_with('z', nspan(npos(9, 2, 3), npos(10, 2, 4))), + lit_with('\n', nspan(npos(10, 2, 4), npos(11, 3, 1))), + lit_with('q', nspan(npos(11, 3, 1), npos(12, 3, 2))), + lit_with('u', nspan(npos(12, 3, 2), npos(13, 3, 3))), + lit_with('u', nspan(npos(13, 3, 3), npos(14, 3, 4))), + lit_with('x', nspan(npos(14, 3, 4), npos(15, 3, 5))), + lit_with('\n', nspan(npos(15, 3, 5), npos(16, 4, 1))), + ] + )) + ); + } + + #[test] + fn parse_uncounted_repetition() { + assert_eq!( + parser(r"a*").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..2), + op: ast::RepetitionOp { + span: span(1..2), + kind: ast::RepetitionKind::ZeroOrMore, + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"a+").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..2), + op: ast::RepetitionOp { + span: span(1..2), + kind: ast::RepetitionKind::OneOrMore, + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + + assert_eq!( + parser(r"a?").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..2), + op: ast::RepetitionOp { + span: span(1..2), + kind: ast::RepetitionKind::ZeroOrOne, + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"a??").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..3), + op: ast::RepetitionOp { + span: span(1..3), + kind: ast::RepetitionKind::ZeroOrOne, + }, + greedy: false, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"a?").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..2), + op: ast::RepetitionOp { + span: span(1..2), + kind: ast::RepetitionKind::ZeroOrOne, + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"a?b").parse(), + Ok(concat( + 0..3, + vec![ + Ast::repetition(ast::Repetition { + span: span(0..2), + op: ast::RepetitionOp { + span: span(1..2), + kind: ast::RepetitionKind::ZeroOrOne, + }, + greedy: true, + ast: Box::new(lit('a', 0)), + }), + lit('b', 2), + ] + )) + ); + assert_eq!( + parser(r"a??b").parse(), + Ok(concat( + 0..4, + vec![ + Ast::repetition(ast::Repetition { + span: span(0..3), + op: ast::RepetitionOp { + span: span(1..3), + kind: ast::RepetitionKind::ZeroOrOne, + }, + greedy: false, + ast: Box::new(lit('a', 0)), + }), + lit('b', 3), + ] + )) + ); + assert_eq!( + parser(r"ab?").parse(), + Ok(concat( + 0..3, + vec![ + lit('a', 0), + Ast::repetition(ast::Repetition { + span: span(1..3), + op: ast::RepetitionOp { + span: span(2..3), + kind: ast::RepetitionKind::ZeroOrOne, + }, + greedy: true, + ast: Box::new(lit('b', 1)), + }), + ] + )) + ); + assert_eq!( + parser(r"(ab)?").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..5), + op: ast::RepetitionOp { + span: span(4..5), + kind: ast::RepetitionKind::ZeroOrOne, + }, + greedy: true, + ast: Box::new(group( + 0..4, + 1, + concat(1..3, vec![lit('a', 1), lit('b', 2),]) + )), + })) + ); + assert_eq!( + parser(r"|a?").parse(), + Ok(alt( + 0..3, + vec![ + Ast::empty(span(0..0)), + Ast::repetition(ast::Repetition { + span: span(1..3), + op: ast::RepetitionOp { + span: span(2..3), + kind: ast::RepetitionKind::ZeroOrOne, + }, + greedy: true, + ast: Box::new(lit('a', 1)), + }), + ] + )) + ); + + assert_eq!( + parser(r"*").parse().unwrap_err(), + TestError { + span: span(0..0), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"(?i)*").parse().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"(*)").parse().unwrap_err(), + TestError { + span: span(1..1), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"(?:?)").parse().unwrap_err(), + TestError { + span: span(3..3), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"+").parse().unwrap_err(), + TestError { + span: span(0..0), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"?").parse().unwrap_err(), + TestError { + span: span(0..0), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"(?)").parse().unwrap_err(), + TestError { + span: span(1..1), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"|*").parse().unwrap_err(), + TestError { + span: span(1..1), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"|+").parse().unwrap_err(), + TestError { + span: span(1..1), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"|?").parse().unwrap_err(), + TestError { + span: span(1..1), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + } + + #[test] + fn parse_counted_repetition() { + assert_eq!( + parser(r"a{5}").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..4), + op: ast::RepetitionOp { + span: span(1..4), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Exactly(5) + ), + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"a{5,}").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..5), + op: ast::RepetitionOp { + span: span(1..5), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::AtLeast(5) + ), + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"a{5,9}").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..6), + op: ast::RepetitionOp { + span: span(1..6), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Bounded(5, 9) + ), + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"a{5}?").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..5), + op: ast::RepetitionOp { + span: span(1..5), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Exactly(5) + ), + }, + greedy: false, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"ab{5}").parse(), + Ok(concat( + 0..5, + vec![ + lit('a', 0), + Ast::repetition(ast::Repetition { + span: span(1..5), + op: ast::RepetitionOp { + span: span(2..5), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Exactly(5) + ), + }, + greedy: true, + ast: Box::new(lit('b', 1)), + }), + ] + )) + ); + assert_eq!( + parser(r"ab{5}c").parse(), + Ok(concat( + 0..6, + vec![ + lit('a', 0), + Ast::repetition(ast::Repetition { + span: span(1..5), + op: ast::RepetitionOp { + span: span(2..5), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Exactly(5) + ), + }, + greedy: true, + ast: Box::new(lit('b', 1)), + }), + lit('c', 5), + ] + )) + ); + + assert_eq!( + parser(r"a{ 5 }").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..6), + op: ast::RepetitionOp { + span: span(1..6), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Exactly(5) + ), + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"a{ 5 , 9 }").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..10), + op: ast::RepetitionOp { + span: span(1..10), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Bounded(5, 9) + ), + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser_empty_min_range(r"a{,9}").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..5), + op: ast::RepetitionOp { + span: span(1..5), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Bounded(0, 9) + ), + }, + greedy: true, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser_ignore_whitespace(r"a{5,9} ?").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..8), + op: ast::RepetitionOp { + span: span(1..8), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Bounded(5, 9) + ), + }, + greedy: false, + ast: Box::new(lit('a', 0)), + })) + ); + assert_eq!( + parser(r"\b{5,9}").parse(), + Ok(Ast::repetition(ast::Repetition { + span: span(0..7), + op: ast::RepetitionOp { + span: span(2..7), + kind: ast::RepetitionKind::Range( + ast::RepetitionRange::Bounded(5, 9) + ), + }, + greedy: true, + ast: Box::new(Ast::assertion(ast::Assertion { + span: span(0..2), + kind: ast::AssertionKind::WordBoundary, + })), + })) + ); + + assert_eq!( + parser(r"(?i){0}").parse().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"(?m){1,1}").parse().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"a{]}").parse().unwrap_err(), + TestError { + span: span(2..2), + kind: ast::ErrorKind::RepetitionCountDecimalEmpty, + } + ); + assert_eq!( + parser(r"a{1,]}").parse().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::RepetitionCountDecimalEmpty, + } + ); + assert_eq!( + parser(r"a{").parse().unwrap_err(), + TestError { + span: span(1..2), + kind: ast::ErrorKind::RepetitionCountUnclosed, + } + ); + assert_eq!( + parser(r"a{}").parse().unwrap_err(), + TestError { + span: span(2..2), + kind: ast::ErrorKind::RepetitionCountDecimalEmpty, + } + ); + assert_eq!( + parser(r"a{a").parse().unwrap_err(), + TestError { + span: span(2..2), + kind: ast::ErrorKind::RepetitionCountDecimalEmpty, + } + ); + assert_eq!( + parser(r"a{9999999999}").parse().unwrap_err(), + TestError { + span: span(2..12), + kind: ast::ErrorKind::DecimalInvalid, + } + ); + assert_eq!( + parser(r"a{9").parse().unwrap_err(), + TestError { + span: span(1..3), + kind: ast::ErrorKind::RepetitionCountUnclosed, + } + ); + assert_eq!( + parser(r"a{9,a").parse().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::RepetitionCountDecimalEmpty, + } + ); + assert_eq!( + parser(r"a{9,9999999999}").parse().unwrap_err(), + TestError { + span: span(4..14), + kind: ast::ErrorKind::DecimalInvalid, + } + ); + assert_eq!( + parser(r"a{9,").parse().unwrap_err(), + TestError { + span: span(1..4), + kind: ast::ErrorKind::RepetitionCountUnclosed, + } + ); + assert_eq!( + parser(r"a{9,11").parse().unwrap_err(), + TestError { + span: span(1..6), + kind: ast::ErrorKind::RepetitionCountUnclosed, + } + ); + assert_eq!( + parser(r"a{2,1}").parse().unwrap_err(), + TestError { + span: span(1..6), + kind: ast::ErrorKind::RepetitionCountInvalid, + } + ); + assert_eq!( + parser(r"{5}").parse().unwrap_err(), + TestError { + span: span(0..0), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + assert_eq!( + parser(r"|{5}").parse().unwrap_err(), + TestError { + span: span(1..1), + kind: ast::ErrorKind::RepetitionMissing, + } + ); + } + + #[test] + fn parse_alternate() { + assert_eq!( + parser(r"a|b").parse(), + Ok(Ast::alternation(ast::Alternation { + span: span(0..3), + asts: vec![lit('a', 0), lit('b', 2)], + })) + ); + assert_eq!( + parser(r"(a|b)").parse(), + Ok(group( + 0..5, + 1, + Ast::alternation(ast::Alternation { + span: span(1..4), + asts: vec![lit('a', 1), lit('b', 3)], + }) + )) + ); + + assert_eq!( + parser(r"a|b|c").parse(), + Ok(Ast::alternation(ast::Alternation { + span: span(0..5), + asts: vec![lit('a', 0), lit('b', 2), lit('c', 4)], + })) + ); + assert_eq!( + parser(r"ax|by|cz").parse(), + Ok(Ast::alternation(ast::Alternation { + span: span(0..8), + asts: vec![ + concat(0..2, vec![lit('a', 0), lit('x', 1)]), + concat(3..5, vec![lit('b', 3), lit('y', 4)]), + concat(6..8, vec![lit('c', 6), lit('z', 7)]), + ], + })) + ); + assert_eq!( + parser(r"(ax|by|cz)").parse(), + Ok(group( + 0..10, + 1, + Ast::alternation(ast::Alternation { + span: span(1..9), + asts: vec![ + concat(1..3, vec![lit('a', 1), lit('x', 2)]), + concat(4..6, vec![lit('b', 4), lit('y', 5)]), + concat(7..9, vec![lit('c', 7), lit('z', 8)]), + ], + }) + )) + ); + assert_eq!( + parser(r"(ax|(by|(cz)))").parse(), + Ok(group( + 0..14, + 1, + alt( + 1..13, + vec![ + concat(1..3, vec![lit('a', 1), lit('x', 2)]), + group( + 4..13, + 2, + alt( + 5..12, + vec![ + concat( + 5..7, + vec![lit('b', 5), lit('y', 6)] + ), + group( + 8..12, + 3, + concat( + 9..11, + vec![lit('c', 9), lit('z', 10),] + ) + ), + ] + ) + ), + ] + ) + )) + ); + + assert_eq!( + parser(r"|").parse(), + Ok(alt( + 0..1, + vec![Ast::empty(span(0..0)), Ast::empty(span(1..1)),] + )) + ); + assert_eq!( + parser(r"||").parse(), + Ok(alt( + 0..2, + vec![ + Ast::empty(span(0..0)), + Ast::empty(span(1..1)), + Ast::empty(span(2..2)), + ] + )) + ); + assert_eq!( + parser(r"a|").parse(), + Ok(alt(0..2, vec![lit('a', 0), Ast::empty(span(2..2)),])) + ); + assert_eq!( + parser(r"|a").parse(), + Ok(alt(0..2, vec![Ast::empty(span(0..0)), lit('a', 1),])) + ); + + assert_eq!( + parser(r"(|)").parse(), + Ok(group( + 0..3, + 1, + alt( + 1..2, + vec![Ast::empty(span(1..1)), Ast::empty(span(2..2)),] + ) + )) + ); + assert_eq!( + parser(r"(a|)").parse(), + Ok(group( + 0..4, + 1, + alt(1..3, vec![lit('a', 1), Ast::empty(span(3..3)),]) + )) + ); + assert_eq!( + parser(r"(|a)").parse(), + Ok(group( + 0..4, + 1, + alt(1..3, vec![Ast::empty(span(1..1)), lit('a', 2),]) + )) + ); + + assert_eq!( + parser(r"a|b)").parse().unwrap_err(), + TestError { + span: span(3..4), + kind: ast::ErrorKind::GroupUnopened, + } + ); + assert_eq!( + parser(r"(a|b").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::GroupUnclosed, + } + ); + } + + #[test] + fn parse_unsupported_lookaround() { + assert_eq!( + parser(r"(?=a)").parse().unwrap_err(), + TestError { + span: span(0..3), + kind: ast::ErrorKind::UnsupportedLookAround, + } + ); + assert_eq!( + parser(r"(?!a)").parse().unwrap_err(), + TestError { + span: span(0..3), + kind: ast::ErrorKind::UnsupportedLookAround, + } + ); + assert_eq!( + parser(r"(?<=a)").parse().unwrap_err(), + TestError { + span: span(0..4), + kind: ast::ErrorKind::UnsupportedLookAround, + } + ); + assert_eq!( + parser(r"(?<!a)").parse().unwrap_err(), + TestError { + span: span(0..4), + kind: ast::ErrorKind::UnsupportedLookAround, + } + ); + } + + #[test] + fn parse_group() { + assert_eq!( + parser("(?i)").parse(), + Ok(Ast::flags(ast::SetFlags { + span: span(0..4), + flags: ast::Flags { + span: span(2..3), + items: vec![ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + }], + }, + })) + ); + assert_eq!( + parser("(?iU)").parse(), + Ok(Ast::flags(ast::SetFlags { + span: span(0..5), + flags: ast::Flags { + span: span(2..4), + items: vec![ + ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + }, + ast::FlagsItem { + span: span(3..4), + kind: ast::FlagsItemKind::Flag( + ast::Flag::SwapGreed + ), + }, + ], + }, + })) + ); + assert_eq!( + parser("(?i-U)").parse(), + Ok(Ast::flags(ast::SetFlags { + span: span(0..6), + flags: ast::Flags { + span: span(2..5), + items: vec![ + ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + }, + ast::FlagsItem { + span: span(3..4), + kind: ast::FlagsItemKind::Negation, + }, + ast::FlagsItem { + span: span(4..5), + kind: ast::FlagsItemKind::Flag( + ast::Flag::SwapGreed + ), + }, + ], + }, + })) + ); + + assert_eq!( + parser("()").parse(), + Ok(Ast::group(ast::Group { + span: span(0..2), + kind: ast::GroupKind::CaptureIndex(1), + ast: Box::new(Ast::empty(span(1..1))), + })) + ); + assert_eq!( + parser("(a)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..3), + kind: ast::GroupKind::CaptureIndex(1), + ast: Box::new(lit('a', 1)), + })) + ); + assert_eq!( + parser("(())").parse(), + Ok(Ast::group(ast::Group { + span: span(0..4), + kind: ast::GroupKind::CaptureIndex(1), + ast: Box::new(Ast::group(ast::Group { + span: span(1..3), + kind: ast::GroupKind::CaptureIndex(2), + ast: Box::new(Ast::empty(span(2..2))), + })), + })) + ); + + assert_eq!( + parser("(?:a)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..5), + kind: ast::GroupKind::NonCapturing(ast::Flags { + span: span(2..2), + items: vec![], + }), + ast: Box::new(lit('a', 3)), + })) + ); + + assert_eq!( + parser("(?i:a)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..6), + kind: ast::GroupKind::NonCapturing(ast::Flags { + span: span(2..3), + items: vec![ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + },], + }), + ast: Box::new(lit('a', 4)), + })) + ); + assert_eq!( + parser("(?i-U:a)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..8), + kind: ast::GroupKind::NonCapturing(ast::Flags { + span: span(2..5), + items: vec![ + ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + }, + ast::FlagsItem { + span: span(3..4), + kind: ast::FlagsItemKind::Negation, + }, + ast::FlagsItem { + span: span(4..5), + kind: ast::FlagsItemKind::Flag( + ast::Flag::SwapGreed + ), + }, + ], + }), + ast: Box::new(lit('a', 6)), + })) + ); + + assert_eq!( + parser("(").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::GroupUnclosed, + } + ); + assert_eq!( + parser("(?").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::GroupUnclosed, + } + ); + assert_eq!( + parser("(?P").parse().unwrap_err(), + TestError { + span: span(2..3), + kind: ast::ErrorKind::FlagUnrecognized, + } + ); + assert_eq!( + parser("(?P<").parse().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::GroupNameUnexpectedEof, + } + ); + assert_eq!( + parser("(a").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::GroupUnclosed, + } + ); + assert_eq!( + parser("(()").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::GroupUnclosed, + } + ); + assert_eq!( + parser(")").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::GroupUnopened, + } + ); + assert_eq!( + parser("a)").parse().unwrap_err(), + TestError { + span: span(1..2), + kind: ast::ErrorKind::GroupUnopened, + } + ); + } + + #[test] + fn parse_capture_name() { + assert_eq!( + parser("(?<a>z)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..7), + kind: ast::GroupKind::CaptureName { + starts_with_p: false, + name: ast::CaptureName { + span: span(3..4), + name: s("a"), + index: 1, + } + }, + ast: Box::new(lit('z', 5)), + })) + ); + assert_eq!( + parser("(?P<a>z)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..8), + kind: ast::GroupKind::CaptureName { + starts_with_p: true, + name: ast::CaptureName { + span: span(4..5), + name: s("a"), + index: 1, + } + }, + ast: Box::new(lit('z', 6)), + })) + ); + assert_eq!( + parser("(?P<abc>z)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..10), + kind: ast::GroupKind::CaptureName { + starts_with_p: true, + name: ast::CaptureName { + span: span(4..7), + name: s("abc"), + index: 1, + } + }, + ast: Box::new(lit('z', 8)), + })) + ); + + assert_eq!( + parser("(?P<a_1>z)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..10), + kind: ast::GroupKind::CaptureName { + starts_with_p: true, + name: ast::CaptureName { + span: span(4..7), + name: s("a_1"), + index: 1, + } + }, + ast: Box::new(lit('z', 8)), + })) + ); + + assert_eq!( + parser("(?P<a.1>z)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..10), + kind: ast::GroupKind::CaptureName { + starts_with_p: true, + name: ast::CaptureName { + span: span(4..7), + name: s("a.1"), + index: 1, + } + }, + ast: Box::new(lit('z', 8)), + })) + ); + + assert_eq!( + parser("(?P<a[1]>z)").parse(), + Ok(Ast::group(ast::Group { + span: span(0..11), + kind: ast::GroupKind::CaptureName { + starts_with_p: true, + name: ast::CaptureName { + span: span(4..8), + name: s("a[1]"), + index: 1, + } + }, + ast: Box::new(lit('z', 9)), + })) + ); + + assert_eq!( + parser("(?P<a¾>)").parse(), + Ok(Ast::group(ast::Group { + span: Span::new( + Position::new(0, 1, 1), + Position::new(9, 1, 9), + ), + kind: ast::GroupKind::CaptureName { + starts_with_p: true, + name: ast::CaptureName { + span: Span::new( + Position::new(4, 1, 5), + Position::new(7, 1, 7), + ), + name: s("a¾"), + index: 1, + } + }, + ast: Box::new(Ast::empty(Span::new( + Position::new(8, 1, 8), + Position::new(8, 1, 8), + ))), + })) + ); + assert_eq!( + parser("(?P<名字>)").parse(), + Ok(Ast::group(ast::Group { + span: Span::new( + Position::new(0, 1, 1), + Position::new(12, 1, 9), + ), + kind: ast::GroupKind::CaptureName { + starts_with_p: true, + name: ast::CaptureName { + span: Span::new( + Position::new(4, 1, 5), + Position::new(10, 1, 7), + ), + name: s("名字"), + index: 1, + } + }, + ast: Box::new(Ast::empty(Span::new( + Position::new(11, 1, 8), + Position::new(11, 1, 8), + ))), + })) + ); + + assert_eq!( + parser("(?P<").parse().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::GroupNameUnexpectedEof, + } + ); + assert_eq!( + parser("(?P<>z)").parse().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::GroupNameEmpty, + } + ); + assert_eq!( + parser("(?P<a").parse().unwrap_err(), + TestError { + span: span(5..5), + kind: ast::ErrorKind::GroupNameUnexpectedEof, + } + ); + assert_eq!( + parser("(?P<ab").parse().unwrap_err(), + TestError { + span: span(6..6), + kind: ast::ErrorKind::GroupNameUnexpectedEof, + } + ); + assert_eq!( + parser("(?P<0a").parse().unwrap_err(), + TestError { + span: span(4..5), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + assert_eq!( + parser("(?P<~").parse().unwrap_err(), + TestError { + span: span(4..5), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + assert_eq!( + parser("(?P<abc~").parse().unwrap_err(), + TestError { + span: span(7..8), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + assert_eq!( + parser("(?P<a>y)(?P<a>z)").parse().unwrap_err(), + TestError { + span: span(12..13), + kind: ast::ErrorKind::GroupNameDuplicate { + original: span(4..5), + }, + } + ); + assert_eq!( + parser("(?P<5>)").parse().unwrap_err(), + TestError { + span: span(4..5), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + assert_eq!( + parser("(?P<5a>)").parse().unwrap_err(), + TestError { + span: span(4..5), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + assert_eq!( + parser("(?P<¾>)").parse().unwrap_err(), + TestError { + span: Span::new( + Position::new(4, 1, 5), + Position::new(6, 1, 6), + ), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + assert_eq!( + parser("(?P<¾a>)").parse().unwrap_err(), + TestError { + span: Span::new( + Position::new(4, 1, 5), + Position::new(6, 1, 6), + ), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + assert_eq!( + parser("(?P<☃>)").parse().unwrap_err(), + TestError { + span: Span::new( + Position::new(4, 1, 5), + Position::new(7, 1, 6), + ), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + assert_eq!( + parser("(?P<a☃>)").parse().unwrap_err(), + TestError { + span: Span::new( + Position::new(5, 1, 6), + Position::new(8, 1, 7), + ), + kind: ast::ErrorKind::GroupNameInvalid, + } + ); + } + + #[test] + fn parse_flags() { + assert_eq!( + parser("i:").parse_flags(), + Ok(ast::Flags { + span: span(0..1), + items: vec![ast::FlagsItem { + span: span(0..1), + kind: ast::FlagsItemKind::Flag(ast::Flag::CaseInsensitive), + }], + }) + ); + assert_eq!( + parser("i)").parse_flags(), + Ok(ast::Flags { + span: span(0..1), + items: vec![ast::FlagsItem { + span: span(0..1), + kind: ast::FlagsItemKind::Flag(ast::Flag::CaseInsensitive), + }], + }) + ); + + assert_eq!( + parser("isU:").parse_flags(), + Ok(ast::Flags { + span: span(0..3), + items: vec![ + ast::FlagsItem { + span: span(0..1), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + }, + ast::FlagsItem { + span: span(1..2), + kind: ast::FlagsItemKind::Flag( + ast::Flag::DotMatchesNewLine + ), + }, + ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag(ast::Flag::SwapGreed), + }, + ], + }) + ); + + assert_eq!( + parser("-isU:").parse_flags(), + Ok(ast::Flags { + span: span(0..4), + items: vec![ + ast::FlagsItem { + span: span(0..1), + kind: ast::FlagsItemKind::Negation, + }, + ast::FlagsItem { + span: span(1..2), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + }, + ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag( + ast::Flag::DotMatchesNewLine + ), + }, + ast::FlagsItem { + span: span(3..4), + kind: ast::FlagsItemKind::Flag(ast::Flag::SwapGreed), + }, + ], + }) + ); + assert_eq!( + parser("i-sU:").parse_flags(), + Ok(ast::Flags { + span: span(0..4), + items: vec![ + ast::FlagsItem { + span: span(0..1), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + }, + ast::FlagsItem { + span: span(1..2), + kind: ast::FlagsItemKind::Negation, + }, + ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag( + ast::Flag::DotMatchesNewLine + ), + }, + ast::FlagsItem { + span: span(3..4), + kind: ast::FlagsItemKind::Flag(ast::Flag::SwapGreed), + }, + ], + }) + ); + assert_eq!( + parser("i-sR:").parse_flags(), + Ok(ast::Flags { + span: span(0..4), + items: vec![ + ast::FlagsItem { + span: span(0..1), + kind: ast::FlagsItemKind::Flag( + ast::Flag::CaseInsensitive + ), + }, + ast::FlagsItem { + span: span(1..2), + kind: ast::FlagsItemKind::Negation, + }, + ast::FlagsItem { + span: span(2..3), + kind: ast::FlagsItemKind::Flag( + ast::Flag::DotMatchesNewLine + ), + }, + ast::FlagsItem { + span: span(3..4), + kind: ast::FlagsItemKind::Flag(ast::Flag::CRLF), + }, + ], + }) + ); + + assert_eq!( + parser("isU").parse_flags().unwrap_err(), + TestError { + span: span(3..3), + kind: ast::ErrorKind::FlagUnexpectedEof, + } + ); + assert_eq!( + parser("isUa:").parse_flags().unwrap_err(), + TestError { + span: span(3..4), + kind: ast::ErrorKind::FlagUnrecognized, + } + ); + assert_eq!( + parser("isUi:").parse_flags().unwrap_err(), + TestError { + span: span(3..4), + kind: ast::ErrorKind::FlagDuplicate { original: span(0..1) }, + } + ); + assert_eq!( + parser("i-sU-i:").parse_flags().unwrap_err(), + TestError { + span: span(4..5), + kind: ast::ErrorKind::FlagRepeatedNegation { + original: span(1..2), + }, + } + ); + assert_eq!( + parser("-)").parse_flags().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::FlagDanglingNegation, + } + ); + assert_eq!( + parser("i-)").parse_flags().unwrap_err(), + TestError { + span: span(1..2), + kind: ast::ErrorKind::FlagDanglingNegation, + } + ); + assert_eq!( + parser("iU-)").parse_flags().unwrap_err(), + TestError { + span: span(2..3), + kind: ast::ErrorKind::FlagDanglingNegation, + } + ); + } + + #[test] + fn parse_flag() { + assert_eq!(parser("i").parse_flag(), Ok(ast::Flag::CaseInsensitive)); + assert_eq!(parser("m").parse_flag(), Ok(ast::Flag::MultiLine)); + assert_eq!(parser("s").parse_flag(), Ok(ast::Flag::DotMatchesNewLine)); + assert_eq!(parser("U").parse_flag(), Ok(ast::Flag::SwapGreed)); + assert_eq!(parser("u").parse_flag(), Ok(ast::Flag::Unicode)); + assert_eq!(parser("R").parse_flag(), Ok(ast::Flag::CRLF)); + assert_eq!(parser("x").parse_flag(), Ok(ast::Flag::IgnoreWhitespace)); + + assert_eq!( + parser("a").parse_flag().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::FlagUnrecognized, + } + ); + assert_eq!( + parser("☃").parse_flag().unwrap_err(), + TestError { + span: span_range("☃", 0..3), + kind: ast::ErrorKind::FlagUnrecognized, + } + ); + } + + #[test] + fn parse_primitive_non_escape() { + assert_eq!( + parser(r".").parse_primitive(), + Ok(Primitive::Dot(span(0..1))) + ); + assert_eq!( + parser(r"^").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..1), + kind: ast::AssertionKind::StartLine, + })) + ); + assert_eq!( + parser(r"$").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..1), + kind: ast::AssertionKind::EndLine, + })) + ); + + assert_eq!( + parser(r"a").parse_primitive(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..1), + kind: ast::LiteralKind::Verbatim, + c: 'a', + })) + ); + assert_eq!( + parser(r"|").parse_primitive(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..1), + kind: ast::LiteralKind::Verbatim, + c: '|', + })) + ); + assert_eq!( + parser(r"☃").parse_primitive(), + Ok(Primitive::Literal(ast::Literal { + span: span_range("☃", 0..3), + kind: ast::LiteralKind::Verbatim, + c: '☃', + })) + ); + } + + #[test] + fn parse_escape() { + assert_eq!( + parser(r"\|").parse_primitive(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..2), + kind: ast::LiteralKind::Meta, + c: '|', + })) + ); + let specials = &[ + (r"\a", '\x07', ast::SpecialLiteralKind::Bell), + (r"\f", '\x0C', ast::SpecialLiteralKind::FormFeed), + (r"\t", '\t', ast::SpecialLiteralKind::Tab), + (r"\n", '\n', ast::SpecialLiteralKind::LineFeed), + (r"\r", '\r', ast::SpecialLiteralKind::CarriageReturn), + (r"\v", '\x0B', ast::SpecialLiteralKind::VerticalTab), + ]; + for &(pat, c, ref kind) in specials { + assert_eq!( + parser(pat).parse_primitive(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..2), + kind: ast::LiteralKind::Special(kind.clone()), + c, + })) + ); + } + assert_eq!( + parser(r"\A").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..2), + kind: ast::AssertionKind::StartText, + })) + ); + assert_eq!( + parser(r"\z").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..2), + kind: ast::AssertionKind::EndText, + })) + ); + assert_eq!( + parser(r"\b").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..2), + kind: ast::AssertionKind::WordBoundary, + })) + ); + assert_eq!( + parser(r"\b{start}").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..9), + kind: ast::AssertionKind::WordBoundaryStart, + })) + ); + assert_eq!( + parser(r"\b{end}").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..7), + kind: ast::AssertionKind::WordBoundaryEnd, + })) + ); + assert_eq!( + parser(r"\b{start-half}").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..14), + kind: ast::AssertionKind::WordBoundaryStartHalf, + })) + ); + assert_eq!( + parser(r"\b{end-half}").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..12), + kind: ast::AssertionKind::WordBoundaryEndHalf, + })) + ); + assert_eq!( + parser(r"\<").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..2), + kind: ast::AssertionKind::WordBoundaryStartAngle, + })) + ); + assert_eq!( + parser(r"\>").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..2), + kind: ast::AssertionKind::WordBoundaryEndAngle, + })) + ); + assert_eq!( + parser(r"\B").parse_primitive(), + Ok(Primitive::Assertion(ast::Assertion { + span: span(0..2), + kind: ast::AssertionKind::NotWordBoundary, + })) + ); + + // We also support superfluous escapes in most cases now too. + for c in ['!', '@', '%', '"', '\'', '/', ' '] { + let pat = format!(r"\{c}"); + assert_eq!( + parser(&pat).parse_primitive(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..2), + kind: ast::LiteralKind::Superfluous, + c, + })) + ); + } + + // Some superfluous escapes, namely [0-9A-Za-z], are still banned. This + // gives flexibility for future evolution. + assert_eq!( + parser(r"\e").parse_escape().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::EscapeUnrecognized, + } + ); + assert_eq!( + parser(r"\y").parse_escape().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::EscapeUnrecognized, + } + ); + + // Starting a special word boundary without any non-whitespace chars + // after the brace makes it ambiguous whether the user meant to write + // a counted repetition (probably not?) or an actual special word + // boundary assertion. + assert_eq!( + parser(r"\b{").parse_escape().unwrap_err(), + TestError { + span: span(0..3), + kind: ast::ErrorKind::SpecialWordOrRepetitionUnexpectedEof, + } + ); + assert_eq!( + parser_ignore_whitespace(r"\b{ ").parse_escape().unwrap_err(), + TestError { + span: span(0..4), + kind: ast::ErrorKind::SpecialWordOrRepetitionUnexpectedEof, + } + ); + // When 'x' is not enabled, the space is seen as a non-[-A-Za-z] char, + // and thus causes the parser to treat it as a counted repetition. + assert_eq!( + parser(r"\b{ ").parse().unwrap_err(), + TestError { + span: span(2..4), + kind: ast::ErrorKind::RepetitionCountUnclosed, + } + ); + // In this case, we got some valid chars that makes it look like the + // user is writing one of the special word boundary assertions, but + // we forget to close the brace. + assert_eq!( + parser(r"\b{foo").parse_escape().unwrap_err(), + TestError { + span: span(2..6), + kind: ast::ErrorKind::SpecialWordBoundaryUnclosed, + } + ); + // We get the same error as above, except it is provoked by seeing a + // char that we know is invalid before seeing a closing brace. + assert_eq!( + parser(r"\b{foo!}").parse_escape().unwrap_err(), + TestError { + span: span(2..6), + kind: ast::ErrorKind::SpecialWordBoundaryUnclosed, + } + ); + // And this one occurs when, syntactically, everything looks okay, but + // we don't use a valid spelling of a word boundary assertion. + assert_eq!( + parser(r"\b{foo}").parse_escape().unwrap_err(), + TestError { + span: span(3..6), + kind: ast::ErrorKind::SpecialWordBoundaryUnrecognized, + } + ); + + // An unfinished escape is illegal. + assert_eq!( + parser(r"\").parse_escape().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + } + + #[test] + fn parse_unsupported_backreference() { + assert_eq!( + parser(r"\0").parse_escape().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::UnsupportedBackreference, + } + ); + assert_eq!( + parser(r"\9").parse_escape().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::UnsupportedBackreference, + } + ); + } + + #[test] + fn parse_octal() { + for i in 0..511 { + let pat = format!(r"\{i:o}"); + assert_eq!( + parser_octal(&pat).parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..pat.len()), + kind: ast::LiteralKind::Octal, + c: char::from_u32(i).unwrap(), + })) + ); + } + assert_eq!( + parser_octal(r"\778").parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..3), + kind: ast::LiteralKind::Octal, + c: '?', + })) + ); + assert_eq!( + parser_octal(r"\7777").parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..4), + kind: ast::LiteralKind::Octal, + c: '\u{01FF}', + })) + ); + assert_eq!( + parser_octal(r"\778").parse(), + Ok(Ast::concat(ast::Concat { + span: span(0..4), + asts: vec![ + Ast::literal(ast::Literal { + span: span(0..3), + kind: ast::LiteralKind::Octal, + c: '?', + }), + Ast::literal(ast::Literal { + span: span(3..4), + kind: ast::LiteralKind::Verbatim, + c: '8', + }), + ], + })) + ); + assert_eq!( + parser_octal(r"\7777").parse(), + Ok(Ast::concat(ast::Concat { + span: span(0..5), + asts: vec![ + Ast::literal(ast::Literal { + span: span(0..4), + kind: ast::LiteralKind::Octal, + c: '\u{01FF}', + }), + Ast::literal(ast::Literal { + span: span(4..5), + kind: ast::LiteralKind::Verbatim, + c: '7', + }), + ], + })) + ); + + assert_eq!( + parser_octal(r"\8").parse_escape().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::EscapeUnrecognized, + } + ); + } + + #[test] + fn parse_hex_two() { + for i in 0..256 { + let pat = format!(r"\x{i:02x}"); + assert_eq!( + parser(&pat).parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..pat.len()), + kind: ast::LiteralKind::HexFixed(ast::HexLiteralKind::X), + c: char::from_u32(i).unwrap(), + })) + ); + } + + assert_eq!( + parser(r"\xF").parse_escape().unwrap_err(), + TestError { + span: span(3..3), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\xG").parse_escape().unwrap_err(), + TestError { + span: span(2..3), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\xFG").parse_escape().unwrap_err(), + TestError { + span: span(3..4), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + } + + #[test] + fn parse_hex_four() { + for i in 0..65536 { + let c = match char::from_u32(i) { + None => continue, + Some(c) => c, + }; + let pat = format!(r"\u{i:04x}"); + assert_eq!( + parser(&pat).parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..pat.len()), + kind: ast::LiteralKind::HexFixed( + ast::HexLiteralKind::UnicodeShort + ), + c, + })) + ); + } + + assert_eq!( + parser(r"\uF").parse_escape().unwrap_err(), + TestError { + span: span(3..3), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\uG").parse_escape().unwrap_err(), + TestError { + span: span(2..3), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\uFG").parse_escape().unwrap_err(), + TestError { + span: span(3..4), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\uFFG").parse_escape().unwrap_err(), + TestError { + span: span(4..5), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\uFFFG").parse_escape().unwrap_err(), + TestError { + span: span(5..6), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\uD800").parse_escape().unwrap_err(), + TestError { + span: span(2..6), + kind: ast::ErrorKind::EscapeHexInvalid, + } + ); + } + + #[test] + fn parse_hex_eight() { + for i in 0..65536 { + let c = match char::from_u32(i) { + None => continue, + Some(c) => c, + }; + let pat = format!(r"\U{i:08x}"); + assert_eq!( + parser(&pat).parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..pat.len()), + kind: ast::LiteralKind::HexFixed( + ast::HexLiteralKind::UnicodeLong + ), + c, + })) + ); + } + + assert_eq!( + parser(r"\UF").parse_escape().unwrap_err(), + TestError { + span: span(3..3), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\UG").parse_escape().unwrap_err(), + TestError { + span: span(2..3), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\UFG").parse_escape().unwrap_err(), + TestError { + span: span(3..4), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\UFFG").parse_escape().unwrap_err(), + TestError { + span: span(4..5), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\UFFFG").parse_escape().unwrap_err(), + TestError { + span: span(5..6), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\UFFFFG").parse_escape().unwrap_err(), + TestError { + span: span(6..7), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\UFFFFFG").parse_escape().unwrap_err(), + TestError { + span: span(7..8), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\UFFFFFFG").parse_escape().unwrap_err(), + TestError { + span: span(8..9), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\UFFFFFFFG").parse_escape().unwrap_err(), + TestError { + span: span(9..10), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + } + + #[test] + fn parse_hex_brace() { + assert_eq!( + parser(r"\u{26c4}").parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..8), + kind: ast::LiteralKind::HexBrace( + ast::HexLiteralKind::UnicodeShort + ), + c: '⛄', + })) + ); + assert_eq!( + parser(r"\U{26c4}").parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..8), + kind: ast::LiteralKind::HexBrace( + ast::HexLiteralKind::UnicodeLong + ), + c: '⛄', + })) + ); + assert_eq!( + parser(r"\x{26c4}").parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..8), + kind: ast::LiteralKind::HexBrace(ast::HexLiteralKind::X), + c: '⛄', + })) + ); + assert_eq!( + parser(r"\x{26C4}").parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..8), + kind: ast::LiteralKind::HexBrace(ast::HexLiteralKind::X), + c: '⛄', + })) + ); + assert_eq!( + parser(r"\x{10fFfF}").parse_escape(), + Ok(Primitive::Literal(ast::Literal { + span: span(0..10), + kind: ast::LiteralKind::HexBrace(ast::HexLiteralKind::X), + c: '\u{10FFFF}', + })) + ); + + assert_eq!( + parser(r"\x").parse_escape().unwrap_err(), + TestError { + span: span(2..2), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\x{").parse_escape().unwrap_err(), + TestError { + span: span(2..3), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\x{FF").parse_escape().unwrap_err(), + TestError { + span: span(2..5), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\x{}").parse_escape().unwrap_err(), + TestError { + span: span(2..4), + kind: ast::ErrorKind::EscapeHexEmpty, + } + ); + assert_eq!( + parser(r"\x{FGF}").parse_escape().unwrap_err(), + TestError { + span: span(4..5), + kind: ast::ErrorKind::EscapeHexInvalidDigit, + } + ); + assert_eq!( + parser(r"\x{FFFFFF}").parse_escape().unwrap_err(), + TestError { + span: span(3..9), + kind: ast::ErrorKind::EscapeHexInvalid, + } + ); + assert_eq!( + parser(r"\x{D800}").parse_escape().unwrap_err(), + TestError { + span: span(3..7), + kind: ast::ErrorKind::EscapeHexInvalid, + } + ); + assert_eq!( + parser(r"\x{FFFFFFFFF}").parse_escape().unwrap_err(), + TestError { + span: span(3..12), + kind: ast::ErrorKind::EscapeHexInvalid, + } + ); + } + + #[test] + fn parse_decimal() { + assert_eq!(parser("123").parse_decimal(), Ok(123)); + assert_eq!(parser("0").parse_decimal(), Ok(0)); + assert_eq!(parser("01").parse_decimal(), Ok(1)); + + assert_eq!( + parser("-1").parse_decimal().unwrap_err(), + TestError { span: span(0..0), kind: ast::ErrorKind::DecimalEmpty } + ); + assert_eq!( + parser("").parse_decimal().unwrap_err(), + TestError { span: span(0..0), kind: ast::ErrorKind::DecimalEmpty } + ); + assert_eq!( + parser("9999999999").parse_decimal().unwrap_err(), + TestError { + span: span(0..10), + kind: ast::ErrorKind::DecimalInvalid, + } + ); + } + + #[test] + fn parse_set_class() { + fn union(span: Span, items: Vec<ast::ClassSetItem>) -> ast::ClassSet { + ast::ClassSet::union(ast::ClassSetUnion { span, items }) + } + + fn intersection( + span: Span, + lhs: ast::ClassSet, + rhs: ast::ClassSet, + ) -> ast::ClassSet { + ast::ClassSet::BinaryOp(ast::ClassSetBinaryOp { + span, + kind: ast::ClassSetBinaryOpKind::Intersection, + lhs: Box::new(lhs), + rhs: Box::new(rhs), + }) + } + + fn difference( + span: Span, + lhs: ast::ClassSet, + rhs: ast::ClassSet, + ) -> ast::ClassSet { + ast::ClassSet::BinaryOp(ast::ClassSetBinaryOp { + span, + kind: ast::ClassSetBinaryOpKind::Difference, + lhs: Box::new(lhs), + rhs: Box::new(rhs), + }) + } + + fn symdifference( + span: Span, + lhs: ast::ClassSet, + rhs: ast::ClassSet, + ) -> ast::ClassSet { + ast::ClassSet::BinaryOp(ast::ClassSetBinaryOp { + span, + kind: ast::ClassSetBinaryOpKind::SymmetricDifference, + lhs: Box::new(lhs), + rhs: Box::new(rhs), + }) + } + + fn itemset(item: ast::ClassSetItem) -> ast::ClassSet { + ast::ClassSet::Item(item) + } + + fn item_ascii(cls: ast::ClassAscii) -> ast::ClassSetItem { + ast::ClassSetItem::Ascii(cls) + } + + fn item_unicode(cls: ast::ClassUnicode) -> ast::ClassSetItem { + ast::ClassSetItem::Unicode(cls) + } + + fn item_perl(cls: ast::ClassPerl) -> ast::ClassSetItem { + ast::ClassSetItem::Perl(cls) + } + + fn item_bracket(cls: ast::ClassBracketed) -> ast::ClassSetItem { + ast::ClassSetItem::Bracketed(Box::new(cls)) + } + + fn lit(span: Span, c: char) -> ast::ClassSetItem { + ast::ClassSetItem::Literal(ast::Literal { + span, + kind: ast::LiteralKind::Verbatim, + c, + }) + } + + fn empty(span: Span) -> ast::ClassSetItem { + ast::ClassSetItem::Empty(span) + } + + fn range(span: Span, start: char, end: char) -> ast::ClassSetItem { + let pos1 = Position { + offset: span.start.offset + start.len_utf8(), + column: span.start.column + 1, + ..span.start + }; + let pos2 = Position { + offset: span.end.offset - end.len_utf8(), + column: span.end.column - 1, + ..span.end + }; + ast::ClassSetItem::Range(ast::ClassSetRange { + span, + start: ast::Literal { + span: Span { end: pos1, ..span }, + kind: ast::LiteralKind::Verbatim, + c: start, + }, + end: ast::Literal { + span: Span { start: pos2, ..span }, + kind: ast::LiteralKind::Verbatim, + c: end, + }, + }) + } + + fn alnum(span: Span, negated: bool) -> ast::ClassAscii { + ast::ClassAscii { span, kind: ast::ClassAsciiKind::Alnum, negated } + } + + fn lower(span: Span, negated: bool) -> ast::ClassAscii { + ast::ClassAscii { span, kind: ast::ClassAsciiKind::Lower, negated } + } + + assert_eq!( + parser("[[:alnum:]]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..11), + negated: false, + kind: itemset(item_ascii(alnum(span(1..10), false))), + })) + ); + assert_eq!( + parser("[[[:alnum:]]]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..13), + negated: false, + kind: itemset(item_bracket(ast::ClassBracketed { + span: span(1..12), + negated: false, + kind: itemset(item_ascii(alnum(span(2..11), false))), + })), + })) + ); + assert_eq!( + parser("[[:alnum:]&&[:lower:]]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..22), + negated: false, + kind: intersection( + span(1..21), + itemset(item_ascii(alnum(span(1..10), false))), + itemset(item_ascii(lower(span(12..21), false))), + ), + })) + ); + assert_eq!( + parser("[[:alnum:]--[:lower:]]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..22), + negated: false, + kind: difference( + span(1..21), + itemset(item_ascii(alnum(span(1..10), false))), + itemset(item_ascii(lower(span(12..21), false))), + ), + })) + ); + assert_eq!( + parser("[[:alnum:]~~[:lower:]]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..22), + negated: false, + kind: symdifference( + span(1..21), + itemset(item_ascii(alnum(span(1..10), false))), + itemset(item_ascii(lower(span(12..21), false))), + ), + })) + ); + + assert_eq!( + parser("[a]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..3), + negated: false, + kind: itemset(lit(span(1..2), 'a')), + })) + ); + assert_eq!( + parser(r"[a\]]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..5), + negated: false, + kind: union( + span(1..4), + vec![ + lit(span(1..2), 'a'), + ast::ClassSetItem::Literal(ast::Literal { + span: span(2..4), + kind: ast::LiteralKind::Meta, + c: ']', + }), + ] + ), + })) + ); + assert_eq!( + parser(r"[a\-z]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..6), + negated: false, + kind: union( + span(1..5), + vec![ + lit(span(1..2), 'a'), + ast::ClassSetItem::Literal(ast::Literal { + span: span(2..4), + kind: ast::LiteralKind::Meta, + c: '-', + }), + lit(span(4..5), 'z'), + ] + ), + })) + ); + assert_eq!( + parser("[ab]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..4), + negated: false, + kind: union( + span(1..3), + vec![lit(span(1..2), 'a'), lit(span(2..3), 'b'),] + ), + })) + ); + assert_eq!( + parser("[a-]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..4), + negated: false, + kind: union( + span(1..3), + vec![lit(span(1..2), 'a'), lit(span(2..3), '-'),] + ), + })) + ); + assert_eq!( + parser("[-a]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..4), + negated: false, + kind: union( + span(1..3), + vec![lit(span(1..2), '-'), lit(span(2..3), 'a'),] + ), + })) + ); + assert_eq!( + parser(r"[\pL]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..5), + negated: false, + kind: itemset(item_unicode(ast::ClassUnicode { + span: span(1..4), + negated: false, + kind: ast::ClassUnicodeKind::OneLetter('L'), + })), + })) + ); + assert_eq!( + parser(r"[\w]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..4), + negated: false, + kind: itemset(item_perl(ast::ClassPerl { + span: span(1..3), + kind: ast::ClassPerlKind::Word, + negated: false, + })), + })) + ); + assert_eq!( + parser(r"[a\wz]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..6), + negated: false, + kind: union( + span(1..5), + vec![ + lit(span(1..2), 'a'), + item_perl(ast::ClassPerl { + span: span(2..4), + kind: ast::ClassPerlKind::Word, + negated: false, + }), + lit(span(4..5), 'z'), + ] + ), + })) + ); + + assert_eq!( + parser("[a-z]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..5), + negated: false, + kind: itemset(range(span(1..4), 'a', 'z')), + })) + ); + assert_eq!( + parser("[a-cx-z]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..8), + negated: false, + kind: union( + span(1..7), + vec![ + range(span(1..4), 'a', 'c'), + range(span(4..7), 'x', 'z'), + ] + ), + })) + ); + assert_eq!( + parser(r"[\w&&a-cx-z]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..12), + negated: false, + kind: intersection( + span(1..11), + itemset(item_perl(ast::ClassPerl { + span: span(1..3), + kind: ast::ClassPerlKind::Word, + negated: false, + })), + union( + span(5..11), + vec![ + range(span(5..8), 'a', 'c'), + range(span(8..11), 'x', 'z'), + ] + ), + ), + })) + ); + assert_eq!( + parser(r"[a-cx-z&&\w]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..12), + negated: false, + kind: intersection( + span(1..11), + union( + span(1..7), + vec![ + range(span(1..4), 'a', 'c'), + range(span(4..7), 'x', 'z'), + ] + ), + itemset(item_perl(ast::ClassPerl { + span: span(9..11), + kind: ast::ClassPerlKind::Word, + negated: false, + })), + ), + })) + ); + assert_eq!( + parser(r"[a--b--c]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..9), + negated: false, + kind: difference( + span(1..8), + difference( + span(1..5), + itemset(lit(span(1..2), 'a')), + itemset(lit(span(4..5), 'b')), + ), + itemset(lit(span(7..8), 'c')), + ), + })) + ); + assert_eq!( + parser(r"[a~~b~~c]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..9), + negated: false, + kind: symdifference( + span(1..8), + symdifference( + span(1..5), + itemset(lit(span(1..2), 'a')), + itemset(lit(span(4..5), 'b')), + ), + itemset(lit(span(7..8), 'c')), + ), + })) + ); + assert_eq!( + parser(r"[\^&&^]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..7), + negated: false, + kind: intersection( + span(1..6), + itemset(ast::ClassSetItem::Literal(ast::Literal { + span: span(1..3), + kind: ast::LiteralKind::Meta, + c: '^', + })), + itemset(lit(span(5..6), '^')), + ), + })) + ); + assert_eq!( + parser(r"[\&&&&]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..7), + negated: false, + kind: intersection( + span(1..6), + itemset(ast::ClassSetItem::Literal(ast::Literal { + span: span(1..3), + kind: ast::LiteralKind::Meta, + c: '&', + })), + itemset(lit(span(5..6), '&')), + ), + })) + ); + assert_eq!( + parser(r"[&&&&]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..6), + negated: false, + kind: intersection( + span(1..5), + intersection( + span(1..3), + itemset(empty(span(1..1))), + itemset(empty(span(3..3))), + ), + itemset(empty(span(5..5))), + ), + })) + ); + + let pat = "[☃-⛄]"; + assert_eq!( + parser(pat).parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span_range(pat, 0..9), + negated: false, + kind: itemset(ast::ClassSetItem::Range(ast::ClassSetRange { + span: span_range(pat, 1..8), + start: ast::Literal { + span: span_range(pat, 1..4), + kind: ast::LiteralKind::Verbatim, + c: '☃', + }, + end: ast::Literal { + span: span_range(pat, 5..8), + kind: ast::LiteralKind::Verbatim, + c: '⛄', + }, + })), + })) + ); + + assert_eq!( + parser(r"[]]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..3), + negated: false, + kind: itemset(lit(span(1..2), ']')), + })) + ); + assert_eq!( + parser(r"[]\[]").parse(), + Ok(Ast::class_bracketed(ast::ClassBracketed { + span: span(0..5), + negated: false, + kind: union( + span(1..4), + vec![ + lit(span(1..2), ']'), + ast::ClassSetItem::Literal(ast::Literal { + span: span(2..4), + kind: ast::LiteralKind::Meta, + c: '[', + }), + ] + ), + })) + ); + assert_eq!( + parser(r"[\[]]").parse(), + Ok(concat( + 0..5, + vec![ + Ast::class_bracketed(ast::ClassBracketed { + span: span(0..4), + negated: false, + kind: itemset(ast::ClassSetItem::Literal( + ast::Literal { + span: span(1..3), + kind: ast::LiteralKind::Meta, + c: '[', + } + )), + }), + Ast::literal(ast::Literal { + span: span(4..5), + kind: ast::LiteralKind::Verbatim, + c: ']', + }), + ] + )) + ); + + assert_eq!( + parser("[").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser("[[").parse().unwrap_err(), + TestError { + span: span(1..2), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser("[[-]").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser("[[[:alnum:]").parse().unwrap_err(), + TestError { + span: span(1..2), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser(r"[\b]").parse().unwrap_err(), + TestError { + span: span(1..3), + kind: ast::ErrorKind::ClassEscapeInvalid, + } + ); + assert_eq!( + parser(r"[\w-a]").parse().unwrap_err(), + TestError { + span: span(1..3), + kind: ast::ErrorKind::ClassRangeLiteral, + } + ); + assert_eq!( + parser(r"[a-\w]").parse().unwrap_err(), + TestError { + span: span(3..5), + kind: ast::ErrorKind::ClassRangeLiteral, + } + ); + assert_eq!( + parser(r"[z-a]").parse().unwrap_err(), + TestError { + span: span(1..4), + kind: ast::ErrorKind::ClassRangeInvalid, + } + ); + + assert_eq!( + parser_ignore_whitespace("[a ").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser_ignore_whitespace("[a- ").parse().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + } + + #[test] + fn parse_set_class_open() { + assert_eq!(parser("[a]").parse_set_class_open(), { + let set = ast::ClassBracketed { + span: span(0..1), + negated: false, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(1..1), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { span: span(1..1), items: vec![] }; + Ok((set, union)) + }); + assert_eq!( + parser_ignore_whitespace("[ a]").parse_set_class_open(), + { + let set = ast::ClassBracketed { + span: span(0..4), + negated: false, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(4..4), + items: vec![], + }), + }; + let union = + ast::ClassSetUnion { span: span(4..4), items: vec![] }; + Ok((set, union)) + } + ); + assert_eq!(parser("[^a]").parse_set_class_open(), { + let set = ast::ClassBracketed { + span: span(0..2), + negated: true, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(2..2), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { span: span(2..2), items: vec![] }; + Ok((set, union)) + }); + assert_eq!( + parser_ignore_whitespace("[ ^ a]").parse_set_class_open(), + { + let set = ast::ClassBracketed { + span: span(0..4), + negated: true, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(4..4), + items: vec![], + }), + }; + let union = + ast::ClassSetUnion { span: span(4..4), items: vec![] }; + Ok((set, union)) + } + ); + assert_eq!(parser("[-a]").parse_set_class_open(), { + let set = ast::ClassBracketed { + span: span(0..2), + negated: false, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(1..1), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { + span: span(1..2), + items: vec![ast::ClassSetItem::Literal(ast::Literal { + span: span(1..2), + kind: ast::LiteralKind::Verbatim, + c: '-', + })], + }; + Ok((set, union)) + }); + assert_eq!( + parser_ignore_whitespace("[ - a]").parse_set_class_open(), + { + let set = ast::ClassBracketed { + span: span(0..4), + negated: false, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(2..2), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { + span: span(2..3), + items: vec![ast::ClassSetItem::Literal(ast::Literal { + span: span(2..3), + kind: ast::LiteralKind::Verbatim, + c: '-', + })], + }; + Ok((set, union)) + } + ); + assert_eq!(parser("[^-a]").parse_set_class_open(), { + let set = ast::ClassBracketed { + span: span(0..3), + negated: true, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(2..2), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { + span: span(2..3), + items: vec![ast::ClassSetItem::Literal(ast::Literal { + span: span(2..3), + kind: ast::LiteralKind::Verbatim, + c: '-', + })], + }; + Ok((set, union)) + }); + assert_eq!(parser("[--a]").parse_set_class_open(), { + let set = ast::ClassBracketed { + span: span(0..3), + negated: false, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(1..1), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { + span: span(1..3), + items: vec![ + ast::ClassSetItem::Literal(ast::Literal { + span: span(1..2), + kind: ast::LiteralKind::Verbatim, + c: '-', + }), + ast::ClassSetItem::Literal(ast::Literal { + span: span(2..3), + kind: ast::LiteralKind::Verbatim, + c: '-', + }), + ], + }; + Ok((set, union)) + }); + assert_eq!(parser("[]a]").parse_set_class_open(), { + let set = ast::ClassBracketed { + span: span(0..2), + negated: false, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(1..1), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { + span: span(1..2), + items: vec![ast::ClassSetItem::Literal(ast::Literal { + span: span(1..2), + kind: ast::LiteralKind::Verbatim, + c: ']', + })], + }; + Ok((set, union)) + }); + assert_eq!( + parser_ignore_whitespace("[ ] a]").parse_set_class_open(), + { + let set = ast::ClassBracketed { + span: span(0..4), + negated: false, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(2..2), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { + span: span(2..3), + items: vec![ast::ClassSetItem::Literal(ast::Literal { + span: span(2..3), + kind: ast::LiteralKind::Verbatim, + c: ']', + })], + }; + Ok((set, union)) + } + ); + assert_eq!(parser("[^]a]").parse_set_class_open(), { + let set = ast::ClassBracketed { + span: span(0..3), + negated: true, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(2..2), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { + span: span(2..3), + items: vec![ast::ClassSetItem::Literal(ast::Literal { + span: span(2..3), + kind: ast::LiteralKind::Verbatim, + c: ']', + })], + }; + Ok((set, union)) + }); + assert_eq!(parser("[-]a]").parse_set_class_open(), { + let set = ast::ClassBracketed { + span: span(0..2), + negated: false, + kind: ast::ClassSet::union(ast::ClassSetUnion { + span: span(1..1), + items: vec![], + }), + }; + let union = ast::ClassSetUnion { + span: span(1..2), + items: vec![ast::ClassSetItem::Literal(ast::Literal { + span: span(1..2), + kind: ast::LiteralKind::Verbatim, + c: '-', + })], + }; + Ok((set, union)) + }); + + assert_eq!( + parser("[").parse_set_class_open().unwrap_err(), + TestError { + span: span(0..1), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser_ignore_whitespace("[ ") + .parse_set_class_open() + .unwrap_err(), + TestError { + span: span(0..5), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser("[^").parse_set_class_open().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser("[]").parse_set_class_open().unwrap_err(), + TestError { + span: span(0..2), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser("[-").parse_set_class_open().unwrap_err(), + TestError { + span: span(0..0), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + assert_eq!( + parser("[--").parse_set_class_open().unwrap_err(), + TestError { + span: span(0..0), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + + // See: https://github.com/rust-lang/regex/issues/792 + assert_eq!( + parser("(?x)[-#]").parse_with_comments().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::ClassUnclosed, + } + ); + } + + #[test] + fn maybe_parse_ascii_class() { + assert_eq!( + parser(r"[:alnum:]").maybe_parse_ascii_class(), + Some(ast::ClassAscii { + span: span(0..9), + kind: ast::ClassAsciiKind::Alnum, + negated: false, + }) + ); + assert_eq!( + parser(r"[:alnum:]A").maybe_parse_ascii_class(), + Some(ast::ClassAscii { + span: span(0..9), + kind: ast::ClassAsciiKind::Alnum, + negated: false, + }) + ); + assert_eq!( + parser(r"[:^alnum:]").maybe_parse_ascii_class(), + Some(ast::ClassAscii { + span: span(0..10), + kind: ast::ClassAsciiKind::Alnum, + negated: true, + }) + ); + + let p = parser(r"[:"); + assert_eq!(p.maybe_parse_ascii_class(), None); + assert_eq!(p.offset(), 0); + + let p = parser(r"[:^"); + assert_eq!(p.maybe_parse_ascii_class(), None); + assert_eq!(p.offset(), 0); + + let p = parser(r"[^:alnum:]"); + assert_eq!(p.maybe_parse_ascii_class(), None); + assert_eq!(p.offset(), 0); + + let p = parser(r"[:alnnum:]"); + assert_eq!(p.maybe_parse_ascii_class(), None); + assert_eq!(p.offset(), 0); + + let p = parser(r"[:alnum]"); + assert_eq!(p.maybe_parse_ascii_class(), None); + assert_eq!(p.offset(), 0); + + let p = parser(r"[:alnum:"); + assert_eq!(p.maybe_parse_ascii_class(), None); + assert_eq!(p.offset(), 0); + } + + #[test] + fn parse_unicode_class() { + assert_eq!( + parser(r"\pN").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..3), + negated: false, + kind: ast::ClassUnicodeKind::OneLetter('N'), + })) + ); + assert_eq!( + parser(r"\PN").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..3), + negated: true, + kind: ast::ClassUnicodeKind::OneLetter('N'), + })) + ); + assert_eq!( + parser(r"\p{N}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..5), + negated: false, + kind: ast::ClassUnicodeKind::Named(s("N")), + })) + ); + assert_eq!( + parser(r"\P{N}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..5), + negated: true, + kind: ast::ClassUnicodeKind::Named(s("N")), + })) + ); + assert_eq!( + parser(r"\p{Greek}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..9), + negated: false, + kind: ast::ClassUnicodeKind::Named(s("Greek")), + })) + ); + + assert_eq!( + parser(r"\p{scx:Katakana}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..16), + negated: false, + kind: ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::Colon, + name: s("scx"), + value: s("Katakana"), + }, + })) + ); + assert_eq!( + parser(r"\p{scx=Katakana}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..16), + negated: false, + kind: ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::Equal, + name: s("scx"), + value: s("Katakana"), + }, + })) + ); + assert_eq!( + parser(r"\p{scx!=Katakana}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..17), + negated: false, + kind: ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::NotEqual, + name: s("scx"), + value: s("Katakana"), + }, + })) + ); + + assert_eq!( + parser(r"\p{:}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..5), + negated: false, + kind: ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::Colon, + name: s(""), + value: s(""), + }, + })) + ); + assert_eq!( + parser(r"\p{=}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..5), + negated: false, + kind: ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::Equal, + name: s(""), + value: s(""), + }, + })) + ); + assert_eq!( + parser(r"\p{!=}").parse_escape(), + Ok(Primitive::Unicode(ast::ClassUnicode { + span: span(0..6), + negated: false, + kind: ast::ClassUnicodeKind::NamedValue { + op: ast::ClassUnicodeOpKind::NotEqual, + name: s(""), + value: s(""), + }, + })) + ); + + assert_eq!( + parser(r"\p").parse_escape().unwrap_err(), + TestError { + span: span(2..2), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\p{").parse_escape().unwrap_err(), + TestError { + span: span(3..3), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\p{N").parse_escape().unwrap_err(), + TestError { + span: span(4..4), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + assert_eq!( + parser(r"\p{Greek").parse_escape().unwrap_err(), + TestError { + span: span(8..8), + kind: ast::ErrorKind::EscapeUnexpectedEof, + } + ); + + assert_eq!( + parser(r"\pNz").parse(), + Ok(Ast::concat(ast::Concat { + span: span(0..4), + asts: vec![ + Ast::class_unicode(ast::ClassUnicode { + span: span(0..3), + negated: false, + kind: ast::ClassUnicodeKind::OneLetter('N'), + }), + Ast::literal(ast::Literal { + span: span(3..4), + kind: ast::LiteralKind::Verbatim, + c: 'z', + }), + ], + })) + ); + assert_eq!( + parser(r"\p{Greek}z").parse(), + Ok(Ast::concat(ast::Concat { + span: span(0..10), + asts: vec![ + Ast::class_unicode(ast::ClassUnicode { + span: span(0..9), + negated: false, + kind: ast::ClassUnicodeKind::Named(s("Greek")), + }), + Ast::literal(ast::Literal { + span: span(9..10), + kind: ast::LiteralKind::Verbatim, + c: 'z', + }), + ], + })) + ); + assert_eq!( + parser(r"\p\{").parse().unwrap_err(), + TestError { + span: span(2..3), + kind: ast::ErrorKind::UnicodeClassInvalid, + } + ); + assert_eq!( + parser(r"\P\{").parse().unwrap_err(), + TestError { + span: span(2..3), + kind: ast::ErrorKind::UnicodeClassInvalid, + } + ); + } + + #[test] + fn parse_perl_class() { + assert_eq!( + parser(r"\d").parse_escape(), + Ok(Primitive::Perl(ast::ClassPerl { + span: span(0..2), + kind: ast::ClassPerlKind::Digit, + negated: false, + })) + ); + assert_eq!( + parser(r"\D").parse_escape(), + Ok(Primitive::Perl(ast::ClassPerl { + span: span(0..2), + kind: ast::ClassPerlKind::Digit, + negated: true, + })) + ); + assert_eq!( + parser(r"\s").parse_escape(), + Ok(Primitive::Perl(ast::ClassPerl { + span: span(0..2), + kind: ast::ClassPerlKind::Space, + negated: false, + })) + ); + assert_eq!( + parser(r"\S").parse_escape(), + Ok(Primitive::Perl(ast::ClassPerl { + span: span(0..2), + kind: ast::ClassPerlKind::Space, + negated: true, + })) + ); + assert_eq!( + parser(r"\w").parse_escape(), + Ok(Primitive::Perl(ast::ClassPerl { + span: span(0..2), + kind: ast::ClassPerlKind::Word, + negated: false, + })) + ); + assert_eq!( + parser(r"\W").parse_escape(), + Ok(Primitive::Perl(ast::ClassPerl { + span: span(0..2), + kind: ast::ClassPerlKind::Word, + negated: true, + })) + ); + + assert_eq!( + parser(r"\d").parse(), + Ok(Ast::class_perl(ast::ClassPerl { + span: span(0..2), + kind: ast::ClassPerlKind::Digit, + negated: false, + })) + ); + assert_eq!( + parser(r"\dz").parse(), + Ok(Ast::concat(ast::Concat { + span: span(0..3), + asts: vec![ + Ast::class_perl(ast::ClassPerl { + span: span(0..2), + kind: ast::ClassPerlKind::Digit, + negated: false, + }), + Ast::literal(ast::Literal { + span: span(2..3), + kind: ast::LiteralKind::Verbatim, + c: 'z', + }), + ], + })) + ); + } + + // This tests a bug fix where the nest limit checker wasn't decrementing + // its depth during post-traversal, which causes long regexes to trip + // the default limit too aggressively. + #[test] + fn regression_454_nest_too_big() { + let pattern = r#" + 2(?: + [45]\d{3}| + 7(?: + 1[0-267]| + 2[0-289]| + 3[0-29]| + 4[01]| + 5[1-3]| + 6[013]| + 7[0178]| + 91 + )| + 8(?: + 0[125]| + [139][1-6]| + 2[0157-9]| + 41| + 6[1-35]| + 7[1-5]| + 8[1-8]| + 90 + )| + 9(?: + 0[0-2]| + 1[0-4]| + 2[568]| + 3[3-6]| + 5[5-7]| + 6[0167]| + 7[15]| + 8[0146-9] + ) + )\d{4} + "#; + assert!(parser_nest_limit(pattern, 50).parse().is_ok()); + } + + // This tests that we treat a trailing `-` in a character class as a + // literal `-` even when whitespace mode is enabled and there is whitespace + // after the trailing `-`. + #[test] + fn regression_455_trailing_dash_ignore_whitespace() { + assert!(parser("(?x)[ / - ]").parse().is_ok()); + assert!(parser("(?x)[ a - ]").parse().is_ok()); + assert!(parser( + "(?x)[ + a + - ] + " + ) + .parse() + .is_ok()); + assert!(parser( + "(?x)[ + a # wat + - ] + " + ) + .parse() + .is_ok()); + + assert!(parser("(?x)[ / -").parse().is_err()); + assert!(parser("(?x)[ / - ").parse().is_err()); + assert!(parser( + "(?x)[ + / - + " + ) + .parse() + .is_err()); + assert!(parser( + "(?x)[ + / - # wat + " + ) + .parse() + .is_err()); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/ast/print.rs b/third_party/crates/regex-syntax-0.8.11/src/ast/print.rs new file mode 100644 index 0000000..556d91f --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/ast/print.rs @@ -0,0 +1,577 @@ +/*! +This module provides a regular expression printer for `Ast`. +*/ + +use core::fmt; + +use crate::ast::{ + self, + visitor::{self, Visitor}, + Ast, +}; + +/// A builder for constructing a printer. +/// +/// Note that since a printer doesn't have any configuration knobs, this type +/// remains unexported. +#[derive(Clone, Debug)] +struct PrinterBuilder { + _priv: (), +} + +impl Default for PrinterBuilder { + fn default() -> PrinterBuilder { + PrinterBuilder::new() + } +} + +impl PrinterBuilder { + fn new() -> PrinterBuilder { + PrinterBuilder { _priv: () } + } + + fn build(&self) -> Printer { + Printer { _priv: () } + } +} + +/// A printer for a regular expression abstract syntax tree. +/// +/// A printer converts an abstract syntax tree (AST) to a regular expression +/// pattern string. This particular printer uses constant stack space and heap +/// space proportional to the size of the AST. +/// +/// This printer will not necessarily preserve the original formatting of the +/// regular expression pattern string. For example, all whitespace and comments +/// are ignored. +#[derive(Debug)] +pub struct Printer { + _priv: (), +} + +impl Printer { + /// Create a new printer. + pub fn new() -> Printer { + PrinterBuilder::new().build() + } + + /// Print the given `Ast` to the given writer. The writer must implement + /// `fmt::Write`. Typical implementations of `fmt::Write` that can be used + /// here are a `fmt::Formatter` (which is available in `fmt::Display` + /// implementations) or a `&mut String`. + pub fn print<W: fmt::Write>(&mut self, ast: &Ast, wtr: W) -> fmt::Result { + visitor::visit(ast, Writer { wtr }) + } +} + +#[derive(Debug)] +struct Writer<W> { + wtr: W, +} + +impl<W: fmt::Write> Visitor for Writer<W> { + type Output = (); + type Err = fmt::Error; + + fn finish(self) -> fmt::Result { + Ok(()) + } + + fn visit_pre(&mut self, ast: &Ast) -> fmt::Result { + match *ast { + Ast::Group(ref x) => self.fmt_group_pre(x), + Ast::ClassBracketed(ref x) => self.fmt_class_bracketed_pre(x), + _ => Ok(()), + } + } + + fn visit_post(&mut self, ast: &Ast) -> fmt::Result { + match *ast { + Ast::Empty(_) => Ok(()), + Ast::Flags(ref x) => self.fmt_set_flags(x), + Ast::Literal(ref x) => self.fmt_literal(x), + Ast::Dot(_) => self.wtr.write_str("."), + Ast::Assertion(ref x) => self.fmt_assertion(x), + Ast::ClassPerl(ref x) => self.fmt_class_perl(x), + Ast::ClassUnicode(ref x) => self.fmt_class_unicode(x), + Ast::ClassBracketed(ref x) => self.fmt_class_bracketed_post(x), + Ast::Repetition(ref x) => self.fmt_repetition(x), + Ast::Group(ref x) => self.fmt_group_post(x), + Ast::Alternation(_) => Ok(()), + Ast::Concat(_) => Ok(()), + } + } + + fn visit_alternation_in(&mut self) -> fmt::Result { + self.wtr.write_str("|") + } + + fn visit_class_set_item_pre( + &mut self, + ast: &ast::ClassSetItem, + ) -> Result<(), Self::Err> { + match *ast { + ast::ClassSetItem::Bracketed(ref x) => { + self.fmt_class_bracketed_pre(x) + } + _ => Ok(()), + } + } + + fn visit_class_set_item_post( + &mut self, + ast: &ast::ClassSetItem, + ) -> Result<(), Self::Err> { + use crate::ast::ClassSetItem::*; + + match *ast { + Empty(_) => Ok(()), + Literal(ref x) => self.fmt_literal(x), + Range(ref x) => { + self.fmt_literal(&x.start)?; + self.wtr.write_str("-")?; + self.fmt_literal(&x.end)?; + Ok(()) + } + Ascii(ref x) => self.fmt_class_ascii(x), + Unicode(ref x) => self.fmt_class_unicode(x), + Perl(ref x) => self.fmt_class_perl(x), + Bracketed(ref x) => self.fmt_class_bracketed_post(x), + Union(_) => Ok(()), + } + } + + fn visit_class_set_binary_op_in( + &mut self, + ast: &ast::ClassSetBinaryOp, + ) -> Result<(), Self::Err> { + self.fmt_class_set_binary_op_kind(&ast.kind) + } +} + +impl<W: fmt::Write> Writer<W> { + fn fmt_group_pre(&mut self, ast: &ast::Group) -> fmt::Result { + use crate::ast::GroupKind::*; + match ast.kind { + CaptureIndex(_) => self.wtr.write_str("("), + CaptureName { ref name, starts_with_p } => { + let start = if starts_with_p { "(?P<" } else { "(?<" }; + self.wtr.write_str(start)?; + self.wtr.write_str(&name.name)?; + self.wtr.write_str(">")?; + Ok(()) + } + NonCapturing(ref flags) => { + self.wtr.write_str("(?")?; + self.fmt_flags(flags)?; + self.wtr.write_str(":")?; + Ok(()) + } + } + } + + fn fmt_group_post(&mut self, _ast: &ast::Group) -> fmt::Result { + self.wtr.write_str(")") + } + + fn fmt_repetition(&mut self, ast: &ast::Repetition) -> fmt::Result { + use crate::ast::RepetitionKind::*; + match ast.op.kind { + ZeroOrOne if ast.greedy => self.wtr.write_str("?"), + ZeroOrOne => self.wtr.write_str("??"), + ZeroOrMore if ast.greedy => self.wtr.write_str("*"), + ZeroOrMore => self.wtr.write_str("*?"), + OneOrMore if ast.greedy => self.wtr.write_str("+"), + OneOrMore => self.wtr.write_str("+?"), + Range(ref x) => { + self.fmt_repetition_range(x)?; + if !ast.greedy { + self.wtr.write_str("?")?; + } + Ok(()) + } + } + } + + fn fmt_repetition_range( + &mut self, + ast: &ast::RepetitionRange, + ) -> fmt::Result { + use crate::ast::RepetitionRange::*; + match *ast { + Exactly(x) => write!(self.wtr, "{{{x}}}"), + AtLeast(x) => write!(self.wtr, "{{{x},}}"), + Bounded(x, y) => write!(self.wtr, "{{{x},{y}}}"), + } + } + + fn fmt_literal(&mut self, ast: &ast::Literal) -> fmt::Result { + use crate::ast::LiteralKind::*; + + match ast.kind { + Verbatim => self.wtr.write_char(ast.c), + Meta | Superfluous => write!(self.wtr, r"\{}", ast.c), + Octal => write!(self.wtr, r"\{:o}", u32::from(ast.c)), + HexFixed(ast::HexLiteralKind::X) => { + write!(self.wtr, r"\x{:02X}", u32::from(ast.c)) + } + HexFixed(ast::HexLiteralKind::UnicodeShort) => { + write!(self.wtr, r"\u{:04X}", u32::from(ast.c)) + } + HexFixed(ast::HexLiteralKind::UnicodeLong) => { + write!(self.wtr, r"\U{:08X}", u32::from(ast.c)) + } + HexBrace(ast::HexLiteralKind::X) => { + write!(self.wtr, r"\x{{{:X}}}", u32::from(ast.c)) + } + HexBrace(ast::HexLiteralKind::UnicodeShort) => { + write!(self.wtr, r"\u{{{:X}}}", u32::from(ast.c)) + } + HexBrace(ast::HexLiteralKind::UnicodeLong) => { + write!(self.wtr, r"\U{{{:X}}}", u32::from(ast.c)) + } + Special(ast::SpecialLiteralKind::Bell) => { + self.wtr.write_str(r"\a") + } + Special(ast::SpecialLiteralKind::FormFeed) => { + self.wtr.write_str(r"\f") + } + Special(ast::SpecialLiteralKind::Tab) => self.wtr.write_str(r"\t"), + Special(ast::SpecialLiteralKind::LineFeed) => { + self.wtr.write_str(r"\n") + } + Special(ast::SpecialLiteralKind::CarriageReturn) => { + self.wtr.write_str(r"\r") + } + Special(ast::SpecialLiteralKind::VerticalTab) => { + self.wtr.write_str(r"\v") + } + Special(ast::SpecialLiteralKind::Space) => { + self.wtr.write_str(r"\ ") + } + } + } + + fn fmt_assertion(&mut self, ast: &ast::Assertion) -> fmt::Result { + use crate::ast::AssertionKind::*; + match ast.kind { + StartLine => self.wtr.write_str("^"), + EndLine => self.wtr.write_str("$"), + StartText => self.wtr.write_str(r"\A"), + EndText => self.wtr.write_str(r"\z"), + WordBoundary => self.wtr.write_str(r"\b"), + NotWordBoundary => self.wtr.write_str(r"\B"), + WordBoundaryStart => self.wtr.write_str(r"\b{start}"), + WordBoundaryEnd => self.wtr.write_str(r"\b{end}"), + WordBoundaryStartAngle => self.wtr.write_str(r"\<"), + WordBoundaryEndAngle => self.wtr.write_str(r"\>"), + WordBoundaryStartHalf => self.wtr.write_str(r"\b{start-half}"), + WordBoundaryEndHalf => self.wtr.write_str(r"\b{end-half}"), + } + } + + fn fmt_set_flags(&mut self, ast: &ast::SetFlags) -> fmt::Result { + self.wtr.write_str("(?")?; + self.fmt_flags(&ast.flags)?; + self.wtr.write_str(")")?; + Ok(()) + } + + fn fmt_flags(&mut self, ast: &ast::Flags) -> fmt::Result { + use crate::ast::{Flag, FlagsItemKind}; + + for item in &ast.items { + match item.kind { + FlagsItemKind::Negation => self.wtr.write_str("-"), + FlagsItemKind::Flag(ref flag) => match *flag { + Flag::CaseInsensitive => self.wtr.write_str("i"), + Flag::MultiLine => self.wtr.write_str("m"), + Flag::DotMatchesNewLine => self.wtr.write_str("s"), + Flag::SwapGreed => self.wtr.write_str("U"), + Flag::Unicode => self.wtr.write_str("u"), + Flag::CRLF => self.wtr.write_str("R"), + Flag::IgnoreWhitespace => self.wtr.write_str("x"), + }, + }?; + } + Ok(()) + } + + fn fmt_class_bracketed_pre( + &mut self, + ast: &ast::ClassBracketed, + ) -> fmt::Result { + if ast.negated { + self.wtr.write_str("[^") + } else { + self.wtr.write_str("[") + } + } + + fn fmt_class_bracketed_post( + &mut self, + _ast: &ast::ClassBracketed, + ) -> fmt::Result { + self.wtr.write_str("]") + } + + fn fmt_class_set_binary_op_kind( + &mut self, + ast: &ast::ClassSetBinaryOpKind, + ) -> fmt::Result { + use crate::ast::ClassSetBinaryOpKind::*; + match *ast { + Intersection => self.wtr.write_str("&&"), + Difference => self.wtr.write_str("--"), + SymmetricDifference => self.wtr.write_str("~~"), + } + } + + fn fmt_class_perl(&mut self, ast: &ast::ClassPerl) -> fmt::Result { + use crate::ast::ClassPerlKind::*; + match ast.kind { + Digit if ast.negated => self.wtr.write_str(r"\D"), + Digit => self.wtr.write_str(r"\d"), + Space if ast.negated => self.wtr.write_str(r"\S"), + Space => self.wtr.write_str(r"\s"), + Word if ast.negated => self.wtr.write_str(r"\W"), + Word => self.wtr.write_str(r"\w"), + } + } + + fn fmt_class_ascii(&mut self, ast: &ast::ClassAscii) -> fmt::Result { + use crate::ast::ClassAsciiKind::*; + match ast.kind { + Alnum if ast.negated => self.wtr.write_str("[:^alnum:]"), + Alnum => self.wtr.write_str("[:alnum:]"), + Alpha if ast.negated => self.wtr.write_str("[:^alpha:]"), + Alpha => self.wtr.write_str("[:alpha:]"), + Ascii if ast.negated => self.wtr.write_str("[:^ascii:]"), + Ascii => self.wtr.write_str("[:ascii:]"), + Blank if ast.negated => self.wtr.write_str("[:^blank:]"), + Blank => self.wtr.write_str("[:blank:]"), + Cntrl if ast.negated => self.wtr.write_str("[:^cntrl:]"), + Cntrl => self.wtr.write_str("[:cntrl:]"), + Digit if ast.negated => self.wtr.write_str("[:^digit:]"), + Digit => self.wtr.write_str("[:digit:]"), + Graph if ast.negated => self.wtr.write_str("[:^graph:]"), + Graph => self.wtr.write_str("[:graph:]"), + Lower if ast.negated => self.wtr.write_str("[:^lower:]"), + Lower => self.wtr.write_str("[:lower:]"), + Print if ast.negated => self.wtr.write_str("[:^print:]"), + Print => self.wtr.write_str("[:print:]"), + Punct if ast.negated => self.wtr.write_str("[:^punct:]"), + Punct => self.wtr.write_str("[:punct:]"), + Space if ast.negated => self.wtr.write_str("[:^space:]"), + Space => self.wtr.write_str("[:space:]"), + Upper if ast.negated => self.wtr.write_str("[:^upper:]"), + Upper => self.wtr.write_str("[:upper:]"), + Word if ast.negated => self.wtr.write_str("[:^word:]"), + Word => self.wtr.write_str("[:word:]"), + Xdigit if ast.negated => self.wtr.write_str("[:^xdigit:]"), + Xdigit => self.wtr.write_str("[:xdigit:]"), + } + } + + fn fmt_class_unicode(&mut self, ast: &ast::ClassUnicode) -> fmt::Result { + use crate::ast::ClassUnicodeKind::*; + use crate::ast::ClassUnicodeOpKind::*; + + if ast.negated { + self.wtr.write_str(r"\P")?; + } else { + self.wtr.write_str(r"\p")?; + } + match ast.kind { + OneLetter(c) => self.wtr.write_char(c), + Named(ref x) => write!(self.wtr, "{{{}}}", x), + NamedValue { op: Equal, ref name, ref value } => { + write!(self.wtr, "{{{}={}}}", name, value) + } + NamedValue { op: Colon, ref name, ref value } => { + write!(self.wtr, "{{{}:{}}}", name, value) + } + NamedValue { op: NotEqual, ref name, ref value } => { + write!(self.wtr, "{{{}!={}}}", name, value) + } + } + } +} + +#[cfg(test)] +mod tests { + use alloc::string::String; + + use crate::ast::parse::ParserBuilder; + + use super::*; + + fn roundtrip(given: &str) { + roundtrip_with(|b| b, given); + } + + fn roundtrip_with<F>(mut f: F, given: &str) + where + F: FnMut(&mut ParserBuilder) -> &mut ParserBuilder, + { + let mut builder = ParserBuilder::new(); + f(&mut builder); + let ast = builder.build().parse(given).unwrap(); + + let mut printer = Printer::new(); + let mut dst = String::new(); + printer.print(&ast, &mut dst).unwrap(); + assert_eq!(given, dst); + } + + #[test] + fn print_literal() { + roundtrip("a"); + roundtrip(r"\["); + roundtrip_with(|b| b.octal(true), r"\141"); + roundtrip(r"\x61"); + roundtrip(r"\x7F"); + roundtrip(r"\u0061"); + roundtrip(r"\U00000061"); + roundtrip(r"\x{61}"); + roundtrip(r"\x{7F}"); + roundtrip(r"\u{61}"); + roundtrip(r"\U{61}"); + + roundtrip(r"\a"); + roundtrip(r"\f"); + roundtrip(r"\t"); + roundtrip(r"\n"); + roundtrip(r"\r"); + roundtrip(r"\v"); + roundtrip(r"(?x)\ "); + } + + #[test] + fn print_dot() { + roundtrip("."); + } + + #[test] + fn print_concat() { + roundtrip("ab"); + roundtrip("abcde"); + roundtrip("a(bcd)ef"); + } + + #[test] + fn print_alternation() { + roundtrip("a|b"); + roundtrip("a|b|c|d|e"); + roundtrip("|a|b|c|d|e"); + roundtrip("|a|b|c|d|e|"); + roundtrip("a(b|c|d)|e|f"); + } + + #[test] + fn print_assertion() { + roundtrip(r"^"); + roundtrip(r"$"); + roundtrip(r"\A"); + roundtrip(r"\z"); + roundtrip(r"\b"); + roundtrip(r"\B"); + } + + #[test] + fn print_repetition() { + roundtrip("a?"); + roundtrip("a??"); + roundtrip("a*"); + roundtrip("a*?"); + roundtrip("a+"); + roundtrip("a+?"); + roundtrip("a{5}"); + roundtrip("a{5}?"); + roundtrip("a{5,}"); + roundtrip("a{5,}?"); + roundtrip("a{5,10}"); + roundtrip("a{5,10}?"); + } + + #[test] + fn print_flags() { + roundtrip("(?i)"); + roundtrip("(?-i)"); + roundtrip("(?s-i)"); + roundtrip("(?-si)"); + roundtrip("(?siUmux)"); + } + + #[test] + fn print_group() { + roundtrip("(?i:a)"); + roundtrip("(?P<foo>a)"); + roundtrip("(?<foo>a)"); + roundtrip("(a)"); + } + + #[test] + fn print_class() { + roundtrip(r"[abc]"); + roundtrip(r"[a-z]"); + roundtrip(r"[^a-z]"); + roundtrip(r"[a-z0-9]"); + roundtrip(r"[-a-z0-9]"); + roundtrip(r"[-a-z0-9]"); + roundtrip(r"[a-z0-9---]"); + roundtrip(r"[a-z&&m-n]"); + roundtrip(r"[[a-z&&m-n]]"); + roundtrip(r"[a-z--m-n]"); + roundtrip(r"[a-z~~m-n]"); + roundtrip(r"[a-z[0-9]]"); + roundtrip(r"[a-z[^0-9]]"); + + roundtrip(r"\d"); + roundtrip(r"\D"); + roundtrip(r"\s"); + roundtrip(r"\S"); + roundtrip(r"\w"); + roundtrip(r"\W"); + + roundtrip(r"[[:alnum:]]"); + roundtrip(r"[[:^alnum:]]"); + roundtrip(r"[[:alpha:]]"); + roundtrip(r"[[:^alpha:]]"); + roundtrip(r"[[:ascii:]]"); + roundtrip(r"[[:^ascii:]]"); + roundtrip(r"[[:blank:]]"); + roundtrip(r"[[:^blank:]]"); + roundtrip(r"[[:cntrl:]]"); + roundtrip(r"[[:^cntrl:]]"); + roundtrip(r"[[:digit:]]"); + roundtrip(r"[[:^digit:]]"); + roundtrip(r"[[:graph:]]"); + roundtrip(r"[[:^graph:]]"); + roundtrip(r"[[:lower:]]"); + roundtrip(r"[[:^lower:]]"); + roundtrip(r"[[:print:]]"); + roundtrip(r"[[:^print:]]"); + roundtrip(r"[[:punct:]]"); + roundtrip(r"[[:^punct:]]"); + roundtrip(r"[[:space:]]"); + roundtrip(r"[[:^space:]]"); + roundtrip(r"[[:upper:]]"); + roundtrip(r"[[:^upper:]]"); + roundtrip(r"[[:word:]]"); + roundtrip(r"[[:^word:]]"); + roundtrip(r"[[:xdigit:]]"); + roundtrip(r"[[:^xdigit:]]"); + + roundtrip(r"\pL"); + roundtrip(r"\PL"); + roundtrip(r"\p{L}"); + roundtrip(r"\P{L}"); + roundtrip(r"\p{X=Y}"); + roundtrip(r"\P{X=Y}"); + roundtrip(r"\p{X:Y}"); + roundtrip(r"\P{X:Y}"); + roundtrip(r"\p{X!=Y}"); + roundtrip(r"\P{X!=Y}"); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/ast/visitor.rs b/third_party/crates/regex-syntax-0.8.11/src/ast/visitor.rs new file mode 100644 index 0000000..36cd713 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/ast/visitor.rs @@ -0,0 +1,522 @@ +use alloc::{vec, vec::Vec}; + +use crate::ast::{self, Ast}; + +/// A trait for visiting an abstract syntax tree (AST) in depth first order. +/// +/// The principle aim of this trait is to enable callers to perform case +/// analysis on an abstract syntax tree without necessarily using recursion. +/// In particular, this permits callers to do case analysis with constant stack +/// usage, which can be important since the size of an abstract syntax tree +/// may be proportional to end user input. +/// +/// Typical usage of this trait involves providing an implementation and then +/// running it using the [`visit`] function. +/// +/// Note that the abstract syntax tree for a regular expression is quite +/// complex. Unless you specifically need it, you might be able to use the much +/// simpler [high-level intermediate representation](crate::hir::Hir) and its +/// [corresponding `Visitor` trait](crate::hir::Visitor) instead. +pub trait Visitor { + /// The result of visiting an AST. + type Output; + /// An error that visiting an AST might return. + type Err; + + /// All implementors of `Visitor` must provide a `finish` method, which + /// yields the result of visiting the AST or an error. + fn finish(self) -> Result<Self::Output, Self::Err>; + + /// This method is called before beginning traversal of the AST. + fn start(&mut self) {} + + /// This method is called on an `Ast` before descending into child `Ast` + /// nodes. + fn visit_pre(&mut self, _ast: &Ast) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called on an `Ast` after descending all of its child + /// `Ast` nodes. + fn visit_post(&mut self, _ast: &Ast) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called between child nodes of an + /// [`Alternation`](ast::Alternation). + fn visit_alternation_in(&mut self) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called between child nodes of a concatenation. + fn visit_concat_in(&mut self) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called on every [`ClassSetItem`](ast::ClassSetItem) + /// before descending into child nodes. + fn visit_class_set_item_pre( + &mut self, + _ast: &ast::ClassSetItem, + ) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called on every [`ClassSetItem`](ast::ClassSetItem) + /// after descending into child nodes. + fn visit_class_set_item_post( + &mut self, + _ast: &ast::ClassSetItem, + ) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called on every + /// [`ClassSetBinaryOp`](ast::ClassSetBinaryOp) before descending into + /// child nodes. + fn visit_class_set_binary_op_pre( + &mut self, + _ast: &ast::ClassSetBinaryOp, + ) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called on every + /// [`ClassSetBinaryOp`](ast::ClassSetBinaryOp) after descending into child + /// nodes. + fn visit_class_set_binary_op_post( + &mut self, + _ast: &ast::ClassSetBinaryOp, + ) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called between the left hand and right hand child nodes + /// of a [`ClassSetBinaryOp`](ast::ClassSetBinaryOp). + fn visit_class_set_binary_op_in( + &mut self, + _ast: &ast::ClassSetBinaryOp, + ) -> Result<(), Self::Err> { + Ok(()) + } +} + +/// Executes an implementation of `Visitor` in constant stack space. +/// +/// This function will visit every node in the given `Ast` while calling the +/// appropriate methods provided by the [`Visitor`] trait. +/// +/// The primary use case for this method is when one wants to perform case +/// analysis over an `Ast` without using a stack size proportional to the depth +/// of the `Ast`. Namely, this method will instead use constant stack size, but +/// will use heap space proportional to the size of the `Ast`. This may be +/// desirable in cases where the size of `Ast` is proportional to end user +/// input. +/// +/// If the visitor returns an error at any point, then visiting is stopped and +/// the error is returned. +pub fn visit<V: Visitor>(ast: &Ast, visitor: V) -> Result<V::Output, V::Err> { + HeapVisitor::new().visit(ast, visitor) +} + +/// HeapVisitor visits every item in an `Ast` recursively using constant stack +/// size and a heap size proportional to the size of the `Ast`. +struct HeapVisitor<'a> { + /// A stack of `Ast` nodes. This is roughly analogous to the call stack + /// used in a typical recursive visitor. + stack: Vec<(&'a Ast, Frame<'a>)>, + /// Similar to the `Ast` stack above, but is used only for character + /// classes. In particular, character classes embed their own mini + /// recursive syntax. + stack_class: Vec<(ClassInduct<'a>, ClassFrame<'a>)>, +} + +/// Represents a single stack frame while performing structural induction over +/// an `Ast`. +enum Frame<'a> { + /// A stack frame allocated just before descending into a repetition + /// operator's child node. + Repetition(&'a ast::Repetition), + /// A stack frame allocated just before descending into a group's child + /// node. + Group(&'a ast::Group), + /// The stack frame used while visiting every child node of a concatenation + /// of expressions. + Concat { + /// The child node we are currently visiting. + head: &'a Ast, + /// The remaining child nodes to visit (which may be empty). + tail: &'a [Ast], + }, + /// The stack frame used while visiting every child node of an alternation + /// of expressions. + Alternation { + /// The child node we are currently visiting. + head: &'a Ast, + /// The remaining child nodes to visit (which may be empty). + tail: &'a [Ast], + }, +} + +/// Represents a single stack frame while performing structural induction over +/// a character class. +enum ClassFrame<'a> { + /// The stack frame used while visiting every child node of a union of + /// character class items. + Union { + /// The child node we are currently visiting. + head: &'a ast::ClassSetItem, + /// The remaining child nodes to visit (which may be empty). + tail: &'a [ast::ClassSetItem], + }, + /// The stack frame used while a binary class operation. + Binary { op: &'a ast::ClassSetBinaryOp }, + /// A stack frame allocated just before descending into a binary operator's + /// left hand child node. + BinaryLHS { + op: &'a ast::ClassSetBinaryOp, + lhs: &'a ast::ClassSet, + rhs: &'a ast::ClassSet, + }, + /// A stack frame allocated just before descending into a binary operator's + /// right hand child node. + BinaryRHS { op: &'a ast::ClassSetBinaryOp, rhs: &'a ast::ClassSet }, +} + +/// A representation of the inductive step when performing structural induction +/// over a character class. +/// +/// Note that there is no analogous explicit type for the inductive step for +/// `Ast` nodes because the inductive step is just an `Ast`. For character +/// classes, the inductive step can produce one of two possible child nodes: +/// an item or a binary operation. (An item cannot be a binary operation +/// because that would imply binary operations can be unioned in the concrete +/// syntax, which is not possible.) +enum ClassInduct<'a> { + Item(&'a ast::ClassSetItem), + BinaryOp(&'a ast::ClassSetBinaryOp), +} + +impl<'a> HeapVisitor<'a> { + fn new() -> HeapVisitor<'a> { + HeapVisitor { stack: vec![], stack_class: vec![] } + } + + fn visit<V: Visitor>( + &mut self, + mut ast: &'a Ast, + mut visitor: V, + ) -> Result<V::Output, V::Err> { + self.stack.clear(); + self.stack_class.clear(); + + visitor.start(); + loop { + visitor.visit_pre(ast)?; + if let Some(x) = self.induct(ast, &mut visitor)? { + let child = x.child(); + self.stack.push((ast, x)); + ast = child; + continue; + } + // No induction means we have a base case, so we can post visit + // it now. + visitor.visit_post(ast)?; + + // At this point, we now try to pop our call stack until it is + // either empty or we hit another inductive case. + loop { + let (post_ast, frame) = match self.stack.pop() { + None => return visitor.finish(), + Some((post_ast, frame)) => (post_ast, frame), + }; + // If this is a concat/alternate, then we might have additional + // inductive steps to process. + if let Some(x) = self.pop(frame) { + match x { + Frame::Alternation { .. } => { + visitor.visit_alternation_in()?; + } + Frame::Concat { .. } => { + visitor.visit_concat_in()?; + } + _ => {} + } + ast = x.child(); + self.stack.push((post_ast, x)); + break; + } + // Otherwise, we've finished visiting all the child nodes for + // this AST, so we can post visit it now. + visitor.visit_post(post_ast)?; + } + } + } + + /// Build a stack frame for the given AST if one is needed (which occurs if + /// and only if there are child nodes in the AST). Otherwise, return None. + /// + /// If this visits a class, then the underlying visitor implementation may + /// return an error which will be passed on here. + fn induct<V: Visitor>( + &mut self, + ast: &'a Ast, + visitor: &mut V, + ) -> Result<Option<Frame<'a>>, V::Err> { + Ok(match *ast { + Ast::ClassBracketed(ref x) => { + self.visit_class(x, visitor)?; + None + } + Ast::Repetition(ref x) => Some(Frame::Repetition(x)), + Ast::Group(ref x) => Some(Frame::Group(x)), + Ast::Concat(ref x) if x.asts.is_empty() => None, + Ast::Concat(ref x) => { + Some(Frame::Concat { head: &x.asts[0], tail: &x.asts[1..] }) + } + Ast::Alternation(ref x) if x.asts.is_empty() => None, + Ast::Alternation(ref x) => Some(Frame::Alternation { + head: &x.asts[0], + tail: &x.asts[1..], + }), + _ => None, + }) + } + + /// Pops the given frame. If the frame has an additional inductive step, + /// then return it, otherwise return `None`. + fn pop(&self, induct: Frame<'a>) -> Option<Frame<'a>> { + match induct { + Frame::Repetition(_) => None, + Frame::Group(_) => None, + Frame::Concat { tail, .. } => { + if tail.is_empty() { + None + } else { + Some(Frame::Concat { head: &tail[0], tail: &tail[1..] }) + } + } + Frame::Alternation { tail, .. } => { + if tail.is_empty() { + None + } else { + Some(Frame::Alternation { + head: &tail[0], + tail: &tail[1..], + }) + } + } + } + } + + fn visit_class<V: Visitor>( + &mut self, + ast: &'a ast::ClassBracketed, + visitor: &mut V, + ) -> Result<(), V::Err> { + let mut ast = ClassInduct::from_bracketed(ast); + loop { + self.visit_class_pre(&ast, visitor)?; + if let Some(x) = self.induct_class(&ast) { + let child = x.child(); + self.stack_class.push((ast, x)); + ast = child; + continue; + } + self.visit_class_post(&ast, visitor)?; + + // At this point, we now try to pop our call stack until it is + // either empty or we hit another inductive case. + loop { + let (post_ast, frame) = match self.stack_class.pop() { + None => return Ok(()), + Some((post_ast, frame)) => (post_ast, frame), + }; + // If this is a union or a binary op, then we might have + // additional inductive steps to process. + if let Some(x) = self.pop_class(frame) { + if let ClassFrame::BinaryRHS { ref op, .. } = x { + visitor.visit_class_set_binary_op_in(op)?; + } + ast = x.child(); + self.stack_class.push((post_ast, x)); + break; + } + // Otherwise, we've finished visiting all the child nodes for + // this class node, so we can post visit it now. + self.visit_class_post(&post_ast, visitor)?; + } + } + } + + /// Call the appropriate `Visitor` methods given an inductive step. + fn visit_class_pre<V: Visitor>( + &self, + ast: &ClassInduct<'a>, + visitor: &mut V, + ) -> Result<(), V::Err> { + match *ast { + ClassInduct::Item(item) => { + visitor.visit_class_set_item_pre(item)?; + } + ClassInduct::BinaryOp(op) => { + visitor.visit_class_set_binary_op_pre(op)?; + } + } + Ok(()) + } + + /// Call the appropriate `Visitor` methods given an inductive step. + fn visit_class_post<V: Visitor>( + &self, + ast: &ClassInduct<'a>, + visitor: &mut V, + ) -> Result<(), V::Err> { + match *ast { + ClassInduct::Item(item) => { + visitor.visit_class_set_item_post(item)?; + } + ClassInduct::BinaryOp(op) => { + visitor.visit_class_set_binary_op_post(op)?; + } + } + Ok(()) + } + + /// Build a stack frame for the given class node if one is needed (which + /// occurs if and only if there are child nodes). Otherwise, return None. + fn induct_class(&self, ast: &ClassInduct<'a>) -> Option<ClassFrame<'a>> { + match *ast { + ClassInduct::Item(&ast::ClassSetItem::Bracketed(ref x)) => { + match x.kind { + ast::ClassSet::Item(ref item) => { + Some(ClassFrame::Union { head: item, tail: &[] }) + } + ast::ClassSet::BinaryOp(ref op) => { + Some(ClassFrame::Binary { op }) + } + } + } + ClassInduct::Item(&ast::ClassSetItem::Union(ref x)) => { + if x.items.is_empty() { + None + } else { + Some(ClassFrame::Union { + head: &x.items[0], + tail: &x.items[1..], + }) + } + } + ClassInduct::BinaryOp(op) => { + Some(ClassFrame::BinaryLHS { op, lhs: &op.lhs, rhs: &op.rhs }) + } + _ => None, + } + } + + /// Pops the given frame. If the frame has an additional inductive step, + /// then return it, otherwise return `None`. + fn pop_class(&self, induct: ClassFrame<'a>) -> Option<ClassFrame<'a>> { + match induct { + ClassFrame::Union { tail, .. } => { + if tail.is_empty() { + None + } else { + Some(ClassFrame::Union { + head: &tail[0], + tail: &tail[1..], + }) + } + } + ClassFrame::Binary { .. } => None, + ClassFrame::BinaryLHS { op, rhs, .. } => { + Some(ClassFrame::BinaryRHS { op, rhs }) + } + ClassFrame::BinaryRHS { .. } => None, + } + } +} + +impl<'a> Frame<'a> { + /// Perform the next inductive step on this frame and return the next + /// child AST node to visit. + fn child(&self) -> &'a Ast { + match *self { + Frame::Repetition(rep) => &rep.ast, + Frame::Group(group) => &group.ast, + Frame::Concat { head, .. } => head, + Frame::Alternation { head, .. } => head, + } + } +} + +impl<'a> ClassFrame<'a> { + /// Perform the next inductive step on this frame and return the next + /// child class node to visit. + fn child(&self) -> ClassInduct<'a> { + match *self { + ClassFrame::Union { head, .. } => ClassInduct::Item(head), + ClassFrame::Binary { op, .. } => ClassInduct::BinaryOp(op), + ClassFrame::BinaryLHS { ref lhs, .. } => { + ClassInduct::from_set(lhs) + } + ClassFrame::BinaryRHS { ref rhs, .. } => { + ClassInduct::from_set(rhs) + } + } + } +} + +impl<'a> ClassInduct<'a> { + fn from_bracketed(ast: &'a ast::ClassBracketed) -> ClassInduct<'a> { + ClassInduct::from_set(&ast.kind) + } + + fn from_set(ast: &'a ast::ClassSet) -> ClassInduct<'a> { + match *ast { + ast::ClassSet::Item(ref item) => ClassInduct::Item(item), + ast::ClassSet::BinaryOp(ref op) => ClassInduct::BinaryOp(op), + } + } +} + +impl<'a> core::fmt::Debug for ClassFrame<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let x = match *self { + ClassFrame::Union { .. } => "Union", + ClassFrame::Binary { .. } => "Binary", + ClassFrame::BinaryLHS { .. } => "BinaryLHS", + ClassFrame::BinaryRHS { .. } => "BinaryRHS", + }; + write!(f, "{x}") + } +} + +impl<'a> core::fmt::Debug for ClassInduct<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let x = match *self { + ClassInduct::Item(it) => match *it { + ast::ClassSetItem::Empty(_) => "Item(Empty)", + ast::ClassSetItem::Literal(_) => "Item(Literal)", + ast::ClassSetItem::Range(_) => "Item(Range)", + ast::ClassSetItem::Ascii(_) => "Item(Ascii)", + ast::ClassSetItem::Perl(_) => "Item(Perl)", + ast::ClassSetItem::Unicode(_) => "Item(Unicode)", + ast::ClassSetItem::Bracketed(_) => "Item(Bracketed)", + ast::ClassSetItem::Union(_) => "Item(Union)", + }, + ClassInduct::BinaryOp(it) => match it.kind { + ast::ClassSetBinaryOpKind::Intersection => { + "BinaryOp(Intersection)" + } + ast::ClassSetBinaryOpKind::Difference => { + "BinaryOp(Difference)" + } + ast::ClassSetBinaryOpKind::SymmetricDifference => { + "BinaryOp(SymmetricDifference)" + } + }, + }; + write!(f, "{x}") + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/debug.rs b/third_party/crates/regex-syntax-0.8.11/src/debug.rs new file mode 100644 index 0000000..7a47d9d --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/debug.rs @@ -0,0 +1,107 @@ +/// A type that wraps a single byte with a convenient fmt::Debug impl that +/// escapes the byte. +pub(crate) struct Byte(pub(crate) u8); + +impl core::fmt::Debug for Byte { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + // Special case ASCII space. It's too hard to read otherwise, so + // put quotes around it. I sometimes wonder whether just '\x20' would + // be better... + if self.0 == b' ' { + return write!(f, "' '"); + } + // 10 bytes is enough to cover any output from ascii::escape_default. + let mut bytes = [0u8; 10]; + let mut len = 0; + for (i, mut b) in core::ascii::escape_default(self.0).enumerate() { + // capitalize \xab to \xAB + if i >= 2 && b'a' <= b && b <= b'f' { + b -= 32; + } + bytes[len] = b; + len += 1; + } + write!(f, "{}", core::str::from_utf8(&bytes[..len]).unwrap()) + } +} + +/// A type that provides a human readable debug impl for arbitrary bytes. +/// +/// This generally works best when the bytes are presumed to be mostly UTF-8, +/// but will work for anything. +/// +/// N.B. This is copied nearly verbatim from regex-automata. Sigh. +pub(crate) struct Bytes<'a>(pub(crate) &'a [u8]); + +impl<'a> core::fmt::Debug for Bytes<'a> { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "\"")?; + // This is a sad re-implementation of a similar impl found in bstr. + let mut bytes = self.0; + while let Some(result) = utf8_decode(bytes) { + let ch = match result { + Ok(ch) => ch, + Err(byte) => { + write!(f, r"\x{byte:02x}")?; + bytes = &bytes[1..]; + continue; + } + }; + bytes = &bytes[ch.len_utf8()..]; + match ch { + '\0' => write!(f, "\\0")?, + // ASCII control characters except \0, \n, \r, \t + '\x01'..='\x08' + | '\x0b' + | '\x0c' + | '\x0e'..='\x19' + | '\x7f' => { + write!(f, "\\x{:02x}", u32::from(ch))?; + } + '\n' | '\r' | '\t' | _ => { + write!(f, "{}", ch.escape_debug())?; + } + } + } + write!(f, "\"")?; + Ok(()) + } +} + +/// Decodes the next UTF-8 encoded codepoint from the given byte slice. +/// +/// If no valid encoding of a codepoint exists at the beginning of the given +/// byte slice, then the first byte is returned instead. +/// +/// This returns `None` if and only if `bytes` is empty. +pub(crate) fn utf8_decode(bytes: &[u8]) -> Option<Result<char, u8>> { + fn len(byte: u8) -> Option<usize> { + if byte <= 0x7F { + return Some(1); + } else if byte & 0b1100_0000 == 0b1000_0000 { + return None; + } else if byte <= 0b1101_1111 { + Some(2) + } else if byte <= 0b1110_1111 { + Some(3) + } else if byte <= 0b1111_0111 { + Some(4) + } else { + None + } + } + + if bytes.is_empty() { + return None; + } + let len = match len(bytes[0]) { + None => return Some(Err(bytes[0])), + Some(len) if len > bytes.len() => return Some(Err(bytes[0])), + Some(1) => return Some(Ok(char::from(bytes[0]))), + Some(len) => len, + }; + match core::str::from_utf8(&bytes[..len]) { + Ok(s) => Some(Ok(s.chars().next().unwrap())), + Err(_) => Some(Err(bytes[0])), + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/either.rs b/third_party/crates/regex-syntax-0.8.11/src/either.rs new file mode 100644 index 0000000..7ae41e4 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/either.rs @@ -0,0 +1,8 @@ +/// A simple binary sum type. +/// +/// This is occasionally useful in an ad hoc fashion. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Either<Left, Right> { + Left(Left), + Right(Right), +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/error.rs b/third_party/crates/regex-syntax-0.8.11/src/error.rs new file mode 100644 index 0000000..21e484d --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/error.rs @@ -0,0 +1,311 @@ +use alloc::{ + format, + string::{String, ToString}, + vec, + vec::Vec, +}; + +use crate::{ast, hir}; + +/// This error type encompasses any error that can be returned by this crate. +/// +/// This error type is marked as `non_exhaustive`. This means that adding a +/// new variant is not considered a breaking change. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Error { + /// An error that occurred while translating concrete syntax into abstract + /// syntax (AST). + Parse(ast::Error), + /// An error that occurred while translating abstract syntax into a high + /// level intermediate representation (HIR). + Translate(hir::Error), +} + +impl From<ast::Error> for Error { + fn from(err: ast::Error) -> Error { + Error::Parse(err) + } +} + +impl From<hir::Error> for Error { + fn from(err: hir::Error) -> Error { + Error::Translate(err) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for Error {} + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match *self { + Error::Parse(ref x) => x.fmt(f), + Error::Translate(ref x) => x.fmt(f), + } + } +} + +/// A helper type for formatting nice error messages. +/// +/// This type is responsible for reporting regex parse errors in a nice human +/// readable format. Most of its complexity is from interspersing notational +/// markers pointing out the position where an error occurred. +#[derive(Debug)] +pub struct Formatter<'e, E> { + /// The original regex pattern in which the error occurred. + pattern: &'e str, + /// The error kind. It must impl fmt::Display. + err: &'e E, + /// The primary span of the error. + span: &'e ast::Span, + /// An auxiliary and optional span, in case the error needs to point to + /// two locations (e.g., when reporting a duplicate capture group name). + aux_span: Option<&'e ast::Span>, +} + +impl<'e> From<&'e ast::Error> for Formatter<'e, ast::ErrorKind> { + fn from(err: &'e ast::Error) -> Self { + Formatter { + pattern: err.pattern(), + err: err.kind(), + span: err.span(), + aux_span: err.auxiliary_span(), + } + } +} + +impl<'e> From<&'e hir::Error> for Formatter<'e, hir::ErrorKind> { + fn from(err: &'e hir::Error) -> Self { + Formatter { + pattern: err.pattern(), + err: err.kind(), + span: err.span(), + aux_span: None, + } + } +} + +impl<'e, E: core::fmt::Display> core::fmt::Display for Formatter<'e, E> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let spans = Spans::from_formatter(self); + if self.pattern.contains('\n') { + let divider = repeat_char('~', 79); + + writeln!(f, "regex parse error:")?; + writeln!(f, "{divider}")?; + let notated = spans.notate(); + write!(f, "{notated}")?; + writeln!(f, "{divider}")?; + // If we have error spans that cover multiple lines, then we just + // note the line numbers. + if !spans.multi_line.is_empty() { + let mut notes = vec![]; + for span in &spans.multi_line { + notes.push(format!( + "on line {} (column {}) through line {} (column {})", + span.start.line, + span.start.column, + span.end.line, + span.end.column - 1 + )); + } + writeln!(f, "{}", notes.join("\n"))?; + } + write!(f, "error: {}", self.err)?; + } else { + writeln!(f, "regex parse error:")?; + let notated = Spans::from_formatter(self).notate(); + write!(f, "{notated}")?; + write!(f, "error: {}", self.err)?; + } + Ok(()) + } +} + +/// This type represents an arbitrary number of error spans in a way that makes +/// it convenient to notate the regex pattern. ("Notate" means "point out +/// exactly where the error occurred in the regex pattern.") +/// +/// Technically, we can only ever have two spans given our current error +/// structure. However, after toiling with a specific algorithm for handling +/// two spans, it became obvious that an algorithm to handle an arbitrary +/// number of spans was actually much simpler. +struct Spans<'p> { + /// The original regex pattern string. + pattern: &'p str, + /// The total width that should be used for line numbers. The width is + /// used for left padding the line numbers for alignment. + /// + /// A value of `0` means line numbers should not be displayed. That is, + /// the pattern is itself only one line. + line_number_width: usize, + /// All error spans that occur on a single line. This sequence always has + /// length equivalent to the number of lines in `pattern`, where the index + /// of the sequence represents a line number, starting at `0`. The spans + /// in each line are sorted in ascending order. + by_line: Vec<Vec<ast::Span>>, + /// All error spans that occur over one or more lines. That is, the start + /// and end position of the span have different line numbers. The spans are + /// sorted in ascending order. + multi_line: Vec<ast::Span>, +} + +impl<'p> Spans<'p> { + /// Build a sequence of spans from a formatter. + fn from_formatter<'e, E: core::fmt::Display>( + fmter: &'p Formatter<'e, E>, + ) -> Spans<'p> { + let mut line_count = fmter.pattern.lines().count(); + // If the pattern ends with a `\n` literal, then our line count is + // off by one, since a span can occur immediately after the last `\n`, + // which is consider to be an additional line. + if fmter.pattern.ends_with('\n') { + line_count += 1; + } + let line_number_width = + if line_count <= 1 { 0 } else { line_count.to_string().len() }; + let mut spans = Spans { + pattern: &fmter.pattern, + line_number_width, + by_line: vec![vec![]; line_count], + multi_line: vec![], + }; + spans.add(fmter.span.clone()); + if let Some(span) = fmter.aux_span { + spans.add(span.clone()); + } + spans + } + + /// Add the given span to this sequence, putting it in the right place. + fn add(&mut self, span: ast::Span) { + // This is grossly inefficient since we sort after each add, but right + // now, we only ever add two spans at most. + if span.is_one_line() { + let i = span.start.line - 1; // because lines are 1-indexed + self.by_line[i].push(span); + self.by_line[i].sort(); + } else { + self.multi_line.push(span); + self.multi_line.sort(); + } + } + + /// Notate the pattern string with carets (`^`) pointing at each span + /// location. This only applies to spans that occur within a single line. + fn notate(&self) -> String { + let mut notated = String::new(); + for (i, line) in self.pattern.lines().enumerate() { + if self.line_number_width > 0 { + notated.push_str(&self.left_pad_line_number(i + 1)); + notated.push_str(": "); + } else { + notated.push_str(" "); + } + notated.push_str(line); + notated.push('\n'); + if let Some(notes) = self.notate_line(i) { + notated.push_str(¬es); + notated.push('\n'); + } + } + notated + } + + /// Return notes for the line indexed at `i` (zero-based). If there are no + /// spans for the given line, then `None` is returned. Otherwise, an + /// appropriately space padded string with correctly positioned `^` is + /// returned, accounting for line numbers. + fn notate_line(&self, i: usize) -> Option<String> { + let spans = &self.by_line[i]; + if spans.is_empty() { + return None; + } + let mut notes = String::new(); + for _ in 0..self.line_number_padding() { + notes.push(' '); + } + let mut pos = 0; + for span in spans { + for _ in pos..(span.start.column - 1) { + notes.push(' '); + pos += 1; + } + let note_len = span.end.column.saturating_sub(span.start.column); + for _ in 0..core::cmp::max(1, note_len) { + notes.push('^'); + pos += 1; + } + } + Some(notes) + } + + /// Left pad the given line number with spaces such that it is aligned with + /// other line numbers. + fn left_pad_line_number(&self, n: usize) -> String { + let n = n.to_string(); + let pad = self.line_number_width.checked_sub(n.len()).unwrap(); + let mut result = repeat_char(' ', pad); + result.push_str(&n); + result + } + + /// Return the line number padding beginning at the start of each line of + /// the pattern. + /// + /// If the pattern is only one line, then this returns a fixed padding + /// for visual indentation. + fn line_number_padding(&self) -> usize { + if self.line_number_width == 0 { + 4 + } else { + 2 + self.line_number_width + } + } +} + +fn repeat_char(c: char, count: usize) -> String { + core::iter::repeat(c).take(count).collect() +} + +#[cfg(test)] +mod tests { + use alloc::string::ToString; + + use crate::ast::parse::Parser; + + fn assert_panic_message(pattern: &str, expected_msg: &str) { + let result = Parser::new().parse(pattern); + match result { + Ok(_) => { + panic!("regex should not have parsed"); + } + Err(err) => { + assert_eq!(err.to_string(), expected_msg.trim()); + } + } + } + + // See: https://github.com/rust-lang/regex/issues/464 + #[test] + fn regression_464() { + let err = Parser::new().parse("a{\n").unwrap_err(); + // This test checks that the error formatter doesn't panic. + assert!(!err.to_string().is_empty()); + } + + // See: https://github.com/rust-lang/regex/issues/545 + #[test] + fn repetition_quantifier_expects_a_valid_decimal() { + assert_panic_message( + r"\\u{[^}]*}", + r#" +regex parse error: + \\u{[^}]*} + ^ +error: repetition quantifier expects a valid decimal +"#, + ); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/hir/interval.rs b/third_party/crates/regex-syntax-0.8.11/src/hir/interval.rs new file mode 100644 index 0000000..baec76d --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/hir/interval.rs @@ -0,0 +1,591 @@ +use core::{char, cmp, fmt::Debug, slice}; + +use alloc::vec::Vec; + +use crate::unicode; + +// This module contains an *internal* implementation of interval sets. +// +// The primary invariant that interval sets guards is canonical ordering. That +// is, every interval set contains an ordered sequence of intervals where +// no two intervals are overlapping or adjacent. While this invariant is +// occasionally broken within the implementation, it should be impossible for +// callers to observe it. +// +// Since case folding (as implemented below) breaks that invariant, we roll +// that into this API even though it is a little out of place in an otherwise +// generic interval set. (Hence the reason why the `unicode` module is imported +// here.) +// +// Some of the implementation complexity here is a result of me wanting to +// preserve the sequential representation without using additional memory. +// In many cases, we do use linear extra memory, but it is at most 2x and it +// is amortized. If we relaxed the memory requirements, this implementation +// could become much simpler. The extra memory is honestly probably OK, but +// character classes (especially of the Unicode variety) can become quite +// large, and it would be nice to keep regex compilation snappy even in debug +// builds. (In the past, I have been careless with this area of code and it has +// caused slow regex compilations in debug mode, so this isn't entirely +// unwarranted.) +// +// Tests on this are relegated to the public API of HIR in src/hir.rs. + +#[derive(Clone, Debug)] +pub struct IntervalSet<I> { + /// A sorted set of non-overlapping ranges. + ranges: Vec<I>, + /// While not required at all for correctness, we keep track of whether an + /// interval set has been case folded or not. This helps us avoid doing + /// redundant work if, for example, a set has already been cased folded. + /// And note that whether a set is folded or not is preserved through + /// all of the pairwise set operations. That is, if both interval sets + /// have been case folded, then any of difference, union, intersection or + /// symmetric difference all produce a case folded set. + /// + /// Note that when this is true, it *must* be the case that the set is case + /// folded. But when it's false, the set *may* be case folded. In other + /// words, we only set this to true when we know it to be case, but we're + /// okay with it being false if it would otherwise be costly to determine + /// whether it should be true. This means code cannot assume that a false + /// value necessarily indicates that the set is not case folded. + /// + /// Bottom line: this is a performance optimization. + folded: bool, +} + +impl<I: Interval> Eq for IntervalSet<I> {} + +// We implement PartialEq manually so that we don't consider the set's internal +// 'folded' property to be part of its identity. The 'folded' property is +// strictly an optimization. +impl<I: Interval> PartialEq for IntervalSet<I> { + fn eq(&self, other: &IntervalSet<I>) -> bool { + self.ranges.eq(&other.ranges) + } +} + +impl<I: Interval> IntervalSet<I> { + /// Create a new set from a sequence of intervals. Each interval is + /// specified as a pair of bounds, where both bounds are inclusive. + /// + /// The given ranges do not need to be in any specific order, and ranges + /// may overlap. + pub fn new<T: IntoIterator<Item = I>>(intervals: T) -> IntervalSet<I> { + let ranges: Vec<I> = intervals.into_iter().collect(); + // An empty set is case folded. + let folded = ranges.is_empty(); + let mut set = IntervalSet { ranges, folded }; + set.canonicalize(); + set + } + + /// Add a new interval to this set. + pub fn push(&mut self, mut interval: I) { + let Err(i) = self.ranges.binary_search(&interval) else { + // Exact match, `interval` is already in the set. + return; + }; + + // The search finds us the first index where the previous interval + // start is less than or equal to the new interval start. Since the + // existing intervals are non-overlapping we only need to try to union + // this single preceding interval + let mut start = i; + if let Some(before_i) = i.checked_sub(1) { + let before = &self.ranges[before_i]; + if let Some(union) = before.union(&interval) { + interval = union; + start = before_i; + } + } + // `interval` may overlap any number of intervals following the + // insertion point so will union each of them until we reach the + // first non-overlapping interval + let mut end = i; + for after_i in i..self.ranges.len() { + let after = &self.ranges[after_i]; + let Some(union) = interval.union(after) else { + break; + }; + interval = union; + end = after_i + 1; + } + self.ranges.splice(start..end, core::iter::once(interval)); + + // We don't know whether the new interval added here is considered + // case folded, so we conservatively assume that the entire set is + // no longer case folded if it was previously. + self.folded = false; + } + + /// Return an iterator over all intervals in this set. + /// + /// The iterator yields intervals in ascending order. + pub fn iter(&self) -> IntervalSetIter<'_, I> { + IntervalSetIter(self.ranges.iter()) + } + + /// Return an immutable slice of intervals in this set. + /// + /// The sequence returned is in canonical ordering. + pub fn intervals(&self) -> &[I] { + &self.ranges + } + + /// Expand this interval set such that it contains all case folded + /// characters. For example, if this class consists of the range `a-z`, + /// then applying case folding will result in the class containing both the + /// ranges `a-z` and `A-Z`. + /// + /// This returns an error if the necessary case mapping data is not + /// available. + pub fn case_fold_simple(&mut self) -> Result<(), unicode::CaseFoldError> { + if self.folded { + return Ok(()); + } + let len = self.ranges.len(); + for i in 0..len { + let range = self.ranges[i]; + if let Err(err) = range.case_fold_simple(&mut self.ranges) { + self.canonicalize(); + return Err(err); + } + } + self.canonicalize(); + self.folded = true; + Ok(()) + } + + /// Union this set with the given set, in place. + pub fn union(&mut self, other: &IntervalSet<I>) { + if other.ranges.is_empty() || self.ranges == other.ranges { + return; + } + // This could almost certainly be done more efficiently. + self.ranges.extend(&other.ranges); + self.canonicalize(); + self.folded = self.folded && other.folded; + } + + /// Intersect this set with the given set, in place. + pub fn intersect(&mut self, other: &IntervalSet<I>) { + if self.ranges.is_empty() { + return; + } + if other.ranges.is_empty() { + self.ranges.clear(); + // An empty set is case folded. + self.folded = true; + return; + } + + // There should be a way to do this in-place with constant memory, + // but I couldn't figure out a simple way to do it. So just append + // the intersection to the end of this range, and then drain it before + // we're done. + let drain_end = self.ranges.len(); + + let mut ita = 0..drain_end; + let mut itb = 0..other.ranges.len(); + let mut a = ita.next().unwrap(); + let mut b = itb.next().unwrap(); + loop { + if let Some(ab) = self.ranges[a].intersect(&other.ranges[b]) { + self.ranges.push(ab); + } + let (it, aorb) = + if self.ranges[a].upper() < other.ranges[b].upper() { + (&mut ita, &mut a) + } else { + (&mut itb, &mut b) + }; + match it.next() { + Some(v) => *aorb = v, + None => break, + } + } + self.ranges.drain(..drain_end); + self.folded = self.folded && other.folded; + } + + /// Subtract the given set from this set, in place. + pub fn difference(&mut self, other: &IntervalSet<I>) { + if self.ranges.is_empty() || other.ranges.is_empty() { + return; + } + + // This algorithm is (to me) surprisingly complex. A search of the + // interwebs indicate that this is a potentially interesting problem. + // Folks seem to suggest interval or segment trees, but I'd like to + // avoid the overhead (both runtime and conceptual) of that. + // + // The following is basically my Shitty First Draft. Therefore, in + // order to grok it, you probably need to read each line carefully. + // Simplifications are most welcome! + // + // Remember, we can assume the canonical format invariant here, which + // says that all ranges are sorted, not overlapping and not adjacent in + // each class. + let drain_end = self.ranges.len(); + let (mut a, mut b) = (0, 0); + 'LOOP: while a < drain_end && b < other.ranges.len() { + // Basically, the easy cases are when neither range overlaps with + // each other. If the `b` range is less than our current `a` + // range, then we can skip it and move on. + if other.ranges[b].upper() < self.ranges[a].lower() { + b += 1; + continue; + } + // ... similarly for the `a` range. If it's less than the smallest + // `b` range, then we can add it as-is. + if self.ranges[a].upper() < other.ranges[b].lower() { + let range = self.ranges[a]; + self.ranges.push(range); + a += 1; + continue; + } + // Otherwise, we have overlapping ranges. + assert!(!self.ranges[a].is_intersection_empty(&other.ranges[b])); + + // This part is tricky and was non-obvious to me without looking + // at explicit examples (see the tests). The trickiness stems from + // two things: 1) subtracting a range from another range could + // yield two ranges and 2) after subtracting a range, it's possible + // that future ranges can have an impact. The loop below advances + // the `b` ranges until they can't possible impact the current + // range. + // + // For example, if our `a` range is `a-t` and our next three `b` + // ranges are `a-c`, `g-i`, `r-t` and `x-z`, then we need to apply + // subtraction three times before moving on to the next `a` range. + let mut range = self.ranges[a]; + while b < other.ranges.len() + && !range.is_intersection_empty(&other.ranges[b]) + { + let old_range = range; + range = match range.difference(&other.ranges[b]) { + (None, None) => { + // We lost the entire range, so move on to the next + // without adding this one. + a += 1; + continue 'LOOP; + } + (Some(range1), None) | (None, Some(range1)) => range1, + (Some(range1), Some(range2)) => { + self.ranges.push(range1); + range2 + } + }; + // It's possible that the `b` range has more to contribute + // here. In particular, if it is greater than the original + // range, then it might impact the next `a` range *and* it + // has impacted the current `a` range as much as possible, + // so we can quit. We don't bump `b` so that the next `a` + // range can apply it. + if other.ranges[b].upper() > old_range.upper() { + break; + } + // Otherwise, the next `b` range might apply to the current + // `a` range. + b += 1; + } + self.ranges.push(range); + a += 1; + } + while a < drain_end { + let range = self.ranges[a]; + self.ranges.push(range); + a += 1; + } + self.ranges.drain(..drain_end); + self.folded = self.folded && other.folded; + } + + /// Compute the symmetric difference of the two sets, in place. + /// + /// This computes the symmetric difference of two interval sets. This + /// removes all elements in this set that are also in the given set, + /// but also adds all elements from the given set that aren't in this + /// set. That is, the set will contain all elements in either set, + /// but will not contain any elements that are in both sets. + pub fn symmetric_difference(&mut self, other: &IntervalSet<I>) { + // TODO(burntsushi): Fix this so that it amortizes allocation. + let mut intersection = self.clone(); + intersection.intersect(other); + self.union(other); + self.difference(&intersection); + } + + /// Negate this interval set. + /// + /// For all `x` where `x` is any element, if `x` was in this set, then it + /// will not be in this set after negation. + pub fn negate(&mut self) { + if self.ranges.is_empty() { + let (min, max) = (I::Bound::min_value(), I::Bound::max_value()); + self.ranges.push(I::create(min, max)); + // The set containing everything must case folded. + self.folded = true; + return; + } + + // There should be a way to do this in-place with constant memory, + // but I couldn't figure out a simple way to do it. So just append + // the negation to the end of this range, and then drain it before + // we're done. + let drain_end = self.ranges.len(); + + // We do checked arithmetic below because of the canonical ordering + // invariant. + if self.ranges[0].lower() > I::Bound::min_value() { + let upper = self.ranges[0].lower().decrement(); + self.ranges.push(I::create(I::Bound::min_value(), upper)); + } + for i in 1..drain_end { + let lower = self.ranges[i - 1].upper().increment(); + let upper = self.ranges[i].lower().decrement(); + self.ranges.push(I::create(lower, upper)); + } + if self.ranges[drain_end - 1].upper() < I::Bound::max_value() { + let lower = self.ranges[drain_end - 1].upper().increment(); + self.ranges.push(I::create(lower, I::Bound::max_value())); + } + self.ranges.drain(..drain_end); + // We don't need to update whether this set is folded or not, because + // it is conservatively preserved through negation. Namely, if a set + // is not folded, then it is possible that its negation is folded, for + // example, [^☃]. But we're fine with assuming that the set is not + // folded in that case. (`folded` permits false negatives but not false + // positives.) + // + // But what about when a set is folded, is its negation also + // necessarily folded? Yes. Because if a set is folded, then for every + // character in the set, it necessarily included its equivalence class + // of case folded characters. Negating it in turn means that all + // equivalence classes in the set are negated, and any equivalence + // class that was previously not in the set is now entirely in the set. + } + + /// Converts this set into a canonical ordering. + fn canonicalize(&mut self) { + if self.is_canonical() { + return; + } + self.ranges.sort(); + assert!(!self.ranges.is_empty()); + + // Is there a way to do this in-place with constant memory? I couldn't + // figure out a way to do it. So just append the canonicalization to + // the end of this range, and then drain it before we're done. + let drain_end = self.ranges.len(); + for oldi in 0..drain_end { + // If we've added at least one new range, then check if we can + // merge this range in the previously added range. + if self.ranges.len() > drain_end { + let (last, rest) = self.ranges.split_last_mut().unwrap(); + if let Some(union) = last.union(&rest[oldi]) { + *last = union; + continue; + } + } + let range = self.ranges[oldi]; + self.ranges.push(range); + } + self.ranges.drain(..drain_end); + } + + /// Returns true if and only if this class is in a canonical ordering. + fn is_canonical(&self) -> bool { + for pair in self.ranges.windows(2) { + if pair[0] >= pair[1] { + return false; + } + if pair[0].is_contiguous(&pair[1]) { + return false; + } + } + true + } +} + +/// An iterator over intervals. +#[derive(Debug)] +pub struct IntervalSetIter<'a, I>(slice::Iter<'a, I>); + +impl<'a, I> Iterator for IntervalSetIter<'a, I> { + type Item = &'a I; + + fn next(&mut self) -> Option<&'a I> { + self.0.next() + } +} + +pub trait Interval: + Clone + Copy + Debug + Default + Eq + PartialEq + PartialOrd + Ord +{ + type Bound: Bound; + + fn lower(&self) -> Self::Bound; + fn upper(&self) -> Self::Bound; + fn set_lower(&mut self, bound: Self::Bound); + fn set_upper(&mut self, bound: Self::Bound); + fn case_fold_simple( + &self, + intervals: &mut Vec<Self>, + ) -> Result<(), unicode::CaseFoldError>; + + /// Create a new interval. + fn create(lower: Self::Bound, upper: Self::Bound) -> Self { + let mut int = Self::default(); + if lower <= upper { + int.set_lower(lower); + int.set_upper(upper); + } else { + int.set_lower(upper); + int.set_upper(lower); + } + int + } + + /// Union the given overlapping range into this range. + /// + /// If the two ranges aren't contiguous, then this returns `None`. + fn union(&self, other: &Self) -> Option<Self> { + if !self.is_contiguous(other) { + return None; + } + let lower = cmp::min(self.lower(), other.lower()); + let upper = cmp::max(self.upper(), other.upper()); + Some(Self::create(lower, upper)) + } + + /// Intersect this range with the given range and return the result. + /// + /// If the intersection is empty, then this returns `None`. + fn intersect(&self, other: &Self) -> Option<Self> { + let lower = cmp::max(self.lower(), other.lower()); + let upper = cmp::min(self.upper(), other.upper()); + if lower <= upper { + Some(Self::create(lower, upper)) + } else { + None + } + } + + /// Subtract the given range from this range and return the resulting + /// ranges. + /// + /// If subtraction would result in an empty range, then no ranges are + /// returned. + fn difference(&self, other: &Self) -> (Option<Self>, Option<Self>) { + if self.is_subset(other) { + return (None, None); + } + if self.is_intersection_empty(other) { + return (Some(self.clone()), None); + } + let add_lower = other.lower() > self.lower(); + let add_upper = other.upper() < self.upper(); + // We know this because !self.is_subset(other) and the ranges have + // a non-empty intersection. + assert!(add_lower || add_upper); + let mut ret = (None, None); + if add_lower { + let upper = other.lower().decrement(); + ret.0 = Some(Self::create(self.lower(), upper)); + } + if add_upper { + let lower = other.upper().increment(); + let range = Self::create(lower, self.upper()); + if ret.0.is_none() { + ret.0 = Some(range); + } else { + ret.1 = Some(range); + } + } + ret + } + + /// Returns true if and only if the two ranges are contiguous. Two ranges + /// are contiguous if and only if the ranges are either overlapping or + /// adjacent. + fn is_contiguous(&self, other: &Self) -> bool { + let lower1 = self.lower().as_u32(); + let upper1 = self.upper().as_u32(); + let lower2 = other.lower().as_u32(); + let upper2 = other.upper().as_u32(); + cmp::max(lower1, lower2) <= cmp::min(upper1, upper2).saturating_add(1) + } + + /// Returns true if and only if the intersection of this range and the + /// other range is empty. + fn is_intersection_empty(&self, other: &Self) -> bool { + let (lower1, upper1) = (self.lower(), self.upper()); + let (lower2, upper2) = (other.lower(), other.upper()); + cmp::max(lower1, lower2) > cmp::min(upper1, upper2) + } + + /// Returns true if and only if this range is a subset of the other range. + fn is_subset(&self, other: &Self) -> bool { + let (lower1, upper1) = (self.lower(), self.upper()); + let (lower2, upper2) = (other.lower(), other.upper()); + (lower2 <= lower1 && lower1 <= upper2) + && (lower2 <= upper1 && upper1 <= upper2) + } +} + +pub trait Bound: + Copy + Clone + Debug + Eq + PartialEq + PartialOrd + Ord +{ + fn min_value() -> Self; + fn max_value() -> Self; + fn as_u32(self) -> u32; + fn increment(self) -> Self; + fn decrement(self) -> Self; +} + +impl Bound for u8 { + fn min_value() -> Self { + u8::MIN + } + fn max_value() -> Self { + u8::MAX + } + fn as_u32(self) -> u32 { + u32::from(self) + } + fn increment(self) -> Self { + self.checked_add(1).unwrap() + } + fn decrement(self) -> Self { + self.checked_sub(1).unwrap() + } +} + +impl Bound for char { + fn min_value() -> Self { + '\x00' + } + fn max_value() -> Self { + '\u{10FFFF}' + } + fn as_u32(self) -> u32 { + u32::from(self) + } + + fn increment(self) -> Self { + match self { + '\u{D7FF}' => '\u{E000}', + c => char::from_u32(u32::from(c).checked_add(1).unwrap()).unwrap(), + } + } + + fn decrement(self) -> Self { + match self { + '\u{E000}' => '\u{D7FF}', + c => char::from_u32(u32::from(c).checked_sub(1).unwrap()).unwrap(), + } + } +} + +// Tests for interval sets are written in src/hir.rs against the public API. diff --git a/third_party/crates/regex-syntax-0.8.11/src/hir/literal.rs b/third_party/crates/regex-syntax-0.8.11/src/hir/literal.rs new file mode 100644 index 0000000..2a6350e --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/hir/literal.rs @@ -0,0 +1,3214 @@ +/*! +Provides literal extraction from `Hir` expressions. + +An [`Extractor`] pulls literals out of [`Hir`] expressions and returns a +[`Seq`] of [`Literal`]s. + +The purpose of literal extraction is generally to provide avenues for +optimizing regex searches. The main idea is that substring searches can be an +order of magnitude faster than a regex search. Therefore, if one can execute +a substring search to find candidate match locations and only run the regex +search at those locations, then it is possible for huge improvements in +performance to be realized. + +With that said, literal optimizations are generally a black art because even +though substring search is generally faster, if the number of candidates +produced is high, then it can create a lot of overhead by ping-ponging between +the substring search and the regex search. + +Here are some heuristics that might be used to help increase the chances of +effective literal optimizations: + +* Stick to small [`Seq`]s. If you search for too many literals, it's likely +to lead to substring search that is only a little faster than a regex search, +and thus the overhead of using literal optimizations in the first place might +make things slower overall. +* The literals in your [`Seq`] shouldn't be too short. In general, longer is +better. A sequence corresponding to single bytes that occur frequently in the +haystack, for example, is probably a bad literal optimization because it's +likely to produce many false positive candidates. Longer literals are less +likely to match, and thus probably produce fewer false positives. +* If it's possible to estimate the approximate frequency of each byte according +to some pre-computed background distribution, it is possible to compute a score +of how "good" a `Seq` is. If a `Seq` isn't good enough, you might consider +skipping the literal optimization and just use the regex engine. + +(It should be noted that there are always pathological cases that can make +any kind of literal optimization be a net slower result. This is why it +might be a good idea to be conservative, or to even provide a means for +literal optimizations to be dynamically disabled if they are determined to be +ineffective according to some measure.) + +You're encouraged to explore the methods on [`Seq`], which permit shrinking +the size of sequences in a preference-order preserving fashion. + +Finally, note that it isn't strictly necessary to use an [`Extractor`]. Namely, +an `Extractor` only uses public APIs of the [`Seq`] and [`Literal`] types, +so it is possible to implement your own extractor. For example, for n-grams +or "inner" literals (i.e., not prefix or suffix literals). The `Extractor` +is mostly responsible for the case analysis over `Hir` expressions. Much of +the "trickier" parts are how to combine literal sequences, and that is all +implemented on [`Seq`]. +*/ + +use core::{cmp, mem, num::NonZeroUsize}; + +use alloc::{vec, vec::Vec}; + +use crate::hir::{self, Hir}; + +/// Extracts prefix or suffix literal sequences from [`Hir`] expressions. +/// +/// Literal extraction is based on the following observations: +/// +/// * Many regexes start with one or a small number of literals. +/// * Substring search for literals is often much faster (sometimes by an order +/// of magnitude) than a regex search. +/// +/// Thus, in many cases, one can search for literals to find candidate starting +/// locations of a match, and then only run the full regex engine at each such +/// location instead of over the full haystack. +/// +/// The main downside of literal extraction is that it can wind up causing a +/// search to be slower overall. For example, if there are many matches or if +/// there are many candidates that don't ultimately lead to a match, then a +/// lot of overhead will be spent in shuffling back-and-forth between substring +/// search and the regex engine. This is the fundamental reason why literal +/// optimizations for regex patterns is sometimes considered a "black art." +/// +/// # Look-around assertions +/// +/// Literal extraction treats all look-around assertions as-if they match every +/// empty string. So for example, the regex `\bquux\b` will yield a sequence +/// containing a single exact literal `quux`. However, not all occurrences +/// of `quux` correspond to a match a of the regex. For example, `\bquux\b` +/// does not match `ZquuxZ` anywhere because `quux` does not fall on a word +/// boundary. +/// +/// In effect, if your regex contains look-around assertions, then a match of +/// an exact literal does not necessarily mean the regex overall matches. So +/// you may still need to run the regex engine in such cases to confirm the +/// match. +/// +/// The precise guarantee you get from a literal sequence is: if every literal +/// in the sequence is exact and the original regex contains zero look-around +/// assertions, then a preference-order multi-substring search of those +/// literals will precisely match a preference-order search of the original +/// regex. +/// +/// # Example +/// +/// This shows how to extract prefixes: +/// +/// ``` +/// use regex_syntax::{hir::literal::{Extractor, Literal, Seq}, parse}; +/// +/// let hir = parse(r"(a|b|c)(x|y|z)[A-Z]+foo")?; +/// let got = Extractor::new().extract(&hir); +/// // All literals returned are "inexact" because none of them reach the +/// // match state. +/// let expected = Seq::from_iter([ +/// Literal::inexact("ax"), +/// Literal::inexact("ay"), +/// Literal::inexact("az"), +/// Literal::inexact("bx"), +/// Literal::inexact("by"), +/// Literal::inexact("bz"), +/// Literal::inexact("cx"), +/// Literal::inexact("cy"), +/// Literal::inexact("cz"), +/// ]); +/// assert_eq!(expected, got); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +/// +/// This shows how to extract suffixes: +/// +/// ``` +/// use regex_syntax::{ +/// hir::literal::{Extractor, ExtractKind, Literal, Seq}, +/// parse, +/// }; +/// +/// let hir = parse(r"foo|[A-Z]+bar")?; +/// let got = Extractor::new().kind(ExtractKind::Suffix).extract(&hir); +/// // Since 'foo' gets to a match state, it is considered exact. But 'bar' +/// // does not because of the '[A-Z]+', and thus is marked inexact. +/// let expected = Seq::from_iter([ +/// Literal::exact("foo"), +/// Literal::inexact("bar"), +/// ]); +/// assert_eq!(expected, got); +/// +/// # Ok::<(), Box<dyn std::error::Error>>(()) +/// ``` +#[derive(Clone, Debug)] +pub struct Extractor { + kind: ExtractKind, + limit_class: usize, + limit_repeat: usize, + limit_literal_len: usize, + limit_total: usize, +} + +impl Extractor { + /// Create a new extractor with a default configuration. + /// + /// The extractor can be optionally configured before calling + /// [`Extractor::extract`] to get a literal sequence. + pub fn new() -> Extractor { + Extractor { + kind: ExtractKind::Prefix, + limit_class: 10, + limit_repeat: 10, + limit_literal_len: 100, + limit_total: 250, + } + } + + /// Execute the extractor and return a sequence of literals. + pub fn extract(&self, hir: &Hir) -> Seq { + use crate::hir::HirKind::*; + + match *hir.kind() { + Empty | Look(_) => Seq::singleton(self::Literal::exact(vec![])), + Literal(hir::Literal(ref bytes)) => { + let mut seq = + Seq::singleton(self::Literal::exact(bytes.to_vec())); + self.enforce_literal_len(&mut seq); + seq + } + Class(hir::Class::Unicode(ref cls)) => { + self.extract_class_unicode(cls) + } + Class(hir::Class::Bytes(ref cls)) => self.extract_class_bytes(cls), + Repetition(ref rep) => self.extract_repetition(rep), + Capture(hir::Capture { ref sub, .. }) => self.extract(sub), + Concat(ref hirs) => match self.kind { + ExtractKind::Prefix => self.extract_concat(hirs.iter()), + ExtractKind::Suffix => self.extract_concat(hirs.iter().rev()), + }, + Alternation(ref hirs) => { + // Unlike concat, we always union starting from the beginning, + // since the beginning corresponds to the highest preference, + // which doesn't change based on forwards vs reverse. + self.extract_alternation(hirs.iter()) + } + } + } + + /// Set the kind of literal sequence to extract from an [`Hir`] expression. + /// + /// The default is to extract prefixes, but suffixes can be selected + /// instead. The contract for prefixes is that every match of the + /// corresponding `Hir` must start with one of the literals in the sequence + /// returned. Moreover, the _order_ of the sequence returned corresponds to + /// the preference order. + /// + /// Suffixes satisfy a similar contract in that every match of the + /// corresponding `Hir` must end with one of the literals in the sequence + /// returned. However, there is no guarantee that the literals are in + /// preference order. + /// + /// Remember that a sequence can be infinite. For example, unless the + /// limits are configured to be impractically large, attempting to extract + /// prefixes (or suffixes) for the pattern `[A-Z]` will return an infinite + /// sequence. Generally speaking, if the sequence returned is infinite, + /// then it is presumed to be unwise to do prefix (or suffix) optimizations + /// for the pattern. + pub fn kind(&mut self, kind: ExtractKind) -> &mut Extractor { + self.kind = kind; + self + } + + /// Configure a limit on the length of the sequence that is permitted for + /// a character class. If a character class exceeds this limit, then the + /// sequence returned for it is infinite. + /// + /// This prevents classes like `[A-Z]` or `\pL` from getting turned into + /// huge and likely unproductive sequences of literals. + /// + /// # Example + /// + /// This example shows how this limit can be lowered to decrease the tolerance + /// for character classes being turned into literal sequences. + /// + /// ``` + /// use regex_syntax::{hir::literal::{Extractor, Seq}, parse}; + /// + /// let hir = parse(r"[0-9]")?; + /// + /// let got = Extractor::new().extract(&hir); + /// let expected = Seq::new([ + /// "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", + /// ]); + /// assert_eq!(expected, got); + /// + /// // Now let's shrink the limit and see how that changes things. + /// let got = Extractor::new().limit_class(4).extract(&hir); + /// let expected = Seq::infinite(); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn limit_class(&mut self, limit: usize) -> &mut Extractor { + self.limit_class = limit; + self + } + + /// Configure a limit on the total number of repetitions that is permitted + /// before literal extraction is stopped. + /// + /// This is useful for limiting things like `(abcde){50}`, or more + /// insidiously, `(?:){1000000000}`. This limit prevents any one single + /// repetition from adding too much to a literal sequence. + /// + /// With this limit set, repetitions that exceed it will be stopped and any + /// literals extracted up to that point will be made inexact. + /// + /// # Example + /// + /// This shows how to decrease the limit and compares it with the default. + /// + /// ``` + /// use regex_syntax::{hir::literal::{Extractor, Literal, Seq}, parse}; + /// + /// let hir = parse(r"(abc){8}")?; + /// + /// let got = Extractor::new().extract(&hir); + /// let expected = Seq::new(["abcabcabcabcabcabcabcabc"]); + /// assert_eq!(expected, got); + /// + /// // Now let's shrink the limit and see how that changes things. + /// let got = Extractor::new().limit_repeat(4).extract(&hir); + /// let expected = Seq::from_iter([ + /// Literal::inexact("abcabcabcabc"), + /// ]); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn limit_repeat(&mut self, limit: usize) -> &mut Extractor { + self.limit_repeat = limit; + self + } + + /// Configure a limit on the maximum length of any literal in a sequence. + /// + /// This is useful for limiting things like `(abcde){5}{5}{5}{5}`. While + /// each repetition or literal in that regex is small, when all the + /// repetitions are applied, one ends up with a literal of length `5^4 = + /// 625`. + /// + /// With this limit set, literals that exceed it will be made inexact and + /// thus prevented from growing. + /// + /// # Example + /// + /// This shows how to decrease the limit and compares it with the default. + /// + /// ``` + /// use regex_syntax::{hir::literal::{Extractor, Literal, Seq}, parse}; + /// + /// let hir = parse(r"(abc){2}{2}{2}")?; + /// + /// let got = Extractor::new().extract(&hir); + /// let expected = Seq::new(["abcabcabcabcabcabcabcabc"]); + /// assert_eq!(expected, got); + /// + /// // Now let's shrink the limit and see how that changes things. + /// let got = Extractor::new().limit_literal_len(14).extract(&hir); + /// let expected = Seq::from_iter([ + /// Literal::inexact("abcabcabcabcab"), + /// ]); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn limit_literal_len(&mut self, limit: usize) -> &mut Extractor { + self.limit_literal_len = limit; + self + } + + /// Configure a limit on the total number of literals that will be + /// returned. + /// + /// This is useful as a practical measure for avoiding the creation of + /// large sequences of literals. While the extractor will automatically + /// handle local creations of large sequences (for example, `[A-Z]` yields + /// an infinite sequence by default), large sequences can be created + /// through non-local means as well. + /// + /// For example, `[ab]{3}{3}` would yield a sequence of length `512 = 2^9` + /// despite each of the repetitions being small on their own. This limit + /// thus represents a "catch all" for avoiding locally small sequences from + /// combining into large sequences. + /// + /// # Example + /// + /// This example shows how reducing the limit will change the literal + /// sequence returned. + /// + /// ``` + /// use regex_syntax::{hir::literal::{Extractor, Literal, Seq}, parse}; + /// + /// let hir = parse(r"[ab]{2}{2}")?; + /// + /// let got = Extractor::new().extract(&hir); + /// let expected = Seq::new([ + /// "aaaa", "aaab", "aaba", "aabb", + /// "abaa", "abab", "abba", "abbb", + /// "baaa", "baab", "baba", "babb", + /// "bbaa", "bbab", "bbba", "bbbb", + /// ]); + /// assert_eq!(expected, got); + /// + /// // The default limit is not too big, but big enough to extract all + /// // literals from '[ab]{2}{2}'. If we shrink the limit to less than 16, + /// // then we'll get a truncated set. Notice that it returns a sequence of + /// // length 4 even though our limit was 10. This is because the sequence + /// // is difficult to increase without blowing the limit. Notice also + /// // that every literal in the sequence is now inexact because they were + /// // stripped of some suffix. + /// let got = Extractor::new().limit_total(10).extract(&hir); + /// let expected = Seq::from_iter([ + /// Literal::inexact("aa"), + /// Literal::inexact("ab"), + /// Literal::inexact("ba"), + /// Literal::inexact("bb"), + /// ]); + /// assert_eq!(expected, got); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn limit_total(&mut self, limit: usize) -> &mut Extractor { + self.limit_total = limit; + self + } + + /// Extract a sequence from the given concatenation. Sequences from each of + /// the child HIR expressions are combined via cross product. + /// + /// This short circuits once the cross product turns into a sequence + /// containing only inexact literals. + fn extract_concat<'a, I: Iterator<Item = &'a Hir>>(&self, it: I) -> Seq { + let mut seq = Seq::singleton(self::Literal::exact(vec![])); + for hir in it { + // If every element in the sequence is inexact, then a cross + // product will always be a no-op. Thus, there is nothing else we + // can add to it and can quit early. Note that this also includes + // infinite sequences. + if seq.is_inexact() { + break; + } + // Note that 'cross' also dispatches based on whether we're + // extracting prefixes or suffixes. + seq = self.cross(seq, &mut self.extract(hir)); + } + seq + } + + /// Extract a sequence from the given alternation. + /// + /// This short circuits once the union turns into an infinite sequence. + fn extract_alternation<'a, I: Iterator<Item = &'a Hir>>( + &self, + it: I, + ) -> Seq { + let mut seq = Seq::empty(); + for hir in it { + // Once our 'seq' is infinite, every subsequent union + // operation on it will itself always result in an + // infinite sequence. Thus, it can never change and we can + // short-circuit. + if !seq.is_finite() { + break; + } + seq = self.union(seq, &mut self.extract(hir)); + } + seq + } + + /// Extract a sequence of literals from the given repetition. We do our + /// best, Some examples: + /// + /// 'a*' => [inexact(a), exact("")] + /// 'a*?' => [exact(""), inexact(a)] + /// 'a+' => [inexact(a)] + /// 'a{3}' => [exact(aaa)] + /// 'a{3,5} => [inexact(aaa)] + /// + /// The key here really is making sure we get the 'inexact' vs 'exact' + /// attributes correct on each of the literals we add. For example, the + /// fact that 'a*' gives us an inexact 'a' and an exact empty string means + /// that a regex like 'ab*c' will result in [inexact(ab), exact(ac)] + /// literals being extracted, which might actually be a better prefilter + /// than just 'a'. + fn extract_repetition(&self, rep: &hir::Repetition) -> Seq { + let mut subseq = self.extract(&rep.sub); + match *rep { + hir::Repetition { min: 0, max, greedy, .. } => { + // When 'max=1', we can retain exactness, since 'a?' is + // equivalent to 'a|'. Similarly below, 'a??' is equivalent to + // '|a'. + if max != Some(1) { + subseq.make_inexact(); + } + let mut empty = Seq::singleton(Literal::exact(vec![])); + if !greedy { + mem::swap(&mut subseq, &mut empty); + } + self.union(subseq, &mut empty) + } + hir::Repetition { min, max: Some(max), .. } if min == max => { + assert!(min > 0); // handled above + let limit = + u32::try_from(self.limit_repeat).unwrap_or(u32::MAX); + let mut seq = Seq::singleton(Literal::exact(vec![])); + for _ in 0..cmp::min(min, limit) { + if seq.is_inexact() { + break; + } + seq = self.cross(seq, &mut subseq.clone()); + } + if usize::try_from(min).is_err() || min > limit { + seq.make_inexact(); + } + seq + } + hir::Repetition { min, .. } => { + assert!(min > 0); // handled above + let limit = + u32::try_from(self.limit_repeat).unwrap_or(u32::MAX); + let mut seq = Seq::singleton(Literal::exact(vec![])); + for _ in 0..cmp::min(min, limit) { + if seq.is_inexact() { + break; + } + seq = self.cross(seq, &mut subseq.clone()); + } + seq.make_inexact(); + seq + } + } + } + + /// Convert the given Unicode class into a sequence of literals if the + /// class is small enough. If the class is too big, return an infinite + /// sequence. + fn extract_class_unicode(&self, cls: &hir::ClassUnicode) -> Seq { + if self.class_over_limit_unicode(cls) { + return Seq::infinite(); + } + let mut seq = Seq::empty(); + for r in cls.iter() { + for ch in r.start()..=r.end() { + seq.push(Literal::from(ch)); + } + } + self.enforce_literal_len(&mut seq); + seq + } + + /// Convert the given byte class into a sequence of literals if the class + /// is small enough. If the class is too big, return an infinite sequence. + fn extract_class_bytes(&self, cls: &hir::ClassBytes) -> Seq { + if self.class_over_limit_bytes(cls) { + return Seq::infinite(); + } + let mut seq = Seq::empty(); + for r in cls.iter() { + for b in r.start()..=r.end() { + seq.push(Literal::from(b)); + } + } + self.enforce_literal_len(&mut seq); + seq + } + + /// Returns true if the given Unicode class exceeds the configured limits + /// on this extractor. + fn class_over_limit_unicode(&self, cls: &hir::ClassUnicode) -> bool { + let mut count = 0; + for r in cls.iter() { + if count > self.limit_class { + return true; + } + count += r.len(); + } + count > self.limit_class + } + + /// Returns true if the given byte class exceeds the configured limits on + /// this extractor. + fn class_over_limit_bytes(&self, cls: &hir::ClassBytes) -> bool { + let mut count = 0; + for r in cls.iter() { + if count > self.limit_class { + return true; + } + count += r.len(); + } + count > self.limit_class + } + + /// Compute the cross product of the two sequences if the result would be + /// within configured limits. Otherwise, make `seq2` infinite and cross the + /// infinite sequence with `seq1`. + fn cross(&self, mut seq1: Seq, seq2: &mut Seq) -> Seq { + if seq1.max_cross_len(seq2).map_or(false, |len| len > self.limit_total) + { + seq2.make_infinite(); + } + if let ExtractKind::Suffix = self.kind { + seq1.cross_reverse(seq2); + } else { + seq1.cross_forward(seq2); + } + assert!(seq1.len().map_or(true, |x| x <= self.limit_total)); + self.enforce_literal_len(&mut seq1); + seq1 + } + + /// Union the two sequences if the result would be within configured + /// limits. Otherwise, make `seq2` infinite and union the infinite sequence + /// with `seq1`. + fn union(&self, mut seq1: Seq, seq2: &mut Seq) -> Seq { + if seq1.max_union_len(seq2).map_or(false, |len| len > self.limit_total) + { + // We try to trim our literal sequences to see if we can make + // room for more literals. The idea is that we'd rather trim down + // literals already in our sequence if it means we can add a few + // more and retain a finite sequence. Otherwise, we'll union with + // an infinite sequence and that infects everything and effectively + // stops literal extraction in its tracks. + // + // We do we keep 4 bytes here? Well, it's a bit of an abstraction + // leakage. Downstream, the literals may wind up getting fed to + // the Teddy algorithm, which supports searching literals up to + // length 4. So that's why we pick that number here. Arguably this + // should be a tunable parameter, but it seems a little tricky to + // describe. And I'm still unsure if this is the right way to go + // about culling literal sequences. + match self.kind { + ExtractKind::Prefix => { + seq1.keep_first_bytes(4); + seq2.keep_first_bytes(4); + } + ExtractKind::Suffix => { + seq1.keep_last_bytes(4); + seq2.keep_last_bytes(4); + } + } + seq1.dedup(); + seq2.dedup(); + if seq1 + .max_union_len(seq2) + .map_or(false, |len| len > self.limit_total) + { + seq2.make_infinite(); + } + } + seq1.union(seq2); + assert!(seq1.len().map_or(true, |x| x <= self.limit_total)); + seq1 + } + + /// Applies the literal length limit to the given sequence. If none of the + /// literals in the sequence exceed the limit, then this is a no-op. + fn enforce_literal_len(&self, seq: &mut Seq) { + let len = self.limit_literal_len; + match self.kind { + ExtractKind::Prefix => seq.keep_first_bytes(len), + ExtractKind::Suffix => seq.keep_last_bytes(len), + } + } +} + +impl Default for Extractor { + fn default() -> Extractor { + Extractor::new() + } +} + +/// The kind of literals to extract from an [`Hir`] expression. +/// +/// The default extraction kind is `Prefix`. +#[non_exhaustive] +#[derive(Clone, Debug)] +pub enum ExtractKind { + /// Extracts only prefix literals from a regex. + Prefix, + /// Extracts only suffix literals from a regex. + /// + /// Note that the sequence returned by suffix literals currently may + /// not correctly represent leftmost-first or "preference" order match + /// semantics. + Suffix, +} + +impl ExtractKind { + /// Returns true if this kind is the `Prefix` variant. + pub fn is_prefix(&self) -> bool { + matches!(*self, ExtractKind::Prefix) + } + + /// Returns true if this kind is the `Suffix` variant. + pub fn is_suffix(&self) -> bool { + matches!(*self, ExtractKind::Suffix) + } +} + +impl Default for ExtractKind { + fn default() -> ExtractKind { + ExtractKind::Prefix + } +} + +/// A sequence of literals. +/// +/// A `Seq` is very much like a set in that it represents a union of its +/// members. That is, it corresponds to a set of literals where at least one +/// must match in order for a particular [`Hir`] expression to match. (Whether +/// this corresponds to the entire `Hir` expression, a prefix of it or a suffix +/// of it depends on how the `Seq` was extracted from the `Hir`.) +/// +/// It is also unlike a set in that multiple identical literals may appear, +/// and that the order of the literals in the `Seq` matters. For example, if +/// the sequence is `[sam, samwise]` and leftmost-first matching is used, then +/// `samwise` can never match and the sequence is equivalent to `[sam]`. +/// +/// # States of a sequence +/// +/// A `Seq` has a few different logical states to consider: +/// +/// * The sequence can represent "any" literal. When this happens, the set does +/// not have a finite size. The purpose of this state is to inhibit callers +/// from making assumptions about what literals are required in order to match +/// a particular [`Hir`] expression. Generally speaking, when a set is in this +/// state, literal optimizations are inhibited. A good example of a regex that +/// will cause this sort of set to appear is `[A-Za-z]`. The character class +/// is just too big (and also too narrow) to be usefully expanded into 52 +/// different literals. (Note that the decision for when a seq should become +/// infinite is determined by the caller. A seq itself has no hard-coded +/// limits.) +/// * The sequence can be empty, in which case, it is an affirmative statement +/// that there are no literals that can match the corresponding `Hir`. +/// Consequently, the `Hir` never matches any input. For example, `[a&&b]`. +/// * The sequence can be non-empty, in which case, at least one of the +/// literals must match in order for the corresponding `Hir` to match. +/// +/// # Example +/// +/// This example shows how literal sequences can be simplified by stripping +/// suffixes and minimizing while maintaining preference order. +/// +/// ``` +/// use regex_syntax::hir::literal::{Literal, Seq}; +/// +/// let mut seq = Seq::new(&[ +/// "farm", +/// "appliance", +/// "faraway", +/// "apple", +/// "fare", +/// "gap", +/// "applicant", +/// "applaud", +/// ]); +/// seq.keep_first_bytes(3); +/// seq.minimize_by_preference(); +/// // Notice that 'far' comes before 'app', which matches the order in the +/// // original sequence. This guarantees that leftmost-first semantics are +/// // not altered by simplifying the set. +/// let expected = Seq::from_iter([ +/// Literal::inexact("far"), +/// Literal::inexact("app"), +/// Literal::exact("gap"), +/// ]); +/// assert_eq!(expected, seq); +/// ``` +#[derive(Clone, Eq, PartialEq)] +pub struct Seq { + /// The members of this seq. + /// + /// When `None`, the seq represents all possible literals. That is, it + /// prevents one from making assumptions about specific literals in the + /// seq, and forces one to treat it as if any literal might be in the seq. + /// + /// Note that `Some(vec![])` is valid and corresponds to the empty seq of + /// literals, i.e., a regex that can never match. For example, `[a&&b]`. + /// It is distinct from `Some(vec![""])`, which corresponds to the seq + /// containing an empty string, which matches at every position. + literals: Option<Vec<Literal>>, +} + +impl Seq { + /// Returns an empty sequence. + /// + /// An empty sequence matches zero literals, and thus corresponds to a + /// regex that itself can never match. + #[inline] + pub fn empty() -> Seq { + Seq { literals: Some(vec![]) } + } + + /// Returns a sequence of literals without a finite size and may contain + /// any literal. + /// + /// A sequence without finite size does not reveal anything about the + /// characteristics of the literals in its set. There are no fixed prefixes + /// or suffixes, nor are lower or upper bounds on the length of the literals + /// in the set known. + /// + /// This is useful to represent constructs in a regex that are "too big" + /// to useful represent as a sequence of literals. For example, `[A-Za-z]`. + /// When sequences get too big, they lose their discriminating nature and + /// are more likely to produce false positives, which in turn makes them + /// less likely to speed up searches. + /// + /// More pragmatically, for many regexes, enumerating all possible literals + /// is itself not possible or might otherwise use too many resources. So + /// constraining the size of sets during extraction is a practical trade + /// off to make. + #[inline] + pub fn infinite() -> Seq { + Seq { literals: None } + } + + /// Returns a sequence containing a single literal. + #[inline] + pub fn singleton(lit: Literal) -> Seq { + Seq { literals: Some(vec![lit]) } + } + + /// Returns a sequence of exact literals from the given byte strings. + #[inline] + pub fn new<I, B>(it: I) -> Seq + where + I: IntoIterator<Item = B>, + B: AsRef<[u8]>, + { + it.into_iter().map(|b| Literal::exact(b.as_ref())).collect() + } + + /// If this is a finite sequence, return its members as a slice of + /// literals. + /// + /// The slice returned may be empty, in which case, there are no literals + /// that can match this sequence. + #[inline] + pub fn literals(&self) -> Option<&[Literal]> { + self.literals.as_deref() + } + + /// Push a literal to the end of this sequence. + /// + /// If this sequence is not finite, then this is a no-op. + /// + /// Similarly, if the most recently added item of this sequence is + /// equivalent to the literal given, then it is not added. This reflects + /// a `Seq`'s "set like" behavior, and represents a practical trade off. + /// Namely, there is never any need to have two adjacent and equivalent + /// literals in the same sequence, _and_ it is easy to detect in some + /// cases. + #[inline] + pub fn push(&mut self, lit: Literal) { + let lits = match self.literals { + None => return, + Some(ref mut lits) => lits, + }; + if lits.last().map_or(false, |m| m == &lit) { + return; + } + lits.push(lit); + } + + /// Make all of the literals in this sequence inexact. + /// + /// This is a no-op if this sequence is not finite. + #[inline] + pub fn make_inexact(&mut self) { + let lits = match self.literals { + None => return, + Some(ref mut lits) => lits, + }; + for lit in lits.iter_mut() { + lit.make_inexact(); + } + } + + /// Converts this sequence to an infinite sequence. + /// + /// This is a no-op if the sequence is already infinite. + #[inline] + pub fn make_infinite(&mut self) { + self.literals = None; + } + + /// Modify this sequence to contain the cross product between it and the + /// sequence given. + /// + /// The cross product only considers literals in this sequence that are + /// exact. That is, inexact literals are not extended. + /// + /// The literals are always drained from `other`, even if none are used. + /// This permits callers to reuse the sequence allocation elsewhere. + /// + /// If this sequence is infinite, then this is a no-op, regardless of what + /// `other` contains (and in this case, the literals are still drained from + /// `other`). If `other` is infinite and this sequence is finite, then this + /// is a no-op, unless this sequence contains a zero-length literal. In + /// which case, the infiniteness of `other` infects this sequence, and this + /// sequence is itself made infinite. + /// + /// Like [`Seq::union`], this may attempt to deduplicate literals. See + /// [`Seq::dedup`] for how deduplication deals with exact and inexact + /// literals. + /// + /// # Example + /// + /// This example shows basic usage and how exact and inexact literals + /// interact. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq1 = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::inexact("bar"), + /// ]); + /// let mut seq2 = Seq::from_iter([ + /// Literal::inexact("quux"), + /// Literal::exact("baz"), + /// ]); + /// seq1.cross_forward(&mut seq2); + /// + /// // The literals are pulled out of seq2. + /// assert_eq!(Some(0), seq2.len()); + /// + /// let expected = Seq::from_iter([ + /// Literal::inexact("fooquux"), + /// Literal::exact("foobaz"), + /// Literal::inexact("bar"), + /// ]); + /// assert_eq!(expected, seq1); + /// ``` + /// + /// This example shows the behavior of when `other` is an infinite + /// sequence. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq1 = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::inexact("bar"), + /// ]); + /// let mut seq2 = Seq::infinite(); + /// seq1.cross_forward(&mut seq2); + /// + /// // When seq2 is infinite, cross product doesn't add anything, but + /// // ensures all members of seq1 are inexact. + /// let expected = Seq::from_iter([ + /// Literal::inexact("foo"), + /// Literal::inexact("bar"), + /// ]); + /// assert_eq!(expected, seq1); + /// ``` + /// + /// This example is like the one above, but shows what happens when this + /// sequence contains an empty string. In this case, an infinite `other` + /// sequence infects this sequence (because the empty string means that + /// there are no finite prefixes): + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq1 = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::exact(""), // inexact provokes same behavior + /// Literal::inexact("bar"), + /// ]); + /// let mut seq2 = Seq::infinite(); + /// seq1.cross_forward(&mut seq2); + /// + /// // seq1 is now infinite! + /// assert!(!seq1.is_finite()); + /// ``` + /// + /// This example shows the behavior of this sequence is infinite. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq1 = Seq::infinite(); + /// let mut seq2 = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::inexact("bar"), + /// ]); + /// seq1.cross_forward(&mut seq2); + /// + /// // seq1 remains unchanged. + /// assert!(!seq1.is_finite()); + /// // Even though the literals in seq2 weren't used, it was still drained. + /// assert_eq!(Some(0), seq2.len()); + /// ``` + #[inline] + pub fn cross_forward(&mut self, other: &mut Seq) { + let (lits1, lits2) = match self.cross_preamble(other) { + None => return, + Some((lits1, lits2)) => (lits1, lits2), + }; + let newcap = lits1.len().saturating_mul(lits2.len()); + for selflit in mem::replace(lits1, Vec::with_capacity(newcap)) { + if !selflit.is_exact() { + lits1.push(selflit); + continue; + } + for otherlit in lits2.iter() { + let mut newlit = Literal::exact(Vec::with_capacity( + selflit.len() + otherlit.len(), + )); + newlit.extend(&selflit); + newlit.extend(&otherlit); + if !otherlit.is_exact() { + newlit.make_inexact(); + } + lits1.push(newlit); + } + } + lits2.drain(..); + self.dedup(); + } + + /// Modify this sequence to contain the cross product between it and + /// the sequence given, where the sequences are treated as suffixes + /// instead of prefixes. Namely, the sequence `other` is *prepended* + /// to `self` (as opposed to `other` being *appended* to `self` in + /// [`Seq::cross_forward`]). + /// + /// The cross product only considers literals in this sequence that are + /// exact. That is, inexact literals are not extended. + /// + /// The literals are always drained from `other`, even if none are used. + /// This permits callers to reuse the sequence allocation elsewhere. + /// + /// If this sequence is infinite, then this is a no-op, regardless of what + /// `other` contains (and in this case, the literals are still drained from + /// `other`). If `other` is infinite and this sequence is finite, then this + /// is a no-op, unless this sequence contains a zero-length literal. In + /// which case, the infiniteness of `other` infects this sequence, and this + /// sequence is itself made infinite. + /// + /// Like [`Seq::union`], this may attempt to deduplicate literals. See + /// [`Seq::dedup`] for how deduplication deals with exact and inexact + /// literals. + /// + /// # Example + /// + /// This example shows basic usage and how exact and inexact literals + /// interact. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq1 = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::inexact("bar"), + /// ]); + /// let mut seq2 = Seq::from_iter([ + /// Literal::inexact("quux"), + /// Literal::exact("baz"), + /// ]); + /// seq1.cross_reverse(&mut seq2); + /// + /// // The literals are pulled out of seq2. + /// assert_eq!(Some(0), seq2.len()); + /// + /// let expected = Seq::from_iter([ + /// Literal::inexact("quuxfoo"), + /// Literal::inexact("bar"), + /// Literal::exact("bazfoo"), + /// ]); + /// assert_eq!(expected, seq1); + /// ``` + /// + /// This example shows the behavior of when `other` is an infinite + /// sequence. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq1 = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::inexact("bar"), + /// ]); + /// let mut seq2 = Seq::infinite(); + /// seq1.cross_reverse(&mut seq2); + /// + /// // When seq2 is infinite, cross product doesn't add anything, but + /// // ensures all members of seq1 are inexact. + /// let expected = Seq::from_iter([ + /// Literal::inexact("foo"), + /// Literal::inexact("bar"), + /// ]); + /// assert_eq!(expected, seq1); + /// ``` + /// + /// This example is like the one above, but shows what happens when this + /// sequence contains an empty string. In this case, an infinite `other` + /// sequence infects this sequence (because the empty string means that + /// there are no finite suffixes): + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq1 = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::exact(""), // inexact provokes same behavior + /// Literal::inexact("bar"), + /// ]); + /// let mut seq2 = Seq::infinite(); + /// seq1.cross_reverse(&mut seq2); + /// + /// // seq1 is now infinite! + /// assert!(!seq1.is_finite()); + /// ``` + /// + /// This example shows the behavior when this sequence is infinite. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq1 = Seq::infinite(); + /// let mut seq2 = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::inexact("bar"), + /// ]); + /// seq1.cross_reverse(&mut seq2); + /// + /// // seq1 remains unchanged. + /// assert!(!seq1.is_finite()); + /// // Even though the literals in seq2 weren't used, it was still drained. + /// assert_eq!(Some(0), seq2.len()); + /// ``` + #[inline] + pub fn cross_reverse(&mut self, other: &mut Seq) { + let (lits1, lits2) = match self.cross_preamble(other) { + None => return, + Some((lits1, lits2)) => (lits1, lits2), + }; + // We basically proceed as we do in 'cross_forward' at this point, + // except that the outer loop is now 'other' and the inner loop is now + // 'self'. That's because 'self' corresponds to suffixes and 'other' + // corresponds to the sequence we want to *prepend* to the suffixes. + let newcap = lits1.len().saturating_mul(lits2.len()); + let selflits = mem::replace(lits1, Vec::with_capacity(newcap)); + for (i, otherlit) in lits2.drain(..).enumerate() { + for selflit in selflits.iter() { + if !selflit.is_exact() { + // If the suffix isn't exact, then we can't prepend + // anything to it. However, we still want to keep it. But + // we only want to keep one of them, to avoid duplication. + // (The duplication is okay from a correctness perspective, + // but wasteful.) + if i == 0 { + lits1.push(selflit.clone()); + } + continue; + } + let mut newlit = Literal::exact(Vec::with_capacity( + otherlit.len() + selflit.len(), + )); + newlit.extend(&otherlit); + newlit.extend(&selflit); + if !otherlit.is_exact() { + newlit.make_inexact(); + } + lits1.push(newlit); + } + } + self.dedup(); + } + + /// A helper function the corresponds to the subtle preamble for both + /// `cross_forward` and `cross_reverse`. In effect, it handles the cases + /// of infinite sequences for both `self` and `other`, as well as ensuring + /// that literals from `other` are drained even if they aren't used. + fn cross_preamble<'a>( + &'a mut self, + other: &'a mut Seq, + ) -> Option<(&'a mut Vec<Literal>, &'a mut Vec<Literal>)> { + let lits2 = match other.literals { + None => { + // If our current seq contains the empty string and the seq + // we're adding matches any literal, then it follows that the + // current seq must now also match any literal. + // + // Otherwise, we just have to make sure everything in this + // sequence is inexact. + if self.min_literal_len() == Some(0) { + *self = Seq::infinite(); + } else { + self.make_inexact(); + } + return None; + } + Some(ref mut lits) => lits, + }; + let lits1 = match self.literals { + None => { + // If we aren't going to make it to the end of this routine + // where lits2 is drained, then we need to do it now. + lits2.drain(..); + return None; + } + Some(ref mut lits) => lits, + }; + Some((lits1, lits2)) + } + + /// Unions the `other` sequence into this one. + /// + /// The literals are always drained out of the given `other` sequence, + /// even if they are being unioned into an infinite sequence. This permits + /// the caller to reuse the `other` sequence in another context. + /// + /// Some literal deduping may be performed. If any deduping happens, + /// any leftmost-first or "preference" order match semantics will be + /// preserved. + /// + /// # Example + /// + /// This example shows basic usage. + /// + /// ``` + /// use regex_syntax::hir::literal::Seq; + /// + /// let mut seq1 = Seq::new(&["foo", "bar"]); + /// let mut seq2 = Seq::new(&["bar", "quux", "foo"]); + /// seq1.union(&mut seq2); + /// + /// // The literals are pulled out of seq2. + /// assert_eq!(Some(0), seq2.len()); + /// + /// // Adjacent literals are deduped, but non-adjacent literals may not be. + /// assert_eq!(Seq::new(&["foo", "bar", "quux", "foo"]), seq1); + /// ``` + /// + /// This example shows that literals are drained from `other` even when + /// they aren't necessarily used. + /// + /// ``` + /// use regex_syntax::hir::literal::Seq; + /// + /// let mut seq1 = Seq::infinite(); + /// // Infinite sequences have no finite length. + /// assert_eq!(None, seq1.len()); + /// + /// let mut seq2 = Seq::new(&["bar", "quux", "foo"]); + /// seq1.union(&mut seq2); + /// + /// // seq1 is still infinite and seq2 has been drained. + /// assert_eq!(None, seq1.len()); + /// assert_eq!(Some(0), seq2.len()); + /// ``` + #[inline] + pub fn union(&mut self, other: &mut Seq) { + let lits2 = match other.literals { + None => { + // Unioning with an infinite sequence always results in an + // infinite sequence. + self.make_infinite(); + return; + } + Some(ref mut lits) => lits.drain(..), + }; + let lits1 = match self.literals { + None => return, + Some(ref mut lits) => lits, + }; + lits1.extend(lits2); + self.dedup(); + } + + /// Unions the `other` sequence into this one by splice the `other` + /// sequence at the position of the first zero-length literal. + /// + /// This is useful for preserving preference order semantics when combining + /// two literal sequences. For example, in the regex `(a||f)+foo`, the + /// correct preference order prefix sequence is `[a, foo, f]`. + /// + /// The literals are always drained out of the given `other` sequence, + /// even if they are being unioned into an infinite sequence. This permits + /// the caller to reuse the `other` sequence in another context. Note that + /// the literals are drained even if no union is performed as well, i.e., + /// when this sequence does not contain a zero-length literal. + /// + /// Some literal deduping may be performed. If any deduping happens, + /// any leftmost-first or "preference" order match semantics will be + /// preserved. + /// + /// # Example + /// + /// This example shows basic usage. + /// + /// ``` + /// use regex_syntax::hir::literal::Seq; + /// + /// let mut seq1 = Seq::new(&["a", "", "f", ""]); + /// let mut seq2 = Seq::new(&["foo"]); + /// seq1.union_into_empty(&mut seq2); + /// + /// // The literals are pulled out of seq2. + /// assert_eq!(Some(0), seq2.len()); + /// // 'foo' gets spliced into seq1 where the first empty string occurs. + /// assert_eq!(Seq::new(&["a", "foo", "f"]), seq1); + /// ``` + /// + /// This example shows that literals are drained from `other` even when + /// they aren't necessarily used. + /// + /// ``` + /// use regex_syntax::hir::literal::Seq; + /// + /// let mut seq1 = Seq::new(&["foo", "bar"]); + /// let mut seq2 = Seq::new(&["bar", "quux", "foo"]); + /// seq1.union_into_empty(&mut seq2); + /// + /// // seq1 has no zero length literals, so no splicing happens. + /// assert_eq!(Seq::new(&["foo", "bar"]), seq1); + /// // Even though no splicing happens, seq2 is still drained. + /// assert_eq!(Some(0), seq2.len()); + /// ``` + #[inline] + pub fn union_into_empty(&mut self, other: &mut Seq) { + let lits2 = other.literals.as_mut().map(|lits| lits.drain(..)); + let lits1 = match self.literals { + None => return, + Some(ref mut lits) => lits, + }; + let first_empty = match lits1.iter().position(|m| m.is_empty()) { + None => return, + Some(i) => i, + }; + let lits2 = match lits2 { + None => { + // Note that we are only here if we've found an empty literal, + // which implies that an infinite sequence infects this seq and + // also turns it into an infinite sequence. + self.literals = None; + return; + } + Some(lits) => lits, + }; + // Clearing out the empties needs to come before the splice because + // the splice might add more empties that we don't want to get rid + // of. Since we're splicing into the position of the first empty, the + // 'first_empty' position computed above is still correct. + lits1.retain(|m| !m.is_empty()); + lits1.splice(first_empty..first_empty, lits2); + self.dedup(); + } + + /// Deduplicate adjacent equivalent literals in this sequence. + /// + /// If adjacent literals are equivalent strings but one is exact and the + /// other inexact, the inexact literal is kept and the exact one is + /// removed. + /// + /// Deduping an infinite sequence is a no-op. + /// + /// # Example + /// + /// This example shows how literals that are duplicate byte strings but + /// are not equivalent with respect to exactness are resolved. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::inexact("foo"), + /// ]); + /// seq.dedup(); + /// + /// assert_eq!(Seq::from_iter([Literal::inexact("foo")]), seq); + /// ``` + #[inline] + pub fn dedup(&mut self) { + if let Some(ref mut lits) = self.literals { + lits.dedup_by(|lit1, lit2| { + if lit1.as_bytes() != lit2.as_bytes() { + return false; + } + if lit1.is_exact() != lit2.is_exact() { + lit1.make_inexact(); + lit2.make_inexact(); + } + true + }); + } + } + + /// Sorts this sequence of literals lexicographically. + /// + /// Note that if, before sorting, if a literal that is a prefix of another + /// literal appears after it, then after sorting, the sequence will not + /// represent the same preference order match semantics. For example, + /// sorting the sequence `[samwise, sam]` yields the sequence `[sam, + /// samwise]`. Under preference order semantics, the latter sequence will + /// never match `samwise` where as the first sequence can. + /// + /// # Example + /// + /// This example shows basic usage. + /// + /// ``` + /// use regex_syntax::hir::literal::Seq; + /// + /// let mut seq = Seq::new(&["foo", "quux", "bar"]); + /// seq.sort(); + /// + /// assert_eq!(Seq::new(&["bar", "foo", "quux"]), seq); + /// ``` + #[inline] + pub fn sort(&mut self) { + if let Some(ref mut lits) = self.literals { + lits.sort(); + } + } + + /// Reverses all of the literals in this sequence. + /// + /// The order of the sequence itself is preserved. + /// + /// # Example + /// + /// This example shows basic usage. + /// + /// ``` + /// use regex_syntax::hir::literal::Seq; + /// + /// let mut seq = Seq::new(&["oof", "rab"]); + /// seq.reverse_literals(); + /// assert_eq!(Seq::new(&["foo", "bar"]), seq); + /// ``` + #[inline] + pub fn reverse_literals(&mut self) { + if let Some(ref mut lits) = self.literals { + for lit in lits.iter_mut() { + lit.reverse(); + } + } + } + + /// Shrinks this seq to its minimal size while respecting the preference + /// order of its literals. + /// + /// While this routine will remove duplicate literals from this seq, it + /// will also remove literals that can never match in a leftmost-first or + /// "preference order" search. Similar to [`Seq::dedup`], if a literal is + /// deduped, then the one that remains is made inexact. + /// + /// This is a no-op on seqs that are empty or not finite. + /// + /// # Example + /// + /// This example shows the difference between `{sam, samwise}` and + /// `{samwise, sam}`. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// // If 'sam' comes before 'samwise' and a preference order search is + /// // executed, then 'samwise' can never match. + /// let mut seq = Seq::new(&["sam", "samwise"]); + /// seq.minimize_by_preference(); + /// assert_eq!(Seq::from_iter([Literal::inexact("sam")]), seq); + /// + /// // But if they are reversed, then it's possible for 'samwise' to match + /// // since it is given higher preference. + /// let mut seq = Seq::new(&["samwise", "sam"]); + /// seq.minimize_by_preference(); + /// assert_eq!(Seq::new(&["samwise", "sam"]), seq); + /// ``` + /// + /// This example shows that if an empty string is in this seq, then + /// anything that comes after it can never match. + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// // An empty string is a prefix of all strings, so it automatically + /// // inhibits any subsequent strings from matching. + /// let mut seq = Seq::new(&["foo", "bar", "", "quux", "fox"]); + /// seq.minimize_by_preference(); + /// let expected = Seq::from_iter([ + /// Literal::exact("foo"), + /// Literal::exact("bar"), + /// Literal::inexact(""), + /// ]); + /// assert_eq!(expected, seq); + /// + /// // And of course, if it's at the beginning, then it makes it impossible + /// // for anything else to match. + /// let mut seq = Seq::new(&["", "foo", "quux", "fox"]); + /// seq.minimize_by_preference(); + /// assert_eq!(Seq::from_iter([Literal::inexact("")]), seq); + /// ``` + #[inline] + pub fn minimize_by_preference(&mut self) { + if let Some(ref mut lits) = self.literals { + PreferenceTrie::minimize(lits, false); + } + } + + /// Trims all literals in this seq such that only the first `len` bytes + /// remain. If a literal has less than or equal to `len` bytes, then it + /// remains unchanged. Otherwise, it is trimmed and made inexact. + /// + /// # Example + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq = Seq::new(&["a", "foo", "quux"]); + /// seq.keep_first_bytes(2); + /// + /// let expected = Seq::from_iter([ + /// Literal::exact("a"), + /// Literal::inexact("fo"), + /// Literal::inexact("qu"), + /// ]); + /// assert_eq!(expected, seq); + /// ``` + #[inline] + pub fn keep_first_bytes(&mut self, len: usize) { + if let Some(ref mut lits) = self.literals { + for m in lits.iter_mut() { + m.keep_first_bytes(len); + } + } + } + + /// Trims all literals in this seq such that only the last `len` bytes + /// remain. If a literal has less than or equal to `len` bytes, then it + /// remains unchanged. Otherwise, it is trimmed and made inexact. + /// + /// # Example + /// + /// ``` + /// use regex_syntax::hir::literal::{Literal, Seq}; + /// + /// let mut seq = Seq::new(&["a", "foo", "quux"]); + /// seq.keep_last_bytes(2); + /// + /// let expected = Seq::from_iter([ + /// Literal::exact("a"), + /// Literal::inexact("oo"), + /// Literal::inexact("ux"), + /// ]); + /// assert_eq!(expected, seq); + /// ``` + #[inline] + pub fn keep_last_bytes(&mut self, len: usize) { + if let Some(ref mut lits) = self.literals { + for m in lits.iter_mut() { + m.keep_last_bytes(len); + } + } + } + + /// Returns true if this sequence is finite. + /// + /// When false, this sequence is infinite and must be treated as if it + /// contains every possible literal. + #[inline] + pub fn is_finite(&self) -> bool { + self.literals.is_some() + } + + /// Returns true if and only if this sequence is finite and empty. + /// + /// An empty sequence never matches anything. It can only be produced by + /// literal extraction when the corresponding regex itself cannot match. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == Some(0) + } + + /// Returns the number of literals in this sequence if the sequence is + /// finite. If the sequence is infinite, then `None` is returned. + #[inline] + pub fn len(&self) -> Option<usize> { + self.literals.as_ref().map(|lits| lits.len()) + } + + /// Returns true if and only if all literals in this sequence are exact. + /// + /// This returns false if the sequence is infinite. + #[inline] + pub fn is_exact(&self) -> bool { + self.literals().map_or(false, |lits| lits.iter().all(|x| x.is_exact())) + } + + /// Returns true if and only if all literals in this sequence are inexact. + /// + /// This returns true if the sequence is infinite. + #[inline] + pub fn is_inexact(&self) -> bool { + self.literals().map_or(true, |lits| lits.iter().all(|x| !x.is_exact())) + } + + /// Return the maximum length of the sequence that would result from + /// unioning `self` with `other`. If either set is infinite, then this + /// returns `None`. + #[inline] + pub fn max_union_len(&self, other: &Seq) -> Option<usize> { + let len1 = self.len()?; + let len2 = other.len()?; + Some(len1.saturating_add(len2)) + } + + /// Return the maximum length of the sequence that would result from the + /// cross product of `self` with `other`. If either set is infinite, then + /// this returns `None`. + #[inline] + pub fn max_cross_len(&self, other: &Seq) -> Option<usize> { + let len1 = self.len()?; + let len2 = other.len()?; + Some(len1.saturating_mul(len2)) + } + + /// Returns the length of the shortest literal in this sequence. + /// + /// If the sequence is infinite or empty, then this returns `None`. + #[inline] + pub fn min_literal_len(&self) -> Option<usize> { + self.literals.as_ref()?.iter().map(|x| x.len()).min() + } + + /// Returns the length of the longest literal in this sequence. + /// + /// If the sequence is infinite or empty, then this returns `None`. + #[inline] + pub fn max_literal_len(&self) -> Option<usize> { + self.literals.as_ref()?.iter().map(|x| x.len()).max() + } + + /// Returns the longest common prefix from this seq. + /// + /// If the seq matches any literal or other contains no literals, then + /// there is no meaningful prefix and this returns `None`. + /// + /// # Example + /// + /// This shows some example seqs and their longest common prefix. + /// + /// ``` + /// use regex_syntax::hir::literal::Seq; + /// + /// let seq = Seq::new(&["foo", "foobar", "fo"]); + /// assert_eq!(Some(&b"fo"[..]), seq.longest_common_prefix()); + /// let seq = Seq::new(&["foo", "foo"]); + /// assert_eq!(Some(&b"foo"[..]), seq.longest_common_prefix()); + /// let seq = Seq::new(&["foo", "bar"]); + /// assert_eq!(Some(&b""[..]), seq.longest_common_prefix()); + /// let seq = Seq::new(&[""]); + /// assert_eq!(Some(&b""[..]), seq.longest_common_prefix()); + /// + /// let seq = Seq::infinite(); + /// assert_eq!(None, seq.longest_common_prefix()); + /// let seq = Seq::empty(); + /// assert_eq!(None, seq.longest_common_prefix()); + /// ``` + #[inline] + pub fn longest_common_prefix(&self) -> Option<&[u8]> { + // If we match everything or match nothing, then there's no meaningful + // longest common prefix. + let lits = match self.literals { + None => return None, + Some(ref lits) => lits, + }; + if lits.len() == 0 { + return None; + } + let base = lits[0].as_bytes(); + let mut len = base.len(); + for m in lits.iter().skip(1) { + len = m + .as_bytes() + .iter() + .zip(base[..len].iter()) + .take_while(|&(a, b)| a == b) + .count(); + if len == 0 { + return Some(&[]); + } + } + Some(&base[..len]) + } + + /// Returns the longest common suffix from this seq. + /// + /// If the seq matches any literal or other contains no literals, then + /// there is no meaningful suffix and this returns `None`. + /// + /// # Example + /// + /// This shows some example seqs and their longest common suffix. + /// + /// ``` + /// use regex_syntax::hir::literal::Seq; + /// + /// let seq = Seq::new(&["oof", "raboof", "of"]); + /// assert_eq!(Some(&b"of"[..]), seq.longest_common_suffix()); + /// let seq = Seq::new(&["foo", "foo"]); + /// assert_eq!(Some(&b"foo"[..]), seq.longest_common_suffix()); + /// let seq = Seq::new(&["foo", "bar"]); + /// assert_eq!(Some(&b""[..]), seq.longest_common_suffix()); + /// let seq = Seq::new(&[""]); + /// assert_eq!(Some(&b""[..]), seq.longest_common_suffix()); + /// + /// let seq = Seq::infinite(); + /// assert_eq!(None, seq.longest_common_suffix()); + /// let seq = Seq::empty(); + /// assert_eq!(None, seq.longest_common_suffix()); + /// ``` + #[inline] + pub fn longest_common_suffix(&self) -> Option<&[u8]> { + // If we match everything or match nothing, then there's no meaningful + // longest common suffix. + let lits = match self.literals { + None => return None, + Some(ref lits) => lits, + }; + if lits.len() == 0 { + return None; + } + let base = lits[0].as_bytes(); + let mut len = base.len(); + for m in lits.iter().skip(1) { + len = m + .as_bytes() + .iter() + .rev() + .zip(base[base.len() - len..].iter().rev()) + .take_while(|&(a, b)| a == b) + .count(); + if len == 0 { + return Some(&[]); + } + } + Some(&base[base.len() - len..]) + } + + /// Optimizes this seq while treating its literals as prefixes and + /// respecting the preference order of its literals. + /// + /// The specific way "optimization" works is meant to be an implementation + /// detail, as it essentially represents a set of heuristics. The goal + /// that optimization tries to accomplish is to make the literals in this + /// set reflect inputs that will result in a more effective prefilter. + /// Principally by reducing the false positive rate of candidates found by + /// the literals in this sequence. That is, when a match of a literal is + /// found, we would like it to be a strong predictor of the overall match + /// of the regex. If it isn't, then much time will be spent starting and + /// stopping the prefilter search and attempting to confirm the match only + /// to have it fail. + /// + /// Some of those heuristics might be: + /// + /// * Identifying a common prefix from a larger sequence of literals, and + /// shrinking the sequence down to that single common prefix. + /// * Rejecting the sequence entirely if it is believed to result in very + /// high false positive rate. When this happens, the sequence is made + /// infinite. + /// * Shrinking the sequence to a smaller number of literals representing + /// prefixes, but not shrinking it so much as to make literals too short. + /// (A sequence with very short literals, of 1 or 2 bytes, will typically + /// result in a higher false positive rate.) + /// + /// Optimization should only be run once extraction is complete. Namely, + /// optimization may make assumptions that do not compose with other + /// operations in the middle of extraction. For example, optimization will + /// reduce `[E(sam), E(samwise)]` to `[E(sam)]`, but such a transformation + /// is only valid if no other extraction will occur. If other extraction + /// may occur, then the correct transformation would be to `[I(sam)]`. + /// + /// The [`Seq::optimize_for_suffix_by_preference`] does the same thing, but + /// for suffixes. + /// + /// # Example + /// + /// This shows how optimization might transform a sequence. Note that + /// the specific behavior is not a documented guarantee. The heuristics + /// used are an implementation detail and may change over time in semver + /// compatible releases. + /// + /// ``` + /// use regex_syntax::hir::literal::{Seq, Literal}; + /// + /// let mut seq = Seq::new(&[ + /// "samantha", + /// "sam", + /// "samwise", + /// "frodo", + /// ]); + /// seq.optimize_for_prefix_by_preference(); + /// assert_eq!(Seq::from_iter([ + /// Literal::exact("samantha"), + /// // Kept exact even though 'samwise' got pruned + /// // because optimization assumes literal extraction + /// // has finished. + /// Literal::exact("sam"), + /// Literal::exact("frodo"), + /// ]), seq); + /// ``` + /// + /// # Example: optimization may make the sequence infinite + /// + /// If the heuristics deem that the sequence could cause a very high false + /// positive rate, then it may make the sequence infinite, effectively + /// disabling its use as a prefilter. + /// + /// ``` + /// use regex_syntax::hir::literal::{Seq, Literal}; + /// + /// let mut seq = Seq::new(&[ + /// "samantha", + /// // An empty string matches at every position, + /// // thus rendering the prefilter completely + /// // ineffective. + /// "", + /// "sam", + /// "samwise", + /// "frodo", + /// ]); + /// seq.optimize_for_prefix_by_preference(); + /// assert!(!seq.is_finite()); + /// ``` + /// + /// Do note that just because there is a `" "` in the sequence, that + /// doesn't mean the sequence will always be made infinite after it is + /// optimized. Namely, if the sequence is considered exact (any match + /// corresponds to an overall match of the original regex), then any match + /// is an overall match, and so the false positive rate is always `0`. + /// + /// To demonstrate this, we remove `samwise` from our sequence. This + /// results in no optimization happening and all literals remain exact. + /// Thus the entire sequence is exact, and it is kept as-is, even though + /// one is an ASCII space: + /// + /// ``` + /// use regex_syntax::hir::literal::{Seq, Literal}; + /// + /// let mut seq = Seq::new(&[ + /// "samantha", + /// " ", + /// "sam", + /// "frodo", + /// ]); + /// seq.optimize_for_prefix_by_preference(); + /// assert!(seq.is_finite()); + /// ``` + #[inline] + pub fn optimize_for_prefix_by_preference(&mut self) { + self.optimize_by_preference(true); + } + + /// Optimizes this seq while treating its literals as suffixes and + /// respecting the preference order of its literals. + /// + /// Optimization should only be run once extraction is complete. + /// + /// The [`Seq::optimize_for_prefix_by_preference`] does the same thing, but + /// for prefixes. See its documentation for more explanation. + #[inline] + pub fn optimize_for_suffix_by_preference(&mut self) { + self.optimize_by_preference(false); + } + + fn optimize_by_preference(&mut self, prefix: bool) { + let origlen = match self.len() { + None => return, + Some(len) => len, + }; + // Just give up now if our sequence contains an empty string. + if self.min_literal_len().map_or(false, |len| len == 0) { + // We squash the sequence so that nobody else gets any bright + // ideas to try and use it. An empty string implies a match at + // every position. A prefilter cannot help you here. + self.make_infinite(); + return; + } + // Make sure we start with the smallest sequence possible. We use a + // special version of preference minimization that retains exactness. + // This is legal because optimization is only expected to occur once + // extraction is complete. + if prefix { + if let Some(ref mut lits) = self.literals { + PreferenceTrie::minimize(lits, true); + } + } + + // Look for a common prefix (or suffix). If we found one of those and + // it's long enough, then it's a good bet that it will be our fastest + // possible prefilter since single-substring search is so fast. + let fix = if prefix { + self.longest_common_prefix() + } else { + self.longest_common_suffix() + }; + if let Some(fix) = fix { + // As a special case, if we have a common prefix and the leading + // byte of that prefix is one that we think probably occurs rarely, + // then strip everything down to just that single byte. This should + // promote the use of memchr. + // + // ... we only do this though if our sequence has more than one + // literal. Otherwise, we'd rather just stick with a single literal + // scan. That is, using memchr is probably better than looking + // for 2 or more literals, but probably not as good as a straight + // memmem search. + // + // ... and also only do this when the prefix is short and probably + // not too discriminatory anyway. If it's longer, then it's + // probably quite discriminatory and thus is likely to have a low + // false positive rate. + if prefix + && origlen > 1 + && fix.len() >= 1 + && fix.len() <= 3 + && rank(fix[0]) < 200 + { + self.keep_first_bytes(1); + self.dedup(); + return; + } + // We only strip down to the common prefix/suffix if we think + // the existing set of literals isn't great, or if the common + // prefix/suffix is expected to be particularly discriminatory. + let isfast = + self.is_exact() && self.len().map_or(false, |len| len <= 16); + let usefix = fix.len() > 4 || (fix.len() > 1 && !isfast); + if usefix { + // If we keep exactly the number of bytes equal to the length + // of the prefix (or suffix), then by the definition of a + // prefix, every literal in the sequence will be equivalent. + // Thus, 'dedup' will leave us with one literal. + // + // We do it this way to avoid an alloc, but also to make sure + // the exactness of literals is kept (or not). + if prefix { + self.keep_first_bytes(fix.len()); + } else { + self.keep_last_bytes(fix.len()); + } + self.dedup(); + assert_eq!(Some(1), self.len()); + // We still fall through here. In particular, we want our + // longest common prefix to be subject to the poison check. + } + } + // If we have an exact sequence, we *probably* just want to keep it + // as-is. But there are some cases where we don't. So we save a copy of + // the exact sequence now, and then try to do some more optimizations + // below. If those don't work out, we go back to this exact sequence. + // + // The specific motivation for this is that we sometimes wind up with + // an exact sequence with a hefty number of literals. Say, 100. If we + // stuck with that, it would be too big for Teddy and would result in + // using Aho-Corasick. Which is fine... but the lazy DFA is plenty + // suitable in such cases. The real issue is that we will wind up not + // using a fast prefilter at all. So in cases like this, even though + // we have an exact sequence, it would be better to try and shrink the + // sequence (which we do below) and use it as a prefilter that can + // produce false positive matches. + // + // But if the shrinking below results in a sequence that "sucks," then + // we don't want to use that because we already have an exact sequence + // in hand. + let exact: Option<Seq> = + if self.is_exact() { Some(self.clone()) } else { None }; + // Now we attempt to shorten the sequence. The idea here is that we + // don't want to look for too many literals, but we want to shorten + // our sequence enough to improve our odds of using better algorithms + // downstream (such as Teddy). + // + // The pair of numbers in this list corresponds to the maximal prefix + // (in bytes) to keep for all literals and the length of the sequence + // at which to do it. + // + // So for example, the pair (3, 500) would mean, "if we have more than + // 500 literals in our sequence, then truncate all of our literals + // such that they are at most 3 bytes in length and the minimize the + // sequence." + const ATTEMPTS: [(usize, usize); 5] = + [(5, 10), (4, 10), (3, 64), (2, 64), (1, 10)]; + for (keep, limit) in ATTEMPTS { + let len = match self.len() { + None => break, + Some(len) => len, + }; + if len <= limit { + break; + } + if prefix { + self.keep_first_bytes(keep); + } else { + self.keep_last_bytes(keep); + } + if prefix { + if let Some(ref mut lits) = self.literals { + PreferenceTrie::minimize(lits, true); + } + } + } + // Check for a poison literal. A poison literal is one that is short + // and is believed to have a very high match count. These poisons + // generally lead to a prefilter with a very high false positive rate, + // and thus overall worse performance. + // + // We do this last because we could have gone from a non-poisonous + // sequence to a poisonous one. Perhaps we should add some code to + // prevent such transitions in the first place, but then again, we + // likely only made the transition in the first place if the sequence + // was itself huge. And huge sequences are themselves poisonous. So... + if let Some(lits) = self.literals() { + if lits.iter().any(|lit| lit.is_poisonous()) { + self.make_infinite(); + } + } + // OK, if we had an exact sequence before attempting more optimizations + // above and our post-optimized sequence sucks for some reason or + // another, then we go back to the exact sequence. + if let Some(exact) = exact { + // If optimizing resulted in dropping our literals, then certainly + // backup and use the exact sequence that we had. + if !self.is_finite() { + *self = exact; + return; + } + // If our optimized sequence contains a short literal, then it's + // *probably* not so great. So throw it away and revert to the + // exact sequence. + if self.min_literal_len().map_or(true, |len| len <= 2) { + *self = exact; + return; + } + // Finally, if our optimized sequence is "big" (i.e., can't use + // Teddy), then also don't use it and rely on the exact sequence. + if self.len().map_or(true, |len| len > 64) { + *self = exact; + return; + } + } + } +} + +impl core::fmt::Debug for Seq { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "Seq")?; + if let Some(lits) = self.literals() { + f.debug_list().entries(lits.iter()).finish() + } else { + write!(f, "[∞]") + } + } +} + +impl FromIterator<Literal> for Seq { + fn from_iter<T: IntoIterator<Item = Literal>>(it: T) -> Seq { + let mut seq = Seq::empty(); + for literal in it { + seq.push(literal); + } + seq + } +} + +/// A single literal extracted from an [`Hir`] expression. +/// +/// A literal is composed of two things: +/// +/// * A sequence of bytes. No guarantees with respect to UTF-8 are provided. +/// In particular, even if the regex a literal is extracted from is UTF-8, the +/// literal extracted may not be valid UTF-8. (For example, if an [`Extractor`] +/// limit resulted in trimming a literal in a way that splits a codepoint.) +/// * Whether the literal is "exact" or not. An "exact" literal means that it +/// has not been trimmed, and may continue to be extended. If a literal is +/// "exact" after visiting the entire `Hir` expression, then this implies that +/// the literal leads to a match state. (Although it doesn't necessarily imply +/// all occurrences of the literal correspond to a match of the regex, since +/// literal extraction ignores look-around assertions.) +#[derive(Clone, Eq, PartialEq, PartialOrd, Ord)] +pub struct Literal { + bytes: Vec<u8>, + exact: bool, +} + +impl Literal { + /// Returns a new exact literal containing the bytes given. + #[inline] + pub fn exact<B: Into<Vec<u8>>>(bytes: B) -> Literal { + Literal { bytes: bytes.into(), exact: true } + } + + /// Returns a new inexact literal containing the bytes given. + #[inline] + pub fn inexact<B: Into<Vec<u8>>>(bytes: B) -> Literal { + Literal { bytes: bytes.into(), exact: false } + } + + /// Returns the bytes in this literal. + #[inline] + pub fn as_bytes(&self) -> &[u8] { + &self.bytes + } + + /// Yields ownership of the bytes inside this literal. + /// + /// Note that this throws away whether the literal is "exact" or not. + #[inline] + pub fn into_bytes(self) -> Vec<u8> { + self.bytes + } + + /// Returns the length of this literal in bytes. + #[inline] + pub fn len(&self) -> usize { + self.as_bytes().len() + } + + /// Returns true if and only if this literal has zero bytes. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns true if and only if this literal is exact. + #[inline] + pub fn is_exact(&self) -> bool { + self.exact + } + + /// Marks this literal as inexact. + /// + /// Inexact literals can never be extended. For example, + /// [`Seq::cross_forward`] will not extend inexact literals. + #[inline] + pub fn make_inexact(&mut self) { + self.exact = false; + } + + /// Reverse the bytes in this literal. + #[inline] + pub fn reverse(&mut self) { + self.bytes.reverse(); + } + + /// Extend this literal with the literal given. + /// + /// If this literal is inexact, then this is a no-op. + #[inline] + pub fn extend(&mut self, lit: &Literal) { + if !self.is_exact() { + return; + } + self.bytes.extend_from_slice(&lit.bytes); + } + + /// Trims this literal such that only the first `len` bytes remain. If + /// this literal has fewer than `len` bytes, then it remains unchanged. + /// Otherwise, the literal is marked as inexact. + #[inline] + pub fn keep_first_bytes(&mut self, len: usize) { + if len >= self.len() { + return; + } + self.make_inexact(); + self.bytes.truncate(len); + } + + /// Trims this literal such that only the last `len` bytes remain. If this + /// literal has fewer than `len` bytes, then it remains unchanged. + /// Otherwise, the literal is marked as inexact. + #[inline] + pub fn keep_last_bytes(&mut self, len: usize) { + if len >= self.len() { + return; + } + self.make_inexact(); + self.bytes.drain(..self.len() - len); + } + + /// Returns true if it is believe that this literal is likely to match very + /// frequently, and is thus not a good candidate for a prefilter. + fn is_poisonous(&self) -> bool { + self.is_empty() || (self.len() == 1 && rank(self.as_bytes()[0]) >= 250) + } +} + +impl From<u8> for Literal { + fn from(byte: u8) -> Literal { + Literal::exact(vec![byte]) + } +} + +impl From<char> for Literal { + fn from(ch: char) -> Literal { + use alloc::string::ToString; + Literal::exact(ch.encode_utf8(&mut [0; 4]).to_string()) + } +} + +impl AsRef<[u8]> for Literal { + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +impl core::fmt::Debug for Literal { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + let tag = if self.exact { "E" } else { "I" }; + f.debug_tuple(tag) + .field(&crate::debug::Bytes(self.as_bytes())) + .finish() + } +} + +/// A "preference" trie that rejects literals that will never match when +/// executing a leftmost first or "preference" search. +/// +/// For example, if 'sam' is inserted, then trying to insert 'samwise' will be +/// rejected because 'samwise' can never match since 'sam' will always take +/// priority. However, if 'samwise' is inserted first, then inserting 'sam' +/// after it is accepted. In this case, either 'samwise' or 'sam' can match in +/// a "preference" search. +/// +/// Note that we only use this trie as a "set." That is, given a sequence of +/// literals, we insert each one in order. An `insert` will reject a literal +/// if a prefix of that literal already exists in the trie. Thus, to rebuild +/// the "minimal" sequence, we simply only keep literals that were successfully +/// inserted. (Since we don't need traversal, one wonders whether we can make +/// some simplifications here, but I haven't given it a ton of thought and I've +/// never seen this show up on a profile. Because of the heuristic limits +/// imposed on literal extractions, the size of the inputs here is usually +/// very small.) +#[derive(Debug)] +struct PreferenceTrie { + /// The states in this trie. The index of a state in this vector is its ID. + states: Vec<State>, + /// This vec indicates which states are match states. It always has + /// the same length as `states` and is indexed by the same state ID. + /// A state with identifier `sid` is a match state if and only if + /// `matches[sid].is_some()`. The option contains the index of the literal + /// corresponding to the match. The index is offset by 1 so that it fits in + /// a NonZeroUsize. + matches: Vec<Option<NonZeroUsize>>, + /// The index to allocate to the next literal added to this trie. Starts at + /// 1 and increments by 1 for every literal successfully added to the trie. + next_literal_index: usize, +} + +/// A single state in a trie. Uses a sparse representation for its transitions. +#[derive(Debug, Default)] +struct State { + /// Sparse representation of the transitions out of this state. Transitions + /// are sorted by byte. There is at most one such transition for any + /// particular byte. + trans: Vec<(u8, usize)>, +} + +impl PreferenceTrie { + /// Minimizes the given sequence of literals while preserving preference + /// order semantics. + /// + /// When `keep_exact` is true, the exactness of every literal retained is + /// kept. This is useful when dealing with a fully extracted `Seq` that + /// only contains exact literals. In that case, we can keep all retained + /// literals as exact because we know we'll never need to match anything + /// after them and because any removed literals are guaranteed to never + /// match. + fn minimize(literals: &mut Vec<Literal>, keep_exact: bool) { + let mut trie = PreferenceTrie { + states: vec![], + matches: vec![], + next_literal_index: 1, + }; + let mut make_inexact = vec![]; + literals.retain_mut(|lit| match trie.insert(lit.as_bytes()) { + Ok(_) => true, + Err(i) => { + if !keep_exact { + make_inexact.push(i.checked_sub(1).unwrap()); + } + false + } + }); + for i in make_inexact { + literals[i].make_inexact(); + } + } + + /// Returns `Ok` if the given byte string is accepted into this trie and + /// `Err` otherwise. The index for the success case corresponds to the + /// index of the literal added. The index for the error case corresponds to + /// the index of the literal already in the trie that prevented the given + /// byte string from being added. (Which implies it is a prefix of the one + /// given.) + /// + /// In short, the byte string given is accepted into the trie if and only + /// if it is possible for it to match when executing a preference order + /// search. + fn insert(&mut self, bytes: &[u8]) -> Result<usize, usize> { + let mut prev = self.root(); + if let Some(idx) = self.matches[prev] { + return Err(idx.get()); + } + for &b in bytes.iter() { + match self.states[prev].trans.binary_search_by_key(&b, |t| t.0) { + Ok(i) => { + prev = self.states[prev].trans[i].1; + if let Some(idx) = self.matches[prev] { + return Err(idx.get()); + } + } + Err(i) => { + let next = self.create_state(); + self.states[prev].trans.insert(i, (b, next)); + prev = next; + } + } + } + let idx = self.next_literal_index; + self.next_literal_index += 1; + self.matches[prev] = NonZeroUsize::new(idx); + Ok(idx) + } + + /// Returns the root state ID, and if it doesn't exist, creates it. + fn root(&mut self) -> usize { + if !self.states.is_empty() { + 0 + } else { + self.create_state() + } + } + + /// Creates a new empty state and returns its ID. + fn create_state(&mut self) -> usize { + let id = self.states.len(); + self.states.push(State::default()); + self.matches.push(None); + id + } +} + +/// Returns the "rank" of the given byte. +/// +/// The minimum rank value is `0` and the maximum rank value is `255`. +/// +/// The rank of a byte is derived from a heuristic background distribution of +/// relative frequencies of bytes. The heuristic says that lower the rank of a +/// byte, the less likely that byte is to appear in any arbitrary haystack. +pub fn rank(byte: u8) -> u8 { + crate::rank::BYTE_FREQUENCIES[usize::from(byte)] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(pattern: &str) -> Hir { + crate::ParserBuilder::new().utf8(false).build().parse(pattern).unwrap() + } + + fn prefixes(pattern: &str) -> Seq { + Extractor::new().kind(ExtractKind::Prefix).extract(&parse(pattern)) + } + + fn suffixes(pattern: &str) -> Seq { + Extractor::new().kind(ExtractKind::Suffix).extract(&parse(pattern)) + } + + fn e(pattern: &str) -> (Seq, Seq) { + (prefixes(pattern), suffixes(pattern)) + } + + #[allow(non_snake_case)] + fn E(x: &str) -> Literal { + Literal::exact(x.as_bytes()) + } + + #[allow(non_snake_case)] + fn I(x: &str) -> Literal { + Literal::inexact(x.as_bytes()) + } + + fn seq<I: IntoIterator<Item = Literal>>(it: I) -> Seq { + Seq::from_iter(it) + } + + fn infinite() -> (Seq, Seq) { + (Seq::infinite(), Seq::infinite()) + } + + fn inexact<I1, I2>(it1: I1, it2: I2) -> (Seq, Seq) + where + I1: IntoIterator<Item = Literal>, + I2: IntoIterator<Item = Literal>, + { + (Seq::from_iter(it1), Seq::from_iter(it2)) + } + + fn exact<B: AsRef<[u8]>, I: IntoIterator<Item = B>>(it: I) -> (Seq, Seq) { + let s1 = Seq::new(it); + let s2 = s1.clone(); + (s1, s2) + } + + fn opt<B: AsRef<[u8]>, I: IntoIterator<Item = B>>(it: I) -> (Seq, Seq) { + let (mut p, mut s) = exact(it); + p.optimize_for_prefix_by_preference(); + s.optimize_for_suffix_by_preference(); + (p, s) + } + + #[test] + fn literal() { + assert_eq!(exact(["a"]), e("a")); + assert_eq!(exact(["aaaaa"]), e("aaaaa")); + assert_eq!(exact(["A", "a"]), e("(?i-u)a")); + assert_eq!(exact(["AB", "Ab", "aB", "ab"]), e("(?i-u)ab")); + assert_eq!(exact(["abC", "abc"]), e("ab(?i-u)c")); + + assert_eq!(exact([b"\xFF"]), e(r"(?-u:\xFF)")); + + #[cfg(feature = "unicode-case")] + { + assert_eq!(exact(["☃"]), e("☃")); + assert_eq!(exact(["☃"]), e("(?i)☃")); + assert_eq!(exact(["☃☃☃☃☃"]), e("☃☃☃☃☃")); + + assert_eq!(exact(["Δ"]), e("Δ")); + assert_eq!(exact(["δ"]), e("δ")); + assert_eq!(exact(["Δ", "δ"]), e("(?i)Δ")); + assert_eq!(exact(["Δ", "δ"]), e("(?i)δ")); + + assert_eq!(exact(["S", "s", "ſ"]), e("(?i)S")); + assert_eq!(exact(["S", "s", "ſ"]), e("(?i)s")); + assert_eq!(exact(["S", "s", "ſ"]), e("(?i)ſ")); + } + + let letters = "ͱͳͷΐάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋ"; + assert_eq!(exact([letters]), e(letters)); + } + + #[test] + fn class() { + assert_eq!(exact(["a", "b", "c"]), e("[abc]")); + assert_eq!(exact(["a1b", "a2b", "a3b"]), e("a[123]b")); + assert_eq!(exact(["δ", "ε"]), e("[εδ]")); + #[cfg(feature = "unicode-case")] + { + assert_eq!(exact(["Δ", "Ε", "δ", "ε", "ϵ"]), e(r"(?i)[εδ]")); + } + } + + #[test] + fn look() { + assert_eq!(exact(["ab"]), e(r"a\Ab")); + assert_eq!(exact(["ab"]), e(r"a\zb")); + assert_eq!(exact(["ab"]), e(r"a(?m:^)b")); + assert_eq!(exact(["ab"]), e(r"a(?m:$)b")); + assert_eq!(exact(["ab"]), e(r"a\bb")); + assert_eq!(exact(["ab"]), e(r"a\Bb")); + assert_eq!(exact(["ab"]), e(r"a(?-u:\b)b")); + assert_eq!(exact(["ab"]), e(r"a(?-u:\B)b")); + + assert_eq!(exact(["ab"]), e(r"^ab")); + assert_eq!(exact(["ab"]), e(r"$ab")); + assert_eq!(exact(["ab"]), e(r"(?m:^)ab")); + assert_eq!(exact(["ab"]), e(r"(?m:$)ab")); + assert_eq!(exact(["ab"]), e(r"\bab")); + assert_eq!(exact(["ab"]), e(r"\Bab")); + assert_eq!(exact(["ab"]), e(r"(?-u:\b)ab")); + assert_eq!(exact(["ab"]), e(r"(?-u:\B)ab")); + + assert_eq!(exact(["ab"]), e(r"ab^")); + assert_eq!(exact(["ab"]), e(r"ab$")); + assert_eq!(exact(["ab"]), e(r"ab(?m:^)")); + assert_eq!(exact(["ab"]), e(r"ab(?m:$)")); + assert_eq!(exact(["ab"]), e(r"ab\b")); + assert_eq!(exact(["ab"]), e(r"ab\B")); + assert_eq!(exact(["ab"]), e(r"ab(?-u:\b)")); + assert_eq!(exact(["ab"]), e(r"ab(?-u:\B)")); + + let expected = (seq([I("aZ"), E("ab")]), seq([I("Zb"), E("ab")])); + assert_eq!(expected, e(r"^aZ*b")); + } + + #[test] + fn repetition() { + assert_eq!(exact(["a", ""]), e(r"a?")); + assert_eq!(exact(["", "a"]), e(r"a??")); + assert_eq!(inexact([I("a"), E("")], [I("a"), E("")]), e(r"a*")); + assert_eq!(inexact([E(""), I("a")], [E(""), I("a")]), e(r"a*?")); + assert_eq!(inexact([I("a")], [I("a")]), e(r"a+")); + assert_eq!(inexact([I("a")], [I("a")]), e(r"(a+)+")); + + assert_eq!(exact(["ab"]), e(r"aZ{0}b")); + assert_eq!(exact(["aZb", "ab"]), e(r"aZ?b")); + assert_eq!(exact(["ab", "aZb"]), e(r"aZ??b")); + assert_eq!( + inexact([I("aZ"), E("ab")], [I("Zb"), E("ab")]), + e(r"aZ*b") + ); + assert_eq!( + inexact([E("ab"), I("aZ")], [E("ab"), I("Zb")]), + e(r"aZ*?b") + ); + assert_eq!(inexact([I("aZ")], [I("Zb")]), e(r"aZ+b")); + assert_eq!(inexact([I("aZ")], [I("Zb")]), e(r"aZ+?b")); + + assert_eq!(exact(["aZZb"]), e(r"aZ{2}b")); + assert_eq!(inexact([I("aZZ")], [I("ZZb")]), e(r"aZ{2,3}b")); + + assert_eq!(exact(["abc", ""]), e(r"(abc)?")); + assert_eq!(exact(["", "abc"]), e(r"(abc)??")); + + assert_eq!(inexact([I("a"), E("b")], [I("ab"), E("b")]), e(r"a*b")); + assert_eq!(inexact([E("b"), I("a")], [E("b"), I("ab")]), e(r"a*?b")); + assert_eq!(inexact([I("ab")], [I("b")]), e(r"ab+")); + assert_eq!(inexact([I("a"), I("b")], [I("b")]), e(r"a*b+")); + + // FIXME: The suffixes for this don't look quite right to me. I think + // the right suffixes would be: [I(ac), I(bc), E(c)]. The main issue I + // think is that suffixes are computed by iterating over concatenations + // in reverse, and then [bc, ac, c] ordering is indeed correct from + // that perspective. We also test a few more equivalent regexes, and + // we get the same result, so it is consistent at least I suppose. + // + // The reason why this isn't an issue is that it only messes up + // preference order, and currently, suffixes are never used in a + // context where preference order matters. For prefixes it matters + // because we sometimes want to use prefilters without confirmation + // when all of the literals are exact (and there's no look-around). But + // we never do that for suffixes. Any time we use suffixes, we always + // include a confirmation step. If that ever changes, then it's likely + // this bug will need to be fixed, but last time I looked, it appears + // hard to do so. + assert_eq!( + inexact([I("a"), I("b"), E("c")], [I("bc"), I("ac"), E("c")]), + e(r"a*b*c") + ); + assert_eq!( + inexact([I("a"), I("b"), E("c")], [I("bc"), I("ac"), E("c")]), + e(r"(a+)?(b+)?c") + ); + assert_eq!( + inexact([I("a"), I("b"), E("c")], [I("bc"), I("ac"), E("c")]), + e(r"(a+|)(b+|)c") + ); + // A few more similarish but not identical regexes. These may have a + // similar problem as above. + assert_eq!( + inexact( + [I("a"), I("b"), I("c"), E("")], + [I("c"), I("b"), I("a"), E("")] + ), + e(r"a*b*c*") + ); + assert_eq!(inexact([I("a"), I("b"), I("c")], [I("c")]), e(r"a*b*c+")); + assert_eq!(inexact([I("a"), I("b")], [I("bc")]), e(r"a*b+c")); + assert_eq!(inexact([I("a"), I("b")], [I("c"), I("b")]), e(r"a*b+c*")); + assert_eq!(inexact([I("ab"), E("a")], [I("b"), E("a")]), e(r"ab*")); + assert_eq!( + inexact([I("ab"), E("ac")], [I("bc"), E("ac")]), + e(r"ab*c") + ); + assert_eq!(inexact([I("ab")], [I("b")]), e(r"ab+")); + assert_eq!(inexact([I("ab")], [I("bc")]), e(r"ab+c")); + + assert_eq!( + inexact([I("z"), E("azb")], [I("zazb"), E("azb")]), + e(r"z*azb") + ); + + let expected = + exact(["aaa", "aab", "aba", "abb", "baa", "bab", "bba", "bbb"]); + assert_eq!(expected, e(r"[ab]{3}")); + let expected = inexact( + [ + I("aaa"), + I("aab"), + I("aba"), + I("abb"), + I("baa"), + I("bab"), + I("bba"), + I("bbb"), + ], + [ + I("aaa"), + I("aab"), + I("aba"), + I("abb"), + I("baa"), + I("bab"), + I("bba"), + I("bbb"), + ], + ); + assert_eq!(expected, e(r"[ab]{3,4}")); + } + + #[test] + fn concat() { + let empty: [&str; 0] = []; + + assert_eq!(exact(["abcxyz"]), e(r"abc()xyz")); + assert_eq!(exact(["abcxyz"]), e(r"(abc)(xyz)")); + assert_eq!(exact(["abcmnoxyz"]), e(r"abc()mno()xyz")); + assert_eq!(exact(empty), e(r"abc[a&&b]xyz")); + assert_eq!(exact(["abcxyz"]), e(r"abc[a&&b]*xyz")); + } + + #[test] + fn alternation() { + assert_eq!(exact(["abc", "mno", "xyz"]), e(r"abc|mno|xyz")); + assert_eq!( + inexact( + [E("abc"), I("mZ"), E("mo"), E("xyz")], + [E("abc"), I("Zo"), E("mo"), E("xyz")] + ), + e(r"abc|mZ*o|xyz") + ); + assert_eq!(exact(["abc", "xyz"]), e(r"abc|M[a&&b]N|xyz")); + assert_eq!(exact(["abc", "MN", "xyz"]), e(r"abc|M[a&&b]*N|xyz")); + + assert_eq!(exact(["aaa", "aaaaa"]), e(r"(?:|aa)aaa")); + assert_eq!( + inexact( + [I("aaa"), E(""), I("aaaaa"), E("aa")], + [I("aaa"), E(""), E("aa")] + ), + e(r"(?:|aa)(?:aaa)*") + ); + assert_eq!( + inexact( + [E(""), I("aaa"), E("aa"), I("aaaaa")], + [E(""), I("aaa"), E("aa")] + ), + e(r"(?:|aa)(?:aaa)*?") + ); + + assert_eq!( + inexact([E("a"), I("b"), E("")], [E("a"), I("b"), E("")]), + e(r"a|b*") + ); + assert_eq!(inexact([E("a"), I("b")], [E("a"), I("b")]), e(r"a|b+")); + + assert_eq!( + inexact([I("a"), E("b"), E("c")], [I("ab"), E("b"), E("c")]), + e(r"a*b|c") + ); + + assert_eq!( + inexact( + [E("a"), E("b"), I("c"), E("")], + [E("a"), E("b"), I("c"), E("")] + ), + e(r"a|(?:b|c*)") + ); + + assert_eq!( + inexact( + [I("a"), I("b"), E("c"), I("a"), I("ab"), E("c")], + [I("ac"), I("bc"), E("c"), I("ac"), I("abc"), E("c")], + ), + e(r"(a|b)*c|(a|ab)*c") + ); + + assert_eq!( + exact(["abef", "abgh", "cdef", "cdgh"]), + e(r"(ab|cd)(ef|gh)") + ); + assert_eq!( + exact([ + "abefij", "abefkl", "abghij", "abghkl", "cdefij", "cdefkl", + "cdghij", "cdghkl", + ]), + e(r"(ab|cd)(ef|gh)(ij|kl)") + ); + + assert_eq!(inexact([E("abab")], [E("abab")]), e(r"(ab){2}")); + + assert_eq!(inexact([I("abab")], [I("abab")]), e(r"(ab){2,3}")); + + assert_eq!(inexact([I("abab")], [I("abab")]), e(r"(ab){2,}")); + } + + #[test] + fn impossible() { + let empty: [&str; 0] = []; + + assert_eq!(exact(empty), e(r"[a&&b]")); + assert_eq!(exact(empty), e(r"a[a&&b]")); + assert_eq!(exact(empty), e(r"[a&&b]b")); + assert_eq!(exact(empty), e(r"a[a&&b]b")); + assert_eq!(exact(["a", "b"]), e(r"a|[a&&b]|b")); + assert_eq!(exact(["a", "b"]), e(r"a|c[a&&b]|b")); + assert_eq!(exact(["a", "b"]), e(r"a|[a&&b]d|b")); + assert_eq!(exact(["a", "b"]), e(r"a|c[a&&b]d|b")); + assert_eq!(exact([""]), e(r"[a&&b]*")); + assert_eq!(exact(["MN"]), e(r"M[a&&b]*N")); + } + + // This tests patterns that contain something that defeats literal + // detection, usually because it would blow some limit on the total number + // of literals that can be returned. + // + // The main idea is that when literal extraction sees something that + // it knows will blow a limit, it replaces it with a marker that says + // "any literal will match here." While not necessarily true, the + // over-estimation is just fine for the purposes of literal extraction, + // because the imprecision doesn't matter: too big is too big. + // + // This is one of the trickier parts of literal extraction, since we need + // to make sure all of our literal extraction operations correctly compose + // with the markers. + #[test] + fn anything() { + assert_eq!(infinite(), e(r".")); + assert_eq!(infinite(), e(r"(?s).")); + assert_eq!(infinite(), e(r"[A-Za-z]")); + assert_eq!(infinite(), e(r"[A-Z]")); + assert_eq!(exact([""]), e(r"[A-Z]{0}")); + assert_eq!(infinite(), e(r"[A-Z]?")); + assert_eq!(infinite(), e(r"[A-Z]*")); + assert_eq!(infinite(), e(r"[A-Z]+")); + assert_eq!((seq([I("1")]), Seq::infinite()), e(r"1[A-Z]")); + assert_eq!((seq([I("1")]), seq([I("2")])), e(r"1[A-Z]2")); + assert_eq!((Seq::infinite(), seq([I("123")])), e(r"[A-Z]+123")); + assert_eq!(infinite(), e(r"[A-Z]+123[A-Z]+")); + assert_eq!(infinite(), e(r"1|[A-Z]|3")); + assert_eq!( + (seq([E("1"), I("2"), E("3")]), Seq::infinite()), + e(r"1|2[A-Z]|3"), + ); + assert_eq!( + (Seq::infinite(), seq([E("1"), I("2"), E("3")])), + e(r"1|[A-Z]2|3"), + ); + assert_eq!( + (seq([E("1"), I("2"), E("4")]), seq([E("1"), I("3"), E("4")])), + e(r"1|2[A-Z]3|4"), + ); + assert_eq!((Seq::infinite(), seq([I("2")])), e(r"(?:|1)[A-Z]2")); + assert_eq!(inexact([I("a")], [I("z")]), e(r"a.z")); + } + + // Like the 'anything' test, but it uses smaller limits in order to test + // the logic for effectively aborting literal extraction when the seqs get + // too big. + #[test] + fn anything_small_limits() { + fn prefixes(pattern: &str) -> Seq { + Extractor::new() + .kind(ExtractKind::Prefix) + .limit_total(10) + .extract(&parse(pattern)) + } + + fn suffixes(pattern: &str) -> Seq { + Extractor::new() + .kind(ExtractKind::Suffix) + .limit_total(10) + .extract(&parse(pattern)) + } + + fn e(pattern: &str) -> (Seq, Seq) { + (prefixes(pattern), suffixes(pattern)) + } + + assert_eq!( + ( + seq([ + I("aaa"), + I("aab"), + I("aba"), + I("abb"), + I("baa"), + I("bab"), + I("bba"), + I("bbb") + ]), + seq([ + I("aaa"), + I("aab"), + I("aba"), + I("abb"), + I("baa"), + I("bab"), + I("bba"), + I("bbb") + ]) + ), + e(r"[ab]{3}{3}") + ); + + assert_eq!(infinite(), e(r"ab|cd|ef|gh|ij|kl|mn|op|qr|st|uv|wx|yz")); + } + + #[test] + fn empty() { + assert_eq!(exact([""]), e(r"")); + assert_eq!(exact([""]), e(r"^")); + assert_eq!(exact([""]), e(r"$")); + assert_eq!(exact([""]), e(r"(?m:^)")); + assert_eq!(exact([""]), e(r"(?m:$)")); + assert_eq!(exact([""]), e(r"\b")); + assert_eq!(exact([""]), e(r"\B")); + assert_eq!(exact([""]), e(r"(?-u:\b)")); + assert_eq!(exact([""]), e(r"(?-u:\B)")); + } + + #[test] + fn odds_and_ends() { + assert_eq!((Seq::infinite(), seq([I("a")])), e(r".a")); + assert_eq!((seq([I("a")]), Seq::infinite()), e(r"a.")); + assert_eq!(infinite(), e(r"a|.")); + assert_eq!(infinite(), e(r".|a")); + + let pat = r"M[ou]'?am+[ae]r .*([AEae]l[- ])?[GKQ]h?[aeu]+([dtz][dhz]?)+af[iy]"; + let expected = inexact( + ["Mo'am", "Moam", "Mu'am", "Muam"].map(I), + [ + "ddafi", "ddafy", "dhafi", "dhafy", "dzafi", "dzafy", "dafi", + "dafy", "tdafi", "tdafy", "thafi", "thafy", "tzafi", "tzafy", + "tafi", "tafy", "zdafi", "zdafy", "zhafi", "zhafy", "zzafi", + "zzafy", "zafi", "zafy", + ] + .map(I), + ); + assert_eq!(expected, e(pat)); + + assert_eq!( + (seq(["fn is_", "fn as_"].map(I)), Seq::infinite()), + e(r"fn is_([A-Z]+)|fn as_([A-Z]+)"), + ); + assert_eq!( + inexact([I("foo")], [I("quux")]), + e(r"foo[A-Z]+bar[A-Z]+quux") + ); + assert_eq!(infinite(), e(r"[A-Z]+bar[A-Z]+")); + assert_eq!( + exact(["Sherlock Holmes"]), + e(r"(?m)^Sherlock Holmes|Sherlock Holmes$") + ); + + assert_eq!(exact(["sa", "sb"]), e(r"\bs(?:[ab])")); + } + + // This tests a specific regex along with some heuristic steps to reduce + // the sequences extracted. This is meant to roughly correspond to the + // types of heuristics used to shrink literal sets in practice. (Shrinking + // is done because you want to balance "spend too much work looking for + // too many literals" and "spend too much work processing false positive + // matches from short literals.") + #[test] + #[cfg(feature = "unicode-case")] + fn holmes() { + let expected = inexact( + ["HOL", "HOl", "HoL", "Hol", "hOL", "hOl", "hoL", "hol"].map(I), + [ + "MES", "MEs", "Eſ", "MeS", "Mes", "eſ", "mES", "mEs", "meS", + "mes", + ] + .map(I), + ); + let (mut prefixes, mut suffixes) = e(r"(?i)Holmes"); + prefixes.keep_first_bytes(3); + suffixes.keep_last_bytes(3); + prefixes.minimize_by_preference(); + suffixes.minimize_by_preference(); + assert_eq!(expected, (prefixes, suffixes)); + } + + // This tests that we get some kind of literals extracted for a beefier + // alternation with case insensitive mode enabled. At one point during + // development, this returned nothing, and motivated some special case + // code in Extractor::union to try and trim down the literal sequences + // if the union would blow the limits set. + #[test] + #[cfg(feature = "unicode-case")] + fn holmes_alt() { + let mut pre = + prefixes(r"(?i)Sherlock|Holmes|Watson|Irene|Adler|John|Baker"); + assert!(pre.len().unwrap() > 0); + pre.optimize_for_prefix_by_preference(); + assert!(pre.len().unwrap() > 0); + } + + // See: https://github.com/rust-lang/regex/security/advisories/GHSA-m5pq-gvj9-9vr8 + // See: CVE-2022-24713 + // + // We test this here to ensure literal extraction completes in reasonable + // time and isn't materially impacted by these sorts of pathological + // repeats. + #[test] + fn crazy_repeats() { + assert_eq!(inexact([E("")], [E("")]), e(r"(?:){4294967295}")); + assert_eq!( + inexact([E("")], [E("")]), + e(r"(?:){64}{64}{64}{64}{64}{64}") + ); + assert_eq!(inexact([E("")], [E("")]), e(r"x{0}{4294967295}")); + assert_eq!(inexact([E("")], [E("")]), e(r"(?:|){4294967295}")); + + assert_eq!( + inexact([E("")], [E("")]), + e(r"(?:){8}{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}") + ); + let repa = "a".repeat(100); + assert_eq!( + inexact([I(&repa)], [I(&repa)]), + e(r"a{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}{8}") + ); + } + + #[test] + fn huge() { + let pat = r#"(?-u) + 2(?: + [45]\d{3}| + 7(?: + 1[0-267]| + 2[0-289]| + 3[0-29]| + 4[01]| + 5[1-3]| + 6[013]| + 7[0178]| + 91 + )| + 8(?: + 0[125]| + [139][1-6]| + 2[0157-9]| + 41| + 6[1-35]| + 7[1-5]| + 8[1-8]| + 90 + )| + 9(?: + 0[0-2]| + 1[0-4]| + 2[568]| + 3[3-6]| + 5[5-7]| + 6[0167]| + 7[15]| + 8[0146-9] + ) + )\d{4}| + 3(?: + 12?[5-7]\d{2}| + 0(?: + 2(?: + [025-79]\d| + [348]\d{1,2} + )| + 3(?: + [2-4]\d| + [56]\d? + ) + )| + 2(?: + 1\d{2}| + 2(?: + [12]\d| + [35]\d{1,2}| + 4\d? + ) + )| + 3(?: + 1\d{2}| + 2(?: + [2356]\d| + 4\d{1,2} + ) + )| + 4(?: + 1\d{2}| + 2(?: + 2\d{1,2}| + [47]| + 5\d{2} + ) + )| + 5(?: + 1\d{2}| + 29 + )| + [67]1\d{2}| + 8(?: + 1\d{2}| + 2(?: + 2\d{2}| + 3| + 4\d + ) + ) + )\d{3}| + 4(?: + 0(?: + 2(?: + [09]\d| + 7 + )| + 33\d{2} + )| + 1\d{3}| + 2(?: + 1\d{2}| + 2(?: + [25]\d?| + [348]\d| + [67]\d{1,2} + ) + )| + 3(?: + 1\d{2}(?: + \d{2} + )?| + 2(?: + [045]\d| + [236-9]\d{1,2} + )| + 32\d{2} + )| + 4(?: + [18]\d{2}| + 2(?: + [2-46]\d{2}| + 3 + )| + 5[25]\d{2} + )| + 5(?: + 1\d{2}| + 2(?: + 3\d| + 5 + ) + )| + 6(?: + [18]\d{2}| + 2(?: + 3(?: + \d{2} + )?| + [46]\d{1,2}| + 5\d{2}| + 7\d + )| + 5(?: + 3\d?| + 4\d| + [57]\d{1,2}| + 6\d{2}| + 8 + ) + )| + 71\d{2}| + 8(?: + [18]\d{2}| + 23\d{2}| + 54\d{2} + )| + 9(?: + [18]\d{2}| + 2[2-5]\d{2}| + 53\d{1,2} + ) + )\d{3}| + 5(?: + 02[03489]\d{2}| + 1\d{2}| + 2(?: + 1\d{2}| + 2(?: + 2(?: + \d{2} + )?| + [457]\d{2} + ) + )| + 3(?: + 1\d{2}| + 2(?: + [37](?: + \d{2} + )?| + [569]\d{2} + ) + )| + 4(?: + 1\d{2}| + 2[46]\d{2} + )| + 5(?: + 1\d{2}| + 26\d{1,2} + )| + 6(?: + [18]\d{2}| + 2| + 53\d{2} + )| + 7(?: + 1| + 24 + )\d{2}| + 8(?: + 1| + 26 + )\d{2}| + 91\d{2} + )\d{3}| + 6(?: + 0(?: + 1\d{2}| + 2(?: + 3\d{2}| + 4\d{1,2} + ) + )| + 2(?: + 2[2-5]\d{2}| + 5(?: + [3-5]\d{2}| + 7 + )| + 8\d{2} + )| + 3(?: + 1| + 2[3478] + )\d{2}| + 4(?: + 1| + 2[34] + )\d{2}| + 5(?: + 1| + 2[47] + )\d{2}| + 6(?: + [18]\d{2}| + 6(?: + 2(?: + 2\d| + [34]\d{2} + )| + 5(?: + [24]\d{2}| + 3\d| + 5\d{1,2} + ) + ) + )| + 72[2-5]\d{2}| + 8(?: + 1\d{2}| + 2[2-5]\d{2} + )| + 9(?: + 1\d{2}| + 2[2-6]\d{2} + ) + )\d{3}| + 7(?: + (?: + 02| + [3-589]1| + 6[12]| + 72[24] + )\d{2}| + 21\d{3}| + 32 + )\d{3}| + 8(?: + (?: + 4[12]| + [5-7]2| + 1\d? + )| + (?: + 0| + 3[12]| + [5-7]1| + 217 + )\d + )\d{4}| + 9(?: + [35]1| + (?: + [024]2| + 81 + )\d| + (?: + 1| + [24]1 + )\d{2} + )\d{3} + "#; + // TODO: This is a good candidate of a seq of literals that could be + // shrunk quite a bit and still be very productive with respect to + // literal optimizations. + let (prefixes, suffixes) = e(pat); + assert!(!suffixes.is_finite()); + assert_eq!(Some(243), prefixes.len()); + } + + #[test] + fn optimize() { + // This gets a common prefix that isn't too short. + let (p, s) = + opt(["foobarfoobar", "foobar", "foobarzfoobar", "foobarfoobar"]); + assert_eq!(seq([I("foobar")]), p); + assert_eq!(seq([I("foobar")]), s); + + // This also finds a common prefix, but since it's only one byte, it + // prefers the multiple literals. + let (p, s) = opt(["abba", "akka", "abccba"]); + assert_eq!(exact(["abba", "akka", "abccba"]), (p, s)); + + let (p, s) = opt(["sam", "samwise"]); + assert_eq!((seq([E("sam")]), seq([E("sam"), E("samwise")])), (p, s)); + + // The empty string is poisonous, so our seq becomes infinite, even + // though all literals are exact. + let (p, s) = opt(["foobarfoo", "foo", "", "foozfoo", "foofoo"]); + assert!(!p.is_finite()); + assert!(!s.is_finite()); + + // A space is also poisonous, so our seq becomes infinite. But this + // only gets triggered when we don't have a completely exact sequence. + // When the sequence is exact, spaces are okay, since we presume that + // any prefilter will match a space more quickly than the regex engine. + // (When the sequence is exact, there's a chance of the prefilter being + // used without needing the regex engine at all.) + let mut p = seq([E("foobarfoo"), I("foo"), E(" "), E("foofoo")]); + p.optimize_for_prefix_by_preference(); + assert!(!p.is_finite()); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/hir/mod.rs b/third_party/crates/regex-syntax-0.8.11/src/hir/mod.rs new file mode 100644 index 0000000..6d57fe3 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/hir/mod.rs @@ -0,0 +1,3873 @@ +/*! +Defines a high-level intermediate (HIR) representation for regular expressions. + +The HIR is represented by the [`Hir`] type, and it principally constructed via +[translation](translate) from an [`Ast`](crate::ast::Ast). Alternatively, users +may use the smart constructors defined on `Hir` to build their own by hand. The +smart constructors simultaneously simplify and "optimize" the HIR, and are also +the same routines used by translation. + +Most regex engines only have an HIR like this, and usually construct it +directly from the concrete syntax. This crate however first parses the +concrete syntax into an `Ast`, and only then creates the HIR from the `Ast`, +as mentioned above. It's done this way to facilitate better error reporting, +and to have a structured representation of a regex that faithfully represents +its concrete syntax. Namely, while an `Hir` value can be converted back to an +equivalent regex pattern string, it is unlikely to look like the original due +to its simplified structure. +*/ + +use core::{char, cmp}; + +use alloc::{ + boxed::Box, + format, + string::{String, ToString}, + vec, + vec::Vec, +}; + +use crate::{ + ast::Span, + hir::interval::{Interval, IntervalSet, IntervalSetIter}, + unicode, +}; + +pub use crate::{ + hir::visitor::{visit, Visitor}, + unicode::CaseFoldError, +}; + +mod interval; +pub mod literal; +pub mod print; +pub mod translate; +mod visitor; + +/// An error that can occur while translating an `Ast` to a `Hir`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Error { + /// The kind of error. + kind: ErrorKind, + /// The original pattern that the translator's Ast was parsed from. Every + /// span in an error is a valid range into this string. + pattern: String, + /// The span of this error, derived from the Ast given to the translator. + span: Span, +} + +impl Error { + /// Return the type of this error. + pub fn kind(&self) -> &ErrorKind { + &self.kind + } + + /// The original pattern string in which this error occurred. + /// + /// Every span reported by this error is reported in terms of this string. + pub fn pattern(&self) -> &str { + &self.pattern + } + + /// Return the span at which this error occurred. + pub fn span(&self) -> &Span { + &self.span + } +} + +/// The type of an error that occurred while building an `Hir`. +/// +/// This error type is marked as `non_exhaustive`. This means that adding a +/// new variant is not considered a breaking change. +#[non_exhaustive] +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ErrorKind { + /// This error occurs when a Unicode feature is used when Unicode + /// support is disabled. For example `(?-u:\pL)` would trigger this error. + UnicodeNotAllowed, + /// This error occurs when translating a pattern that could match a byte + /// sequence that isn't UTF-8 and `utf8` was enabled. + InvalidUtf8, + /// This error occurs when one uses a non-ASCII byte for a line terminator, + /// but where Unicode mode is enabled and UTF-8 mode is disabled. + InvalidLineTerminator, + /// This occurs when an unrecognized Unicode property name could not + /// be found. + UnicodePropertyNotFound, + /// This occurs when an unrecognized Unicode property value could not + /// be found. + UnicodePropertyValueNotFound, + /// This occurs when a Unicode-aware Perl character class (`\w`, `\s` or + /// `\d`) could not be found. This can occur when the `unicode-perl` + /// crate feature is not enabled. + UnicodePerlClassNotFound, + /// This occurs when the Unicode simple case mapping tables are not + /// available, and the regular expression required Unicode aware case + /// insensitivity. + UnicodeCaseUnavailable, +} + +#[cfg(feature = "std")] +impl std::error::Error for Error {} + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::error::Formatter::from(self).fmt(f) + } +} + +impl core::fmt::Display for ErrorKind { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + use self::ErrorKind::*; + + let msg = match *self { + UnicodeNotAllowed => "Unicode not allowed here", + InvalidUtf8 => "pattern can match invalid UTF-8", + InvalidLineTerminator => "invalid line terminator, must be ASCII", + UnicodePropertyNotFound => "Unicode property not found", + UnicodePropertyValueNotFound => "Unicode property value not found", + UnicodePerlClassNotFound => { + "Unicode-aware Perl class not found \ + (make sure the unicode-perl feature is enabled)" + } + UnicodeCaseUnavailable => { + "Unicode-aware case insensitivity matching is not available \ + (make sure the unicode-case feature is enabled)" + } + }; + f.write_str(msg) + } +} + +/// A high-level intermediate representation (HIR) for a regular expression. +/// +/// An HIR value is a combination of a [`HirKind`] and a set of [`Properties`]. +/// An `HirKind` indicates what kind of regular expression it is (a literal, +/// a repetition, a look-around assertion, etc.), where as a `Properties` +/// describes various facts about the regular expression. For example, whether +/// it matches UTF-8 or if it matches the empty string. +/// +/// The HIR of a regular expression represents an intermediate step between +/// its abstract syntax (a structured description of the concrete syntax) and +/// an actual regex matcher. The purpose of HIR is to make regular expressions +/// easier to analyze. In particular, the AST is much more complex than the +/// HIR. For example, while an AST supports arbitrarily nested character +/// classes, the HIR will flatten all nested classes into a single set. The HIR +/// will also "compile away" every flag present in the concrete syntax. For +/// example, users of HIR expressions never need to worry about case folding; +/// it is handled automatically by the translator (e.g., by translating +/// `(?i:A)` to `[aA]`). +/// +/// The specific type of an HIR expression can be accessed via its `kind` +/// or `into_kind` methods. This extra level of indirection exists for two +/// reasons: +/// +/// 1. Construction of an HIR expression *must* use the constructor methods on +/// this `Hir` type instead of building the `HirKind` values directly. This +/// permits construction to enforce invariants like "concatenations always +/// consist of two or more sub-expressions." +/// 2. Every HIR expression contains attributes that are defined inductively, +/// and can be computed cheaply during the construction process. For example, +/// one such attribute is whether the expression must match at the beginning of +/// the haystack. +/// +/// In particular, if you have an `HirKind` value, then there is intentionally +/// no way to build an `Hir` value from it. You instead need to do case +/// analysis on the `HirKind` value and build the `Hir` value using its smart +/// constructors. +/// +/// # UTF-8 +/// +/// If the HIR was produced by a translator with +/// [`TranslatorBuilder::utf8`](translate::TranslatorBuilder::utf8) enabled, +/// then the HIR is guaranteed to match UTF-8 exclusively for all non-empty +/// matches. +/// +/// For empty matches, those can occur at any position. It is the +/// responsibility of the regex engine to determine whether empty matches are +/// permitted between the code units of a single codepoint. +/// +/// # Stack space +/// +/// This type defines its own destructor that uses constant stack space and +/// heap space proportional to the size of the HIR. +/// +/// Also, an `Hir`'s `fmt::Display` implementation prints an HIR as a regular +/// expression pattern string, and uses constant stack space and heap space +/// proportional to the size of the `Hir`. The regex it prints is guaranteed to +/// be _semantically_ equivalent to the original concrete syntax, but it may +/// look very different. (And potentially not practically readable by a human.) +/// +/// An `Hir`'s `fmt::Debug` implementation currently does not use constant +/// stack space. The implementation will also suppress some details (such as +/// the `Properties` inlined into every `Hir` value to make it less noisy). +#[derive(Clone, Eq, PartialEq)] +pub struct Hir { + /// The underlying HIR kind. + kind: HirKind, + /// Analysis info about this HIR, computed during construction. + props: Properties, +} + +/// Methods for accessing the underlying `HirKind` and `Properties`. +impl Hir { + /// Returns a reference to the underlying HIR kind. + pub fn kind(&self) -> &HirKind { + &self.kind + } + + /// Consumes ownership of this HIR expression and returns its underlying + /// `HirKind`. + pub fn into_kind(mut self) -> HirKind { + core::mem::replace(&mut self.kind, HirKind::Empty) + } + + /// Returns the properties computed for this `Hir`. + pub fn properties(&self) -> &Properties { + &self.props + } + + /// Splits this HIR into its constituent parts. + /// + /// This is useful because `let Hir { kind, props } = hir;` does not work + /// because of `Hir`'s custom `Drop` implementation. + fn into_parts(mut self) -> (HirKind, Properties) { + ( + core::mem::replace(&mut self.kind, HirKind::Empty), + core::mem::replace(&mut self.props, Properties::empty()), + ) + } +} + +/// Smart constructors for HIR values. +/// +/// These constructors are called "smart" because they do inductive work or +/// simplifications. For example, calling `Hir::repetition` with a repetition +/// like `a{0}` will actually return a `Hir` with a `HirKind::Empty` kind +/// since it is equivalent to an empty regex. Another example is calling +/// `Hir::concat(vec![expr])`. Instead of getting a `HirKind::Concat`, you'll +/// just get back the original `expr` since it's precisely equivalent. +/// +/// Smart constructors enable maintaining invariants about the HIR data type +/// while also simultaneously keeping the representation as simple as possible. +impl Hir { + /// Returns an empty HIR expression. + /// + /// An empty HIR expression always matches, including the empty string. + #[inline] + pub fn empty() -> Hir { + let props = Properties::empty(); + Hir { kind: HirKind::Empty, props } + } + + /// Returns an HIR expression that can never match anything. That is, + /// the size of the set of strings in the language described by the HIR + /// returned is `0`. + /// + /// This is distinct from [`Hir::empty`] in that the empty string matches + /// the HIR returned by `Hir::empty`. That is, the set of strings in the + /// language describe described by `Hir::empty` is non-empty. + /// + /// Note that currently, the HIR returned uses an empty character class to + /// indicate that nothing can match. An equivalent expression that cannot + /// match is an empty alternation, but all such "fail" expressions are + /// normalized (via smart constructors) to empty character classes. This is + /// because empty character classes can be spelled in the concrete syntax + /// of a regex (e.g., `\P{any}` or `(?-u:[^\x00-\xFF])` or `[a&&b]`), but + /// empty alternations cannot. + #[inline] + pub fn fail() -> Hir { + let class = Class::Bytes(ClassBytes::empty()); + let props = Properties::class(&class); + // We can't just call Hir::class here because it defers to Hir::fail + // in order to canonicalize the Hir value used to represent "cannot + // match." + Hir { kind: HirKind::Class(class), props } + } + + /// Creates a literal HIR expression. + /// + /// This accepts anything that can be converted into a `Box<[u8]>`. + /// + /// Note that there is no mechanism for storing a `char` or a `Box<str>` + /// in an HIR. Everything is "just bytes." Whether a `Literal` (or + /// any HIR node) matches valid UTF-8 exclusively can be queried via + /// [`Properties::is_utf8`]. + /// + /// # Example + /// + /// This example shows that concatenations of `Literal` HIR values will + /// automatically get flattened and combined together. So for example, even + /// if you concat multiple `Literal` values that are themselves not valid + /// UTF-8, they might add up to valid UTF-8. This also demonstrates just + /// how "smart" Hir's smart constructors are. + /// + /// ``` + /// use regex_syntax::hir::{Hir, HirKind, Literal}; + /// + /// let literals = vec![ + /// Hir::literal([0xE2]), + /// Hir::literal([0x98]), + /// Hir::literal([0x83]), + /// ]; + /// // Each literal, on its own, is invalid UTF-8. + /// assert!(literals.iter().all(|hir| !hir.properties().is_utf8())); + /// + /// let concat = Hir::concat(literals); + /// // But the concatenation is valid UTF-8! + /// assert!(concat.properties().is_utf8()); + /// + /// // And also notice that the literals have been concatenated into a + /// // single `Literal`, to the point where there is no explicit `Concat`! + /// let expected = HirKind::Literal(Literal(Box::from("☃".as_bytes()))); + /// assert_eq!(&expected, concat.kind()); + /// ``` + /// + /// # Example: building a literal from a `char` + /// + /// This example shows how to build a single `Hir` literal from a `char` + /// value. Since a [`Literal`] is just bytes, we just need to UTF-8 + /// encode a `char` value: + /// + /// ``` + /// use regex_syntax::hir::{Hir, HirKind, Literal}; + /// + /// let ch = '☃'; + /// let got = Hir::literal(ch.encode_utf8(&mut [0; 4]).as_bytes()); + /// + /// let expected = HirKind::Literal(Literal(Box::from("☃".as_bytes()))); + /// assert_eq!(&expected, got.kind()); + /// ``` + #[inline] + pub fn literal<B: Into<Box<[u8]>>>(lit: B) -> Hir { + let bytes = lit.into(); + if bytes.is_empty() { + return Hir::empty(); + } + + let lit = Literal(bytes); + let props = Properties::literal(&lit); + Hir { kind: HirKind::Literal(lit), props } + } + + /// Creates a class HIR expression. The class may either be defined over + /// ranges of Unicode codepoints or ranges of raw byte values. + /// + /// Note that an empty class is permitted. An empty class is equivalent to + /// `Hir::fail()`. + #[inline] + pub fn class(class: Class) -> Hir { + if class.is_empty() { + return Hir::fail(); + } else if let Some(bytes) = class.literal() { + return Hir::literal(bytes); + } + let props = Properties::class(&class); + Hir { kind: HirKind::Class(class), props } + } + + /// Creates a look-around assertion HIR expression. + #[inline] + pub fn look(look: Look) -> Hir { + let props = Properties::look(look); + Hir { kind: HirKind::Look(look), props } + } + + /// Creates a repetition HIR expression. + #[inline] + pub fn repetition(mut rep: Repetition) -> Hir { + // If the sub-expression of a repetition can only match the empty + // string, then we force its maximum to be at most 1. + if rep.sub.properties().maximum_len() == Some(0) { + rep.min = cmp::min(rep.min, 1); + rep.max = rep.max.map(|n| cmp::min(n, 1)).or(Some(1)); + } + // The regex 'a{0}' is always equivalent to the empty regex. This is + // true even when 'a' is an expression that never matches anything + // (like '\P{any}'). + // + // Additionally, the regex 'a{1}' is always equivalent to 'a'. + if rep.min == 0 && rep.max == Some(0) { + return Hir::empty(); + } else if rep.min == 1 && rep.max == Some(1) { + return *rep.sub; + } + let props = Properties::repetition(&rep); + Hir { kind: HirKind::Repetition(rep), props } + } + + /// Creates a capture HIR expression. + /// + /// Note that there is no explicit HIR value for a non-capturing group. + /// Since a non-capturing group only exists to override precedence in the + /// concrete syntax and since an HIR already does its own grouping based on + /// what is parsed, there is no need to explicitly represent non-capturing + /// groups in the HIR. + #[inline] + pub fn capture(capture: Capture) -> Hir { + let props = Properties::capture(&capture); + Hir { kind: HirKind::Capture(capture), props } + } + + /// Returns the concatenation of the given expressions. + /// + /// This attempts to flatten and simplify the concatenation as appropriate. + /// + /// # Example + /// + /// This shows a simple example of basic flattening of both concatenations + /// and literals. + /// + /// ``` + /// use regex_syntax::hir::Hir; + /// + /// let hir = Hir::concat(vec![ + /// Hir::concat(vec![ + /// Hir::literal([b'a']), + /// Hir::literal([b'b']), + /// Hir::literal([b'c']), + /// ]), + /// Hir::concat(vec![ + /// Hir::literal([b'x']), + /// Hir::literal([b'y']), + /// Hir::literal([b'z']), + /// ]), + /// ]); + /// let expected = Hir::literal("abcxyz".as_bytes()); + /// assert_eq!(expected, hir); + /// ``` + pub fn concat(subs: Vec<Hir>) -> Hir { + // We rebuild the concatenation by simplifying it. Would be nice to do + // it in place, but that seems a little tricky? + let mut new = vec![]; + // This gobbles up any adjacent literals in a concatenation and smushes + // them together. Basically, when we see a literal, we add its bytes + // to 'prior_lit', and whenever we see anything else, we first take + // any bytes in 'prior_lit' and add it to the 'new' concatenation. + let mut prior_lit: Option<Vec<u8>> = None; + for sub in subs { + let (kind, props) = sub.into_parts(); + match kind { + HirKind::Literal(Literal(bytes)) => { + if let Some(ref mut prior_bytes) = prior_lit { + prior_bytes.extend_from_slice(&bytes); + } else { + prior_lit = Some(bytes.to_vec()); + } + } + // We also flatten concats that are direct children of another + // concat. We only need to do this one level deep since + // Hir::concat is the only way to build concatenations, and so + // flattening happens inductively. + HirKind::Concat(subs2) => { + for sub2 in subs2 { + let (kind2, props2) = sub2.into_parts(); + match kind2 { + HirKind::Literal(Literal(bytes)) => { + if let Some(ref mut prior_bytes) = prior_lit { + prior_bytes.extend_from_slice(&bytes); + } else { + prior_lit = Some(bytes.to_vec()); + } + } + kind2 => { + if let Some(prior_bytes) = prior_lit.take() { + new.push(Hir::literal(prior_bytes)); + } + new.push(Hir { kind: kind2, props: props2 }); + } + } + } + } + // We can just skip empty HIRs. + HirKind::Empty => {} + kind => { + if let Some(prior_bytes) = prior_lit.take() { + new.push(Hir::literal(prior_bytes)); + } + new.push(Hir { kind, props }); + } + } + } + if let Some(prior_bytes) = prior_lit.take() { + new.push(Hir::literal(prior_bytes)); + } + if new.is_empty() { + return Hir::empty(); + } else if new.len() == 1 { + return new.pop().unwrap(); + } + let props = Properties::concat(&new); + Hir { kind: HirKind::Concat(new), props } + } + + /// Returns the alternation of the given expressions. + /// + /// This flattens and simplifies the alternation as appropriate. This may + /// include factoring out common prefixes or even rewriting the alternation + /// as a character class. + /// + /// Note that an empty alternation is equivalent to `Hir::fail()`. (It + /// is not possible for one to write an empty alternation, or even an + /// alternation with a single sub-expression, in the concrete syntax of a + /// regex.) + /// + /// # Example + /// + /// This is a simple example showing how an alternation might get + /// simplified. + /// + /// ``` + /// use regex_syntax::hir::{Hir, Class, ClassUnicode, ClassUnicodeRange}; + /// + /// let hir = Hir::alternation(vec![ + /// Hir::literal([b'a']), + /// Hir::literal([b'b']), + /// Hir::literal([b'c']), + /// Hir::literal([b'd']), + /// Hir::literal([b'e']), + /// Hir::literal([b'f']), + /// ]); + /// let expected = Hir::class(Class::Unicode(ClassUnicode::new([ + /// ClassUnicodeRange::new('a', 'f'), + /// ]))); + /// assert_eq!(expected, hir); + /// ``` + /// + /// And another example showing how common prefixes might get factored + /// out. + /// + /// ``` + /// use regex_syntax::hir::{Hir, Class, ClassUnicode, ClassUnicodeRange}; + /// + /// let hir = Hir::alternation(vec![ + /// Hir::concat(vec![ + /// Hir::literal("abc".as_bytes()), + /// Hir::class(Class::Unicode(ClassUnicode::new([ + /// ClassUnicodeRange::new('A', 'Z'), + /// ]))), + /// ]), + /// Hir::concat(vec![ + /// Hir::literal("abc".as_bytes()), + /// Hir::class(Class::Unicode(ClassUnicode::new([ + /// ClassUnicodeRange::new('a', 'z'), + /// ]))), + /// ]), + /// ]); + /// let expected = Hir::concat(vec![ + /// Hir::literal("abc".as_bytes()), + /// Hir::alternation(vec![ + /// Hir::class(Class::Unicode(ClassUnicode::new([ + /// ClassUnicodeRange::new('A', 'Z'), + /// ]))), + /// Hir::class(Class::Unicode(ClassUnicode::new([ + /// ClassUnicodeRange::new('a', 'z'), + /// ]))), + /// ]), + /// ]); + /// assert_eq!(expected, hir); + /// ``` + /// + /// Note that these sorts of simplifications are not guaranteed. + pub fn alternation(subs: Vec<Hir>) -> Hir { + // We rebuild the alternation by simplifying it. We proceed similarly + // as the concatenation case. But in this case, there's no literal + // simplification happening. We're just flattening alternations. + let mut new = Vec::with_capacity(subs.len()); + for sub in subs { + let (kind, props) = sub.into_parts(); + match kind { + HirKind::Alternation(subs2) => { + new.extend(subs2); + } + kind => { + new.push(Hir { kind, props }); + } + } + } + if new.is_empty() { + return Hir::fail(); + } else if new.len() == 1 { + return new.pop().unwrap(); + } + // Now that it's completely flattened, look for the special case of + // 'char1|char2|...|charN' and collapse that into a class. Note that + // we look for 'char' first and then bytes. The issue here is that if + // we find both non-ASCII codepoints and non-ASCII singleton bytes, + // then it isn't actually possible to smush them into a single class. + // (Because classes are either "all codepoints" or "all bytes." You + // can have a class that both matches non-ASCII but valid UTF-8 and + // invalid UTF-8.) So we look for all chars and then all bytes, and + // don't handle anything else. + if let Some(singletons) = singleton_chars(&new) { + let it = singletons + .into_iter() + .map(|ch| ClassUnicodeRange { start: ch, end: ch }); + return Hir::class(Class::Unicode(ClassUnicode::new(it))); + } + if let Some(singletons) = singleton_bytes(&new) { + let it = singletons + .into_iter() + .map(|b| ClassBytesRange { start: b, end: b }); + return Hir::class(Class::Bytes(ClassBytes::new(it))); + } + // Similar to singleton chars, we can also look for alternations of + // classes. Those can be smushed into a single class. + if let Some(cls) = class_chars(&new) { + return Hir::class(cls); + } + if let Some(cls) = class_bytes(&new) { + return Hir::class(cls); + } + // Factor out a common prefix if we can, which might potentially + // simplify the expression and unlock other optimizations downstream. + // It also might generally make NFA matching and DFA construction + // faster by reducing the scope of branching in the regex. + new = match lift_common_prefix(new) { + Ok(hir) => return hir, + Err(unchanged) => unchanged, + }; + let props = Properties::alternation(&new); + Hir { kind: HirKind::Alternation(new), props } + } + + /// Returns an HIR expression for `.`. + /// + /// * [`Dot::AnyChar`] maps to `(?su-R:.)`. + /// * [`Dot::AnyByte`] maps to `(?s-Ru:.)`. + /// * [`Dot::AnyCharExceptLF`] maps to `(?u-Rs:.)`. + /// * [`Dot::AnyCharExceptCRLF`] maps to `(?Ru-s:.)`. + /// * [`Dot::AnyByteExceptLF`] maps to `(?-Rsu:.)`. + /// * [`Dot::AnyByteExceptCRLF`] maps to `(?R-su:.)`. + /// + /// # Example + /// + /// Note that this is a convenience routine for constructing the correct + /// character class based on the value of `Dot`. There is no explicit "dot" + /// HIR value. It is just an abbreviation for a common character class. + /// + /// ``` + /// use regex_syntax::hir::{Hir, Dot, Class, ClassBytes, ClassBytesRange}; + /// + /// let hir = Hir::dot(Dot::AnyByte); + /// let expected = Hir::class(Class::Bytes(ClassBytes::new([ + /// ClassBytesRange::new(0x00, 0xFF), + /// ]))); + /// assert_eq!(expected, hir); + /// ``` + #[inline] + pub fn dot(dot: Dot) -> Hir { + match dot { + Dot::AnyChar => Hir::class(Class::Unicode(ClassUnicode::new([ + ClassUnicodeRange::new('\0', '\u{10FFFF}'), + ]))), + Dot::AnyByte => Hir::class(Class::Bytes(ClassBytes::new([ + ClassBytesRange::new(b'\0', b'\xFF'), + ]))), + Dot::AnyCharExcept(ch) => { + let mut cls = + ClassUnicode::new([ClassUnicodeRange::new(ch, ch)]); + cls.negate(); + Hir::class(Class::Unicode(cls)) + } + Dot::AnyCharExceptLF => { + Hir::class(Class::Unicode(ClassUnicode::new([ + ClassUnicodeRange::new('\0', '\x09'), + ClassUnicodeRange::new('\x0B', '\u{10FFFF}'), + ]))) + } + Dot::AnyCharExceptCRLF => { + Hir::class(Class::Unicode(ClassUnicode::new([ + ClassUnicodeRange::new('\0', '\x09'), + ClassUnicodeRange::new('\x0B', '\x0C'), + ClassUnicodeRange::new('\x0E', '\u{10FFFF}'), + ]))) + } + Dot::AnyByteExcept(byte) => { + let mut cls = + ClassBytes::new([ClassBytesRange::new(byte, byte)]); + cls.negate(); + Hir::class(Class::Bytes(cls)) + } + Dot::AnyByteExceptLF => { + Hir::class(Class::Bytes(ClassBytes::new([ + ClassBytesRange::new(b'\0', b'\x09'), + ClassBytesRange::new(b'\x0B', b'\xFF'), + ]))) + } + Dot::AnyByteExceptCRLF => { + Hir::class(Class::Bytes(ClassBytes::new([ + ClassBytesRange::new(b'\0', b'\x09'), + ClassBytesRange::new(b'\x0B', b'\x0C'), + ClassBytesRange::new(b'\x0E', b'\xFF'), + ]))) + } + } + } +} + +/// The underlying kind of an arbitrary [`Hir`] expression. +/// +/// An `HirKind` is principally useful for doing case analysis on the type +/// of a regular expression. If you're looking to build new `Hir` values, +/// then you _must_ use the smart constructors defined on `Hir`, like +/// [`Hir::repetition`], to build new `Hir` values. The API intentionally does +/// not expose any way of building an `Hir` directly from an `HirKind`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum HirKind { + /// The empty regular expression, which matches everything, including the + /// empty string. + Empty, + /// A literal string that matches exactly these bytes. + Literal(Literal), + /// A single character class that matches any of the characters in the + /// class. A class can either consist of Unicode scalar values as + /// characters, or it can use bytes. + /// + /// A class may be empty. In which case, it matches nothing. + Class(Class), + /// A look-around assertion. A look-around match always has zero length. + Look(Look), + /// A repetition operation applied to a sub-expression. + Repetition(Repetition), + /// A capturing group, which contains a sub-expression. + Capture(Capture), + /// A concatenation of expressions. + /// + /// A concatenation matches only if each of its sub-expressions match one + /// after the other. + /// + /// Concatenations are guaranteed by `Hir`'s smart constructors to always + /// have at least two sub-expressions. + Concat(Vec<Hir>), + /// An alternation of expressions. + /// + /// An alternation matches only if at least one of its sub-expressions + /// match. If multiple sub-expressions match, then the leftmost is + /// preferred. + /// + /// Alternations are guaranteed by `Hir`'s smart constructors to always + /// have at least two sub-expressions. + Alternation(Vec<Hir>), +} + +impl HirKind { + /// Returns a slice of this kind's sub-expressions, if any. + pub fn subs(&self) -> &[Hir] { + use core::slice::from_ref; + + match *self { + HirKind::Empty + | HirKind::Literal(_) + | HirKind::Class(_) + | HirKind::Look(_) => &[], + HirKind::Repetition(Repetition { ref sub, .. }) => from_ref(sub), + HirKind::Capture(Capture { ref sub, .. }) => from_ref(sub), + HirKind::Concat(ref subs) => subs, + HirKind::Alternation(ref subs) => subs, + } + } +} + +impl core::fmt::Debug for Hir { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + self.kind.fmt(f) + } +} + +/// Print a display representation of this Hir. +/// +/// The result of this is a valid regular expression pattern string. +/// +/// This implementation uses constant stack space and heap space proportional +/// to the size of the `Hir`. +impl core::fmt::Display for Hir { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + crate::hir::print::Printer::new().print(self, f) + } +} + +/// The high-level intermediate representation of a literal. +/// +/// A literal corresponds to `0` or more bytes that should be matched +/// literally. The smart constructors defined on `Hir` will automatically +/// concatenate adjacent literals into one literal, and will even automatically +/// replace empty literals with `Hir::empty()`. +/// +/// Note that despite a literal being represented by a sequence of bytes, its +/// `Debug` implementation will attempt to print it as a normal string. (That +/// is, not a sequence of decimal numbers.) +#[derive(Clone, Eq, PartialEq)] +pub struct Literal(pub Box<[u8]>); + +impl core::fmt::Debug for Literal { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + crate::debug::Bytes(&self.0).fmt(f) + } +} + +/// The high-level intermediate representation of a character class. +/// +/// A character class corresponds to a set of characters. A character is either +/// defined by a Unicode scalar value or a byte. +/// +/// A character class, regardless of its character type, is represented by a +/// sequence of non-overlapping non-adjacent ranges of characters. +/// +/// There are no guarantees about which class variant is used. Generally +/// speaking, the Unicode variant is used whenever a class needs to contain +/// non-ASCII Unicode scalar values. But the Unicode variant can be used even +/// when Unicode mode is disabled. For example, at the time of writing, the +/// regex `(?-u:a|\xc2\xa0)` will compile down to HIR for the Unicode class +/// `[a\u00A0]` due to optimizations. +/// +/// Note that `Bytes` variant may be produced even when it exclusively matches +/// valid UTF-8. This is because a `Bytes` variant represents an intention by +/// the author of the regular expression to disable Unicode mode, which in turn +/// impacts the semantics of case insensitive matching. For example, `(?i)k` +/// and `(?i-u)k` will not match the same set of strings. +#[derive(Clone, Eq, PartialEq)] +pub enum Class { + /// A set of characters represented by Unicode scalar values. + Unicode(ClassUnicode), + /// A set of characters represented by arbitrary bytes (one byte per + /// character). + Bytes(ClassBytes), +} + +impl Class { + /// Apply Unicode simple case folding to this character class, in place. + /// The character class will be expanded to include all simple case folded + /// character variants. + /// + /// If this is a byte oriented character class, then this will be limited + /// to the ASCII ranges `A-Z` and `a-z`. + /// + /// # Panics + /// + /// This routine panics when the case mapping data necessary for this + /// routine to complete is unavailable. This occurs when the `unicode-case` + /// feature is not enabled and the underlying class is Unicode oriented. + /// + /// Callers should prefer using `try_case_fold_simple` instead, which will + /// return an error instead of panicking. + pub fn case_fold_simple(&mut self) { + match *self { + Class::Unicode(ref mut x) => x.case_fold_simple(), + Class::Bytes(ref mut x) => x.case_fold_simple(), + } + } + + /// Apply Unicode simple case folding to this character class, in place. + /// The character class will be expanded to include all simple case folded + /// character variants. + /// + /// If this is a byte oriented character class, then this will be limited + /// to the ASCII ranges `A-Z` and `a-z`. + /// + /// # Error + /// + /// This routine returns an error when the case mapping data necessary + /// for this routine to complete is unavailable. This occurs when the + /// `unicode-case` feature is not enabled and the underlying class is + /// Unicode oriented. + pub fn try_case_fold_simple( + &mut self, + ) -> core::result::Result<(), CaseFoldError> { + match *self { + Class::Unicode(ref mut x) => x.try_case_fold_simple()?, + Class::Bytes(ref mut x) => x.case_fold_simple(), + } + Ok(()) + } + + /// Negate this character class in place. + /// + /// After completion, this character class will contain precisely the + /// characters that weren't previously in the class. + pub fn negate(&mut self) { + match *self { + Class::Unicode(ref mut x) => x.negate(), + Class::Bytes(ref mut x) => x.negate(), + } + } + + /// Returns true if and only if this character class will only ever match + /// valid UTF-8. + /// + /// A character class can match invalid UTF-8 only when the following + /// conditions are met: + /// + /// 1. The translator was configured to permit generating an expression + /// that can match invalid UTF-8. (By default, this is disabled.) + /// 2. Unicode mode (via the `u` flag) was disabled either in the concrete + /// syntax or in the parser builder. By default, Unicode mode is + /// enabled. + pub fn is_utf8(&self) -> bool { + match *self { + Class::Unicode(_) => true, + Class::Bytes(ref x) => x.is_ascii(), + } + } + + /// Returns the length, in bytes, of the smallest string matched by this + /// character class. + /// + /// For non-empty byte oriented classes, this always returns `1`. For + /// non-empty Unicode oriented classes, this can return `1`, `2`, `3` or + /// `4`. For empty classes, `None` is returned. It is impossible for `0` to + /// be returned. + /// + /// # Example + /// + /// This example shows some examples of regexes and their corresponding + /// minimum length, if any. + /// + /// ``` + /// use regex_syntax::{hir::Properties, parse}; + /// + /// // The empty string has a min length of 0. + /// let hir = parse(r"")?; + /// assert_eq!(Some(0), hir.properties().minimum_len()); + /// // As do other types of regexes that only match the empty string. + /// let hir = parse(r"^$\b\B")?; + /// assert_eq!(Some(0), hir.properties().minimum_len()); + /// // A regex that can match the empty string but match more is still 0. + /// let hir = parse(r"a*")?; + /// assert_eq!(Some(0), hir.properties().minimum_len()); + /// // A regex that matches nothing has no minimum defined. + /// let hir = parse(r"[a&&b]")?; + /// assert_eq!(None, hir.properties().minimum_len()); + /// // Character classes usually have a minimum length of 1. + /// let hir = parse(r"\w")?; + /// assert_eq!(Some(1), hir.properties().minimum_len()); + /// // But sometimes Unicode classes might be bigger! + /// let hir = parse(r"\p{Cyrillic}")?; + /// assert_eq!(Some(2), hir.properties().minimum_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn minimum_len(&self) -> Option<usize> { + match *self { + Class::Unicode(ref x) => x.minimum_len(), + Class::Bytes(ref x) => x.minimum_len(), + } + } + + /// Returns the length, in bytes, of the longest string matched by this + /// character class. + /// + /// For non-empty byte oriented classes, this always returns `1`. For + /// non-empty Unicode oriented classes, this can return `1`, `2`, `3` or + /// `4`. For empty classes, `None` is returned. It is impossible for `0` to + /// be returned. + /// + /// # Example + /// + /// This example shows some examples of regexes and their corresponding + /// maximum length, if any. + /// + /// ``` + /// use regex_syntax::{hir::Properties, parse}; + /// + /// // The empty string has a max length of 0. + /// let hir = parse(r"")?; + /// assert_eq!(Some(0), hir.properties().maximum_len()); + /// // As do other types of regexes that only match the empty string. + /// let hir = parse(r"^$\b\B")?; + /// assert_eq!(Some(0), hir.properties().maximum_len()); + /// // A regex that matches nothing has no maximum defined. + /// let hir = parse(r"[a&&b]")?; + /// assert_eq!(None, hir.properties().maximum_len()); + /// // Bounded repeats work as you expect. + /// let hir = parse(r"x{2,10}")?; + /// assert_eq!(Some(10), hir.properties().maximum_len()); + /// // An unbounded repeat means there is no maximum. + /// let hir = parse(r"x{2,}")?; + /// assert_eq!(None, hir.properties().maximum_len()); + /// // With Unicode enabled, \w can match up to 4 bytes! + /// let hir = parse(r"\w")?; + /// assert_eq!(Some(4), hir.properties().maximum_len()); + /// // Without Unicode enabled, \w matches at most 1 byte. + /// let hir = parse(r"(?-u)\w")?; + /// assert_eq!(Some(1), hir.properties().maximum_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn maximum_len(&self) -> Option<usize> { + match *self { + Class::Unicode(ref x) => x.maximum_len(), + Class::Bytes(ref x) => x.maximum_len(), + } + } + + /// Returns true if and only if this character class is empty. That is, + /// it has no elements. + /// + /// An empty character can never match anything, including an empty string. + pub fn is_empty(&self) -> bool { + match *self { + Class::Unicode(ref x) => x.ranges().is_empty(), + Class::Bytes(ref x) => x.ranges().is_empty(), + } + } + + /// If this class consists of exactly one element (whether a codepoint or a + /// byte), then return it as a literal byte string. + /// + /// If this class is empty or contains more than one element, then `None` + /// is returned. + pub fn literal(&self) -> Option<Vec<u8>> { + match *self { + Class::Unicode(ref x) => x.literal(), + Class::Bytes(ref x) => x.literal(), + } + } +} + +impl core::fmt::Debug for Class { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + use crate::debug::Byte; + + let mut fmter = f.debug_set(); + match *self { + Class::Unicode(ref cls) => { + for r in cls.ranges().iter() { + fmter.entry(&(r.start..=r.end)); + } + } + Class::Bytes(ref cls) => { + for r in cls.ranges().iter() { + fmter.entry(&(Byte(r.start)..=Byte(r.end))); + } + } + } + fmter.finish() + } +} + +/// A set of characters represented by Unicode scalar values. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ClassUnicode { + set: IntervalSet<ClassUnicodeRange>, +} + +impl ClassUnicode { + /// Create a new class from a sequence of ranges. + /// + /// The given ranges do not need to be in any specific order, and ranges + /// may overlap. Ranges will automatically be sorted into a canonical + /// non-overlapping order. + pub fn new<I>(ranges: I) -> ClassUnicode + where + I: IntoIterator<Item = ClassUnicodeRange>, + { + ClassUnicode { set: IntervalSet::new(ranges) } + } + + /// Create a new class with no ranges. + /// + /// An empty class matches nothing. That is, it is equivalent to + /// [`Hir::fail`]. + pub fn empty() -> ClassUnicode { + ClassUnicode::new(vec![]) + } + + /// Add a new range to this set. + pub fn push(&mut self, range: ClassUnicodeRange) { + self.set.push(range); + } + + /// Return an iterator over all ranges in this class. + /// + /// The iterator yields ranges in ascending order. + pub fn iter(&self) -> ClassUnicodeIter<'_> { + ClassUnicodeIter(self.set.iter()) + } + + /// Return the underlying ranges as a slice. + pub fn ranges(&self) -> &[ClassUnicodeRange] { + self.set.intervals() + } + + /// Expand this character class such that it contains all case folded + /// characters, according to Unicode's "simple" mapping. For example, if + /// this class consists of the range `a-z`, then applying case folding will + /// result in the class containing both the ranges `a-z` and `A-Z`. + /// + /// # Panics + /// + /// This routine panics when the case mapping data necessary for this + /// routine to complete is unavailable. This occurs when the `unicode-case` + /// feature is not enabled. + /// + /// Callers should prefer using `try_case_fold_simple` instead, which will + /// return an error instead of panicking. + pub fn case_fold_simple(&mut self) { + self.set + .case_fold_simple() + .expect("unicode-case feature must be enabled"); + } + + /// Expand this character class such that it contains all case folded + /// characters, according to Unicode's "simple" mapping. For example, if + /// this class consists of the range `a-z`, then applying case folding will + /// result in the class containing both the ranges `a-z` and `A-Z`. + /// + /// # Error + /// + /// This routine returns an error when the case mapping data necessary + /// for this routine to complete is unavailable. This occurs when the + /// `unicode-case` feature is not enabled. + pub fn try_case_fold_simple( + &mut self, + ) -> core::result::Result<(), CaseFoldError> { + self.set.case_fold_simple() + } + + /// Negate this character class. + /// + /// For all `c` where `c` is a Unicode scalar value, if `c` was in this + /// set, then it will not be in this set after negation. + pub fn negate(&mut self) { + self.set.negate(); + } + + /// Union this character class with the given character class, in place. + pub fn union(&mut self, other: &ClassUnicode) { + self.set.union(&other.set); + } + + /// Intersect this character class with the given character class, in + /// place. + pub fn intersect(&mut self, other: &ClassUnicode) { + self.set.intersect(&other.set); + } + + /// Subtract the given character class from this character class, in place. + pub fn difference(&mut self, other: &ClassUnicode) { + self.set.difference(&other.set); + } + + /// Compute the symmetric difference of the given character classes, in + /// place. + /// + /// This computes the symmetric difference of two character classes. This + /// removes all elements in this class that are also in the given class, + /// but all adds all elements from the given class that aren't in this + /// class. That is, the class will contain all elements in either class, + /// but will not contain any elements that are in both classes. + pub fn symmetric_difference(&mut self, other: &ClassUnicode) { + self.set.symmetric_difference(&other.set); + } + + /// Returns true if and only if this character class will either match + /// nothing or only ASCII bytes. Stated differently, this returns false + /// if and only if this class contains a non-ASCII codepoint. + pub fn is_ascii(&self) -> bool { + self.set.intervals().last().map_or(true, |r| r.end <= '\x7F') + } + + /// Returns the length, in bytes, of the smallest string matched by this + /// character class. + /// + /// Returns `None` when the class is empty. + pub fn minimum_len(&self) -> Option<usize> { + let first = self.ranges().get(0)?; + // Correct because c1 < c2 implies c1.len_utf8() < c2.len_utf8(). + Some(first.start.len_utf8()) + } + + /// Returns the length, in bytes, of the longest string matched by this + /// character class. + /// + /// Returns `None` when the class is empty. + pub fn maximum_len(&self) -> Option<usize> { + let last = self.ranges().last()?; + // Correct because c1 < c2 implies c1.len_utf8() < c2.len_utf8(). + Some(last.end.len_utf8()) + } + + /// If this class consists of exactly one codepoint, then return it as + /// a literal byte string. + /// + /// If this class is empty or contains more than one codepoint, then `None` + /// is returned. + pub fn literal(&self) -> Option<Vec<u8>> { + let rs = self.ranges(); + if rs.len() == 1 && rs[0].start == rs[0].end { + Some(rs[0].start.encode_utf8(&mut [0; 4]).to_string().into_bytes()) + } else { + None + } + } + + /// If this class consists of only ASCII ranges, then return its + /// corresponding and equivalent byte class. + pub fn to_byte_class(&self) -> Option<ClassBytes> { + if !self.is_ascii() { + return None; + } + Some(ClassBytes::new(self.ranges().iter().map(|r| { + // Since we are guaranteed that our codepoint range is ASCII, the + // 'u8::try_from' calls below are guaranteed to be correct. + ClassBytesRange { + start: u8::try_from(r.start).unwrap(), + end: u8::try_from(r.end).unwrap(), + } + }))) + } +} + +/// An iterator over all ranges in a Unicode character class. +/// +/// The lifetime `'a` refers to the lifetime of the underlying class. +#[derive(Debug)] +pub struct ClassUnicodeIter<'a>(IntervalSetIter<'a, ClassUnicodeRange>); + +impl<'a> Iterator for ClassUnicodeIter<'a> { + type Item = &'a ClassUnicodeRange; + + fn next(&mut self) -> Option<&'a ClassUnicodeRange> { + self.0.next() + } +} + +/// A single range of characters represented by Unicode scalar values. +/// +/// The range is closed. That is, the start and end of the range are included +/// in the range. +#[derive(Clone, Copy, Default, Eq, PartialEq, PartialOrd, Ord)] +pub struct ClassUnicodeRange { + start: char, + end: char, +} + +impl core::fmt::Debug for ClassUnicodeRange { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let start = if !self.start.is_whitespace() && !self.start.is_control() + { + self.start.to_string() + } else { + format!("0x{:X}", u32::from(self.start)) + }; + let end = if !self.end.is_whitespace() && !self.end.is_control() { + self.end.to_string() + } else { + format!("0x{:X}", u32::from(self.end)) + }; + f.debug_struct("ClassUnicodeRange") + .field("start", &start) + .field("end", &end) + .finish() + } +} + +impl Interval for ClassUnicodeRange { + type Bound = char; + + #[inline] + fn lower(&self) -> char { + self.start + } + #[inline] + fn upper(&self) -> char { + self.end + } + #[inline] + fn set_lower(&mut self, bound: char) { + self.start = bound; + } + #[inline] + fn set_upper(&mut self, bound: char) { + self.end = bound; + } + + /// Apply simple case folding to this Unicode scalar value range. + /// + /// Additional ranges are appended to the given vector. Canonical ordering + /// is *not* maintained in the given vector. + fn case_fold_simple( + &self, + ranges: &mut Vec<ClassUnicodeRange>, + ) -> Result<(), unicode::CaseFoldError> { + let mut folder = unicode::SimpleCaseFolder::new()?; + if !folder.overlaps(self.start, self.end) { + return Ok(()); + } + let (start, end) = (u32::from(self.start), u32::from(self.end)); + for cp in (start..=end).filter_map(char::from_u32) { + for &cp_folded in folder.mapping(cp) { + ranges.push(ClassUnicodeRange::new(cp_folded, cp_folded)); + } + } + Ok(()) + } +} + +impl ClassUnicodeRange { + /// Create a new Unicode scalar value range for a character class. + /// + /// The returned range is always in a canonical form. That is, the range + /// returned always satisfies the invariant that `start <= end`. + pub fn new(start: char, end: char) -> ClassUnicodeRange { + ClassUnicodeRange::create(start, end) + } + + /// Return the start of this range. + /// + /// The start of a range is always less than or equal to the end of the + /// range. + pub fn start(&self) -> char { + self.start + } + + /// Return the end of this range. + /// + /// The end of a range is always greater than or equal to the start of the + /// range. + pub fn end(&self) -> char { + self.end + } + + /// Returns the number of codepoints in this range. + pub fn len(&self) -> usize { + let diff = 1 + u32::from(self.end) - u32::from(self.start); + // This is likely to panic in 16-bit targets since a usize can only fit + // 2^16. It's not clear what to do here, other than to return an error + // when building a Unicode class that contains a range whose length + // overflows usize. (Which, to be honest, is probably quite common on + // 16-bit targets. For example, this would imply that '.' and '\p{any}' + // would be impossible to build.) + usize::try_from(diff).expect("char class len fits in usize") + } +} + +/// A set of characters represented by arbitrary bytes. +/// +/// Each byte corresponds to one character. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ClassBytes { + set: IntervalSet<ClassBytesRange>, +} + +impl ClassBytes { + /// Create a new class from a sequence of ranges. + /// + /// The given ranges do not need to be in any specific order, and ranges + /// may overlap. Ranges will automatically be sorted into a canonical + /// non-overlapping order. + pub fn new<I>(ranges: I) -> ClassBytes + where + I: IntoIterator<Item = ClassBytesRange>, + { + ClassBytes { set: IntervalSet::new(ranges) } + } + + /// Create a new class with no ranges. + /// + /// An empty class matches nothing. That is, it is equivalent to + /// [`Hir::fail`]. + pub fn empty() -> ClassBytes { + ClassBytes::new(vec![]) + } + + /// Add a new range to this set. + pub fn push(&mut self, range: ClassBytesRange) { + self.set.push(range); + } + + /// Return an iterator over all ranges in this class. + /// + /// The iterator yields ranges in ascending order. + pub fn iter(&self) -> ClassBytesIter<'_> { + ClassBytesIter(self.set.iter()) + } + + /// Return the underlying ranges as a slice. + pub fn ranges(&self) -> &[ClassBytesRange] { + self.set.intervals() + } + + /// Expand this character class such that it contains all case folded + /// characters. For example, if this class consists of the range `a-z`, + /// then applying case folding will result in the class containing both the + /// ranges `a-z` and `A-Z`. + /// + /// Note that this only applies ASCII case folding, which is limited to the + /// characters `a-z` and `A-Z`. + pub fn case_fold_simple(&mut self) { + self.set.case_fold_simple().expect("ASCII case folding never fails"); + } + + /// Negate this byte class. + /// + /// For all `b` where `b` is a any byte, if `b` was in this set, then it + /// will not be in this set after negation. + pub fn negate(&mut self) { + self.set.negate(); + } + + /// Union this byte class with the given byte class, in place. + pub fn union(&mut self, other: &ClassBytes) { + self.set.union(&other.set); + } + + /// Intersect this byte class with the given byte class, in place. + pub fn intersect(&mut self, other: &ClassBytes) { + self.set.intersect(&other.set); + } + + /// Subtract the given byte class from this byte class, in place. + pub fn difference(&mut self, other: &ClassBytes) { + self.set.difference(&other.set); + } + + /// Compute the symmetric difference of the given byte classes, in place. + /// + /// This computes the symmetric difference of two byte classes. This + /// removes all elements in this class that are also in the given class, + /// but all adds all elements from the given class that aren't in this + /// class. That is, the class will contain all elements in either class, + /// but will not contain any elements that are in both classes. + pub fn symmetric_difference(&mut self, other: &ClassBytes) { + self.set.symmetric_difference(&other.set); + } + + /// Returns true if and only if this character class will either match + /// nothing or only ASCII bytes. Stated differently, this returns false + /// if and only if this class contains a non-ASCII byte. + pub fn is_ascii(&self) -> bool { + self.set.intervals().last().map_or(true, |r| r.end <= 0x7F) + } + + /// Returns the length, in bytes, of the smallest string matched by this + /// character class. + /// + /// Returns `None` when the class is empty. + pub fn minimum_len(&self) -> Option<usize> { + if self.ranges().is_empty() { + None + } else { + Some(1) + } + } + + /// Returns the length, in bytes, of the longest string matched by this + /// character class. + /// + /// Returns `None` when the class is empty. + pub fn maximum_len(&self) -> Option<usize> { + if self.ranges().is_empty() { + None + } else { + Some(1) + } + } + + /// If this class consists of exactly one byte, then return it as + /// a literal byte string. + /// + /// If this class is empty or contains more than one byte, then `None` + /// is returned. + pub fn literal(&self) -> Option<Vec<u8>> { + let rs = self.ranges(); + if rs.len() == 1 && rs[0].start == rs[0].end { + Some(vec![rs[0].start]) + } else { + None + } + } + + /// If this class consists of only ASCII ranges, then return its + /// corresponding and equivalent Unicode class. + pub fn to_unicode_class(&self) -> Option<ClassUnicode> { + if !self.is_ascii() { + return None; + } + Some(ClassUnicode::new(self.ranges().iter().map(|r| { + // Since we are guaranteed that our byte range is ASCII, the + // 'char::from' calls below are correct and will not erroneously + // convert a raw byte value into its corresponding codepoint. + ClassUnicodeRange { + start: char::from(r.start), + end: char::from(r.end), + } + }))) + } +} + +/// An iterator over all ranges in a byte character class. +/// +/// The lifetime `'a` refers to the lifetime of the underlying class. +#[derive(Debug)] +pub struct ClassBytesIter<'a>(IntervalSetIter<'a, ClassBytesRange>); + +impl<'a> Iterator for ClassBytesIter<'a> { + type Item = &'a ClassBytesRange; + + fn next(&mut self) -> Option<&'a ClassBytesRange> { + self.0.next() + } +} + +/// A single range of characters represented by arbitrary bytes. +/// +/// The range is closed. That is, the start and end of the range are included +/// in the range. +#[derive(Clone, Copy, Default, Eq, PartialEq, PartialOrd, Ord)] +pub struct ClassBytesRange { + start: u8, + end: u8, +} + +impl Interval for ClassBytesRange { + type Bound = u8; + + #[inline] + fn lower(&self) -> u8 { + self.start + } + #[inline] + fn upper(&self) -> u8 { + self.end + } + #[inline] + fn set_lower(&mut self, bound: u8) { + self.start = bound; + } + #[inline] + fn set_upper(&mut self, bound: u8) { + self.end = bound; + } + + /// Apply simple case folding to this byte range. Only ASCII case mappings + /// (for a-z) are applied. + /// + /// Additional ranges are appended to the given vector. Canonical ordering + /// is *not* maintained in the given vector. + fn case_fold_simple( + &self, + ranges: &mut Vec<ClassBytesRange>, + ) -> Result<(), unicode::CaseFoldError> { + if !ClassBytesRange::new(b'a', b'z').is_intersection_empty(self) { + let lower = cmp::max(self.start, b'a'); + let upper = cmp::min(self.end, b'z'); + ranges.push(ClassBytesRange::new(lower - 32, upper - 32)); + } + if !ClassBytesRange::new(b'A', b'Z').is_intersection_empty(self) { + let lower = cmp::max(self.start, b'A'); + let upper = cmp::min(self.end, b'Z'); + ranges.push(ClassBytesRange::new(lower + 32, upper + 32)); + } + Ok(()) + } +} + +impl ClassBytesRange { + /// Create a new byte range for a character class. + /// + /// The returned range is always in a canonical form. That is, the range + /// returned always satisfies the invariant that `start <= end`. + pub fn new(start: u8, end: u8) -> ClassBytesRange { + ClassBytesRange::create(start, end) + } + + /// Return the start of this range. + /// + /// The start of a range is always less than or equal to the end of the + /// range. + pub fn start(&self) -> u8 { + self.start + } + + /// Return the end of this range. + /// + /// The end of a range is always greater than or equal to the start of the + /// range. + pub fn end(&self) -> u8 { + self.end + } + + /// Returns the number of bytes in this range. + pub fn len(&self) -> usize { + usize::from(self.end.checked_sub(self.start).unwrap()) + .checked_add(1) + .unwrap() + } +} + +impl core::fmt::Debug for ClassBytesRange { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("ClassBytesRange") + .field("start", &crate::debug::Byte(self.start)) + .field("end", &crate::debug::Byte(self.end)) + .finish() + } +} + +/// The high-level intermediate representation for a look-around assertion. +/// +/// An assertion match is always zero-length. Also called an "empty match." +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Look { + /// Match the beginning of text. Specifically, this matches at the starting + /// position of the input. + Start = 1 << 0, + /// Match the end of text. Specifically, this matches at the ending + /// position of the input. + End = 1 << 1, + /// Match the beginning of a line or the beginning of text. Specifically, + /// this matches at the starting position of the input, or at the position + /// immediately following a `\n` character. + StartLF = 1 << 2, + /// Match the end of a line or the end of text. Specifically, this matches + /// at the end position of the input, or at the position immediately + /// preceding a `\n` character. + EndLF = 1 << 3, + /// Match the beginning of a line or the beginning of text. Specifically, + /// this matches at the starting position of the input, or at the position + /// immediately following either a `\r` or `\n` character, but never after + /// a `\r` when a `\n` follows. + StartCRLF = 1 << 4, + /// Match the end of a line or the end of text. Specifically, this matches + /// at the end position of the input, or at the position immediately + /// preceding a `\r` or `\n` character, but never before a `\n` when a `\r` + /// precedes it. + EndCRLF = 1 << 5, + /// Match an ASCII-only word boundary. That is, this matches a position + /// where the left adjacent character and right adjacent character + /// correspond to a word and non-word or a non-word and word character. + WordAscii = 1 << 6, + /// Match an ASCII-only negation of a word boundary. + WordAsciiNegate = 1 << 7, + /// Match a Unicode-aware word boundary. That is, this matches a position + /// where the left adjacent character and right adjacent character + /// correspond to a word and non-word or a non-word and word character. + WordUnicode = 1 << 8, + /// Match a Unicode-aware negation of a word boundary. + WordUnicodeNegate = 1 << 9, + /// Match the start of an ASCII-only word boundary. That is, this matches a + /// position at either the beginning of the haystack or where the previous + /// character is not a word character and the following character is a word + /// character. + WordStartAscii = 1 << 10, + /// Match the end of an ASCII-only word boundary. That is, this matches + /// a position at either the end of the haystack or where the previous + /// character is a word character and the following character is not a word + /// character. + WordEndAscii = 1 << 11, + /// Match the start of a Unicode word boundary. That is, this matches a + /// position at either the beginning of the haystack or where the previous + /// character is not a word character and the following character is a word + /// character. + WordStartUnicode = 1 << 12, + /// Match the end of a Unicode word boundary. That is, this matches a + /// position at either the end of the haystack or where the previous + /// character is a word character and the following character is not a word + /// character. + WordEndUnicode = 1 << 13, + /// Match the start half of an ASCII-only word boundary. That is, this + /// matches a position at either the beginning of the haystack or where the + /// previous character is not a word character. + WordStartHalfAscii = 1 << 14, + /// Match the end half of an ASCII-only word boundary. That is, this + /// matches a position at either the end of the haystack or where the + /// following character is not a word character. + WordEndHalfAscii = 1 << 15, + /// Match the start half of a Unicode word boundary. That is, this matches + /// a position at either the beginning of the haystack or where the + /// previous character is not a word character. + WordStartHalfUnicode = 1 << 16, + /// Match the end half of a Unicode word boundary. That is, this matches + /// a position at either the end of the haystack or where the following + /// character is not a word character. + WordEndHalfUnicode = 1 << 17, +} + +impl Look { + /// Flip the look-around assertion to its equivalent for reverse searches. + /// For example, `StartLF` gets translated to `EndLF`. + /// + /// Some assertions, such as `WordUnicode`, remain the same since they + /// match the same positions regardless of the direction of the search. + #[inline] + pub const fn reversed(self) -> Look { + match self { + Look::Start => Look::End, + Look::End => Look::Start, + Look::StartLF => Look::EndLF, + Look::EndLF => Look::StartLF, + Look::StartCRLF => Look::EndCRLF, + Look::EndCRLF => Look::StartCRLF, + Look::WordAscii => Look::WordAscii, + Look::WordAsciiNegate => Look::WordAsciiNegate, + Look::WordUnicode => Look::WordUnicode, + Look::WordUnicodeNegate => Look::WordUnicodeNegate, + Look::WordStartAscii => Look::WordEndAscii, + Look::WordEndAscii => Look::WordStartAscii, + Look::WordStartUnicode => Look::WordEndUnicode, + Look::WordEndUnicode => Look::WordStartUnicode, + Look::WordStartHalfAscii => Look::WordEndHalfAscii, + Look::WordEndHalfAscii => Look::WordStartHalfAscii, + Look::WordStartHalfUnicode => Look::WordEndHalfUnicode, + Look::WordEndHalfUnicode => Look::WordStartHalfUnicode, + } + } + + /// Return the underlying representation of this look-around enumeration + /// as an integer. Giving the return value to the [`Look::from_repr`] + /// constructor is guaranteed to return the same look-around variant that + /// one started with within a semver compatible release of this crate. + #[inline] + pub const fn as_repr(self) -> u32 { + // AFAIK, 'as' is the only way to zero-cost convert an int enum to an + // actual int. + self as u32 + } + + /// Given the underlying representation of a `Look` value, return the + /// corresponding `Look` value if the representation is valid. Otherwise + /// `None` is returned. + #[inline] + pub const fn from_repr(repr: u32) -> Option<Look> { + match repr { + 0b00_0000_0000_0000_0001 => Some(Look::Start), + 0b00_0000_0000_0000_0010 => Some(Look::End), + 0b00_0000_0000_0000_0100 => Some(Look::StartLF), + 0b00_0000_0000_0000_1000 => Some(Look::EndLF), + 0b00_0000_0000_0001_0000 => Some(Look::StartCRLF), + 0b00_0000_0000_0010_0000 => Some(Look::EndCRLF), + 0b00_0000_0000_0100_0000 => Some(Look::WordAscii), + 0b00_0000_0000_1000_0000 => Some(Look::WordAsciiNegate), + 0b00_0000_0001_0000_0000 => Some(Look::WordUnicode), + 0b00_0000_0010_0000_0000 => Some(Look::WordUnicodeNegate), + 0b00_0000_0100_0000_0000 => Some(Look::WordStartAscii), + 0b00_0000_1000_0000_0000 => Some(Look::WordEndAscii), + 0b00_0001_0000_0000_0000 => Some(Look::WordStartUnicode), + 0b00_0010_0000_0000_0000 => Some(Look::WordEndUnicode), + 0b00_0100_0000_0000_0000 => Some(Look::WordStartHalfAscii), + 0b00_1000_0000_0000_0000 => Some(Look::WordEndHalfAscii), + 0b01_0000_0000_0000_0000 => Some(Look::WordStartHalfUnicode), + 0b10_0000_0000_0000_0000 => Some(Look::WordEndHalfUnicode), + _ => None, + } + } + + /// Returns a convenient single codepoint representation of this + /// look-around assertion. Each assertion is guaranteed to be represented + /// by a distinct character. + /// + /// This is useful for succinctly representing a look-around assertion in + /// human friendly but succinct output intended for a programmer working on + /// regex internals. + #[inline] + pub const fn as_char(self) -> char { + match self { + Look::Start => 'A', + Look::End => 'z', + Look::StartLF => '^', + Look::EndLF => '$', + Look::StartCRLF => 'r', + Look::EndCRLF => 'R', + Look::WordAscii => 'b', + Look::WordAsciiNegate => 'B', + Look::WordUnicode => '𝛃', + Look::WordUnicodeNegate => '𝚩', + Look::WordStartAscii => '<', + Look::WordEndAscii => '>', + Look::WordStartUnicode => '〈', + Look::WordEndUnicode => '〉', + Look::WordStartHalfAscii => '◁', + Look::WordEndHalfAscii => '▷', + Look::WordStartHalfUnicode => '◀', + Look::WordEndHalfUnicode => '▶', + } + } +} + +/// The high-level intermediate representation for a capturing group. +/// +/// A capturing group always has an index and a child expression. It may +/// also have a name associated with it (e.g., `(?P<foo>\w)`), but it's not +/// necessary. +/// +/// Note that there is no explicit representation of a non-capturing group +/// in a `Hir`. Instead, non-capturing grouping is handled automatically by +/// the recursive structure of the `Hir` itself. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Capture { + /// The capture index of the capture. + pub index: u32, + /// The name of the capture, if it exists. + pub name: Option<Box<str>>, + /// The expression inside the capturing group, which may be empty. + pub sub: Box<Hir>, +} + +/// The high-level intermediate representation of a repetition operator. +/// +/// A repetition operator permits the repetition of an arbitrary +/// sub-expression. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Repetition { + /// The minimum range of the repetition. + /// + /// Note that special cases like `?`, `+` and `*` all get translated into + /// the ranges `{0,1}`, `{1,}` and `{0,}`, respectively. + /// + /// When `min` is zero, this expression can match the empty string + /// regardless of what its sub-expression is. + pub min: u32, + /// The maximum range of the repetition. + /// + /// Note that when `max` is `None`, `min` acts as a lower bound but where + /// there is no upper bound. For something like `x{5}` where the min and + /// max are equivalent, `min` will be set to `5` and `max` will be set to + /// `Some(5)`. + pub max: Option<u32>, + /// Whether this repetition operator is greedy or not. A greedy operator + /// will match as much as it can. A non-greedy operator will match as + /// little as it can. + /// + /// Typically, operators are greedy by default and are only non-greedy when + /// a `?` suffix is used, e.g., `(expr)*` is greedy while `(expr)*?` is + /// not. However, this can be inverted via the `U` "ungreedy" flag. + pub greedy: bool, + /// The expression being repeated. + pub sub: Box<Hir>, +} + +impl Repetition { + /// Returns a new repetition with the same `min`, `max` and `greedy` + /// values, but with its sub-expression replaced with the one given. + pub fn with(&self, sub: Hir) -> Repetition { + Repetition { + min: self.min, + max: self.max, + greedy: self.greedy, + sub: Box::new(sub), + } + } +} + +/// A type describing the different flavors of `.`. +/// +/// This type is meant to be used with [`Hir::dot`], which is a convenience +/// routine for building HIR values derived from the `.` regex. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Dot { + /// Matches the UTF-8 encoding of any Unicode scalar value. + /// + /// This is equivalent to `(?su:.)` and also `\p{any}`. + AnyChar, + /// Matches any byte value. + /// + /// This is equivalent to `(?s-u:.)` and also `(?-u:[\x00-\xFF])`. + AnyByte, + /// Matches the UTF-8 encoding of any Unicode scalar value except for the + /// `char` given. + /// + /// This is equivalent to using `(?u-s:.)` with the line terminator set + /// to a particular ASCII byte. (Because of peculiarities in the regex + /// engines, a line terminator must be a single byte. It follows that when + /// UTF-8 mode is enabled, this single byte must also be a Unicode scalar + /// value. That is, ti must be ASCII.) + /// + /// (This and `AnyCharExceptLF` both exist because of legacy reasons. + /// `AnyCharExceptLF` will be dropped in the next breaking change release.) + AnyCharExcept(char), + /// Matches the UTF-8 encoding of any Unicode scalar value except for `\n`. + /// + /// This is equivalent to `(?u-s:.)` and also `[\p{any}--\n]`. + AnyCharExceptLF, + /// Matches the UTF-8 encoding of any Unicode scalar value except for `\r` + /// and `\n`. + /// + /// This is equivalent to `(?uR-s:.)` and also `[\p{any}--\r\n]`. + AnyCharExceptCRLF, + /// Matches any byte value except for the `u8` given. + /// + /// This is equivalent to using `(?-us:.)` with the line terminator set + /// to a particular ASCII byte. (Because of peculiarities in the regex + /// engines, a line terminator must be a single byte. It follows that when + /// UTF-8 mode is enabled, this single byte must also be a Unicode scalar + /// value. That is, ti must be ASCII.) + /// + /// (This and `AnyByteExceptLF` both exist because of legacy reasons. + /// `AnyByteExceptLF` will be dropped in the next breaking change release.) + AnyByteExcept(u8), + /// Matches any byte value except for `\n`. + /// + /// This is equivalent to `(?-su:.)` and also `(?-u:[[\x00-\xFF]--\n])`. + AnyByteExceptLF, + /// Matches any byte value except for `\r` and `\n`. + /// + /// This is equivalent to `(?R-su:.)` and also `(?-u:[[\x00-\xFF]--\r\n])`. + AnyByteExceptCRLF, +} + +/// A custom `Drop` impl is used for `HirKind` such that it uses constant stack +/// space but heap space proportional to the depth of the total `Hir`. +impl Drop for Hir { + fn drop(&mut self) { + use core::mem; + + match *self.kind() { + HirKind::Empty + | HirKind::Literal(_) + | HirKind::Class(_) + | HirKind::Look(_) => return, + HirKind::Capture(ref x) if x.sub.kind.subs().is_empty() => return, + HirKind::Repetition(ref x) if x.sub.kind.subs().is_empty() => { + return + } + HirKind::Concat(ref x) if x.is_empty() => return, + HirKind::Alternation(ref x) if x.is_empty() => return, + _ => {} + } + + let mut stack = vec![mem::replace(self, Hir::empty())]; + while let Some(mut expr) = stack.pop() { + match expr.kind { + HirKind::Empty + | HirKind::Literal(_) + | HirKind::Class(_) + | HirKind::Look(_) => {} + HirKind::Capture(ref mut x) => { + stack.push(mem::replace(&mut x.sub, Hir::empty())); + } + HirKind::Repetition(ref mut x) => { + stack.push(mem::replace(&mut x.sub, Hir::empty())); + } + HirKind::Concat(ref mut x) => { + stack.extend(x.drain(..)); + } + HirKind::Alternation(ref mut x) => { + stack.extend(x.drain(..)); + } + } + } + } +} + +/// A type that collects various properties of an HIR value. +/// +/// Properties are always scalar values and represent meta data that is +/// computed inductively on an HIR value. Properties are defined for all +/// HIR values. +/// +/// All methods on a `Properties` value take constant time and are meant to +/// be cheap to call. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Properties(Box<PropertiesI>); + +/// The property definition. It is split out so that we can box it, and +/// there by make `Properties` use less stack size. This is kind-of important +/// because every HIR value has a `Properties` attached to it. +/// +/// This does have the unfortunate consequence that creating any HIR value +/// always leads to at least one alloc for properties, but this is generally +/// true anyway (for pretty much all HirKinds except for look-arounds). +#[derive(Clone, Debug, Eq, PartialEq)] +struct PropertiesI { + minimum_len: Option<usize>, + maximum_len: Option<usize>, + look_set: LookSet, + look_set_prefix: LookSet, + look_set_suffix: LookSet, + look_set_prefix_any: LookSet, + look_set_suffix_any: LookSet, + utf8: bool, + explicit_captures_len: usize, + static_explicit_captures_len: Option<usize>, + literal: bool, + alternation_literal: bool, +} + +impl Properties { + /// Returns the length (in bytes) of the smallest string matched by this + /// HIR. + /// + /// A return value of `0` is possible and occurs when the HIR can match an + /// empty string. + /// + /// `None` is returned when there is no minimum length. This occurs in + /// precisely the cases where the HIR matches nothing. i.e., The language + /// the regex matches is empty. An example of such a regex is `\P{any}`. + #[inline] + pub fn minimum_len(&self) -> Option<usize> { + self.0.minimum_len + } + + /// Returns the length (in bytes) of the longest string matched by this + /// HIR. + /// + /// A return value of `0` is possible and occurs when nothing longer than + /// the empty string is in the language described by this HIR. + /// + /// `None` is returned when there is no longest matching string. This + /// occurs when the HIR matches nothing or when there is no upper bound on + /// the length of matching strings. Example of such regexes are `\P{any}` + /// (matches nothing) and `a+` (has no upper bound). + #[inline] + pub fn maximum_len(&self) -> Option<usize> { + self.0.maximum_len + } + + /// Returns a set of all look-around assertions that appear at least once + /// in this HIR value. + #[inline] + pub fn look_set(&self) -> LookSet { + self.0.look_set + } + + /// Returns a set of all look-around assertions that appear as a prefix for + /// this HIR value. That is, the set returned corresponds to the set of + /// assertions that must be passed before matching any bytes in a haystack. + /// + /// For example, `hir.look_set_prefix().contains(Look::Start)` returns true + /// if and only if the HIR is fully anchored at the start. + #[inline] + pub fn look_set_prefix(&self) -> LookSet { + self.0.look_set_prefix + } + + /// Returns a set of all look-around assertions that appear as a _possible_ + /// prefix for this HIR value. That is, the set returned corresponds to the + /// set of assertions that _may_ be passed before matching any bytes in a + /// haystack. + /// + /// For example, `hir.look_set_prefix_any().contains(Look::Start)` returns + /// true if and only if it's possible for the regex to match through a + /// anchored assertion before consuming any input. + #[inline] + pub fn look_set_prefix_any(&self) -> LookSet { + self.0.look_set_prefix_any + } + + /// Returns a set of all look-around assertions that appear as a suffix for + /// this HIR value. That is, the set returned corresponds to the set of + /// assertions that must be passed in order to be considered a match after + /// all other consuming HIR expressions. + /// + /// For example, `hir.look_set_suffix().contains(Look::End)` returns true + /// if and only if the HIR is fully anchored at the end. + #[inline] + pub fn look_set_suffix(&self) -> LookSet { + self.0.look_set_suffix + } + + /// Returns a set of all look-around assertions that appear as a _possible_ + /// suffix for this HIR value. That is, the set returned corresponds to the + /// set of assertions that _may_ be passed before matching any bytes in a + /// haystack. + /// + /// For example, `hir.look_set_suffix_any().contains(Look::End)` returns + /// true if and only if it's possible for the regex to match through a + /// anchored assertion at the end of a match without consuming any input. + #[inline] + pub fn look_set_suffix_any(&self) -> LookSet { + self.0.look_set_suffix_any + } + + /// Return true if and only if the corresponding HIR will always match + /// valid UTF-8. + /// + /// When this returns false, then it is possible for this HIR expression to + /// match invalid UTF-8, including by matching between the code units of + /// a single UTF-8 encoded codepoint. + /// + /// Note that this returns true even when the corresponding HIR can match + /// the empty string. Since an empty string can technically appear between + /// UTF-8 code units, it is possible for a match to be reported that splits + /// a codepoint which could in turn be considered matching invalid UTF-8. + /// However, it is generally assumed that such empty matches are handled + /// specially by the search routine if it is absolutely required that + /// matches not split a codepoint. + /// + /// # Example + /// + /// This code example shows the UTF-8 property of a variety of patterns. + /// + /// ``` + /// use regex_syntax::{ParserBuilder, parse}; + /// + /// // Examples of 'is_utf8() == true'. + /// assert!(parse(r"a")?.properties().is_utf8()); + /// assert!(parse(r"[^a]")?.properties().is_utf8()); + /// assert!(parse(r".")?.properties().is_utf8()); + /// assert!(parse(r"\W")?.properties().is_utf8()); + /// assert!(parse(r"\b")?.properties().is_utf8()); + /// assert!(parse(r"\B")?.properties().is_utf8()); + /// assert!(parse(r"(?-u)\b")?.properties().is_utf8()); + /// assert!(parse(r"(?-u)\B")?.properties().is_utf8()); + /// // Unicode mode is enabled by default, and in + /// // that mode, all \x hex escapes are treated as + /// // codepoints. So this actually matches the UTF-8 + /// // encoding of U+00FF. + /// assert!(parse(r"\xFF")?.properties().is_utf8()); + /// + /// // Now we show examples of 'is_utf8() == false'. + /// // The only way to do this is to force the parser + /// // to permit invalid UTF-8, otherwise all of these + /// // would fail to parse! + /// let parse = |pattern| { + /// ParserBuilder::new().utf8(false).build().parse(pattern) + /// }; + /// assert!(!parse(r"(?-u)[^a]")?.properties().is_utf8()); + /// assert!(!parse(r"(?-u).")?.properties().is_utf8()); + /// assert!(!parse(r"(?-u)\W")?.properties().is_utf8()); + /// // Conversely to the equivalent example above, + /// // when Unicode mode is disabled, \x hex escapes + /// // are treated as their raw byte values. + /// assert!(!parse(r"(?-u)\xFF")?.properties().is_utf8()); + /// // Note that just because we disabled UTF-8 in the + /// // parser doesn't mean we still can't use Unicode. + /// // It is enabled by default, so \xFF is still + /// // equivalent to matching the UTF-8 encoding of + /// // U+00FF by default. + /// assert!(parse(r"\xFF")?.properties().is_utf8()); + /// // Even though we use raw bytes that individually + /// // are not valid UTF-8, when combined together, the + /// // overall expression *does* match valid UTF-8! + /// assert!(parse(r"(?-u)\xE2\x98\x83")?.properties().is_utf8()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn is_utf8(&self) -> bool { + self.0.utf8 + } + + /// Returns the total number of explicit capturing groups in the + /// corresponding HIR. + /// + /// Note that this does not include the implicit capturing group + /// corresponding to the entire match that is typically included by regex + /// engines. + /// + /// # Example + /// + /// This method will return `0` for `a` and `1` for `(a)`: + /// + /// ``` + /// use regex_syntax::parse; + /// + /// assert_eq!(0, parse("a")?.properties().explicit_captures_len()); + /// assert_eq!(1, parse("(a)")?.properties().explicit_captures_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn explicit_captures_len(&self) -> usize { + self.0.explicit_captures_len + } + + /// Returns the total number of explicit capturing groups that appear in + /// every possible match. + /// + /// If the number of capture groups can vary depending on the match, then + /// this returns `None`. That is, a value is only returned when the number + /// of matching groups is invariant or "static." + /// + /// Note that this does not include the implicit capturing group + /// corresponding to the entire match. + /// + /// # Example + /// + /// This shows a few cases where a static number of capture groups is + /// available and a few cases where it is not. + /// + /// ``` + /// use regex_syntax::parse; + /// + /// let len = |pattern| { + /// parse(pattern).map(|h| { + /// h.properties().static_explicit_captures_len() + /// }) + /// }; + /// + /// assert_eq!(Some(0), len("a")?); + /// assert_eq!(Some(1), len("(a)")?); + /// assert_eq!(Some(1), len("(a)|(b)")?); + /// assert_eq!(Some(2), len("(a)(b)|(c)(d)")?); + /// assert_eq!(None, len("(a)|b")?); + /// assert_eq!(None, len("a|(b)")?); + /// assert_eq!(None, len("(b)*")?); + /// assert_eq!(Some(1), len("(b)+")?); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + #[inline] + pub fn static_explicit_captures_len(&self) -> Option<usize> { + self.0.static_explicit_captures_len + } + + /// Return true if and only if this HIR is a simple literal. This is + /// only true when this HIR expression is either itself a `Literal` or a + /// concatenation of only `Literal`s. + /// + /// For example, `f` and `foo` are literals, but `f+`, `(foo)`, `foo()` and + /// the empty string are not (even though they contain sub-expressions that + /// are literals). + #[inline] + pub fn is_literal(&self) -> bool { + self.0.literal + } + + /// Return true if and only if this HIR is either a simple literal or an + /// alternation of simple literals. This is only + /// true when this HIR expression is either itself a `Literal` or a + /// concatenation of only `Literal`s or an alternation of only `Literal`s. + /// + /// For example, `f`, `foo`, `a|b|c`, and `foo|bar|baz` are alternation + /// literals, but `f+`, `(foo)`, `foo()`, and the empty pattern are not + /// (even though that contain sub-expressions that are literals). + #[inline] + pub fn is_alternation_literal(&self) -> bool { + self.0.alternation_literal + } + + /// Returns the total amount of heap memory usage, in bytes, used by this + /// `Properties` value. + #[inline] + pub fn memory_usage(&self) -> usize { + core::mem::size_of::<PropertiesI>() + } + + /// Returns a new set of properties that corresponds to the union of the + /// iterator of properties given. + /// + /// This is useful when one has multiple `Hir` expressions and wants + /// to combine them into a single alternation without constructing the + /// corresponding `Hir`. This routine provides a way of combining the + /// properties of each `Hir` expression into one set of properties + /// representing the union of those expressions. + /// + /// # Example: union with HIRs that never match + /// + /// This example shows that unioning properties together with one that + /// represents a regex that never matches will "poison" certain attributes, + /// like the minimum and maximum lengths. + /// + /// ``` + /// use regex_syntax::{hir::Properties, parse}; + /// + /// let hir1 = parse("ab?c?")?; + /// assert_eq!(Some(1), hir1.properties().minimum_len()); + /// assert_eq!(Some(3), hir1.properties().maximum_len()); + /// + /// let hir2 = parse(r"[a&&b]")?; + /// assert_eq!(None, hir2.properties().minimum_len()); + /// assert_eq!(None, hir2.properties().maximum_len()); + /// + /// let hir3 = parse(r"wxy?z?")?; + /// assert_eq!(Some(2), hir3.properties().minimum_len()); + /// assert_eq!(Some(4), hir3.properties().maximum_len()); + /// + /// let unioned = Properties::union([ + /// hir1.properties(), + /// hir2.properties(), + /// hir3.properties(), + /// ]); + /// assert_eq!(None, unioned.minimum_len()); + /// assert_eq!(None, unioned.maximum_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + /// + /// The maximum length can also be "poisoned" by a pattern that has no + /// upper bound on the length of a match. The minimum length remains + /// unaffected: + /// + /// ``` + /// use regex_syntax::{hir::Properties, parse}; + /// + /// let hir1 = parse("ab?c?")?; + /// assert_eq!(Some(1), hir1.properties().minimum_len()); + /// assert_eq!(Some(3), hir1.properties().maximum_len()); + /// + /// let hir2 = parse(r"a+")?; + /// assert_eq!(Some(1), hir2.properties().minimum_len()); + /// assert_eq!(None, hir2.properties().maximum_len()); + /// + /// let hir3 = parse(r"wxy?z?")?; + /// assert_eq!(Some(2), hir3.properties().minimum_len()); + /// assert_eq!(Some(4), hir3.properties().maximum_len()); + /// + /// let unioned = Properties::union([ + /// hir1.properties(), + /// hir2.properties(), + /// hir3.properties(), + /// ]); + /// assert_eq!(Some(1), unioned.minimum_len()); + /// assert_eq!(None, unioned.maximum_len()); + /// + /// # Ok::<(), Box<dyn std::error::Error>>(()) + /// ``` + pub fn union<I, P>(props: I) -> Properties + where + I: IntoIterator<Item = P>, + P: core::borrow::Borrow<Properties>, + { + let mut it = props.into_iter().peekable(); + // While empty alternations aren't possible, we still behave as if they + // are. When we have an empty alternate, then clearly the look-around + // prefix and suffix is empty. Otherwise, it is the intersection of all + // prefixes and suffixes (respectively) of the branches. + let fix = if it.peek().is_none() { + LookSet::empty() + } else { + LookSet::full() + }; + // And also, an empty alternate means we have 0 static capture groups, + // but we otherwise start with the number corresponding to the first + // alternate. If any subsequent alternate has a different number of + // static capture groups, then we overall have a variation and not a + // static number of groups. + let static_explicit_captures_len = + it.peek().and_then(|p| p.borrow().static_explicit_captures_len()); + // The base case is an empty alternation, which matches nothing. + // Note though that empty alternations aren't possible, because the + // Hir::alternation smart constructor rewrites those as empty character + // classes. + let mut props = PropertiesI { + minimum_len: None, + maximum_len: None, + look_set: LookSet::empty(), + look_set_prefix: fix, + look_set_suffix: fix, + look_set_prefix_any: LookSet::empty(), + look_set_suffix_any: LookSet::empty(), + utf8: true, + explicit_captures_len: 0, + static_explicit_captures_len, + literal: false, + alternation_literal: true, + }; + let (mut min_poisoned, mut max_poisoned) = (false, false); + // Handle properties that need to visit every child hir. + for prop in it { + let p = prop.borrow(); + props.look_set.set_union(p.look_set()); + props.look_set_prefix.set_intersect(p.look_set_prefix()); + props.look_set_suffix.set_intersect(p.look_set_suffix()); + props.look_set_prefix_any.set_union(p.look_set_prefix_any()); + props.look_set_suffix_any.set_union(p.look_set_suffix_any()); + props.utf8 = props.utf8 && p.is_utf8(); + props.explicit_captures_len = props + .explicit_captures_len + .saturating_add(p.explicit_captures_len()); + if props.static_explicit_captures_len + != p.static_explicit_captures_len() + { + props.static_explicit_captures_len = None; + } + props.alternation_literal = + props.alternation_literal && p.is_literal(); + if !min_poisoned { + if let Some(xmin) = p.minimum_len() { + if props.minimum_len.map_or(true, |pmin| xmin < pmin) { + props.minimum_len = Some(xmin); + } + } else { + props.minimum_len = None; + min_poisoned = true; + } + } + if !max_poisoned { + if let Some(xmax) = p.maximum_len() { + if props.maximum_len.map_or(true, |pmax| xmax > pmax) { + props.maximum_len = Some(xmax); + } + } else { + props.maximum_len = None; + max_poisoned = true; + } + } + } + Properties(Box::new(props)) + } +} + +impl Properties { + /// Create a new set of HIR properties for an empty regex. + fn empty() -> Properties { + let inner = PropertiesI { + minimum_len: Some(0), + maximum_len: Some(0), + look_set: LookSet::empty(), + look_set_prefix: LookSet::empty(), + look_set_suffix: LookSet::empty(), + look_set_prefix_any: LookSet::empty(), + look_set_suffix_any: LookSet::empty(), + // It is debatable whether an empty regex always matches at valid + // UTF-8 boundaries. Strictly speaking, at a byte oriented view, + // it is clearly false. There are, for example, many empty strings + // between the bytes encoding a '☃'. + // + // However, when Unicode mode is enabled, the fundamental atom + // of matching is really a codepoint. And in that scenario, an + // empty regex is defined to only match at valid UTF-8 boundaries + // and to never split a codepoint. It just so happens that this + // enforcement is somewhat tricky to do for regexes that match + // the empty string inside regex engines themselves. It usually + // requires some layer above the regex engine to filter out such + // matches. + // + // In any case, 'true' is really the only coherent option. If it + // were false, for example, then 'a*' would also need to be false + // since it too can match the empty string. + utf8: true, + explicit_captures_len: 0, + static_explicit_captures_len: Some(0), + literal: false, + alternation_literal: false, + }; + Properties(Box::new(inner)) + } + + /// Create a new set of HIR properties for a literal regex. + fn literal(lit: &Literal) -> Properties { + let inner = PropertiesI { + minimum_len: Some(lit.0.len()), + maximum_len: Some(lit.0.len()), + look_set: LookSet::empty(), + look_set_prefix: LookSet::empty(), + look_set_suffix: LookSet::empty(), + look_set_prefix_any: LookSet::empty(), + look_set_suffix_any: LookSet::empty(), + utf8: core::str::from_utf8(&lit.0).is_ok(), + explicit_captures_len: 0, + static_explicit_captures_len: Some(0), + literal: true, + alternation_literal: true, + }; + Properties(Box::new(inner)) + } + + /// Create a new set of HIR properties for a character class. + fn class(class: &Class) -> Properties { + let inner = PropertiesI { + minimum_len: class.minimum_len(), + maximum_len: class.maximum_len(), + look_set: LookSet::empty(), + look_set_prefix: LookSet::empty(), + look_set_suffix: LookSet::empty(), + look_set_prefix_any: LookSet::empty(), + look_set_suffix_any: LookSet::empty(), + utf8: class.is_utf8(), + explicit_captures_len: 0, + static_explicit_captures_len: Some(0), + literal: false, + alternation_literal: false, + }; + Properties(Box::new(inner)) + } + + /// Create a new set of HIR properties for a look-around assertion. + fn look(look: Look) -> Properties { + let inner = PropertiesI { + minimum_len: Some(0), + maximum_len: Some(0), + look_set: LookSet::singleton(look), + look_set_prefix: LookSet::singleton(look), + look_set_suffix: LookSet::singleton(look), + look_set_prefix_any: LookSet::singleton(look), + look_set_suffix_any: LookSet::singleton(look), + // This requires a little explanation. Basically, we don't consider + // matching an empty string to be equivalent to matching invalid + // UTF-8, even though technically matching every empty string will + // split the UTF-8 encoding of a single codepoint when treating a + // UTF-8 encoded string as a sequence of bytes. Our defense here is + // that in such a case, a codepoint should logically be treated as + // the fundamental atom for matching, and thus the only valid match + // points are between codepoints and not bytes. + // + // More practically, this is true here because it's also true + // for 'Hir::empty()', otherwise something like 'a*' would be + // considered to match invalid UTF-8. That in turn makes this + // property borderline useless. + utf8: true, + explicit_captures_len: 0, + static_explicit_captures_len: Some(0), + literal: false, + alternation_literal: false, + }; + Properties(Box::new(inner)) + } + + /// Create a new set of HIR properties for a repetition. + fn repetition(rep: &Repetition) -> Properties { + let p = rep.sub.properties(); + let minimum_len = p.minimum_len().map(|child_min| { + let rep_min = usize::try_from(rep.min).unwrap_or(usize::MAX); + child_min.saturating_mul(rep_min) + }); + let maximum_len = rep.max.and_then(|rep_max| { + let rep_max = usize::try_from(rep_max).ok()?; + let child_max = p.maximum_len()?; + child_max.checked_mul(rep_max) + }); + + let mut inner = PropertiesI { + minimum_len, + maximum_len, + look_set: p.look_set(), + look_set_prefix: LookSet::empty(), + look_set_suffix: LookSet::empty(), + look_set_prefix_any: p.look_set_prefix_any(), + look_set_suffix_any: p.look_set_suffix_any(), + utf8: p.is_utf8(), + explicit_captures_len: p.explicit_captures_len(), + static_explicit_captures_len: p.static_explicit_captures_len(), + literal: false, + alternation_literal: false, + }; + // If the repetition operator can match the empty string, then its + // lookset prefix and suffixes themselves remain empty since they are + // no longer required to match. + if rep.min > 0 { + inner.look_set_prefix = p.look_set_prefix(); + inner.look_set_suffix = p.look_set_suffix(); + } + // If the static captures len of the sub-expression is not known or + // is greater than zero, then it automatically propagates to the + // repetition, regardless of the repetition. Otherwise, it might + // change, but only when the repetition can match 0 times. + if rep.min == 0 + && inner.static_explicit_captures_len.map_or(false, |len| len > 0) + { + // If we require a match 0 times, then our captures len is + // guaranteed to be zero. Otherwise, if we *can* match the empty + // string, then it's impossible to know how many captures will be + // in the resulting match. + if rep.max == Some(0) { + inner.static_explicit_captures_len = Some(0); + } else { + inner.static_explicit_captures_len = None; + } + } + Properties(Box::new(inner)) + } + + /// Create a new set of HIR properties for a capture. + fn capture(capture: &Capture) -> Properties { + let p = capture.sub.properties(); + Properties(Box::new(PropertiesI { + explicit_captures_len: p.explicit_captures_len().saturating_add(1), + static_explicit_captures_len: p + .static_explicit_captures_len() + .map(|len| len.saturating_add(1)), + literal: false, + alternation_literal: false, + ..*p.0.clone() + })) + } + + /// Create a new set of HIR properties for a concatenation. + fn concat(concat: &[Hir]) -> Properties { + // The base case is an empty concatenation, which matches the empty + // string. Note though that empty concatenations aren't possible, + // because the Hir::concat smart constructor rewrites those as + // Hir::empty. + let mut props = PropertiesI { + minimum_len: Some(0), + maximum_len: Some(0), + look_set: LookSet::empty(), + look_set_prefix: LookSet::empty(), + look_set_suffix: LookSet::empty(), + look_set_prefix_any: LookSet::empty(), + look_set_suffix_any: LookSet::empty(), + utf8: true, + explicit_captures_len: 0, + static_explicit_captures_len: Some(0), + literal: true, + alternation_literal: true, + }; + // Handle properties that need to visit every child hir. + for x in concat.iter() { + let p = x.properties(); + props.look_set.set_union(p.look_set()); + props.utf8 = props.utf8 && p.is_utf8(); + props.explicit_captures_len = props + .explicit_captures_len + .saturating_add(p.explicit_captures_len()); + props.static_explicit_captures_len = p + .static_explicit_captures_len() + .and_then(|len1| { + Some((len1, props.static_explicit_captures_len?)) + }) + .and_then(|(len1, len2)| Some(len1.saturating_add(len2))); + props.literal = props.literal && p.is_literal(); + props.alternation_literal = + props.alternation_literal && p.is_alternation_literal(); + if let Some(minimum_len) = props.minimum_len { + match p.minimum_len() { + None => props.minimum_len = None, + Some(len) => { + // We use saturating arithmetic here because the + // minimum is just a lower bound. We can't go any + // higher than what our number types permit. + props.minimum_len = + Some(minimum_len.saturating_add(len)); + } + } + } + if let Some(maximum_len) = props.maximum_len { + match p.maximum_len() { + None => props.maximum_len = None, + Some(len) => { + props.maximum_len = maximum_len.checked_add(len) + } + } + } + } + // Handle the prefix properties, which only requires visiting + // child exprs until one matches more than the empty string. + let mut it = concat.iter(); + while let Some(x) = it.next() { + props.look_set_prefix.set_union(x.properties().look_set_prefix()); + props + .look_set_prefix_any + .set_union(x.properties().look_set_prefix_any()); + if x.properties().maximum_len().map_or(true, |x| x > 0) { + break; + } + } + // Same thing for the suffix properties, but in reverse. + let mut it = concat.iter().rev(); + while let Some(x) = it.next() { + props.look_set_suffix.set_union(x.properties().look_set_suffix()); + props + .look_set_suffix_any + .set_union(x.properties().look_set_suffix_any()); + if x.properties().maximum_len().map_or(true, |x| x > 0) { + break; + } + } + Properties(Box::new(props)) + } + + /// Create a new set of HIR properties for a concatenation. + fn alternation(alts: &[Hir]) -> Properties { + Properties::union(alts.iter().map(|hir| hir.properties())) + } +} + +/// A set of look-around assertions. +/// +/// This is useful for efficiently tracking look-around assertions. For +/// example, an [`Hir`] provides properties that return `LookSet`s. +#[derive(Clone, Copy, Default, Eq, PartialEq)] +pub struct LookSet { + /// The underlying representation this set is exposed to make it possible + /// to store it somewhere efficiently. The representation is that + /// of a bitset, where each assertion occupies bit `i` where `i = + /// Look::as_repr()`. + /// + /// Note that users of this internal representation must permit the full + /// range of `u16` values to be represented. For example, even if the + /// current implementation only makes use of the 10 least significant bits, + /// it may use more bits in a future semver compatible release. + pub bits: u32, +} + +impl LookSet { + /// Create an empty set of look-around assertions. + #[inline] + pub fn empty() -> LookSet { + LookSet { bits: 0 } + } + + /// Create a full set of look-around assertions. + /// + /// This set contains all possible look-around assertions. + #[inline] + pub fn full() -> LookSet { + LookSet { bits: !0 } + } + + /// Create a look-around set containing the look-around assertion given. + /// + /// This is a convenience routine for creating an empty set and inserting + /// one look-around assertions. + #[inline] + pub fn singleton(look: Look) -> LookSet { + LookSet::empty().insert(look) + } + + /// Returns the total number of look-around assertions in this set. + #[inline] + pub fn len(self) -> usize { + // OK because max value always fits in a u8, which in turn always + // fits in a usize, regardless of target. + usize::try_from(self.bits.count_ones()).unwrap() + } + + /// Returns true if and only if this set is empty. + #[inline] + pub fn is_empty(self) -> bool { + self.len() == 0 + } + + /// Returns true if and only if the given look-around assertion is in this + /// set. + #[inline] + pub fn contains(self, look: Look) -> bool { + self.bits & look.as_repr() != 0 + } + + /// Returns true if and only if this set contains any anchor assertions. + /// This includes both "start/end of haystack" and "start/end of line." + #[inline] + pub fn contains_anchor(&self) -> bool { + self.contains_anchor_haystack() || self.contains_anchor_line() + } + + /// Returns true if and only if this set contains any "start/end of + /// haystack" anchors. This doesn't include "start/end of line" anchors. + #[inline] + pub fn contains_anchor_haystack(&self) -> bool { + self.contains(Look::Start) || self.contains(Look::End) + } + + /// Returns true if and only if this set contains any "start/end of line" + /// anchors. This doesn't include "start/end of haystack" anchors. This + /// includes both `\n` line anchors and CRLF (`\r\n`) aware line anchors. + #[inline] + pub fn contains_anchor_line(&self) -> bool { + self.contains(Look::StartLF) + || self.contains(Look::EndLF) + || self.contains(Look::StartCRLF) + || self.contains(Look::EndCRLF) + } + + /// Returns true if and only if this set contains any "start/end of line" + /// anchors that only treat `\n` as line terminators. This does not include + /// haystack anchors or CRLF aware line anchors. + #[inline] + pub fn contains_anchor_lf(&self) -> bool { + self.contains(Look::StartLF) || self.contains(Look::EndLF) + } + + /// Returns true if and only if this set contains any "start/end of line" + /// anchors that are CRLF-aware. This doesn't include "start/end of + /// haystack" or "start/end of line-feed" anchors. + #[inline] + pub fn contains_anchor_crlf(&self) -> bool { + self.contains(Look::StartCRLF) || self.contains(Look::EndCRLF) + } + + /// Returns true if and only if this set contains any word boundary or + /// negated word boundary assertions. This include both Unicode and ASCII + /// word boundaries. + #[inline] + pub fn contains_word(self) -> bool { + self.contains_word_unicode() || self.contains_word_ascii() + } + + /// Returns true if and only if this set contains any Unicode word boundary + /// or negated Unicode word boundary assertions. + #[inline] + pub fn contains_word_unicode(self) -> bool { + self.contains(Look::WordUnicode) + || self.contains(Look::WordUnicodeNegate) + || self.contains(Look::WordStartUnicode) + || self.contains(Look::WordEndUnicode) + || self.contains(Look::WordStartHalfUnicode) + || self.contains(Look::WordEndHalfUnicode) + } + + /// Returns true if and only if this set contains any ASCII word boundary + /// or negated ASCII word boundary assertions. + #[inline] + pub fn contains_word_ascii(self) -> bool { + self.contains(Look::WordAscii) + || self.contains(Look::WordAsciiNegate) + || self.contains(Look::WordStartAscii) + || self.contains(Look::WordEndAscii) + || self.contains(Look::WordStartHalfAscii) + || self.contains(Look::WordEndHalfAscii) + } + + /// Returns an iterator over all of the look-around assertions in this set. + #[inline] + pub fn iter(self) -> LookSetIter { + LookSetIter { set: self } + } + + /// Return a new set that is equivalent to the original, but with the given + /// assertion added to it. If the assertion is already in the set, then the + /// returned set is equivalent to the original. + #[inline] + pub fn insert(self, look: Look) -> LookSet { + LookSet { bits: self.bits | look.as_repr() } + } + + /// Updates this set in place with the result of inserting the given + /// assertion into this set. + #[inline] + pub fn set_insert(&mut self, look: Look) { + *self = self.insert(look); + } + + /// Return a new set that is equivalent to the original, but with the given + /// assertion removed from it. If the assertion is not in the set, then the + /// returned set is equivalent to the original. + #[inline] + pub fn remove(self, look: Look) -> LookSet { + LookSet { bits: self.bits & !look.as_repr() } + } + + /// Updates this set in place with the result of removing the given + /// assertion from this set. + #[inline] + pub fn set_remove(&mut self, look: Look) { + *self = self.remove(look); + } + + /// Returns a new set that is the result of subtracting the given set from + /// this set. + #[inline] + pub fn subtract(self, other: LookSet) -> LookSet { + LookSet { bits: self.bits & !other.bits } + } + + /// Updates this set in place with the result of subtracting the given set + /// from this set. + #[inline] + pub fn set_subtract(&mut self, other: LookSet) { + *self = self.subtract(other); + } + + /// Returns a new set that is the union of this and the one given. + #[inline] + pub fn union(self, other: LookSet) -> LookSet { + LookSet { bits: self.bits | other.bits } + } + + /// Updates this set in place with the result of unioning it with the one + /// given. + #[inline] + pub fn set_union(&mut self, other: LookSet) { + *self = self.union(other); + } + + /// Returns a new set that is the intersection of this and the one given. + #[inline] + pub fn intersect(self, other: LookSet) -> LookSet { + LookSet { bits: self.bits & other.bits } + } + + /// Updates this set in place with the result of intersecting it with the + /// one given. + #[inline] + pub fn set_intersect(&mut self, other: LookSet) { + *self = self.intersect(other); + } + + /// Return a `LookSet` from the slice given as a native endian 32-bit + /// integer. + /// + /// # Panics + /// + /// This panics if `slice.len() < 4`. + #[inline] + pub fn read_repr(slice: &[u8]) -> LookSet { + let bits = u32::from_ne_bytes(slice[..4].try_into().unwrap()); + LookSet { bits } + } + + /// Write a `LookSet` as a native endian 32-bit integer to the beginning + /// of the slice given. + /// + /// # Panics + /// + /// This panics if `slice.len() < 4`. + #[inline] + pub fn write_repr(self, slice: &mut [u8]) { + let raw = self.bits.to_ne_bytes(); + slice[0] = raw[0]; + slice[1] = raw[1]; + slice[2] = raw[2]; + slice[3] = raw[3]; + } +} + +impl core::fmt::Debug for LookSet { + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + if self.is_empty() { + return write!(f, "∅"); + } + for look in self.iter() { + write!(f, "{}", look.as_char())?; + } + Ok(()) + } +} + +/// An iterator over all look-around assertions in a [`LookSet`]. +/// +/// This iterator is created by [`LookSet::iter`]. +#[derive(Clone, Debug)] +pub struct LookSetIter { + set: LookSet, +} + +impl Iterator for LookSetIter { + type Item = Look; + + #[inline] + fn next(&mut self) -> Option<Look> { + if self.set.is_empty() { + return None; + } + // We'll never have more than u8::MAX distinct look-around assertions, + // so 'bit' will always fit into a u16. + let bit = u16::try_from(self.set.bits.trailing_zeros()).unwrap(); + let look = Look::from_repr(1 << bit)?; + self.set = self.set.remove(look); + Some(look) + } +} + +/// Given a sequence of HIR values where each value corresponds to a Unicode +/// class (or an all-ASCII byte class), return a single Unicode class +/// corresponding to the union of the classes found. +fn class_chars(hirs: &[Hir]) -> Option<Class> { + let mut cls = ClassUnicode::new(vec![]); + for hir in hirs.iter() { + match *hir.kind() { + HirKind::Class(Class::Unicode(ref cls2)) => { + cls.union(cls2); + } + HirKind::Class(Class::Bytes(ref cls2)) => { + cls.union(&cls2.to_unicode_class()?); + } + _ => return None, + }; + } + Some(Class::Unicode(cls)) +} + +/// Given a sequence of HIR values where each value corresponds to a byte class +/// (or an all-ASCII Unicode class), return a single byte class corresponding +/// to the union of the classes found. +fn class_bytes(hirs: &[Hir]) -> Option<Class> { + let mut cls = ClassBytes::new(vec![]); + for hir in hirs.iter() { + match *hir.kind() { + HirKind::Class(Class::Unicode(ref cls2)) => { + cls.union(&cls2.to_byte_class()?); + } + HirKind::Class(Class::Bytes(ref cls2)) => { + cls.union(cls2); + } + _ => return None, + }; + } + Some(Class::Bytes(cls)) +} + +/// Given a sequence of HIR values where each value corresponds to a literal +/// that is a single `char`, return that sequence of `char`s. Otherwise return +/// None. No deduplication is done. +fn singleton_chars(hirs: &[Hir]) -> Option<Vec<char>> { + let mut singletons = vec![]; + for hir in hirs.iter() { + let literal = match *hir.kind() { + HirKind::Literal(Literal(ref bytes)) => bytes, + _ => return None, + }; + let ch = match crate::debug::utf8_decode(literal) { + None => return None, + Some(Err(_)) => return None, + Some(Ok(ch)) => ch, + }; + if literal.len() != ch.len_utf8() { + return None; + } + singletons.push(ch); + } + Some(singletons) +} + +/// Given a sequence of HIR values where each value corresponds to a literal +/// that is a single byte, return that sequence of bytes. Otherwise return +/// None. No deduplication is done. +fn singleton_bytes(hirs: &[Hir]) -> Option<Vec<u8>> { + let mut singletons = vec![]; + for hir in hirs.iter() { + let literal = match *hir.kind() { + HirKind::Literal(Literal(ref bytes)) => bytes, + _ => return None, + }; + if literal.len() != 1 { + return None; + } + singletons.push(literal[0]); + } + Some(singletons) +} + +/// Looks for a common prefix in the list of alternation branches given. If one +/// is found, then an equivalent but (hopefully) simplified Hir is returned. +/// Otherwise, the original given list of branches is returned unmodified. +/// +/// This is not quite as good as it could be. Right now, it requires that +/// all branches are 'Concat' expressions. It also doesn't do well with +/// literals. For example, given 'foofoo|foobar', it will not refactor it to +/// 'foo(?:foo|bar)' because literals are flattened into their own special +/// concatenation. (One wonders if perhaps 'Literal' should be a single atom +/// instead of a string of bytes because of this. Otherwise, handling the +/// current representation in this routine will be pretty gnarly. Sigh.) +fn lift_common_prefix(hirs: Vec<Hir>) -> Result<Hir, Vec<Hir>> { + if hirs.len() <= 1 { + return Err(hirs); + } + let mut prefix = match hirs[0].kind() { + HirKind::Concat(ref xs) => &**xs, + _ => return Err(hirs), + }; + if prefix.is_empty() { + return Err(hirs); + } + for h in hirs.iter().skip(1) { + let concat = match h.kind() { + HirKind::Concat(ref xs) => xs, + _ => return Err(hirs), + }; + let common_len = prefix + .iter() + .zip(concat.iter()) + .take_while(|(x, y)| x == y) + .count(); + prefix = &prefix[..common_len]; + if prefix.is_empty() { + return Err(hirs); + } + } + let len = prefix.len(); + assert_ne!(0, len); + let mut prefix_concat = vec![]; + let mut suffix_alts = vec![]; + for h in hirs { + let mut concat = match h.into_kind() { + HirKind::Concat(xs) => xs, + // We required all sub-expressions to be + // concats above, so we're only here if we + // have a concat. + _ => unreachable!(), + }; + suffix_alts.push(Hir::concat(concat.split_off(len))); + if prefix_concat.is_empty() { + prefix_concat = concat; + } + } + let mut concat = prefix_concat; + concat.push(Hir::alternation(suffix_alts)); + Ok(Hir::concat(concat)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn uclass(ranges: &[(char, char)]) -> ClassUnicode { + let ranges: Vec<ClassUnicodeRange> = ranges + .iter() + .map(|&(s, e)| ClassUnicodeRange::new(s, e)) + .collect(); + ClassUnicode::new(ranges) + } + + fn bclass(ranges: &[(u8, u8)]) -> ClassBytes { + let ranges: Vec<ClassBytesRange> = + ranges.iter().map(|&(s, e)| ClassBytesRange::new(s, e)).collect(); + ClassBytes::new(ranges) + } + + fn uranges(cls: &ClassUnicode) -> Vec<(char, char)> { + cls.iter().map(|x| (x.start(), x.end())).collect() + } + + #[cfg(feature = "unicode-case")] + fn ucasefold(cls: &ClassUnicode) -> ClassUnicode { + let mut cls_ = cls.clone(); + cls_.case_fold_simple(); + cls_ + } + + fn uunion(cls1: &ClassUnicode, cls2: &ClassUnicode) -> ClassUnicode { + let mut cls_ = cls1.clone(); + cls_.union(cls2); + cls_ + } + + fn uintersect(cls1: &ClassUnicode, cls2: &ClassUnicode) -> ClassUnicode { + let mut cls_ = cls1.clone(); + cls_.intersect(cls2); + cls_ + } + + fn udifference(cls1: &ClassUnicode, cls2: &ClassUnicode) -> ClassUnicode { + let mut cls_ = cls1.clone(); + cls_.difference(cls2); + cls_ + } + + fn usymdifference( + cls1: &ClassUnicode, + cls2: &ClassUnicode, + ) -> ClassUnicode { + let mut cls_ = cls1.clone(); + cls_.symmetric_difference(cls2); + cls_ + } + + fn unegate(cls: &ClassUnicode) -> ClassUnicode { + let mut cls_ = cls.clone(); + cls_.negate(); + cls_ + } + + fn branges(cls: &ClassBytes) -> Vec<(u8, u8)> { + cls.iter().map(|x| (x.start(), x.end())).collect() + } + + fn bcasefold(cls: &ClassBytes) -> ClassBytes { + let mut cls_ = cls.clone(); + cls_.case_fold_simple(); + cls_ + } + + fn bunion(cls1: &ClassBytes, cls2: &ClassBytes) -> ClassBytes { + let mut cls_ = cls1.clone(); + cls_.union(cls2); + cls_ + } + + fn bintersect(cls1: &ClassBytes, cls2: &ClassBytes) -> ClassBytes { + let mut cls_ = cls1.clone(); + cls_.intersect(cls2); + cls_ + } + + fn bdifference(cls1: &ClassBytes, cls2: &ClassBytes) -> ClassBytes { + let mut cls_ = cls1.clone(); + cls_.difference(cls2); + cls_ + } + + fn bsymdifference(cls1: &ClassBytes, cls2: &ClassBytes) -> ClassBytes { + let mut cls_ = cls1.clone(); + cls_.symmetric_difference(cls2); + cls_ + } + + fn bnegate(cls: &ClassBytes) -> ClassBytes { + let mut cls_ = cls.clone(); + cls_.negate(); + cls_ + } + + #[test] + fn class_range_canonical_unicode() { + let range = ClassUnicodeRange::new('\u{00FF}', '\0'); + assert_eq!('\0', range.start()); + assert_eq!('\u{00FF}', range.end()); + } + + #[test] + fn class_range_canonical_bytes() { + let range = ClassBytesRange::new(b'\xFF', b'\0'); + assert_eq!(b'\0', range.start()); + assert_eq!(b'\xFF', range.end()); + } + + #[test] + fn class_canonicalize_unicode() { + let cls = uclass(&[('a', 'c'), ('x', 'z')]); + let expected = vec![('a', 'c'), ('x', 'z')]; + assert_eq!(expected, uranges(&cls)); + + let cls = uclass(&[('x', 'z'), ('a', 'c')]); + let expected = vec![('a', 'c'), ('x', 'z')]; + assert_eq!(expected, uranges(&cls)); + + let cls = uclass(&[('x', 'z'), ('w', 'y')]); + let expected = vec![('w', 'z')]; + assert_eq!(expected, uranges(&cls)); + + let cls = uclass(&[ + ('c', 'f'), + ('a', 'g'), + ('d', 'j'), + ('a', 'c'), + ('m', 'p'), + ('l', 's'), + ]); + let expected = vec![('a', 'j'), ('l', 's')]; + assert_eq!(expected, uranges(&cls)); + + let cls = uclass(&[('x', 'z'), ('u', 'w')]); + let expected = vec![('u', 'z')]; + assert_eq!(expected, uranges(&cls)); + + let cls = uclass(&[('\x00', '\u{10FFFF}'), ('\x00', '\u{10FFFF}')]); + let expected = vec![('\x00', '\u{10FFFF}')]; + assert_eq!(expected, uranges(&cls)); + + let cls = uclass(&[('a', 'a'), ('b', 'b')]); + let expected = vec![('a', 'b')]; + assert_eq!(expected, uranges(&cls)); + } + + #[test] + fn class_canonicalize_bytes() { + let cls = bclass(&[(b'a', b'c'), (b'x', b'z')]); + let expected = vec![(b'a', b'c'), (b'x', b'z')]; + assert_eq!(expected, branges(&cls)); + + let cls = bclass(&[(b'x', b'z'), (b'a', b'c')]); + let expected = vec![(b'a', b'c'), (b'x', b'z')]; + assert_eq!(expected, branges(&cls)); + + let cls = bclass(&[(b'x', b'z'), (b'w', b'y')]); + let expected = vec![(b'w', b'z')]; + assert_eq!(expected, branges(&cls)); + + let cls = bclass(&[ + (b'c', b'f'), + (b'a', b'g'), + (b'd', b'j'), + (b'a', b'c'), + (b'm', b'p'), + (b'l', b's'), + ]); + let expected = vec![(b'a', b'j'), (b'l', b's')]; + assert_eq!(expected, branges(&cls)); + + let cls = bclass(&[(b'x', b'z'), (b'u', b'w')]); + let expected = vec![(b'u', b'z')]; + assert_eq!(expected, branges(&cls)); + + let cls = bclass(&[(b'\x00', b'\xFF'), (b'\x00', b'\xFF')]); + let expected = vec![(b'\x00', b'\xFF')]; + assert_eq!(expected, branges(&cls)); + + let cls = bclass(&[(b'a', b'a'), (b'b', b'b')]); + let expected = vec![(b'a', b'b')]; + assert_eq!(expected, branges(&cls)); + } + + #[test] + #[cfg(feature = "unicode-case")] + fn class_case_fold_unicode() { + let cls = uclass(&[ + ('C', 'F'), + ('A', 'G'), + ('D', 'J'), + ('A', 'C'), + ('M', 'P'), + ('L', 'S'), + ('c', 'f'), + ]); + let expected = uclass(&[ + ('A', 'J'), + ('L', 'S'), + ('a', 'j'), + ('l', 's'), + ('\u{17F}', '\u{17F}'), + ]); + assert_eq!(expected, ucasefold(&cls)); + + let cls = uclass(&[('A', 'Z')]); + let expected = uclass(&[ + ('A', 'Z'), + ('a', 'z'), + ('\u{17F}', '\u{17F}'), + ('\u{212A}', '\u{212A}'), + ]); + assert_eq!(expected, ucasefold(&cls)); + + let cls = uclass(&[('a', 'z')]); + let expected = uclass(&[ + ('A', 'Z'), + ('a', 'z'), + ('\u{17F}', '\u{17F}'), + ('\u{212A}', '\u{212A}'), + ]); + assert_eq!(expected, ucasefold(&cls)); + + let cls = uclass(&[('A', 'A'), ('_', '_')]); + let expected = uclass(&[('A', 'A'), ('_', '_'), ('a', 'a')]); + assert_eq!(expected, ucasefold(&cls)); + + let cls = uclass(&[('A', 'A'), ('=', '=')]); + let expected = uclass(&[('=', '='), ('A', 'A'), ('a', 'a')]); + assert_eq!(expected, ucasefold(&cls)); + + let cls = uclass(&[('\x00', '\x10')]); + assert_eq!(cls, ucasefold(&cls)); + + let cls = uclass(&[('k', 'k')]); + let expected = + uclass(&[('K', 'K'), ('k', 'k'), ('\u{212A}', '\u{212A}')]); + assert_eq!(expected, ucasefold(&cls)); + + let cls = uclass(&[('@', '@')]); + assert_eq!(cls, ucasefold(&cls)); + } + + #[test] + #[cfg(not(feature = "unicode-case"))] + fn class_case_fold_unicode_disabled() { + let mut cls = uclass(&[ + ('C', 'F'), + ('A', 'G'), + ('D', 'J'), + ('A', 'C'), + ('M', 'P'), + ('L', 'S'), + ('c', 'f'), + ]); + assert!(cls.try_case_fold_simple().is_err()); + } + + #[test] + #[should_panic] + #[cfg(not(feature = "unicode-case"))] + fn class_case_fold_unicode_disabled_panics() { + let mut cls = uclass(&[ + ('C', 'F'), + ('A', 'G'), + ('D', 'J'), + ('A', 'C'), + ('M', 'P'), + ('L', 'S'), + ('c', 'f'), + ]); + cls.case_fold_simple(); + } + + #[test] + fn class_case_fold_bytes() { + let cls = bclass(&[ + (b'C', b'F'), + (b'A', b'G'), + (b'D', b'J'), + (b'A', b'C'), + (b'M', b'P'), + (b'L', b'S'), + (b'c', b'f'), + ]); + let expected = + bclass(&[(b'A', b'J'), (b'L', b'S'), (b'a', b'j'), (b'l', b's')]); + assert_eq!(expected, bcasefold(&cls)); + + let cls = bclass(&[(b'A', b'Z')]); + let expected = bclass(&[(b'A', b'Z'), (b'a', b'z')]); + assert_eq!(expected, bcasefold(&cls)); + + let cls = bclass(&[(b'a', b'z')]); + let expected = bclass(&[(b'A', b'Z'), (b'a', b'z')]); + assert_eq!(expected, bcasefold(&cls)); + + let cls = bclass(&[(b'A', b'A'), (b'_', b'_')]); + let expected = bclass(&[(b'A', b'A'), (b'_', b'_'), (b'a', b'a')]); + assert_eq!(expected, bcasefold(&cls)); + + let cls = bclass(&[(b'A', b'A'), (b'=', b'=')]); + let expected = bclass(&[(b'=', b'='), (b'A', b'A'), (b'a', b'a')]); + assert_eq!(expected, bcasefold(&cls)); + + let cls = bclass(&[(b'\x00', b'\x10')]); + assert_eq!(cls, bcasefold(&cls)); + + let cls = bclass(&[(b'k', b'k')]); + let expected = bclass(&[(b'K', b'K'), (b'k', b'k')]); + assert_eq!(expected, bcasefold(&cls)); + + let cls = bclass(&[(b'@', b'@')]); + assert_eq!(cls, bcasefold(&cls)); + } + + #[test] + fn class_negate_unicode() { + let cls = uclass(&[('a', 'a')]); + let expected = uclass(&[('\x00', '\x60'), ('\x62', '\u{10FFFF}')]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('a', 'a'), ('b', 'b')]); + let expected = uclass(&[('\x00', '\x60'), ('\x63', '\u{10FFFF}')]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('a', 'c'), ('x', 'z')]); + let expected = uclass(&[ + ('\x00', '\x60'), + ('\x64', '\x77'), + ('\x7B', '\u{10FFFF}'), + ]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('\x00', 'a')]); + let expected = uclass(&[('\x62', '\u{10FFFF}')]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('a', '\u{10FFFF}')]); + let expected = uclass(&[('\x00', '\x60')]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('\x00', '\u{10FFFF}')]); + let expected = uclass(&[]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[]); + let expected = uclass(&[('\x00', '\u{10FFFF}')]); + assert_eq!(expected, unegate(&cls)); + + let cls = + uclass(&[('\x00', '\u{10FFFD}'), ('\u{10FFFF}', '\u{10FFFF}')]); + let expected = uclass(&[('\u{10FFFE}', '\u{10FFFE}')]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('\x00', '\u{D7FF}')]); + let expected = uclass(&[('\u{E000}', '\u{10FFFF}')]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('\x00', '\u{D7FE}')]); + let expected = uclass(&[('\u{D7FF}', '\u{10FFFF}')]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('\u{E000}', '\u{10FFFF}')]); + let expected = uclass(&[('\x00', '\u{D7FF}')]); + assert_eq!(expected, unegate(&cls)); + + let cls = uclass(&[('\u{E001}', '\u{10FFFF}')]); + let expected = uclass(&[('\x00', '\u{E000}')]); + assert_eq!(expected, unegate(&cls)); + } + + #[test] + fn class_negate_bytes() { + let cls = bclass(&[(b'a', b'a')]); + let expected = bclass(&[(b'\x00', b'\x60'), (b'\x62', b'\xFF')]); + assert_eq!(expected, bnegate(&cls)); + + let cls = bclass(&[(b'a', b'a'), (b'b', b'b')]); + let expected = bclass(&[(b'\x00', b'\x60'), (b'\x63', b'\xFF')]); + assert_eq!(expected, bnegate(&cls)); + + let cls = bclass(&[(b'a', b'c'), (b'x', b'z')]); + let expected = bclass(&[ + (b'\x00', b'\x60'), + (b'\x64', b'\x77'), + (b'\x7B', b'\xFF'), + ]); + assert_eq!(expected, bnegate(&cls)); + + let cls = bclass(&[(b'\x00', b'a')]); + let expected = bclass(&[(b'\x62', b'\xFF')]); + assert_eq!(expected, bnegate(&cls)); + + let cls = bclass(&[(b'a', b'\xFF')]); + let expected = bclass(&[(b'\x00', b'\x60')]); + assert_eq!(expected, bnegate(&cls)); + + let cls = bclass(&[(b'\x00', b'\xFF')]); + let expected = bclass(&[]); + assert_eq!(expected, bnegate(&cls)); + + let cls = bclass(&[]); + let expected = bclass(&[(b'\x00', b'\xFF')]); + assert_eq!(expected, bnegate(&cls)); + + let cls = bclass(&[(b'\x00', b'\xFD'), (b'\xFF', b'\xFF')]); + let expected = bclass(&[(b'\xFE', b'\xFE')]); + assert_eq!(expected, bnegate(&cls)); + } + + #[test] + fn class_union_unicode() { + let cls1 = uclass(&[('a', 'g'), ('m', 't'), ('A', 'C')]); + let cls2 = uclass(&[('a', 'z')]); + let expected = uclass(&[('a', 'z'), ('A', 'C')]); + assert_eq!(expected, uunion(&cls1, &cls2)); + } + + #[test] + fn class_union_bytes() { + let cls1 = bclass(&[(b'a', b'g'), (b'm', b't'), (b'A', b'C')]); + let cls2 = bclass(&[(b'a', b'z')]); + let expected = bclass(&[(b'a', b'z'), (b'A', b'C')]); + assert_eq!(expected, bunion(&cls1, &cls2)); + } + + #[test] + fn class_intersect_unicode() { + let cls1 = uclass(&[]); + let cls2 = uclass(&[('a', 'a')]); + let expected = uclass(&[]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'a')]); + let cls2 = uclass(&[('a', 'a')]); + let expected = uclass(&[('a', 'a')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'a')]); + let cls2 = uclass(&[('b', 'b')]); + let expected = uclass(&[]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'a')]); + let cls2 = uclass(&[('a', 'c')]); + let expected = uclass(&[('a', 'a')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b')]); + let cls2 = uclass(&[('a', 'c')]); + let expected = uclass(&[('a', 'b')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b')]); + let cls2 = uclass(&[('b', 'c')]); + let expected = uclass(&[('b', 'b')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b')]); + let cls2 = uclass(&[('c', 'd')]); + let expected = uclass(&[]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('b', 'c')]); + let cls2 = uclass(&[('a', 'd')]); + let expected = uclass(&[('b', 'c')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b'), ('d', 'e'), ('g', 'h')]); + let cls2 = uclass(&[('a', 'h')]); + let expected = uclass(&[('a', 'b'), ('d', 'e'), ('g', 'h')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b'), ('d', 'e'), ('g', 'h')]); + let cls2 = uclass(&[('a', 'b'), ('d', 'e'), ('g', 'h')]); + let expected = uclass(&[('a', 'b'), ('d', 'e'), ('g', 'h')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b'), ('g', 'h')]); + let cls2 = uclass(&[('d', 'e'), ('k', 'l')]); + let expected = uclass(&[]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b'), ('d', 'e'), ('g', 'h')]); + let cls2 = uclass(&[('h', 'h')]); + let expected = uclass(&[('h', 'h')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b'), ('e', 'f'), ('i', 'j')]); + let cls2 = uclass(&[('c', 'd'), ('g', 'h'), ('k', 'l')]); + let expected = uclass(&[]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'b'), ('c', 'd'), ('e', 'f')]); + let cls2 = uclass(&[('b', 'c'), ('d', 'e'), ('f', 'g')]); + let expected = uclass(&[('b', 'f')]); + assert_eq!(expected, uintersect(&cls1, &cls2)); + } + + #[test] + fn class_intersect_bytes() { + let cls1 = bclass(&[]); + let cls2 = bclass(&[(b'a', b'a')]); + let expected = bclass(&[]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'a')]); + let cls2 = bclass(&[(b'a', b'a')]); + let expected = bclass(&[(b'a', b'a')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'a')]); + let cls2 = bclass(&[(b'b', b'b')]); + let expected = bclass(&[]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'a')]); + let cls2 = bclass(&[(b'a', b'c')]); + let expected = bclass(&[(b'a', b'a')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b')]); + let cls2 = bclass(&[(b'a', b'c')]); + let expected = bclass(&[(b'a', b'b')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b')]); + let cls2 = bclass(&[(b'b', b'c')]); + let expected = bclass(&[(b'b', b'b')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b')]); + let cls2 = bclass(&[(b'c', b'd')]); + let expected = bclass(&[]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'b', b'c')]); + let cls2 = bclass(&[(b'a', b'd')]); + let expected = bclass(&[(b'b', b'c')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b'), (b'd', b'e'), (b'g', b'h')]); + let cls2 = bclass(&[(b'a', b'h')]); + let expected = bclass(&[(b'a', b'b'), (b'd', b'e'), (b'g', b'h')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b'), (b'd', b'e'), (b'g', b'h')]); + let cls2 = bclass(&[(b'a', b'b'), (b'd', b'e'), (b'g', b'h')]); + let expected = bclass(&[(b'a', b'b'), (b'd', b'e'), (b'g', b'h')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b'), (b'g', b'h')]); + let cls2 = bclass(&[(b'd', b'e'), (b'k', b'l')]); + let expected = bclass(&[]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b'), (b'd', b'e'), (b'g', b'h')]); + let cls2 = bclass(&[(b'h', b'h')]); + let expected = bclass(&[(b'h', b'h')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b'), (b'e', b'f'), (b'i', b'j')]); + let cls2 = bclass(&[(b'c', b'd'), (b'g', b'h'), (b'k', b'l')]); + let expected = bclass(&[]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'b'), (b'c', b'd'), (b'e', b'f')]); + let cls2 = bclass(&[(b'b', b'c'), (b'd', b'e'), (b'f', b'g')]); + let expected = bclass(&[(b'b', b'f')]); + assert_eq!(expected, bintersect(&cls1, &cls2)); + } + + #[test] + fn class_difference_unicode() { + let cls1 = uclass(&[('a', 'a')]); + let cls2 = uclass(&[('a', 'a')]); + let expected = uclass(&[]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'a')]); + let cls2 = uclass(&[]); + let expected = uclass(&[('a', 'a')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[]); + let cls2 = uclass(&[('a', 'a')]); + let expected = uclass(&[]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'z')]); + let cls2 = uclass(&[('a', 'a')]); + let expected = uclass(&[('b', 'z')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'z')]); + let cls2 = uclass(&[('z', 'z')]); + let expected = uclass(&[('a', 'y')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'z')]); + let cls2 = uclass(&[('m', 'm')]); + let expected = uclass(&[('a', 'l'), ('n', 'z')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'c'), ('g', 'i'), ('r', 't')]); + let cls2 = uclass(&[('a', 'z')]); + let expected = uclass(&[]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'c'), ('g', 'i'), ('r', 't')]); + let cls2 = uclass(&[('d', 'v')]); + let expected = uclass(&[('a', 'c')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'c'), ('g', 'i'), ('r', 't')]); + let cls2 = uclass(&[('b', 'g'), ('s', 'u')]); + let expected = uclass(&[('a', 'a'), ('h', 'i'), ('r', 'r')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'c'), ('g', 'i'), ('r', 't')]); + let cls2 = uclass(&[('b', 'd'), ('e', 'g'), ('s', 'u')]); + let expected = uclass(&[('a', 'a'), ('h', 'i'), ('r', 'r')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('x', 'z')]); + let cls2 = uclass(&[('a', 'c'), ('e', 'g'), ('s', 'u')]); + let expected = uclass(&[('x', 'z')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + + let cls1 = uclass(&[('a', 'z')]); + let cls2 = uclass(&[('a', 'c'), ('e', 'g'), ('s', 'u')]); + let expected = uclass(&[('d', 'd'), ('h', 'r'), ('v', 'z')]); + assert_eq!(expected, udifference(&cls1, &cls2)); + } + + #[test] + fn class_difference_bytes() { + let cls1 = bclass(&[(b'a', b'a')]); + let cls2 = bclass(&[(b'a', b'a')]); + let expected = bclass(&[]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'a')]); + let cls2 = bclass(&[]); + let expected = bclass(&[(b'a', b'a')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[]); + let cls2 = bclass(&[(b'a', b'a')]); + let expected = bclass(&[]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'z')]); + let cls2 = bclass(&[(b'a', b'a')]); + let expected = bclass(&[(b'b', b'z')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'z')]); + let cls2 = bclass(&[(b'z', b'z')]); + let expected = bclass(&[(b'a', b'y')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'z')]); + let cls2 = bclass(&[(b'm', b'm')]); + let expected = bclass(&[(b'a', b'l'), (b'n', b'z')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'c'), (b'g', b'i'), (b'r', b't')]); + let cls2 = bclass(&[(b'a', b'z')]); + let expected = bclass(&[]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'c'), (b'g', b'i'), (b'r', b't')]); + let cls2 = bclass(&[(b'd', b'v')]); + let expected = bclass(&[(b'a', b'c')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'c'), (b'g', b'i'), (b'r', b't')]); + let cls2 = bclass(&[(b'b', b'g'), (b's', b'u')]); + let expected = bclass(&[(b'a', b'a'), (b'h', b'i'), (b'r', b'r')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'c'), (b'g', b'i'), (b'r', b't')]); + let cls2 = bclass(&[(b'b', b'd'), (b'e', b'g'), (b's', b'u')]); + let expected = bclass(&[(b'a', b'a'), (b'h', b'i'), (b'r', b'r')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'x', b'z')]); + let cls2 = bclass(&[(b'a', b'c'), (b'e', b'g'), (b's', b'u')]); + let expected = bclass(&[(b'x', b'z')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + + let cls1 = bclass(&[(b'a', b'z')]); + let cls2 = bclass(&[(b'a', b'c'), (b'e', b'g'), (b's', b'u')]); + let expected = bclass(&[(b'd', b'd'), (b'h', b'r'), (b'v', b'z')]); + assert_eq!(expected, bdifference(&cls1, &cls2)); + } + + #[test] + fn class_symmetric_difference_unicode() { + let cls1 = uclass(&[('a', 'm')]); + let cls2 = uclass(&[('g', 't')]); + let expected = uclass(&[('a', 'f'), ('n', 't')]); + assert_eq!(expected, usymdifference(&cls1, &cls2)); + } + + #[test] + fn class_symmetric_difference_bytes() { + let cls1 = bclass(&[(b'a', b'm')]); + let cls2 = bclass(&[(b'g', b't')]); + let expected = bclass(&[(b'a', b'f'), (b'n', b't')]); + assert_eq!(expected, bsymdifference(&cls1, &cls2)); + } + + // We use a thread with an explicit stack size to test that our destructor + // for Hir can handle arbitrarily sized expressions in constant stack + // space. In case we run on a platform without threads (WASM?), we limit + // this test to Windows/Unix. + #[test] + #[cfg(any(unix, windows))] + fn no_stack_overflow_on_drop() { + use std::thread; + + let run = || { + let mut expr = Hir::empty(); + for _ in 0..100 { + expr = Hir::capture(Capture { + index: 1, + name: None, + sub: Box::new(expr), + }); + expr = Hir::repetition(Repetition { + min: 0, + max: Some(1), + greedy: true, + sub: Box::new(expr), + }); + + expr = Hir { + kind: HirKind::Concat(vec![expr]), + props: Properties::empty(), + }; + expr = Hir { + kind: HirKind::Alternation(vec![expr]), + props: Properties::empty(), + }; + } + assert!(!matches!(*expr.kind(), HirKind::Empty)); + }; + + // We run our test on a thread with a small stack size so we can + // force the issue more easily. + // + // NOTE(2023-03-21): See the corresponding test in 'crate::ast::tests' + // for context on the specific stack size chosen here. + thread::Builder::new() + .stack_size(16 << 10) + .spawn(run) + .unwrap() + .join() + .unwrap(); + } + + #[test] + fn look_set_iter() { + let set = LookSet::empty(); + assert_eq!(0, set.iter().count()); + + let set = LookSet::full(); + assert_eq!(18, set.iter().count()); + + let set = + LookSet::empty().insert(Look::StartLF).insert(Look::WordUnicode); + assert_eq!(2, set.iter().count()); + + let set = LookSet::empty().insert(Look::StartLF); + assert_eq!(1, set.iter().count()); + + let set = LookSet::empty().insert(Look::WordAsciiNegate); + assert_eq!(1, set.iter().count()); + } + + #[test] + fn look_set_debug() { + let res = format!("{:?}", LookSet::empty()); + assert_eq!("∅", res); + let res = format!("{:?}", LookSet::full()); + assert_eq!("Az^$rRbB𝛃𝚩<>〈〉◁▷◀▶", res); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/hir/print.rs b/third_party/crates/regex-syntax-0.8.11/src/hir/print.rs new file mode 100644 index 0000000..89db08c2 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/hir/print.rs @@ -0,0 +1,608 @@ +/*! +This module provides a regular expression printer for `Hir`. +*/ + +use core::fmt; + +use crate::{ + hir::{ + self, + visitor::{self, Visitor}, + Hir, HirKind, + }, + is_meta_character, +}; + +/// A builder for constructing a printer. +/// +/// Note that since a printer doesn't have any configuration knobs, this type +/// remains unexported. +#[derive(Clone, Debug)] +struct PrinterBuilder { + _priv: (), +} + +impl Default for PrinterBuilder { + fn default() -> PrinterBuilder { + PrinterBuilder::new() + } +} + +impl PrinterBuilder { + fn new() -> PrinterBuilder { + PrinterBuilder { _priv: () } + } + + fn build(&self) -> Printer { + Printer { _priv: () } + } +} + +/// A printer for a regular expression's high-level intermediate +/// representation. +/// +/// A printer converts a high-level intermediate representation (HIR) to a +/// regular expression pattern string. This particular printer uses constant +/// stack space and heap space proportional to the size of the HIR. +/// +/// Since this printer is only using the HIR, the pattern it prints will likely +/// not resemble the original pattern at all. For example, a pattern like +/// `\pL` will have its entire class written out. +/// +/// The purpose of this printer is to provide a means to mutate an HIR and then +/// build a regular expression from the result of that mutation. (A regex +/// library could provide a constructor from this HIR explicitly, but that +/// creates an unnecessary public coupling between the regex library and this +/// specific HIR representation.) +#[derive(Debug)] +pub struct Printer { + _priv: (), +} + +impl Printer { + /// Create a new printer. + pub fn new() -> Printer { + PrinterBuilder::new().build() + } + + /// Print the given `Ast` to the given writer. The writer must implement + /// `fmt::Write`. Typical implementations of `fmt::Write` that can be used + /// here are a `fmt::Formatter` (which is available in `fmt::Display` + /// implementations) or a `&mut String`. + pub fn print<W: fmt::Write>(&mut self, hir: &Hir, wtr: W) -> fmt::Result { + visitor::visit(hir, Writer { wtr }) + } +} + +#[derive(Debug)] +struct Writer<W> { + wtr: W, +} + +impl<W: fmt::Write> Visitor for Writer<W> { + type Output = (); + type Err = fmt::Error; + + fn finish(self) -> fmt::Result { + Ok(()) + } + + fn visit_pre(&mut self, hir: &Hir) -> fmt::Result { + match *hir.kind() { + HirKind::Empty => { + // Technically an empty sub-expression could be "printed" by + // just ignoring it, but in practice, you could have a + // repetition operator attached to an empty expression, and you + // really need something in the concrete syntax to make that + // work as you'd expect. + self.wtr.write_str(r"(?:)")?; + } + // Repetition operators are strictly suffix oriented. + HirKind::Repetition(_) => {} + HirKind::Literal(hir::Literal(ref bytes)) => { + // See the comment on the 'Concat' and 'Alternation' case below + // for why we put parens here. Literals are, conceptually, + // a special case of concatenation where each element is a + // character. The HIR flattens this into a Box<[u8]>, but we + // still need to treat it like a concatenation for correct + // printing. As a special case, we don't write parens if there + // is only one character. One character means there is no + // concat so we don't need parens. Adding parens would still be + // correct, but we drop them here because it tends to create + // rather noisy regexes even in simple cases. + let result = core::str::from_utf8(bytes); + let len = result.map_or(bytes.len(), |s| s.chars().count()); + if len > 1 { + self.wtr.write_str(r"(?:")?; + } + match result { + Ok(string) => { + for c in string.chars() { + self.write_literal_char(c)?; + } + } + Err(_) => { + for &b in bytes.iter() { + self.write_literal_byte(b)?; + } + } + } + if len > 1 { + self.wtr.write_str(r")")?; + } + } + HirKind::Class(hir::Class::Unicode(ref cls)) => { + if cls.ranges().is_empty() { + return self.wtr.write_str("[a&&b]"); + } + self.wtr.write_str("[")?; + for range in cls.iter() { + if range.start() == range.end() { + self.write_literal_char(range.start())?; + } else if u32::from(range.start()) + 1 + == u32::from(range.end()) + { + self.write_literal_char(range.start())?; + self.write_literal_char(range.end())?; + } else { + self.write_literal_char(range.start())?; + self.wtr.write_str("-")?; + self.write_literal_char(range.end())?; + } + } + self.wtr.write_str("]")?; + } + HirKind::Class(hir::Class::Bytes(ref cls)) => { + if cls.ranges().is_empty() { + return self.wtr.write_str("[a&&b]"); + } + self.wtr.write_str("(?-u:[")?; + for range in cls.iter() { + if range.start() == range.end() { + self.write_literal_class_byte(range.start())?; + } else if range.start() + 1 == range.end() { + self.write_literal_class_byte(range.start())?; + self.write_literal_class_byte(range.end())?; + } else { + self.write_literal_class_byte(range.start())?; + self.wtr.write_str("-")?; + self.write_literal_class_byte(range.end())?; + } + } + self.wtr.write_str("])")?; + } + HirKind::Look(ref look) => match *look { + hir::Look::Start => { + self.wtr.write_str(r"\A")?; + } + hir::Look::End => { + self.wtr.write_str(r"\z")?; + } + hir::Look::StartLF => { + self.wtr.write_str("(?m:^)")?; + } + hir::Look::EndLF => { + self.wtr.write_str("(?m:$)")?; + } + hir::Look::StartCRLF => { + self.wtr.write_str("(?mR:^)")?; + } + hir::Look::EndCRLF => { + self.wtr.write_str("(?mR:$)")?; + } + hir::Look::WordAscii => { + self.wtr.write_str(r"(?-u:\b)")?; + } + hir::Look::WordAsciiNegate => { + self.wtr.write_str(r"(?-u:\B)")?; + } + hir::Look::WordUnicode => { + self.wtr.write_str(r"\b")?; + } + hir::Look::WordUnicodeNegate => { + self.wtr.write_str(r"\B")?; + } + hir::Look::WordStartAscii => { + self.wtr.write_str(r"(?-u:\b{start})")?; + } + hir::Look::WordEndAscii => { + self.wtr.write_str(r"(?-u:\b{end})")?; + } + hir::Look::WordStartUnicode => { + self.wtr.write_str(r"\b{start}")?; + } + hir::Look::WordEndUnicode => { + self.wtr.write_str(r"\b{end}")?; + } + hir::Look::WordStartHalfAscii => { + self.wtr.write_str(r"(?-u:\b{start-half})")?; + } + hir::Look::WordEndHalfAscii => { + self.wtr.write_str(r"(?-u:\b{end-half})")?; + } + hir::Look::WordStartHalfUnicode => { + self.wtr.write_str(r"\b{start-half}")?; + } + hir::Look::WordEndHalfUnicode => { + self.wtr.write_str(r"\b{end-half}")?; + } + }, + HirKind::Capture(hir::Capture { ref name, .. }) => { + self.wtr.write_str("(")?; + if let Some(ref name) = *name { + write!(self.wtr, "?P<{name}>")?; + } + } + // Why do this? Wrapping concats and alts in non-capturing groups + // is not *always* necessary, but is sometimes necessary. For + // example, 'concat(a, alt(b, c))' should be written as 'a(?:b|c)' + // and not 'ab|c'. The former is clearly the intended meaning, but + // the latter is actually 'alt(concat(a, b), c)'. + // + // It would be possible to only group these things in cases where + // it's strictly necessary, but it requires knowing the parent + // expression. And since this technique is simpler and always + // correct, we take this route. More to the point, it is a non-goal + // of an HIR printer to show a nice easy-to-read regex. Indeed, + // its construction forbids it from doing so. Therefore, inserting + // extra groups where they aren't necessary is perfectly okay. + HirKind::Concat(_) | HirKind::Alternation(_) => { + self.wtr.write_str(r"(?:")?; + } + } + Ok(()) + } + + fn visit_post(&mut self, hir: &Hir) -> fmt::Result { + match *hir.kind() { + // Handled during visit_pre + HirKind::Empty + | HirKind::Literal(_) + | HirKind::Class(_) + | HirKind::Look(_) => {} + HirKind::Repetition(ref x) => { + match (x.min, x.max) { + (0, Some(1)) => { + self.wtr.write_str("?")?; + } + (0, None) => { + self.wtr.write_str("*")?; + } + (1, None) => { + self.wtr.write_str("+")?; + } + (1, Some(1)) => { + // 'a{1}' and 'a{1}?' are exactly equivalent to 'a'. + return Ok(()); + } + (m, None) => { + write!(self.wtr, "{{{m},}}")?; + } + (m, Some(n)) if m == n => { + write!(self.wtr, "{{{m}}}")?; + // a{m} and a{m}? are always exactly equivalent. + return Ok(()); + } + (m, Some(n)) => { + write!(self.wtr, "{{{m},{n}}}")?; + } + } + if !x.greedy { + self.wtr.write_str("?")?; + } + } + HirKind::Capture(_) + | HirKind::Concat(_) + | HirKind::Alternation(_) => { + self.wtr.write_str(r")")?; + } + } + Ok(()) + } + + fn visit_alternation_in(&mut self) -> fmt::Result { + self.wtr.write_str("|") + } +} + +impl<W: fmt::Write> Writer<W> { + fn write_literal_char(&mut self, c: char) -> fmt::Result { + if is_meta_character(c) { + self.wtr.write_str("\\")?; + } + self.wtr.write_char(c) + } + + fn write_literal_byte(&mut self, b: u8) -> fmt::Result { + if b <= 0x7F && !b.is_ascii_control() && !b.is_ascii_whitespace() { + self.write_literal_char(char::try_from(b).unwrap()) + } else { + write!(self.wtr, "(?-u:\\x{b:02X})") + } + } + + fn write_literal_class_byte(&mut self, b: u8) -> fmt::Result { + if b <= 0x7F && !b.is_ascii_control() && !b.is_ascii_whitespace() { + self.write_literal_char(char::try_from(b).unwrap()) + } else { + write!(self.wtr, "\\x{b:02X}") + } + } +} + +#[cfg(test)] +mod tests { + use alloc::{ + boxed::Box, + string::{String, ToString}, + }; + + use crate::ParserBuilder; + + use super::*; + + fn roundtrip(given: &str, expected: &str) { + roundtrip_with(|b| b, given, expected); + } + + fn roundtrip_bytes(given: &str, expected: &str) { + roundtrip_with(|b| b.utf8(false), given, expected); + } + + fn roundtrip_with<F>(mut f: F, given: &str, expected: &str) + where + F: FnMut(&mut ParserBuilder) -> &mut ParserBuilder, + { + let mut builder = ParserBuilder::new(); + f(&mut builder); + let hir = builder.build().parse(given).unwrap(); + + let mut printer = Printer::new(); + let mut dst = String::new(); + printer.print(&hir, &mut dst).unwrap(); + + // Check that the result is actually valid. + builder.build().parse(&dst).unwrap(); + + assert_eq!(expected, dst); + } + + #[test] + fn print_literal() { + roundtrip("a", "a"); + roundtrip(r"\xff", "\u{FF}"); + roundtrip_bytes(r"\xff", "\u{FF}"); + roundtrip_bytes(r"(?-u)\xff", r"(?-u:\xFF)"); + roundtrip("☃", "☃"); + } + + #[test] + fn print_class() { + roundtrip(r"[a]", r"a"); + roundtrip(r"[ab]", r"[ab]"); + roundtrip(r"[a-z]", r"[a-z]"); + roundtrip(r"[a-z--b-c--x-y]", r"[ad-wz]"); + roundtrip(r"[^\x01-\u{10FFFF}]", "\u{0}"); + roundtrip(r"[-]", r"\-"); + roundtrip(r"[☃-⛄]", r"[☃-⛄]"); + + roundtrip(r"(?-u)[a]", r"a"); + roundtrip(r"(?-u)[ab]", r"(?-u:[ab])"); + roundtrip(r"(?-u)[a-z]", r"(?-u:[a-z])"); + roundtrip_bytes(r"(?-u)[a-\xFF]", r"(?-u:[a-\xFF])"); + + // The following test that the printer escapes meta characters + // in character classes. + roundtrip(r"[\[]", r"\["); + roundtrip(r"[Z-_]", r"[Z-_]"); + roundtrip(r"[Z-_--Z]", r"[\[-_]"); + + // The following test that the printer escapes meta characters + // in byte oriented character classes. + roundtrip_bytes(r"(?-u)[\[]", r"\["); + roundtrip_bytes(r"(?-u)[Z-_]", r"(?-u:[Z-_])"); + roundtrip_bytes(r"(?-u)[Z-_--Z]", r"(?-u:[\[-_])"); + + // This tests that an empty character class is correctly roundtripped. + #[cfg(feature = "unicode-gencat")] + roundtrip(r"\P{any}", r"[a&&b]"); + roundtrip_bytes(r"(?-u)[^\x00-\xFF]", r"[a&&b]"); + } + + #[test] + fn print_anchor() { + roundtrip(r"^", r"\A"); + roundtrip(r"$", r"\z"); + roundtrip(r"(?m)^", r"(?m:^)"); + roundtrip(r"(?m)$", r"(?m:$)"); + } + + #[test] + fn print_word_boundary() { + roundtrip(r"\b", r"\b"); + roundtrip(r"\B", r"\B"); + roundtrip(r"(?-u)\b", r"(?-u:\b)"); + roundtrip_bytes(r"(?-u)\B", r"(?-u:\B)"); + } + + #[test] + fn print_repetition() { + roundtrip("a?", "a?"); + roundtrip("a??", "a??"); + roundtrip("(?U)a?", "a??"); + + roundtrip("a*", "a*"); + roundtrip("a*?", "a*?"); + roundtrip("(?U)a*", "a*?"); + + roundtrip("a+", "a+"); + roundtrip("a+?", "a+?"); + roundtrip("(?U)a+", "a+?"); + + roundtrip("a{1}", "a"); + roundtrip("a{2}", "a{2}"); + roundtrip("a{1,}", "a+"); + roundtrip("a{1,5}", "a{1,5}"); + roundtrip("a{1}?", "a"); + roundtrip("a{2}?", "a{2}"); + roundtrip("a{1,}?", "a+?"); + roundtrip("a{1,5}?", "a{1,5}?"); + roundtrip("(?U)a{1}", "a"); + roundtrip("(?U)a{2}", "a{2}"); + roundtrip("(?U)a{1,}", "a+?"); + roundtrip("(?U)a{1,5}", "a{1,5}?"); + + // Test that various zero-length repetitions always translate to an + // empty regex. This is more a property of HIR's smart constructors + // than the printer though. + roundtrip("a{0}", "(?:)"); + roundtrip("(?:ab){0}", "(?:)"); + #[cfg(feature = "unicode-gencat")] + { + roundtrip(r"\p{any}{0}", "(?:)"); + roundtrip(r"\P{any}{0}", "(?:)"); + } + } + + #[test] + fn print_group() { + roundtrip("()", "((?:))"); + roundtrip("(?P<foo>)", "(?P<foo>(?:))"); + roundtrip("(?:)", "(?:)"); + + roundtrip("(a)", "(a)"); + roundtrip("(?P<foo>a)", "(?P<foo>a)"); + roundtrip("(?:a)", "a"); + + roundtrip("((((a))))", "((((a))))"); + } + + #[test] + fn print_alternation() { + roundtrip("|", "(?:(?:)|(?:))"); + roundtrip("||", "(?:(?:)|(?:)|(?:))"); + + roundtrip("a|b", "[ab]"); + roundtrip("ab|cd", "(?:(?:ab)|(?:cd))"); + roundtrip("a|b|c", "[a-c]"); + roundtrip("ab|cd|ef", "(?:(?:ab)|(?:cd)|(?:ef))"); + roundtrip("foo|bar|quux", "(?:(?:foo)|(?:bar)|(?:quux))"); + } + + // This is a regression test that stresses a peculiarity of how the HIR + // is both constructed and printed. Namely, it is legal for a repetition + // to directly contain a concatenation. This particular construct isn't + // really possible to build from the concrete syntax directly, since you'd + // be forced to put the concatenation into (at least) a non-capturing + // group. Concurrently, the printer doesn't consider this case and just + // kind of naively prints the child expression and tacks on the repetition + // operator. + // + // As a result, if you attached '+' to a 'concat(a, b)', the printer gives + // you 'ab+', but clearly it really should be '(?:ab)+'. + // + // This bug isn't easy to surface because most ways of building an HIR + // come directly from the concrete syntax, and as mentioned above, it just + // isn't possible to build this kind of HIR from the concrete syntax. + // Nevertheless, this is definitely a bug. + // + // See: https://github.com/rust-lang/regex/issues/731 + #[test] + fn regression_repetition_concat() { + let expr = Hir::concat(alloc::vec![ + Hir::literal("x".as_bytes()), + Hir::repetition(hir::Repetition { + min: 1, + max: None, + greedy: true, + sub: Box::new(Hir::literal("ab".as_bytes())), + }), + Hir::literal("y".as_bytes()), + ]); + assert_eq!(r"(?:x(?:ab)+y)", expr.to_string()); + + let expr = Hir::concat(alloc::vec![ + Hir::look(hir::Look::Start), + Hir::repetition(hir::Repetition { + min: 1, + max: None, + greedy: true, + sub: Box::new(Hir::concat(alloc::vec![ + Hir::look(hir::Look::Start), + Hir::look(hir::Look::End), + ])), + }), + Hir::look(hir::Look::End), + ]); + assert_eq!(r"(?:\A\A\z\z)", expr.to_string()); + } + + // Just like regression_repetition_concat, but with the repetition using + // an alternation as a child expression instead. + // + // See: https://github.com/rust-lang/regex/issues/731 + #[test] + fn regression_repetition_alternation() { + let expr = Hir::concat(alloc::vec![ + Hir::literal("ab".as_bytes()), + Hir::repetition(hir::Repetition { + min: 1, + max: None, + greedy: true, + sub: Box::new(Hir::alternation(alloc::vec![ + Hir::literal("cd".as_bytes()), + Hir::literal("ef".as_bytes()), + ])), + }), + Hir::literal("gh".as_bytes()), + ]); + assert_eq!(r"(?:(?:ab)(?:(?:cd)|(?:ef))+(?:gh))", expr.to_string()); + + let expr = Hir::concat(alloc::vec![ + Hir::look(hir::Look::Start), + Hir::repetition(hir::Repetition { + min: 1, + max: None, + greedy: true, + sub: Box::new(Hir::alternation(alloc::vec![ + Hir::look(hir::Look::Start), + Hir::look(hir::Look::End), + ])), + }), + Hir::look(hir::Look::End), + ]); + assert_eq!(r"(?:\A(?:\A|\z)\z)", expr.to_string()); + } + + // This regression test is very similar in flavor to + // regression_repetition_concat in that the root of the issue lies in a + // peculiarity of how the HIR is represented and how the printer writes it + // out. Like the other regression, this one is also rooted in the fact that + // you can't produce the peculiar HIR from the concrete syntax. Namely, you + // just can't have a 'concat(a, alt(b, c))' because the 'alt' will normally + // be in (at least) a non-capturing group. Why? Because the '|' has very + // low precedence (lower that concatenation), and so something like 'ab|c' + // is actually 'alt(ab, c)'. + // + // See: https://github.com/rust-lang/regex/issues/516 + #[test] + fn regression_alternation_concat() { + let expr = Hir::concat(alloc::vec![ + Hir::literal("ab".as_bytes()), + Hir::alternation(alloc::vec![ + Hir::literal("mn".as_bytes()), + Hir::literal("xy".as_bytes()), + ]), + ]); + assert_eq!(r"(?:(?:ab)(?:(?:mn)|(?:xy)))", expr.to_string()); + + let expr = Hir::concat(alloc::vec![ + Hir::look(hir::Look::Start), + Hir::alternation(alloc::vec![ + Hir::look(hir::Look::Start), + Hir::look(hir::Look::End), + ]), + ]); + assert_eq!(r"(?:\A(?:\A|\z))", expr.to_string()); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/hir/translate.rs b/third_party/crates/regex-syntax-0.8.11/src/hir/translate.rs new file mode 100644 index 0000000..91a5e17 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/hir/translate.rs @@ -0,0 +1,3744 @@ +/*! +Defines a translator that converts an `Ast` to an `Hir`. +*/ + +use core::cell::{Cell, RefCell}; + +use alloc::{boxed::Box, string::ToString, vec, vec::Vec}; + +use crate::{ + ast::{self, Ast, Span, Visitor}, + either::Either, + hir::{self, Error, ErrorKind, Hir, HirKind}, + unicode::{self, ClassQuery}, +}; + +type Result<T> = core::result::Result<T, Error>; + +/// A builder for constructing an AST->HIR translator. +#[derive(Clone, Debug)] +pub struct TranslatorBuilder { + utf8: bool, + line_terminator: u8, + flags: Flags, +} + +impl Default for TranslatorBuilder { + fn default() -> TranslatorBuilder { + TranslatorBuilder::new() + } +} + +impl TranslatorBuilder { + /// Create a new translator builder with a default configuration. + pub fn new() -> TranslatorBuilder { + TranslatorBuilder { + utf8: true, + line_terminator: b'\n', + flags: Flags::default(), + } + } + + /// Build a translator using the current configuration. + pub fn build(&self) -> Translator { + Translator { + stack: RefCell::new(vec![]), + flags: Cell::new(self.flags), + utf8: self.utf8, + line_terminator: self.line_terminator, + } + } + + /// When disabled, translation will permit the construction of a regular + /// expression that may match invalid UTF-8. + /// + /// When enabled (the default), the translator is guaranteed to produce an + /// expression that, for non-empty matches, will only ever produce spans + /// that are entirely valid UTF-8 (otherwise, the translator will return an + /// error). + /// + /// Perhaps surprisingly, when UTF-8 is enabled, an empty regex or even + /// a negated ASCII word boundary (uttered as `(?-u:\B)` in the concrete + /// syntax) will be allowed even though they can produce matches that split + /// a UTF-8 encoded codepoint. This only applies to zero-width or "empty" + /// matches, and it is expected that the regex engine itself must handle + /// these cases if necessary (perhaps by suppressing any zero-width matches + /// that split a codepoint). + pub fn utf8(&mut self, yes: bool) -> &mut TranslatorBuilder { + self.utf8 = yes; + self + } + + /// Sets the line terminator for use with `(?u-s:.)` and `(?-us:.)`. + /// + /// Namely, instead of `.` (by default) matching everything except for `\n`, + /// this will cause `.` to match everything except for the byte given. + /// + /// If `.` is used in a context where Unicode mode is enabled and this byte + /// isn't ASCII, then an error will be returned. When Unicode mode is + /// disabled, then any byte is permitted, but will return an error if UTF-8 + /// mode is enabled and it is a non-ASCII byte. + /// + /// In short, any ASCII value for a line terminator is always okay. But a + /// non-ASCII byte might result in an error depending on whether Unicode + /// mode or UTF-8 mode are enabled. + /// + /// Note that if `R` mode is enabled then it always takes precedence and + /// the line terminator will be treated as `\r` and `\n` simultaneously. + /// + /// Note also that this *doesn't* impact the look-around assertions + /// `(?m:^)` and `(?m:$)`. That's usually controlled by additional + /// configuration in the regex engine itself. + pub fn line_terminator(&mut self, byte: u8) -> &mut TranslatorBuilder { + self.line_terminator = byte; + self + } + + /// Enable or disable the case insensitive flag (`i`) by default. + pub fn case_insensitive(&mut self, yes: bool) -> &mut TranslatorBuilder { + self.flags.case_insensitive = if yes { Some(true) } else { None }; + self + } + + /// Enable or disable the multi-line matching flag (`m`) by default. + pub fn multi_line(&mut self, yes: bool) -> &mut TranslatorBuilder { + self.flags.multi_line = if yes { Some(true) } else { None }; + self + } + + /// Enable or disable the "dot matches any character" flag (`s`) by + /// default. + pub fn dot_matches_new_line( + &mut self, + yes: bool, + ) -> &mut TranslatorBuilder { + self.flags.dot_matches_new_line = if yes { Some(true) } else { None }; + self + } + + /// Enable or disable the CRLF mode flag (`R`) by default. + pub fn crlf(&mut self, yes: bool) -> &mut TranslatorBuilder { + self.flags.crlf = if yes { Some(true) } else { None }; + self + } + + /// Enable or disable the "swap greed" flag (`U`) by default. + pub fn swap_greed(&mut self, yes: bool) -> &mut TranslatorBuilder { + self.flags.swap_greed = if yes { Some(true) } else { None }; + self + } + + /// Enable or disable the Unicode flag (`u`) by default. + pub fn unicode(&mut self, yes: bool) -> &mut TranslatorBuilder { + self.flags.unicode = if yes { None } else { Some(false) }; + self + } +} + +/// A translator maps abstract syntax to a high level intermediate +/// representation. +/// +/// A translator may be benefit from reuse. That is, a translator can translate +/// many abstract syntax trees. +/// +/// A `Translator` can be configured in more detail via a +/// [`TranslatorBuilder`]. +#[derive(Clone, Debug)] +pub struct Translator { + /// Our call stack, but on the heap. + stack: RefCell<Vec<HirFrame>>, + /// The current flag settings. + flags: Cell<Flags>, + /// Whether we're allowed to produce HIR that can match arbitrary bytes. + utf8: bool, + /// The line terminator to use for `.`. + line_terminator: u8, +} + +impl Translator { + /// Create a new translator using the default configuration. + pub fn new() -> Translator { + TranslatorBuilder::new().build() + } + + /// Translate the given abstract syntax tree (AST) into a high level + /// intermediate representation (HIR). + /// + /// If there was a problem doing the translation, then an HIR-specific + /// error is returned. + /// + /// The original pattern string used to produce the `Ast` *must* also be + /// provided. The translator does not use the pattern string during any + /// correct translation, but is used for error reporting. + pub fn translate(&mut self, pattern: &str, ast: &Ast) -> Result<Hir> { + ast::visit(ast, TranslatorI::new(self, pattern)) + } +} + +/// An HirFrame is a single stack frame, represented explicitly, which is +/// created for each item in the Ast that we traverse. +/// +/// Note that technically, this type doesn't represent our entire stack +/// frame. In particular, the Ast visitor represents any state associated with +/// traversing the Ast itself. +#[derive(Clone, Debug)] +enum HirFrame { + /// An arbitrary HIR expression. These get pushed whenever we hit a base + /// case in the Ast. They get popped after an inductive (i.e., recursive) + /// step is complete. + Expr(Hir), + /// A literal that is being constructed, character by character, from the + /// AST. We need this because the AST gives each individual character its + /// own node. So as we see characters, we peek at the top-most HirFrame. + /// If it's a literal, then we add to it. Otherwise, we push a new literal. + /// When it comes time to pop it, we convert it to an Hir via Hir::literal. + Literal(Vec<u8>), + /// A Unicode character class. This frame is mutated as we descend into + /// the Ast of a character class (which is itself its own mini recursive + /// structure). + ClassUnicode(hir::ClassUnicode), + /// A byte-oriented character class. This frame is mutated as we descend + /// into the Ast of a character class (which is itself its own mini + /// recursive structure). + /// + /// Byte character classes are created when Unicode mode (`u`) is disabled. + /// If `utf8` is enabled (the default), then a byte character is only + /// permitted to match ASCII text. + ClassBytes(hir::ClassBytes), + /// This is pushed whenever a repetition is observed. After visiting every + /// sub-expression in the repetition, the translator's stack is expected to + /// have this sentinel at the top. + /// + /// This sentinel only exists to stop other things (like flattening + /// literals) from reaching across repetition operators. + Repetition, + /// This is pushed on to the stack upon first seeing any kind of capture, + /// indicated by parentheses (including non-capturing groups). It is popped + /// upon leaving a group. + Group { + /// The old active flags when this group was opened. + /// + /// If this group sets flags, then the new active flags are set to the + /// result of merging the old flags with the flags introduced by this + /// group. If the group doesn't set any flags, then this is simply + /// equivalent to whatever flags were set when the group was opened. + /// + /// When this group is popped, the active flags should be restored to + /// the flags set here. + /// + /// The "active" flags correspond to whatever flags are set in the + /// Translator. + old_flags: Flags, + }, + /// This is pushed whenever a concatenation is observed. After visiting + /// every sub-expression in the concatenation, the translator's stack is + /// popped until it sees a Concat frame. + Concat, + /// This is pushed whenever an alternation is observed. After visiting + /// every sub-expression in the alternation, the translator's stack is + /// popped until it sees an Alternation frame. + Alternation, + /// This is pushed immediately before each sub-expression in an + /// alternation. This separates the branches of an alternation on the + /// stack and prevents literal flattening from reaching across alternation + /// branches. + /// + /// It is popped after each expression in a branch until an 'Alternation' + /// frame is observed when doing a post visit on an alternation. + AlternationBranch, +} + +impl HirFrame { + /// Assert that the current stack frame is an Hir expression and return it. + fn unwrap_expr(self) -> Hir { + match self { + HirFrame::Expr(expr) => expr, + HirFrame::Literal(lit) => Hir::literal(lit), + _ => panic!("tried to unwrap expr from HirFrame, got: {self:?}"), + } + } + + /// Assert that the current stack frame is a Unicode class expression and + /// return it. + fn unwrap_class_unicode(self) -> hir::ClassUnicode { + match self { + HirFrame::ClassUnicode(cls) => cls, + _ => panic!( + "tried to unwrap Unicode class \ + from HirFrame, got: {:?}", + self + ), + } + } + + /// Assert that the current stack frame is a byte class expression and + /// return it. + fn unwrap_class_bytes(self) -> hir::ClassBytes { + match self { + HirFrame::ClassBytes(cls) => cls, + _ => panic!( + "tried to unwrap byte class \ + from HirFrame, got: {:?}", + self + ), + } + } + + /// Assert that the current stack frame is a repetition sentinel. If it + /// isn't, then panic. + fn unwrap_repetition(self) { + match self { + HirFrame::Repetition => {} + _ => { + panic!( + "tried to unwrap repetition from HirFrame, got: {self:?}" + ) + } + } + } + + /// Assert that the current stack frame is a group indicator and return + /// its corresponding flags (the flags that were active at the time the + /// group was entered). + fn unwrap_group(self) -> Flags { + match self { + HirFrame::Group { old_flags } => old_flags, + _ => { + panic!("tried to unwrap group from HirFrame, got: {self:?}") + } + } + } + + /// Assert that the current stack frame is an alternation pipe sentinel. If + /// it isn't, then panic. + fn unwrap_alternation_pipe(self) { + match self { + HirFrame::AlternationBranch => {} + _ => { + panic!("tried to unwrap alt pipe from HirFrame, got: {self:?}") + } + } + } +} + +impl<'t, 'p> Visitor for TranslatorI<'t, 'p> { + type Output = Hir; + type Err = Error; + + fn finish(self) -> Result<Hir> { + // ... otherwise, we should have exactly one HIR on the stack. + assert_eq!(self.trans().stack.borrow().len(), 1); + Ok(self.pop().unwrap().unwrap_expr()) + } + + fn visit_pre(&mut self, ast: &Ast) -> Result<()> { + match *ast { + Ast::ClassBracketed(_) => { + if self.flags().unicode() { + let cls = hir::ClassUnicode::empty(); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let cls = hir::ClassBytes::empty(); + self.push(HirFrame::ClassBytes(cls)); + } + } + Ast::Repetition(_) => self.push(HirFrame::Repetition), + Ast::Group(ref x) => { + let old_flags = x + .flags() + .map(|ast| self.set_flags(ast)) + .unwrap_or_else(|| self.flags()); + self.push(HirFrame::Group { old_flags }); + } + Ast::Concat(_) => { + self.push(HirFrame::Concat); + } + Ast::Alternation(ref x) => { + self.push(HirFrame::Alternation); + if !x.asts.is_empty() { + self.push(HirFrame::AlternationBranch); + } + } + _ => {} + } + Ok(()) + } + + fn visit_post(&mut self, ast: &Ast) -> Result<()> { + match *ast { + Ast::Empty(_) => { + self.push(HirFrame::Expr(Hir::empty())); + } + Ast::Flags(ref x) => { + self.set_flags(&x.flags); + // Flags in the AST are generally considered directives and + // not actual sub-expressions. However, they can be used in + // the concrete syntax like `((?i))`, and we need some kind of + // indication of an expression there, and Empty is the correct + // choice. + // + // There can also be things like `(?i)+`, but we rule those out + // in the parser. In the future, we might allow them for + // consistency sake. + self.push(HirFrame::Expr(Hir::empty())); + } + Ast::Literal(ref x) => match self.ast_literal_to_scalar(x)? { + Either::Right(byte) => self.push_byte(byte), + Either::Left(ch) => match self.case_fold_char(x.span, ch)? { + None => self.push_char(ch), + Some(expr) => self.push(HirFrame::Expr(expr)), + }, + }, + Ast::Dot(ref span) => { + self.push(HirFrame::Expr(self.hir_dot(**span)?)); + } + Ast::Assertion(ref x) => { + self.push(HirFrame::Expr(self.hir_assertion(x)?)); + } + Ast::ClassPerl(ref x) => { + if self.flags().unicode() { + let cls = self.hir_perl_unicode_class(x)?; + let hcls = hir::Class::Unicode(cls); + self.push(HirFrame::Expr(Hir::class(hcls))); + } else { + let cls = self.hir_perl_byte_class(x)?; + let hcls = hir::Class::Bytes(cls); + self.push(HirFrame::Expr(Hir::class(hcls))); + } + } + Ast::ClassUnicode(ref x) => { + let cls = hir::Class::Unicode(self.hir_unicode_class(x)?); + self.push(HirFrame::Expr(Hir::class(cls))); + } + Ast::ClassBracketed(ref ast) => { + if self.flags().unicode() { + let mut cls = self.pop().unwrap().unwrap_class_unicode(); + self.unicode_fold_and_negate( + &ast.span, + ast.negated, + &mut cls, + )?; + let expr = Hir::class(hir::Class::Unicode(cls)); + self.push(HirFrame::Expr(expr)); + } else { + let mut cls = self.pop().unwrap().unwrap_class_bytes(); + self.bytes_fold_and_negate( + &ast.span, + ast.negated, + &mut cls, + )?; + let expr = Hir::class(hir::Class::Bytes(cls)); + self.push(HirFrame::Expr(expr)); + } + } + Ast::Repetition(ref x) => { + let expr = self.pop().unwrap().unwrap_expr(); + self.pop().unwrap().unwrap_repetition(); + self.push(HirFrame::Expr(self.hir_repetition(x, expr))); + } + Ast::Group(ref x) => { + let expr = self.pop().unwrap().unwrap_expr(); + let old_flags = self.pop().unwrap().unwrap_group(); + self.trans().flags.set(old_flags); + self.push(HirFrame::Expr(self.hir_capture(x, expr))); + } + Ast::Concat(_) => { + let mut exprs = vec![]; + while let Some(expr) = self.pop_concat_expr() { + if !matches!(*expr.kind(), HirKind::Empty) { + exprs.push(expr); + } + } + exprs.reverse(); + self.push(HirFrame::Expr(Hir::concat(exprs))); + } + Ast::Alternation(_) => { + let mut exprs = vec![]; + while let Some(expr) = self.pop_alt_expr() { + self.pop().unwrap().unwrap_alternation_pipe(); + exprs.push(expr); + } + exprs.reverse(); + self.push(HirFrame::Expr(Hir::alternation(exprs))); + } + } + Ok(()) + } + + fn visit_alternation_in(&mut self) -> Result<()> { + self.push(HirFrame::AlternationBranch); + Ok(()) + } + + fn visit_class_set_item_pre( + &mut self, + ast: &ast::ClassSetItem, + ) -> Result<()> { + match *ast { + ast::ClassSetItem::Bracketed(_) => { + if self.flags().unicode() { + let cls = hir::ClassUnicode::empty(); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let cls = hir::ClassBytes::empty(); + self.push(HirFrame::ClassBytes(cls)); + } + } + // We needn't handle the Union case here since the visitor will + // do it for us. + _ => {} + } + Ok(()) + } + + fn visit_class_set_item_post( + &mut self, + ast: &ast::ClassSetItem, + ) -> Result<()> { + match *ast { + ast::ClassSetItem::Empty(_) => {} + ast::ClassSetItem::Literal(ref x) => { + if self.flags().unicode() { + let mut cls = self.pop().unwrap().unwrap_class_unicode(); + cls.push(hir::ClassUnicodeRange::new(x.c, x.c)); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let mut cls = self.pop().unwrap().unwrap_class_bytes(); + let byte = self.class_literal_byte(x)?; + cls.push(hir::ClassBytesRange::new(byte, byte)); + self.push(HirFrame::ClassBytes(cls)); + } + } + ast::ClassSetItem::Range(ref x) => { + if self.flags().unicode() { + let mut cls = self.pop().unwrap().unwrap_class_unicode(); + cls.push(hir::ClassUnicodeRange::new(x.start.c, x.end.c)); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let mut cls = self.pop().unwrap().unwrap_class_bytes(); + let start = self.class_literal_byte(&x.start)?; + let end = self.class_literal_byte(&x.end)?; + cls.push(hir::ClassBytesRange::new(start, end)); + self.push(HirFrame::ClassBytes(cls)); + } + } + ast::ClassSetItem::Ascii(ref x) => { + if self.flags().unicode() { + let xcls = self.hir_ascii_unicode_class(x)?; + let mut cls = self.pop().unwrap().unwrap_class_unicode(); + cls.union(&xcls); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let xcls = self.hir_ascii_byte_class(x)?; + let mut cls = self.pop().unwrap().unwrap_class_bytes(); + cls.union(&xcls); + self.push(HirFrame::ClassBytes(cls)); + } + } + ast::ClassSetItem::Unicode(ref x) => { + let xcls = self.hir_unicode_class(x)?; + let mut cls = self.pop().unwrap().unwrap_class_unicode(); + cls.union(&xcls); + self.push(HirFrame::ClassUnicode(cls)); + } + ast::ClassSetItem::Perl(ref x) => { + if self.flags().unicode() { + let xcls = self.hir_perl_unicode_class(x)?; + let mut cls = self.pop().unwrap().unwrap_class_unicode(); + cls.union(&xcls); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let xcls = self.hir_perl_byte_class(x)?; + let mut cls = self.pop().unwrap().unwrap_class_bytes(); + cls.union(&xcls); + self.push(HirFrame::ClassBytes(cls)); + } + } + ast::ClassSetItem::Bracketed(ref ast) => { + if self.flags().unicode() { + let mut cls1 = self.pop().unwrap().unwrap_class_unicode(); + self.unicode_fold_and_negate( + &ast.span, + ast.negated, + &mut cls1, + )?; + + let mut cls2 = self.pop().unwrap().unwrap_class_unicode(); + cls2.union(&cls1); + self.push(HirFrame::ClassUnicode(cls2)); + } else { + let mut cls1 = self.pop().unwrap().unwrap_class_bytes(); + self.bytes_fold_and_negate( + &ast.span, + ast.negated, + &mut cls1, + )?; + + let mut cls2 = self.pop().unwrap().unwrap_class_bytes(); + cls2.union(&cls1); + self.push(HirFrame::ClassBytes(cls2)); + } + } + // This is handled automatically by the visitor. + ast::ClassSetItem::Union(_) => {} + } + Ok(()) + } + + fn visit_class_set_binary_op_pre( + &mut self, + _op: &ast::ClassSetBinaryOp, + ) -> Result<()> { + if self.flags().unicode() { + let cls = hir::ClassUnicode::empty(); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let cls = hir::ClassBytes::empty(); + self.push(HirFrame::ClassBytes(cls)); + } + Ok(()) + } + + fn visit_class_set_binary_op_in( + &mut self, + _op: &ast::ClassSetBinaryOp, + ) -> Result<()> { + if self.flags().unicode() { + let cls = hir::ClassUnicode::empty(); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let cls = hir::ClassBytes::empty(); + self.push(HirFrame::ClassBytes(cls)); + } + Ok(()) + } + + fn visit_class_set_binary_op_post( + &mut self, + op: &ast::ClassSetBinaryOp, + ) -> Result<()> { + use crate::ast::ClassSetBinaryOpKind::*; + + if self.flags().unicode() { + let mut rhs = self.pop().unwrap().unwrap_class_unicode(); + let mut lhs = self.pop().unwrap().unwrap_class_unicode(); + let mut cls = self.pop().unwrap().unwrap_class_unicode(); + if self.flags().case_insensitive() { + rhs.try_case_fold_simple().map_err(|_| { + self.error( + op.rhs.span().clone(), + ErrorKind::UnicodeCaseUnavailable, + ) + })?; + lhs.try_case_fold_simple().map_err(|_| { + self.error( + op.lhs.span().clone(), + ErrorKind::UnicodeCaseUnavailable, + ) + })?; + } + match op.kind { + Intersection => lhs.intersect(&rhs), + Difference => lhs.difference(&rhs), + SymmetricDifference => lhs.symmetric_difference(&rhs), + } + cls.union(&lhs); + self.push(HirFrame::ClassUnicode(cls)); + } else { + let mut rhs = self.pop().unwrap().unwrap_class_bytes(); + let mut lhs = self.pop().unwrap().unwrap_class_bytes(); + let mut cls = self.pop().unwrap().unwrap_class_bytes(); + if self.flags().case_insensitive() { + rhs.case_fold_simple(); + lhs.case_fold_simple(); + } + match op.kind { + Intersection => lhs.intersect(&rhs), + Difference => lhs.difference(&rhs), + SymmetricDifference => lhs.symmetric_difference(&rhs), + } + cls.union(&lhs); + self.push(HirFrame::ClassBytes(cls)); + } + Ok(()) + } +} + +/// The internal implementation of a translator. +/// +/// This type is responsible for carrying around the original pattern string, +/// which is not tied to the internal state of a translator. +/// +/// A TranslatorI exists for the time it takes to translate a single Ast. +#[derive(Clone, Debug)] +struct TranslatorI<'t, 'p> { + trans: &'t Translator, + pattern: &'p str, +} + +impl<'t, 'p> TranslatorI<'t, 'p> { + /// Build a new internal translator. + fn new(trans: &'t Translator, pattern: &'p str) -> TranslatorI<'t, 'p> { + TranslatorI { trans, pattern } + } + + /// Return a reference to the underlying translator. + fn trans(&self) -> &Translator { + &self.trans + } + + /// Push the given frame on to the call stack. + fn push(&self, frame: HirFrame) { + self.trans().stack.borrow_mut().push(frame); + } + + /// Push the given literal char on to the call stack. + /// + /// If the top-most element of the stack is a literal, then the char + /// is appended to the end of that literal. Otherwise, a new literal + /// containing just the given char is pushed to the top of the stack. + fn push_char(&self, ch: char) { + let mut buf = [0; 4]; + let bytes = ch.encode_utf8(&mut buf).as_bytes(); + let mut stack = self.trans().stack.borrow_mut(); + if let Some(HirFrame::Literal(ref mut literal)) = stack.last_mut() { + literal.extend_from_slice(bytes); + } else { + stack.push(HirFrame::Literal(bytes.to_vec())); + } + } + + /// Push the given literal byte on to the call stack. + /// + /// If the top-most element of the stack is a literal, then the byte + /// is appended to the end of that literal. Otherwise, a new literal + /// containing just the given byte is pushed to the top of the stack. + fn push_byte(&self, byte: u8) { + let mut stack = self.trans().stack.borrow_mut(); + if let Some(HirFrame::Literal(ref mut literal)) = stack.last_mut() { + literal.push(byte); + } else { + stack.push(HirFrame::Literal(vec![byte])); + } + } + + /// Pop the top of the call stack. If the call stack is empty, return None. + fn pop(&self) -> Option<HirFrame> { + self.trans().stack.borrow_mut().pop() + } + + /// Pop an HIR expression from the top of the stack for a concatenation. + /// + /// This returns None if the stack is empty or when a concat frame is seen. + /// Otherwise, it panics if it could not find an HIR expression. + fn pop_concat_expr(&self) -> Option<Hir> { + let frame = self.pop()?; + match frame { + HirFrame::Concat => None, + HirFrame::Expr(expr) => Some(expr), + HirFrame::Literal(lit) => Some(Hir::literal(lit)), + HirFrame::ClassUnicode(_) => { + unreachable!("expected expr or concat, got Unicode class") + } + HirFrame::ClassBytes(_) => { + unreachable!("expected expr or concat, got byte class") + } + HirFrame::Repetition => { + unreachable!("expected expr or concat, got repetition") + } + HirFrame::Group { .. } => { + unreachable!("expected expr or concat, got group") + } + HirFrame::Alternation => { + unreachable!("expected expr or concat, got alt marker") + } + HirFrame::AlternationBranch => { + unreachable!("expected expr or concat, got alt branch marker") + } + } + } + + /// Pop an HIR expression from the top of the stack for an alternation. + /// + /// This returns None if the stack is empty or when an alternation frame is + /// seen. Otherwise, it panics if it could not find an HIR expression. + fn pop_alt_expr(&self) -> Option<Hir> { + let frame = self.pop()?; + match frame { + HirFrame::Alternation => None, + HirFrame::Expr(expr) => Some(expr), + HirFrame::Literal(lit) => Some(Hir::literal(lit)), + HirFrame::ClassUnicode(_) => { + unreachable!("expected expr or alt, got Unicode class") + } + HirFrame::ClassBytes(_) => { + unreachable!("expected expr or alt, got byte class") + } + HirFrame::Repetition => { + unreachable!("expected expr or alt, got repetition") + } + HirFrame::Group { .. } => { + unreachable!("expected expr or alt, got group") + } + HirFrame::Concat => { + unreachable!("expected expr or alt, got concat marker") + } + HirFrame::AlternationBranch => { + unreachable!("expected expr or alt, got alt branch marker") + } + } + } + + /// Create a new error with the given span and error type. + fn error(&self, span: Span, kind: ErrorKind) -> Error { + Error { kind, pattern: self.pattern.to_string(), span } + } + + /// Return a copy of the active flags. + fn flags(&self) -> Flags { + self.trans().flags.get() + } + + /// Set the flags of this translator from the flags set in the given AST. + /// Then, return the old flags. + fn set_flags(&self, ast_flags: &ast::Flags) -> Flags { + let old_flags = self.flags(); + let mut new_flags = Flags::from_ast(ast_flags); + new_flags.merge(&old_flags); + self.trans().flags.set(new_flags); + old_flags + } + + /// Convert an Ast literal to its scalar representation. + /// + /// When Unicode mode is enabled, then this always succeeds and returns a + /// `char` (Unicode scalar value). + /// + /// When Unicode mode is disabled, then a `char` will still be returned + /// whenever possible. A byte is returned only when invalid UTF-8 is + /// allowed and when the byte is not ASCII. Otherwise, a non-ASCII byte + /// will result in an error when invalid UTF-8 is not allowed. + fn ast_literal_to_scalar( + &self, + lit: &ast::Literal, + ) -> Result<Either<char, u8>> { + if self.flags().unicode() { + return Ok(Either::Left(lit.c)); + } + let byte = match lit.byte() { + None => return Ok(Either::Left(lit.c)), + Some(byte) => byte, + }; + if byte <= 0x7F { + return Ok(Either::Left(char::try_from(byte).unwrap())); + } + if self.trans().utf8 { + return Err(self.error(lit.span, ErrorKind::InvalidUtf8)); + } + Ok(Either::Right(byte)) + } + + fn case_fold_char(&self, span: Span, c: char) -> Result<Option<Hir>> { + if !self.flags().case_insensitive() { + return Ok(None); + } + if self.flags().unicode() { + // If case folding won't do anything, then don't bother trying. + let map = unicode::SimpleCaseFolder::new() + .map(|f| f.overlaps(c, c)) + .map_err(|_| { + self.error(span, ErrorKind::UnicodeCaseUnavailable) + })?; + if !map { + return Ok(None); + } + let mut cls = + hir::ClassUnicode::new(vec![hir::ClassUnicodeRange::new( + c, c, + )]); + cls.try_case_fold_simple().map_err(|_| { + self.error(span, ErrorKind::UnicodeCaseUnavailable) + })?; + Ok(Some(Hir::class(hir::Class::Unicode(cls)))) + } else { + if !c.is_ascii() { + return Ok(None); + } + // If case folding won't do anything, then don't bother trying. + match c { + 'A'..='Z' | 'a'..='z' => {} + _ => return Ok(None), + } + let mut cls = + hir::ClassBytes::new(vec![hir::ClassBytesRange::new( + // OK because 'c.len_utf8() == 1' which in turn implies + // that 'c' is ASCII. + u8::try_from(c).unwrap(), + u8::try_from(c).unwrap(), + )]); + cls.case_fold_simple(); + Ok(Some(Hir::class(hir::Class::Bytes(cls)))) + } + } + + fn hir_dot(&self, span: Span) -> Result<Hir> { + let (utf8, lineterm, flags) = + (self.trans().utf8, self.trans().line_terminator, self.flags()); + if utf8 && (!flags.unicode() || !lineterm.is_ascii()) { + return Err(self.error(span, ErrorKind::InvalidUtf8)); + } + let dot = if flags.dot_matches_new_line() { + if flags.unicode() { + hir::Dot::AnyChar + } else { + hir::Dot::AnyByte + } + } else { + if flags.unicode() { + if flags.crlf() { + hir::Dot::AnyCharExceptCRLF + } else { + if !lineterm.is_ascii() { + return Err( + self.error(span, ErrorKind::InvalidLineTerminator) + ); + } + hir::Dot::AnyCharExcept(char::from(lineterm)) + } + } else { + if flags.crlf() { + hir::Dot::AnyByteExceptCRLF + } else { + hir::Dot::AnyByteExcept(lineterm) + } + } + }; + Ok(Hir::dot(dot)) + } + + fn hir_assertion(&self, asst: &ast::Assertion) -> Result<Hir> { + let unicode = self.flags().unicode(); + let multi_line = self.flags().multi_line(); + let crlf = self.flags().crlf(); + Ok(match asst.kind { + ast::AssertionKind::StartLine => Hir::look(if multi_line { + if crlf { + hir::Look::StartCRLF + } else { + hir::Look::StartLF + } + } else { + hir::Look::Start + }), + ast::AssertionKind::EndLine => Hir::look(if multi_line { + if crlf { + hir::Look::EndCRLF + } else { + hir::Look::EndLF + } + } else { + hir::Look::End + }), + ast::AssertionKind::StartText => Hir::look(hir::Look::Start), + ast::AssertionKind::EndText => Hir::look(hir::Look::End), + ast::AssertionKind::WordBoundary => Hir::look(if unicode { + hir::Look::WordUnicode + } else { + hir::Look::WordAscii + }), + ast::AssertionKind::NotWordBoundary => Hir::look(if unicode { + hir::Look::WordUnicodeNegate + } else { + hir::Look::WordAsciiNegate + }), + ast::AssertionKind::WordBoundaryStart + | ast::AssertionKind::WordBoundaryStartAngle => { + Hir::look(if unicode { + hir::Look::WordStartUnicode + } else { + hir::Look::WordStartAscii + }) + } + ast::AssertionKind::WordBoundaryEnd + | ast::AssertionKind::WordBoundaryEndAngle => { + Hir::look(if unicode { + hir::Look::WordEndUnicode + } else { + hir::Look::WordEndAscii + }) + } + ast::AssertionKind::WordBoundaryStartHalf => { + Hir::look(if unicode { + hir::Look::WordStartHalfUnicode + } else { + hir::Look::WordStartHalfAscii + }) + } + ast::AssertionKind::WordBoundaryEndHalf => Hir::look(if unicode { + hir::Look::WordEndHalfUnicode + } else { + hir::Look::WordEndHalfAscii + }), + }) + } + + fn hir_capture(&self, group: &ast::Group, expr: Hir) -> Hir { + let (index, name) = match group.kind { + ast::GroupKind::CaptureIndex(index) => (index, None), + ast::GroupKind::CaptureName { ref name, .. } => { + (name.index, Some(name.name.clone().into_boxed_str())) + } + // The HIR doesn't need to use non-capturing groups, since the way + // in which the data type is defined handles this automatically. + ast::GroupKind::NonCapturing(_) => return expr, + }; + Hir::capture(hir::Capture { index, name, sub: Box::new(expr) }) + } + + fn hir_repetition(&self, rep: &ast::Repetition, expr: Hir) -> Hir { + let (min, max) = match rep.op.kind { + ast::RepetitionKind::ZeroOrOne => (0, Some(1)), + ast::RepetitionKind::ZeroOrMore => (0, None), + ast::RepetitionKind::OneOrMore => (1, None), + ast::RepetitionKind::Range(ast::RepetitionRange::Exactly(m)) => { + (m, Some(m)) + } + ast::RepetitionKind::Range(ast::RepetitionRange::AtLeast(m)) => { + (m, None) + } + ast::RepetitionKind::Range(ast::RepetitionRange::Bounded( + m, + n, + )) => (m, Some(n)), + }; + let greedy = + if self.flags().swap_greed() { !rep.greedy } else { rep.greedy }; + Hir::repetition(hir::Repetition { + min, + max, + greedy, + sub: Box::new(expr), + }) + } + + fn hir_unicode_class( + &self, + ast_class: &ast::ClassUnicode, + ) -> Result<hir::ClassUnicode> { + use crate::ast::ClassUnicodeKind::*; + + if !self.flags().unicode() { + return Err( + self.error(ast_class.span, ErrorKind::UnicodeNotAllowed) + ); + } + let query = match ast_class.kind { + OneLetter(name) => ClassQuery::OneLetter(name), + Named(ref name) => ClassQuery::Binary(name), + NamedValue { ref name, ref value, .. } => ClassQuery::ByValue { + property_name: name, + property_value: value, + }, + }; + let mut result = self.convert_unicode_class_error( + &ast_class.span, + unicode::class(query), + ); + if let Ok(ref mut class) = result { + self.unicode_fold_and_negate( + &ast_class.span, + ast_class.is_negated(), + class, + )?; + } + result + } + + fn hir_ascii_unicode_class( + &self, + ast: &ast::ClassAscii, + ) -> Result<hir::ClassUnicode> { + let mut cls = hir::ClassUnicode::new( + ascii_class_as_chars(&ast.kind) + .map(|(s, e)| hir::ClassUnicodeRange::new(s, e)), + ); + self.unicode_fold_and_negate(&ast.span, ast.negated, &mut cls)?; + Ok(cls) + } + + fn hir_ascii_byte_class( + &self, + ast: &ast::ClassAscii, + ) -> Result<hir::ClassBytes> { + let mut cls = hir::ClassBytes::new( + ascii_class(&ast.kind) + .map(|(s, e)| hir::ClassBytesRange::new(s, e)), + ); + self.bytes_fold_and_negate(&ast.span, ast.negated, &mut cls)?; + Ok(cls) + } + + fn hir_perl_unicode_class( + &self, + ast_class: &ast::ClassPerl, + ) -> Result<hir::ClassUnicode> { + use crate::ast::ClassPerlKind::*; + + assert!(self.flags().unicode()); + let result = match ast_class.kind { + Digit => unicode::perl_digit(), + Space => unicode::perl_space(), + Word => unicode::perl_word(), + }; + let mut class = + self.convert_unicode_class_error(&ast_class.span, result)?; + // We needn't apply case folding here because the Perl Unicode classes + // are already closed under Unicode simple case folding. + if ast_class.negated { + class.negate(); + } + Ok(class) + } + + fn hir_perl_byte_class( + &self, + ast_class: &ast::ClassPerl, + ) -> Result<hir::ClassBytes> { + use crate::ast::ClassPerlKind::*; + + assert!(!self.flags().unicode()); + let mut class = match ast_class.kind { + Digit => hir_ascii_class_bytes(&ast::ClassAsciiKind::Digit), + Space => hir_ascii_class_bytes(&ast::ClassAsciiKind::Space), + Word => hir_ascii_class_bytes(&ast::ClassAsciiKind::Word), + }; + // We needn't apply case folding here because the Perl ASCII classes + // are already closed (under ASCII case folding). + if ast_class.negated { + class.negate(); + } + // Negating a Perl byte class is likely to cause it to match invalid + // UTF-8. That's only OK if the translator is configured to allow such + // things. + if self.trans().utf8 && !class.is_ascii() { + return Err(self.error(ast_class.span, ErrorKind::InvalidUtf8)); + } + Ok(class) + } + + /// Converts the given Unicode specific error to an HIR translation error. + /// + /// The span given should approximate the position at which an error would + /// occur. + fn convert_unicode_class_error( + &self, + span: &Span, + result: core::result::Result<hir::ClassUnicode, unicode::Error>, + ) -> Result<hir::ClassUnicode> { + result.map_err(|err| { + let sp = span.clone(); + match err { + unicode::Error::PropertyNotFound => { + self.error(sp, ErrorKind::UnicodePropertyNotFound) + } + unicode::Error::PropertyValueNotFound => { + self.error(sp, ErrorKind::UnicodePropertyValueNotFound) + } + unicode::Error::PerlClassNotFound => { + self.error(sp, ErrorKind::UnicodePerlClassNotFound) + } + } + }) + } + + fn unicode_fold_and_negate( + &self, + span: &Span, + negated: bool, + class: &mut hir::ClassUnicode, + ) -> Result<()> { + // Note that we must apply case folding before negation! + // Consider `(?i)[^x]`. If we applied negation first, then + // the result would be the character class that matched any + // Unicode scalar value. + if self.flags().case_insensitive() { + class.try_case_fold_simple().map_err(|_| { + self.error(span.clone(), ErrorKind::UnicodeCaseUnavailable) + })?; + } + if negated { + class.negate(); + } + Ok(()) + } + + fn bytes_fold_and_negate( + &self, + span: &Span, + negated: bool, + class: &mut hir::ClassBytes, + ) -> Result<()> { + // Note that we must apply case folding before negation! + // Consider `(?i)[^x]`. If we applied negation first, then + // the result would be the character class that matched any + // Unicode scalar value. + if self.flags().case_insensitive() { + class.case_fold_simple(); + } + if negated { + class.negate(); + } + if self.trans().utf8 && !class.is_ascii() { + return Err(self.error(span.clone(), ErrorKind::InvalidUtf8)); + } + Ok(()) + } + + /// Return a scalar byte value suitable for use as a literal in a byte + /// character class. + fn class_literal_byte(&self, ast: &ast::Literal) -> Result<u8> { + match self.ast_literal_to_scalar(ast)? { + Either::Right(byte) => Ok(byte), + Either::Left(ch) => { + if ch.is_ascii() { + Ok(u8::try_from(ch).unwrap()) + } else { + // We can't feasibly support Unicode in + // byte oriented classes. Byte classes don't + // do Unicode case folding. + Err(self.error(ast.span, ErrorKind::UnicodeNotAllowed)) + } + } + } + } +} + +/// A translator's representation of a regular expression's flags at any given +/// moment in time. +/// +/// Each flag can be in one of three states: absent, present but disabled or +/// present but enabled. +#[derive(Clone, Copy, Debug, Default)] +struct Flags { + case_insensitive: Option<bool>, + multi_line: Option<bool>, + dot_matches_new_line: Option<bool>, + swap_greed: Option<bool>, + unicode: Option<bool>, + crlf: Option<bool>, + // Note that `ignore_whitespace` is omitted here because it is handled + // entirely in the parser. +} + +impl Flags { + fn from_ast(ast: &ast::Flags) -> Flags { + let mut flags = Flags::default(); + let mut enable = true; + for item in &ast.items { + match item.kind { + ast::FlagsItemKind::Negation => { + enable = false; + } + ast::FlagsItemKind::Flag(ast::Flag::CaseInsensitive) => { + flags.case_insensitive = Some(enable); + } + ast::FlagsItemKind::Flag(ast::Flag::MultiLine) => { + flags.multi_line = Some(enable); + } + ast::FlagsItemKind::Flag(ast::Flag::DotMatchesNewLine) => { + flags.dot_matches_new_line = Some(enable); + } + ast::FlagsItemKind::Flag(ast::Flag::SwapGreed) => { + flags.swap_greed = Some(enable); + } + ast::FlagsItemKind::Flag(ast::Flag::Unicode) => { + flags.unicode = Some(enable); + } + ast::FlagsItemKind::Flag(ast::Flag::CRLF) => { + flags.crlf = Some(enable); + } + ast::FlagsItemKind::Flag(ast::Flag::IgnoreWhitespace) => {} + } + } + flags + } + + fn merge(&mut self, previous: &Flags) { + if self.case_insensitive.is_none() { + self.case_insensitive = previous.case_insensitive; + } + if self.multi_line.is_none() { + self.multi_line = previous.multi_line; + } + if self.dot_matches_new_line.is_none() { + self.dot_matches_new_line = previous.dot_matches_new_line; + } + if self.swap_greed.is_none() { + self.swap_greed = previous.swap_greed; + } + if self.unicode.is_none() { + self.unicode = previous.unicode; + } + if self.crlf.is_none() { + self.crlf = previous.crlf; + } + } + + fn case_insensitive(&self) -> bool { + self.case_insensitive.unwrap_or(false) + } + + fn multi_line(&self) -> bool { + self.multi_line.unwrap_or(false) + } + + fn dot_matches_new_line(&self) -> bool { + self.dot_matches_new_line.unwrap_or(false) + } + + fn swap_greed(&self) -> bool { + self.swap_greed.unwrap_or(false) + } + + fn unicode(&self) -> bool { + self.unicode.unwrap_or(true) + } + + fn crlf(&self) -> bool { + self.crlf.unwrap_or(false) + } +} + +fn hir_ascii_class_bytes(kind: &ast::ClassAsciiKind) -> hir::ClassBytes { + let ranges: Vec<_> = ascii_class(kind) + .map(|(s, e)| hir::ClassBytesRange::new(s, e)) + .collect(); + hir::ClassBytes::new(ranges) +} + +fn ascii_class(kind: &ast::ClassAsciiKind) -> impl Iterator<Item = (u8, u8)> { + use crate::ast::ClassAsciiKind::*; + + let slice: &'static [(u8, u8)] = match *kind { + Alnum => &[(b'0', b'9'), (b'A', b'Z'), (b'a', b'z')], + Alpha => &[(b'A', b'Z'), (b'a', b'z')], + Ascii => &[(b'\x00', b'\x7F')], + Blank => &[(b'\t', b'\t'), (b' ', b' ')], + Cntrl => &[(b'\x00', b'\x1F'), (b'\x7F', b'\x7F')], + Digit => &[(b'0', b'9')], + Graph => &[(b'!', b'~')], + Lower => &[(b'a', b'z')], + Print => &[(b' ', b'~')], + Punct => &[(b'!', b'/'), (b':', b'@'), (b'[', b'`'), (b'{', b'~')], + Space => &[ + (b'\t', b'\t'), + (b'\n', b'\n'), + (b'\x0B', b'\x0B'), + (b'\x0C', b'\x0C'), + (b'\r', b'\r'), + (b' ', b' '), + ], + Upper => &[(b'A', b'Z')], + Word => &[(b'0', b'9'), (b'A', b'Z'), (b'_', b'_'), (b'a', b'z')], + Xdigit => &[(b'0', b'9'), (b'A', b'F'), (b'a', b'f')], + }; + slice.iter().copied() +} + +fn ascii_class_as_chars( + kind: &ast::ClassAsciiKind, +) -> impl Iterator<Item = (char, char)> { + ascii_class(kind).map(|(s, e)| (char::from(s), char::from(e))) +} + +#[cfg(test)] +mod tests { + use crate::{ + ast::{parse::ParserBuilder, Position}, + hir::{Look, Properties}, + }; + + use super::*; + + // We create these errors to compare with real hir::Errors in the tests. + // We define equality between TestError and hir::Error to disregard the + // pattern string in hir::Error, which is annoying to provide in tests. + #[derive(Clone, Debug)] + struct TestError { + span: Span, + kind: hir::ErrorKind, + } + + impl PartialEq<hir::Error> for TestError { + fn eq(&self, other: &hir::Error) -> bool { + self.span == other.span && self.kind == other.kind + } + } + + impl PartialEq<TestError> for hir::Error { + fn eq(&self, other: &TestError) -> bool { + self.span == other.span && self.kind == other.kind + } + } + + fn parse(pattern: &str) -> Ast { + ParserBuilder::new().octal(true).build().parse(pattern).unwrap() + } + + fn t(pattern: &str) -> Hir { + TranslatorBuilder::new() + .utf8(true) + .build() + .translate(pattern, &parse(pattern)) + .unwrap() + } + + fn t_err(pattern: &str) -> hir::Error { + TranslatorBuilder::new() + .utf8(true) + .build() + .translate(pattern, &parse(pattern)) + .unwrap_err() + } + + fn t_bytes(pattern: &str) -> Hir { + TranslatorBuilder::new() + .utf8(false) + .build() + .translate(pattern, &parse(pattern)) + .unwrap() + } + + fn props(pattern: &str) -> Properties { + t(pattern).properties().clone() + } + + fn props_bytes(pattern: &str) -> Properties { + t_bytes(pattern).properties().clone() + } + + fn hir_lit(s: &str) -> Hir { + hir_blit(s.as_bytes()) + } + + fn hir_blit(s: &[u8]) -> Hir { + Hir::literal(s) + } + + fn hir_capture(index: u32, expr: Hir) -> Hir { + Hir::capture(hir::Capture { index, name: None, sub: Box::new(expr) }) + } + + fn hir_capture_name(index: u32, name: &str, expr: Hir) -> Hir { + Hir::capture(hir::Capture { + index, + name: Some(name.into()), + sub: Box::new(expr), + }) + } + + fn hir_quest(greedy: bool, expr: Hir) -> Hir { + Hir::repetition(hir::Repetition { + min: 0, + max: Some(1), + greedy, + sub: Box::new(expr), + }) + } + + fn hir_star(greedy: bool, expr: Hir) -> Hir { + Hir::repetition(hir::Repetition { + min: 0, + max: None, + greedy, + sub: Box::new(expr), + }) + } + + fn hir_plus(greedy: bool, expr: Hir) -> Hir { + Hir::repetition(hir::Repetition { + min: 1, + max: None, + greedy, + sub: Box::new(expr), + }) + } + + fn hir_range(greedy: bool, min: u32, max: Option<u32>, expr: Hir) -> Hir { + Hir::repetition(hir::Repetition { + min, + max, + greedy, + sub: Box::new(expr), + }) + } + + fn hir_alt(alts: Vec<Hir>) -> Hir { + Hir::alternation(alts) + } + + fn hir_cat(exprs: Vec<Hir>) -> Hir { + Hir::concat(exprs) + } + + #[allow(dead_code)] + fn hir_uclass_query(query: ClassQuery<'_>) -> Hir { + Hir::class(hir::Class::Unicode(unicode::class(query).unwrap())) + } + + #[allow(dead_code)] + fn hir_uclass_perl_word() -> Hir { + Hir::class(hir::Class::Unicode(unicode::perl_word().unwrap())) + } + + fn hir_ascii_uclass(kind: &ast::ClassAsciiKind) -> Hir { + Hir::class(hir::Class::Unicode(hir::ClassUnicode::new( + ascii_class_as_chars(kind) + .map(|(s, e)| hir::ClassUnicodeRange::new(s, e)), + ))) + } + + fn hir_ascii_bclass(kind: &ast::ClassAsciiKind) -> Hir { + Hir::class(hir::Class::Bytes(hir::ClassBytes::new( + ascii_class(kind).map(|(s, e)| hir::ClassBytesRange::new(s, e)), + ))) + } + + fn hir_uclass(ranges: &[(char, char)]) -> Hir { + Hir::class(uclass(ranges)) + } + + fn hir_bclass(ranges: &[(u8, u8)]) -> Hir { + Hir::class(bclass(ranges)) + } + + fn hir_case_fold(expr: Hir) -> Hir { + match expr.into_kind() { + HirKind::Class(mut cls) => { + cls.case_fold_simple(); + Hir::class(cls) + } + _ => panic!("cannot case fold non-class Hir expr"), + } + } + + fn hir_negate(expr: Hir) -> Hir { + match expr.into_kind() { + HirKind::Class(mut cls) => { + cls.negate(); + Hir::class(cls) + } + _ => panic!("cannot negate non-class Hir expr"), + } + } + + fn uclass(ranges: &[(char, char)]) -> hir::Class { + let ranges: Vec<hir::ClassUnicodeRange> = ranges + .iter() + .map(|&(s, e)| hir::ClassUnicodeRange::new(s, e)) + .collect(); + hir::Class::Unicode(hir::ClassUnicode::new(ranges)) + } + + fn bclass(ranges: &[(u8, u8)]) -> hir::Class { + let ranges: Vec<hir::ClassBytesRange> = ranges + .iter() + .map(|&(s, e)| hir::ClassBytesRange::new(s, e)) + .collect(); + hir::Class::Bytes(hir::ClassBytes::new(ranges)) + } + + #[cfg(feature = "unicode-case")] + fn class_case_fold(mut cls: hir::Class) -> Hir { + cls.case_fold_simple(); + Hir::class(cls) + } + + fn class_negate(mut cls: hir::Class) -> Hir { + cls.negate(); + Hir::class(cls) + } + + #[allow(dead_code)] + fn hir_union(expr1: Hir, expr2: Hir) -> Hir { + use crate::hir::Class::{Bytes, Unicode}; + + match (expr1.into_kind(), expr2.into_kind()) { + (HirKind::Class(Unicode(mut c1)), HirKind::Class(Unicode(c2))) => { + c1.union(&c2); + Hir::class(hir::Class::Unicode(c1)) + } + (HirKind::Class(Bytes(mut c1)), HirKind::Class(Bytes(c2))) => { + c1.union(&c2); + Hir::class(hir::Class::Bytes(c1)) + } + _ => panic!("cannot union non-class Hir exprs"), + } + } + + #[allow(dead_code)] + fn hir_difference(expr1: Hir, expr2: Hir) -> Hir { + use crate::hir::Class::{Bytes, Unicode}; + + match (expr1.into_kind(), expr2.into_kind()) { + (HirKind::Class(Unicode(mut c1)), HirKind::Class(Unicode(c2))) => { + c1.difference(&c2); + Hir::class(hir::Class::Unicode(c1)) + } + (HirKind::Class(Bytes(mut c1)), HirKind::Class(Bytes(c2))) => { + c1.difference(&c2); + Hir::class(hir::Class::Bytes(c1)) + } + _ => panic!("cannot difference non-class Hir exprs"), + } + } + + fn hir_look(look: hir::Look) -> Hir { + Hir::look(look) + } + + #[test] + fn empty() { + assert_eq!(t(""), Hir::empty()); + assert_eq!(t("(?i)"), Hir::empty()); + assert_eq!(t("()"), hir_capture(1, Hir::empty())); + assert_eq!(t("(?:)"), Hir::empty()); + assert_eq!(t("(?P<wat>)"), hir_capture_name(1, "wat", Hir::empty())); + assert_eq!(t("|"), hir_alt(vec![Hir::empty(), Hir::empty()])); + assert_eq!( + t("()|()"), + hir_alt(vec![ + hir_capture(1, Hir::empty()), + hir_capture(2, Hir::empty()), + ]) + ); + assert_eq!( + t("(|b)"), + hir_capture(1, hir_alt(vec![Hir::empty(), hir_lit("b"),])) + ); + assert_eq!( + t("(a|)"), + hir_capture(1, hir_alt(vec![hir_lit("a"), Hir::empty(),])) + ); + assert_eq!( + t("(a||c)"), + hir_capture( + 1, + hir_alt(vec![hir_lit("a"), Hir::empty(), hir_lit("c"),]) + ) + ); + assert_eq!( + t("(||)"), + hir_capture( + 1, + hir_alt(vec![Hir::empty(), Hir::empty(), Hir::empty(),]) + ) + ); + } + + #[test] + fn literal() { + assert_eq!(t("a"), hir_lit("a")); + assert_eq!(t("(?-u)a"), hir_lit("a")); + assert_eq!(t("☃"), hir_lit("☃")); + assert_eq!(t("abcd"), hir_lit("abcd")); + + assert_eq!(t_bytes("(?-u)a"), hir_lit("a")); + assert_eq!(t_bytes("(?-u)\x61"), hir_lit("a")); + assert_eq!(t_bytes(r"(?-u)\x61"), hir_lit("a")); + assert_eq!(t_bytes(r"(?-u)\xFF"), hir_blit(b"\xFF")); + + assert_eq!(t("(?-u)☃"), hir_lit("☃")); + assert_eq!( + t_err(r"(?-u)\xFF"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(5, 1, 6), + Position::new(9, 1, 10) + ), + } + ); + } + + #[test] + fn literal_case_insensitive() { + #[cfg(feature = "unicode-case")] + assert_eq!(t("(?i)a"), hir_uclass(&[('A', 'A'), ('a', 'a'),])); + #[cfg(feature = "unicode-case")] + assert_eq!(t("(?i:a)"), hir_uclass(&[('A', 'A'), ('a', 'a')])); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("a(?i)a(?-i)a"), + hir_cat(vec![ + hir_lit("a"), + hir_uclass(&[('A', 'A'), ('a', 'a')]), + hir_lit("a"), + ]) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)ab@c"), + hir_cat(vec![ + hir_uclass(&[('A', 'A'), ('a', 'a')]), + hir_uclass(&[('B', 'B'), ('b', 'b')]), + hir_lit("@"), + hir_uclass(&[('C', 'C'), ('c', 'c')]), + ]) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)β"), + hir_uclass(&[('Β', 'Β'), ('β', 'β'), ('ϐ', 'ϐ'),]) + ); + + assert_eq!(t("(?i-u)a"), hir_bclass(&[(b'A', b'A'), (b'a', b'a'),])); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?-u)a(?i)a(?-i)a"), + hir_cat(vec![ + hir_lit("a"), + hir_bclass(&[(b'A', b'A'), (b'a', b'a')]), + hir_lit("a"), + ]) + ); + assert_eq!( + t("(?i-u)ab@c"), + hir_cat(vec![ + hir_bclass(&[(b'A', b'A'), (b'a', b'a')]), + hir_bclass(&[(b'B', b'B'), (b'b', b'b')]), + hir_lit("@"), + hir_bclass(&[(b'C', b'C'), (b'c', b'c')]), + ]) + ); + + assert_eq!( + t_bytes("(?i-u)a"), + hir_bclass(&[(b'A', b'A'), (b'a', b'a'),]) + ); + assert_eq!( + t_bytes("(?i-u)\x61"), + hir_bclass(&[(b'A', b'A'), (b'a', b'a'),]) + ); + assert_eq!( + t_bytes(r"(?i-u)\x61"), + hir_bclass(&[(b'A', b'A'), (b'a', b'a'),]) + ); + assert_eq!(t_bytes(r"(?i-u)\xFF"), hir_blit(b"\xFF")); + + assert_eq!(t("(?i-u)β"), hir_lit("β"),); + } + + #[test] + fn dot() { + assert_eq!( + t("."), + hir_uclass(&[('\0', '\t'), ('\x0B', '\u{10FFFF}')]) + ); + assert_eq!( + t("(?R)."), + hir_uclass(&[ + ('\0', '\t'), + ('\x0B', '\x0C'), + ('\x0E', '\u{10FFFF}'), + ]) + ); + assert_eq!(t("(?s)."), hir_uclass(&[('\0', '\u{10FFFF}')])); + assert_eq!(t("(?Rs)."), hir_uclass(&[('\0', '\u{10FFFF}')])); + assert_eq!( + t_bytes("(?-u)."), + hir_bclass(&[(b'\0', b'\t'), (b'\x0B', b'\xFF')]) + ); + assert_eq!( + t_bytes("(?R-u)."), + hir_bclass(&[ + (b'\0', b'\t'), + (b'\x0B', b'\x0C'), + (b'\x0E', b'\xFF'), + ]) + ); + assert_eq!(t_bytes("(?s-u)."), hir_bclass(&[(b'\0', b'\xFF'),])); + assert_eq!(t_bytes("(?Rs-u)."), hir_bclass(&[(b'\0', b'\xFF'),])); + + // If invalid UTF-8 isn't allowed, then non-Unicode `.` isn't allowed. + assert_eq!( + t_err("(?-u)."), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(5, 1, 6), + Position::new(6, 1, 7) + ), + } + ); + assert_eq!( + t_err("(?R-u)."), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(6, 1, 7), + Position::new(7, 1, 8) + ), + } + ); + assert_eq!( + t_err("(?s-u)."), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(6, 1, 7), + Position::new(7, 1, 8) + ), + } + ); + assert_eq!( + t_err("(?Rs-u)."), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(7, 1, 8), + Position::new(8, 1, 9) + ), + } + ); + } + + #[test] + fn assertions() { + assert_eq!(t("^"), hir_look(hir::Look::Start)); + assert_eq!(t("$"), hir_look(hir::Look::End)); + assert_eq!(t(r"\A"), hir_look(hir::Look::Start)); + assert_eq!(t(r"\z"), hir_look(hir::Look::End)); + assert_eq!(t("(?m)^"), hir_look(hir::Look::StartLF)); + assert_eq!(t("(?m)$"), hir_look(hir::Look::EndLF)); + assert_eq!(t(r"(?m)\A"), hir_look(hir::Look::Start)); + assert_eq!(t(r"(?m)\z"), hir_look(hir::Look::End)); + + assert_eq!(t(r"\b"), hir_look(hir::Look::WordUnicode)); + assert_eq!(t(r"\B"), hir_look(hir::Look::WordUnicodeNegate)); + assert_eq!(t(r"(?-u)\b"), hir_look(hir::Look::WordAscii)); + assert_eq!(t(r"(?-u)\B"), hir_look(hir::Look::WordAsciiNegate)); + } + + #[test] + fn group() { + assert_eq!(t("(a)"), hir_capture(1, hir_lit("a"))); + assert_eq!( + t("(a)(b)"), + hir_cat(vec![ + hir_capture(1, hir_lit("a")), + hir_capture(2, hir_lit("b")), + ]) + ); + assert_eq!( + t("(a)|(b)"), + hir_alt(vec![ + hir_capture(1, hir_lit("a")), + hir_capture(2, hir_lit("b")), + ]) + ); + assert_eq!(t("(?P<foo>)"), hir_capture_name(1, "foo", Hir::empty())); + assert_eq!(t("(?P<foo>a)"), hir_capture_name(1, "foo", hir_lit("a"))); + assert_eq!( + t("(?P<foo>a)(?P<bar>b)"), + hir_cat(vec![ + hir_capture_name(1, "foo", hir_lit("a")), + hir_capture_name(2, "bar", hir_lit("b")), + ]) + ); + assert_eq!(t("(?:)"), Hir::empty()); + assert_eq!(t("(?:a)"), hir_lit("a")); + assert_eq!( + t("(?:a)(b)"), + hir_cat(vec![hir_lit("a"), hir_capture(1, hir_lit("b")),]) + ); + assert_eq!( + t("(a)(?:b)(c)"), + hir_cat(vec![ + hir_capture(1, hir_lit("a")), + hir_lit("b"), + hir_capture(2, hir_lit("c")), + ]) + ); + assert_eq!( + t("(a)(?P<foo>b)(c)"), + hir_cat(vec![ + hir_capture(1, hir_lit("a")), + hir_capture_name(2, "foo", hir_lit("b")), + hir_capture(3, hir_lit("c")), + ]) + ); + assert_eq!(t("()"), hir_capture(1, Hir::empty())); + assert_eq!(t("((?i))"), hir_capture(1, Hir::empty())); + assert_eq!(t("((?x))"), hir_capture(1, Hir::empty())); + assert_eq!( + t("(((?x)))"), + hir_capture(1, hir_capture(2, Hir::empty())) + ); + } + + #[test] + fn line_anchors() { + assert_eq!(t("^"), hir_look(hir::Look::Start)); + assert_eq!(t("$"), hir_look(hir::Look::End)); + assert_eq!(t(r"\A"), hir_look(hir::Look::Start)); + assert_eq!(t(r"\z"), hir_look(hir::Look::End)); + + assert_eq!(t(r"(?m)\A"), hir_look(hir::Look::Start)); + assert_eq!(t(r"(?m)\z"), hir_look(hir::Look::End)); + assert_eq!(t("(?m)^"), hir_look(hir::Look::StartLF)); + assert_eq!(t("(?m)$"), hir_look(hir::Look::EndLF)); + + assert_eq!(t(r"(?R)\A"), hir_look(hir::Look::Start)); + assert_eq!(t(r"(?R)\z"), hir_look(hir::Look::End)); + assert_eq!(t("(?R)^"), hir_look(hir::Look::Start)); + assert_eq!(t("(?R)$"), hir_look(hir::Look::End)); + + assert_eq!(t(r"(?Rm)\A"), hir_look(hir::Look::Start)); + assert_eq!(t(r"(?Rm)\z"), hir_look(hir::Look::End)); + assert_eq!(t("(?Rm)^"), hir_look(hir::Look::StartCRLF)); + assert_eq!(t("(?Rm)$"), hir_look(hir::Look::EndCRLF)); + } + + #[test] + fn flags() { + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i:a)a"), + hir_cat( + vec![hir_uclass(&[('A', 'A'), ('a', 'a')]), hir_lit("a"),] + ) + ); + assert_eq!( + t("(?i-u:a)β"), + hir_cat(vec![ + hir_bclass(&[(b'A', b'A'), (b'a', b'a')]), + hir_lit("β"), + ]) + ); + assert_eq!( + t("(?:(?i-u)a)b"), + hir_cat(vec![ + hir_bclass(&[(b'A', b'A'), (b'a', b'a')]), + hir_lit("b"), + ]) + ); + assert_eq!( + t("((?i-u)a)b"), + hir_cat(vec![ + hir_capture(1, hir_bclass(&[(b'A', b'A'), (b'a', b'a')])), + hir_lit("b"), + ]) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)(?-i:a)a"), + hir_cat( + vec![hir_lit("a"), hir_uclass(&[('A', 'A'), ('a', 'a')]),] + ) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?im)a^"), + hir_cat(vec![ + hir_uclass(&[('A', 'A'), ('a', 'a')]), + hir_look(hir::Look::StartLF), + ]) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?im)a^(?i-m)a^"), + hir_cat(vec![ + hir_uclass(&[('A', 'A'), ('a', 'a')]), + hir_look(hir::Look::StartLF), + hir_uclass(&[('A', 'A'), ('a', 'a')]), + hir_look(hir::Look::Start), + ]) + ); + assert_eq!( + t("(?U)a*a*?(?-U)a*a*?"), + hir_cat(vec![ + hir_star(false, hir_lit("a")), + hir_star(true, hir_lit("a")), + hir_star(true, hir_lit("a")), + hir_star(false, hir_lit("a")), + ]) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?:a(?i)a)a"), + hir_cat(vec![ + hir_cat(vec![ + hir_lit("a"), + hir_uclass(&[('A', 'A'), ('a', 'a')]), + ]), + hir_lit("a"), + ]) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)(?:a(?-i)a)a"), + hir_cat(vec![ + hir_cat(vec![ + hir_uclass(&[('A', 'A'), ('a', 'a')]), + hir_lit("a"), + ]), + hir_uclass(&[('A', 'A'), ('a', 'a')]), + ]) + ); + } + + #[test] + fn escape() { + assert_eq!( + t(r"\\\.\+\*\?\(\)\|\[\]\{\}\^\$\#"), + hir_lit(r"\.+*?()|[]{}^$#") + ); + } + + #[test] + fn repetition() { + assert_eq!(t("a?"), hir_quest(true, hir_lit("a"))); + assert_eq!(t("a*"), hir_star(true, hir_lit("a"))); + assert_eq!(t("a+"), hir_plus(true, hir_lit("a"))); + assert_eq!(t("a??"), hir_quest(false, hir_lit("a"))); + assert_eq!(t("a*?"), hir_star(false, hir_lit("a"))); + assert_eq!(t("a+?"), hir_plus(false, hir_lit("a"))); + + assert_eq!(t("a{1}"), hir_range(true, 1, Some(1), hir_lit("a"),)); + assert_eq!(t("a{1,}"), hir_range(true, 1, None, hir_lit("a"),)); + assert_eq!(t("a{1,2}"), hir_range(true, 1, Some(2), hir_lit("a"),)); + assert_eq!(t("a{1}?"), hir_range(false, 1, Some(1), hir_lit("a"),)); + assert_eq!(t("a{1,}?"), hir_range(false, 1, None, hir_lit("a"),)); + assert_eq!(t("a{1,2}?"), hir_range(false, 1, Some(2), hir_lit("a"),)); + + assert_eq!( + t("ab?"), + hir_cat(vec![hir_lit("a"), hir_quest(true, hir_lit("b")),]) + ); + assert_eq!(t("(ab)?"), hir_quest(true, hir_capture(1, hir_lit("ab")))); + assert_eq!( + t("a|b?"), + hir_alt(vec![hir_lit("a"), hir_quest(true, hir_lit("b")),]) + ); + } + + #[test] + fn cat_alt() { + let a = || hir_look(hir::Look::Start); + let b = || hir_look(hir::Look::End); + let c = || hir_look(hir::Look::WordUnicode); + let d = || hir_look(hir::Look::WordUnicodeNegate); + + assert_eq!(t("(^$)"), hir_capture(1, hir_cat(vec![a(), b()]))); + assert_eq!(t("^|$"), hir_alt(vec![a(), b()])); + assert_eq!(t(r"^|$|\b"), hir_alt(vec![a(), b(), c()])); + assert_eq!( + t(r"^$|$\b|\b\B"), + hir_alt(vec![ + hir_cat(vec![a(), b()]), + hir_cat(vec![b(), c()]), + hir_cat(vec![c(), d()]), + ]) + ); + assert_eq!(t("(^|$)"), hir_capture(1, hir_alt(vec![a(), b()]))); + assert_eq!( + t(r"(^|$|\b)"), + hir_capture(1, hir_alt(vec![a(), b(), c()])) + ); + assert_eq!( + t(r"(^$|$\b|\b\B)"), + hir_capture( + 1, + hir_alt(vec![ + hir_cat(vec![a(), b()]), + hir_cat(vec![b(), c()]), + hir_cat(vec![c(), d()]), + ]) + ) + ); + assert_eq!( + t(r"(^$|($\b|(\b\B)))"), + hir_capture( + 1, + hir_alt(vec![ + hir_cat(vec![a(), b()]), + hir_capture( + 2, + hir_alt(vec![ + hir_cat(vec![b(), c()]), + hir_capture(3, hir_cat(vec![c(), d()])), + ]) + ), + ]) + ) + ); + } + + // Tests the HIR transformation of things like '[a-z]|[A-Z]' into + // '[A-Za-z]'. In other words, an alternation of just classes is always + // equivalent to a single class corresponding to the union of the branches + // in that class. (Unless some branches match invalid UTF-8 and others + // match non-ASCII Unicode.) + #[test] + fn cat_class_flattened() { + assert_eq!(t(r"[a-z]|[A-Z]"), hir_uclass(&[('A', 'Z'), ('a', 'z')])); + // Combining all of the letter properties should give us the one giant + // letter property. + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"(?x) + \p{Lowercase_Letter} + |\p{Uppercase_Letter} + |\p{Titlecase_Letter} + |\p{Modifier_Letter} + |\p{Other_Letter} + "), + hir_uclass_query(ClassQuery::Binary("letter")) + ); + // Byte classes that can truly match invalid UTF-8 cannot be combined + // with Unicode classes. + assert_eq!( + t_bytes(r"[Δδ]|(?-u:[\x90-\xFF])|[Λλ]"), + hir_alt(vec![ + hir_uclass(&[('Δ', 'Δ'), ('δ', 'δ')]), + hir_bclass(&[(b'\x90', b'\xFF')]), + hir_uclass(&[('Λ', 'Λ'), ('λ', 'λ')]), + ]) + ); + // Byte classes on their own can be combined, even if some are ASCII + // and others are invalid UTF-8. + assert_eq!( + t_bytes(r"[a-z]|(?-u:[\x90-\xFF])|[A-Z]"), + hir_bclass(&[(b'A', b'Z'), (b'a', b'z'), (b'\x90', b'\xFF')]), + ); + } + + #[test] + fn class_ascii() { + assert_eq!( + t("[[:alnum:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Alnum) + ); + assert_eq!( + t("[[:alpha:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Alpha) + ); + assert_eq!( + t("[[:ascii:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Ascii) + ); + assert_eq!( + t("[[:blank:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Blank) + ); + assert_eq!( + t("[[:cntrl:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Cntrl) + ); + assert_eq!( + t("[[:digit:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Digit) + ); + assert_eq!( + t("[[:graph:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Graph) + ); + assert_eq!( + t("[[:lower:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Lower) + ); + assert_eq!( + t("[[:print:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Print) + ); + assert_eq!( + t("[[:punct:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Punct) + ); + assert_eq!( + t("[[:space:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Space) + ); + assert_eq!( + t("[[:upper:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Upper) + ); + assert_eq!( + t("[[:word:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Word) + ); + assert_eq!( + t("[[:xdigit:]]"), + hir_ascii_uclass(&ast::ClassAsciiKind::Xdigit) + ); + + assert_eq!( + t("[[:^lower:]]"), + hir_negate(hir_ascii_uclass(&ast::ClassAsciiKind::Lower)) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[[:lower:]]"), + hir_uclass(&[ + ('A', 'Z'), + ('a', 'z'), + ('\u{17F}', '\u{17F}'), + ('\u{212A}', '\u{212A}'), + ]) + ); + + assert_eq!( + t("(?-u)[[:lower:]]"), + hir_ascii_bclass(&ast::ClassAsciiKind::Lower) + ); + assert_eq!( + t("(?i-u)[[:lower:]]"), + hir_case_fold(hir_ascii_bclass(&ast::ClassAsciiKind::Lower)) + ); + + assert_eq!( + t_err("(?-u)[[:^lower:]]"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(6, 1, 7), + Position::new(16, 1, 17) + ), + } + ); + assert_eq!( + t_err("(?i-u)[[:^lower:]]"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(7, 1, 8), + Position::new(17, 1, 18) + ), + } + ); + } + + #[test] + fn class_ascii_multiple() { + // See: https://github.com/rust-lang/regex/issues/680 + assert_eq!( + t("[[:alnum:][:^ascii:]]"), + hir_union( + hir_ascii_uclass(&ast::ClassAsciiKind::Alnum), + hir_uclass(&[('\u{80}', '\u{10FFFF}')]), + ), + ); + assert_eq!( + t_bytes("(?-u)[[:alnum:][:^ascii:]]"), + hir_union( + hir_ascii_bclass(&ast::ClassAsciiKind::Alnum), + hir_bclass(&[(0x80, 0xFF)]), + ), + ); + } + + #[test] + #[cfg(feature = "unicode-perl")] + fn class_perl_unicode() { + // Unicode + assert_eq!(t(r"\d"), hir_uclass_query(ClassQuery::Binary("digit"))); + assert_eq!(t(r"\s"), hir_uclass_query(ClassQuery::Binary("space"))); + assert_eq!(t(r"\w"), hir_uclass_perl_word()); + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)\d"), + hir_uclass_query(ClassQuery::Binary("digit")) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)\s"), + hir_uclass_query(ClassQuery::Binary("space")) + ); + #[cfg(feature = "unicode-case")] + assert_eq!(t(r"(?i)\w"), hir_uclass_perl_word()); + + // Unicode, negated + assert_eq!( + t(r"\D"), + hir_negate(hir_uclass_query(ClassQuery::Binary("digit"))) + ); + assert_eq!( + t(r"\S"), + hir_negate(hir_uclass_query(ClassQuery::Binary("space"))) + ); + assert_eq!(t(r"\W"), hir_negate(hir_uclass_perl_word())); + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)\D"), + hir_negate(hir_uclass_query(ClassQuery::Binary("digit"))) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)\S"), + hir_negate(hir_uclass_query(ClassQuery::Binary("space"))) + ); + #[cfg(feature = "unicode-case")] + assert_eq!(t(r"(?i)\W"), hir_negate(hir_uclass_perl_word())); + } + + #[test] + fn class_perl_ascii() { + // ASCII only + assert_eq!( + t(r"(?-u)\d"), + hir_ascii_bclass(&ast::ClassAsciiKind::Digit) + ); + assert_eq!( + t(r"(?-u)\s"), + hir_ascii_bclass(&ast::ClassAsciiKind::Space) + ); + assert_eq!( + t(r"(?-u)\w"), + hir_ascii_bclass(&ast::ClassAsciiKind::Word) + ); + assert_eq!( + t(r"(?i-u)\d"), + hir_ascii_bclass(&ast::ClassAsciiKind::Digit) + ); + assert_eq!( + t(r"(?i-u)\s"), + hir_ascii_bclass(&ast::ClassAsciiKind::Space) + ); + assert_eq!( + t(r"(?i-u)\w"), + hir_ascii_bclass(&ast::ClassAsciiKind::Word) + ); + + // ASCII only, negated + assert_eq!( + t_bytes(r"(?-u)\D"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Digit)) + ); + assert_eq!( + t_bytes(r"(?-u)\S"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Space)) + ); + assert_eq!( + t_bytes(r"(?-u)\W"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Word)) + ); + assert_eq!( + t_bytes(r"(?i-u)\D"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Digit)) + ); + assert_eq!( + t_bytes(r"(?i-u)\S"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Space)) + ); + assert_eq!( + t_bytes(r"(?i-u)\W"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Word)) + ); + + // ASCII only, negated, with UTF-8 mode enabled. + // In this case, negating any Perl class results in an error because + // all such classes can match invalid UTF-8. + assert_eq!( + t_err(r"(?-u)\D"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(5, 1, 6), + Position::new(7, 1, 8), + ), + }, + ); + assert_eq!( + t_err(r"(?-u)\S"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(5, 1, 6), + Position::new(7, 1, 8), + ), + }, + ); + assert_eq!( + t_err(r"(?-u)\W"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(5, 1, 6), + Position::new(7, 1, 8), + ), + }, + ); + assert_eq!( + t_err(r"(?i-u)\D"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(6, 1, 7), + Position::new(8, 1, 9), + ), + }, + ); + assert_eq!( + t_err(r"(?i-u)\S"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(6, 1, 7), + Position::new(8, 1, 9), + ), + }, + ); + assert_eq!( + t_err(r"(?i-u)\W"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(6, 1, 7), + Position::new(8, 1, 9), + ), + }, + ); + } + + #[test] + #[cfg(not(feature = "unicode-perl"))] + fn class_perl_word_disabled() { + assert_eq!( + t_err(r"\w"), + TestError { + kind: hir::ErrorKind::UnicodePerlClassNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(2, 1, 3) + ), + } + ); + } + + #[test] + #[cfg(all(not(feature = "unicode-perl"), not(feature = "unicode-bool")))] + fn class_perl_space_disabled() { + assert_eq!( + t_err(r"\s"), + TestError { + kind: hir::ErrorKind::UnicodePerlClassNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(2, 1, 3) + ), + } + ); + } + + #[test] + #[cfg(all( + not(feature = "unicode-perl"), + not(feature = "unicode-gencat") + ))] + fn class_perl_digit_disabled() { + assert_eq!( + t_err(r"\d"), + TestError { + kind: hir::ErrorKind::UnicodePerlClassNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(2, 1, 3) + ), + } + ); + } + + #[test] + #[cfg(feature = "unicode-gencat")] + fn class_unicode_gencat() { + assert_eq!(t(r"\pZ"), hir_uclass_query(ClassQuery::Binary("Z"))); + assert_eq!(t(r"\pz"), hir_uclass_query(ClassQuery::Binary("Z"))); + assert_eq!( + t(r"\p{Separator}"), + hir_uclass_query(ClassQuery::Binary("Z")) + ); + assert_eq!( + t(r"\p{se PaRa ToR}"), + hir_uclass_query(ClassQuery::Binary("Z")) + ); + assert_eq!( + t(r"\p{gc:Separator}"), + hir_uclass_query(ClassQuery::Binary("Z")) + ); + assert_eq!( + t(r"\p{gc=Separator}"), + hir_uclass_query(ClassQuery::Binary("Z")) + ); + assert_eq!( + t(r"\p{gc!=Separator}"), + hir_negate(hir_uclass_query(ClassQuery::Binary("Z"))) + ); + assert_eq!( + t(r"\p{Other}"), + hir_uclass_query(ClassQuery::Binary("Other")) + ); + assert_eq!(t(r"\pC"), hir_uclass_query(ClassQuery::Binary("Other"))); + + assert_eq!( + t(r"\PZ"), + hir_negate(hir_uclass_query(ClassQuery::Binary("Z"))) + ); + assert_eq!( + t(r"\P{separator}"), + hir_negate(hir_uclass_query(ClassQuery::Binary("Z"))) + ); + assert_eq!( + t(r"\P{gc!=separator}"), + hir_uclass_query(ClassQuery::Binary("Z")) + ); + + assert_eq!(t(r"\p{any}"), hir_uclass_query(ClassQuery::Binary("Any"))); + assert_eq!( + t(r"\p{assigned}"), + hir_uclass_query(ClassQuery::Binary("Assigned")) + ); + assert_eq!( + t(r"\p{ascii}"), + hir_uclass_query(ClassQuery::Binary("ASCII")) + ); + assert_eq!( + t(r"\p{gc:any}"), + hir_uclass_query(ClassQuery::Binary("Any")) + ); + assert_eq!( + t(r"\p{gc:assigned}"), + hir_uclass_query(ClassQuery::Binary("Assigned")) + ); + assert_eq!( + t(r"\p{gc:ascii}"), + hir_uclass_query(ClassQuery::Binary("ASCII")) + ); + + assert_eq!( + t_err(r"(?-u)\pZ"), + TestError { + kind: hir::ErrorKind::UnicodeNotAllowed, + span: Span::new( + Position::new(5, 1, 6), + Position::new(8, 1, 9) + ), + } + ); + assert_eq!( + t_err(r"(?-u)\p{Separator}"), + TestError { + kind: hir::ErrorKind::UnicodeNotAllowed, + span: Span::new( + Position::new(5, 1, 6), + Position::new(18, 1, 19) + ), + } + ); + assert_eq!( + t_err(r"\pE"), + TestError { + kind: hir::ErrorKind::UnicodePropertyNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(3, 1, 4) + ), + } + ); + assert_eq!( + t_err(r"\p{Foo}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(7, 1, 8) + ), + } + ); + assert_eq!( + t_err(r"\p{gc:Foo}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyValueNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(10, 1, 11) + ), + } + ); + } + + #[test] + #[cfg(not(feature = "unicode-gencat"))] + fn class_unicode_gencat_disabled() { + assert_eq!( + t_err(r"\p{Separator}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(13, 1, 14) + ), + } + ); + + assert_eq!( + t_err(r"\p{Any}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(7, 1, 8) + ), + } + ); + } + + #[test] + #[cfg(feature = "unicode-script")] + fn class_unicode_script() { + assert_eq!( + t(r"\p{Greek}"), + hir_uclass_query(ClassQuery::Binary("Greek")) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)\p{Greek}"), + hir_case_fold(hir_uclass_query(ClassQuery::Binary("Greek"))) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)\P{Greek}"), + hir_negate(hir_case_fold(hir_uclass_query(ClassQuery::Binary( + "Greek" + )))) + ); + + assert_eq!( + t_err(r"\p{sc:Foo}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyValueNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(10, 1, 11) + ), + } + ); + assert_eq!( + t_err(r"\p{scx:Foo}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyValueNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(11, 1, 12) + ), + } + ); + } + + #[test] + #[cfg(not(feature = "unicode-script"))] + fn class_unicode_script_disabled() { + assert_eq!( + t_err(r"\p{Greek}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(9, 1, 10) + ), + } + ); + + assert_eq!( + t_err(r"\p{scx:Greek}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(13, 1, 14) + ), + } + ); + } + + #[test] + #[cfg(feature = "unicode-age")] + fn class_unicode_age() { + assert_eq!( + t_err(r"\p{age:Foo}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyValueNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(11, 1, 12) + ), + } + ); + } + + #[test] + #[cfg(feature = "unicode-gencat")] + fn class_unicode_any_empty() { + assert_eq!(t(r"\P{any}"), hir_uclass(&[]),); + } + + #[test] + #[cfg(not(feature = "unicode-age"))] + fn class_unicode_age_disabled() { + assert_eq!( + t_err(r"\p{age:3.0}"), + TestError { + kind: hir::ErrorKind::UnicodePropertyNotFound, + span: Span::new( + Position::new(0, 1, 1), + Position::new(11, 1, 12) + ), + } + ); + } + + #[test] + fn class_bracketed() { + assert_eq!(t("[a]"), hir_lit("a")); + assert_eq!(t("[ab]"), hir_uclass(&[('a', 'b')])); + assert_eq!(t("[^[a]]"), class_negate(uclass(&[('a', 'a')]))); + assert_eq!(t("[a-z]"), hir_uclass(&[('a', 'z')])); + assert_eq!(t("[a-fd-h]"), hir_uclass(&[('a', 'h')])); + assert_eq!(t("[a-fg-m]"), hir_uclass(&[('a', 'm')])); + assert_eq!(t(r"[\x00]"), hir_uclass(&[('\0', '\0')])); + assert_eq!(t(r"[\n]"), hir_uclass(&[('\n', '\n')])); + assert_eq!(t("[\n]"), hir_uclass(&[('\n', '\n')])); + #[cfg(any(feature = "unicode-perl", feature = "unicode-gencat"))] + assert_eq!(t(r"[\d]"), hir_uclass_query(ClassQuery::Binary("digit"))); + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"[\pZ]"), + hir_uclass_query(ClassQuery::Binary("separator")) + ); + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"[\p{separator}]"), + hir_uclass_query(ClassQuery::Binary("separator")) + ); + #[cfg(any(feature = "unicode-perl", feature = "unicode-gencat"))] + assert_eq!(t(r"[^\D]"), hir_uclass_query(ClassQuery::Binary("digit"))); + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"[^\PZ]"), + hir_uclass_query(ClassQuery::Binary("separator")) + ); + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"[^\P{separator}]"), + hir_uclass_query(ClassQuery::Binary("separator")) + ); + #[cfg(all( + feature = "unicode-case", + any(feature = "unicode-perl", feature = "unicode-gencat") + ))] + assert_eq!( + t(r"(?i)[^\D]"), + hir_uclass_query(ClassQuery::Binary("digit")) + ); + #[cfg(all(feature = "unicode-case", feature = "unicode-script"))] + assert_eq!( + t(r"(?i)[^\P{greek}]"), + hir_case_fold(hir_uclass_query(ClassQuery::Binary("greek"))) + ); + + assert_eq!(t("(?-u)[a]"), hir_bclass(&[(b'a', b'a')])); + assert_eq!(t(r"(?-u)[\x00]"), hir_bclass(&[(b'\0', b'\0')])); + assert_eq!(t_bytes(r"(?-u)[\xFF]"), hir_bclass(&[(b'\xFF', b'\xFF')])); + + #[cfg(feature = "unicode-case")] + assert_eq!(t("(?i)[a]"), hir_uclass(&[('A', 'A'), ('a', 'a')])); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[k]"), + hir_uclass(&[('K', 'K'), ('k', 'k'), ('\u{212A}', '\u{212A}'),]) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[β]"), + hir_uclass(&[('Β', 'Β'), ('β', 'β'), ('ϐ', 'ϐ'),]) + ); + assert_eq!(t("(?i-u)[k]"), hir_bclass(&[(b'K', b'K'), (b'k', b'k'),])); + + assert_eq!(t("[^a]"), class_negate(uclass(&[('a', 'a')]))); + assert_eq!(t(r"[^\x00]"), class_negate(uclass(&[('\0', '\0')]))); + assert_eq!( + t_bytes("(?-u)[^a]"), + class_negate(bclass(&[(b'a', b'a')])) + ); + #[cfg(any(feature = "unicode-perl", feature = "unicode-gencat"))] + assert_eq!( + t(r"[^\d]"), + hir_negate(hir_uclass_query(ClassQuery::Binary("digit"))) + ); + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"[^\pZ]"), + hir_negate(hir_uclass_query(ClassQuery::Binary("separator"))) + ); + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"[^\p{separator}]"), + hir_negate(hir_uclass_query(ClassQuery::Binary("separator"))) + ); + #[cfg(all(feature = "unicode-case", feature = "unicode-script"))] + assert_eq!( + t(r"(?i)[^\p{greek}]"), + hir_negate(hir_case_fold(hir_uclass_query(ClassQuery::Binary( + "greek" + )))) + ); + #[cfg(all(feature = "unicode-case", feature = "unicode-script"))] + assert_eq!( + t(r"(?i)[\P{greek}]"), + hir_negate(hir_case_fold(hir_uclass_query(ClassQuery::Binary( + "greek" + )))) + ); + + // Test some weird cases. + assert_eq!(t(r"[\[]"), hir_uclass(&[('[', '[')])); + + assert_eq!(t(r"[&]"), hir_uclass(&[('&', '&')])); + assert_eq!(t(r"[\&]"), hir_uclass(&[('&', '&')])); + assert_eq!(t(r"[\&\&]"), hir_uclass(&[('&', '&')])); + assert_eq!(t(r"[\x00-&]"), hir_uclass(&[('\0', '&')])); + assert_eq!(t(r"[&-\xFF]"), hir_uclass(&[('&', '\u{FF}')])); + + assert_eq!(t(r"[~]"), hir_uclass(&[('~', '~')])); + assert_eq!(t(r"[\~]"), hir_uclass(&[('~', '~')])); + assert_eq!(t(r"[\~\~]"), hir_uclass(&[('~', '~')])); + assert_eq!(t(r"[\x00-~]"), hir_uclass(&[('\0', '~')])); + assert_eq!(t(r"[~-\xFF]"), hir_uclass(&[('~', '\u{FF}')])); + + assert_eq!(t(r"[-]"), hir_uclass(&[('-', '-')])); + assert_eq!(t(r"[\-]"), hir_uclass(&[('-', '-')])); + assert_eq!(t(r"[\-\-]"), hir_uclass(&[('-', '-')])); + assert_eq!(t(r"[\x00-\-]"), hir_uclass(&[('\0', '-')])); + assert_eq!(t(r"[\--\xFF]"), hir_uclass(&[('-', '\u{FF}')])); + + assert_eq!( + t_err("(?-u)[^a]"), + TestError { + kind: hir::ErrorKind::InvalidUtf8, + span: Span::new( + Position::new(5, 1, 6), + Position::new(9, 1, 10) + ), + } + ); + #[cfg(any(feature = "unicode-perl", feature = "unicode-bool"))] + assert_eq!(t(r"[^\s\S]"), hir_uclass(&[]),); + #[cfg(any(feature = "unicode-perl", feature = "unicode-bool"))] + assert_eq!(t_bytes(r"(?-u)[^\s\S]"), hir_bclass(&[]),); + } + + #[test] + fn class_bracketed_union() { + assert_eq!(t("[a-zA-Z]"), hir_uclass(&[('A', 'Z'), ('a', 'z')])); + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"[a\pZb]"), + hir_union( + hir_uclass(&[('a', 'b')]), + hir_uclass_query(ClassQuery::Binary("separator")) + ) + ); + #[cfg(all(feature = "unicode-gencat", feature = "unicode-script"))] + assert_eq!( + t(r"[\pZ\p{Greek}]"), + hir_union( + hir_uclass_query(ClassQuery::Binary("greek")), + hir_uclass_query(ClassQuery::Binary("separator")) + ) + ); + #[cfg(all( + feature = "unicode-age", + feature = "unicode-gencat", + feature = "unicode-script" + ))] + assert_eq!( + t(r"[\p{age:3.0}\pZ\p{Greek}]"), + hir_union( + hir_uclass_query(ClassQuery::ByValue { + property_name: "age", + property_value: "3.0", + }), + hir_union( + hir_uclass_query(ClassQuery::Binary("greek")), + hir_uclass_query(ClassQuery::Binary("separator")) + ) + ) + ); + #[cfg(all( + feature = "unicode-age", + feature = "unicode-gencat", + feature = "unicode-script" + ))] + assert_eq!( + t(r"[[[\p{age:3.0}\pZ]\p{Greek}][\p{Cyrillic}]]"), + hir_union( + hir_uclass_query(ClassQuery::ByValue { + property_name: "age", + property_value: "3.0", + }), + hir_union( + hir_uclass_query(ClassQuery::Binary("cyrillic")), + hir_union( + hir_uclass_query(ClassQuery::Binary("greek")), + hir_uclass_query(ClassQuery::Binary("separator")) + ) + ) + ) + ); + + #[cfg(all( + feature = "unicode-age", + feature = "unicode-case", + feature = "unicode-gencat", + feature = "unicode-script" + ))] + assert_eq!( + t(r"(?i)[\p{age:3.0}\pZ\p{Greek}]"), + hir_case_fold(hir_union( + hir_uclass_query(ClassQuery::ByValue { + property_name: "age", + property_value: "3.0", + }), + hir_union( + hir_uclass_query(ClassQuery::Binary("greek")), + hir_uclass_query(ClassQuery::Binary("separator")) + ) + )) + ); + #[cfg(all( + feature = "unicode-age", + feature = "unicode-gencat", + feature = "unicode-script" + ))] + assert_eq!( + t(r"[^\p{age:3.0}\pZ\p{Greek}]"), + hir_negate(hir_union( + hir_uclass_query(ClassQuery::ByValue { + property_name: "age", + property_value: "3.0", + }), + hir_union( + hir_uclass_query(ClassQuery::Binary("greek")), + hir_uclass_query(ClassQuery::Binary("separator")) + ) + )) + ); + #[cfg(all( + feature = "unicode-age", + feature = "unicode-case", + feature = "unicode-gencat", + feature = "unicode-script" + ))] + assert_eq!( + t(r"(?i)[^\p{age:3.0}\pZ\p{Greek}]"), + hir_negate(hir_case_fold(hir_union( + hir_uclass_query(ClassQuery::ByValue { + property_name: "age", + property_value: "3.0", + }), + hir_union( + hir_uclass_query(ClassQuery::Binary("greek")), + hir_uclass_query(ClassQuery::Binary("separator")) + ) + ))) + ); + } + + #[test] + fn class_bracketed_nested() { + assert_eq!(t(r"[a[^c]]"), class_negate(uclass(&[('c', 'c')]))); + assert_eq!(t(r"[a-b[^c]]"), class_negate(uclass(&[('c', 'c')]))); + assert_eq!(t(r"[a-c[^c]]"), class_negate(uclass(&[]))); + + assert_eq!(t(r"[^a[^c]]"), hir_uclass(&[('c', 'c')])); + assert_eq!(t(r"[^a-b[^c]]"), hir_uclass(&[('c', 'c')])); + + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)[a[^c]]"), + hir_negate(class_case_fold(uclass(&[('c', 'c')]))) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)[a-b[^c]]"), + hir_negate(class_case_fold(uclass(&[('c', 'c')]))) + ); + + #[cfg(feature = "unicode-case")] + assert_eq!(t(r"(?i)[^a[^c]]"), hir_uclass(&[('C', 'C'), ('c', 'c')])); + #[cfg(feature = "unicode-case")] + assert_eq!( + t(r"(?i)[^a-b[^c]]"), + hir_uclass(&[('C', 'C'), ('c', 'c')]) + ); + + assert_eq!(t(r"[^a-c[^c]]"), hir_uclass(&[]),); + #[cfg(feature = "unicode-case")] + assert_eq!(t(r"(?i)[^a-c[^c]]"), hir_uclass(&[]),); + } + + #[test] + fn class_bracketed_intersect() { + assert_eq!(t("[abc&&b-c]"), hir_uclass(&[('b', 'c')])); + assert_eq!(t("[abc&&[b-c]]"), hir_uclass(&[('b', 'c')])); + assert_eq!(t("[[abc]&&[b-c]]"), hir_uclass(&[('b', 'c')])); + assert_eq!(t("[a-z&&b-y&&c-x]"), hir_uclass(&[('c', 'x')])); + assert_eq!(t("[c-da-b&&a-d]"), hir_uclass(&[('a', 'd')])); + assert_eq!(t("[a-d&&c-da-b]"), hir_uclass(&[('a', 'd')])); + assert_eq!(t(r"[a-z&&a-c]"), hir_uclass(&[('a', 'c')])); + assert_eq!(t(r"[[a-z&&a-c]]"), hir_uclass(&[('a', 'c')])); + assert_eq!(t(r"[^[a-z&&a-c]]"), hir_negate(hir_uclass(&[('a', 'c')]))); + + assert_eq!(t("(?-u)[abc&&b-c]"), hir_bclass(&[(b'b', b'c')])); + assert_eq!(t("(?-u)[abc&&[b-c]]"), hir_bclass(&[(b'b', b'c')])); + assert_eq!(t("(?-u)[[abc]&&[b-c]]"), hir_bclass(&[(b'b', b'c')])); + assert_eq!(t("(?-u)[a-z&&b-y&&c-x]"), hir_bclass(&[(b'c', b'x')])); + assert_eq!(t("(?-u)[c-da-b&&a-d]"), hir_bclass(&[(b'a', b'd')])); + assert_eq!(t("(?-u)[a-d&&c-da-b]"), hir_bclass(&[(b'a', b'd')])); + + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[abc&&b-c]"), + hir_case_fold(hir_uclass(&[('b', 'c')])) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[abc&&[b-c]]"), + hir_case_fold(hir_uclass(&[('b', 'c')])) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[[abc]&&[b-c]]"), + hir_case_fold(hir_uclass(&[('b', 'c')])) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[a-z&&b-y&&c-x]"), + hir_case_fold(hir_uclass(&[('c', 'x')])) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[c-da-b&&a-d]"), + hir_case_fold(hir_uclass(&[('a', 'd')])) + ); + #[cfg(feature = "unicode-case")] + assert_eq!( + t("(?i)[a-d&&c-da-b]"), + hir_case_fold(hir_uclass(&[('a', 'd')])) + ); + + assert_eq!( + t("(?i-u)[abc&&b-c]"), + hir_case_fold(hir_bclass(&[(b'b', b'c')])) + ); + assert_eq!( + t("(?i-u)[abc&&[b-c]]"), + hir_case_fold(hir_bclass(&[(b'b', b'c')])) + ); + assert_eq!( + t("(?i-u)[[abc]&&[b-c]]"), + hir_case_fold(hir_bclass(&[(b'b', b'c')])) + ); + assert_eq!( + t("(?i-u)[a-z&&b-y&&c-x]"), + hir_case_fold(hir_bclass(&[(b'c', b'x')])) + ); + assert_eq!( + t("(?i-u)[c-da-b&&a-d]"), + hir_case_fold(hir_bclass(&[(b'a', b'd')])) + ); + assert_eq!( + t("(?i-u)[a-d&&c-da-b]"), + hir_case_fold(hir_bclass(&[(b'a', b'd')])) + ); + + // In `[a^]`, `^` does not need to be escaped, so it makes sense that + // `^` is also allowed to be unescaped after `&&`. + assert_eq!(t(r"[\^&&^]"), hir_uclass(&[('^', '^')])); + // `]` needs to be escaped after `&&` since it's not at start of class. + assert_eq!(t(r"[]&&\]]"), hir_uclass(&[(']', ']')])); + assert_eq!(t(r"[-&&-]"), hir_uclass(&[('-', '-')])); + assert_eq!(t(r"[\&&&&]"), hir_uclass(&[('&', '&')])); + assert_eq!(t(r"[\&&&\&]"), hir_uclass(&[('&', '&')])); + // Test precedence. + assert_eq!( + t(r"[a-w&&[^c-g]z]"), + hir_uclass(&[('a', 'b'), ('h', 'w')]) + ); + } + + #[test] + fn class_bracketed_intersect_negate() { + #[cfg(feature = "unicode-perl")] + assert_eq!( + t(r"[^\w&&\d]"), + hir_negate(hir_uclass_query(ClassQuery::Binary("digit"))) + ); + assert_eq!(t(r"[^[a-z&&a-c]]"), hir_negate(hir_uclass(&[('a', 'c')]))); + #[cfg(feature = "unicode-perl")] + assert_eq!( + t(r"[^[\w&&\d]]"), + hir_negate(hir_uclass_query(ClassQuery::Binary("digit"))) + ); + #[cfg(feature = "unicode-perl")] + assert_eq!( + t(r"[^[^\w&&\d]]"), + hir_uclass_query(ClassQuery::Binary("digit")) + ); + #[cfg(feature = "unicode-perl")] + assert_eq!(t(r"[[[^\w]&&[^\d]]]"), hir_negate(hir_uclass_perl_word())); + + #[cfg(feature = "unicode-perl")] + assert_eq!( + t_bytes(r"(?-u)[^\w&&\d]"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Digit)) + ); + assert_eq!( + t_bytes(r"(?-u)[^[a-z&&a-c]]"), + hir_negate(hir_bclass(&[(b'a', b'c')])) + ); + assert_eq!( + t_bytes(r"(?-u)[^[\w&&\d]]"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Digit)) + ); + assert_eq!( + t_bytes(r"(?-u)[^[^\w&&\d]]"), + hir_ascii_bclass(&ast::ClassAsciiKind::Digit) + ); + assert_eq!( + t_bytes(r"(?-u)[[[^\w]&&[^\d]]]"), + hir_negate(hir_ascii_bclass(&ast::ClassAsciiKind::Word)) + ); + } + + #[test] + fn class_bracketed_difference() { + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"[\pL--[:ascii:]]"), + hir_difference( + hir_uclass_query(ClassQuery::Binary("letter")), + hir_uclass(&[('\0', '\x7F')]) + ) + ); + + assert_eq!( + t(r"(?-u)[[:alpha:]--[:lower:]]"), + hir_bclass(&[(b'A', b'Z')]) + ); + } + + #[test] + fn class_bracketed_symmetric_difference() { + #[cfg(feature = "unicode-script")] + assert_eq!( + t(r"[\p{sc:Greek}~~\p{scx:Greek}]"), + // Class({ + // '·'..='·', + // '\u{300}'..='\u{301}', + // '\u{304}'..='\u{304}', + // '\u{306}'..='\u{306}', + // '\u{308}'..='\u{308}', + // '\u{313}'..='\u{313}', + // '\u{342}'..='\u{342}', + // '\u{345}'..='\u{345}', + // 'ʹ'..='ʹ', + // '\u{1dc0}'..='\u{1dc1}', + // '⁝'..='⁝', + // }) + hir_uclass(&[ + ('·', '·'), + ('\u{0300}', '\u{0301}'), + ('\u{0304}', '\u{0304}'), + ('\u{0306}', '\u{0306}'), + ('\u{0308}', '\u{0308}'), + ('\u{0313}', '\u{0313}'), + ('\u{0342}', '\u{0342}'), + ('\u{0345}', '\u{0345}'), + ('ʹ', 'ʹ'), + ('\u{1DC0}', '\u{1DC1}'), + ('⁝', '⁝'), + ]) + ); + assert_eq!(t(r"[a-g~~c-j]"), hir_uclass(&[('a', 'b'), ('h', 'j')])); + + assert_eq!( + t(r"(?-u)[a-g~~c-j]"), + hir_bclass(&[(b'a', b'b'), (b'h', b'j')]) + ); + } + + #[test] + fn ignore_whitespace() { + assert_eq!(t(r"(?x)\12 3"), hir_lit("\n3")); + assert_eq!(t(r"(?x)\x { 53 }"), hir_lit("S")); + assert_eq!( + t(r"(?x)\x # comment +{ # comment + 53 # comment +} #comment"), + hir_lit("S") + ); + + assert_eq!(t(r"(?x)\x 53"), hir_lit("S")); + assert_eq!( + t(r"(?x)\x # comment + 53 # comment"), + hir_lit("S") + ); + assert_eq!(t(r"(?x)\x5 3"), hir_lit("S")); + + #[cfg(feature = "unicode-gencat")] + assert_eq!( + t(r"(?x)\p # comment +{ # comment + Separator # comment +} # comment"), + hir_uclass_query(ClassQuery::Binary("separator")) + ); + + assert_eq!( + t(r"(?x)a # comment +{ # comment + 5 # comment + , # comment + 10 # comment +} # comment"), + hir_range(true, 5, Some(10), hir_lit("a")) + ); + + assert_eq!(t(r"(?x)a\ # hi there"), hir_lit("a ")); + } + + #[test] + fn analysis_is_utf8() { + // Positive examples. + assert!(props_bytes(r"a").is_utf8()); + assert!(props_bytes(r"ab").is_utf8()); + assert!(props_bytes(r"(?-u)a").is_utf8()); + assert!(props_bytes(r"(?-u)ab").is_utf8()); + assert!(props_bytes(r"\xFF").is_utf8()); + assert!(props_bytes(r"\xFF\xFF").is_utf8()); + assert!(props_bytes(r"[^a]").is_utf8()); + assert!(props_bytes(r"[^a][^a]").is_utf8()); + assert!(props_bytes(r"\b").is_utf8()); + assert!(props_bytes(r"\B").is_utf8()); + assert!(props_bytes(r"(?-u)\b").is_utf8()); + assert!(props_bytes(r"(?-u)\B").is_utf8()); + + // Negative examples. + assert!(!props_bytes(r"(?-u)\xFF").is_utf8()); + assert!(!props_bytes(r"(?-u)\xFF\xFF").is_utf8()); + assert!(!props_bytes(r"(?-u)[^a]").is_utf8()); + assert!(!props_bytes(r"(?-u)[^a][^a]").is_utf8()); + } + + #[test] + fn analysis_captures_len() { + assert_eq!(0, props(r"a").explicit_captures_len()); + assert_eq!(0, props(r"(?:a)").explicit_captures_len()); + assert_eq!(0, props(r"(?i-u:a)").explicit_captures_len()); + assert_eq!(0, props(r"(?i-u)a").explicit_captures_len()); + assert_eq!(1, props(r"(a)").explicit_captures_len()); + assert_eq!(1, props(r"(?P<foo>a)").explicit_captures_len()); + assert_eq!(1, props(r"()").explicit_captures_len()); + assert_eq!(1, props(r"()a").explicit_captures_len()); + assert_eq!(1, props(r"(a)+").explicit_captures_len()); + assert_eq!(2, props(r"(a)(b)").explicit_captures_len()); + assert_eq!(2, props(r"(a)|(b)").explicit_captures_len()); + assert_eq!(2, props(r"((a))").explicit_captures_len()); + assert_eq!(1, props(r"([a&&b])").explicit_captures_len()); + } + + #[test] + fn analysis_static_captures_len() { + let len = |pattern| props(pattern).static_explicit_captures_len(); + assert_eq!(Some(0), len(r"")); + assert_eq!(Some(0), len(r"foo|bar")); + assert_eq!(None, len(r"(foo)|bar")); + assert_eq!(None, len(r"foo|(bar)")); + assert_eq!(Some(1), len(r"(foo|bar)")); + assert_eq!(Some(1), len(r"(a|b|c|d|e|f)")); + assert_eq!(Some(1), len(r"(a)|(b)|(c)|(d)|(e)|(f)")); + assert_eq!(Some(2), len(r"(a)(b)|(c)(d)|(e)(f)")); + assert_eq!(Some(6), len(r"(a)(b)(c)(d)(e)(f)")); + assert_eq!(Some(3), len(r"(a)(b)(extra)|(a)(b)()")); + assert_eq!(Some(3), len(r"(a)(b)((?:extra)?)")); + assert_eq!(None, len(r"(a)(b)(extra)?")); + assert_eq!(Some(1), len(r"(foo)|(bar)")); + assert_eq!(Some(2), len(r"(foo)(bar)")); + assert_eq!(Some(2), len(r"(foo)+(bar)")); + assert_eq!(None, len(r"(foo)*(bar)")); + assert_eq!(Some(0), len(r"(foo)?{0}")); + assert_eq!(None, len(r"(foo)?{1}")); + assert_eq!(Some(1), len(r"(foo){1}")); + assert_eq!(Some(1), len(r"(foo){1,}")); + assert_eq!(Some(1), len(r"(foo){1,}?")); + assert_eq!(None, len(r"(foo){1,}??")); + assert_eq!(None, len(r"(foo){0,}")); + assert_eq!(Some(1), len(r"(foo)(?:bar)")); + assert_eq!(Some(2), len(r"(foo(?:bar)+)(?:baz(boo))")); + assert_eq!(Some(2), len(r"(?P<bar>foo)(?:bar)(bal|loon)")); + assert_eq!( + Some(2), + len(r#"<(a)[^>]+href="([^"]+)"|<(img)[^>]+src="([^"]+)""#) + ); + } + + #[test] + fn analysis_is_all_assertions() { + // Positive examples. + let p = props(r"\b"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"\B"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"^"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"$"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"\A"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"\z"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"$^\z\A\b\B"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"$|^|\z|\A|\b|\B"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"^$|$^"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + let p = props(r"((\b)+())*^"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(0)); + + // Negative examples. + let p = props(r"^a"); + assert!(!p.look_set().is_empty()); + assert_eq!(p.minimum_len(), Some(1)); + } + + #[test] + fn analysis_look_set_prefix_any() { + let p = props(r"(?-u)(?i:(?:\b|_)win(?:32|64|dows)?(?:\b|_))"); + assert!(p.look_set_prefix_any().contains(Look::WordAscii)); + } + + #[test] + fn analysis_is_anchored() { + let is_start = |p| props(p).look_set_prefix().contains(Look::Start); + let is_end = |p| props(p).look_set_suffix().contains(Look::End); + + // Positive examples. + assert!(is_start(r"^")); + assert!(is_end(r"$")); + + assert!(is_start(r"^^")); + assert!(props(r"$$").look_set_suffix().contains(Look::End)); + + assert!(is_start(r"^$")); + assert!(is_end(r"^$")); + + assert!(is_start(r"^foo")); + assert!(is_end(r"foo$")); + + assert!(is_start(r"^foo|^bar")); + assert!(is_end(r"foo$|bar$")); + + assert!(is_start(r"^(foo|bar)")); + assert!(is_end(r"(foo|bar)$")); + + assert!(is_start(r"^+")); + assert!(is_end(r"$+")); + assert!(is_start(r"^++")); + assert!(is_end(r"$++")); + assert!(is_start(r"(^)+")); + assert!(is_end(r"($)+")); + + assert!(is_start(r"$^")); + assert!(is_start(r"$^")); + assert!(is_start(r"$^|^$")); + assert!(is_end(r"$^|^$")); + + assert!(is_start(r"\b^")); + assert!(is_end(r"$\b")); + assert!(is_start(r"^(?m:^)")); + assert!(is_end(r"(?m:$)$")); + assert!(is_start(r"(?m:^)^")); + assert!(is_end(r"$(?m:$)")); + + // Negative examples. + assert!(!is_start(r"(?m)^")); + assert!(!is_end(r"(?m)$")); + assert!(!is_start(r"(?m:^$)|$^")); + assert!(!is_end(r"(?m:^$)|$^")); + assert!(!is_start(r"$^|(?m:^$)")); + assert!(!is_end(r"$^|(?m:^$)")); + + assert!(!is_start(r"a^")); + assert!(!is_start(r"$a")); + + assert!(!is_end(r"a^")); + assert!(!is_end(r"$a")); + + assert!(!is_start(r"^foo|bar")); + assert!(!is_end(r"foo|bar$")); + + assert!(!is_start(r"^*")); + assert!(!is_end(r"$*")); + assert!(!is_start(r"^*+")); + assert!(!is_end(r"$*+")); + assert!(!is_start(r"^+*")); + assert!(!is_end(r"$+*")); + assert!(!is_start(r"(^)*")); + assert!(!is_end(r"($)*")); + } + + #[test] + fn analysis_is_any_anchored() { + let is_start = |p| props(p).look_set().contains(Look::Start); + let is_end = |p| props(p).look_set().contains(Look::End); + + // Positive examples. + assert!(is_start(r"^")); + assert!(is_end(r"$")); + assert!(is_start(r"\A")); + assert!(is_end(r"\z")); + + // Negative examples. + assert!(!is_start(r"(?m)^")); + assert!(!is_end(r"(?m)$")); + assert!(!is_start(r"$")); + assert!(!is_end(r"^")); + } + + #[test] + fn analysis_can_empty() { + // Positive examples. + let assert_empty = + |p| assert_eq!(Some(0), props_bytes(p).minimum_len()); + assert_empty(r""); + assert_empty(r"()"); + assert_empty(r"()*"); + assert_empty(r"()+"); + assert_empty(r"()?"); + assert_empty(r"a*"); + assert_empty(r"a?"); + assert_empty(r"a{0}"); + assert_empty(r"a{0,}"); + assert_empty(r"a{0,1}"); + assert_empty(r"a{0,10}"); + #[cfg(feature = "unicode-gencat")] + assert_empty(r"\pL*"); + assert_empty(r"a*|b"); + assert_empty(r"b|a*"); + assert_empty(r"a|"); + assert_empty(r"|a"); + assert_empty(r"a||b"); + assert_empty(r"a*a?(abcd)*"); + assert_empty(r"^"); + assert_empty(r"$"); + assert_empty(r"(?m)^"); + assert_empty(r"(?m)$"); + assert_empty(r"\A"); + assert_empty(r"\z"); + assert_empty(r"\B"); + assert_empty(r"(?-u)\B"); + assert_empty(r"\b"); + assert_empty(r"(?-u)\b"); + + // Negative examples. + let assert_non_empty = + |p| assert_ne!(Some(0), props_bytes(p).minimum_len()); + assert_non_empty(r"a+"); + assert_non_empty(r"a{1}"); + assert_non_empty(r"a{1,}"); + assert_non_empty(r"a{1,2}"); + assert_non_empty(r"a{1,10}"); + assert_non_empty(r"b|a"); + assert_non_empty(r"a*a+(abcd)*"); + #[cfg(feature = "unicode-gencat")] + assert_non_empty(r"\P{any}"); + assert_non_empty(r"[a--a]"); + assert_non_empty(r"[a&&b]"); + } + + #[test] + fn analysis_is_literal() { + // Positive examples. + assert!(props(r"a").is_literal()); + assert!(props(r"ab").is_literal()); + assert!(props(r"abc").is_literal()); + assert!(props(r"(?m)abc").is_literal()); + assert!(props(r"(?:a)").is_literal()); + assert!(props(r"foo(?:a)").is_literal()); + assert!(props(r"(?:a)foo").is_literal()); + assert!(props(r"[a]").is_literal()); + + // Negative examples. + assert!(!props(r"").is_literal()); + assert!(!props(r"^").is_literal()); + assert!(!props(r"a|b").is_literal()); + assert!(!props(r"(a)").is_literal()); + assert!(!props(r"a+").is_literal()); + assert!(!props(r"foo(a)").is_literal()); + assert!(!props(r"(a)foo").is_literal()); + assert!(!props(r"[ab]").is_literal()); + } + + #[test] + fn analysis_is_alternation_literal() { + // Positive examples. + assert!(props(r"a").is_alternation_literal()); + assert!(props(r"ab").is_alternation_literal()); + assert!(props(r"abc").is_alternation_literal()); + assert!(props(r"(?m)abc").is_alternation_literal()); + assert!(props(r"foo|bar").is_alternation_literal()); + assert!(props(r"foo|bar|baz").is_alternation_literal()); + assert!(props(r"[a]").is_alternation_literal()); + assert!(props(r"(?:ab)|cd").is_alternation_literal()); + assert!(props(r"ab|(?:cd)").is_alternation_literal()); + + // Negative examples. + assert!(!props(r"").is_alternation_literal()); + assert!(!props(r"^").is_alternation_literal()); + assert!(!props(r"(a)").is_alternation_literal()); + assert!(!props(r"a+").is_alternation_literal()); + assert!(!props(r"foo(a)").is_alternation_literal()); + assert!(!props(r"(a)foo").is_alternation_literal()); + assert!(!props(r"[ab]").is_alternation_literal()); + assert!(!props(r"[ab]|b").is_alternation_literal()); + assert!(!props(r"a|[ab]").is_alternation_literal()); + assert!(!props(r"(a)|b").is_alternation_literal()); + assert!(!props(r"a|(b)").is_alternation_literal()); + assert!(!props(r"a|b").is_alternation_literal()); + assert!(!props(r"a|b|c").is_alternation_literal()); + assert!(!props(r"[a]|b").is_alternation_literal()); + assert!(!props(r"a|[b]").is_alternation_literal()); + assert!(!props(r"(?:a)|b").is_alternation_literal()); + assert!(!props(r"a|(?:b)").is_alternation_literal()); + assert!(!props(r"(?:z|xx)@|xx").is_alternation_literal()); + } + + // This tests that the smart Hir::repetition constructors does some basic + // simplifications. + #[test] + fn smart_repetition() { + assert_eq!(t(r"a{0}"), Hir::empty()); + assert_eq!(t(r"a{1}"), hir_lit("a")); + assert_eq!(t(r"\B{32111}"), hir_look(hir::Look::WordUnicodeNegate)); + } + + // This tests that the smart Hir::concat constructor simplifies the given + // exprs in a way we expect. + #[test] + fn smart_concat() { + assert_eq!(t(""), Hir::empty()); + assert_eq!(t("(?:)"), Hir::empty()); + assert_eq!(t("abc"), hir_lit("abc")); + assert_eq!(t("(?:foo)(?:bar)"), hir_lit("foobar")); + assert_eq!(t("quux(?:foo)(?:bar)baz"), hir_lit("quuxfoobarbaz")); + assert_eq!( + t("foo(?:bar^baz)quux"), + hir_cat(vec![ + hir_lit("foobar"), + hir_look(hir::Look::Start), + hir_lit("bazquux"), + ]) + ); + assert_eq!( + t("foo(?:ba(?:r^b)az)quux"), + hir_cat(vec![ + hir_lit("foobar"), + hir_look(hir::Look::Start), + hir_lit("bazquux"), + ]) + ); + } + + // This tests that the smart Hir::alternation constructor simplifies the + // given exprs in a way we expect. + #[test] + fn smart_alternation() { + assert_eq!( + t("(?:foo)|(?:bar)"), + hir_alt(vec![hir_lit("foo"), hir_lit("bar")]) + ); + assert_eq!( + t("quux|(?:abc|def|xyz)|baz"), + hir_alt(vec![ + hir_lit("quux"), + hir_lit("abc"), + hir_lit("def"), + hir_lit("xyz"), + hir_lit("baz"), + ]) + ); + assert_eq!( + t("quux|(?:abc|(?:def|mno)|xyz)|baz"), + hir_alt(vec![ + hir_lit("quux"), + hir_lit("abc"), + hir_lit("def"), + hir_lit("mno"), + hir_lit("xyz"), + hir_lit("baz"), + ]) + ); + assert_eq!( + t("a|b|c|d|e|f|x|y|z"), + hir_uclass(&[('a', 'f'), ('x', 'z')]), + ); + // Tests that we lift common prefixes out of an alternation. + assert_eq!( + t("[A-Z]foo|[A-Z]quux"), + hir_cat(vec![ + hir_uclass(&[('A', 'Z')]), + hir_alt(vec![hir_lit("foo"), hir_lit("quux")]), + ]), + ); + assert_eq!( + t("[A-Z][A-Z]|[A-Z]quux"), + hir_cat(vec![ + hir_uclass(&[('A', 'Z')]), + hir_alt(vec![hir_uclass(&[('A', 'Z')]), hir_lit("quux")]), + ]), + ); + assert_eq!( + t("[A-Z][A-Z]|[A-Z][A-Z]quux"), + hir_cat(vec![ + hir_uclass(&[('A', 'Z')]), + hir_uclass(&[('A', 'Z')]), + hir_alt(vec![Hir::empty(), hir_lit("quux")]), + ]), + ); + assert_eq!( + t("[A-Z]foo|[A-Z]foobar"), + hir_cat(vec![ + hir_uclass(&[('A', 'Z')]), + hir_alt(vec![hir_lit("foo"), hir_lit("foobar")]), + ]), + ); + } + + #[test] + fn regression_alt_empty_concat() { + use crate::ast::{self, Ast}; + + let span = Span::splat(Position::new(0, 0, 0)); + let ast = Ast::alternation(ast::Alternation { + span, + asts: vec![Ast::concat(ast::Concat { span, asts: vec![] })], + }); + + let mut t = Translator::new(); + assert_eq!(Ok(Hir::empty()), t.translate("", &ast)); + } + + #[test] + fn regression_empty_alt() { + use crate::ast::{self, Ast}; + + let span = Span::splat(Position::new(0, 0, 0)); + let ast = Ast::concat(ast::Concat { + span, + asts: vec![Ast::alternation(ast::Alternation { + span, + asts: vec![], + })], + }); + + let mut t = Translator::new(); + assert_eq!(Ok(Hir::fail()), t.translate("", &ast)); + } + + #[test] + fn regression_singleton_alt() { + use crate::{ + ast::{self, Ast}, + hir::Dot, + }; + + let span = Span::splat(Position::new(0, 0, 0)); + let ast = Ast::concat(ast::Concat { + span, + asts: vec![Ast::alternation(ast::Alternation { + span, + asts: vec![Ast::dot(span)], + })], + }); + + let mut t = Translator::new(); + assert_eq!(Ok(Hir::dot(Dot::AnyCharExceptLF)), t.translate("", &ast)); + } + + // See: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=63168 + #[test] + fn regression_fuzz_match() { + let pat = "[(\u{6} \0-\u{afdf5}] \0 "; + let ast = ParserBuilder::new() + .octal(false) + .ignore_whitespace(true) + .build() + .parse(pat) + .unwrap(); + let hir = TranslatorBuilder::new() + .utf8(true) + .case_insensitive(false) + .multi_line(false) + .dot_matches_new_line(false) + .swap_greed(true) + .unicode(true) + .build() + .translate(pat, &ast) + .unwrap(); + assert_eq!( + hir, + Hir::concat(vec![ + hir_uclass(&[('\0', '\u{afdf5}')]), + hir_lit("\0"), + ]) + ); + } + + // See: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=63155 + #[cfg(feature = "unicode")] + #[test] + fn regression_fuzz_difference1() { + let pat = r"\W\W|\W[^\v--\W\W\P{Script_Extensions:Pau_Cin_Hau}\u10A1A1-\U{3E3E3}--~~~~--~~~~~~~~------~~~~~~--~~~~~~]*"; + let _ = t(pat); // shouldn't panic + } + + // See: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=63153 + #[test] + fn regression_fuzz_char_decrement1() { + let pat = "w[w[^w?\rw\rw[^w?\rw[^w?\rw[^w?\rw[^w?\rw[^w?\rw[^w?\r\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0w?\rw[^w?\rw[^w?\rw[^w\0\0\u{1}\0]\0\0-*\0]\0\0\0\0\0\0\u{1}\0]\0\0-*\0]\0\0\0\0\0\u{1}\0]\0\0\0\0\0\0\0\0\0*\0\0\u{1}\0]\0\0-*\0][^w?\rw[^w?\rw[^w?\rw[^w?\rw[^w?\rw[^w?\rw[^w\0\0\u{1}\0]\0\0-*\0]\0\0\0\0\0\0\u{1}\0]\0\0-*\0]\0\0\0\0\0\u{1}\0]\0\0\0\0\0\0\0\0\0x\0\0\u{1}\0]\0\0-*\0]\0\0\0\0\0\0\0\0\0*??\0\u{7f}{2}\u{10}??\0\0\0\0\0\0\0\0\0\u{3}\0\0\0}\0-*\0]\0\0\0\0\0\0\u{1}\0]\0\0-*\0]\0\0\0\0\0\0\u{1}\0]\0\0-*\0]\0\0\0\0\0\u{1}\0]\0\0-*\0]\0\0\0\0\0\0\0\u{1}\0]\0\u{1}\u{1}H-i]-]\0\0\0\0\u{1}\0]\0\0\0\u{1}\0]\0\0-*\0\0\0\0\u{1}9-\u{7f}]\0'|-\u{7f}]\0'|(?i-ux)[-\u{7f}]\0'\u{3}\0\0\0}\0-*\0]<D\0\0\0\0\0\0\u{1}]\0\0\0\0]\0\0-*\0]\0\0 "; + let _ = t(pat); // shouldn't panic + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/hir/visitor.rs b/third_party/crates/regex-syntax-0.8.11/src/hir/visitor.rs new file mode 100644 index 0000000..f30f0a1 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/hir/visitor.rs @@ -0,0 +1,215 @@ +use alloc::{vec, vec::Vec}; + +use crate::hir::{self, Hir, HirKind}; + +/// A trait for visiting the high-level IR (HIR) in depth first order. +/// +/// The principle aim of this trait is to enable callers to perform case +/// analysis on a high-level intermediate representation of a regular +/// expression without necessarily using recursion. In particular, this permits +/// callers to do case analysis with constant stack usage, which can be +/// important since the size of an HIR may be proportional to end user input. +/// +/// Typical usage of this trait involves providing an implementation and then +/// running it using the [`visit`] function. +pub trait Visitor { + /// The result of visiting an HIR. + type Output; + /// An error that visiting an HIR might return. + type Err; + + /// All implementors of `Visitor` must provide a `finish` method, which + /// yields the result of visiting the HIR or an error. + fn finish(self) -> Result<Self::Output, Self::Err>; + + /// This method is called before beginning traversal of the HIR. + fn start(&mut self) {} + + /// This method is called on an `Hir` before descending into child `Hir` + /// nodes. + fn visit_pre(&mut self, _hir: &Hir) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called on an `Hir` after descending all of its child + /// `Hir` nodes. + fn visit_post(&mut self, _hir: &Hir) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called between child nodes of an alternation. + fn visit_alternation_in(&mut self) -> Result<(), Self::Err> { + Ok(()) + } + + /// This method is called between child nodes of a concatenation. + fn visit_concat_in(&mut self) -> Result<(), Self::Err> { + Ok(()) + } +} + +/// Executes an implementation of `Visitor` in constant stack space. +/// +/// This function will visit every node in the given `Hir` while calling +/// appropriate methods provided by the [`Visitor`] trait. +/// +/// The primary use case for this method is when one wants to perform case +/// analysis over an `Hir` without using a stack size proportional to the depth +/// of the `Hir`. Namely, this method will instead use constant stack space, +/// but will use heap space proportional to the size of the `Hir`. This may be +/// desirable in cases where the size of `Hir` is proportional to end user +/// input. +/// +/// If the visitor returns an error at any point, then visiting is stopped and +/// the error is returned. +pub fn visit<V: Visitor>(hir: &Hir, visitor: V) -> Result<V::Output, V::Err> { + HeapVisitor::new().visit(hir, visitor) +} + +/// HeapVisitor visits every item in an `Hir` recursively using constant stack +/// size and a heap size proportional to the size of the `Hir`. +struct HeapVisitor<'a> { + /// A stack of `Hir` nodes. This is roughly analogous to the call stack + /// used in a typical recursive visitor. + stack: Vec<(&'a Hir, Frame<'a>)>, +} + +/// Represents a single stack frame while performing structural induction over +/// an `Hir`. +enum Frame<'a> { + /// A stack frame allocated just before descending into a repetition + /// operator's child node. + Repetition(&'a hir::Repetition), + /// A stack frame allocated just before descending into a capture's child + /// node. + Capture(&'a hir::Capture), + /// The stack frame used while visiting every child node of a concatenation + /// of expressions. + Concat { + /// The child node we are currently visiting. + head: &'a Hir, + /// The remaining child nodes to visit (which may be empty). + tail: &'a [Hir], + }, + /// The stack frame used while visiting every child node of an alternation + /// of expressions. + Alternation { + /// The child node we are currently visiting. + head: &'a Hir, + /// The remaining child nodes to visit (which may be empty). + tail: &'a [Hir], + }, +} + +impl<'a> HeapVisitor<'a> { + fn new() -> HeapVisitor<'a> { + HeapVisitor { stack: vec![] } + } + + fn visit<V: Visitor>( + &mut self, + mut hir: &'a Hir, + mut visitor: V, + ) -> Result<V::Output, V::Err> { + self.stack.clear(); + + visitor.start(); + loop { + visitor.visit_pre(hir)?; + if let Some(x) = self.induct(hir) { + let child = x.child(); + self.stack.push((hir, x)); + hir = child; + continue; + } + // No induction means we have a base case, so we can post visit + // it now. + visitor.visit_post(hir)?; + + // At this point, we now try to pop our call stack until it is + // either empty or we hit another inductive case. + loop { + let (post_hir, frame) = match self.stack.pop() { + None => return visitor.finish(), + Some((post_hir, frame)) => (post_hir, frame), + }; + // If this is a concat/alternate, then we might have additional + // inductive steps to process. + if let Some(x) = self.pop(frame) { + match x { + Frame::Alternation { .. } => { + visitor.visit_alternation_in()?; + } + Frame::Concat { .. } => { + visitor.visit_concat_in()?; + } + _ => {} + } + hir = x.child(); + self.stack.push((post_hir, x)); + break; + } + // Otherwise, we've finished visiting all the child nodes for + // this HIR, so we can post visit it now. + visitor.visit_post(post_hir)?; + } + } + } + + /// Build a stack frame for the given HIR if one is needed (which occurs if + /// and only if there are child nodes in the HIR). Otherwise, return None. + fn induct(&mut self, hir: &'a Hir) -> Option<Frame<'a>> { + match *hir.kind() { + HirKind::Repetition(ref x) => Some(Frame::Repetition(x)), + HirKind::Capture(ref x) => Some(Frame::Capture(x)), + HirKind::Concat(ref x) if x.is_empty() => None, + HirKind::Concat(ref x) => { + Some(Frame::Concat { head: &x[0], tail: &x[1..] }) + } + HirKind::Alternation(ref x) if x.is_empty() => None, + HirKind::Alternation(ref x) => { + Some(Frame::Alternation { head: &x[0], tail: &x[1..] }) + } + _ => None, + } + } + + /// Pops the given frame. If the frame has an additional inductive step, + /// then return it, otherwise return `None`. + fn pop(&self, induct: Frame<'a>) -> Option<Frame<'a>> { + match induct { + Frame::Repetition(_) => None, + Frame::Capture(_) => None, + Frame::Concat { tail, .. } => { + if tail.is_empty() { + None + } else { + Some(Frame::Concat { head: &tail[0], tail: &tail[1..] }) + } + } + Frame::Alternation { tail, .. } => { + if tail.is_empty() { + None + } else { + Some(Frame::Alternation { + head: &tail[0], + tail: &tail[1..], + }) + } + } + } + } +} + +impl<'a> Frame<'a> { + /// Perform the next inductive step on this frame and return the next + /// child HIR node to visit. + fn child(&self) -> &'a Hir { + match *self { + Frame::Repetition(rep) => &rep.sub, + Frame::Capture(capture) => &capture.sub, + Frame::Concat { head, .. } => head, + Frame::Alternation { head, .. } => head, + } + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/lib.rs b/third_party/crates/regex-syntax-0.8.11/src/lib.rs new file mode 100644 index 0000000..a4512e2 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/lib.rs @@ -0,0 +1,433 @@ +/*! +This crate provides a robust regular expression parser. + +This crate defines two primary types: + +* [`Ast`](ast::Ast) is the abstract syntax of a regular expression. + An abstract syntax corresponds to a *structured representation* of the + concrete syntax of a regular expression, where the concrete syntax is the + pattern string itself (e.g., `foo(bar)+`). Given some abstract syntax, it + can be converted back to the original concrete syntax (modulo some details, + like whitespace). To a first approximation, the abstract syntax is complex + and difficult to analyze. +* [`Hir`](hir::Hir) is the high-level intermediate representation + ("HIR" or "high-level IR" for short) of regular expression. It corresponds to + an intermediate state of a regular expression that sits between the abstract + syntax and the low level compiled opcodes that are eventually responsible for + executing a regular expression search. Given some high-level IR, it is not + possible to produce the original concrete syntax (although it is possible to + produce an equivalent concrete syntax, but it will likely scarcely resemble + the original pattern). To a first approximation, the high-level IR is simple + and easy to analyze. + +These two types come with conversion routines: + +* An [`ast::parse::Parser`] converts concrete syntax (a `&str`) to an +[`Ast`](ast::Ast). +* A [`hir::translate::Translator`] converts an [`Ast`](ast::Ast) to a +[`Hir`](hir::Hir). + +As a convenience, the above two conversion routines are combined into one via +the top-level [`Parser`] type. This `Parser` will first convert your pattern to +an `Ast` and then convert the `Ast` to an `Hir`. It's also exposed as top-level +[`parse`] free function. + + +# Example + +This example shows how to parse a pattern string into its HIR: + +``` +use regex_syntax::{hir::Hir, parse}; + +let hir = parse("a|b")?; +assert_eq!(hir, Hir::alternation(vec![ + Hir::literal("a".as_bytes()), + Hir::literal("b".as_bytes()), +])); +# Ok::<(), Box<dyn std::error::Error>>(()) +``` + + +# Concrete syntax supported + +The concrete syntax is documented as part of the public API of the +[`regex` crate](https://docs.rs/regex/%2A/regex/#syntax). + + +# Input safety + +A key feature of this library is that it is safe to use with end user facing +input. This plays a significant role in the internal implementation. In +particular: + +1. Parsers provide a `nest_limit` option that permits callers to control how + deeply nested a regular expression is allowed to be. This makes it possible + to do case analysis over an `Ast` or an `Hir` using recursion without + worrying about stack overflow. +2. Since relying on a particular stack size is brittle, this crate goes to + great lengths to ensure that all interactions with both the `Ast` and the + `Hir` do not use recursion. Namely, they use constant stack space and heap + space proportional to the size of the original pattern string (in bytes). + This includes the type's corresponding destructors. (One exception to this + is literal extraction, but this will eventually get fixed.) + + +# Error reporting + +The `Display` implementations on all `Error` types exposed in this library +provide nice human readable errors that are suitable for showing to end users +in a monospace font. + + +# Literal extraction + +This crate provides limited support for [literal extraction from `Hir` +values](hir::literal). Be warned that literal extraction uses recursion, and +therefore, stack size proportional to the size of the `Hir`. + +The purpose of literal extraction is to speed up searches. That is, if you +know a regular expression must match a prefix or suffix literal, then it is +often quicker to search for instances of that literal, and then confirm or deny +the match using the full regular expression engine. These optimizations are +done automatically in the `regex` crate. + + +# Crate features + +An important feature provided by this crate is its Unicode support. This +includes things like case folding, boolean properties, general categories, +scripts and Unicode-aware support for the Perl classes `\w`, `\s` and `\d`. +However, a downside of this support is that it requires bundling several +Unicode data tables that are substantial in size. + +A fair number of use cases do not require full Unicode support. For this +reason, this crate exposes a number of features to control which Unicode +data is available. + +If a regular expression attempts to use a Unicode feature that is not available +because the corresponding crate feature was disabled, then translating that +regular expression to an `Hir` will return an error. (It is still possible +construct an `Ast` for such a regular expression, since Unicode data is not +used until translation to an `Hir`.) Stated differently, enabling or disabling +any of the features below can only add or subtract from the total set of valid +regular expressions. Enabling or disabling a feature will never modify the +match semantics of a regular expression. + +The following features are available: + +* **std** - + Enables support for the standard library. This feature is enabled by default. + When disabled, only `core` and `alloc` are used. Otherwise, enabling `std` + generally just enables `std::error::Error` trait impls for the various error + types. +* **unicode** - + Enables all Unicode features. This feature is enabled by default, and will + always cover all Unicode features, even if more are added in the future. +* **unicode-age** - + Provide the data for the + [Unicode `Age` property](https://www.unicode.org/reports/tr44/tr44-24.html#Character_Age). + This makes it possible to use classes like `\p{Age:6.0}` to refer to all + codepoints first introduced in Unicode 6.0 +* **unicode-bool** - + Provide the data for numerous Unicode boolean properties. The full list + is not included here, but contains properties like `Alphabetic`, `Emoji`, + `Lowercase`, `Math`, `Uppercase` and `White_Space`. +* **unicode-case** - + Provide the data for case insensitive matching using + [Unicode's "simple loose matches" specification](https://www.unicode.org/reports/tr18/#Simple_Loose_Matches). +* **unicode-gencat** - + Provide the data for + [Unicode general categories](https://www.unicode.org/reports/tr44/tr44-24.html#General_Category_Values). + This includes, but is not limited to, `Decimal_Number`, `Letter`, + `Math_Symbol`, `Number` and `Punctuation`. +* **unicode-perl** - + Provide the data for supporting the Unicode-aware Perl character classes, + corresponding to `\w`, `\s` and `\d`. This is also necessary for using + Unicode-aware word boundary assertions. Note that if this feature is + disabled, the `\s` and `\d` character classes are still available if the + `unicode-bool` and `unicode-gencat` features are enabled, respectively. +* **unicode-script** - + Provide the data for + [Unicode scripts and script extensions](https://www.unicode.org/reports/tr24/). + This includes, but is not limited to, `Arabic`, `Cyrillic`, `Hebrew`, + `Latin` and `Thai`. +* **unicode-segment** - + Provide the data necessary to provide the properties used to implement the + [Unicode text segmentation algorithms](https://www.unicode.org/reports/tr29/). + This enables using classes like `\p{gcb=Extend}`, `\p{wb=Katakana}` and + `\p{sb=ATerm}`. +* **arbitrary** - + Enabling this feature introduces a public dependency on the + [`arbitrary`](https://crates.io/crates/arbitrary) + crate. Namely, it implements the `Arbitrary` trait from that crate for the + [`Ast`](crate::ast::Ast) type. This feature is disabled by default. +*/ + +#![no_std] +#![forbid(unsafe_code)] +#![deny(missing_docs, rustdoc::broken_intra_doc_links)] +#![warn(missing_debug_implementations)] +// This adds Cargo feature annotations to items in the rustdoc output. Which is +// sadly hugely beneficial for this crate due to the number of features. +#![cfg_attr(docsrs_regex, feature(doc_cfg))] + +#[cfg(any(test, feature = "std"))] +extern crate std; + +extern crate alloc; + +pub use crate::{ + error::Error, + parser::{parse, Parser, ParserBuilder}, + unicode::UnicodeWordError, +}; + +use alloc::string::String; + +pub mod ast; +mod debug; +mod either; +mod error; +pub mod hir; +mod parser; +mod rank; +mod unicode; +mod unicode_tables; +pub mod utf8; + +/// Escapes all regular expression meta characters in `text`. +/// +/// The string returned may be safely used as a literal in a regular +/// expression. +pub fn escape(text: &str) -> String { + let mut quoted = String::new(); + escape_into(text, &mut quoted); + quoted +} + +/// Escapes all meta characters in `text` and writes the result into `buf`. +/// +/// This will append escape characters into the given buffer. The characters +/// that are appended are safe to use as a literal in a regular expression. +pub fn escape_into(text: &str, buf: &mut String) { + buf.reserve(text.len()); + for c in text.chars() { + if is_meta_character(c) { + buf.push('\\'); + } + buf.push(c); + } +} + +/// Returns true if the given character has significance in a regex. +/// +/// Generally speaking, these are the only characters which _must_ be escaped +/// in order to match their literal meaning. For example, to match a literal +/// `|`, one could write `\|`. Sometimes escaping isn't always necessary. For +/// example, `-` is treated as a meta character because of its significance +/// for writing ranges inside of character classes, but the regex `-` will +/// match a literal `-` because `-` has no special meaning outside of character +/// classes. +/// +/// In order to determine whether a character may be escaped at all, the +/// [`is_escapeable_character`] routine should be used. The difference between +/// `is_meta_character` and `is_escapeable_character` is that the latter will +/// return true for some characters that are _not_ meta characters. For +/// example, `%` and `\%` both match a literal `%` in all contexts. In other +/// words, `is_escapeable_character` includes "superfluous" escapes. +/// +/// Note that the set of characters for which this function returns `true` or +/// `false` is fixed and won't change in a semver compatible release. (In this +/// case, "semver compatible release" actually refers to the `regex` crate +/// itself, since reducing or expanding the set of meta characters would be a +/// breaking change for not just `regex-syntax` but also `regex` itself.) +/// +/// # Example +/// +/// ``` +/// use regex_syntax::is_meta_character; +/// +/// assert!(is_meta_character('?')); +/// assert!(is_meta_character('-')); +/// assert!(is_meta_character('&')); +/// assert!(is_meta_character('#')); +/// +/// assert!(!is_meta_character('%')); +/// assert!(!is_meta_character('/')); +/// assert!(!is_meta_character('!')); +/// assert!(!is_meta_character('"')); +/// assert!(!is_meta_character('e')); +/// ``` +pub fn is_meta_character(c: char) -> bool { + match c { + '\\' | '.' | '+' | '*' | '?' | '(' | ')' | '|' | '[' | ']' | '{' + | '}' | '^' | '$' | '#' | '&' | '-' | '~' => true, + _ => false, + } +} + +/// Returns true if the given character can be escaped in a regex. +/// +/// This returns true in all cases that `is_meta_character` returns true, but +/// also returns true in some cases where `is_meta_character` returns false. +/// For example, `%` is not a meta character, but it is escapable. That is, +/// `%` and `\%` both match a literal `%` in all contexts. +/// +/// The purpose of this routine is to provide knowledge about what characters +/// may be escaped. Namely, most regex engines permit "superfluous" escapes +/// where characters without any special significance may be escaped even +/// though there is no actual _need_ to do so. +/// +/// This will return false for some characters. For example, `e` is not +/// escapable. Therefore, `\e` will either result in a parse error (which is +/// true today), or it could backwards compatibly evolve into a new construct +/// with its own meaning. Indeed, that is the purpose of banning _some_ +/// superfluous escapes: it provides a way to evolve the syntax in a compatible +/// manner. +/// +/// # Example +/// +/// ``` +/// use regex_syntax::is_escapeable_character; +/// +/// assert!(is_escapeable_character('?')); +/// assert!(is_escapeable_character('-')); +/// assert!(is_escapeable_character('&')); +/// assert!(is_escapeable_character('#')); +/// assert!(is_escapeable_character('%')); +/// assert!(is_escapeable_character('/')); +/// assert!(is_escapeable_character('!')); +/// assert!(is_escapeable_character('"')); +/// +/// assert!(!is_escapeable_character('e')); +/// ``` +pub fn is_escapeable_character(c: char) -> bool { + // Certainly escapable if it's a meta character. + if is_meta_character(c) { + return true; + } + // Any character that isn't ASCII is definitely not escapable. There's + // no real need to allow things like \☃ right? + if !c.is_ascii() { + return false; + } + // Otherwise, we basically say that everything is escapable unless it's a + // letter or digit. Things like \3 are either octal (when enabled) or an + // error, and we should keep it that way. Otherwise, letters are reserved + // for adding new syntax in a backwards compatible way. + match c { + '0'..='9' | 'A'..='Z' | 'a'..='z' => false, + // While not currently supported, we keep these as not escapable to + // give us some flexibility with respect to supporting the \< and + // \> word boundary assertions in the future. By rejecting them as + // escapable, \< and \> will result in a parse error. Thus, we can + // turn them into something else in the future without it being a + // backwards incompatible change. + // + // OK, now we support \< and \>, and we need to retain them as *not* + // escapable here since the escape sequence is significant. + '<' | '>' => false, + _ => true, + } +} + +/// Returns true if and only if the given character is a Unicode word +/// character. +/// +/// A Unicode word character is defined by +/// [UTS#18 Annex C](https://unicode.org/reports/tr18/#Compatibility_Properties). +/// In particular, a character +/// is considered a word character if it is in either of the `Alphabetic` or +/// `Join_Control` properties, or is in one of the `Decimal_Number`, `Mark` +/// or `Connector_Punctuation` general categories. +/// +/// # Panics +/// +/// If the `unicode-perl` feature is not enabled, then this function +/// panics. For this reason, it is recommended that callers use +/// [`try_is_word_character`] instead. +pub fn is_word_character(c: char) -> bool { + try_is_word_character(c).expect("unicode-perl feature must be enabled") +} + +/// Returns true if and only if the given character is a Unicode word +/// character. +/// +/// A Unicode word character is defined by +/// [UTS#18 Annex C](https://unicode.org/reports/tr18/#Compatibility_Properties). +/// In particular, a character +/// is considered a word character if it is in either of the `Alphabetic` or +/// `Join_Control` properties, or is in one of the `Decimal_Number`, `Mark` +/// or `Connector_Punctuation` general categories. +/// +/// # Errors +/// +/// If the `unicode-perl` feature is not enabled, then this function always +/// returns an error. +pub fn try_is_word_character( + c: char, +) -> core::result::Result<bool, UnicodeWordError> { + unicode::is_word_character(c) +} + +/// Returns true if and only if the given character is an ASCII word character. +/// +/// An ASCII word character is defined by the following character class: +/// `[_0-9a-zA-Z]`. +pub fn is_word_byte(c: u8) -> bool { + match c { + b'_' | b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' => true, + _ => false, + } +} + +#[cfg(test)] +mod tests { + use alloc::string::ToString; + + use super::*; + + #[test] + fn escape_meta() { + assert_eq!( + escape(r"\.+*?()|[]{}^$#&-~"), + r"\\\.\+\*\?\(\)\|\[\]\{\}\^\$\#\&\-\~".to_string() + ); + } + + #[test] + fn word_byte() { + assert!(is_word_byte(b'a')); + assert!(!is_word_byte(b'-')); + } + + #[test] + #[cfg(feature = "unicode-perl")] + fn word_char() { + assert!(is_word_character('a'), "ASCII"); + assert!(is_word_character('à'), "Latin-1"); + assert!(is_word_character('β'), "Greek"); + assert!(is_word_character('\u{11011}'), "Brahmi (Unicode 6.0)"); + assert!(is_word_character('\u{11611}'), "Modi (Unicode 7.0)"); + assert!(is_word_character('\u{11711}'), "Ahom (Unicode 8.0)"); + assert!(is_word_character('\u{17828}'), "Tangut (Unicode 9.0)"); + assert!(is_word_character('\u{1B1B1}'), "Nushu (Unicode 10.0)"); + assert!(is_word_character('\u{16E40}'), "Medefaidrin (Unicode 11.0)"); + assert!(!is_word_character('-')); + assert!(!is_word_character('☃')); + } + + #[test] + #[should_panic] + #[cfg(not(feature = "unicode-perl"))] + fn word_char_disabled_panic() { + assert!(is_word_character('a')); + } + + #[test] + #[cfg(not(feature = "unicode-perl"))] + fn word_char_disabled_error() { + assert!(try_is_word_character('a').is_err()); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/parser.rs b/third_party/crates/regex-syntax-0.8.11/src/parser.rs new file mode 100644 index 0000000..f482b84 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/parser.rs @@ -0,0 +1,254 @@ +use crate::{ast, hir, Error}; + +/// A convenience routine for parsing a regex using default options. +/// +/// This is equivalent to `Parser::new().parse(pattern)`. +/// +/// If you need to set non-default options, then use a [`ParserBuilder`]. +/// +/// This routine returns an [`Hir`](hir::Hir) value. Namely, it automatically +/// parses the pattern as an [`Ast`](ast::Ast) and then invokes the translator +/// to convert the `Ast` into an `Hir`. If you need access to the `Ast`, then +/// you should use a [`ast::parse::Parser`]. +pub fn parse(pattern: &str) -> Result<hir::Hir, Error> { + Parser::new().parse(pattern) +} + +/// A builder for a regular expression parser. +/// +/// This builder permits modifying configuration options for the parser. +/// +/// This type combines the builder options for both the [AST +/// `ParserBuilder`](ast::parse::ParserBuilder) and the [HIR +/// `TranslatorBuilder`](hir::translate::TranslatorBuilder). +#[derive(Clone, Debug, Default)] +pub struct ParserBuilder { + ast: ast::parse::ParserBuilder, + hir: hir::translate::TranslatorBuilder, +} + +impl ParserBuilder { + /// Create a new parser builder with a default configuration. + pub fn new() -> ParserBuilder { + ParserBuilder::default() + } + + /// Build a parser from this configuration with the given pattern. + pub fn build(&self) -> Parser { + Parser { ast: self.ast.build(), hir: self.hir.build() } + } + + /// Set the nesting limit for this parser. + /// + /// The nesting limit controls how deep the abstract syntax tree is allowed + /// to be. If the AST exceeds the given limit (e.g., with too many nested + /// groups), then an error is returned by the parser. + /// + /// The purpose of this limit is to act as a heuristic to prevent stack + /// overflow for consumers that do structural induction on an `Ast` using + /// explicit recursion. While this crate never does this (instead using + /// constant stack space and moving the call stack to the heap), other + /// crates may. + /// + /// This limit is not checked until the entire Ast is parsed. Therefore, + /// if callers want to put a limit on the amount of heap space used, then + /// they should impose a limit on the length, in bytes, of the concrete + /// pattern string. In particular, this is viable since this parser + /// implementation will limit itself to heap space proportional to the + /// length of the pattern string. + /// + /// Note that a nest limit of `0` will return a nest limit error for most + /// patterns but not all. For example, a nest limit of `0` permits `a` but + /// not `ab`, since `ab` requires a concatenation, which results in a nest + /// depth of `1`. In general, a nest limit is not something that manifests + /// in an obvious way in the concrete syntax, therefore, it should not be + /// used in a granular way. + pub fn nest_limit(&mut self, limit: u32) -> &mut ParserBuilder { + self.ast.nest_limit(limit); + self + } + + /// Whether to support octal syntax or not. + /// + /// Octal syntax is a little-known way of uttering Unicode codepoints in + /// a regular expression. For example, `a`, `\x61`, `\u0061` and + /// `\141` are all equivalent regular expressions, where the last example + /// shows octal syntax. + /// + /// While supporting octal syntax isn't in and of itself a problem, it does + /// make good error messages harder. That is, in PCRE based regex engines, + /// syntax like `\0` invokes a backreference, which is explicitly + /// unsupported in Rust's regex engine. However, many users expect it to + /// be supported. Therefore, when octal support is disabled, the error + /// message will explicitly mention that backreferences aren't supported. + /// + /// Octal syntax is disabled by default. + pub fn octal(&mut self, yes: bool) -> &mut ParserBuilder { + self.ast.octal(yes); + self + } + + /// When disabled, translation will permit the construction of a regular + /// expression that may match invalid UTF-8. + /// + /// When enabled (the default), the translator is guaranteed to produce an + /// expression that, for non-empty matches, will only ever produce spans + /// that are entirely valid UTF-8 (otherwise, the translator will return an + /// error). + /// + /// Perhaps surprisingly, when UTF-8 is enabled, an empty regex or even + /// a negated ASCII word boundary (uttered as `(?-u:\B)` in the concrete + /// syntax) will be allowed even though they can produce matches that split + /// a UTF-8 encoded codepoint. This only applies to zero-width or "empty" + /// matches, and it is expected that the regex engine itself must handle + /// these cases if necessary (perhaps by suppressing any zero-width matches + /// that split a codepoint). + pub fn utf8(&mut self, yes: bool) -> &mut ParserBuilder { + self.hir.utf8(yes); + self + } + + /// Enable verbose mode in the regular expression. + /// + /// When enabled, verbose mode permits insignificant whitespace in many + /// places in the regular expression, as well as comments. Comments are + /// started using `#` and continue until the end of the line. + /// + /// By default, this is disabled. It may be selectively enabled in the + /// regular expression by using the `x` flag regardless of this setting. + pub fn ignore_whitespace(&mut self, yes: bool) -> &mut ParserBuilder { + self.ast.ignore_whitespace(yes); + self + } + + /// Enable or disable the case insensitive flag by default. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `i` flag. + pub fn case_insensitive(&mut self, yes: bool) -> &mut ParserBuilder { + self.hir.case_insensitive(yes); + self + } + + /// Enable or disable the multi-line matching flag by default. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `m` flag. + pub fn multi_line(&mut self, yes: bool) -> &mut ParserBuilder { + self.hir.multi_line(yes); + self + } + + /// Enable or disable the "dot matches any character" flag by default. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `s` flag. + pub fn dot_matches_new_line(&mut self, yes: bool) -> &mut ParserBuilder { + self.hir.dot_matches_new_line(yes); + self + } + + /// Enable or disable the CRLF mode flag by default. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `R` flag. + /// + /// When CRLF mode is enabled, the following happens: + /// + /// * Unless `dot_matches_new_line` is enabled, `.` will match any character + /// except for `\r` and `\n`. + /// * When `multi_line` mode is enabled, `^` and `$` will treat `\r\n`, + /// `\r` and `\n` as line terminators. And in particular, neither will + /// match between a `\r` and a `\n`. + pub fn crlf(&mut self, yes: bool) -> &mut ParserBuilder { + self.hir.crlf(yes); + self + } + + /// Sets the line terminator for use with `(?u-s:.)` and `(?-us:.)`. + /// + /// Namely, instead of `.` (by default) matching everything except for `\n`, + /// this will cause `.` to match everything except for the byte given. + /// + /// If `.` is used in a context where Unicode mode is enabled and this byte + /// isn't ASCII, then an error will be returned. When Unicode mode is + /// disabled, then any byte is permitted, but will return an error if UTF-8 + /// mode is enabled and it is a non-ASCII byte. + /// + /// In short, any ASCII value for a line terminator is always okay. But a + /// non-ASCII byte might result in an error depending on whether Unicode + /// mode or UTF-8 mode are enabled. + /// + /// Note that if `R` mode is enabled then it always takes precedence and + /// the line terminator will be treated as `\r` and `\n` simultaneously. + /// + /// Note also that this *doesn't* impact the look-around assertions + /// `(?m:^)` and `(?m:$)`. That's usually controlled by additional + /// configuration in the regex engine itself. + pub fn line_terminator(&mut self, byte: u8) -> &mut ParserBuilder { + self.hir.line_terminator(byte); + self + } + + /// Enable or disable the "swap greed" flag by default. + /// + /// By default this is disabled. It may alternatively be selectively + /// enabled in the regular expression itself via the `U` flag. + pub fn swap_greed(&mut self, yes: bool) -> &mut ParserBuilder { + self.hir.swap_greed(yes); + self + } + + /// Enable or disable the Unicode flag (`u`) by default. + /// + /// By default this is **enabled**. It may alternatively be selectively + /// disabled in the regular expression itself via the `u` flag. + /// + /// Note that unless `utf8` is disabled (it's enabled by default), a + /// regular expression will fail to parse if Unicode mode is disabled and a + /// sub-expression could possibly match invalid UTF-8. + pub fn unicode(&mut self, yes: bool) -> &mut ParserBuilder { + self.hir.unicode(yes); + self + } +} + +/// A convenience parser for regular expressions. +/// +/// This parser takes as input a regular expression pattern string (the +/// "concrete syntax") and returns a high-level intermediate representation +/// (the HIR) suitable for most types of analysis. In particular, this parser +/// hides the intermediate state of producing an AST (the "abstract syntax"). +/// The AST is itself far more complex than the HIR, so this parser serves as a +/// convenience for never having to deal with it at all. +/// +/// If callers have more fine grained use cases that need an AST, then please +/// see the [`ast::parse`] module. +/// +/// A `Parser` can be configured in more detail via a [`ParserBuilder`]. +#[derive(Clone, Debug)] +pub struct Parser { + ast: ast::parse::Parser, + hir: hir::translate::Translator, +} + +impl Parser { + /// Create a new parser with a default configuration. + /// + /// The parser can be run with `parse` method. The parse method returns + /// a high level intermediate representation of the given regular + /// expression. + /// + /// To set configuration options on the parser, use [`ParserBuilder`]. + pub fn new() -> Parser { + ParserBuilder::new().build() + } + + /// Parse the regular expression into a high level intermediate + /// representation. + pub fn parse(&mut self, pattern: &str) -> Result<hir::Hir, Error> { + let ast = self.ast.parse(pattern)?; + let hir = self.hir.translate(pattern, &ast)?; + Ok(hir) + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/rank.rs b/third_party/crates/regex-syntax-0.8.11/src/rank.rs new file mode 100644 index 0000000..ccb25a2 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/rank.rs @@ -0,0 +1,258 @@ +pub(crate) const BYTE_FREQUENCIES: [u8; 256] = [ + 55, // '\x00' + 52, // '\x01' + 51, // '\x02' + 50, // '\x03' + 49, // '\x04' + 48, // '\x05' + 47, // '\x06' + 46, // '\x07' + 45, // '\x08' + 103, // '\t' + 242, // '\n' + 66, // '\x0b' + 67, // '\x0c' + 229, // '\r' + 44, // '\x0e' + 43, // '\x0f' + 42, // '\x10' + 41, // '\x11' + 40, // '\x12' + 39, // '\x13' + 38, // '\x14' + 37, // '\x15' + 36, // '\x16' + 35, // '\x17' + 34, // '\x18' + 33, // '\x19' + 56, // '\x1a' + 32, // '\x1b' + 31, // '\x1c' + 30, // '\x1d' + 29, // '\x1e' + 28, // '\x1f' + 255, // ' ' + 148, // '!' + 164, // '"' + 149, // '#' + 136, // '$' + 160, // '%' + 155, // '&' + 173, // "'" + 221, // '(' + 222, // ')' + 134, // '*' + 122, // '+' + 232, // ',' + 202, // '-' + 215, // '.' + 224, // '/' + 208, // '0' + 220, // '1' + 204, // '2' + 187, // '3' + 183, // '4' + 179, // '5' + 177, // '6' + 168, // '7' + 178, // '8' + 200, // '9' + 226, // ':' + 195, // ';' + 154, // '<' + 184, // '=' + 174, // '>' + 126, // '?' + 120, // '@' + 191, // 'A' + 157, // 'B' + 194, // 'C' + 170, // 'D' + 189, // 'E' + 162, // 'F' + 161, // 'G' + 150, // 'H' + 193, // 'I' + 142, // 'J' + 137, // 'K' + 171, // 'L' + 176, // 'M' + 185, // 'N' + 167, // 'O' + 186, // 'P' + 112, // 'Q' + 175, // 'R' + 192, // 'S' + 188, // 'T' + 156, // 'U' + 140, // 'V' + 143, // 'W' + 123, // 'X' + 133, // 'Y' + 128, // 'Z' + 147, // '[' + 138, // '\\' + 146, // ']' + 114, // '^' + 223, // '_' + 151, // '`' + 249, // 'a' + 216, // 'b' + 238, // 'c' + 236, // 'd' + 253, // 'e' + 227, // 'f' + 218, // 'g' + 230, // 'h' + 247, // 'i' + 135, // 'j' + 180, // 'k' + 241, // 'l' + 233, // 'm' + 246, // 'n' + 244, // 'o' + 231, // 'p' + 139, // 'q' + 245, // 'r' + 243, // 's' + 251, // 't' + 235, // 'u' + 201, // 'v' + 196, // 'w' + 240, // 'x' + 214, // 'y' + 152, // 'z' + 182, // '{' + 205, // '|' + 181, // '}' + 127, // '~' + 27, // '\x7f' + 212, // '\x80' + 211, // '\x81' + 210, // '\x82' + 213, // '\x83' + 228, // '\x84' + 197, // '\x85' + 169, // '\x86' + 159, // '\x87' + 131, // '\x88' + 172, // '\x89' + 105, // '\x8a' + 80, // '\x8b' + 98, // '\x8c' + 96, // '\x8d' + 97, // '\x8e' + 81, // '\x8f' + 207, // '\x90' + 145, // '\x91' + 116, // '\x92' + 115, // '\x93' + 144, // '\x94' + 130, // '\x95' + 153, // '\x96' + 121, // '\x97' + 107, // '\x98' + 132, // '\x99' + 109, // '\x9a' + 110, // '\x9b' + 124, // '\x9c' + 111, // '\x9d' + 82, // '\x9e' + 108, // '\x9f' + 118, // '\xa0' + 141, // '¡' + 113, // '¢' + 129, // '£' + 119, // '¤' + 125, // '¥' + 165, // '¦' + 117, // '§' + 92, // '¨' + 106, // '©' + 83, // 'ª' + 72, // '«' + 99, // '¬' + 93, // '\xad' + 65, // '®' + 79, // '¯' + 166, // '°' + 237, // '±' + 163, // '²' + 199, // '³' + 190, // '´' + 225, // 'µ' + 209, // '¶' + 203, // '·' + 198, // '¸' + 217, // '¹' + 219, // 'º' + 206, // '»' + 234, // '¼' + 248, // '½' + 158, // '¾' + 239, // '¿' + 255, // 'À' + 255, // 'Á' + 255, // 'Â' + 255, // 'Ã' + 255, // 'Ä' + 255, // 'Å' + 255, // 'Æ' + 255, // 'Ç' + 255, // 'È' + 255, // 'É' + 255, // 'Ê' + 255, // 'Ë' + 255, // 'Ì' + 255, // 'Í' + 255, // 'Î' + 255, // 'Ï' + 255, // 'Ð' + 255, // 'Ñ' + 255, // 'Ò' + 255, // 'Ó' + 255, // 'Ô' + 255, // 'Õ' + 255, // 'Ö' + 255, // '×' + 255, // 'Ø' + 255, // 'Ù' + 255, // 'Ú' + 255, // 'Û' + 255, // 'Ü' + 255, // 'Ý' + 255, // 'Þ' + 255, // 'ß' + 255, // 'à' + 255, // 'á' + 255, // 'â' + 255, // 'ã' + 255, // 'ä' + 255, // 'å' + 255, // 'æ' + 255, // 'ç' + 255, // 'è' + 255, // 'é' + 255, // 'ê' + 255, // 'ë' + 255, // 'ì' + 255, // 'í' + 255, // 'î' + 255, // 'ï' + 255, // 'ð' + 255, // 'ñ' + 255, // 'ò' + 255, // 'ó' + 255, // 'ô' + 255, // 'õ' + 255, // 'ö' + 255, // '÷' + 255, // 'ø' + 255, // 'ù' + 255, // 'ú' + 255, // 'û' + 255, // 'ü' + 255, // 'ý' + 255, // 'þ' + 255, // 'ÿ' +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode.rs new file mode 100644 index 0000000..07f7819 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode.rs @@ -0,0 +1,1041 @@ +use alloc::{ + string::{String, ToString}, + vec::Vec, +}; + +use crate::hir; + +/// An inclusive range of codepoints from a generated file (hence the static +/// lifetime). +type Range = &'static [(char, char)]; + +/// An error that occurs when dealing with Unicode. +/// +/// We don't impl the Error trait here because these always get converted +/// into other public errors. (This error type isn't exported.) +#[derive(Debug)] +pub enum Error { + PropertyNotFound, + PropertyValueNotFound, + // Not used when unicode-perl is enabled. + #[allow(dead_code)] + PerlClassNotFound, +} + +/// An error that occurs when Unicode-aware simple case folding fails. +/// +/// This error can occur when the case mapping tables necessary for Unicode +/// aware case folding are unavailable. This only occurs when the +/// `unicode-case` feature is disabled. (The feature is enabled by default.) +#[derive(Debug)] +pub struct CaseFoldError(()); + +#[cfg(feature = "std")] +impl std::error::Error for CaseFoldError {} + +impl core::fmt::Display for CaseFoldError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "Unicode-aware case folding is not available \ + (probably because the unicode-case feature is not enabled)" + ) + } +} + +/// An error that occurs when the Unicode-aware `\w` class is unavailable. +/// +/// This error can occur when the data tables necessary for the Unicode aware +/// Perl character class `\w` are unavailable. This only occurs when the +/// `unicode-perl` feature is disabled. (The feature is enabled by default.) +#[derive(Debug)] +pub struct UnicodeWordError(()); + +#[cfg(feature = "std")] +impl std::error::Error for UnicodeWordError {} + +impl core::fmt::Display for UnicodeWordError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "Unicode-aware \\w class is not available \ + (probably because the unicode-perl feature is not enabled)" + ) + } +} + +/// A state oriented traverser of the simple case folding table. +/// +/// A case folder can be constructed via `SimpleCaseFolder::new()`, which will +/// return an error if the underlying case folding table is unavailable. +/// +/// After construction, it is expected that callers will use +/// `SimpleCaseFolder::mapping` by calling it with codepoints in strictly +/// increasing order. For example, calling it on `b` and then on `a` is illegal +/// and will result in a panic. +/// +/// The main idea of this type is that it tries hard to make mapping lookups +/// fast by exploiting the structure of the underlying table, and the ordering +/// assumption enables this. +#[derive(Debug)] +pub struct SimpleCaseFolder { + /// The simple case fold table. It's a sorted association list, where the + /// keys are Unicode scalar values and the values are the corresponding + /// equivalence class (not including the key) of the "simple" case folded + /// Unicode scalar values. + table: &'static [(char, &'static [char])], + /// The last codepoint that was used for a lookup. + last: Option<char>, + /// The index to the entry in `table` corresponding to the smallest key `k` + /// such that `k > k0`, where `k0` is the most recent key lookup. Note that + /// in particular, `k0` may not be in the table! + next: usize, +} + +impl SimpleCaseFolder { + /// Create a new simple case folder, returning an error if the underlying + /// case folding table is unavailable. + pub fn new() -> Result<SimpleCaseFolder, CaseFoldError> { + #[cfg(not(feature = "unicode-case"))] + { + Err(CaseFoldError(())) + } + #[cfg(feature = "unicode-case")] + { + Ok(SimpleCaseFolder { + table: crate::unicode_tables::case_folding_simple::CASE_FOLDING_SIMPLE, + last: None, + next: 0, + }) + } + } + + /// Return the equivalence class of case folded codepoints for the given + /// codepoint. The equivalence class returned never includes the codepoint + /// given. If the given codepoint has no case folded codepoints (i.e., + /// no entry in the underlying case folding table), then this returns an + /// empty slice. + /// + /// # Panics + /// + /// This panics when called with a `c` that is less than or equal to the + /// previous call. In other words, callers need to use this method with + /// strictly increasing values of `c`. + pub fn mapping(&mut self, c: char) -> &'static [char] { + if let Some(last) = self.last { + assert!( + last < c, + "got codepoint U+{:X} which occurs before \ + last codepoint U+{:X}", + u32::from(c), + u32::from(last), + ); + } + self.last = Some(c); + if self.next >= self.table.len() { + return &[]; + } + let (k, v) = self.table[self.next]; + if k == c { + self.next += 1; + return v; + } + match self.get(c) { + Err(i) => { + self.next = i; + &[] + } + Ok(i) => { + // Since we require lookups to proceed + // in order, anything we find should be + // after whatever we thought might be + // next. Otherwise, the caller is either + // going out of order or we would have + // found our next key at 'self.next'. + assert!(i > self.next); + self.next = i + 1; + self.table[i].1 + } + } + } + + /// Returns true if and only if the given range overlaps with any region + /// of the underlying case folding table. That is, when true, there exists + /// at least one codepoint in the inclusive range `[start, end]` that has + /// a non-trivial equivalence class of case folded codepoints. Conversely, + /// when this returns false, all codepoints in the range `[start, end]` + /// correspond to the trivial equivalence class of case folded codepoints, + /// i.e., itself. + /// + /// This is useful to call before iterating over the codepoints in the + /// range and looking up the mapping for each. If you know none of the + /// mappings will return anything, then you might be able to skip doing it + /// altogether. + /// + /// # Panics + /// + /// This panics when `end < start`. + pub fn overlaps(&self, start: char, end: char) -> bool { + use core::cmp::Ordering; + + assert!(start <= end); + self.table + .binary_search_by(|&(c, _)| { + if start <= c && c <= end { + Ordering::Equal + } else if c > end { + Ordering::Greater + } else { + Ordering::Less + } + }) + .is_ok() + } + + /// Returns the index at which `c` occurs in the simple case fold table. If + /// `c` does not occur, then this returns an `i` such that `table[i-1].0 < + /// c` and `table[i].0 > c`. + fn get(&self, c: char) -> Result<usize, usize> { + self.table.binary_search_by_key(&c, |&(c1, _)| c1) + } +} + +/// A query for finding a character class defined by Unicode. This supports +/// either use of a property name directly, or lookup by property value. The +/// former generally refers to Binary properties (see UTS#44, Table 8), but +/// as a special exception (see UTS#18, Section 1.2) both general categories +/// (an enumeration) and scripts (a catalog) are supported as if each of their +/// possible values were a binary property. +/// +/// In all circumstances, property names and values are normalized and +/// canonicalized. That is, `GC == gc == GeneralCategory == general_category`. +/// +/// The lifetime `'a` refers to the shorter of the lifetimes of property name +/// and property value. +#[derive(Debug)] +pub enum ClassQuery<'a> { + /// Return a class corresponding to a Unicode binary property, named by + /// a single letter. + OneLetter(char), + /// Return a class corresponding to a Unicode binary property. + /// + /// Note that, by special exception (see UTS#18, Section 1.2), both + /// general category values and script values are permitted here as if + /// they were a binary property. + Binary(&'a str), + /// Return a class corresponding to all codepoints whose property + /// (identified by `property_name`) corresponds to the given value + /// (identified by `property_value`). + ByValue { + /// A property name. + property_name: &'a str, + /// A property value. + property_value: &'a str, + }, +} + +impl<'a> ClassQuery<'a> { + fn canonicalize(&self) -> Result<CanonicalClassQuery, Error> { + match *self { + ClassQuery::OneLetter(c) => self.canonical_binary(&c.to_string()), + ClassQuery::Binary(name) => self.canonical_binary(name), + ClassQuery::ByValue { property_name, property_value } => { + let property_name = symbolic_name_normalize(property_name); + let property_value = symbolic_name_normalize(property_value); + + let canon_name = match canonical_prop(&property_name)? { + None => return Err(Error::PropertyNotFound), + Some(canon_name) => canon_name, + }; + Ok(match canon_name { + "General_Category" => { + let canon = match canonical_gencat(&property_value)? { + None => return Err(Error::PropertyValueNotFound), + Some(canon) => canon, + }; + CanonicalClassQuery::GeneralCategory(canon) + } + "Script" => { + let canon = match canonical_script(&property_value)? { + None => return Err(Error::PropertyValueNotFound), + Some(canon) => canon, + }; + CanonicalClassQuery::Script(canon) + } + _ => { + let vals = match property_values(canon_name)? { + None => return Err(Error::PropertyValueNotFound), + Some(vals) => vals, + }; + let canon_val = + match canonical_value(vals, &property_value) { + None => { + return Err(Error::PropertyValueNotFound) + } + Some(canon_val) => canon_val, + }; + CanonicalClassQuery::ByValue { + property_name: canon_name, + property_value: canon_val, + } + } + }) + } + } + } + + fn canonical_binary( + &self, + name: &str, + ) -> Result<CanonicalClassQuery, Error> { + let norm = symbolic_name_normalize(name); + + // This is a special case where 'cf' refers to the 'Format' general + // category, but where the 'cf' abbreviation is also an abbreviation + // for the 'Case_Folding' property. But we want to treat it as + // a general category. (Currently, we don't even support the + // 'Case_Folding' property. But if we do in the future, users will be + // required to spell it out.) + // + // Also 'sc' refers to the 'Currency_Symbol' general category, but is + // also the abbreviation for the 'Script' property. So we avoid calling + // 'canonical_prop' for it too, which would erroneously normalize it + // to 'Script'. + // + // Another case: 'lc' is an abbreviation for the 'Cased_Letter' + // general category, but is also an abbreviation for the 'Lowercase_Mapping' + // property. We don't currently support the latter, so as with 'cf' + // above, we treat 'lc' as 'Cased_Letter'. + if norm != "cf" && norm != "sc" && norm != "lc" { + if let Some(canon) = canonical_prop(&norm)? { + return Ok(CanonicalClassQuery::Binary(canon)); + } + } + if let Some(canon) = canonical_gencat(&norm)? { + return Ok(CanonicalClassQuery::GeneralCategory(canon)); + } + if let Some(canon) = canonical_script(&norm)? { + return Ok(CanonicalClassQuery::Script(canon)); + } + Err(Error::PropertyNotFound) + } +} + +/// Like ClassQuery, but its parameters have been canonicalized. This also +/// differentiates binary properties from flattened general categories and +/// scripts. +#[derive(Debug, Eq, PartialEq)] +enum CanonicalClassQuery { + /// The canonical binary property name. + Binary(&'static str), + /// The canonical general category name. + GeneralCategory(&'static str), + /// The canonical script name. + Script(&'static str), + /// An arbitrary association between property and value, both of which + /// have been canonicalized. + /// + /// Note that by construction, the property name of ByValue will never + /// be General_Category or Script. Those two cases are subsumed by the + /// eponymous variants. + ByValue { + /// The canonical property name. + property_name: &'static str, + /// The canonical property value. + property_value: &'static str, + }, +} + +/// Looks up a Unicode class given a query. If one doesn't exist, then +/// `None` is returned. +pub fn class(query: ClassQuery<'_>) -> Result<hir::ClassUnicode, Error> { + use self::CanonicalClassQuery::*; + + match query.canonicalize()? { + Binary(name) => bool_property(name), + GeneralCategory(name) => gencat(name), + Script(name) => script(name), + ByValue { property_name: "Age", property_value } => { + let mut class = hir::ClassUnicode::empty(); + for set in ages(property_value)? { + class.union(&hir_class(set)); + } + Ok(class) + } + ByValue { property_name: "Script_Extensions", property_value } => { + script_extension(property_value) + } + ByValue { + property_name: "Grapheme_Cluster_Break", + property_value, + } => gcb(property_value), + ByValue { property_name: "Sentence_Break", property_value } => { + sb(property_value) + } + ByValue { property_name: "Word_Break", property_value } => { + wb(property_value) + } + _ => { + // What else should we support? + Err(Error::PropertyNotFound) + } + } +} + +/// Returns a Unicode aware class for \w. +/// +/// This returns an error if the data is not available for \w. +pub fn perl_word() -> Result<hir::ClassUnicode, Error> { + #[cfg(not(feature = "unicode-perl"))] + fn imp() -> Result<hir::ClassUnicode, Error> { + Err(Error::PerlClassNotFound) + } + + #[cfg(feature = "unicode-perl")] + fn imp() -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::perl_word::PERL_WORD; + Ok(hir_class(PERL_WORD)) + } + + imp() +} + +/// Returns a Unicode aware class for \s. +/// +/// This returns an error if the data is not available for \s. +pub fn perl_space() -> Result<hir::ClassUnicode, Error> { + #[cfg(not(any(feature = "unicode-perl", feature = "unicode-bool")))] + fn imp() -> Result<hir::ClassUnicode, Error> { + Err(Error::PerlClassNotFound) + } + + #[cfg(all(feature = "unicode-perl", not(feature = "unicode-bool")))] + fn imp() -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::perl_space::WHITE_SPACE; + Ok(hir_class(WHITE_SPACE)) + } + + #[cfg(feature = "unicode-bool")] + fn imp() -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::property_bool::WHITE_SPACE; + Ok(hir_class(WHITE_SPACE)) + } + + imp() +} + +/// Returns a Unicode aware class for \d. +/// +/// This returns an error if the data is not available for \d. +pub fn perl_digit() -> Result<hir::ClassUnicode, Error> { + #[cfg(not(any(feature = "unicode-perl", feature = "unicode-gencat")))] + fn imp() -> Result<hir::ClassUnicode, Error> { + Err(Error::PerlClassNotFound) + } + + #[cfg(all(feature = "unicode-perl", not(feature = "unicode-gencat")))] + fn imp() -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::perl_decimal::DECIMAL_NUMBER; + Ok(hir_class(DECIMAL_NUMBER)) + } + + #[cfg(feature = "unicode-gencat")] + fn imp() -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::general_category::DECIMAL_NUMBER; + Ok(hir_class(DECIMAL_NUMBER)) + } + + imp() +} + +/// Build a Unicode HIR class from a sequence of Unicode scalar value ranges. +pub fn hir_class(ranges: &[(char, char)]) -> hir::ClassUnicode { + let hir_ranges: Vec<hir::ClassUnicodeRange> = ranges + .iter() + .map(|&(s, e)| hir::ClassUnicodeRange::new(s, e)) + .collect(); + hir::ClassUnicode::new(hir_ranges) +} + +/// Returns true only if the given codepoint is in the `\w` character class. +/// +/// If the `unicode-perl` feature is not enabled, then this returns an error. +pub fn is_word_character(c: char) -> Result<bool, UnicodeWordError> { + #[cfg(not(feature = "unicode-perl"))] + fn imp(_: char) -> Result<bool, UnicodeWordError> { + Err(UnicodeWordError(())) + } + + #[cfg(feature = "unicode-perl")] + fn imp(c: char) -> Result<bool, UnicodeWordError> { + use crate::{is_word_byte, unicode_tables::perl_word::PERL_WORD}; + + if u8::try_from(c).map_or(false, is_word_byte) { + return Ok(true); + } + Ok(PERL_WORD + .binary_search_by(|&(start, end)| { + use core::cmp::Ordering; + + if start <= c && c <= end { + Ordering::Equal + } else if start > c { + Ordering::Greater + } else { + Ordering::Less + } + }) + .is_ok()) + } + + imp(c) +} + +/// A mapping of property values for a specific property. +/// +/// The first element of each tuple is a normalized property value while the +/// second element of each tuple is the corresponding canonical property +/// value. +type PropertyValues = &'static [(&'static str, &'static str)]; + +fn canonical_gencat( + normalized_value: &str, +) -> Result<Option<&'static str>, Error> { + Ok(match normalized_value { + "any" => Some("Any"), + "assigned" => Some("Assigned"), + "ascii" => Some("ASCII"), + _ => { + let gencats = property_values("General_Category")?.unwrap(); + canonical_value(gencats, normalized_value) + } + }) +} + +fn canonical_script( + normalized_value: &str, +) -> Result<Option<&'static str>, Error> { + let scripts = property_values("Script")?.unwrap(); + Ok(canonical_value(scripts, normalized_value)) +} + +/// Find the canonical property name for the given normalized property name. +/// +/// If no such property exists, then `None` is returned. +/// +/// The normalized property name must have been normalized according to +/// UAX44 LM3, which can be done using `symbolic_name_normalize`. +/// +/// If the property names data is not available, then an error is returned. +fn canonical_prop( + normalized_name: &str, +) -> Result<Option<&'static str>, Error> { + #[cfg(not(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", + )))] + fn imp(_: &str) -> Result<Option<&'static str>, Error> { + Err(Error::PropertyNotFound) + } + + #[cfg(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", + ))] + fn imp(name: &str) -> Result<Option<&'static str>, Error> { + use crate::unicode_tables::property_names::PROPERTY_NAMES; + + Ok(PROPERTY_NAMES + .binary_search_by_key(&name, |&(n, _)| n) + .ok() + .map(|i| PROPERTY_NAMES[i].1)) + } + + imp(normalized_name) +} + +/// Find the canonical property value for the given normalized property +/// value. +/// +/// The given property values should correspond to the values for the property +/// under question, which can be found using `property_values`. +/// +/// If no such property value exists, then `None` is returned. +/// +/// The normalized property value must have been normalized according to +/// UAX44 LM3, which can be done using `symbolic_name_normalize`. +fn canonical_value( + vals: PropertyValues, + normalized_value: &str, +) -> Option<&'static str> { + vals.binary_search_by_key(&normalized_value, |&(n, _)| n) + .ok() + .map(|i| vals[i].1) +} + +/// Return the table of property values for the given property name. +/// +/// If the property values data is not available, then an error is returned. +fn property_values( + canonical_property_name: &'static str, +) -> Result<Option<PropertyValues>, Error> { + #[cfg(not(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", + )))] + fn imp(_: &'static str) -> Result<Option<PropertyValues>, Error> { + Err(Error::PropertyValueNotFound) + } + + #[cfg(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", + ))] + fn imp(name: &'static str) -> Result<Option<PropertyValues>, Error> { + use crate::unicode_tables::property_values::PROPERTY_VALUES; + + Ok(PROPERTY_VALUES + .binary_search_by_key(&name, |&(n, _)| n) + .ok() + .map(|i| PROPERTY_VALUES[i].1)) + } + + imp(canonical_property_name) +} + +// This is only used in some cases, but small enough to just let it be dead +// instead of figuring out (and maintaining) the right set of features. +#[allow(dead_code)] +fn property_set( + name_map: &'static [(&'static str, Range)], + canonical: &'static str, +) -> Option<Range> { + name_map + .binary_search_by_key(&canonical, |x| x.0) + .ok() + .map(|i| name_map[i].1) +} + +/// Returns an iterator over Unicode Age sets. Each item corresponds to a set +/// of codepoints that were added in a particular revision of Unicode. The +/// iterator yields items in chronological order. +/// +/// If the given age value isn't valid or if the data isn't available, then an +/// error is returned instead. +fn ages(canonical_age: &str) -> Result<impl Iterator<Item = Range>, Error> { + #[cfg(not(feature = "unicode-age"))] + fn imp(_: &str) -> Result<impl Iterator<Item = Range>, Error> { + use core::option::IntoIter; + Err::<IntoIter<Range>, _>(Error::PropertyNotFound) + } + + #[cfg(feature = "unicode-age")] + fn imp(canonical_age: &str) -> Result<impl Iterator<Item = Range>, Error> { + use crate::unicode_tables::age; + + const AGES: &[(&str, Range)] = &[ + ("V1_1", age::V1_1), + ("V2_0", age::V2_0), + ("V2_1", age::V2_1), + ("V3_0", age::V3_0), + ("V3_1", age::V3_1), + ("V3_2", age::V3_2), + ("V4_0", age::V4_0), + ("V4_1", age::V4_1), + ("V5_0", age::V5_0), + ("V5_1", age::V5_1), + ("V5_2", age::V5_2), + ("V6_0", age::V6_0), + ("V6_1", age::V6_1), + ("V6_2", age::V6_2), + ("V6_3", age::V6_3), + ("V7_0", age::V7_0), + ("V8_0", age::V8_0), + ("V9_0", age::V9_0), + ("V10_0", age::V10_0), + ("V11_0", age::V11_0), + ("V12_0", age::V12_0), + ("V12_1", age::V12_1), + ("V13_0", age::V13_0), + ("V14_0", age::V14_0), + ("V15_0", age::V15_0), + ("V15_1", age::V15_1), + ("V16_0", age::V16_0), + ]; + assert_eq!(AGES.len(), age::BY_NAME.len(), "ages are out of sync"); + + let pos = AGES.iter().position(|&(age, _)| canonical_age == age); + match pos { + None => Err(Error::PropertyValueNotFound), + Some(i) => Ok(AGES[..=i].iter().map(|&(_, classes)| classes)), + } + } + + imp(canonical_age) +} + +/// Returns the Unicode HIR class corresponding to the given general category. +/// +/// Name canonicalization is assumed to be performed by the caller. +/// +/// If the given general category could not be found, or if the general +/// category data is not available, then an error is returned. +fn gencat(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { + #[cfg(not(feature = "unicode-gencat"))] + fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> { + Err(Error::PropertyNotFound) + } + + #[cfg(feature = "unicode-gencat")] + fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::general_category::BY_NAME; + match name { + "ASCII" => Ok(hir_class(&[('\0', '\x7F')])), + "Any" => Ok(hir_class(&[('\0', '\u{10FFFF}')])), + "Assigned" => { + let mut cls = gencat("Unassigned")?; + cls.negate(); + Ok(cls) + } + name => property_set(BY_NAME, name) + .map(hir_class) + .ok_or(Error::PropertyValueNotFound), + } + } + + match canonical_name { + "Decimal_Number" => perl_digit(), + name => imp(name), + } +} + +/// Returns the Unicode HIR class corresponding to the given script. +/// +/// Name canonicalization is assumed to be performed by the caller. +/// +/// If the given script could not be found, or if the script data is not +/// available, then an error is returned. +fn script(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { + #[cfg(not(feature = "unicode-script"))] + fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> { + Err(Error::PropertyNotFound) + } + + #[cfg(feature = "unicode-script")] + fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::script::BY_NAME; + property_set(BY_NAME, name) + .map(hir_class) + .ok_or(Error::PropertyValueNotFound) + } + + imp(canonical_name) +} + +/// Returns the Unicode HIR class corresponding to the given script extension. +/// +/// Name canonicalization is assumed to be performed by the caller. +/// +/// If the given script extension could not be found, or if the script data is +/// not available, then an error is returned. +fn script_extension( + canonical_name: &'static str, +) -> Result<hir::ClassUnicode, Error> { + #[cfg(not(feature = "unicode-script"))] + fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> { + Err(Error::PropertyNotFound) + } + + #[cfg(feature = "unicode-script")] + fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::script_extension::BY_NAME; + property_set(BY_NAME, name) + .map(hir_class) + .ok_or(Error::PropertyValueNotFound) + } + + imp(canonical_name) +} + +/// Returns the Unicode HIR class corresponding to the given Unicode boolean +/// property. +/// +/// Name canonicalization is assumed to be performed by the caller. +/// +/// If the given boolean property could not be found, or if the boolean +/// property data is not available, then an error is returned. +fn bool_property( + canonical_name: &'static str, +) -> Result<hir::ClassUnicode, Error> { + #[cfg(not(feature = "unicode-bool"))] + fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> { + Err(Error::PropertyNotFound) + } + + #[cfg(feature = "unicode-bool")] + fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::property_bool::BY_NAME; + property_set(BY_NAME, name) + .map(hir_class) + .ok_or(Error::PropertyNotFound) + } + + match canonical_name { + "Decimal_Number" => perl_digit(), + "White_Space" => perl_space(), + name => imp(name), + } +} + +/// Returns the Unicode HIR class corresponding to the given grapheme cluster +/// break property. +/// +/// Name canonicalization is assumed to be performed by the caller. +/// +/// If the given property could not be found, or if the corresponding data is +/// not available, then an error is returned. +fn gcb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { + #[cfg(not(feature = "unicode-segment"))] + fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> { + Err(Error::PropertyNotFound) + } + + #[cfg(feature = "unicode-segment")] + fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::grapheme_cluster_break::BY_NAME; + property_set(BY_NAME, name) + .map(hir_class) + .ok_or(Error::PropertyValueNotFound) + } + + imp(canonical_name) +} + +/// Returns the Unicode HIR class corresponding to the given word break +/// property. +/// +/// Name canonicalization is assumed to be performed by the caller. +/// +/// If the given property could not be found, or if the corresponding data is +/// not available, then an error is returned. +fn wb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { + #[cfg(not(feature = "unicode-segment"))] + fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> { + Err(Error::PropertyNotFound) + } + + #[cfg(feature = "unicode-segment")] + fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::word_break::BY_NAME; + property_set(BY_NAME, name) + .map(hir_class) + .ok_or(Error::PropertyValueNotFound) + } + + imp(canonical_name) +} + +/// Returns the Unicode HIR class corresponding to the given sentence +/// break property. +/// +/// Name canonicalization is assumed to be performed by the caller. +/// +/// If the given property could not be found, or if the corresponding data is +/// not available, then an error is returned. +fn sb(canonical_name: &'static str) -> Result<hir::ClassUnicode, Error> { + #[cfg(not(feature = "unicode-segment"))] + fn imp(_: &'static str) -> Result<hir::ClassUnicode, Error> { + Err(Error::PropertyNotFound) + } + + #[cfg(feature = "unicode-segment")] + fn imp(name: &'static str) -> Result<hir::ClassUnicode, Error> { + use crate::unicode_tables::sentence_break::BY_NAME; + property_set(BY_NAME, name) + .map(hir_class) + .ok_or(Error::PropertyValueNotFound) + } + + imp(canonical_name) +} + +/// Like symbolic_name_normalize_bytes, but operates on a string. +fn symbolic_name_normalize(x: &str) -> String { + let mut tmp = x.as_bytes().to_vec(); + let len = symbolic_name_normalize_bytes(&mut tmp).len(); + tmp.truncate(len); + // This should always succeed because `symbolic_name_normalize_bytes` + // guarantees that `&tmp[..len]` is always valid UTF-8. + // + // N.B. We could avoid the additional UTF-8 check here, but it's unlikely + // to be worth skipping the additional safety check. A benchmark must + // justify it first. + String::from_utf8(tmp).unwrap() +} + +/// Normalize the given symbolic name in place according to UAX44-LM3. +/// +/// A "symbolic name" typically corresponds to property names and property +/// value aliases. Note, though, that it should not be applied to property +/// string values. +/// +/// The slice returned is guaranteed to be valid UTF-8 for all possible values +/// of `slice`. +/// +/// See: https://unicode.org/reports/tr44/#UAX44-LM3 +fn symbolic_name_normalize_bytes(slice: &mut [u8]) -> &mut [u8] { + // I couldn't find a place in the standard that specified that property + // names/aliases had a particular structure (unlike character names), but + // we assume that it's ASCII only and drop anything that isn't ASCII. + let mut start = 0; + let mut starts_with_is = false; + if slice.len() >= 2 { + // Ignore any "is" prefix. + starts_with_is = slice[0..2] == b"is"[..] + || slice[0..2] == b"IS"[..] + || slice[0..2] == b"iS"[..] + || slice[0..2] == b"Is"[..]; + if starts_with_is { + start = 2; + } + } + let mut next_write = 0; + for i in start..slice.len() { + // VALIDITY ARGUMENT: To guarantee that the resulting slice is valid + // UTF-8, we ensure that the slice contains only ASCII bytes. In + // particular, we drop every non-ASCII byte from the normalized string. + let b = slice[i]; + if b == b' ' || b == b'_' || b == b'-' { + continue; + } else if b'A' <= b && b <= b'Z' { + slice[next_write] = b + (b'a' - b'A'); + next_write += 1; + } else if b <= 0x7F { + slice[next_write] = b; + next_write += 1; + } + } + // Special case: ISO_Comment has a 'isc' abbreviation. Since we generally + // ignore 'is' prefixes, the 'isc' abbreviation gets caught in the cross + // fire and ends up creating an alias for 'c' to 'ISO_Comment', but it + // is actually an alias for the 'Other' general category. + if starts_with_is && next_write == 1 && slice[0] == b'c' { + slice[0] = b'i'; + slice[1] = b's'; + slice[2] = b'c'; + next_write = 3; + } + &mut slice[..next_write] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(feature = "unicode-case")] + fn simple_fold_ok(c: char) -> impl Iterator<Item = char> { + SimpleCaseFolder::new().unwrap().mapping(c).iter().copied() + } + + #[cfg(feature = "unicode-case")] + fn contains_case_map(start: char, end: char) -> bool { + SimpleCaseFolder::new().unwrap().overlaps(start, end) + } + + #[test] + #[cfg(feature = "unicode-case")] + fn simple_fold_k() { + let xs: Vec<char> = simple_fold_ok('k').collect(); + assert_eq!(xs, alloc::vec!['K', 'K']); + + let xs: Vec<char> = simple_fold_ok('K').collect(); + assert_eq!(xs, alloc::vec!['k', 'K']); + + let xs: Vec<char> = simple_fold_ok('K').collect(); + assert_eq!(xs, alloc::vec!['K', 'k']); + } + + #[test] + #[cfg(feature = "unicode-case")] + fn simple_fold_a() { + let xs: Vec<char> = simple_fold_ok('a').collect(); + assert_eq!(xs, alloc::vec!['A']); + + let xs: Vec<char> = simple_fold_ok('A').collect(); + assert_eq!(xs, alloc::vec!['a']); + } + + #[test] + #[cfg(not(feature = "unicode-case"))] + fn simple_fold_disabled() { + assert!(SimpleCaseFolder::new().is_err()); + } + + #[test] + #[cfg(feature = "unicode-case")] + fn range_contains() { + assert!(contains_case_map('A', 'A')); + assert!(contains_case_map('Z', 'Z')); + assert!(contains_case_map('A', 'Z')); + assert!(contains_case_map('@', 'A')); + assert!(contains_case_map('Z', '[')); + assert!(contains_case_map('☃', 'Ⰰ')); + + assert!(!contains_case_map('[', '[')); + assert!(!contains_case_map('[', '`')); + + assert!(!contains_case_map('☃', '☃')); + } + + #[test] + #[cfg(feature = "unicode-gencat")] + fn regression_466() { + use super::{CanonicalClassQuery, ClassQuery}; + + let q = ClassQuery::OneLetter('C'); + assert_eq!( + q.canonicalize().unwrap(), + CanonicalClassQuery::GeneralCategory("Other") + ); + } + + #[test] + fn sym_normalize() { + let sym_norm = symbolic_name_normalize; + + assert_eq!(sym_norm("Line_Break"), "linebreak"); + assert_eq!(sym_norm("Line-break"), "linebreak"); + assert_eq!(sym_norm("linebreak"), "linebreak"); + assert_eq!(sym_norm("BA"), "ba"); + assert_eq!(sym_norm("ba"), "ba"); + assert_eq!(sym_norm("Greek"), "greek"); + assert_eq!(sym_norm("isGreek"), "greek"); + assert_eq!(sym_norm("IS_Greek"), "greek"); + assert_eq!(sym_norm("isc"), "isc"); + assert_eq!(sym_norm("is c"), "isc"); + assert_eq!(sym_norm("is_c"), "isc"); + } + + #[test] + fn valid_utf8_symbolic() { + let mut x = b"abc\xFFxyz".to_vec(); + let y = symbolic_name_normalize_bytes(&mut x); + assert_eq!(y, b"abcxyz"); + } +} diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/LICENSE-UNICODE b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/LICENSE-UNICODE new file mode 100644 index 0000000..b82826b --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/LICENSE-UNICODE @@ -0,0 +1,57 @@ +UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE + +Unicode Data Files include all data files under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and +http://www.unicode.org/utility/trac/browser/. + +Unicode Data Files do not include PDF online code charts under the +directory http://www.unicode.org/Public/. + +Software includes any source code published in the Unicode Standard +or under the directories +http://www.unicode.org/Public/, http://www.unicode.org/reports/, +http://www.unicode.org/cldr/data/, http://source.icu-project.org/repos/icu/, and +http://www.unicode.org/utility/trac/browser/. + +NOTICE TO USER: Carefully read the following legal agreement. +BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S +DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), +YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. +IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE +THE DATA FILES OR SOFTWARE. + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2018 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that either +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, or +(b) this copyright and permission notice appear in associated +Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/age.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/age.rs new file mode 100644 index 0000000..466510c --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/age.rs @@ -0,0 +1,1846 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate age ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = &[ + ("V10_0", V10_0), + ("V11_0", V11_0), + ("V12_0", V12_0), + ("V12_1", V12_1), + ("V13_0", V13_0), + ("V14_0", V14_0), + ("V15_0", V15_0), + ("V15_1", V15_1), + ("V16_0", V16_0), + ("V1_1", V1_1), + ("V2_0", V2_0), + ("V2_1", V2_1), + ("V3_0", V3_0), + ("V3_1", V3_1), + ("V3_2", V3_2), + ("V4_0", V4_0), + ("V4_1", V4_1), + ("V5_0", V5_0), + ("V5_1", V5_1), + ("V5_2", V5_2), + ("V6_0", V6_0), + ("V6_1", V6_1), + ("V6_2", V6_2), + ("V6_3", V6_3), + ("V7_0", V7_0), + ("V8_0", V8_0), + ("V9_0", V9_0), +]; + +pub const V10_0: &'static [(char, char)] = &[ + ('ࡠ', 'ࡪ'), + ('ৼ', '৽'), + ('\u{afa}', '\u{aff}'), + ('\u{d00}', '\u{d00}'), + ('\u{d3b}', '\u{d3c}'), + ('᳷', '᳷'), + ('\u{1df6}', '\u{1df9}'), + ('₿', '₿'), + ('⏿', '⏿'), + ('⯒', '⯒'), + ('⹅', '⹉'), + ('ㄮ', 'ㄮ'), + ('鿖', '鿪'), + ('𐌭', '𐌯'), + ('𑨀', '\u{11a47}'), + ('𑩐', '𑪃'), + ('𑪆', '𑪜'), + ('𑪞', '𑪢'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d47}'), + ('𑵐', '𑵙'), + ('𖿡', '𖿡'), + ('𛀂', '𛄞'), + ('𛅰', '𛋻'), + ('🉠', '🉥'), + ('🛓', '🛔'), + ('🛷', '🛸'), + ('🤀', '🤋'), + ('🤟', '🤟'), + ('🤨', '🤯'), + ('🤱', '🤲'), + ('🥌', '🥌'), + ('🥟', '🥫'), + ('🦒', '🦗'), + ('🧐', '🧦'), + ('𬺰', '𮯠'), +]; + +pub const V11_0: &'static [(char, char)] = &[ + ('ՠ', 'ՠ'), + ('ֈ', 'ֈ'), + ('ׯ', 'ׯ'), + ('\u{7fd}', '߿'), + ('\u{8d3}', '\u{8d3}'), + ('\u{9fe}', '\u{9fe}'), + ('੶', '੶'), + ('\u{c04}', '\u{c04}'), + ('಄', '಄'), + ('ᡸ', 'ᡸ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('⮺', '⮼'), + ('⯓', '⯫'), + ('⯰', '⯾'), + ('⹊', '⹎'), + ('ㄯ', 'ㄯ'), + ('鿫', '鿯'), + ('ꞯ', 'ꞯ'), + ('Ꞹ', 'ꞹ'), + ('ꣾ', '\u{a8ff}'), + ('𐨴', '𐨵'), + ('𐩈', '𐩈'), + ('𐴀', '\u{10d27}'), + ('𐴰', '𐴹'), + ('𐼀', '𐼧'), + ('𐼰', '𐽙'), + ('\u{110cd}', '\u{110cd}'), + ('𑅄', '𑅆'), + ('\u{1133b}', '\u{1133b}'), + ('\u{1145e}', '\u{1145e}'), + ('𑜚', '𑜚'), + ('𑠀', '𑠻'), + ('𑪝', '𑪝'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶘'), + ('𑶠', '𑶩'), + ('𑻠', '𑻸'), + ('𖹀', '𖺚'), + ('𘟭', '𘟱'), + ('𝋠', '𝋳'), + ('𝍲', '𝍸'), + ('𞱱', '𞲴'), + ('🄯', '🄯'), + ('🛹', '🛹'), + ('🟕', '🟘'), + ('🥍', '🥏'), + ('🥬', '🥰'), + ('🥳', '🥶'), + ('🥺', '🥺'), + ('🥼', '🥿'), + ('🦘', '🦢'), + ('🦰', '🦹'), + ('🧁', '🧂'), + ('🧧', '🧿'), + ('🩠', '🩭'), +]; + +pub const V12_0: &'static [(char, char)] = &[ + ('౷', '౷'), + ('ຆ', 'ຆ'), + ('ຉ', 'ຉ'), + ('ຌ', 'ຌ'), + ('ຎ', 'ຓ'), + ('ຘ', 'ຘ'), + ('ຠ', 'ຠ'), + ('ຨ', 'ຩ'), + ('ຬ', 'ຬ'), + ('\u{eba}', '\u{eba}'), + ('ᳺ', 'ᳺ'), + ('⯉', '⯉'), + ('⯿', '⯿'), + ('⹏', '⹏'), + ('Ꞻ', 'ꞿ'), + ('Ꟃ', 'Ᶎ'), + ('ꭦ', 'ꭧ'), + ('𐿠', '𐿶'), + ('𑑟', '𑑟'), + ('𑚸', '𑚸'), + ('𑦠', '𑦧'), + ('𑦪', '\u{119d7}'), + ('\u{119da}', '𑧤'), + ('𑪄', '𑪅'), + ('𑿀', '𑿱'), + ('𑿿', '𑿿'), + ('\u{13430}', '\u{13438}'), + ('𖽅', '𖽊'), + ('\u{16f4f}', '\u{16f4f}'), + ('𖽿', '𖾇'), + ('𖿢', '𖿣'), + ('𘟲', '𘟷'), + ('𛅐', '𛅒'), + ('𛅤', '𛅧'), + ('𞄀', '𞄬'), + ('\u{1e130}', '𞄽'), + ('𞅀', '𞅉'), + ('𞅎', '𞅏'), + ('𞋀', '𞋹'), + ('𞋿', '𞋿'), + ('𞥋', '𞥋'), + ('𞴁', '𞴽'), + ('🅬', '🅬'), + ('🛕', '🛕'), + ('🛺', '🛺'), + ('🟠', '🟫'), + ('🤍', '🤏'), + ('🤿', '🤿'), + ('🥱', '🥱'), + ('🥻', '🥻'), + ('🦥', '🦪'), + ('🦮', '🦯'), + ('🦺', '🦿'), + ('🧃', '🧊'), + ('🧍', '🧏'), + ('🨀', '🩓'), + ('🩰', '🩳'), + ('🩸', '🩺'), + ('🪀', '🪂'), + ('🪐', '🪕'), +]; + +pub const V12_1: &'static [(char, char)] = &[('㋿', '㋿')]; + +pub const V13_0: &'static [(char, char)] = &[ + ('ࢾ', 'ࣇ'), + ('\u{b55}', '\u{b55}'), + ('ഄ', 'ഄ'), + ('\u{d81}', '\u{d81}'), + ('\u{1abf}', '\u{1ac0}'), + ('⮗', '⮗'), + ('⹐', '⹒'), + ('ㆻ', 'ㆿ'), + ('䶶', '䶿'), + ('鿰', '鿼'), + ('Ꟈ', 'ꟊ'), + ('Ꟶ', 'ꟶ'), + ('\u{a82c}', '\u{a82c}'), + ('ꭨ', '꭫'), + ('𐆜', '𐆜'), + ('𐺀', '𐺩'), + ('\u{10eab}', '𐺭'), + ('𐺰', '𐺱'), + ('𐾰', '𐿋'), + ('𑅇', '𑅇'), + ('𑇎', '\u{111cf}'), + ('𑑚', '𑑚'), + ('𑑠', '𑑡'), + ('𑤀', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '𑥆'), + ('𑥐', '𑥙'), + ('𑾰', '𑾰'), + ('\u{16fe4}', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('𘫳', '𘳕'), + ('𘴀', '𘴈'), + ('🄍', '🄏'), + ('🅭', '🅯'), + ('🆭', '🆭'), + ('🛖', '🛗'), + ('🛻', '🛼'), + ('🢰', '🢱'), + ('🤌', '🤌'), + ('🥲', '🥲'), + ('🥷', '🥸'), + ('🦣', '🦤'), + ('🦫', '🦭'), + ('🧋', '🧋'), + ('🩴', '🩴'), + ('🪃', '🪆'), + ('🪖', '🪨'), + ('🪰', '🪶'), + ('🫀', '🫂'), + ('🫐', '🫖'), + ('🬀', '🮒'), + ('🮔', '🯊'), + ('🯰', '🯹'), + ('𪛗', '𪛝'), + ('𰀀', '𱍊'), +]; + +pub const V14_0: &'static [(char, char)] = &[ + ('؝', '؝'), + ('ࡰ', 'ࢎ'), + ('\u{890}', '\u{891}'), + ('\u{898}', '\u{89f}'), + ('ࢵ', 'ࢵ'), + ('ࣈ', '\u{8d2}'), + ('\u{c3c}', '\u{c3c}'), + ('ౝ', 'ౝ'), + ('ೝ', 'ೝ'), + ('ᜍ', 'ᜍ'), + ('\u{1715}', '\u{1715}'), + ('ᜟ', 'ᜟ'), + ('\u{180f}', '\u{180f}'), + ('\u{1ac1}', '\u{1ace}'), + ('ᭌ', 'ᭌ'), + ('᭽', '᭾'), + ('\u{1dfa}', '\u{1dfa}'), + ('⃀', '⃀'), + ('Ⱟ', 'Ⱟ'), + ('ⱟ', 'ⱟ'), + ('⹓', '⹝'), + ('鿽', '鿿'), + ('Ꟁ', 'ꟁ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'ꟙ'), + ('ꟲ', 'ꟴ'), + ('﯂', '﯂'), + ('﵀', '﵏'), + ('﷏', '﷏'), + ('﷾', '﷿'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐽰', '𐾉'), + ('\u{11070}', '𑁵'), + ('\u{110c2}', '\u{110c2}'), + ('𑚹', '𑚹'), + ('𑝀', '𑝆'), + ('𑪰', '𑪿'), + ('𒾐', '𒿲'), + ('𖩰', '𖪾'), + ('𖫀', '𖫉'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛄟', '𛄢'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('𜽐', '𜿃'), + ('𝇩', '𝇪'), + ('𝼀', '𝼞'), + ('𞊐', '\u{1e2ae}'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('🛝', '🛟'), + ('🟰', '🟰'), + ('🥹', '🥹'), + ('🧌', '🧌'), + ('🩻', '🩼'), + ('🪩', '🪬'), + ('🪷', '🪺'), + ('🫃', '🫅'), + ('🫗', '🫙'), + ('🫠', '🫧'), + ('🫰', '🫶'), + ('𪛞', '𪛟'), + ('𫜵', '𫜸'), +]; + +pub const V15_0: &'static [(char, char)] = &[ + ('ೳ', 'ೳ'), + ('\u{ece}', '\u{ece}'), + ('\u{10efd}', '\u{10eff}'), + ('𑈿', '\u{11241}'), + ('𑬀', '𑬉'), + ('\u{11f00}', '𑼐'), + ('𑼒', '\u{11f3a}'), + ('𑼾', '𑽙'), + ('𓐯', '𓐯'), + ('\u{13439}', '\u{13455}'), + ('𛄲', '𛄲'), + ('𛅕', '𛅕'), + ('𝋀', '𝋓'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), + ('𞓐', '𞓹'), + ('🛜', '🛜'), + ('🝴', '🝶'), + ('🝻', '🝿'), + ('🟙', '🟙'), + ('🩵', '🩷'), + ('🪇', '🪈'), + ('🪭', '🪯'), + ('🪻', '🪽'), + ('🪿', '🪿'), + ('🫎', '🫏'), + ('🫚', '🫛'), + ('🫨', '🫨'), + ('🫷', '🫸'), + ('𫜹', '𫜹'), + ('𱍐', '𲎯'), +]; + +pub const V15_1: &'static [(char, char)] = + &[('⿼', '⿿'), ('㇯', '㇯'), ('𮯰', '𮹝')]; + +pub const V16_0: &'static [(char, char)] = &[ + ('\u{897}', '\u{897}'), + ('᭎', '᭏'), + ('᭿', '᭿'), + ('Ᲊ', 'ᲊ'), + ('␧', '␩'), + ('㇤', '㇥'), + ('Ɤ', 'ꟍ'), + ('Ꟛ', 'Ƛ'), + ('𐗀', '𐗳'), + ('𐵀', '𐵥'), + ('\u{10d69}', '𐶅'), + ('𐶎', '𐶏'), + ('𐻂', '𐻄'), + ('\u{10efc}', '\u{10efc}'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏕'), + ('𑏗', '𑏘'), + ('\u{113e1}', '\u{113e2}'), + ('𑛐', '𑛣'), + ('𑯀', '𑯡'), + ('𑯰', '𑯹'), + ('\u{11f5a}', '\u{11f5a}'), + ('𓑠', '𔏺'), + ('𖄀', '𖄹'), + ('𖵀', '𖵹'), + ('𘳿', '𘳿'), + ('𜰀', '𜳹'), + ('𜴀', '𜺳'), + ('𞗐', '𞗺'), + ('𞗿', '𞗿'), + ('🢲', '🢻'), + ('🣀', '🣁'), + ('🪉', '🪉'), + ('🪏', '🪏'), + ('🪾', '🪾'), + ('🫆', '🫆'), + ('🫜', '🫜'), + ('🫟', '🫟'), + ('🫩', '🫩'), + ('🯋', '🯯'), +]; + +pub const V1_1: &'static [(char, char)] = &[ + ('\0', 'ǵ'), + ('Ǻ', 'ȗ'), + ('ɐ', 'ʨ'), + ('ʰ', '˞'), + ('ˠ', '˩'), + ('\u{300}', '\u{345}'), + ('\u{360}', '\u{361}'), + ('ʹ', '͵'), + ('ͺ', 'ͺ'), + (';', ';'), + ('΄', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ώ'), + ('ϐ', 'ϖ'), + ('Ϛ', 'Ϛ'), + ('Ϝ', 'Ϝ'), + ('Ϟ', 'Ϟ'), + ('Ϡ', 'Ϡ'), + ('Ϣ', 'ϳ'), + ('Ё', 'Ќ'), + ('Ў', 'я'), + ('ё', 'ќ'), + ('ў', '\u{486}'), + ('Ґ', 'ӄ'), + ('Ӈ', 'ӈ'), + ('Ӌ', 'ӌ'), + ('Ӑ', 'ӫ'), + ('Ӯ', 'ӵ'), + ('Ӹ', 'ӹ'), + ('Ա', 'Ֆ'), + ('ՙ', '՟'), + ('ա', 'և'), + ('։', '։'), + ('\u{5b0}', '\u{5b9}'), + ('\u{5bb}', '׃'), + ('א', 'ת'), + ('װ', '״'), + ('،', '،'), + ('؛', '؛'), + ('؟', '؟'), + ('ء', 'غ'), + ('ـ', '\u{652}'), + ('٠', '٭'), + ('\u{670}', 'ڷ'), + ('ں', 'ھ'), + ('ۀ', 'ێ'), + ('ې', '\u{6ed}'), + ('۰', '۹'), + ('\u{901}', 'ः'), + ('अ', 'ह'), + ('\u{93c}', '\u{94d}'), + ('ॐ', '\u{954}'), + ('क़', '॰'), + ('\u{981}', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9be}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', '\u{9cd}'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', '\u{9e3}'), + ('০', '৺'), + ('\u{a02}', '\u{a02}'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('੦', 'ੴ'), + ('\u{a81}', 'ઃ'), + ('અ', 'ઋ'), + ('ઍ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('\u{abc}', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('ૐ', 'ૐ'), + ('ૠ', 'ૠ'), + ('૦', '૯'), + ('\u{b01}', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଶ', 'ହ'), + ('\u{b3c}', '\u{b43}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b56}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୡ'), + ('୦', '୰'), + ('\u{b82}', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'வ'), + ('ஷ', 'ஹ'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('\u{bd7}', '\u{bd7}'), + ('௧', '௲'), + ('ఁ', 'ః'), + ('అ', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'ళ'), + ('వ', 'హ'), + ('\u{c3e}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('ౠ', 'ౡ'), + ('౦', '౯'), + ('ಂ', 'ಃ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಾ', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('ೞ', 'ೞ'), + ('ೠ', 'ೡ'), + ('೦', '೯'), + ('ം', 'ഃ'), + ('അ', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ന'), + ('പ', 'ഹ'), + ('\u{d3e}', '\u{d43}'), + ('െ', 'ൈ'), + ('ൊ', '\u{d4d}'), + ('\u{d57}', '\u{d57}'), + ('ൠ', 'ൡ'), + ('൦', '൯'), + ('ก', '\u{e3a}'), + ('฿', '๛'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ງ', 'ຈ'), + ('ຊ', 'ຊ'), + ('ຍ', 'ຍ'), + ('ດ', 'ທ'), + ('ນ', 'ຟ'), + ('ມ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ວ'), + ('ສ', 'ຫ'), + ('ອ', '\u{eb9}'), + ('\u{ebb}', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('\u{ec8}', '\u{ecd}'), + ('໐', '໙'), + ('ໜ', 'ໝ'), + ('Ⴀ', 'Ⴥ'), + ('ა', 'ჶ'), + ('჻', '჻'), + ('ᄀ', 'ᅙ'), + ('ᅟ', 'ᆢ'), + ('ᆨ', 'ᇹ'), + ('Ḁ', 'ẚ'), + ('Ạ', 'ỹ'), + ('ἀ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ῄ'), + ('ῆ', 'ΐ'), + ('ῖ', 'Ί'), + ('῝', '`'), + ('ῲ', 'ῴ'), + ('ῶ', '῾'), + ('\u{2000}', '\u{202e}'), + ('‰', '⁆'), + ('\u{206a}', '⁰'), + ('⁴', '₎'), + ('₠', '₪'), + ('\u{20d0}', '\u{20e1}'), + ('℀', 'ℸ'), + ('⅓', 'ↂ'), + ('←', '⇪'), + ('∀', '⋱'), + ('⌀', '⌀'), + ('⌂', '⍺'), + ('␀', '␤'), + ('⑀', '⑊'), + ('①', '⓪'), + ('─', '▕'), + ('■', '◯'), + ('☀', '☓'), + ('☚', '♯'), + ('✁', '✄'), + ('✆', '✉'), + ('✌', '✧'), + ('✩', '❋'), + ('❍', '❍'), + ('❏', '❒'), + ('❖', '❖'), + ('❘', '❞'), + ('❡', '❧'), + ('❶', '➔'), + ('➘', '➯'), + ('➱', '➾'), + ('\u{3000}', '〷'), + ('〿', '〿'), + ('ぁ', 'ゔ'), + ('\u{3099}', 'ゞ'), + ('ァ', 'ヾ'), + ('ㄅ', 'ㄬ'), + ('ㄱ', 'ㆎ'), + ('㆐', '㆟'), + ('㈀', '㈜'), + ('㈠', '㉃'), + ('㉠', '㉻'), + ('㉿', '㊰'), + ('㋀', '㋋'), + ('㋐', '㋾'), + ('㌀', '㍶'), + ('㍻', '㏝'), + ('㏠', '㏾'), + ('一', '龥'), + ('\u{e000}', '鶴'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('\u{fb1e}', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', '﴿'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('\u{fe20}', '\u{fe23}'), + ('︰', '﹄'), + ('﹉', '﹒'), + ('﹔', '﹦'), + ('﹨', '﹫'), + ('ﹰ', 'ﹲ'), + ('ﹴ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('\u{feff}', '\u{feff}'), + ('！', '～'), + ('｡', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('￠', '￦'), + ('￨', '￮'), + ('�', '\u{ffff}'), +]; + +pub const V2_0: &'static [(char, char)] = &[ + ('\u{591}', '\u{5a1}'), + ('\u{5a3}', '\u{5af}'), + ('\u{5c4}', '\u{5c4}'), + ('ༀ', 'ཇ'), + ('ཉ', 'ཀྵ'), + ('\u{f71}', 'ྋ'), + ('\u{f90}', '\u{f95}'), + ('\u{f97}', '\u{f97}'), + ('\u{f99}', '\u{fad}'), + ('\u{fb1}', '\u{fb7}'), + ('\u{fb9}', '\u{fb9}'), + ('ẛ', 'ẛ'), + ('₫', '₫'), + ('가', '힣'), + ('\u{1fffe}', '\u{1ffff}'), + ('\u{2fffe}', '\u{2ffff}'), + ('\u{3fffe}', '\u{3ffff}'), + ('\u{4fffe}', '\u{4ffff}'), + ('\u{5fffe}', '\u{5ffff}'), + ('\u{6fffe}', '\u{6ffff}'), + ('\u{7fffe}', '\u{7ffff}'), + ('\u{8fffe}', '\u{8ffff}'), + ('\u{9fffe}', '\u{9ffff}'), + ('\u{afffe}', '\u{affff}'), + ('\u{bfffe}', '\u{bffff}'), + ('\u{cfffe}', '\u{cffff}'), + ('\u{dfffe}', '\u{dffff}'), + ('\u{efffe}', '\u{10ffff}'), +]; + +pub const V2_1: &'static [(char, char)] = &[('€', '€'), ('', '')]; + +pub const V3_0: &'static [(char, char)] = &[ + ('Ƕ', 'ǹ'), + ('Ș', 'ȟ'), + ('Ȣ', 'ȳ'), + ('ʩ', 'ʭ'), + ('˟', '˟'), + ('˪', 'ˮ'), + ('\u{346}', '\u{34e}'), + ('\u{362}', '\u{362}'), + ('ϗ', 'ϗ'), + ('ϛ', 'ϛ'), + ('ϝ', 'ϝ'), + ('ϟ', 'ϟ'), + ('ϡ', 'ϡ'), + ('Ѐ', 'Ѐ'), + ('Ѝ', 'Ѝ'), + ('ѐ', 'ѐ'), + ('ѝ', 'ѝ'), + ('\u{488}', '\u{489}'), + ('Ҍ', 'ҏ'), + ('Ӭ', 'ӭ'), + ('֊', '֊'), + ('\u{653}', '\u{655}'), + ('ڸ', 'ڹ'), + ('ڿ', 'ڿ'), + ('ۏ', 'ۏ'), + ('ۺ', '۾'), + ('܀', '܍'), + ('\u{70f}', 'ܬ'), + ('\u{730}', '\u{74a}'), + ('ހ', '\u{7b0}'), + ('ං', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('ෲ', '෴'), + ('ཪ', 'ཪ'), + ('\u{f96}', '\u{f96}'), + ('\u{fae}', '\u{fb0}'), + ('\u{fb8}', '\u{fb8}'), + ('\u{fba}', '\u{fbc}'), + ('྾', '࿌'), + ('࿏', '࿏'), + ('က', 'အ'), + ('ဣ', 'ဧ'), + ('ဩ', 'ဪ'), + ('ာ', '\u{1032}'), + ('\u{1036}', '\u{1039}'), + ('၀', '\u{1059}'), + ('ሀ', 'ሆ'), + ('ለ', 'ቆ'), + ('ቈ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኆ'), + ('ኈ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኮ'), + ('ኰ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዎ'), + ('ዐ', 'ዖ'), + ('ዘ', 'ዮ'), + ('ደ', 'ጎ'), + ('ጐ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ጞ'), + ('ጠ', 'ፆ'), + ('ፈ', 'ፚ'), + ('፡', '፼'), + ('Ꭰ', 'Ᏼ'), + ('ᐁ', 'ᙶ'), + ('\u{1680}', '᚜'), + ('ᚠ', 'ᛰ'), + ('ក', 'ៜ'), + ('០', '៩'), + ('᠀', '\u{180e}'), + ('᠐', '᠙'), + ('ᠠ', 'ᡷ'), + ('ᢀ', '\u{18a9}'), + ('\u{202f}', '\u{202f}'), + ('⁈', '⁍'), + ('₭', '₯'), + ('\u{20e2}', '\u{20e3}'), + ('ℹ', '℺'), + ('Ↄ', 'Ↄ'), + ('⇫', '⇳'), + ('⌁', '⌁'), + ('⍻', '⍻'), + ('⍽', '⎚'), + ('␥', '␦'), + ('◰', '◷'), + ('☙', '☙'), + ('♰', '♱'), + ('⠀', '⣿'), + ('⺀', '⺙'), + ('⺛', '⻳'), + ('⼀', '⿕'), + ('⿰', '⿻'), + ('〸', '〺'), + ('〾', '〾'), + ('ㆠ', 'ㆷ'), + ('㐀', '䶵'), + ('ꀀ', 'ꒌ'), + ('꒐', '꒡'), + ('꒤', '꒳'), + ('꒵', '꓀'), + ('꓂', '꓄'), + ('꓆', '꓆'), + ('יִ', 'יִ'), + ('\u{fff9}', '\u{fffb}'), +]; + +pub const V3_1: &'static [(char, char)] = &[ + ('ϴ', 'ϵ'), + ('\u{fdd0}', '\u{fdef}'), + ('𐌀', '𐌞'), + ('𐌠', '𐌣'), + ('𐌰', '𐍊'), + ('𐐀', '𐐥'), + ('𐐨', '𐑍'), + ('𝀀', '𝃵'), + ('𝄀', '𝄦'), + ('𝄪', '𝇝'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓀'), + ('𝓂', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚣'), + ('𝚨', '𝟉'), + ('𝟎', '𝟿'), + ('𠀀', '𪛖'), + ('丽', '𪘀'), + ('\u{e0001}', '\u{e0001}'), + ('\u{e0020}', '\u{e007f}'), +]; + +pub const V3_2: &'static [(char, char)] = &[ + ('Ƞ', 'Ƞ'), + ('\u{34f}', '\u{34f}'), + ('\u{363}', '\u{36f}'), + ('Ϙ', 'ϙ'), + ('϶', '϶'), + ('Ҋ', 'ҋ'), + ('Ӆ', 'ӆ'), + ('Ӊ', 'ӊ'), + ('Ӎ', 'ӎ'), + ('Ԁ', 'ԏ'), + ('ٮ', 'ٯ'), + ('ޱ', 'ޱ'), + ('ჷ', 'ჸ'), + ('ᜀ', 'ᜌ'), + ('ᜎ', '\u{1714}'), + ('ᜠ', '᜶'), + ('ᝀ', '\u{1753}'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('\u{1772}', '\u{1773}'), + ('⁇', '⁇'), + ('⁎', '⁒'), + ('⁗', '⁗'), + ('\u{205f}', '\u{2063}'), + ('ⁱ', 'ⁱ'), + ('₰', '₱'), + ('\u{20e4}', '\u{20ea}'), + ('ℽ', '⅋'), + ('⇴', '⇿'), + ('⋲', '⋿'), + ('⍼', '⍼'), + ('⎛', '⏎'), + ('⓫', '⓾'), + ('▖', '▟'), + ('◸', '◿'), + ('☖', '☗'), + ('♲', '♽'), + ('⚀', '⚉'), + ('❨', '❵'), + ('⟐', '⟫'), + ('⟰', '⟿'), + ('⤀', '⫿'), + ('〻', '〽'), + ('ゕ', 'ゖ'), + ('ゟ', '゠'), + ('ヿ', 'ヿ'), + ('ㇰ', 'ㇿ'), + ('㉑', '㉟'), + ('㊱', '㊿'), + ('꒢', '꒣'), + ('꒴', '꒴'), + ('꓁', '꓁'), + ('꓅', '꓅'), + ('侮', '頻'), + ('﷼', '﷼'), + ('\u{fe00}', '\u{fe0f}'), + ('﹅', '﹆'), + ('ﹳ', 'ﹳ'), + ('｟', '｠'), +]; + +pub const V4_0: &'static [(char, char)] = &[ + ('ȡ', 'ȡ'), + ('ȴ', 'ȶ'), + ('ʮ', 'ʯ'), + ('˯', '˿'), + ('\u{350}', '\u{357}'), + ('\u{35d}', '\u{35f}'), + ('Ϸ', 'ϻ'), + ('\u{600}', '\u{603}'), + ('؍', '\u{615}'), + ('\u{656}', '\u{658}'), + ('ۮ', 'ۯ'), + ('ۿ', 'ۿ'), + ('ܭ', 'ܯ'), + ('ݍ', 'ݏ'), + ('ऄ', 'ऄ'), + ('ঽ', 'ঽ'), + ('\u{a01}', '\u{a01}'), + ('ਃ', 'ਃ'), + ('ઌ', 'ઌ'), + ('ૡ', '\u{ae3}'), + ('૱', '૱'), + ('ଵ', 'ଵ'), + ('ୱ', 'ୱ'), + ('௳', '௺'), + ('\u{cbc}', 'ಽ'), + ('\u{17dd}', '\u{17dd}'), + ('៰', '៹'), + ('ᤀ', 'ᤜ'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('᥀', '᥀'), + ('᥄', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('᧠', '᧿'), + ('ᴀ', 'ᵫ'), + ('⁓', '⁔'), + ('℻', '℻'), + ('⏏', '⏐'), + ('⓿', '⓿'), + ('☔', '☕'), + ('⚊', '⚑'), + ('⚠', '⚡'), + ('⬀', '⬍'), + ('㈝', '㈞'), + ('㉐', '㉐'), + ('㉼', '㉽'), + ('㋌', '㋏'), + ('㍷', '㍺'), + ('㏞', '㏟'), + ('㏿', '㏿'), + ('䷀', '䷿'), + ('﷽', '﷽'), + ('﹇', '﹈'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐄀', '𐄂'), + ('𐄇', '𐄳'), + ('𐄷', '𐄿'), + ('𐎀', '𐎝'), + ('𐎟', '𐎟'), + ('𐐦', '𐐧'), + ('𐑎', '𐒝'), + ('𐒠', '𐒩'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐠿'), + ('𝌀', '𝍖'), + ('𝓁', '𝓁'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const V4_1: &'static [(char, char)] = &[ + ('ȷ', 'Ɂ'), + ('\u{358}', '\u{35c}'), + ('ϼ', 'Ͽ'), + ('Ӷ', 'ӷ'), + ('\u{5a2}', '\u{5a2}'), + ('\u{5c5}', '\u{5c7}'), + ('؋', '؋'), + ('؞', '؞'), + ('\u{659}', '\u{65e}'), + ('ݐ', 'ݭ'), + ('ॽ', 'ॽ'), + ('ৎ', 'ৎ'), + ('ஶ', 'ஶ'), + ('௦', '௦'), + ('࿐', '࿑'), + ('ჹ', 'ჺ'), + ('ჼ', 'ჼ'), + ('ሇ', 'ሇ'), + ('ቇ', 'ቇ'), + ('ኇ', 'ኇ'), + ('ኯ', 'ኯ'), + ('ዏ', 'ዏ'), + ('ዯ', 'ዯ'), + ('ጏ', 'ጏ'), + ('ጟ', 'ጟ'), + ('ፇ', 'ፇ'), + ('\u{135f}', '፠'), + ('ᎀ', '᎙'), + ('ᦀ', 'ᦩ'), + ('ᦰ', 'ᧉ'), + ('᧐', '᧙'), + ('᧞', '᧟'), + ('ᨀ', '\u{1a1b}'), + ('᨞', '᨟'), + ('ᵬ', '\u{1dc3}'), + ('⁕', '⁖'), + ('⁘', '⁞'), + ('ₐ', 'ₔ'), + ('₲', '₵'), + ('\u{20eb}', '\u{20eb}'), + ('ℼ', 'ℼ'), + ('⅌', '⅌'), + ('⏑', '⏛'), + ('☘', '☘'), + ('♾', '♿'), + ('⚒', '⚜'), + ('⚢', '⚱'), + ('⟀', '⟆'), + ('⬎', '⬓'), + ('Ⰰ', 'Ⱞ'), + ('ⰰ', 'ⱞ'), + ('Ⲁ', '⳪'), + ('⳹', 'ⴥ'), + ('ⴰ', 'ⵥ'), + ('ⵯ', 'ⵯ'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('⸀', '⸗'), + ('⸜', '⸝'), + ('㇀', '㇏'), + ('㉾', '㉾'), + ('龦', '龻'), + ('꜀', '꜖'), + ('ꠀ', '꠫'), + ('並', '龎'), + ('︐', '︙'), + ('𐅀', '𐆊'), + ('𐎠', '𐏃'), + ('𐏈', '𐏕'), + ('𐨀', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨳'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '𐩇'), + ('𐩐', '𐩘'), + ('𝈀', '𝉅'), + ('𝚤', '𝚥'), +]; + +pub const V5_0: &'static [(char, char)] = &[ + ('ɂ', 'ɏ'), + ('ͻ', 'ͽ'), + ('ӏ', 'ӏ'), + ('Ӻ', 'ӿ'), + ('Ԑ', 'ԓ'), + ('\u{5ba}', '\u{5ba}'), + ('߀', 'ߺ'), + ('ॻ', 'ॼ'), + ('ॾ', 'ॿ'), + ('\u{ce2}', '\u{ce3}'), + ('ೱ', 'ೲ'), + ('\u{1b00}', 'ᭋ'), + ('᭐', '᭼'), + ('\u{1dc4}', '\u{1dca}'), + ('\u{1dfe}', '\u{1dff}'), + ('\u{20ec}', '\u{20ef}'), + ('⅍', 'ⅎ'), + ('ↄ', 'ↄ'), + ('⏜', '⏧'), + ('⚲', '⚲'), + ('⟇', '⟊'), + ('⬔', '⬚'), + ('⬠', '⬣'), + ('Ⱡ', 'ⱬ'), + ('ⱴ', 'ⱷ'), + ('ꜗ', 'ꜚ'), + ('꜠', '꜡'), + ('ꡀ', '꡷'), + ('𐤀', '𐤙'), + ('𐤟', '𐤟'), + ('𒀀', '𒍮'), + ('𒐀', '𒑢'), + ('𒑰', '𒑳'), + ('𝍠', '𝍱'), + ('𝟊', '𝟋'), +]; + +pub const V5_1: &'static [(char, char)] = &[ + ('Ͱ', 'ͳ'), + ('Ͷ', 'ͷ'), + ('Ϗ', 'Ϗ'), + ('\u{487}', '\u{487}'), + ('Ԕ', 'ԣ'), + ('؆', '؊'), + ('\u{616}', '\u{61a}'), + ('ػ', 'ؿ'), + ('ݮ', 'ݿ'), + ('ॱ', 'ॲ'), + ('\u{a51}', '\u{a51}'), + ('\u{a75}', '\u{a75}'), + ('\u{b44}', '\u{b44}'), + ('\u{b62}', '\u{b63}'), + ('ௐ', 'ௐ'), + ('ఽ', 'ఽ'), + ('ౘ', 'ౙ'), + ('\u{c62}', '\u{c63}'), + ('౸', '౿'), + ('ഽ', 'ഽ'), + ('\u{d44}', '\u{d44}'), + ('\u{d62}', '\u{d63}'), + ('൰', '൵'), + ('൹', 'ൿ'), + ('ཫ', 'ཬ'), + ('࿎', '࿎'), + ('࿒', '࿔'), + ('ဢ', 'ဢ'), + ('ဨ', 'ဨ'), + ('ါ', 'ါ'), + ('\u{1033}', '\u{1035}'), + ('\u{103a}', 'ဿ'), + ('ၚ', '႙'), + ('႞', '႟'), + ('ᢪ', 'ᢪ'), + ('\u{1b80}', '\u{1baa}'), + ('ᮮ', '᮹'), + ('ᰀ', '\u{1c37}'), + ('᰻', '᱉'), + ('ᱍ', '᱿'), + ('\u{1dcb}', '\u{1de6}'), + ('ẜ', 'ẟ'), + ('Ỻ', 'ỿ'), + ('\u{2064}', '\u{2064}'), + ('\u{20f0}', '\u{20f0}'), + ('⅏', '⅏'), + ('ↅ', 'ↈ'), + ('⚝', '⚝'), + ('⚳', '⚼'), + ('⛀', '⛃'), + ('⟌', '⟌'), + ('⟬', '⟯'), + ('⬛', '⬟'), + ('⬤', '⭌'), + ('⭐', '⭔'), + ('Ɑ', 'Ɐ'), + ('ⱱ', 'ⱳ'), + ('ⱸ', 'ⱽ'), + ('\u{2de0}', '\u{2dff}'), + ('⸘', '⸛'), + ('⸞', '⸰'), + ('ㄭ', 'ㄭ'), + ('㇐', '㇣'), + ('龼', '鿃'), + ('ꔀ', 'ꘫ'), + ('Ꙁ', 'ꙟ'), + ('Ꙣ', '꙳'), + ('\u{a67c}', 'ꚗ'), + ('ꜛ', 'ꜟ'), + ('Ꜣ', 'ꞌ'), + ('ꟻ', 'ꟿ'), + ('ꢀ', '\u{a8c4}'), + ('꣎', '꣙'), + ('꤀', '\u{a953}'), + ('꥟', '꥟'), + ('ꨀ', '\u{aa36}'), + ('ꩀ', 'ꩍ'), + ('꩐', '꩙'), + ('꩜', '꩟'), + ('\u{fe24}', '\u{fe26}'), + ('𐆐', '𐆛'), + ('𐇐', '\u{101fd}'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐤠', '𐤹'), + ('𐤿', '𐤿'), + ('𝄩', '𝄩'), + ('🀀', '🀫'), + ('🀰', '🂓'), +]; + +pub const V5_2: &'static [(char, char)] = &[ + ('Ԥ', 'ԥ'), + ('ࠀ', '\u{82d}'), + ('࠰', '࠾'), + ('\u{900}', '\u{900}'), + ('ॎ', 'ॎ'), + ('\u{955}', '\u{955}'), + ('ॹ', 'ॺ'), + ('৻', '৻'), + ('࿕', '࿘'), + ('ႚ', '\u{109d}'), + ('ᅚ', 'ᅞ'), + ('ᆣ', 'ᆧ'), + ('ᇺ', 'ᇿ'), + ('᐀', '᐀'), + ('ᙷ', 'ᙿ'), + ('ᢰ', 'ᣵ'), + ('ᦪ', 'ᦫ'), + ('᧚', '᧚'), + ('ᨠ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '᪉'), + ('᪐', '᪙'), + ('᪠', '᪭'), + ('\u{1cd0}', 'ᳲ'), + ('\u{1dfd}', '\u{1dfd}'), + ('₶', '₸'), + ('⅐', '⅒'), + ('↉', '↉'), + ('⏨', '⏨'), + ('⚞', '⚟'), + ('⚽', '⚿'), + ('⛄', '⛍'), + ('⛏', '⛡'), + ('⛣', '⛣'), + ('⛨', '⛿'), + ('❗', '❗'), + ('⭕', '⭙'), + ('Ɒ', 'Ɒ'), + ('Ȿ', 'Ɀ'), + ('Ⳬ', '\u{2cf1}'), + ('⸱', '⸱'), + ('㉄', '㉏'), + ('鿄', '鿋'), + ('ꓐ', '꓿'), + ('ꚠ', '꛷'), + ('꠰', '꠹'), + ('\u{a8e0}', 'ꣻ'), + ('ꥠ', 'ꥼ'), + ('\u{a980}', '꧍'), + ('ꧏ', '꧙'), + ('꧞', '꧟'), + ('ꩠ', 'ꩻ'), + ('ꪀ', 'ꫂ'), + ('ꫛ', '꫟'), + ('ꯀ', '\u{abed}'), + ('꯰', '꯹'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('恵', '舘'), + ('𐡀', '𐡕'), + ('𐡗', '𐡟'), + ('𐤚', '𐤛'), + ('𐩠', '𐩿'), + ('𐬀', '𐬵'), + ('𐬹', '𐭕'), + ('𐭘', '𐭲'), + ('𐭸', '𐭿'), + ('𐰀', '𐱈'), + ('𐹠', '𐹾'), + ('\u{11080}', '𑃁'), + ('𓀀', '𓐮'), + ('🄀', '🄊'), + ('🄐', '🄮'), + ('🄱', '🄱'), + ('🄽', '🄽'), + ('🄿', '🄿'), + ('🅂', '🅂'), + ('🅆', '🅆'), + ('🅊', '🅎'), + ('🅗', '🅗'), + ('🅟', '🅟'), + ('🅹', '🅹'), + ('🅻', '🅼'), + ('🅿', '🅿'), + ('🆊', '🆍'), + ('🆐', '🆐'), + ('🈀', '🈀'), + ('🈐', '🈱'), + ('🉀', '🉈'), + ('𪜀', '𫜴'), +]; + +pub const V6_0: &'static [(char, char)] = &[ + ('Ԧ', 'ԧ'), + ('ؠ', 'ؠ'), + ('\u{65f}', '\u{65f}'), + ('ࡀ', '\u{85b}'), + ('࡞', '࡞'), + ('\u{93a}', 'ऻ'), + ('ॏ', 'ॏ'), + ('\u{956}', '\u{957}'), + ('ॳ', 'ॷ'), + ('୲', '୷'), + ('ഩ', 'ഩ'), + ('ഺ', 'ഺ'), + ('ൎ', 'ൎ'), + ('ྌ', '\u{f8f}'), + ('࿙', '࿚'), + ('\u{135d}', '\u{135e}'), + ('ᯀ', '\u{1bf3}'), + ('᯼', '᯿'), + ('\u{1dfc}', '\u{1dfc}'), + ('ₕ', 'ₜ'), + ('₹', '₹'), + ('⏩', '⏳'), + ('⛎', '⛎'), + ('⛢', '⛢'), + ('⛤', '⛧'), + ('✅', '✅'), + ('✊', '✋'), + ('✨', '✨'), + ('❌', '❌'), + ('❎', '❎'), + ('❓', '❕'), + ('❟', '❠'), + ('➕', '➗'), + ('➰', '➰'), + ('➿', '➿'), + ('⟎', '⟏'), + ('⵰', '⵰'), + ('\u{2d7f}', '\u{2d7f}'), + ('ㆸ', 'ㆺ'), + ('Ꙡ', 'ꙡ'), + ('Ɥ', 'ꞎ'), + ('Ꞑ', 'ꞑ'), + ('Ꞡ', 'ꞩ'), + ('ꟺ', 'ꟺ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('﮲', '﯁'), + ('𑀀', '𑁍'), + ('𑁒', '𑁯'), + ('𖠀', '𖨸'), + ('𛀀', '𛀁'), + ('🂠', '🂮'), + ('🂱', '🂾'), + ('🃁', '🃏'), + ('🃑', '🃟'), + ('🄰', '🄰'), + ('🄲', '🄼'), + ('🄾', '🄾'), + ('🅀', '🅁'), + ('🅃', '🅅'), + ('🅇', '🅉'), + ('🅏', '🅖'), + ('🅘', '🅞'), + ('🅠', '🅩'), + ('🅰', '🅸'), + ('🅺', '🅺'), + ('🅽', '🅾'), + ('🆀', '🆉'), + ('🆎', '🆏'), + ('🆑', '🆚'), + ('🇦', '🇿'), + ('🈁', '🈂'), + ('🈲', '🈺'), + ('🉐', '🉑'), + ('🌀', '🌠'), + ('🌰', '🌵'), + ('🌷', '🍼'), + ('🎀', '🎓'), + ('🎠', '🏄'), + ('🏆', '🏊'), + ('🏠', '🏰'), + ('🐀', '🐾'), + ('👀', '👀'), + ('👂', '📷'), + ('📹', '📼'), + ('🔀', '🔽'), + ('🕐', '🕧'), + ('🗻', '🗿'), + ('😁', '😐'), + ('😒', '😔'), + ('😖', '😖'), + ('😘', '😘'), + ('😚', '😚'), + ('😜', '😞'), + ('😠', '😥'), + ('😨', '😫'), + ('😭', '😭'), + ('😰', '😳'), + ('😵', '🙀'), + ('🙅', '🙏'), + ('🚀', '🛅'), + ('🜀', '🝳'), + ('𫝀', '𫠝'), +]; + +pub const V6_1: &'static [(char, char)] = &[ + ('֏', '֏'), + ('\u{604}', '\u{604}'), + ('ࢠ', 'ࢠ'), + ('ࢢ', 'ࢬ'), + ('\u{8e4}', '\u{8fe}'), + ('૰', '૰'), + ('ໞ', 'ໟ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ჽ', 'ჿ'), + ('\u{1bab}', '\u{1bad}'), + ('ᮺ', 'ᮿ'), + ('᳀', '᳇'), + ('ᳳ', 'ᳶ'), + ('⟋', '⟋'), + ('⟍', '⟍'), + ('Ⳳ', 'ⳳ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⵦ', 'ⵧ'), + ('⸲', '⸻'), + ('鿌', '鿌'), + ('\u{a674}', '\u{a67b}'), + ('\u{a69f}', '\u{a69f}'), + ('Ꞓ', 'ꞓ'), + ('Ɦ', 'Ɦ'), + ('ꟸ', 'ꟹ'), + ('ꫠ', '\u{aaf6}'), + ('郞', '隷'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𑃐', '𑃨'), + ('𑃰', '𑃹'), + ('\u{11100}', '\u{11134}'), + ('𑄶', '𑅃'), + ('\u{11180}', '𑇈'), + ('𑇐', '𑇙'), + ('𑚀', '\u{116b7}'), + ('𑛀', '𑛉'), + ('𖼀', '𖽄'), + ('𖽐', '𖽾'), + ('\u{16f8f}', '𖾟'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𞻰', '𞻱'), + ('🅪', '🅫'), + ('🕀', '🕃'), + ('😀', '😀'), + ('😑', '😑'), + ('😕', '😕'), + ('😗', '😗'), + ('😙', '😙'), + ('😛', '😛'), + ('😟', '😟'), + ('😦', '😧'), + ('😬', '😬'), + ('😮', '😯'), + ('😴', '😴'), +]; + +pub const V6_2: &'static [(char, char)] = &[('₺', '₺')]; + +pub const V6_3: &'static [(char, char)] = + &[('\u{61c}', '\u{61c}'), ('\u{2066}', '\u{2069}')]; + +pub const V7_0: &'static [(char, char)] = &[ + ('Ϳ', 'Ϳ'), + ('Ԩ', 'ԯ'), + ('֍', '֎'), + ('\u{605}', '\u{605}'), + ('ࢡ', 'ࢡ'), + ('ࢭ', 'ࢲ'), + ('\u{8ff}', '\u{8ff}'), + ('ॸ', 'ॸ'), + ('ঀ', 'ঀ'), + ('\u{c00}', '\u{c00}'), + ('ఴ', 'ఴ'), + ('\u{c81}', '\u{c81}'), + ('\u{d01}', '\u{d01}'), + ('෦', '෯'), + ('ᛱ', 'ᛸ'), + ('ᤝ', 'ᤞ'), + ('\u{1ab0}', '\u{1abe}'), + ('\u{1cf8}', '\u{1cf9}'), + ('\u{1de7}', '\u{1df5}'), + ('₻', '₽'), + ('⏴', '⏺'), + ('✀', '✀'), + ('⭍', '⭏'), + ('⭚', '⭳'), + ('⭶', '⮕'), + ('⮘', '⮹'), + ('⮽', '⯈'), + ('⯊', '⯑'), + ('⸼', '⹂'), + ('Ꚙ', 'ꚝ'), + ('ꞔ', 'ꞟ'), + ('Ɜ', 'Ɬ'), + ('Ʞ', 'Ʇ'), + ('ꟷ', 'ꟷ'), + ('ꧠ', 'ꧾ'), + ('\u{aa7c}', 'ꩿ'), + ('ꬰ', 'ꭟ'), + ('ꭤ', 'ꭥ'), + ('\u{fe27}', '\u{fe2d}'), + ('𐆋', '𐆌'), + ('𐆠', '𐆠'), + ('\u{102e0}', '𐋻'), + ('𐌟', '𐌟'), + ('𐍐', '\u{1037a}'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕯', '𐕯'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐡠', '𐢞'), + ('𐢧', '𐢯'), + ('𐪀', '𐪟'), + ('𐫀', '\u{10ae6}'), + ('𐫫', '𐫶'), + ('𐮀', '𐮑'), + ('𐮙', '𐮜'), + ('𐮩', '𐮯'), + ('\u{1107f}', '\u{1107f}'), + ('𑅐', '𑅶'), + ('𑇍', '𑇍'), + ('𑇚', '𑇚'), + ('𑇡', '𑇴'), + ('𑈀', '𑈑'), + ('𑈓', '𑈽'), + ('𑊰', '\u{112ea}'), + ('𑋰', '𑋹'), + ('\u{11301}', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('\u{1133c}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('𑍝', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('𑒀', '𑓇'), + ('𑓐', '𑓙'), + ('𑖀', '\u{115b5}'), + ('𑖸', '𑗉'), + ('𑘀', '𑙄'), + ('𑙐', '𑙙'), + ('𑢠', '𑣲'), + ('𑣿', '𑣿'), + ('𑫀', '𑫸'), + ('𒍯', '𒎘'), + ('𒑣', '𒑮'), + ('𒑴', '𒑴'), + ('𖩀', '𖩞'), + ('𖩠', '𖩩'), + ('𖩮', '𖩯'), + ('𖫐', '𖫭'), + ('\u{16af0}', '𖫵'), + ('𖬀', '𖭅'), + ('𖭐', '𖭙'), + ('𖭛', '𖭡'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𛲜', '\u{1bca3}'), + ('𞠀', '𞣄'), + ('𞣇', '\u{1e8d6}'), + ('🂿', '🂿'), + ('🃠', '🃵'), + ('🄋', '🄌'), + ('🌡', '🌬'), + ('🌶', '🌶'), + ('🍽', '🍽'), + ('🎔', '🎟'), + ('🏅', '🏅'), + ('🏋', '🏎'), + ('🏔', '🏟'), + ('🏱', '🏷'), + ('🐿', '🐿'), + ('👁', '👁'), + ('📸', '📸'), + ('📽', '📾'), + ('🔾', '🔿'), + ('🕄', '🕊'), + ('🕨', '🕹'), + ('🕻', '🖣'), + ('🖥', '🗺'), + ('🙁', '🙂'), + ('🙐', '🙿'), + ('🛆', '🛏'), + ('🛠', '🛬'), + ('🛰', '🛳'), + ('🞀', '🟔'), + ('🠀', '🠋'), + ('🠐', '🡇'), + ('🡐', '🡙'), + ('🡠', '🢇'), + ('🢐', '🢭'), +]; + +pub const V8_0: &'static [(char, char)] = &[ + ('ࢳ', 'ࢴ'), + ('\u{8e3}', '\u{8e3}'), + ('ૹ', 'ૹ'), + ('ౚ', 'ౚ'), + ('ൟ', 'ൟ'), + ('Ᏽ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('₾', '₾'), + ('↊', '↋'), + ('⯬', '⯯'), + ('鿍', '鿕'), + ('\u{a69e}', '\u{a69e}'), + ('ꞏ', 'ꞏ'), + ('Ʝ', 'ꞷ'), + ('꣼', 'ꣽ'), + ('ꭠ', 'ꭣ'), + ('ꭰ', 'ꮿ'), + ('\u{fe2e}', '\u{fe2f}'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐣻', '𐣿'), + ('𐦼', '𐦽'), + ('𐧀', '𐧏'), + ('𐧒', '𐧿'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐳺', '𐳿'), + ('\u{111c9}', '\u{111cc}'), + ('𑇛', '𑇟'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊩'), + ('\u{11300}', '\u{11300}'), + ('𑍐', '𑍐'), + ('𑗊', '\u{115dd}'), + ('𑜀', '𑜙'), + ('\u{1171d}', '\u{1172b}'), + ('𑜰', '𑜿'), + ('𒎙', '𒎙'), + ('𒒀', '𒕃'), + ('𔐀', '𔙆'), + ('𝇞', '𝇨'), + ('𝠀', '𝪋'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('🌭', '🌯'), + ('🍾', '🍿'), + ('🏏', '🏓'), + ('🏸', '🏿'), + ('📿', '📿'), + ('🕋', '🕏'), + ('🙃', '🙄'), + ('🛐', '🛐'), + ('🤐', '🤘'), + ('🦀', '🦄'), + ('🧀', '🧀'), + ('𫠠', '𬺡'), +]; + +pub const V9_0: &'static [(char, char)] = &[ + ('ࢶ', 'ࢽ'), + ('\u{8d4}', '\u{8e2}'), + ('ಀ', 'ಀ'), + ('൏', '൏'), + ('ൔ', 'ൖ'), + ('൘', '൞'), + ('൶', '൸'), + ('ᲀ', 'ᲈ'), + ('\u{1dfb}', '\u{1dfb}'), + ('⏻', '⏾'), + ('⹃', '⹄'), + ('Ɪ', 'Ɪ'), + ('\u{a8c5}', '\u{a8c5}'), + ('𐆍', '𐆎'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('\u{1123e}', '\u{1123e}'), + ('𑐀', '𑑙'), + ('𑑛', '𑑛'), + ('𑑝', '𑑝'), + ('𑙠', '𑙬'), + ('𑰀', '𑰈'), + ('𑰊', '\u{11c36}'), + ('\u{11c38}', '𑱅'), + ('𑱐', '𑱬'), + ('𑱰', '𑲏'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('𖿠', '𖿠'), + ('𗀀', '𘟬'), + ('𘠀', '𘫲'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('𞤀', '\u{1e94a}'), + ('𞥐', '𞥙'), + ('𞥞', '𞥟'), + ('🆛', '🆬'), + ('🈻', '🈻'), + ('🕺', '🕺'), + ('🖤', '🖤'), + ('🛑', '🛒'), + ('🛴', '🛶'), + ('🤙', '🤞'), + ('🤠', '🤧'), + ('🤰', '🤰'), + ('🤳', '🤾'), + ('🥀', '🥋'), + ('🥐', '🥞'), + ('🦅', '🦑'), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/case_folding_simple.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/case_folding_simple.rs new file mode 100644 index 0000000..07f6ff2 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/case_folding_simple.rs @@ -0,0 +1,2948 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate case-folding-simple ucd-16.0.0 --chars --all-pairs +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const CASE_FOLDING_SIMPLE: &'static [(char, &'static [char])] = &[ + ('A', &['a']), + ('B', &['b']), + ('C', &['c']), + ('D', &['d']), + ('E', &['e']), + ('F', &['f']), + ('G', &['g']), + ('H', &['h']), + ('I', &['i']), + ('J', &['j']), + ('K', &['k', 'K']), + ('L', &['l']), + ('M', &['m']), + ('N', &['n']), + ('O', &['o']), + ('P', &['p']), + ('Q', &['q']), + ('R', &['r']), + ('S', &['s', 'ſ']), + ('T', &['t']), + ('U', &['u']), + ('V', &['v']), + ('W', &['w']), + ('X', &['x']), + ('Y', &['y']), + ('Z', &['z']), + ('a', &['A']), + ('b', &['B']), + ('c', &['C']), + ('d', &['D']), + ('e', &['E']), + ('f', &['F']), + ('g', &['G']), + ('h', &['H']), + ('i', &['I']), + ('j', &['J']), + ('k', &['K', 'K']), + ('l', &['L']), + ('m', &['M']), + ('n', &['N']), + ('o', &['O']), + ('p', &['P']), + ('q', &['Q']), + ('r', &['R']), + ('s', &['S', 'ſ']), + ('t', &['T']), + ('u', &['U']), + ('v', &['V']), + ('w', &['W']), + ('x', &['X']), + ('y', &['Y']), + ('z', &['Z']), + ('µ', &['Μ', 'μ']), + ('À', &['à']), + ('Á', &['á']), + ('Â', &['â']), + ('Ã', &['ã']), + ('Ä', &['ä']), + ('Å', &['å', 'Å']), + ('Æ', &['æ']), + ('Ç', &['ç']), + ('È', &['è']), + ('É', &['é']), + ('Ê', &['ê']), + ('Ë', &['ë']), + ('Ì', &['ì']), + ('Í', &['í']), + ('Î', &['î']), + ('Ï', &['ï']), + ('Ð', &['ð']), + ('Ñ', &['ñ']), + ('Ò', &['ò']), + ('Ó', &['ó']), + ('Ô', &['ô']), + ('Õ', &['õ']), + ('Ö', &['ö']), + ('Ø', &['ø']), + ('Ù', &['ù']), + ('Ú', &['ú']), + ('Û', &['û']), + ('Ü', &['ü']), + ('Ý', &['ý']), + ('Þ', &['þ']), + ('ß', &['ẞ']), + ('à', &['À']), + ('á', &['Á']), + ('â', &['Â']), + ('ã', &['Ã']), + ('ä', &['Ä']), + ('å', &['Å', 'Å']), + ('æ', &['Æ']), + ('ç', &['Ç']), + ('è', &['È']), + ('é', &['É']), + ('ê', &['Ê']), + ('ë', &['Ë']), + ('ì', &['Ì']), + ('í', &['Í']), + ('î', &['Î']), + ('ï', &['Ï']), + ('ð', &['Ð']), + ('ñ', &['Ñ']), + ('ò', &['Ò']), + ('ó', &['Ó']), + ('ô', &['Ô']), + ('õ', &['Õ']), + ('ö', &['Ö']), + ('ø', &['Ø']), + ('ù', &['Ù']), + ('ú', &['Ú']), + ('û', &['Û']), + ('ü', &['Ü']), + ('ý', &['Ý']), + ('þ', &['Þ']), + ('ÿ', &['Ÿ']), + ('Ā', &['ā']), + ('ā', &['Ā']), + ('Ă', &['ă']), + ('ă', &['Ă']), + ('Ą', &['ą']), + ('ą', &['Ą']), + ('Ć', &['ć']), + ('ć', &['Ć']), + ('Ĉ', &['ĉ']), + ('ĉ', &['Ĉ']), + ('Ċ', &['ċ']), + ('ċ', &['Ċ']), + ('Č', &['č']), + ('č', &['Č']), + ('Ď', &['ď']), + ('ď', &['Ď']), + ('Đ', &['đ']), + ('đ', &['Đ']), + ('Ē', &['ē']), + ('ē', &['Ē']), + ('Ĕ', &['ĕ']), + ('ĕ', &['Ĕ']), + ('Ė', &['ė']), + ('ė', &['Ė']), + ('Ę', &['ę']), + ('ę', &['Ę']), + ('Ě', &['ě']), + ('ě', &['Ě']), + ('Ĝ', &['ĝ']), + ('ĝ', &['Ĝ']), + ('Ğ', &['ğ']), + ('ğ', &['Ğ']), + ('Ġ', &['ġ']), + ('ġ', &['Ġ']), + ('Ģ', &['ģ']), + ('ģ', &['Ģ']), + ('Ĥ', &['ĥ']), + ('ĥ', &['Ĥ']), + ('Ħ', &['ħ']), + ('ħ', &['Ħ']), + ('Ĩ', &['ĩ']), + ('ĩ', &['Ĩ']), + ('Ī', &['ī']), + ('ī', &['Ī']), + ('Ĭ', &['ĭ']), + ('ĭ', &['Ĭ']), + ('Į', &['į']), + ('į', &['Į']), + ('IJ', &['ij']), + ('ij', &['IJ']), + ('Ĵ', &['ĵ']), + ('ĵ', &['Ĵ']), + ('Ķ', &['ķ']), + ('ķ', &['Ķ']), + ('Ĺ', &['ĺ']), + ('ĺ', &['Ĺ']), + ('Ļ', &['ļ']), + ('ļ', &['Ļ']), + ('Ľ', &['ľ']), + ('ľ', &['Ľ']), + ('Ŀ', &['ŀ']), + ('ŀ', &['Ŀ']), + ('Ł', &['ł']), + ('ł', &['Ł']), + ('Ń', &['ń']), + ('ń', &['Ń']), + ('Ņ', &['ņ']), + ('ņ', &['Ņ']), + ('Ň', &['ň']), + ('ň', &['Ň']), + ('Ŋ', &['ŋ']), + ('ŋ', &['Ŋ']), + ('Ō', &['ō']), + ('ō', &['Ō']), + ('Ŏ', &['ŏ']), + ('ŏ', &['Ŏ']), + ('Ő', &['ő']), + ('ő', &['Ő']), + ('Œ', &['œ']), + ('œ', &['Œ']), + ('Ŕ', &['ŕ']), + ('ŕ', &['Ŕ']), + ('Ŗ', &['ŗ']), + ('ŗ', &['Ŗ']), + ('Ř', &['ř']), + ('ř', &['Ř']), + ('Ś', &['ś']), + ('ś', &['Ś']), + ('Ŝ', &['ŝ']), + ('ŝ', &['Ŝ']), + ('Ş', &['ş']), + ('ş', &['Ş']), + ('Š', &['š']), + ('š', &['Š']), + ('Ţ', &['ţ']), + ('ţ', &['Ţ']), + ('Ť', &['ť']), + ('ť', &['Ť']), + ('Ŧ', &['ŧ']), + ('ŧ', &['Ŧ']), + ('Ũ', &['ũ']), + ('ũ', &['Ũ']), + ('Ū', &['ū']), + ('ū', &['Ū']), + ('Ŭ', &['ŭ']), + ('ŭ', &['Ŭ']), + ('Ů', &['ů']), + ('ů', &['Ů']), + ('Ű', &['ű']), + ('ű', &['Ű']), + ('Ų', &['ų']), + ('ų', &['Ų']), + ('Ŵ', &['ŵ']), + ('ŵ', &['Ŵ']), + ('Ŷ', &['ŷ']), + ('ŷ', &['Ŷ']), + ('Ÿ', &['ÿ']), + ('Ź', &['ź']), + ('ź', &['Ź']), + ('Ż', &['ż']), + ('ż', &['Ż']), + ('Ž', &['ž']), + ('ž', &['Ž']), + ('ſ', &['S', 's']), + ('ƀ', &['Ƀ']), + ('Ɓ', &['ɓ']), + ('Ƃ', &['ƃ']), + ('ƃ', &['Ƃ']), + ('Ƅ', &['ƅ']), + ('ƅ', &['Ƅ']), + ('Ɔ', &['ɔ']), + ('Ƈ', &['ƈ']), + ('ƈ', &['Ƈ']), + ('Ɖ', &['ɖ']), + ('Ɗ', &['ɗ']), + ('Ƌ', &['ƌ']), + ('ƌ', &['Ƌ']), + ('Ǝ', &['ǝ']), + ('Ə', &['ə']), + ('Ɛ', &['ɛ']), + ('Ƒ', &['ƒ']), + ('ƒ', &['Ƒ']), + ('Ɠ', &['ɠ']), + ('Ɣ', &['ɣ']), + ('ƕ', &['Ƕ']), + ('Ɩ', &['ɩ']), + ('Ɨ', &['ɨ']), + ('Ƙ', &['ƙ']), + ('ƙ', &['Ƙ']), + ('ƚ', &['Ƚ']), + ('ƛ', &['Ƛ']), + ('Ɯ', &['ɯ']), + ('Ɲ', &['ɲ']), + ('ƞ', &['Ƞ']), + ('Ɵ', &['ɵ']), + ('Ơ', &['ơ']), + ('ơ', &['Ơ']), + ('Ƣ', &['ƣ']), + ('ƣ', &['Ƣ']), + ('Ƥ', &['ƥ']), + ('ƥ', &['Ƥ']), + ('Ʀ', &['ʀ']), + ('Ƨ', &['ƨ']), + ('ƨ', &['Ƨ']), + ('Ʃ', &['ʃ']), + ('Ƭ', &['ƭ']), + ('ƭ', &['Ƭ']), + ('Ʈ', &['ʈ']), + ('Ư', &['ư']), + ('ư', &['Ư']), + ('Ʊ', &['ʊ']), + ('Ʋ', &['ʋ']), + ('Ƴ', &['ƴ']), + ('ƴ', &['Ƴ']), + ('Ƶ', &['ƶ']), + ('ƶ', &['Ƶ']), + ('Ʒ', &['ʒ']), + ('Ƹ', &['ƹ']), + ('ƹ', &['Ƹ']), + ('Ƽ', &['ƽ']), + ('ƽ', &['Ƽ']), + ('ƿ', &['Ƿ']), + ('DŽ', &['Dž', 'dž']), + ('Dž', &['DŽ', 'dž']), + ('dž', &['DŽ', 'Dž']), + ('LJ', &['Lj', 'lj']), + ('Lj', &['LJ', 'lj']), + ('lj', &['LJ', 'Lj']), + ('NJ', &['Nj', 'nj']), + ('Nj', &['NJ', 'nj']), + ('nj', &['NJ', 'Nj']), + ('Ǎ', &['ǎ']), + ('ǎ', &['Ǎ']), + ('Ǐ', &['ǐ']), + ('ǐ', &['Ǐ']), + ('Ǒ', &['ǒ']), + ('ǒ', &['Ǒ']), + ('Ǔ', &['ǔ']), + ('ǔ', &['Ǔ']), + ('Ǖ', &['ǖ']), + ('ǖ', &['Ǖ']), + ('Ǘ', &['ǘ']), + ('ǘ', &['Ǘ']), + ('Ǚ', &['ǚ']), + ('ǚ', &['Ǚ']), + ('Ǜ', &['ǜ']), + ('ǜ', &['Ǜ']), + ('ǝ', &['Ǝ']), + ('Ǟ', &['ǟ']), + ('ǟ', &['Ǟ']), + ('Ǡ', &['ǡ']), + ('ǡ', &['Ǡ']), + ('Ǣ', &['ǣ']), + ('ǣ', &['Ǣ']), + ('Ǥ', &['ǥ']), + ('ǥ', &['Ǥ']), + ('Ǧ', &['ǧ']), + ('ǧ', &['Ǧ']), + ('Ǩ', &['ǩ']), + ('ǩ', &['Ǩ']), + ('Ǫ', &['ǫ']), + ('ǫ', &['Ǫ']), + ('Ǭ', &['ǭ']), + ('ǭ', &['Ǭ']), + ('Ǯ', &['ǯ']), + ('ǯ', &['Ǯ']), + ('DZ', &['Dz', 'dz']), + ('Dz', &['DZ', 'dz']), + ('dz', &['DZ', 'Dz']), + ('Ǵ', &['ǵ']), + ('ǵ', &['Ǵ']), + ('Ƕ', &['ƕ']), + ('Ƿ', &['ƿ']), + ('Ǹ', &['ǹ']), + ('ǹ', &['Ǹ']), + ('Ǻ', &['ǻ']), + ('ǻ', &['Ǻ']), + ('Ǽ', &['ǽ']), + ('ǽ', &['Ǽ']), + ('Ǿ', &['ǿ']), + ('ǿ', &['Ǿ']), + ('Ȁ', &['ȁ']), + ('ȁ', &['Ȁ']), + ('Ȃ', &['ȃ']), + ('ȃ', &['Ȃ']), + ('Ȅ', &['ȅ']), + ('ȅ', &['Ȅ']), + ('Ȇ', &['ȇ']), + ('ȇ', &['Ȇ']), + ('Ȉ', &['ȉ']), + ('ȉ', &['Ȉ']), + ('Ȋ', &['ȋ']), + ('ȋ', &['Ȋ']), + ('Ȍ', &['ȍ']), + ('ȍ', &['Ȍ']), + ('Ȏ', &['ȏ']), + ('ȏ', &['Ȏ']), + ('Ȑ', &['ȑ']), + ('ȑ', &['Ȑ']), + ('Ȓ', &['ȓ']), + ('ȓ', &['Ȓ']), + ('Ȕ', &['ȕ']), + ('ȕ', &['Ȕ']), + ('Ȗ', &['ȗ']), + ('ȗ', &['Ȗ']), + ('Ș', &['ș']), + ('ș', &['Ș']), + ('Ț', &['ț']), + ('ț', &['Ț']), + ('Ȝ', &['ȝ']), + ('ȝ', &['Ȝ']), + ('Ȟ', &['ȟ']), + ('ȟ', &['Ȟ']), + ('Ƞ', &['ƞ']), + ('Ȣ', &['ȣ']), + ('ȣ', &['Ȣ']), + ('Ȥ', &['ȥ']), + ('ȥ', &['Ȥ']), + ('Ȧ', &['ȧ']), + ('ȧ', &['Ȧ']), + ('Ȩ', &['ȩ']), + ('ȩ', &['Ȩ']), + ('Ȫ', &['ȫ']), + ('ȫ', &['Ȫ']), + ('Ȭ', &['ȭ']), + ('ȭ', &['Ȭ']), + ('Ȯ', &['ȯ']), + ('ȯ', &['Ȯ']), + ('Ȱ', &['ȱ']), + ('ȱ', &['Ȱ']), + ('Ȳ', &['ȳ']), + ('ȳ', &['Ȳ']), + ('Ⱥ', &['ⱥ']), + ('Ȼ', &['ȼ']), + ('ȼ', &['Ȼ']), + ('Ƚ', &['ƚ']), + ('Ⱦ', &['ⱦ']), + ('ȿ', &['Ȿ']), + ('ɀ', &['Ɀ']), + ('Ɂ', &['ɂ']), + ('ɂ', &['Ɂ']), + ('Ƀ', &['ƀ']), + ('Ʉ', &['ʉ']), + ('Ʌ', &['ʌ']), + ('Ɇ', &['ɇ']), + ('ɇ', &['Ɇ']), + ('Ɉ', &['ɉ']), + ('ɉ', &['Ɉ']), + ('Ɋ', &['ɋ']), + ('ɋ', &['Ɋ']), + ('Ɍ', &['ɍ']), + ('ɍ', &['Ɍ']), + ('Ɏ', &['ɏ']), + ('ɏ', &['Ɏ']), + ('ɐ', &['Ɐ']), + ('ɑ', &['Ɑ']), + ('ɒ', &['Ɒ']), + ('ɓ', &['Ɓ']), + ('ɔ', &['Ɔ']), + ('ɖ', &['Ɖ']), + ('ɗ', &['Ɗ']), + ('ə', &['Ə']), + ('ɛ', &['Ɛ']), + ('ɜ', &['Ɜ']), + ('ɠ', &['Ɠ']), + ('ɡ', &['Ɡ']), + ('ɣ', &['Ɣ']), + ('ɤ', &['Ɤ']), + ('ɥ', &['Ɥ']), + ('ɦ', &['Ɦ']), + ('ɨ', &['Ɨ']), + ('ɩ', &['Ɩ']), + ('ɪ', &['Ɪ']), + ('ɫ', &['Ɫ']), + ('ɬ', &['Ɬ']), + ('ɯ', &['Ɯ']), + ('ɱ', &['Ɱ']), + ('ɲ', &['Ɲ']), + ('ɵ', &['Ɵ']), + ('ɽ', &['Ɽ']), + ('ʀ', &['Ʀ']), + ('ʂ', &['Ʂ']), + ('ʃ', &['Ʃ']), + ('ʇ', &['Ʇ']), + ('ʈ', &['Ʈ']), + ('ʉ', &['Ʉ']), + ('ʊ', &['Ʊ']), + ('ʋ', &['Ʋ']), + ('ʌ', &['Ʌ']), + ('ʒ', &['Ʒ']), + ('ʝ', &['Ʝ']), + ('ʞ', &['Ʞ']), + ('\u{345}', &['Ι', 'ι', 'ι']), + ('Ͱ', &['ͱ']), + ('ͱ', &['Ͱ']), + ('Ͳ', &['ͳ']), + ('ͳ', &['Ͳ']), + ('Ͷ', &['ͷ']), + ('ͷ', &['Ͷ']), + ('ͻ', &['Ͻ']), + ('ͼ', &['Ͼ']), + ('ͽ', &['Ͽ']), + ('Ϳ', &['ϳ']), + ('Ά', &['ά']), + ('Έ', &['έ']), + ('Ή', &['ή']), + ('Ί', &['ί']), + ('Ό', &['ό']), + ('Ύ', &['ύ']), + ('Ώ', &['ώ']), + ('ΐ', &['ΐ']), + ('Α', &['α']), + ('Β', &['β', 'ϐ']), + ('Γ', &['γ']), + ('Δ', &['δ']), + ('Ε', &['ε', 'ϵ']), + ('Ζ', &['ζ']), + ('Η', &['η']), + ('Θ', &['θ', 'ϑ', 'ϴ']), + ('Ι', &['\u{345}', 'ι', 'ι']), + ('Κ', &['κ', 'ϰ']), + ('Λ', &['λ']), + ('Μ', &['µ', 'μ']), + ('Ν', &['ν']), + ('Ξ', &['ξ']), + ('Ο', &['ο']), + ('Π', &['π', 'ϖ']), + ('Ρ', &['ρ', 'ϱ']), + ('Σ', &['ς', 'σ']), + ('Τ', &['τ']), + ('Υ', &['υ']), + ('Φ', &['φ', 'ϕ']), + ('Χ', &['χ']), + ('Ψ', &['ψ']), + ('Ω', &['ω', 'Ω']), + ('Ϊ', &['ϊ']), + ('Ϋ', &['ϋ']), + ('ά', &['Ά']), + ('έ', &['Έ']), + ('ή', &['Ή']), + ('ί', &['Ί']), + ('ΰ', &['ΰ']), + ('α', &['Α']), + ('β', &['Β', 'ϐ']), + ('γ', &['Γ']), + ('δ', &['Δ']), + ('ε', &['Ε', 'ϵ']), + ('ζ', &['Ζ']), + ('η', &['Η']), + ('θ', &['Θ', 'ϑ', 'ϴ']), + ('ι', &['\u{345}', 'Ι', 'ι']), + ('κ', &['Κ', 'ϰ']), + ('λ', &['Λ']), + ('μ', &['µ', 'Μ']), + ('ν', &['Ν']), + ('ξ', &['Ξ']), + ('ο', &['Ο']), + ('π', &['Π', 'ϖ']), + ('ρ', &['Ρ', 'ϱ']), + ('ς', &['Σ', 'σ']), + ('σ', &['Σ', 'ς']), + ('τ', &['Τ']), + ('υ', &['Υ']), + ('φ', &['Φ', 'ϕ']), + ('χ', &['Χ']), + ('ψ', &['Ψ']), + ('ω', &['Ω', 'Ω']), + ('ϊ', &['Ϊ']), + ('ϋ', &['Ϋ']), + ('ό', &['Ό']), + ('ύ', &['Ύ']), + ('ώ', &['Ώ']), + ('Ϗ', &['ϗ']), + ('ϐ', &['Β', 'β']), + ('ϑ', &['Θ', 'θ', 'ϴ']), + ('ϕ', &['Φ', 'φ']), + ('ϖ', &['Π', 'π']), + ('ϗ', &['Ϗ']), + ('Ϙ', &['ϙ']), + ('ϙ', &['Ϙ']), + ('Ϛ', &['ϛ']), + ('ϛ', &['Ϛ']), + ('Ϝ', &['ϝ']), + ('ϝ', &['Ϝ']), + ('Ϟ', &['ϟ']), + ('ϟ', &['Ϟ']), + ('Ϡ', &['ϡ']), + ('ϡ', &['Ϡ']), + ('Ϣ', &['ϣ']), + ('ϣ', &['Ϣ']), + ('Ϥ', &['ϥ']), + ('ϥ', &['Ϥ']), + ('Ϧ', &['ϧ']), + ('ϧ', &['Ϧ']), + ('Ϩ', &['ϩ']), + ('ϩ', &['Ϩ']), + ('Ϫ', &['ϫ']), + ('ϫ', &['Ϫ']), + ('Ϭ', &['ϭ']), + ('ϭ', &['Ϭ']), + ('Ϯ', &['ϯ']), + ('ϯ', &['Ϯ']), + ('ϰ', &['Κ', 'κ']), + ('ϱ', &['Ρ', 'ρ']), + ('ϲ', &['Ϲ']), + ('ϳ', &['Ϳ']), + ('ϴ', &['Θ', 'θ', 'ϑ']), + ('ϵ', &['Ε', 'ε']), + ('Ϸ', &['ϸ']), + ('ϸ', &['Ϸ']), + ('Ϲ', &['ϲ']), + ('Ϻ', &['ϻ']), + ('ϻ', &['Ϻ']), + ('Ͻ', &['ͻ']), + ('Ͼ', &['ͼ']), + ('Ͽ', &['ͽ']), + ('Ѐ', &['ѐ']), + ('Ё', &['ё']), + ('Ђ', &['ђ']), + ('Ѓ', &['ѓ']), + ('Є', &['є']), + ('Ѕ', &['ѕ']), + ('І', &['і']), + ('Ї', &['ї']), + ('Ј', &['ј']), + ('Љ', &['љ']), + ('Њ', &['њ']), + ('Ћ', &['ћ']), + ('Ќ', &['ќ']), + ('Ѝ', &['ѝ']), + ('Ў', &['ў']), + ('Џ', &['џ']), + ('А', &['а']), + ('Б', &['б']), + ('В', &['в', 'ᲀ']), + ('Г', &['г']), + ('Д', &['д', 'ᲁ']), + ('Е', &['е']), + ('Ж', &['ж']), + ('З', &['з']), + ('И', &['и']), + ('Й', &['й']), + ('К', &['к']), + ('Л', &['л']), + ('М', &['м']), + ('Н', &['н']), + ('О', &['о', 'ᲂ']), + ('П', &['п']), + ('Р', &['р']), + ('С', &['с', 'ᲃ']), + ('Т', &['т', 'ᲄ', 'ᲅ']), + ('У', &['у']), + ('Ф', &['ф']), + ('Х', &['х']), + ('Ц', &['ц']), + ('Ч', &['ч']), + ('Ш', &['ш']), + ('Щ', &['щ']), + ('Ъ', &['ъ', 'ᲆ']), + ('Ы', &['ы']), + ('Ь', &['ь']), + ('Э', &['э']), + ('Ю', &['ю']), + ('Я', &['я']), + ('а', &['А']), + ('б', &['Б']), + ('в', &['В', 'ᲀ']), + ('г', &['Г']), + ('д', &['Д', 'ᲁ']), + ('е', &['Е']), + ('ж', &['Ж']), + ('з', &['З']), + ('и', &['И']), + ('й', &['Й']), + ('к', &['К']), + ('л', &['Л']), + ('м', &['М']), + ('н', &['Н']), + ('о', &['О', 'ᲂ']), + ('п', &['П']), + ('р', &['Р']), + ('с', &['С', 'ᲃ']), + ('т', &['Т', 'ᲄ', 'ᲅ']), + ('у', &['У']), + ('ф', &['Ф']), + ('х', &['Х']), + ('ц', &['Ц']), + ('ч', &['Ч']), + ('ш', &['Ш']), + ('щ', &['Щ']), + ('ъ', &['Ъ', 'ᲆ']), + ('ы', &['Ы']), + ('ь', &['Ь']), + ('э', &['Э']), + ('ю', &['Ю']), + ('я', &['Я']), + ('ѐ', &['Ѐ']), + ('ё', &['Ё']), + ('ђ', &['Ђ']), + ('ѓ', &['Ѓ']), + ('є', &['Є']), + ('ѕ', &['Ѕ']), + ('і', &['І']), + ('ї', &['Ї']), + ('ј', &['Ј']), + ('љ', &['Љ']), + ('њ', &['Њ']), + ('ћ', &['Ћ']), + ('ќ', &['Ќ']), + ('ѝ', &['Ѝ']), + ('ў', &['Ў']), + ('џ', &['Џ']), + ('Ѡ', &['ѡ']), + ('ѡ', &['Ѡ']), + ('Ѣ', &['ѣ', 'ᲇ']), + ('ѣ', &['Ѣ', 'ᲇ']), + ('Ѥ', &['ѥ']), + ('ѥ', &['Ѥ']), + ('Ѧ', &['ѧ']), + ('ѧ', &['Ѧ']), + ('Ѩ', &['ѩ']), + ('ѩ', &['Ѩ']), + ('Ѫ', &['ѫ']), + ('ѫ', &['Ѫ']), + ('Ѭ', &['ѭ']), + ('ѭ', &['Ѭ']), + ('Ѯ', &['ѯ']), + ('ѯ', &['Ѯ']), + ('Ѱ', &['ѱ']), + ('ѱ', &['Ѱ']), + ('Ѳ', &['ѳ']), + ('ѳ', &['Ѳ']), + ('Ѵ', &['ѵ']), + ('ѵ', &['Ѵ']), + ('Ѷ', &['ѷ']), + ('ѷ', &['Ѷ']), + ('Ѹ', &['ѹ']), + ('ѹ', &['Ѹ']), + ('Ѻ', &['ѻ']), + ('ѻ', &['Ѻ']), + ('Ѽ', &['ѽ']), + ('ѽ', &['Ѽ']), + ('Ѿ', &['ѿ']), + ('ѿ', &['Ѿ']), + ('Ҁ', &['ҁ']), + ('ҁ', &['Ҁ']), + ('Ҋ', &['ҋ']), + ('ҋ', &['Ҋ']), + ('Ҍ', &['ҍ']), + ('ҍ', &['Ҍ']), + ('Ҏ', &['ҏ']), + ('ҏ', &['Ҏ']), + ('Ґ', &['ґ']), + ('ґ', &['Ґ']), + ('Ғ', &['ғ']), + ('ғ', &['Ғ']), + ('Ҕ', &['ҕ']), + ('ҕ', &['Ҕ']), + ('Җ', &['җ']), + ('җ', &['Җ']), + ('Ҙ', &['ҙ']), + ('ҙ', &['Ҙ']), + ('Қ', &['қ']), + ('қ', &['Қ']), + ('Ҝ', &['ҝ']), + ('ҝ', &['Ҝ']), + ('Ҟ', &['ҟ']), + ('ҟ', &['Ҟ']), + ('Ҡ', &['ҡ']), + ('ҡ', &['Ҡ']), + ('Ң', &['ң']), + ('ң', &['Ң']), + ('Ҥ', &['ҥ']), + ('ҥ', &['Ҥ']), + ('Ҧ', &['ҧ']), + ('ҧ', &['Ҧ']), + ('Ҩ', &['ҩ']), + ('ҩ', &['Ҩ']), + ('Ҫ', &['ҫ']), + ('ҫ', &['Ҫ']), + ('Ҭ', &['ҭ']), + ('ҭ', &['Ҭ']), + ('Ү', &['ү']), + ('ү', &['Ү']), + ('Ұ', &['ұ']), + ('ұ', &['Ұ']), + ('Ҳ', &['ҳ']), + ('ҳ', &['Ҳ']), + ('Ҵ', &['ҵ']), + ('ҵ', &['Ҵ']), + ('Ҷ', &['ҷ']), + ('ҷ', &['Ҷ']), + ('Ҹ', &['ҹ']), + ('ҹ', &['Ҹ']), + ('Һ', &['һ']), + ('һ', &['Һ']), + ('Ҽ', &['ҽ']), + ('ҽ', &['Ҽ']), + ('Ҿ', &['ҿ']), + ('ҿ', &['Ҿ']), + ('Ӏ', &['ӏ']), + ('Ӂ', &['ӂ']), + ('ӂ', &['Ӂ']), + ('Ӄ', &['ӄ']), + ('ӄ', &['Ӄ']), + ('Ӆ', &['ӆ']), + ('ӆ', &['Ӆ']), + ('Ӈ', &['ӈ']), + ('ӈ', &['Ӈ']), + ('Ӊ', &['ӊ']), + ('ӊ', &['Ӊ']), + ('Ӌ', &['ӌ']), + ('ӌ', &['Ӌ']), + ('Ӎ', &['ӎ']), + ('ӎ', &['Ӎ']), + ('ӏ', &['Ӏ']), + ('Ӑ', &['ӑ']), + ('ӑ', &['Ӑ']), + ('Ӓ', &['ӓ']), + ('ӓ', &['Ӓ']), + ('Ӕ', &['ӕ']), + ('ӕ', &['Ӕ']), + ('Ӗ', &['ӗ']), + ('ӗ', &['Ӗ']), + ('Ә', &['ә']), + ('ә', &['Ә']), + ('Ӛ', &['ӛ']), + ('ӛ', &['Ӛ']), + ('Ӝ', &['ӝ']), + ('ӝ', &['Ӝ']), + ('Ӟ', &['ӟ']), + ('ӟ', &['Ӟ']), + ('Ӡ', &['ӡ']), + ('ӡ', &['Ӡ']), + ('Ӣ', &['ӣ']), + ('ӣ', &['Ӣ']), + ('Ӥ', &['ӥ']), + ('ӥ', &['Ӥ']), + ('Ӧ', &['ӧ']), + ('ӧ', &['Ӧ']), + ('Ө', &['ө']), + ('ө', &['Ө']), + ('Ӫ', &['ӫ']), + ('ӫ', &['Ӫ']), + ('Ӭ', &['ӭ']), + ('ӭ', &['Ӭ']), + ('Ӯ', &['ӯ']), + ('ӯ', &['Ӯ']), + ('Ӱ', &['ӱ']), + ('ӱ', &['Ӱ']), + ('Ӳ', &['ӳ']), + ('ӳ', &['Ӳ']), + ('Ӵ', &['ӵ']), + ('ӵ', &['Ӵ']), + ('Ӷ', &['ӷ']), + ('ӷ', &['Ӷ']), + ('Ӹ', &['ӹ']), + ('ӹ', &['Ӹ']), + ('Ӻ', &['ӻ']), + ('ӻ', &['Ӻ']), + ('Ӽ', &['ӽ']), + ('ӽ', &['Ӽ']), + ('Ӿ', &['ӿ']), + ('ӿ', &['Ӿ']), + ('Ԁ', &['ԁ']), + ('ԁ', &['Ԁ']), + ('Ԃ', &['ԃ']), + ('ԃ', &['Ԃ']), + ('Ԅ', &['ԅ']), + ('ԅ', &['Ԅ']), + ('Ԇ', &['ԇ']), + ('ԇ', &['Ԇ']), + ('Ԉ', &['ԉ']), + ('ԉ', &['Ԉ']), + ('Ԋ', &['ԋ']), + ('ԋ', &['Ԋ']), + ('Ԍ', &['ԍ']), + ('ԍ', &['Ԍ']), + ('Ԏ', &['ԏ']), + ('ԏ', &['Ԏ']), + ('Ԑ', &['ԑ']), + ('ԑ', &['Ԑ']), + ('Ԓ', &['ԓ']), + ('ԓ', &['Ԓ']), + ('Ԕ', &['ԕ']), + ('ԕ', &['Ԕ']), + ('Ԗ', &['ԗ']), + ('ԗ', &['Ԗ']), + ('Ԙ', &['ԙ']), + ('ԙ', &['Ԙ']), + ('Ԛ', &['ԛ']), + ('ԛ', &['Ԛ']), + ('Ԝ', &['ԝ']), + ('ԝ', &['Ԝ']), + ('Ԟ', &['ԟ']), + ('ԟ', &['Ԟ']), + ('Ԡ', &['ԡ']), + ('ԡ', &['Ԡ']), + ('Ԣ', &['ԣ']), + ('ԣ', &['Ԣ']), + ('Ԥ', &['ԥ']), + ('ԥ', &['Ԥ']), + ('Ԧ', &['ԧ']), + ('ԧ', &['Ԧ']), + ('Ԩ', &['ԩ']), + ('ԩ', &['Ԩ']), + ('Ԫ', &['ԫ']), + ('ԫ', &['Ԫ']), + ('Ԭ', &['ԭ']), + ('ԭ', &['Ԭ']), + ('Ԯ', &['ԯ']), + ('ԯ', &['Ԯ']), + ('Ա', &['ա']), + ('Բ', &['բ']), + ('Գ', &['գ']), + ('Դ', &['դ']), + ('Ե', &['ե']), + ('Զ', &['զ']), + ('Է', &['է']), + ('Ը', &['ը']), + ('Թ', &['թ']), + ('Ժ', &['ժ']), + ('Ի', &['ի']), + ('Լ', &['լ']), + ('Խ', &['խ']), + ('Ծ', &['ծ']), + ('Կ', &['կ']), + ('Հ', &['հ']), + ('Ձ', &['ձ']), + ('Ղ', &['ղ']), + ('Ճ', &['ճ']), + ('Մ', &['մ']), + ('Յ', &['յ']), + ('Ն', &['ն']), + ('Շ', &['շ']), + ('Ո', &['ո']), + ('Չ', &['չ']), + ('Պ', &['պ']), + ('Ջ', &['ջ']), + ('Ռ', &['ռ']), + ('Ս', &['ս']), + ('Վ', &['վ']), + ('Տ', &['տ']), + ('Ր', &['ր']), + ('Ց', &['ց']), + ('Ւ', &['ւ']), + ('Փ', &['փ']), + ('Ք', &['ք']), + ('Օ', &['օ']), + ('Ֆ', &['ֆ']), + ('ա', &['Ա']), + ('բ', &['Բ']), + ('գ', &['Գ']), + ('դ', &['Դ']), + ('ե', &['Ե']), + ('զ', &['Զ']), + ('է', &['Է']), + ('ը', &['Ը']), + ('թ', &['Թ']), + ('ժ', &['Ժ']), + ('ի', &['Ի']), + ('լ', &['Լ']), + ('խ', &['Խ']), + ('ծ', &['Ծ']), + ('կ', &['Կ']), + ('հ', &['Հ']), + ('ձ', &['Ձ']), + ('ղ', &['Ղ']), + ('ճ', &['Ճ']), + ('մ', &['Մ']), + ('յ', &['Յ']), + ('ն', &['Ն']), + ('շ', &['Շ']), + ('ո', &['Ո']), + ('չ', &['Չ']), + ('պ', &['Պ']), + ('ջ', &['Ջ']), + ('ռ', &['Ռ']), + ('ս', &['Ս']), + ('վ', &['Վ']), + ('տ', &['Տ']), + ('ր', &['Ր']), + ('ց', &['Ց']), + ('ւ', &['Ւ']), + ('փ', &['Փ']), + ('ք', &['Ք']), + ('օ', &['Օ']), + ('ֆ', &['Ֆ']), + ('Ⴀ', &['ⴀ']), + ('Ⴁ', &['ⴁ']), + ('Ⴂ', &['ⴂ']), + ('Ⴃ', &['ⴃ']), + ('Ⴄ', &['ⴄ']), + ('Ⴅ', &['ⴅ']), + ('Ⴆ', &['ⴆ']), + ('Ⴇ', &['ⴇ']), + ('Ⴈ', &['ⴈ']), + ('Ⴉ', &['ⴉ']), + ('Ⴊ', &['ⴊ']), + ('Ⴋ', &['ⴋ']), + ('Ⴌ', &['ⴌ']), + ('Ⴍ', &['ⴍ']), + ('Ⴎ', &['ⴎ']), + ('Ⴏ', &['ⴏ']), + ('Ⴐ', &['ⴐ']), + ('Ⴑ', &['ⴑ']), + ('Ⴒ', &['ⴒ']), + ('Ⴓ', &['ⴓ']), + ('Ⴔ', &['ⴔ']), + ('Ⴕ', &['ⴕ']), + ('Ⴖ', &['ⴖ']), + ('Ⴗ', &['ⴗ']), + ('Ⴘ', &['ⴘ']), + ('Ⴙ', &['ⴙ']), + ('Ⴚ', &['ⴚ']), + ('Ⴛ', &['ⴛ']), + ('Ⴜ', &['ⴜ']), + ('Ⴝ', &['ⴝ']), + ('Ⴞ', &['ⴞ']), + ('Ⴟ', &['ⴟ']), + ('Ⴠ', &['ⴠ']), + ('Ⴡ', &['ⴡ']), + ('Ⴢ', &['ⴢ']), + ('Ⴣ', &['ⴣ']), + ('Ⴤ', &['ⴤ']), + ('Ⴥ', &['ⴥ']), + ('Ⴧ', &['ⴧ']), + ('Ⴭ', &['ⴭ']), + ('ა', &['Ა']), + ('ბ', &['Ბ']), + ('გ', &['Გ']), + ('დ', &['Დ']), + ('ე', &['Ე']), + ('ვ', &['Ვ']), + ('ზ', &['Ზ']), + ('თ', &['Თ']), + ('ი', &['Ი']), + ('კ', &['Კ']), + ('ლ', &['Ლ']), + ('მ', &['Მ']), + ('ნ', &['Ნ']), + ('ო', &['Ო']), + ('პ', &['Პ']), + ('ჟ', &['Ჟ']), + ('რ', &['Რ']), + ('ს', &['Ს']), + ('ტ', &['Ტ']), + ('უ', &['Უ']), + ('ფ', &['Ფ']), + ('ქ', &['Ქ']), + ('ღ', &['Ღ']), + ('ყ', &['Ყ']), + ('შ', &['Შ']), + ('ჩ', &['Ჩ']), + ('ც', &['Ც']), + ('ძ', &['Ძ']), + ('წ', &['Წ']), + ('ჭ', &['Ჭ']), + ('ხ', &['Ხ']), + ('ჯ', &['Ჯ']), + ('ჰ', &['Ჰ']), + ('ჱ', &['Ჱ']), + ('ჲ', &['Ჲ']), + ('ჳ', &['Ჳ']), + ('ჴ', &['Ჴ']), + ('ჵ', &['Ჵ']), + ('ჶ', &['Ჶ']), + ('ჷ', &['Ჷ']), + ('ჸ', &['Ჸ']), + ('ჹ', &['Ჹ']), + ('ჺ', &['Ჺ']), + ('ჽ', &['Ჽ']), + ('ჾ', &['Ჾ']), + ('ჿ', &['Ჿ']), + ('Ꭰ', &['ꭰ']), + ('Ꭱ', &['ꭱ']), + ('Ꭲ', &['ꭲ']), + ('Ꭳ', &['ꭳ']), + ('Ꭴ', &['ꭴ']), + ('Ꭵ', &['ꭵ']), + ('Ꭶ', &['ꭶ']), + ('Ꭷ', &['ꭷ']), + ('Ꭸ', &['ꭸ']), + ('Ꭹ', &['ꭹ']), + ('Ꭺ', &['ꭺ']), + ('Ꭻ', &['ꭻ']), + ('Ꭼ', &['ꭼ']), + ('Ꭽ', &['ꭽ']), + ('Ꭾ', &['ꭾ']), + ('Ꭿ', &['ꭿ']), + ('Ꮀ', &['ꮀ']), + ('Ꮁ', &['ꮁ']), + ('Ꮂ', &['ꮂ']), + ('Ꮃ', &['ꮃ']), + ('Ꮄ', &['ꮄ']), + ('Ꮅ', &['ꮅ']), + ('Ꮆ', &['ꮆ']), + ('Ꮇ', &['ꮇ']), + ('Ꮈ', &['ꮈ']), + ('Ꮉ', &['ꮉ']), + ('Ꮊ', &['ꮊ']), + ('Ꮋ', &['ꮋ']), + ('Ꮌ', &['ꮌ']), + ('Ꮍ', &['ꮍ']), + ('Ꮎ', &['ꮎ']), + ('Ꮏ', &['ꮏ']), + ('Ꮐ', &['ꮐ']), + ('Ꮑ', &['ꮑ']), + ('Ꮒ', &['ꮒ']), + ('Ꮓ', &['ꮓ']), + ('Ꮔ', &['ꮔ']), + ('Ꮕ', &['ꮕ']), + ('Ꮖ', &['ꮖ']), + ('Ꮗ', &['ꮗ']), + ('Ꮘ', &['ꮘ']), + ('Ꮙ', &['ꮙ']), + ('Ꮚ', &['ꮚ']), + ('Ꮛ', &['ꮛ']), + ('Ꮜ', &['ꮜ']), + ('Ꮝ', &['ꮝ']), + ('Ꮞ', &['ꮞ']), + ('Ꮟ', &['ꮟ']), + ('Ꮠ', &['ꮠ']), + ('Ꮡ', &['ꮡ']), + ('Ꮢ', &['ꮢ']), + ('Ꮣ', &['ꮣ']), + ('Ꮤ', &['ꮤ']), + ('Ꮥ', &['ꮥ']), + ('Ꮦ', &['ꮦ']), + ('Ꮧ', &['ꮧ']), + ('Ꮨ', &['ꮨ']), + ('Ꮩ', &['ꮩ']), + ('Ꮪ', &['ꮪ']), + ('Ꮫ', &['ꮫ']), + ('Ꮬ', &['ꮬ']), + ('Ꮭ', &['ꮭ']), + ('Ꮮ', &['ꮮ']), + ('Ꮯ', &['ꮯ']), + ('Ꮰ', &['ꮰ']), + ('Ꮱ', &['ꮱ']), + ('Ꮲ', &['ꮲ']), + ('Ꮳ', &['ꮳ']), + ('Ꮴ', &['ꮴ']), + ('Ꮵ', &['ꮵ']), + ('Ꮶ', &['ꮶ']), + ('Ꮷ', &['ꮷ']), + ('Ꮸ', &['ꮸ']), + ('Ꮹ', &['ꮹ']), + ('Ꮺ', &['ꮺ']), + ('Ꮻ', &['ꮻ']), + ('Ꮼ', &['ꮼ']), + ('Ꮽ', &['ꮽ']), + ('Ꮾ', &['ꮾ']), + ('Ꮿ', &['ꮿ']), + ('Ᏸ', &['ᏸ']), + ('Ᏹ', &['ᏹ']), + ('Ᏺ', &['ᏺ']), + ('Ᏻ', &['ᏻ']), + ('Ᏼ', &['ᏼ']), + ('Ᏽ', &['ᏽ']), + ('ᏸ', &['Ᏸ']), + ('ᏹ', &['Ᏹ']), + ('ᏺ', &['Ᏺ']), + ('ᏻ', &['Ᏻ']), + ('ᏼ', &['Ᏼ']), + ('ᏽ', &['Ᏽ']), + ('ᲀ', &['В', 'в']), + ('ᲁ', &['Д', 'д']), + ('ᲂ', &['О', 'о']), + ('ᲃ', &['С', 'с']), + ('ᲄ', &['Т', 'т', 'ᲅ']), + ('ᲅ', &['Т', 'т', 'ᲄ']), + ('ᲆ', &['Ъ', 'ъ']), + ('ᲇ', &['Ѣ', 'ѣ']), + ('ᲈ', &['Ꙋ', 'ꙋ']), + ('Ᲊ', &['ᲊ']), + ('ᲊ', &['Ᲊ']), + ('Ა', &['ა']), + ('Ბ', &['ბ']), + ('Გ', &['გ']), + ('Დ', &['დ']), + ('Ე', &['ე']), + ('Ვ', &['ვ']), + ('Ზ', &['ზ']), + ('Თ', &['თ']), + ('Ი', &['ი']), + ('Კ', &['კ']), + ('Ლ', &['ლ']), + ('Მ', &['მ']), + ('Ნ', &['ნ']), + ('Ო', &['ო']), + ('Პ', &['პ']), + ('Ჟ', &['ჟ']), + ('Რ', &['რ']), + ('Ს', &['ს']), + ('Ტ', &['ტ']), + ('Უ', &['უ']), + ('Ფ', &['ფ']), + ('Ქ', &['ქ']), + ('Ღ', &['ღ']), + ('Ყ', &['ყ']), + ('Შ', &['შ']), + ('Ჩ', &['ჩ']), + ('Ც', &['ც']), + ('Ძ', &['ძ']), + ('Წ', &['წ']), + ('Ჭ', &['ჭ']), + ('Ხ', &['ხ']), + ('Ჯ', &['ჯ']), + ('Ჰ', &['ჰ']), + ('Ჱ', &['ჱ']), + ('Ჲ', &['ჲ']), + ('Ჳ', &['ჳ']), + ('Ჴ', &['ჴ']), + ('Ჵ', &['ჵ']), + ('Ჶ', &['ჶ']), + ('Ჷ', &['ჷ']), + ('Ჸ', &['ჸ']), + ('Ჹ', &['ჹ']), + ('Ჺ', &['ჺ']), + ('Ჽ', &['ჽ']), + ('Ჾ', &['ჾ']), + ('Ჿ', &['ჿ']), + ('ᵹ', &['Ᵹ']), + ('ᵽ', &['Ᵽ']), + ('ᶎ', &['Ᶎ']), + ('Ḁ', &['ḁ']), + ('ḁ', &['Ḁ']), + ('Ḃ', &['ḃ']), + ('ḃ', &['Ḃ']), + ('Ḅ', &['ḅ']), + ('ḅ', &['Ḅ']), + ('Ḇ', &['ḇ']), + ('ḇ', &['Ḇ']), + ('Ḉ', &['ḉ']), + ('ḉ', &['Ḉ']), + ('Ḋ', &['ḋ']), + ('ḋ', &['Ḋ']), + ('Ḍ', &['ḍ']), + ('ḍ', &['Ḍ']), + ('Ḏ', &['ḏ']), + ('ḏ', &['Ḏ']), + ('Ḑ', &['ḑ']), + ('ḑ', &['Ḑ']), + ('Ḓ', &['ḓ']), + ('ḓ', &['Ḓ']), + ('Ḕ', &['ḕ']), + ('ḕ', &['Ḕ']), + ('Ḗ', &['ḗ']), + ('ḗ', &['Ḗ']), + ('Ḙ', &['ḙ']), + ('ḙ', &['Ḙ']), + ('Ḛ', &['ḛ']), + ('ḛ', &['Ḛ']), + ('Ḝ', &['ḝ']), + ('ḝ', &['Ḝ']), + ('Ḟ', &['ḟ']), + ('ḟ', &['Ḟ']), + ('Ḡ', &['ḡ']), + ('ḡ', &['Ḡ']), + ('Ḣ', &['ḣ']), + ('ḣ', &['Ḣ']), + ('Ḥ', &['ḥ']), + ('ḥ', &['Ḥ']), + ('Ḧ', &['ḧ']), + ('ḧ', &['Ḧ']), + ('Ḩ', &['ḩ']), + ('ḩ', &['Ḩ']), + ('Ḫ', &['ḫ']), + ('ḫ', &['Ḫ']), + ('Ḭ', &['ḭ']), + ('ḭ', &['Ḭ']), + ('Ḯ', &['ḯ']), + ('ḯ', &['Ḯ']), + ('Ḱ', &['ḱ']), + ('ḱ', &['Ḱ']), + ('Ḳ', &['ḳ']), + ('ḳ', &['Ḳ']), + ('Ḵ', &['ḵ']), + ('ḵ', &['Ḵ']), + ('Ḷ', &['ḷ']), + ('ḷ', &['Ḷ']), + ('Ḹ', &['ḹ']), + ('ḹ', &['Ḹ']), + ('Ḻ', &['ḻ']), + ('ḻ', &['Ḻ']), + ('Ḽ', &['ḽ']), + ('ḽ', &['Ḽ']), + ('Ḿ', &['ḿ']), + ('ḿ', &['Ḿ']), + ('Ṁ', &['ṁ']), + ('ṁ', &['Ṁ']), + ('Ṃ', &['ṃ']), + ('ṃ', &['Ṃ']), + ('Ṅ', &['ṅ']), + ('ṅ', &['Ṅ']), + ('Ṇ', &['ṇ']), + ('ṇ', &['Ṇ']), + ('Ṉ', &['ṉ']), + ('ṉ', &['Ṉ']), + ('Ṋ', &['ṋ']), + ('ṋ', &['Ṋ']), + ('Ṍ', &['ṍ']), + ('ṍ', &['Ṍ']), + ('Ṏ', &['ṏ']), + ('ṏ', &['Ṏ']), + ('Ṑ', &['ṑ']), + ('ṑ', &['Ṑ']), + ('Ṓ', &['ṓ']), + ('ṓ', &['Ṓ']), + ('Ṕ', &['ṕ']), + ('ṕ', &['Ṕ']), + ('Ṗ', &['ṗ']), + ('ṗ', &['Ṗ']), + ('Ṙ', &['ṙ']), + ('ṙ', &['Ṙ']), + ('Ṛ', &['ṛ']), + ('ṛ', &['Ṛ']), + ('Ṝ', &['ṝ']), + ('ṝ', &['Ṝ']), + ('Ṟ', &['ṟ']), + ('ṟ', &['Ṟ']), + ('Ṡ', &['ṡ', 'ẛ']), + ('ṡ', &['Ṡ', 'ẛ']), + ('Ṣ', &['ṣ']), + ('ṣ', &['Ṣ']), + ('Ṥ', &['ṥ']), + ('ṥ', &['Ṥ']), + ('Ṧ', &['ṧ']), + ('ṧ', &['Ṧ']), + ('Ṩ', &['ṩ']), + ('ṩ', &['Ṩ']), + ('Ṫ', &['ṫ']), + ('ṫ', &['Ṫ']), + ('Ṭ', &['ṭ']), + ('ṭ', &['Ṭ']), + ('Ṯ', &['ṯ']), + ('ṯ', &['Ṯ']), + ('Ṱ', &['ṱ']), + ('ṱ', &['Ṱ']), + ('Ṳ', &['ṳ']), + ('ṳ', &['Ṳ']), + ('Ṵ', &['ṵ']), + ('ṵ', &['Ṵ']), + ('Ṷ', &['ṷ']), + ('ṷ', &['Ṷ']), + ('Ṹ', &['ṹ']), + ('ṹ', &['Ṹ']), + ('Ṻ', &['ṻ']), + ('ṻ', &['Ṻ']), + ('Ṽ', &['ṽ']), + ('ṽ', &['Ṽ']), + ('Ṿ', &['ṿ']), + ('ṿ', &['Ṿ']), + ('Ẁ', &['ẁ']), + ('ẁ', &['Ẁ']), + ('Ẃ', &['ẃ']), + ('ẃ', &['Ẃ']), + ('Ẅ', &['ẅ']), + ('ẅ', &['Ẅ']), + ('Ẇ', &['ẇ']), + ('ẇ', &['Ẇ']), + ('Ẉ', &['ẉ']), + ('ẉ', &['Ẉ']), + ('Ẋ', &['ẋ']), + ('ẋ', &['Ẋ']), + ('Ẍ', &['ẍ']), + ('ẍ', &['Ẍ']), + ('Ẏ', &['ẏ']), + ('ẏ', &['Ẏ']), + ('Ẑ', &['ẑ']), + ('ẑ', &['Ẑ']), + ('Ẓ', &['ẓ']), + ('ẓ', &['Ẓ']), + ('Ẕ', &['ẕ']), + ('ẕ', &['Ẕ']), + ('ẛ', &['Ṡ', 'ṡ']), + ('ẞ', &['ß']), + ('Ạ', &['ạ']), + ('ạ', &['Ạ']), + ('Ả', &['ả']), + ('ả', &['Ả']), + ('Ấ', &['ấ']), + ('ấ', &['Ấ']), + ('Ầ', &['ầ']), + ('ầ', &['Ầ']), + ('Ẩ', &['ẩ']), + ('ẩ', &['Ẩ']), + ('Ẫ', &['ẫ']), + ('ẫ', &['Ẫ']), + ('Ậ', &['ậ']), + ('ậ', &['Ậ']), + ('Ắ', &['ắ']), + ('ắ', &['Ắ']), + ('Ằ', &['ằ']), + ('ằ', &['Ằ']), + ('Ẳ', &['ẳ']), + ('ẳ', &['Ẳ']), + ('Ẵ', &['ẵ']), + ('ẵ', &['Ẵ']), + ('Ặ', &['ặ']), + ('ặ', &['Ặ']), + ('Ẹ', &['ẹ']), + ('ẹ', &['Ẹ']), + ('Ẻ', &['ẻ']), + ('ẻ', &['Ẻ']), + ('Ẽ', &['ẽ']), + ('ẽ', &['Ẽ']), + ('Ế', &['ế']), + ('ế', &['Ế']), + ('Ề', &['ề']), + ('ề', &['Ề']), + ('Ể', &['ể']), + ('ể', &['Ể']), + ('Ễ', &['ễ']), + ('ễ', &['Ễ']), + ('Ệ', &['ệ']), + ('ệ', &['Ệ']), + ('Ỉ', &['ỉ']), + ('ỉ', &['Ỉ']), + ('Ị', &['ị']), + ('ị', &['Ị']), + ('Ọ', &['ọ']), + ('ọ', &['Ọ']), + ('Ỏ', &['ỏ']), + ('ỏ', &['Ỏ']), + ('Ố', &['ố']), + ('ố', &['Ố']), + ('Ồ', &['ồ']), + ('ồ', &['Ồ']), + ('Ổ', &['ổ']), + ('ổ', &['Ổ']), + ('Ỗ', &['ỗ']), + ('ỗ', &['Ỗ']), + ('Ộ', &['ộ']), + ('ộ', &['Ộ']), + ('Ớ', &['ớ']), + ('ớ', &['Ớ']), + ('Ờ', &['ờ']), + ('ờ', &['Ờ']), + ('Ở', &['ở']), + ('ở', &['Ở']), + ('Ỡ', &['ỡ']), + ('ỡ', &['Ỡ']), + ('Ợ', &['ợ']), + ('ợ', &['Ợ']), + ('Ụ', &['ụ']), + ('ụ', &['Ụ']), + ('Ủ', &['ủ']), + ('ủ', &['Ủ']), + ('Ứ', &['ứ']), + ('ứ', &['Ứ']), + ('Ừ', &['ừ']), + ('ừ', &['Ừ']), + ('Ử', &['ử']), + ('ử', &['Ử']), + ('Ữ', &['ữ']), + ('ữ', &['Ữ']), + ('Ự', &['ự']), + ('ự', &['Ự']), + ('Ỳ', &['ỳ']), + ('ỳ', &['Ỳ']), + ('Ỵ', &['ỵ']), + ('ỵ', &['Ỵ']), + ('Ỷ', &['ỷ']), + ('ỷ', &['Ỷ']), + ('Ỹ', &['ỹ']), + ('ỹ', &['Ỹ']), + ('Ỻ', &['ỻ']), + ('ỻ', &['Ỻ']), + ('Ỽ', &['ỽ']), + ('ỽ', &['Ỽ']), + ('Ỿ', &['ỿ']), + ('ỿ', &['Ỿ']), + ('ἀ', &['Ἀ']), + ('ἁ', &['Ἁ']), + ('ἂ', &['Ἂ']), + ('ἃ', &['Ἃ']), + ('ἄ', &['Ἄ']), + ('ἅ', &['Ἅ']), + ('ἆ', &['Ἆ']), + ('ἇ', &['Ἇ']), + ('Ἀ', &['ἀ']), + ('Ἁ', &['ἁ']), + ('Ἂ', &['ἂ']), + ('Ἃ', &['ἃ']), + ('Ἄ', &['ἄ']), + ('Ἅ', &['ἅ']), + ('Ἆ', &['ἆ']), + ('Ἇ', &['ἇ']), + ('ἐ', &['Ἐ']), + ('ἑ', &['Ἑ']), + ('ἒ', &['Ἒ']), + ('ἓ', &['Ἓ']), + ('ἔ', &['Ἔ']), + ('ἕ', &['Ἕ']), + ('Ἐ', &['ἐ']), + ('Ἑ', &['ἑ']), + ('Ἒ', &['ἒ']), + ('Ἓ', &['ἓ']), + ('Ἔ', &['ἔ']), + ('Ἕ', &['ἕ']), + ('ἠ', &['Ἠ']), + ('ἡ', &['Ἡ']), + ('ἢ', &['Ἢ']), + ('ἣ', &['Ἣ']), + ('ἤ', &['Ἤ']), + ('ἥ', &['Ἥ']), + ('ἦ', &['Ἦ']), + ('ἧ', &['Ἧ']), + ('Ἠ', &['ἠ']), + ('Ἡ', &['ἡ']), + ('Ἢ', &['ἢ']), + ('Ἣ', &['ἣ']), + ('Ἤ', &['ἤ']), + ('Ἥ', &['ἥ']), + ('Ἦ', &['ἦ']), + ('Ἧ', &['ἧ']), + ('ἰ', &['Ἰ']), + ('ἱ', &['Ἱ']), + ('ἲ', &['Ἲ']), + ('ἳ', &['Ἳ']), + ('ἴ', &['Ἴ']), + ('ἵ', &['Ἵ']), + ('ἶ', &['Ἶ']), + ('ἷ', &['Ἷ']), + ('Ἰ', &['ἰ']), + ('Ἱ', &['ἱ']), + ('Ἲ', &['ἲ']), + ('Ἳ', &['ἳ']), + ('Ἴ', &['ἴ']), + ('Ἵ', &['ἵ']), + ('Ἶ', &['ἶ']), + ('Ἷ', &['ἷ']), + ('ὀ', &['Ὀ']), + ('ὁ', &['Ὁ']), + ('ὂ', &['Ὂ']), + ('ὃ', &['Ὃ']), + ('ὄ', &['Ὄ']), + ('ὅ', &['Ὅ']), + ('Ὀ', &['ὀ']), + ('Ὁ', &['ὁ']), + ('Ὂ', &['ὂ']), + ('Ὃ', &['ὃ']), + ('Ὄ', &['ὄ']), + ('Ὅ', &['ὅ']), + ('ὑ', &['Ὑ']), + ('ὓ', &['Ὓ']), + ('ὕ', &['Ὕ']), + ('ὗ', &['Ὗ']), + ('Ὑ', &['ὑ']), + ('Ὓ', &['ὓ']), + ('Ὕ', &['ὕ']), + ('Ὗ', &['ὗ']), + ('ὠ', &['Ὠ']), + ('ὡ', &['Ὡ']), + ('ὢ', &['Ὢ']), + ('ὣ', &['Ὣ']), + ('ὤ', &['Ὤ']), + ('ὥ', &['Ὥ']), + ('ὦ', &['Ὦ']), + ('ὧ', &['Ὧ']), + ('Ὠ', &['ὠ']), + ('Ὡ', &['ὡ']), + ('Ὢ', &['ὢ']), + ('Ὣ', &['ὣ']), + ('Ὤ', &['ὤ']), + ('Ὥ', &['ὥ']), + ('Ὦ', &['ὦ']), + ('Ὧ', &['ὧ']), + ('ὰ', &['Ὰ']), + ('ά', &['Ά']), + ('ὲ', &['Ὲ']), + ('έ', &['Έ']), + ('ὴ', &['Ὴ']), + ('ή', &['Ή']), + ('ὶ', &['Ὶ']), + ('ί', &['Ί']), + ('ὸ', &['Ὸ']), + ('ό', &['Ό']), + ('ὺ', &['Ὺ']), + ('ύ', &['Ύ']), + ('ὼ', &['Ὼ']), + ('ώ', &['Ώ']), + ('ᾀ', &['ᾈ']), + ('ᾁ', &['ᾉ']), + ('ᾂ', &['ᾊ']), + ('ᾃ', &['ᾋ']), + ('ᾄ', &['ᾌ']), + ('ᾅ', &['ᾍ']), + ('ᾆ', &['ᾎ']), + ('ᾇ', &['ᾏ']), + ('ᾈ', &['ᾀ']), + ('ᾉ', &['ᾁ']), + ('ᾊ', &['ᾂ']), + ('ᾋ', &['ᾃ']), + ('ᾌ', &['ᾄ']), + ('ᾍ', &['ᾅ']), + ('ᾎ', &['ᾆ']), + ('ᾏ', &['ᾇ']), + ('ᾐ', &['ᾘ']), + ('ᾑ', &['ᾙ']), + ('ᾒ', &['ᾚ']), + ('ᾓ', &['ᾛ']), + ('ᾔ', &['ᾜ']), + ('ᾕ', &['ᾝ']), + ('ᾖ', &['ᾞ']), + ('ᾗ', &['ᾟ']), + ('ᾘ', &['ᾐ']), + ('ᾙ', &['ᾑ']), + ('ᾚ', &['ᾒ']), + ('ᾛ', &['ᾓ']), + ('ᾜ', &['ᾔ']), + ('ᾝ', &['ᾕ']), + ('ᾞ', &['ᾖ']), + ('ᾟ', &['ᾗ']), + ('ᾠ', &['ᾨ']), + ('ᾡ', &['ᾩ']), + ('ᾢ', &['ᾪ']), + ('ᾣ', &['ᾫ']), + ('ᾤ', &['ᾬ']), + ('ᾥ', &['ᾭ']), + ('ᾦ', &['ᾮ']), + ('ᾧ', &['ᾯ']), + ('ᾨ', &['ᾠ']), + ('ᾩ', &['ᾡ']), + ('ᾪ', &['ᾢ']), + ('ᾫ', &['ᾣ']), + ('ᾬ', &['ᾤ']), + ('ᾭ', &['ᾥ']), + ('ᾮ', &['ᾦ']), + ('ᾯ', &['ᾧ']), + ('ᾰ', &['Ᾰ']), + ('ᾱ', &['Ᾱ']), + ('ᾳ', &['ᾼ']), + ('Ᾰ', &['ᾰ']), + ('Ᾱ', &['ᾱ']), + ('Ὰ', &['ὰ']), + ('Ά', &['ά']), + ('ᾼ', &['ᾳ']), + ('ι', &['\u{345}', 'Ι', 'ι']), + ('ῃ', &['ῌ']), + ('Ὲ', &['ὲ']), + ('Έ', &['έ']), + ('Ὴ', &['ὴ']), + ('Ή', &['ή']), + ('ῌ', &['ῃ']), + ('ῐ', &['Ῐ']), + ('ῑ', &['Ῑ']), + ('ΐ', &['ΐ']), + ('Ῐ', &['ῐ']), + ('Ῑ', &['ῑ']), + ('Ὶ', &['ὶ']), + ('Ί', &['ί']), + ('ῠ', &['Ῠ']), + ('ῡ', &['Ῡ']), + ('ΰ', &['ΰ']), + ('ῥ', &['Ῥ']), + ('Ῠ', &['ῠ']), + ('Ῡ', &['ῡ']), + ('Ὺ', &['ὺ']), + ('Ύ', &['ύ']), + ('Ῥ', &['ῥ']), + ('ῳ', &['ῼ']), + ('Ὸ', &['ὸ']), + ('Ό', &['ό']), + ('Ὼ', &['ὼ']), + ('Ώ', &['ώ']), + ('ῼ', &['ῳ']), + ('Ω', &['Ω', 'ω']), + ('K', &['K', 'k']), + ('Å', &['Å', 'å']), + ('Ⅎ', &['ⅎ']), + ('ⅎ', &['Ⅎ']), + ('Ⅰ', &['ⅰ']), + ('Ⅱ', &['ⅱ']), + ('Ⅲ', &['ⅲ']), + ('Ⅳ', &['ⅳ']), + ('Ⅴ', &['ⅴ']), + ('Ⅵ', &['ⅵ']), + ('Ⅶ', &['ⅶ']), + ('Ⅷ', &['ⅷ']), + ('Ⅸ', &['ⅸ']), + ('Ⅹ', &['ⅹ']), + ('Ⅺ', &['ⅺ']), + ('Ⅻ', &['ⅻ']), + ('Ⅼ', &['ⅼ']), + ('Ⅽ', &['ⅽ']), + ('Ⅾ', &['ⅾ']), + ('Ⅿ', &['ⅿ']), + ('ⅰ', &['Ⅰ']), + ('ⅱ', &['Ⅱ']), + ('ⅲ', &['Ⅲ']), + ('ⅳ', &['Ⅳ']), + ('ⅴ', &['Ⅴ']), + ('ⅵ', &['Ⅵ']), + ('ⅶ', &['Ⅶ']), + ('ⅷ', &['Ⅷ']), + ('ⅸ', &['Ⅸ']), + ('ⅹ', &['Ⅹ']), + ('ⅺ', &['Ⅺ']), + ('ⅻ', &['Ⅻ']), + ('ⅼ', &['Ⅼ']), + ('ⅽ', &['Ⅽ']), + ('ⅾ', &['Ⅾ']), + ('ⅿ', &['Ⅿ']), + ('Ↄ', &['ↄ']), + ('ↄ', &['Ↄ']), + ('Ⓐ', &['ⓐ']), + ('Ⓑ', &['ⓑ']), + ('Ⓒ', &['ⓒ']), + ('Ⓓ', &['ⓓ']), + ('Ⓔ', &['ⓔ']), + ('Ⓕ', &['ⓕ']), + ('Ⓖ', &['ⓖ']), + ('Ⓗ', &['ⓗ']), + ('Ⓘ', &['ⓘ']), + ('Ⓙ', &['ⓙ']), + ('Ⓚ', &['ⓚ']), + ('Ⓛ', &['ⓛ']), + ('Ⓜ', &['ⓜ']), + ('Ⓝ', &['ⓝ']), + ('Ⓞ', &['ⓞ']), + ('Ⓟ', &['ⓟ']), + ('Ⓠ', &['ⓠ']), + ('Ⓡ', &['ⓡ']), + ('Ⓢ', &['ⓢ']), + ('Ⓣ', &['ⓣ']), + ('Ⓤ', &['ⓤ']), + ('Ⓥ', &['ⓥ']), + ('Ⓦ', &['ⓦ']), + ('Ⓧ', &['ⓧ']), + ('Ⓨ', &['ⓨ']), + ('Ⓩ', &['ⓩ']), + ('ⓐ', &['Ⓐ']), + ('ⓑ', &['Ⓑ']), + ('ⓒ', &['Ⓒ']), + ('ⓓ', &['Ⓓ']), + ('ⓔ', &['Ⓔ']), + ('ⓕ', &['Ⓕ']), + ('ⓖ', &['Ⓖ']), + ('ⓗ', &['Ⓗ']), + ('ⓘ', &['Ⓘ']), + ('ⓙ', &['Ⓙ']), + ('ⓚ', &['Ⓚ']), + ('ⓛ', &['Ⓛ']), + ('ⓜ', &['Ⓜ']), + ('ⓝ', &['Ⓝ']), + ('ⓞ', &['Ⓞ']), + ('ⓟ', &['Ⓟ']), + ('ⓠ', &['Ⓠ']), + ('ⓡ', &['Ⓡ']), + ('ⓢ', &['Ⓢ']), + ('ⓣ', &['Ⓣ']), + ('ⓤ', &['Ⓤ']), + ('ⓥ', &['Ⓥ']), + ('ⓦ', &['Ⓦ']), + ('ⓧ', &['Ⓧ']), + ('ⓨ', &['Ⓨ']), + ('ⓩ', &['Ⓩ']), + ('Ⰰ', &['ⰰ']), + ('Ⰱ', &['ⰱ']), + ('Ⰲ', &['ⰲ']), + ('Ⰳ', &['ⰳ']), + ('Ⰴ', &['ⰴ']), + ('Ⰵ', &['ⰵ']), + ('Ⰶ', &['ⰶ']), + ('Ⰷ', &['ⰷ']), + ('Ⰸ', &['ⰸ']), + ('Ⰹ', &['ⰹ']), + ('Ⰺ', &['ⰺ']), + ('Ⰻ', &['ⰻ']), + ('Ⰼ', &['ⰼ']), + ('Ⰽ', &['ⰽ']), + ('Ⰾ', &['ⰾ']), + ('Ⰿ', &['ⰿ']), + ('Ⱀ', &['ⱀ']), + ('Ⱁ', &['ⱁ']), + ('Ⱂ', &['ⱂ']), + ('Ⱃ', &['ⱃ']), + ('Ⱄ', &['ⱄ']), + ('Ⱅ', &['ⱅ']), + ('Ⱆ', &['ⱆ']), + ('Ⱇ', &['ⱇ']), + ('Ⱈ', &['ⱈ']), + ('Ⱉ', &['ⱉ']), + ('Ⱊ', &['ⱊ']), + ('Ⱋ', &['ⱋ']), + ('Ⱌ', &['ⱌ']), + ('Ⱍ', &['ⱍ']), + ('Ⱎ', &['ⱎ']), + ('Ⱏ', &['ⱏ']), + ('Ⱐ', &['ⱐ']), + ('Ⱑ', &['ⱑ']), + ('Ⱒ', &['ⱒ']), + ('Ⱓ', &['ⱓ']), + ('Ⱔ', &['ⱔ']), + ('Ⱕ', &['ⱕ']), + ('Ⱖ', &['ⱖ']), + ('Ⱗ', &['ⱗ']), + ('Ⱘ', &['ⱘ']), + ('Ⱙ', &['ⱙ']), + ('Ⱚ', &['ⱚ']), + ('Ⱛ', &['ⱛ']), + ('Ⱜ', &['ⱜ']), + ('Ⱝ', &['ⱝ']), + ('Ⱞ', &['ⱞ']), + ('Ⱟ', &['ⱟ']), + ('ⰰ', &['Ⰰ']), + ('ⰱ', &['Ⰱ']), + ('ⰲ', &['Ⰲ']), + ('ⰳ', &['Ⰳ']), + ('ⰴ', &['Ⰴ']), + ('ⰵ', &['Ⰵ']), + ('ⰶ', &['Ⰶ']), + ('ⰷ', &['Ⰷ']), + ('ⰸ', &['Ⰸ']), + ('ⰹ', &['Ⰹ']), + ('ⰺ', &['Ⰺ']), + ('ⰻ', &['Ⰻ']), + ('ⰼ', &['Ⰼ']), + ('ⰽ', &['Ⰽ']), + ('ⰾ', &['Ⰾ']), + ('ⰿ', &['Ⰿ']), + ('ⱀ', &['Ⱀ']), + ('ⱁ', &['Ⱁ']), + ('ⱂ', &['Ⱂ']), + ('ⱃ', &['Ⱃ']), + ('ⱄ', &['Ⱄ']), + ('ⱅ', &['Ⱅ']), + ('ⱆ', &['Ⱆ']), + ('ⱇ', &['Ⱇ']), + ('ⱈ', &['Ⱈ']), + ('ⱉ', &['Ⱉ']), + ('ⱊ', &['Ⱊ']), + ('ⱋ', &['Ⱋ']), + ('ⱌ', &['Ⱌ']), + ('ⱍ', &['Ⱍ']), + ('ⱎ', &['Ⱎ']), + ('ⱏ', &['Ⱏ']), + ('ⱐ', &['Ⱐ']), + ('ⱑ', &['Ⱑ']), + ('ⱒ', &['Ⱒ']), + ('ⱓ', &['Ⱓ']), + ('ⱔ', &['Ⱔ']), + ('ⱕ', &['Ⱕ']), + ('ⱖ', &['Ⱖ']), + ('ⱗ', &['Ⱗ']), + ('ⱘ', &['Ⱘ']), + ('ⱙ', &['Ⱙ']), + ('ⱚ', &['Ⱚ']), + ('ⱛ', &['Ⱛ']), + ('ⱜ', &['Ⱜ']), + ('ⱝ', &['Ⱝ']), + ('ⱞ', &['Ⱞ']), + ('ⱟ', &['Ⱟ']), + ('Ⱡ', &['ⱡ']), + ('ⱡ', &['Ⱡ']), + ('Ɫ', &['ɫ']), + ('Ᵽ', &['ᵽ']), + ('Ɽ', &['ɽ']), + ('ⱥ', &['Ⱥ']), + ('ⱦ', &['Ⱦ']), + ('Ⱨ', &['ⱨ']), + ('ⱨ', &['Ⱨ']), + ('Ⱪ', &['ⱪ']), + ('ⱪ', &['Ⱪ']), + ('Ⱬ', &['ⱬ']), + ('ⱬ', &['Ⱬ']), + ('Ɑ', &['ɑ']), + ('Ɱ', &['ɱ']), + ('Ɐ', &['ɐ']), + ('Ɒ', &['ɒ']), + ('Ⱳ', &['ⱳ']), + ('ⱳ', &['Ⱳ']), + ('Ⱶ', &['ⱶ']), + ('ⱶ', &['Ⱶ']), + ('Ȿ', &['ȿ']), + ('Ɀ', &['ɀ']), + ('Ⲁ', &['ⲁ']), + ('ⲁ', &['Ⲁ']), + ('Ⲃ', &['ⲃ']), + ('ⲃ', &['Ⲃ']), + ('Ⲅ', &['ⲅ']), + ('ⲅ', &['Ⲅ']), + ('Ⲇ', &['ⲇ']), + ('ⲇ', &['Ⲇ']), + ('Ⲉ', &['ⲉ']), + ('ⲉ', &['Ⲉ']), + ('Ⲋ', &['ⲋ']), + ('ⲋ', &['Ⲋ']), + ('Ⲍ', &['ⲍ']), + ('ⲍ', &['Ⲍ']), + ('Ⲏ', &['ⲏ']), + ('ⲏ', &['Ⲏ']), + ('Ⲑ', &['ⲑ']), + ('ⲑ', &['Ⲑ']), + ('Ⲓ', &['ⲓ']), + ('ⲓ', &['Ⲓ']), + ('Ⲕ', &['ⲕ']), + ('ⲕ', &['Ⲕ']), + ('Ⲗ', &['ⲗ']), + ('ⲗ', &['Ⲗ']), + ('Ⲙ', &['ⲙ']), + ('ⲙ', &['Ⲙ']), + ('Ⲛ', &['ⲛ']), + ('ⲛ', &['Ⲛ']), + ('Ⲝ', &['ⲝ']), + ('ⲝ', &['Ⲝ']), + ('Ⲟ', &['ⲟ']), + ('ⲟ', &['Ⲟ']), + ('Ⲡ', &['ⲡ']), + ('ⲡ', &['Ⲡ']), + ('Ⲣ', &['ⲣ']), + ('ⲣ', &['Ⲣ']), + ('Ⲥ', &['ⲥ']), + ('ⲥ', &['Ⲥ']), + ('Ⲧ', &['ⲧ']), + ('ⲧ', &['Ⲧ']), + ('Ⲩ', &['ⲩ']), + ('ⲩ', &['Ⲩ']), + ('Ⲫ', &['ⲫ']), + ('ⲫ', &['Ⲫ']), + ('Ⲭ', &['ⲭ']), + ('ⲭ', &['Ⲭ']), + ('Ⲯ', &['ⲯ']), + ('ⲯ', &['Ⲯ']), + ('Ⲱ', &['ⲱ']), + ('ⲱ', &['Ⲱ']), + ('Ⲳ', &['ⲳ']), + ('ⲳ', &['Ⲳ']), + ('Ⲵ', &['ⲵ']), + ('ⲵ', &['Ⲵ']), + ('Ⲷ', &['ⲷ']), + ('ⲷ', &['Ⲷ']), + ('Ⲹ', &['ⲹ']), + ('ⲹ', &['Ⲹ']), + ('Ⲻ', &['ⲻ']), + ('ⲻ', &['Ⲻ']), + ('Ⲽ', &['ⲽ']), + ('ⲽ', &['Ⲽ']), + ('Ⲿ', &['ⲿ']), + ('ⲿ', &['Ⲿ']), + ('Ⳁ', &['ⳁ']), + ('ⳁ', &['Ⳁ']), + ('Ⳃ', &['ⳃ']), + ('ⳃ', &['Ⳃ']), + ('Ⳅ', &['ⳅ']), + ('ⳅ', &['Ⳅ']), + ('Ⳇ', &['ⳇ']), + ('ⳇ', &['Ⳇ']), + ('Ⳉ', &['ⳉ']), + ('ⳉ', &['Ⳉ']), + ('Ⳋ', &['ⳋ']), + ('ⳋ', &['Ⳋ']), + ('Ⳍ', &['ⳍ']), + ('ⳍ', &['Ⳍ']), + ('Ⳏ', &['ⳏ']), + ('ⳏ', &['Ⳏ']), + ('Ⳑ', &['ⳑ']), + ('ⳑ', &['Ⳑ']), + ('Ⳓ', &['ⳓ']), + ('ⳓ', &['Ⳓ']), + ('Ⳕ', &['ⳕ']), + ('ⳕ', &['Ⳕ']), + ('Ⳗ', &['ⳗ']), + ('ⳗ', &['Ⳗ']), + ('Ⳙ', &['ⳙ']), + ('ⳙ', &['Ⳙ']), + ('Ⳛ', &['ⳛ']), + ('ⳛ', &['Ⳛ']), + ('Ⳝ', &['ⳝ']), + ('ⳝ', &['Ⳝ']), + ('Ⳟ', &['ⳟ']), + ('ⳟ', &['Ⳟ']), + ('Ⳡ', &['ⳡ']), + ('ⳡ', &['Ⳡ']), + ('Ⳣ', &['ⳣ']), + ('ⳣ', &['Ⳣ']), + ('Ⳬ', &['ⳬ']), + ('ⳬ', &['Ⳬ']), + ('Ⳮ', &['ⳮ']), + ('ⳮ', &['Ⳮ']), + ('Ⳳ', &['ⳳ']), + ('ⳳ', &['Ⳳ']), + ('ⴀ', &['Ⴀ']), + ('ⴁ', &['Ⴁ']), + ('ⴂ', &['Ⴂ']), + ('ⴃ', &['Ⴃ']), + ('ⴄ', &['Ⴄ']), + ('ⴅ', &['Ⴅ']), + ('ⴆ', &['Ⴆ']), + ('ⴇ', &['Ⴇ']), + ('ⴈ', &['Ⴈ']), + ('ⴉ', &['Ⴉ']), + ('ⴊ', &['Ⴊ']), + ('ⴋ', &['Ⴋ']), + ('ⴌ', &['Ⴌ']), + ('ⴍ', &['Ⴍ']), + ('ⴎ', &['Ⴎ']), + ('ⴏ', &['Ⴏ']), + ('ⴐ', &['Ⴐ']), + ('ⴑ', &['Ⴑ']), + ('ⴒ', &['Ⴒ']), + ('ⴓ', &['Ⴓ']), + ('ⴔ', &['Ⴔ']), + ('ⴕ', &['Ⴕ']), + ('ⴖ', &['Ⴖ']), + ('ⴗ', &['Ⴗ']), + ('ⴘ', &['Ⴘ']), + ('ⴙ', &['Ⴙ']), + ('ⴚ', &['Ⴚ']), + ('ⴛ', &['Ⴛ']), + ('ⴜ', &['Ⴜ']), + ('ⴝ', &['Ⴝ']), + ('ⴞ', &['Ⴞ']), + ('ⴟ', &['Ⴟ']), + ('ⴠ', &['Ⴠ']), + ('ⴡ', &['Ⴡ']), + ('ⴢ', &['Ⴢ']), + ('ⴣ', &['Ⴣ']), + ('ⴤ', &['Ⴤ']), + ('ⴥ', &['Ⴥ']), + ('ⴧ', &['Ⴧ']), + ('ⴭ', &['Ⴭ']), + ('Ꙁ', &['ꙁ']), + ('ꙁ', &['Ꙁ']), + ('Ꙃ', &['ꙃ']), + ('ꙃ', &['Ꙃ']), + ('Ꙅ', &['ꙅ']), + ('ꙅ', &['Ꙅ']), + ('Ꙇ', &['ꙇ']), + ('ꙇ', &['Ꙇ']), + ('Ꙉ', &['ꙉ']), + ('ꙉ', &['Ꙉ']), + ('Ꙋ', &['ᲈ', 'ꙋ']), + ('ꙋ', &['ᲈ', 'Ꙋ']), + ('Ꙍ', &['ꙍ']), + ('ꙍ', &['Ꙍ']), + ('Ꙏ', &['ꙏ']), + ('ꙏ', &['Ꙏ']), + ('Ꙑ', &['ꙑ']), + ('ꙑ', &['Ꙑ']), + ('Ꙓ', &['ꙓ']), + ('ꙓ', &['Ꙓ']), + ('Ꙕ', &['ꙕ']), + ('ꙕ', &['Ꙕ']), + ('Ꙗ', &['ꙗ']), + ('ꙗ', &['Ꙗ']), + ('Ꙙ', &['ꙙ']), + ('ꙙ', &['Ꙙ']), + ('Ꙛ', &['ꙛ']), + ('ꙛ', &['Ꙛ']), + ('Ꙝ', &['ꙝ']), + ('ꙝ', &['Ꙝ']), + ('Ꙟ', &['ꙟ']), + ('ꙟ', &['Ꙟ']), + ('Ꙡ', &['ꙡ']), + ('ꙡ', &['Ꙡ']), + ('Ꙣ', &['ꙣ']), + ('ꙣ', &['Ꙣ']), + ('Ꙥ', &['ꙥ']), + ('ꙥ', &['Ꙥ']), + ('Ꙧ', &['ꙧ']), + ('ꙧ', &['Ꙧ']), + ('Ꙩ', &['ꙩ']), + ('ꙩ', &['Ꙩ']), + ('Ꙫ', &['ꙫ']), + ('ꙫ', &['Ꙫ']), + ('Ꙭ', &['ꙭ']), + ('ꙭ', &['Ꙭ']), + ('Ꚁ', &['ꚁ']), + ('ꚁ', &['Ꚁ']), + ('Ꚃ', &['ꚃ']), + ('ꚃ', &['Ꚃ']), + ('Ꚅ', &['ꚅ']), + ('ꚅ', &['Ꚅ']), + ('Ꚇ', &['ꚇ']), + ('ꚇ', &['Ꚇ']), + ('Ꚉ', &['ꚉ']), + ('ꚉ', &['Ꚉ']), + ('Ꚋ', &['ꚋ']), + ('ꚋ', &['Ꚋ']), + ('Ꚍ', &['ꚍ']), + ('ꚍ', &['Ꚍ']), + ('Ꚏ', &['ꚏ']), + ('ꚏ', &['Ꚏ']), + ('Ꚑ', &['ꚑ']), + ('ꚑ', &['Ꚑ']), + ('Ꚓ', &['ꚓ']), + ('ꚓ', &['Ꚓ']), + ('Ꚕ', &['ꚕ']), + ('ꚕ', &['Ꚕ']), + ('Ꚗ', &['ꚗ']), + ('ꚗ', &['Ꚗ']), + ('Ꚙ', &['ꚙ']), + ('ꚙ', &['Ꚙ']), + ('Ꚛ', &['ꚛ']), + ('ꚛ', &['Ꚛ']), + ('Ꜣ', &['ꜣ']), + ('ꜣ', &['Ꜣ']), + ('Ꜥ', &['ꜥ']), + ('ꜥ', &['Ꜥ']), + ('Ꜧ', &['ꜧ']), + ('ꜧ', &['Ꜧ']), + ('Ꜩ', &['ꜩ']), + ('ꜩ', &['Ꜩ']), + ('Ꜫ', &['ꜫ']), + ('ꜫ', &['Ꜫ']), + ('Ꜭ', &['ꜭ']), + ('ꜭ', &['Ꜭ']), + ('Ꜯ', &['ꜯ']), + ('ꜯ', &['Ꜯ']), + ('Ꜳ', &['ꜳ']), + ('ꜳ', &['Ꜳ']), + ('Ꜵ', &['ꜵ']), + ('ꜵ', &['Ꜵ']), + ('Ꜷ', &['ꜷ']), + ('ꜷ', &['Ꜷ']), + ('Ꜹ', &['ꜹ']), + ('ꜹ', &['Ꜹ']), + ('Ꜻ', &['ꜻ']), + ('ꜻ', &['Ꜻ']), + ('Ꜽ', &['ꜽ']), + ('ꜽ', &['Ꜽ']), + ('Ꜿ', &['ꜿ']), + ('ꜿ', &['Ꜿ']), + ('Ꝁ', &['ꝁ']), + ('ꝁ', &['Ꝁ']), + ('Ꝃ', &['ꝃ']), + ('ꝃ', &['Ꝃ']), + ('Ꝅ', &['ꝅ']), + ('ꝅ', &['Ꝅ']), + ('Ꝇ', &['ꝇ']), + ('ꝇ', &['Ꝇ']), + ('Ꝉ', &['ꝉ']), + ('ꝉ', &['Ꝉ']), + ('Ꝋ', &['ꝋ']), + ('ꝋ', &['Ꝋ']), + ('Ꝍ', &['ꝍ']), + ('ꝍ', &['Ꝍ']), + ('Ꝏ', &['ꝏ']), + ('ꝏ', &['Ꝏ']), + ('Ꝑ', &['ꝑ']), + ('ꝑ', &['Ꝑ']), + ('Ꝓ', &['ꝓ']), + ('ꝓ', &['Ꝓ']), + ('Ꝕ', &['ꝕ']), + ('ꝕ', &['Ꝕ']), + ('Ꝗ', &['ꝗ']), + ('ꝗ', &['Ꝗ']), + ('Ꝙ', &['ꝙ']), + ('ꝙ', &['Ꝙ']), + ('Ꝛ', &['ꝛ']), + ('ꝛ', &['Ꝛ']), + ('Ꝝ', &['ꝝ']), + ('ꝝ', &['Ꝝ']), + ('Ꝟ', &['ꝟ']), + ('ꝟ', &['Ꝟ']), + ('Ꝡ', &['ꝡ']), + ('ꝡ', &['Ꝡ']), + ('Ꝣ', &['ꝣ']), + ('ꝣ', &['Ꝣ']), + ('Ꝥ', &['ꝥ']), + ('ꝥ', &['Ꝥ']), + ('Ꝧ', &['ꝧ']), + ('ꝧ', &['Ꝧ']), + ('Ꝩ', &['ꝩ']), + ('ꝩ', &['Ꝩ']), + ('Ꝫ', &['ꝫ']), + ('ꝫ', &['Ꝫ']), + ('Ꝭ', &['ꝭ']), + ('ꝭ', &['Ꝭ']), + ('Ꝯ', &['ꝯ']), + ('ꝯ', &['Ꝯ']), + ('Ꝺ', &['ꝺ']), + ('ꝺ', &['Ꝺ']), + ('Ꝼ', &['ꝼ']), + ('ꝼ', &['Ꝼ']), + ('Ᵹ', &['ᵹ']), + ('Ꝿ', &['ꝿ']), + ('ꝿ', &['Ꝿ']), + ('Ꞁ', &['ꞁ']), + ('ꞁ', &['Ꞁ']), + ('Ꞃ', &['ꞃ']), + ('ꞃ', &['Ꞃ']), + ('Ꞅ', &['ꞅ']), + ('ꞅ', &['Ꞅ']), + ('Ꞇ', &['ꞇ']), + ('ꞇ', &['Ꞇ']), + ('Ꞌ', &['ꞌ']), + ('ꞌ', &['Ꞌ']), + ('Ɥ', &['ɥ']), + ('Ꞑ', &['ꞑ']), + ('ꞑ', &['Ꞑ']), + ('Ꞓ', &['ꞓ']), + ('ꞓ', &['Ꞓ']), + ('ꞔ', &['Ꞔ']), + ('Ꞗ', &['ꞗ']), + ('ꞗ', &['Ꞗ']), + ('Ꞙ', &['ꞙ']), + ('ꞙ', &['Ꞙ']), + ('Ꞛ', &['ꞛ']), + ('ꞛ', &['Ꞛ']), + ('Ꞝ', &['ꞝ']), + ('ꞝ', &['Ꞝ']), + ('Ꞟ', &['ꞟ']), + ('ꞟ', &['Ꞟ']), + ('Ꞡ', &['ꞡ']), + ('ꞡ', &['Ꞡ']), + ('Ꞣ', &['ꞣ']), + ('ꞣ', &['Ꞣ']), + ('Ꞥ', &['ꞥ']), + ('ꞥ', &['Ꞥ']), + ('Ꞧ', &['ꞧ']), + ('ꞧ', &['Ꞧ']), + ('Ꞩ', &['ꞩ']), + ('ꞩ', &['Ꞩ']), + ('Ɦ', &['ɦ']), + ('Ɜ', &['ɜ']), + ('Ɡ', &['ɡ']), + ('Ɬ', &['ɬ']), + ('Ɪ', &['ɪ']), + ('Ʞ', &['ʞ']), + ('Ʇ', &['ʇ']), + ('Ʝ', &['ʝ']), + ('Ꭓ', &['ꭓ']), + ('Ꞵ', &['ꞵ']), + ('ꞵ', &['Ꞵ']), + ('Ꞷ', &['ꞷ']), + ('ꞷ', &['Ꞷ']), + ('Ꞹ', &['ꞹ']), + ('ꞹ', &['Ꞹ']), + ('Ꞻ', &['ꞻ']), + ('ꞻ', &['Ꞻ']), + ('Ꞽ', &['ꞽ']), + ('ꞽ', &['Ꞽ']), + ('Ꞿ', &['ꞿ']), + ('ꞿ', &['Ꞿ']), + ('Ꟁ', &['ꟁ']), + ('ꟁ', &['Ꟁ']), + ('Ꟃ', &['ꟃ']), + ('ꟃ', &['Ꟃ']), + ('Ꞔ', &['ꞔ']), + ('Ʂ', &['ʂ']), + ('Ᶎ', &['ᶎ']), + ('Ꟈ', &['ꟈ']), + ('ꟈ', &['Ꟈ']), + ('Ꟊ', &['ꟊ']), + ('ꟊ', &['Ꟊ']), + ('Ɤ', &['ɤ']), + ('Ꟍ', &['ꟍ']), + ('ꟍ', &['Ꟍ']), + ('Ꟑ', &['ꟑ']), + ('ꟑ', &['Ꟑ']), + ('Ꟗ', &['ꟗ']), + ('ꟗ', &['Ꟗ']), + ('Ꟙ', &['ꟙ']), + ('ꟙ', &['Ꟙ']), + ('Ꟛ', &['ꟛ']), + ('ꟛ', &['Ꟛ']), + ('Ƛ', &['ƛ']), + ('Ꟶ', &['ꟶ']), + ('ꟶ', &['Ꟶ']), + ('ꭓ', &['Ꭓ']), + ('ꭰ', &['Ꭰ']), + ('ꭱ', &['Ꭱ']), + ('ꭲ', &['Ꭲ']), + ('ꭳ', &['Ꭳ']), + ('ꭴ', &['Ꭴ']), + ('ꭵ', &['Ꭵ']), + ('ꭶ', &['Ꭶ']), + ('ꭷ', &['Ꭷ']), + ('ꭸ', &['Ꭸ']), + ('ꭹ', &['Ꭹ']), + ('ꭺ', &['Ꭺ']), + ('ꭻ', &['Ꭻ']), + ('ꭼ', &['Ꭼ']), + ('ꭽ', &['Ꭽ']), + ('ꭾ', &['Ꭾ']), + ('ꭿ', &['Ꭿ']), + ('ꮀ', &['Ꮀ']), + ('ꮁ', &['Ꮁ']), + ('ꮂ', &['Ꮂ']), + ('ꮃ', &['Ꮃ']), + ('ꮄ', &['Ꮄ']), + ('ꮅ', &['Ꮅ']), + ('ꮆ', &['Ꮆ']), + ('ꮇ', &['Ꮇ']), + ('ꮈ', &['Ꮈ']), + ('ꮉ', &['Ꮉ']), + ('ꮊ', &['Ꮊ']), + ('ꮋ', &['Ꮋ']), + ('ꮌ', &['Ꮌ']), + ('ꮍ', &['Ꮍ']), + ('ꮎ', &['Ꮎ']), + ('ꮏ', &['Ꮏ']), + ('ꮐ', &['Ꮐ']), + ('ꮑ', &['Ꮑ']), + ('ꮒ', &['Ꮒ']), + ('ꮓ', &['Ꮓ']), + ('ꮔ', &['Ꮔ']), + ('ꮕ', &['Ꮕ']), + ('ꮖ', &['Ꮖ']), + ('ꮗ', &['Ꮗ']), + ('ꮘ', &['Ꮘ']), + ('ꮙ', &['Ꮙ']), + ('ꮚ', &['Ꮚ']), + ('ꮛ', &['Ꮛ']), + ('ꮜ', &['Ꮜ']), + ('ꮝ', &['Ꮝ']), + ('ꮞ', &['Ꮞ']), + ('ꮟ', &['Ꮟ']), + ('ꮠ', &['Ꮠ']), + ('ꮡ', &['Ꮡ']), + ('ꮢ', &['Ꮢ']), + ('ꮣ', &['Ꮣ']), + ('ꮤ', &['Ꮤ']), + ('ꮥ', &['Ꮥ']), + ('ꮦ', &['Ꮦ']), + ('ꮧ', &['Ꮧ']), + ('ꮨ', &['Ꮨ']), + ('ꮩ', &['Ꮩ']), + ('ꮪ', &['Ꮪ']), + ('ꮫ', &['Ꮫ']), + ('ꮬ', &['Ꮬ']), + ('ꮭ', &['Ꮭ']), + ('ꮮ', &['Ꮮ']), + ('ꮯ', &['Ꮯ']), + ('ꮰ', &['Ꮰ']), + ('ꮱ', &['Ꮱ']), + ('ꮲ', &['Ꮲ']), + ('ꮳ', &['Ꮳ']), + ('ꮴ', &['Ꮴ']), + ('ꮵ', &['Ꮵ']), + ('ꮶ', &['Ꮶ']), + ('ꮷ', &['Ꮷ']), + ('ꮸ', &['Ꮸ']), + ('ꮹ', &['Ꮹ']), + ('ꮺ', &['Ꮺ']), + ('ꮻ', &['Ꮻ']), + ('ꮼ', &['Ꮼ']), + ('ꮽ', &['Ꮽ']), + ('ꮾ', &['Ꮾ']), + ('ꮿ', &['Ꮿ']), + ('ſt', &['st']), + ('st', &['ſt']), + ('Ａ', &['ａ']), + ('Ｂ', &['ｂ']), + ('Ｃ', &['ｃ']), + ('Ｄ', &['ｄ']), + ('Ｅ', &['ｅ']), + ('Ｆ', &['ｆ']), + ('Ｇ', &['ｇ']), + ('Ｈ', &['ｈ']), + ('Ｉ', &['ｉ']), + ('Ｊ', &['ｊ']), + ('Ｋ', &['ｋ']), + ('Ｌ', &['ｌ']), + ('Ｍ', &['ｍ']), + ('Ｎ', &['ｎ']), + ('Ｏ', &['ｏ']), + ('Ｐ', &['ｐ']), + ('Ｑ', &['ｑ']), + ('Ｒ', &['ｒ']), + ('Ｓ', &['ｓ']), + ('Ｔ', &['ｔ']), + ('Ｕ', &['ｕ']), + ('Ｖ', &['ｖ']), + ('Ｗ', &['ｗ']), + ('Ｘ', &['ｘ']), + ('Ｙ', &['ｙ']), + ('Ｚ', &['ｚ']), + ('ａ', &['Ａ']), + ('ｂ', &['Ｂ']), + ('ｃ', &['Ｃ']), + ('ｄ', &['Ｄ']), + ('ｅ', &['Ｅ']), + ('ｆ', &['Ｆ']), + ('ｇ', &['Ｇ']), + ('ｈ', &['Ｈ']), + ('ｉ', &['Ｉ']), + ('ｊ', &['Ｊ']), + ('ｋ', &['Ｋ']), + ('ｌ', &['Ｌ']), + ('ｍ', &['Ｍ']), + ('ｎ', &['Ｎ']), + ('ｏ', &['Ｏ']), + ('ｐ', &['Ｐ']), + ('ｑ', &['Ｑ']), + ('ｒ', &['Ｒ']), + ('ｓ', &['Ｓ']), + ('ｔ', &['Ｔ']), + ('ｕ', &['Ｕ']), + ('ｖ', &['Ｖ']), + ('ｗ', &['Ｗ']), + ('ｘ', &['Ｘ']), + ('ｙ', &['Ｙ']), + ('ｚ', &['Ｚ']), + ('𐐀', &['𐐨']), + ('𐐁', &['𐐩']), + ('𐐂', &['𐐪']), + ('𐐃', &['𐐫']), + ('𐐄', &['𐐬']), + ('𐐅', &['𐐭']), + ('𐐆', &['𐐮']), + ('𐐇', &['𐐯']), + ('𐐈', &['𐐰']), + ('𐐉', &['𐐱']), + ('𐐊', &['𐐲']), + ('𐐋', &['𐐳']), + ('𐐌', &['𐐴']), + ('𐐍', &['𐐵']), + ('𐐎', &['𐐶']), + ('𐐏', &['𐐷']), + ('𐐐', &['𐐸']), + ('𐐑', &['𐐹']), + ('𐐒', &['𐐺']), + ('𐐓', &['𐐻']), + ('𐐔', &['𐐼']), + ('𐐕', &['𐐽']), + ('𐐖', &['𐐾']), + ('𐐗', &['𐐿']), + ('𐐘', &['𐑀']), + ('𐐙', &['𐑁']), + ('𐐚', &['𐑂']), + ('𐐛', &['𐑃']), + ('𐐜', &['𐑄']), + ('𐐝', &['𐑅']), + ('𐐞', &['𐑆']), + ('𐐟', &['𐑇']), + ('𐐠', &['𐑈']), + ('𐐡', &['𐑉']), + ('𐐢', &['𐑊']), + ('𐐣', &['𐑋']), + ('𐐤', &['𐑌']), + ('𐐥', &['𐑍']), + ('𐐦', &['𐑎']), + ('𐐧', &['𐑏']), + ('𐐨', &['𐐀']), + ('𐐩', &['𐐁']), + ('𐐪', &['𐐂']), + ('𐐫', &['𐐃']), + ('𐐬', &['𐐄']), + ('𐐭', &['𐐅']), + ('𐐮', &['𐐆']), + ('𐐯', &['𐐇']), + ('𐐰', &['𐐈']), + ('𐐱', &['𐐉']), + ('𐐲', &['𐐊']), + ('𐐳', &['𐐋']), + ('𐐴', &['𐐌']), + ('𐐵', &['𐐍']), + ('𐐶', &['𐐎']), + ('𐐷', &['𐐏']), + ('𐐸', &['𐐐']), + ('𐐹', &['𐐑']), + ('𐐺', &['𐐒']), + ('𐐻', &['𐐓']), + ('𐐼', &['𐐔']), + ('𐐽', &['𐐕']), + ('𐐾', &['𐐖']), + ('𐐿', &['𐐗']), + ('𐑀', &['𐐘']), + ('𐑁', &['𐐙']), + ('𐑂', &['𐐚']), + ('𐑃', &['𐐛']), + ('𐑄', &['𐐜']), + ('𐑅', &['𐐝']), + ('𐑆', &['𐐞']), + ('𐑇', &['𐐟']), + ('𐑈', &['𐐠']), + ('𐑉', &['𐐡']), + ('𐑊', &['𐐢']), + ('𐑋', &['𐐣']), + ('𐑌', &['𐐤']), + ('𐑍', &['𐐥']), + ('𐑎', &['𐐦']), + ('𐑏', &['𐐧']), + ('𐒰', &['𐓘']), + ('𐒱', &['𐓙']), + ('𐒲', &['𐓚']), + ('𐒳', &['𐓛']), + ('𐒴', &['𐓜']), + ('𐒵', &['𐓝']), + ('𐒶', &['𐓞']), + ('𐒷', &['𐓟']), + ('𐒸', &['𐓠']), + ('𐒹', &['𐓡']), + ('𐒺', &['𐓢']), + ('𐒻', &['𐓣']), + ('𐒼', &['𐓤']), + ('𐒽', &['𐓥']), + ('𐒾', &['𐓦']), + ('𐒿', &['𐓧']), + ('𐓀', &['𐓨']), + ('𐓁', &['𐓩']), + ('𐓂', &['𐓪']), + ('𐓃', &['𐓫']), + ('𐓄', &['𐓬']), + ('𐓅', &['𐓭']), + ('𐓆', &['𐓮']), + ('𐓇', &['𐓯']), + ('𐓈', &['𐓰']), + ('𐓉', &['𐓱']), + ('𐓊', &['𐓲']), + ('𐓋', &['𐓳']), + ('𐓌', &['𐓴']), + ('𐓍', &['𐓵']), + ('𐓎', &['𐓶']), + ('𐓏', &['𐓷']), + ('𐓐', &['𐓸']), + ('𐓑', &['𐓹']), + ('𐓒', &['𐓺']), + ('𐓓', &['𐓻']), + ('𐓘', &['𐒰']), + ('𐓙', &['𐒱']), + ('𐓚', &['𐒲']), + ('𐓛', &['𐒳']), + ('𐓜', &['𐒴']), + ('𐓝', &['𐒵']), + ('𐓞', &['𐒶']), + ('𐓟', &['𐒷']), + ('𐓠', &['𐒸']), + ('𐓡', &['𐒹']), + ('𐓢', &['𐒺']), + ('𐓣', &['𐒻']), + ('𐓤', &['𐒼']), + ('𐓥', &['𐒽']), + ('𐓦', &['𐒾']), + ('𐓧', &['𐒿']), + ('𐓨', &['𐓀']), + ('𐓩', &['𐓁']), + ('𐓪', &['𐓂']), + ('𐓫', &['𐓃']), + ('𐓬', &['𐓄']), + ('𐓭', &['𐓅']), + ('𐓮', &['𐓆']), + ('𐓯', &['𐓇']), + ('𐓰', &['𐓈']), + ('𐓱', &['𐓉']), + ('𐓲', &['𐓊']), + ('𐓳', &['𐓋']), + ('𐓴', &['𐓌']), + ('𐓵', &['𐓍']), + ('𐓶', &['𐓎']), + ('𐓷', &['𐓏']), + ('𐓸', &['𐓐']), + ('𐓹', &['𐓑']), + ('𐓺', &['𐓒']), + ('𐓻', &['𐓓']), + ('𐕰', &['𐖗']), + ('𐕱', &['𐖘']), + ('𐕲', &['𐖙']), + ('𐕳', &['𐖚']), + ('𐕴', &['𐖛']), + ('𐕵', &['𐖜']), + ('𐕶', &['𐖝']), + ('𐕷', &['𐖞']), + ('𐕸', &['𐖟']), + ('𐕹', &['𐖠']), + ('𐕺', &['𐖡']), + ('𐕼', &['𐖣']), + ('𐕽', &['𐖤']), + ('𐕾', &['𐖥']), + ('𐕿', &['𐖦']), + ('𐖀', &['𐖧']), + ('𐖁', &['𐖨']), + ('𐖂', &['𐖩']), + ('𐖃', &['𐖪']), + ('𐖄', &['𐖫']), + ('𐖅', &['𐖬']), + ('𐖆', &['𐖭']), + ('𐖇', &['𐖮']), + ('𐖈', &['𐖯']), + ('𐖉', &['𐖰']), + ('𐖊', &['𐖱']), + ('𐖌', &['𐖳']), + ('𐖍', &['𐖴']), + ('𐖎', &['𐖵']), + ('𐖏', &['𐖶']), + ('𐖐', &['𐖷']), + ('𐖑', &['𐖸']), + ('𐖒', &['𐖹']), + ('𐖔', &['𐖻']), + ('𐖕', &['𐖼']), + ('𐖗', &['𐕰']), + ('𐖘', &['𐕱']), + ('𐖙', &['𐕲']), + ('𐖚', &['𐕳']), + ('𐖛', &['𐕴']), + ('𐖜', &['𐕵']), + ('𐖝', &['𐕶']), + ('𐖞', &['𐕷']), + ('𐖟', &['𐕸']), + ('𐖠', &['𐕹']), + ('𐖡', &['𐕺']), + ('𐖣', &['𐕼']), + ('𐖤', &['𐕽']), + ('𐖥', &['𐕾']), + ('𐖦', &['𐕿']), + ('𐖧', &['𐖀']), + ('𐖨', &['𐖁']), + ('𐖩', &['𐖂']), + ('𐖪', &['𐖃']), + ('𐖫', &['𐖄']), + ('𐖬', &['𐖅']), + ('𐖭', &['𐖆']), + ('𐖮', &['𐖇']), + ('𐖯', &['𐖈']), + ('𐖰', &['𐖉']), + ('𐖱', &['𐖊']), + ('𐖳', &['𐖌']), + ('𐖴', &['𐖍']), + ('𐖵', &['𐖎']), + ('𐖶', &['𐖏']), + ('𐖷', &['𐖐']), + ('𐖸', &['𐖑']), + ('𐖹', &['𐖒']), + ('𐖻', &['𐖔']), + ('𐖼', &['𐖕']), + ('𐲀', &['𐳀']), + ('𐲁', &['𐳁']), + ('𐲂', &['𐳂']), + ('𐲃', &['𐳃']), + ('𐲄', &['𐳄']), + ('𐲅', &['𐳅']), + ('𐲆', &['𐳆']), + ('𐲇', &['𐳇']), + ('𐲈', &['𐳈']), + ('𐲉', &['𐳉']), + ('𐲊', &['𐳊']), + ('𐲋', &['𐳋']), + ('𐲌', &['𐳌']), + ('𐲍', &['𐳍']), + ('𐲎', &['𐳎']), + ('𐲏', &['𐳏']), + ('𐲐', &['𐳐']), + ('𐲑', &['𐳑']), + ('𐲒', &['𐳒']), + ('𐲓', &['𐳓']), + ('𐲔', &['𐳔']), + ('𐲕', &['𐳕']), + ('𐲖', &['𐳖']), + ('𐲗', &['𐳗']), + ('𐲘', &['𐳘']), + ('𐲙', &['𐳙']), + ('𐲚', &['𐳚']), + ('𐲛', &['𐳛']), + ('𐲜', &['𐳜']), + ('𐲝', &['𐳝']), + ('𐲞', &['𐳞']), + ('𐲟', &['𐳟']), + ('𐲠', &['𐳠']), + ('𐲡', &['𐳡']), + ('𐲢', &['𐳢']), + ('𐲣', &['𐳣']), + ('𐲤', &['𐳤']), + ('𐲥', &['𐳥']), + ('𐲦', &['𐳦']), + ('𐲧', &['𐳧']), + ('𐲨', &['𐳨']), + ('𐲩', &['𐳩']), + ('𐲪', &['𐳪']), + ('𐲫', &['𐳫']), + ('𐲬', &['𐳬']), + ('𐲭', &['𐳭']), + ('𐲮', &['𐳮']), + ('𐲯', &['𐳯']), + ('𐲰', &['𐳰']), + ('𐲱', &['𐳱']), + ('𐲲', &['𐳲']), + ('𐳀', &['𐲀']), + ('𐳁', &['𐲁']), + ('𐳂', &['𐲂']), + ('𐳃', &['𐲃']), + ('𐳄', &['𐲄']), + ('𐳅', &['𐲅']), + ('𐳆', &['𐲆']), + ('𐳇', &['𐲇']), + ('𐳈', &['𐲈']), + ('𐳉', &['𐲉']), + ('𐳊', &['𐲊']), + ('𐳋', &['𐲋']), + ('𐳌', &['𐲌']), + ('𐳍', &['𐲍']), + ('𐳎', &['𐲎']), + ('𐳏', &['𐲏']), + ('𐳐', &['𐲐']), + ('𐳑', &['𐲑']), + ('𐳒', &['𐲒']), + ('𐳓', &['𐲓']), + ('𐳔', &['𐲔']), + ('𐳕', &['𐲕']), + ('𐳖', &['𐲖']), + ('𐳗', &['𐲗']), + ('𐳘', &['𐲘']), + ('𐳙', &['𐲙']), + ('𐳚', &['𐲚']), + ('𐳛', &['𐲛']), + ('𐳜', &['𐲜']), + ('𐳝', &['𐲝']), + ('𐳞', &['𐲞']), + ('𐳟', &['𐲟']), + ('𐳠', &['𐲠']), + ('𐳡', &['𐲡']), + ('𐳢', &['𐲢']), + ('𐳣', &['𐲣']), + ('𐳤', &['𐲤']), + ('𐳥', &['𐲥']), + ('𐳦', &['𐲦']), + ('𐳧', &['𐲧']), + ('𐳨', &['𐲨']), + ('𐳩', &['𐲩']), + ('𐳪', &['𐲪']), + ('𐳫', &['𐲫']), + ('𐳬', &['𐲬']), + ('𐳭', &['𐲭']), + ('𐳮', &['𐲮']), + ('𐳯', &['𐲯']), + ('𐳰', &['𐲰']), + ('𐳱', &['𐲱']), + ('𐳲', &['𐲲']), + ('𐵐', &['𐵰']), + ('𐵑', &['𐵱']), + ('𐵒', &['𐵲']), + ('𐵓', &['𐵳']), + ('𐵔', &['𐵴']), + ('𐵕', &['𐵵']), + ('𐵖', &['𐵶']), + ('𐵗', &['𐵷']), + ('𐵘', &['𐵸']), + ('𐵙', &['𐵹']), + ('𐵚', &['𐵺']), + ('𐵛', &['𐵻']), + ('𐵜', &['𐵼']), + ('𐵝', &['𐵽']), + ('𐵞', &['𐵾']), + ('𐵟', &['𐵿']), + ('𐵠', &['𐶀']), + ('𐵡', &['𐶁']), + ('𐵢', &['𐶂']), + ('𐵣', &['𐶃']), + ('𐵤', &['𐶄']), + ('𐵥', &['𐶅']), + ('𐵰', &['𐵐']), + ('𐵱', &['𐵑']), + ('𐵲', &['𐵒']), + ('𐵳', &['𐵓']), + ('𐵴', &['𐵔']), + ('𐵵', &['𐵕']), + ('𐵶', &['𐵖']), + ('𐵷', &['𐵗']), + ('𐵸', &['𐵘']), + ('𐵹', &['𐵙']), + ('𐵺', &['𐵚']), + ('𐵻', &['𐵛']), + ('𐵼', &['𐵜']), + ('𐵽', &['𐵝']), + ('𐵾', &['𐵞']), + ('𐵿', &['𐵟']), + ('𐶀', &['𐵠']), + ('𐶁', &['𐵡']), + ('𐶂', &['𐵢']), + ('𐶃', &['𐵣']), + ('𐶄', &['𐵤']), + ('𐶅', &['𐵥']), + ('𑢠', &['𑣀']), + ('𑢡', &['𑣁']), + ('𑢢', &['𑣂']), + ('𑢣', &['𑣃']), + ('𑢤', &['𑣄']), + ('𑢥', &['𑣅']), + ('𑢦', &['𑣆']), + ('𑢧', &['𑣇']), + ('𑢨', &['𑣈']), + ('𑢩', &['𑣉']), + ('𑢪', &['𑣊']), + ('𑢫', &['𑣋']), + ('𑢬', &['𑣌']), + ('𑢭', &['𑣍']), + ('𑢮', &['𑣎']), + ('𑢯', &['𑣏']), + ('𑢰', &['𑣐']), + ('𑢱', &['𑣑']), + ('𑢲', &['𑣒']), + ('𑢳', &['𑣓']), + ('𑢴', &['𑣔']), + ('𑢵', &['𑣕']), + ('𑢶', &['𑣖']), + ('𑢷', &['𑣗']), + ('𑢸', &['𑣘']), + ('𑢹', &['𑣙']), + ('𑢺', &['𑣚']), + ('𑢻', &['𑣛']), + ('𑢼', &['𑣜']), + ('𑢽', &['𑣝']), + ('𑢾', &['𑣞']), + ('𑢿', &['𑣟']), + ('𑣀', &['𑢠']), + ('𑣁', &['𑢡']), + ('𑣂', &['𑢢']), + ('𑣃', &['𑢣']), + ('𑣄', &['𑢤']), + ('𑣅', &['𑢥']), + ('𑣆', &['𑢦']), + ('𑣇', &['𑢧']), + ('𑣈', &['𑢨']), + ('𑣉', &['𑢩']), + ('𑣊', &['𑢪']), + ('𑣋', &['𑢫']), + ('𑣌', &['𑢬']), + ('𑣍', &['𑢭']), + ('𑣎', &['𑢮']), + ('𑣏', &['𑢯']), + ('𑣐', &['𑢰']), + ('𑣑', &['𑢱']), + ('𑣒', &['𑢲']), + ('𑣓', &['𑢳']), + ('𑣔', &['𑢴']), + ('𑣕', &['𑢵']), + ('𑣖', &['𑢶']), + ('𑣗', &['𑢷']), + ('𑣘', &['𑢸']), + ('𑣙', &['𑢹']), + ('𑣚', &['𑢺']), + ('𑣛', &['𑢻']), + ('𑣜', &['𑢼']), + ('𑣝', &['𑢽']), + ('𑣞', &['𑢾']), + ('𑣟', &['𑢿']), + ('𖹀', &['𖹠']), + ('𖹁', &['𖹡']), + ('𖹂', &['𖹢']), + ('𖹃', &['𖹣']), + ('𖹄', &['𖹤']), + ('𖹅', &['𖹥']), + ('𖹆', &['𖹦']), + ('𖹇', &['𖹧']), + ('𖹈', &['𖹨']), + ('𖹉', &['𖹩']), + ('𖹊', &['𖹪']), + ('𖹋', &['𖹫']), + ('𖹌', &['𖹬']), + ('𖹍', &['𖹭']), + ('𖹎', &['𖹮']), + ('𖹏', &['𖹯']), + ('𖹐', &['𖹰']), + ('𖹑', &['𖹱']), + ('𖹒', &['𖹲']), + ('𖹓', &['𖹳']), + ('𖹔', &['𖹴']), + ('𖹕', &['𖹵']), + ('𖹖', &['𖹶']), + ('𖹗', &['𖹷']), + ('𖹘', &['𖹸']), + ('𖹙', &['𖹹']), + ('𖹚', &['𖹺']), + ('𖹛', &['𖹻']), + ('𖹜', &['𖹼']), + ('𖹝', &['𖹽']), + ('𖹞', &['𖹾']), + ('𖹟', &['𖹿']), + ('𖹠', &['𖹀']), + ('𖹡', &['𖹁']), + ('𖹢', &['𖹂']), + ('𖹣', &['𖹃']), + ('𖹤', &['𖹄']), + ('𖹥', &['𖹅']), + ('𖹦', &['𖹆']), + ('𖹧', &['𖹇']), + ('𖹨', &['𖹈']), + ('𖹩', &['𖹉']), + ('𖹪', &['𖹊']), + ('𖹫', &['𖹋']), + ('𖹬', &['𖹌']), + ('𖹭', &['𖹍']), + ('𖹮', &['𖹎']), + ('𖹯', &['𖹏']), + ('𖹰', &['𖹐']), + ('𖹱', &['𖹑']), + ('𖹲', &['𖹒']), + ('𖹳', &['𖹓']), + ('𖹴', &['𖹔']), + ('𖹵', &['𖹕']), + ('𖹶', &['𖹖']), + ('𖹷', &['𖹗']), + ('𖹸', &['𖹘']), + ('𖹹', &['𖹙']), + ('𖹺', &['𖹚']), + ('𖹻', &['𖹛']), + ('𖹼', &['𖹜']), + ('𖹽', &['𖹝']), + ('𖹾', &['𖹞']), + ('𖹿', &['𖹟']), + ('𞤀', &['𞤢']), + ('𞤁', &['𞤣']), + ('𞤂', &['𞤤']), + ('𞤃', &['𞤥']), + ('𞤄', &['𞤦']), + ('𞤅', &['𞤧']), + ('𞤆', &['𞤨']), + ('𞤇', &['𞤩']), + ('𞤈', &['𞤪']), + ('𞤉', &['𞤫']), + ('𞤊', &['𞤬']), + ('𞤋', &['𞤭']), + ('𞤌', &['𞤮']), + ('𞤍', &['𞤯']), + ('𞤎', &['𞤰']), + ('𞤏', &['𞤱']), + ('𞤐', &['𞤲']), + ('𞤑', &['𞤳']), + ('𞤒', &['𞤴']), + ('𞤓', &['𞤵']), + ('𞤔', &['𞤶']), + ('𞤕', &['𞤷']), + ('𞤖', &['𞤸']), + ('𞤗', &['𞤹']), + ('𞤘', &['𞤺']), + ('𞤙', &['𞤻']), + ('𞤚', &['𞤼']), + ('𞤛', &['𞤽']), + ('𞤜', &['𞤾']), + ('𞤝', &['𞤿']), + ('𞤞', &['𞥀']), + ('𞤟', &['𞥁']), + ('𞤠', &['𞥂']), + ('𞤡', &['𞥃']), + ('𞤢', &['𞤀']), + ('𞤣', &['𞤁']), + ('𞤤', &['𞤂']), + ('𞤥', &['𞤃']), + ('𞤦', &['𞤄']), + ('𞤧', &['𞤅']), + ('𞤨', &['𞤆']), + ('𞤩', &['𞤇']), + ('𞤪', &['𞤈']), + ('𞤫', &['𞤉']), + ('𞤬', &['𞤊']), + ('𞤭', &['𞤋']), + ('𞤮', &['𞤌']), + ('𞤯', &['𞤍']), + ('𞤰', &['𞤎']), + ('𞤱', &['𞤏']), + ('𞤲', &['𞤐']), + ('𞤳', &['𞤑']), + ('𞤴', &['𞤒']), + ('𞤵', &['𞤓']), + ('𞤶', &['𞤔']), + ('𞤷', &['𞤕']), + ('𞤸', &['𞤖']), + ('𞤹', &['𞤗']), + ('𞤺', &['𞤘']), + ('𞤻', &['𞤙']), + ('𞤼', &['𞤚']), + ('𞤽', &['𞤛']), + ('𞤾', &['𞤜']), + ('𞤿', &['𞤝']), + ('𞥀', &['𞤞']), + ('𞥁', &['𞤟']), + ('𞥂', &['𞤠']), + ('𞥃', &['𞤡']), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/general_category.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/general_category.rs new file mode 100644 index 0000000..6ff6b53 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/general_category.rs @@ -0,0 +1,6717 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate general-category ucd-16.0.0 --chars --exclude surrogate +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = &[ + ("Cased_Letter", CASED_LETTER), + ("Close_Punctuation", CLOSE_PUNCTUATION), + ("Connector_Punctuation", CONNECTOR_PUNCTUATION), + ("Control", CONTROL), + ("Currency_Symbol", CURRENCY_SYMBOL), + ("Dash_Punctuation", DASH_PUNCTUATION), + ("Decimal_Number", DECIMAL_NUMBER), + ("Enclosing_Mark", ENCLOSING_MARK), + ("Final_Punctuation", FINAL_PUNCTUATION), + ("Format", FORMAT), + ("Initial_Punctuation", INITIAL_PUNCTUATION), + ("Letter", LETTER), + ("Letter_Number", LETTER_NUMBER), + ("Line_Separator", LINE_SEPARATOR), + ("Lowercase_Letter", LOWERCASE_LETTER), + ("Mark", MARK), + ("Math_Symbol", MATH_SYMBOL), + ("Modifier_Letter", MODIFIER_LETTER), + ("Modifier_Symbol", MODIFIER_SYMBOL), + ("Nonspacing_Mark", NONSPACING_MARK), + ("Number", NUMBER), + ("Open_Punctuation", OPEN_PUNCTUATION), + ("Other", OTHER), + ("Other_Letter", OTHER_LETTER), + ("Other_Number", OTHER_NUMBER), + ("Other_Punctuation", OTHER_PUNCTUATION), + ("Other_Symbol", OTHER_SYMBOL), + ("Paragraph_Separator", PARAGRAPH_SEPARATOR), + ("Private_Use", PRIVATE_USE), + ("Punctuation", PUNCTUATION), + ("Separator", SEPARATOR), + ("Space_Separator", SPACE_SEPARATOR), + ("Spacing_Mark", SPACING_MARK), + ("Symbol", SYMBOL), + ("Titlecase_Letter", TITLECASE_LETTER), + ("Unassigned", UNASSIGNED), + ("Uppercase_Letter", UPPERCASE_LETTER), +]; + +pub const CASED_LETTER: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('µ', 'µ'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ƺ'), + ('Ƽ', 'ƿ'), + ('DŽ', 'ʓ'), + ('ʕ', 'ʯ'), + ('Ͱ', 'ͳ'), + ('Ͷ', 'ͷ'), + ('ͻ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՠ', 'ֈ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჽ', 'ჿ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᴀ', 'ᴫ'), + ('ᵫ', 'ᵷ'), + ('ᵹ', 'ᶚ'), + ('Ḁ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℯ', 'ℴ'), + ('ℹ', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ↄ', 'ↄ'), + ('Ⰰ', 'ⱻ'), + ('Ȿ', 'ⳤ'), + ('Ⳬ', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('Ꙁ', 'ꙭ'), + ('Ꚁ', 'ꚛ'), + ('Ꜣ', 'ꝯ'), + ('ꝱ', 'ꞇ'), + ('Ꞌ', 'ꞎ'), + ('Ꞑ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('Ꟶ', 'ꟶ'), + ('ꟺ', 'ꟺ'), + ('ꬰ', 'ꭚ'), + ('ꭠ', 'ꭨ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('𐐀', '𐑏'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐵐', '𐵥'), + ('𐵰', '𐶅'), + ('𑢠', '𑣟'), + ('𖹀', '𖹿'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝼀', '𝼉'), + ('𝼋', '𝼞'), + ('𝼥', '𝼪'), + ('𞤀', '𞥃'), +]; + +pub const CLOSE_PUNCTUATION: &'static [(char, char)] = &[ + (')', ')'), + (']', ']'), + ('}', '}'), + ('༻', '༻'), + ('༽', '༽'), + ('᚜', '᚜'), + ('⁆', '⁆'), + ('⁾', '⁾'), + ('₎', '₎'), + ('⌉', '⌉'), + ('⌋', '⌋'), + ('〉', '〉'), + ('❩', '❩'), + ('❫', '❫'), + ('❭', '❭'), + ('❯', '❯'), + ('❱', '❱'), + ('❳', '❳'), + ('❵', '❵'), + ('⟆', '⟆'), + ('⟧', '⟧'), + ('⟩', '⟩'), + ('⟫', '⟫'), + ('⟭', '⟭'), + ('⟯', '⟯'), + ('⦄', '⦄'), + ('⦆', '⦆'), + ('⦈', '⦈'), + ('⦊', '⦊'), + ('⦌', '⦌'), + ('⦎', '⦎'), + ('⦐', '⦐'), + ('⦒', '⦒'), + ('⦔', '⦔'), + ('⦖', '⦖'), + ('⦘', '⦘'), + ('⧙', '⧙'), + ('⧛', '⧛'), + ('⧽', '⧽'), + ('⸣', '⸣'), + ('⸥', '⸥'), + ('⸧', '⸧'), + ('⸩', '⸩'), + ('⹖', '⹖'), + ('⹘', '⹘'), + ('⹚', '⹚'), + ('⹜', '⹜'), + ('〉', '〉'), + ('》', '》'), + ('」', '」'), + ('』', '』'), + ('】', '】'), + ('〕', '〕'), + ('〗', '〗'), + ('〙', '〙'), + ('〛', '〛'), + ('〞', '〟'), + ('﴾', '﴾'), + ('︘', '︘'), + ('︶', '︶'), + ('︸', '︸'), + ('︺', '︺'), + ('︼', '︼'), + ('︾', '︾'), + ('﹀', '﹀'), + ('﹂', '﹂'), + ('﹄', '﹄'), + ('﹈', '﹈'), + ('﹚', '﹚'), + ('﹜', '﹜'), + ('﹞', '﹞'), + ('）', '）'), + ('］', '］'), + ('｝', '｝'), + ('｠', '｠'), + ('｣', '｣'), +]; + +pub const CONNECTOR_PUNCTUATION: &'static [(char, char)] = &[ + ('_', '_'), + ('‿', '⁀'), + ('⁔', '⁔'), + ('︳', '︴'), + ('﹍', '﹏'), + ('＿', '＿'), +]; + +pub const CONTROL: &'static [(char, char)] = + &[('\0', '\u{1f}'), ('\u{7f}', '\u{9f}')]; + +pub const CURRENCY_SYMBOL: &'static [(char, char)] = &[ + ('$', '$'), + ('¢', '¥'), + ('֏', '֏'), + ('؋', '؋'), + ('߾', '߿'), + ('৲', '৳'), + ('৻', '৻'), + ('૱', '૱'), + ('௹', '௹'), + ('฿', '฿'), + ('៛', '៛'), + ('₠', '⃀'), + ('꠸', '꠸'), + ('﷼', '﷼'), + ('﹩', '﹩'), + ('＄', '＄'), + ('￠', '￡'), + ('￥', '￦'), + ('𑿝', '𑿠'), + ('𞋿', '𞋿'), + ('𞲰', '𞲰'), +]; + +pub const DASH_PUNCTUATION: &'static [(char, char)] = &[ + ('-', '-'), + ('֊', '֊'), + ('־', '־'), + ('᐀', '᐀'), + ('᠆', '᠆'), + ('‐', '―'), + ('⸗', '⸗'), + ('⸚', '⸚'), + ('⸺', '⸻'), + ('⹀', '⹀'), + ('⹝', '⹝'), + ('〜', '〜'), + ('〰', '〰'), + ('゠', '゠'), + ('︱', '︲'), + ('﹘', '﹘'), + ('﹣', '﹣'), + ('－', '－'), + ('𐵮', '𐵮'), + ('𐺭', '𐺭'), +]; + +pub const DECIMAL_NUMBER: &'static [(char, char)] = &[ + ('0', '9'), + ('٠', '٩'), + ('۰', '۹'), + ('߀', '߉'), + ('०', '९'), + ('০', '৯'), + ('੦', '੯'), + ('૦', '૯'), + ('୦', '୯'), + ('௦', '௯'), + ('౦', '౯'), + ('೦', '೯'), + ('൦', '൯'), + ('෦', '෯'), + ('๐', '๙'), + ('໐', '໙'), + ('༠', '༩'), + ('၀', '၉'), + ('႐', '႙'), + ('០', '៩'), + ('᠐', '᠙'), + ('᥆', '᥏'), + ('᧐', '᧙'), + ('᪀', '᪉'), + ('᪐', '᪙'), + ('᭐', '᭙'), + ('᮰', '᮹'), + ('᱀', '᱉'), + ('᱐', '᱙'), + ('꘠', '꘩'), + ('꣐', '꣙'), + ('꤀', '꤉'), + ('꧐', '꧙'), + ('꧰', '꧹'), + ('꩐', '꩙'), + ('꯰', '꯹'), + ('０', '９'), + ('𐒠', '𐒩'), + ('𐴰', '𐴹'), + ('𐵀', '𐵉'), + ('𑁦', '𑁯'), + ('𑃰', '𑃹'), + ('𑄶', '𑄿'), + ('𑇐', '𑇙'), + ('𑋰', '𑋹'), + ('𑑐', '𑑙'), + ('𑓐', '𑓙'), + ('𑙐', '𑙙'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜰', '𑜹'), + ('𑣠', '𑣩'), + ('𑥐', '𑥙'), + ('𑯰', '𑯹'), + ('𑱐', '𑱙'), + ('𑵐', '𑵙'), + ('𑶠', '𑶩'), + ('𑽐', '𑽙'), + ('𖄰', '𖄹'), + ('𖩠', '𖩩'), + ('𖫀', '𖫉'), + ('𖭐', '𖭙'), + ('𖵰', '𖵹'), + ('𜳰', '𜳹'), + ('𝟎', '𝟿'), + ('𞅀', '𞅉'), + ('𞋰', '𞋹'), + ('𞓰', '𞓹'), + ('𞗱', '𞗺'), + ('𞥐', '𞥙'), + ('🯰', '🯹'), +]; + +pub const ENCLOSING_MARK: &'static [(char, char)] = &[ + ('\u{488}', '\u{489}'), + ('\u{1abe}', '\u{1abe}'), + ('\u{20dd}', '\u{20e0}'), + ('\u{20e2}', '\u{20e4}'), + ('\u{a670}', '\u{a672}'), +]; + +pub const FINAL_PUNCTUATION: &'static [(char, char)] = &[ + ('»', '»'), + ('’', '’'), + ('”', '”'), + ('›', '›'), + ('⸃', '⸃'), + ('⸅', '⸅'), + ('⸊', '⸊'), + ('⸍', '⸍'), + ('⸝', '⸝'), + ('⸡', '⸡'), +]; + +pub const FORMAT: &'static [(char, char)] = &[ + ('\u{ad}', '\u{ad}'), + ('\u{600}', '\u{605}'), + ('\u{61c}', '\u{61c}'), + ('\u{6dd}', '\u{6dd}'), + ('\u{70f}', '\u{70f}'), + ('\u{890}', '\u{891}'), + ('\u{8e2}', '\u{8e2}'), + ('\u{180e}', '\u{180e}'), + ('\u{200b}', '\u{200f}'), + ('\u{202a}', '\u{202e}'), + ('\u{2060}', '\u{2064}'), + ('\u{2066}', '\u{206f}'), + ('\u{feff}', '\u{feff}'), + ('\u{fff9}', '\u{fffb}'), + ('\u{110bd}', '\u{110bd}'), + ('\u{110cd}', '\u{110cd}'), + ('\u{13430}', '\u{1343f}'), + ('\u{1bca0}', '\u{1bca3}'), + ('\u{1d173}', '\u{1d17a}'), + ('\u{e0001}', '\u{e0001}'), + ('\u{e0020}', '\u{e007f}'), +]; + +pub const INITIAL_PUNCTUATION: &'static [(char, char)] = &[ + ('«', '«'), + ('‘', '‘'), + ('‛', '“'), + ('‟', '‟'), + ('‹', '‹'), + ('⸂', '⸂'), + ('⸄', '⸄'), + ('⸉', '⸉'), + ('⸌', '⸌'), + ('⸜', '⸜'), + ('⸠', '⸠'), +]; + +pub const LETTER: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('Ͱ', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', 'ՙ'), + ('ՠ', 'ֈ'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('ؠ', 'ي'), + ('ٮ', 'ٯ'), + ('ٱ', 'ۓ'), + ('ە', 'ە'), + ('ۥ', 'ۦ'), + ('ۮ', 'ۯ'), + ('ۺ', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', 'ܐ'), + ('ܒ', 'ܯ'), + ('ݍ', 'ޥ'), + ('ޱ', 'ޱ'), + ('ߊ', 'ߪ'), + ('ߴ', 'ߵ'), + ('ߺ', 'ߺ'), + ('ࠀ', 'ࠕ'), + ('ࠚ', 'ࠚ'), + ('ࠤ', 'ࠤ'), + ('ࠨ', 'ࠨ'), + ('ࡀ', 'ࡘ'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('ࢠ', 'ࣉ'), + ('ऄ', 'ह'), + ('ऽ', 'ऽ'), + ('ॐ', 'ॐ'), + ('क़', 'ॡ'), + ('ॱ', 'ঀ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('ঽ', 'ঽ'), + ('ৎ', 'ৎ'), + ('ড়', 'ঢ়'), + ('য়', 'ৡ'), + ('ৰ', 'ৱ'), + ('ৼ', 'ৼ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('ੲ', 'ੴ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('ઽ', 'ઽ'), + ('ૐ', 'ૐ'), + ('ૠ', 'ૡ'), + ('ૹ', 'ૹ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('ଽ', 'ଽ'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୡ'), + ('ୱ', 'ୱ'), + ('ஃ', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('ௐ', 'ௐ'), + ('అ', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('ఽ', 'ఽ'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', 'ౡ'), + ('ಀ', 'ಀ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಽ', 'ಽ'), + ('ೝ', 'ೞ'), + ('ೠ', 'ೡ'), + ('ೱ', 'ೲ'), + ('ഄ', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ഺ'), + ('ഽ', 'ഽ'), + ('ൎ', 'ൎ'), + ('ൔ', 'ൖ'), + ('ൟ', 'ൡ'), + ('ൺ', 'ൿ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('ก', 'ะ'), + ('า', 'ำ'), + ('เ', 'ๆ'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ະ'), + ('າ', 'ຳ'), + ('ຽ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('ཀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('ྈ', 'ྌ'), + ('က', 'ဪ'), + ('ဿ', 'ဿ'), + ('ၐ', 'ၕ'), + ('ၚ', 'ၝ'), + ('ၡ', 'ၡ'), + ('ၥ', 'ၦ'), + ('ၮ', 'ၰ'), + ('ၵ', 'ႁ'), + ('ႎ', 'ႎ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛱ', 'ᛸ'), + ('ᜀ', 'ᜑ'), + ('ᜟ', 'ᜱ'), + ('ᝀ', 'ᝑ'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('ក', 'ឳ'), + ('ៗ', 'ៗ'), + ('ៜ', 'ៜ'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢄ'), + ('ᢇ', 'ᢨ'), + ('ᢪ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('ᥐ', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('ᨀ', 'ᨖ'), + ('ᨠ', 'ᩔ'), + ('ᪧ', 'ᪧ'), + ('ᬅ', 'ᬳ'), + ('ᭅ', 'ᭌ'), + ('ᮃ', 'ᮠ'), + ('ᮮ', 'ᮯ'), + ('ᮺ', 'ᯥ'), + ('ᰀ', 'ᰣ'), + ('ᱍ', 'ᱏ'), + ('ᱚ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', 'ᳶ'), + ('ᳺ', 'ᳺ'), + ('ᴀ', 'ᶿ'), + ('Ḁ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℯ', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ↄ', 'ↄ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('ⸯ', 'ⸯ'), + ('々', '〆'), + ('〱', '〵'), + ('〻', '〼'), + ('ぁ', 'ゖ'), + ('ゝ', 'ゟ'), + ('ァ', 'ヺ'), + ('ー', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘟ'), + ('ꘪ', 'ꘫ'), + ('Ꙁ', 'ꙮ'), + ('ꙿ', 'ꚝ'), + ('ꚠ', 'ꛥ'), + ('ꜗ', 'ꜟ'), + ('Ꜣ', 'ꞈ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠁ'), + ('ꠃ', 'ꠅ'), + ('ꠇ', 'ꠊ'), + ('ꠌ', 'ꠢ'), + ('ꡀ', 'ꡳ'), + ('ꢂ', 'ꢳ'), + ('ꣲ', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', 'ꣾ'), + ('ꤊ', 'ꤥ'), + ('ꤰ', 'ꥆ'), + ('ꥠ', 'ꥼ'), + ('ꦄ', 'ꦲ'), + ('ꧏ', 'ꧏ'), + ('ꧠ', 'ꧤ'), + ('ꧦ', 'ꧯ'), + ('ꧺ', 'ꧾ'), + ('ꨀ', 'ꨨ'), + ('ꩀ', 'ꩂ'), + ('ꩄ', 'ꩋ'), + ('ꩠ', 'ꩶ'), + ('ꩺ', 'ꩺ'), + ('ꩾ', 'ꪯ'), + ('ꪱ', 'ꪱ'), + ('ꪵ', 'ꪶ'), + ('ꪹ', 'ꪽ'), + ('ꫀ', 'ꫀ'), + ('ꫂ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫪ'), + ('ꫲ', 'ꫴ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꯢ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'יִ'), + ('ײַ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('ｦ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐌀', '𐌟'), + ('𐌭', '𐍀'), + ('𐍂', '𐍉'), + ('𐍐', '𐍵'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐐀', '𐒝'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '𐨀'), + ('𐨐', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '𐫤'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '𐴣'), + ('𐵊', '𐵥'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('𐼀', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '𐽅'), + ('𐽰', '𐾁'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀃', '𑀷'), + ('𑁱', '𑁲'), + ('𑁵', '𑁵'), + ('𑂃', '𑂯'), + ('𑃐', '𑃨'), + ('𑄃', '𑄦'), + ('𑅄', '𑅄'), + ('𑅇', '𑅇'), + ('𑅐', '𑅲'), + ('𑅶', '𑅶'), + ('𑆃', '𑆲'), + ('𑇁', '𑇄'), + ('𑇚', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '𑈫'), + ('𑈿', '𑉀'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '𑋞'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('𑌽', '𑌽'), + ('𑍐', '𑍐'), + ('𑍝', '𑍡'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '𑎷'), + ('𑏑', '𑏑'), + ('𑏓', '𑏓'), + ('𑐀', '𑐴'), + ('𑑇', '𑑊'), + ('𑑟', '𑑡'), + ('𑒀', '𑒯'), + ('𑓄', '𑓅'), + ('𑓇', '𑓇'), + ('𑖀', '𑖮'), + ('𑗘', '𑗛'), + ('𑘀', '𑘯'), + ('𑙄', '𑙄'), + ('𑚀', '𑚪'), + ('𑚸', '𑚸'), + ('𑜀', '𑜚'), + ('𑝀', '𑝆'), + ('𑠀', '𑠫'), + ('𑢠', '𑣟'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤯'), + ('𑤿', '𑤿'), + ('𑥁', '𑥁'), + ('𑦠', '𑦧'), + ('𑦪', '𑧐'), + ('𑧡', '𑧡'), + ('𑧣', '𑧣'), + ('𑨀', '𑨀'), + ('𑨋', '𑨲'), + ('𑨺', '𑨺'), + ('𑩐', '𑩐'), + ('𑩜', '𑪉'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑰀', '𑰈'), + ('𑰊', '𑰮'), + ('𑱀', '𑱀'), + ('𑱲', '𑲏'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '𑴰'), + ('𑵆', '𑵆'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶉'), + ('𑶘', '𑶘'), + ('𑻠', '𑻲'), + ('𑼂', '𑼂'), + ('𑼄', '𑼐'), + ('𑼒', '𑼳'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('𓑁', '𓑆'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄝'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩰', '𖪾'), + ('𖫐', '𖫭'), + ('𖬀', '𖬯'), + ('𖭀', '𖭃'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('𖽐', '𖽐'), + ('𖾓', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '𖿣'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('𞄀', '𞄬'), + ('𞄷', '𞄽'), + ('𞅎', '𞅎'), + ('𞊐', '𞊭'), + ('𞋀', '𞋫'), + ('𞓐', '𞓫'), + ('𞗐', '𞗭'), + ('𞗰', '𞗰'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('𞤀', '𞥃'), + ('𞥋', '𞥋'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const LETTER_NUMBER: &'static [(char, char)] = &[ + ('ᛮ', 'ᛰ'), + ('Ⅰ', 'ↂ'), + ('ↅ', 'ↈ'), + ('〇', '〇'), + ('〡', '〩'), + ('〸', '〺'), + ('ꛦ', 'ꛯ'), + ('𐅀', '𐅴'), + ('𐍁', '𐍁'), + ('𐍊', '𐍊'), + ('𐏑', '𐏕'), + ('𒐀', '𒑮'), +]; + +pub const LINE_SEPARATOR: &'static [(char, char)] = + &[('\u{2028}', '\u{2028}')]; + +pub const LOWERCASE_LETTER: &'static [(char, char)] = &[ + ('a', 'z'), + ('µ', 'µ'), + ('ß', 'ö'), + ('ø', 'ÿ'), + ('ā', 'ā'), + ('ă', 'ă'), + ('ą', 'ą'), + ('ć', 'ć'), + ('ĉ', 'ĉ'), + ('ċ', 'ċ'), + ('č', 'č'), + ('ď', 'ď'), + ('đ', 'đ'), + ('ē', 'ē'), + ('ĕ', 'ĕ'), + ('ė', 'ė'), + ('ę', 'ę'), + ('ě', 'ě'), + ('ĝ', 'ĝ'), + ('ğ', 'ğ'), + ('ġ', 'ġ'), + ('ģ', 'ģ'), + ('ĥ', 'ĥ'), + ('ħ', 'ħ'), + ('ĩ', 'ĩ'), + ('ī', 'ī'), + ('ĭ', 'ĭ'), + ('į', 'į'), + ('ı', 'ı'), + ('ij', 'ij'), + ('ĵ', 'ĵ'), + ('ķ', 'ĸ'), + ('ĺ', 'ĺ'), + ('ļ', 'ļ'), + ('ľ', 'ľ'), + ('ŀ', 'ŀ'), + ('ł', 'ł'), + ('ń', 'ń'), + ('ņ', 'ņ'), + ('ň', 'ʼn'), + ('ŋ', 'ŋ'), + ('ō', 'ō'), + ('ŏ', 'ŏ'), + ('ő', 'ő'), + ('œ', 'œ'), + ('ŕ', 'ŕ'), + ('ŗ', 'ŗ'), + ('ř', 'ř'), + ('ś', 'ś'), + ('ŝ', 'ŝ'), + ('ş', 'ş'), + ('š', 'š'), + ('ţ', 'ţ'), + ('ť', 'ť'), + ('ŧ', 'ŧ'), + ('ũ', 'ũ'), + ('ū', 'ū'), + ('ŭ', 'ŭ'), + ('ů', 'ů'), + ('ű', 'ű'), + ('ų', 'ų'), + ('ŵ', 'ŵ'), + ('ŷ', 'ŷ'), + ('ź', 'ź'), + ('ż', 'ż'), + ('ž', 'ƀ'), + ('ƃ', 'ƃ'), + ('ƅ', 'ƅ'), + ('ƈ', 'ƈ'), + ('ƌ', 'ƍ'), + ('ƒ', 'ƒ'), + ('ƕ', 'ƕ'), + ('ƙ', 'ƛ'), + ('ƞ', 'ƞ'), + ('ơ', 'ơ'), + ('ƣ', 'ƣ'), + ('ƥ', 'ƥ'), + ('ƨ', 'ƨ'), + ('ƪ', 'ƫ'), + ('ƭ', 'ƭ'), + ('ư', 'ư'), + ('ƴ', 'ƴ'), + ('ƶ', 'ƶ'), + ('ƹ', 'ƺ'), + ('ƽ', 'ƿ'), + ('dž', 'dž'), + ('lj', 'lj'), + ('nj', 'nj'), + ('ǎ', 'ǎ'), + ('ǐ', 'ǐ'), + ('ǒ', 'ǒ'), + ('ǔ', 'ǔ'), + ('ǖ', 'ǖ'), + ('ǘ', 'ǘ'), + ('ǚ', 'ǚ'), + ('ǜ', 'ǝ'), + ('ǟ', 'ǟ'), + ('ǡ', 'ǡ'), + ('ǣ', 'ǣ'), + ('ǥ', 'ǥ'), + ('ǧ', 'ǧ'), + ('ǩ', 'ǩ'), + ('ǫ', 'ǫ'), + ('ǭ', 'ǭ'), + ('ǯ', 'ǰ'), + ('dz', 'dz'), + ('ǵ', 'ǵ'), + ('ǹ', 'ǹ'), + ('ǻ', 'ǻ'), + ('ǽ', 'ǽ'), + ('ǿ', 'ǿ'), + ('ȁ', 'ȁ'), + ('ȃ', 'ȃ'), + ('ȅ', 'ȅ'), + ('ȇ', 'ȇ'), + ('ȉ', 'ȉ'), + ('ȋ', 'ȋ'), + ('ȍ', 'ȍ'), + ('ȏ', 'ȏ'), + ('ȑ', 'ȑ'), + ('ȓ', 'ȓ'), + ('ȕ', 'ȕ'), + ('ȗ', 'ȗ'), + ('ș', 'ș'), + ('ț', 'ț'), + ('ȝ', 'ȝ'), + ('ȟ', 'ȟ'), + ('ȡ', 'ȡ'), + ('ȣ', 'ȣ'), + ('ȥ', 'ȥ'), + ('ȧ', 'ȧ'), + ('ȩ', 'ȩ'), + ('ȫ', 'ȫ'), + ('ȭ', 'ȭ'), + ('ȯ', 'ȯ'), + ('ȱ', 'ȱ'), + ('ȳ', 'ȹ'), + ('ȼ', 'ȼ'), + ('ȿ', 'ɀ'), + ('ɂ', 'ɂ'), + ('ɇ', 'ɇ'), + ('ɉ', 'ɉ'), + ('ɋ', 'ɋ'), + ('ɍ', 'ɍ'), + ('ɏ', 'ʓ'), + ('ʕ', 'ʯ'), + ('ͱ', 'ͱ'), + ('ͳ', 'ͳ'), + ('ͷ', 'ͷ'), + ('ͻ', 'ͽ'), + ('ΐ', 'ΐ'), + ('ά', 'ώ'), + ('ϐ', 'ϑ'), + ('ϕ', 'ϗ'), + ('ϙ', 'ϙ'), + ('ϛ', 'ϛ'), + ('ϝ', 'ϝ'), + ('ϟ', 'ϟ'), + ('ϡ', 'ϡ'), + ('ϣ', 'ϣ'), + ('ϥ', 'ϥ'), + ('ϧ', 'ϧ'), + ('ϩ', 'ϩ'), + ('ϫ', 'ϫ'), + ('ϭ', 'ϭ'), + ('ϯ', 'ϳ'), + ('ϵ', 'ϵ'), + ('ϸ', 'ϸ'), + ('ϻ', 'ϼ'), + ('а', 'џ'), + ('ѡ', 'ѡ'), + ('ѣ', 'ѣ'), + ('ѥ', 'ѥ'), + ('ѧ', 'ѧ'), + ('ѩ', 'ѩ'), + ('ѫ', 'ѫ'), + ('ѭ', 'ѭ'), + ('ѯ', 'ѯ'), + ('ѱ', 'ѱ'), + ('ѳ', 'ѳ'), + ('ѵ', 'ѵ'), + ('ѷ', 'ѷ'), + ('ѹ', 'ѹ'), + ('ѻ', 'ѻ'), + ('ѽ', 'ѽ'), + ('ѿ', 'ѿ'), + ('ҁ', 'ҁ'), + ('ҋ', 'ҋ'), + ('ҍ', 'ҍ'), + ('ҏ', 'ҏ'), + ('ґ', 'ґ'), + ('ғ', 'ғ'), + ('ҕ', 'ҕ'), + ('җ', 'җ'), + ('ҙ', 'ҙ'), + ('қ', 'қ'), + ('ҝ', 'ҝ'), + ('ҟ', 'ҟ'), + ('ҡ', 'ҡ'), + ('ң', 'ң'), + ('ҥ', 'ҥ'), + ('ҧ', 'ҧ'), + ('ҩ', 'ҩ'), + ('ҫ', 'ҫ'), + ('ҭ', 'ҭ'), + ('ү', 'ү'), + ('ұ', 'ұ'), + ('ҳ', 'ҳ'), + ('ҵ', 'ҵ'), + ('ҷ', 'ҷ'), + ('ҹ', 'ҹ'), + ('һ', 'һ'), + ('ҽ', 'ҽ'), + ('ҿ', 'ҿ'), + ('ӂ', 'ӂ'), + ('ӄ', 'ӄ'), + ('ӆ', 'ӆ'), + ('ӈ', 'ӈ'), + ('ӊ', 'ӊ'), + ('ӌ', 'ӌ'), + ('ӎ', 'ӏ'), + ('ӑ', 'ӑ'), + ('ӓ', 'ӓ'), + ('ӕ', 'ӕ'), + ('ӗ', 'ӗ'), + ('ә', 'ә'), + ('ӛ', 'ӛ'), + ('ӝ', 'ӝ'), + ('ӟ', 'ӟ'), + ('ӡ', 'ӡ'), + ('ӣ', 'ӣ'), + ('ӥ', 'ӥ'), + ('ӧ', 'ӧ'), + ('ө', 'ө'), + ('ӫ', 'ӫ'), + ('ӭ', 'ӭ'), + ('ӯ', 'ӯ'), + ('ӱ', 'ӱ'), + ('ӳ', 'ӳ'), + ('ӵ', 'ӵ'), + ('ӷ', 'ӷ'), + ('ӹ', 'ӹ'), + ('ӻ', 'ӻ'), + ('ӽ', 'ӽ'), + ('ӿ', 'ӿ'), + ('ԁ', 'ԁ'), + ('ԃ', 'ԃ'), + ('ԅ', 'ԅ'), + ('ԇ', 'ԇ'), + ('ԉ', 'ԉ'), + ('ԋ', 'ԋ'), + ('ԍ', 'ԍ'), + ('ԏ', 'ԏ'), + ('ԑ', 'ԑ'), + ('ԓ', 'ԓ'), + ('ԕ', 'ԕ'), + ('ԗ', 'ԗ'), + ('ԙ', 'ԙ'), + ('ԛ', 'ԛ'), + ('ԝ', 'ԝ'), + ('ԟ', 'ԟ'), + ('ԡ', 'ԡ'), + ('ԣ', 'ԣ'), + ('ԥ', 'ԥ'), + ('ԧ', 'ԧ'), + ('ԩ', 'ԩ'), + ('ԫ', 'ԫ'), + ('ԭ', 'ԭ'), + ('ԯ', 'ԯ'), + ('ՠ', 'ֈ'), + ('ა', 'ჺ'), + ('ჽ', 'ჿ'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'ᲈ'), + ('ᲊ', 'ᲊ'), + ('ᴀ', 'ᴫ'), + ('ᵫ', 'ᵷ'), + ('ᵹ', 'ᶚ'), + ('ḁ', 'ḁ'), + ('ḃ', 'ḃ'), + ('ḅ', 'ḅ'), + ('ḇ', 'ḇ'), + ('ḉ', 'ḉ'), + ('ḋ', 'ḋ'), + ('ḍ', 'ḍ'), + ('ḏ', 'ḏ'), + ('ḑ', 'ḑ'), + ('ḓ', 'ḓ'), + ('ḕ', 'ḕ'), + ('ḗ', 'ḗ'), + ('ḙ', 'ḙ'), + ('ḛ', 'ḛ'), + ('ḝ', 'ḝ'), + ('ḟ', 'ḟ'), + ('ḡ', 'ḡ'), + ('ḣ', 'ḣ'), + ('ḥ', 'ḥ'), + ('ḧ', 'ḧ'), + ('ḩ', 'ḩ'), + ('ḫ', 'ḫ'), + ('ḭ', 'ḭ'), + ('ḯ', 'ḯ'), + ('ḱ', 'ḱ'), + ('ḳ', 'ḳ'), + ('ḵ', 'ḵ'), + ('ḷ', 'ḷ'), + ('ḹ', 'ḹ'), + ('ḻ', 'ḻ'), + ('ḽ', 'ḽ'), + ('ḿ', 'ḿ'), + ('ṁ', 'ṁ'), + ('ṃ', 'ṃ'), + ('ṅ', 'ṅ'), + ('ṇ', 'ṇ'), + ('ṉ', 'ṉ'), + ('ṋ', 'ṋ'), + ('ṍ', 'ṍ'), + ('ṏ', 'ṏ'), + ('ṑ', 'ṑ'), + ('ṓ', 'ṓ'), + ('ṕ', 'ṕ'), + ('ṗ', 'ṗ'), + ('ṙ', 'ṙ'), + ('ṛ', 'ṛ'), + ('ṝ', 'ṝ'), + ('ṟ', 'ṟ'), + ('ṡ', 'ṡ'), + ('ṣ', 'ṣ'), + ('ṥ', 'ṥ'), + ('ṧ', 'ṧ'), + ('ṩ', 'ṩ'), + ('ṫ', 'ṫ'), + ('ṭ', 'ṭ'), + ('ṯ', 'ṯ'), + ('ṱ', 'ṱ'), + ('ṳ', 'ṳ'), + ('ṵ', 'ṵ'), + ('ṷ', 'ṷ'), + ('ṹ', 'ṹ'), + ('ṻ', 'ṻ'), + ('ṽ', 'ṽ'), + ('ṿ', 'ṿ'), + ('ẁ', 'ẁ'), + ('ẃ', 'ẃ'), + ('ẅ', 'ẅ'), + ('ẇ', 'ẇ'), + ('ẉ', 'ẉ'), + ('ẋ', 'ẋ'), + ('ẍ', 'ẍ'), + ('ẏ', 'ẏ'), + ('ẑ', 'ẑ'), + ('ẓ', 'ẓ'), + ('ẕ', 'ẝ'), + ('ẟ', 'ẟ'), + ('ạ', 'ạ'), + ('ả', 'ả'), + ('ấ', 'ấ'), + ('ầ', 'ầ'), + ('ẩ', 'ẩ'), + ('ẫ', 'ẫ'), + ('ậ', 'ậ'), + ('ắ', 'ắ'), + ('ằ', 'ằ'), + ('ẳ', 'ẳ'), + ('ẵ', 'ẵ'), + ('ặ', 'ặ'), + ('ẹ', 'ẹ'), + ('ẻ', 'ẻ'), + ('ẽ', 'ẽ'), + ('ế', 'ế'), + ('ề', 'ề'), + ('ể', 'ể'), + ('ễ', 'ễ'), + ('ệ', 'ệ'), + ('ỉ', 'ỉ'), + ('ị', 'ị'), + ('ọ', 'ọ'), + ('ỏ', 'ỏ'), + ('ố', 'ố'), + ('ồ', 'ồ'), + ('ổ', 'ổ'), + ('ỗ', 'ỗ'), + ('ộ', 'ộ'), + ('ớ', 'ớ'), + ('ờ', 'ờ'), + ('ở', 'ở'), + ('ỡ', 'ỡ'), + ('ợ', 'ợ'), + ('ụ', 'ụ'), + ('ủ', 'ủ'), + ('ứ', 'ứ'), + ('ừ', 'ừ'), + ('ử', 'ử'), + ('ữ', 'ữ'), + ('ự', 'ự'), + ('ỳ', 'ỳ'), + ('ỵ', 'ỵ'), + ('ỷ', 'ỷ'), + ('ỹ', 'ỹ'), + ('ỻ', 'ỻ'), + ('ỽ', 'ỽ'), + ('ỿ', 'ἇ'), + ('ἐ', 'ἕ'), + ('ἠ', 'ἧ'), + ('ἰ', 'ἷ'), + ('ὀ', 'ὅ'), + ('ὐ', 'ὗ'), + ('ὠ', 'ὧ'), + ('ὰ', 'ώ'), + ('ᾀ', 'ᾇ'), + ('ᾐ', 'ᾗ'), + ('ᾠ', 'ᾧ'), + ('ᾰ', 'ᾴ'), + ('ᾶ', 'ᾷ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῇ'), + ('ῐ', 'ΐ'), + ('ῖ', 'ῗ'), + ('ῠ', 'ῧ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῷ'), + ('ℊ', 'ℊ'), + ('ℎ', 'ℏ'), + ('ℓ', 'ℓ'), + ('ℯ', 'ℯ'), + ('ℴ', 'ℴ'), + ('ℹ', 'ℹ'), + ('ℼ', 'ℽ'), + ('ⅆ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('ↄ', 'ↄ'), + ('ⰰ', 'ⱟ'), + ('ⱡ', 'ⱡ'), + ('ⱥ', 'ⱦ'), + ('ⱨ', 'ⱨ'), + ('ⱪ', 'ⱪ'), + ('ⱬ', 'ⱬ'), + ('ⱱ', 'ⱱ'), + ('ⱳ', 'ⱴ'), + ('ⱶ', 'ⱻ'), + ('ⲁ', 'ⲁ'), + ('ⲃ', 'ⲃ'), + ('ⲅ', 'ⲅ'), + ('ⲇ', 'ⲇ'), + ('ⲉ', 'ⲉ'), + ('ⲋ', 'ⲋ'), + ('ⲍ', 'ⲍ'), + ('ⲏ', 'ⲏ'), + ('ⲑ', 'ⲑ'), + ('ⲓ', 'ⲓ'), + ('ⲕ', 'ⲕ'), + ('ⲗ', 'ⲗ'), + ('ⲙ', 'ⲙ'), + ('ⲛ', 'ⲛ'), + ('ⲝ', 'ⲝ'), + ('ⲟ', 'ⲟ'), + ('ⲡ', 'ⲡ'), + ('ⲣ', 'ⲣ'), + ('ⲥ', 'ⲥ'), + ('ⲧ', 'ⲧ'), + ('ⲩ', 'ⲩ'), + ('ⲫ', 'ⲫ'), + ('ⲭ', 'ⲭ'), + ('ⲯ', 'ⲯ'), + ('ⲱ', 'ⲱ'), + ('ⲳ', 'ⲳ'), + ('ⲵ', 'ⲵ'), + ('ⲷ', 'ⲷ'), + ('ⲹ', 'ⲹ'), + ('ⲻ', 'ⲻ'), + ('ⲽ', 'ⲽ'), + ('ⲿ', 'ⲿ'), + ('ⳁ', 'ⳁ'), + ('ⳃ', 'ⳃ'), + ('ⳅ', 'ⳅ'), + ('ⳇ', 'ⳇ'), + ('ⳉ', 'ⳉ'), + ('ⳋ', 'ⳋ'), + ('ⳍ', 'ⳍ'), + ('ⳏ', 'ⳏ'), + ('ⳑ', 'ⳑ'), + ('ⳓ', 'ⳓ'), + ('ⳕ', 'ⳕ'), + ('ⳗ', 'ⳗ'), + ('ⳙ', 'ⳙ'), + ('ⳛ', 'ⳛ'), + ('ⳝ', 'ⳝ'), + ('ⳟ', 'ⳟ'), + ('ⳡ', 'ⳡ'), + ('ⳣ', 'ⳤ'), + ('ⳬ', 'ⳬ'), + ('ⳮ', 'ⳮ'), + ('ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ꙁ', 'ꙁ'), + ('ꙃ', 'ꙃ'), + ('ꙅ', 'ꙅ'), + ('ꙇ', 'ꙇ'), + ('ꙉ', 'ꙉ'), + ('ꙋ', 'ꙋ'), + ('ꙍ', 'ꙍ'), + ('ꙏ', 'ꙏ'), + ('ꙑ', 'ꙑ'), + ('ꙓ', 'ꙓ'), + ('ꙕ', 'ꙕ'), + ('ꙗ', 'ꙗ'), + ('ꙙ', 'ꙙ'), + ('ꙛ', 'ꙛ'), + ('ꙝ', 'ꙝ'), + ('ꙟ', 'ꙟ'), + ('ꙡ', 'ꙡ'), + ('ꙣ', 'ꙣ'), + ('ꙥ', 'ꙥ'), + ('ꙧ', 'ꙧ'), + ('ꙩ', 'ꙩ'), + ('ꙫ', 'ꙫ'), + ('ꙭ', 'ꙭ'), + ('ꚁ', 'ꚁ'), + ('ꚃ', 'ꚃ'), + ('ꚅ', 'ꚅ'), + ('ꚇ', 'ꚇ'), + ('ꚉ', 'ꚉ'), + ('ꚋ', 'ꚋ'), + ('ꚍ', 'ꚍ'), + ('ꚏ', 'ꚏ'), + ('ꚑ', 'ꚑ'), + ('ꚓ', 'ꚓ'), + ('ꚕ', 'ꚕ'), + ('ꚗ', 'ꚗ'), + ('ꚙ', 'ꚙ'), + ('ꚛ', 'ꚛ'), + ('ꜣ', 'ꜣ'), + ('ꜥ', 'ꜥ'), + ('ꜧ', 'ꜧ'), + ('ꜩ', 'ꜩ'), + ('ꜫ', 'ꜫ'), + ('ꜭ', 'ꜭ'), + ('ꜯ', 'ꜱ'), + ('ꜳ', 'ꜳ'), + ('ꜵ', 'ꜵ'), + ('ꜷ', 'ꜷ'), + ('ꜹ', 'ꜹ'), + ('ꜻ', 'ꜻ'), + ('ꜽ', 'ꜽ'), + ('ꜿ', 'ꜿ'), + ('ꝁ', 'ꝁ'), + ('ꝃ', 'ꝃ'), + ('ꝅ', 'ꝅ'), + ('ꝇ', 'ꝇ'), + ('ꝉ', 'ꝉ'), + ('ꝋ', 'ꝋ'), + ('ꝍ', 'ꝍ'), + ('ꝏ', 'ꝏ'), + ('ꝑ', 'ꝑ'), + ('ꝓ', 'ꝓ'), + ('ꝕ', 'ꝕ'), + ('ꝗ', 'ꝗ'), + ('ꝙ', 'ꝙ'), + ('ꝛ', 'ꝛ'), + ('ꝝ', 'ꝝ'), + ('ꝟ', 'ꝟ'), + ('ꝡ', 'ꝡ'), + ('ꝣ', 'ꝣ'), + ('ꝥ', 'ꝥ'), + ('ꝧ', 'ꝧ'), + ('ꝩ', 'ꝩ'), + ('ꝫ', 'ꝫ'), + ('ꝭ', 'ꝭ'), + ('ꝯ', 'ꝯ'), + ('ꝱ', 'ꝸ'), + ('ꝺ', 'ꝺ'), + ('ꝼ', 'ꝼ'), + ('ꝿ', 'ꝿ'), + ('ꞁ', 'ꞁ'), + ('ꞃ', 'ꞃ'), + ('ꞅ', 'ꞅ'), + ('ꞇ', 'ꞇ'), + ('ꞌ', 'ꞌ'), + ('ꞎ', 'ꞎ'), + ('ꞑ', 'ꞑ'), + ('ꞓ', 'ꞕ'), + ('ꞗ', 'ꞗ'), + ('ꞙ', 'ꞙ'), + ('ꞛ', 'ꞛ'), + ('ꞝ', 'ꞝ'), + ('ꞟ', 'ꞟ'), + ('ꞡ', 'ꞡ'), + ('ꞣ', 'ꞣ'), + ('ꞥ', 'ꞥ'), + ('ꞧ', 'ꞧ'), + ('ꞩ', 'ꞩ'), + ('ꞯ', 'ꞯ'), + ('ꞵ', 'ꞵ'), + ('ꞷ', 'ꞷ'), + ('ꞹ', 'ꞹ'), + ('ꞻ', 'ꞻ'), + ('ꞽ', 'ꞽ'), + ('ꞿ', 'ꞿ'), + ('ꟁ', 'ꟁ'), + ('ꟃ', 'ꟃ'), + ('ꟈ', 'ꟈ'), + ('ꟊ', 'ꟊ'), + ('ꟍ', 'ꟍ'), + ('ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'ꟕ'), + ('ꟗ', 'ꟗ'), + ('ꟙ', 'ꟙ'), + ('ꟛ', 'ꟛ'), + ('ꟶ', 'ꟶ'), + ('ꟺ', 'ꟺ'), + ('ꬰ', 'ꭚ'), + ('ꭠ', 'ꭨ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('ａ', 'ｚ'), + ('𐐨', '𐑏'), + ('𐓘', '𐓻'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐳀', '𐳲'), + ('𐵰', '𐶅'), + ('𑣀', '𑣟'), + ('𖹠', '𖹿'), + ('𝐚', '𝐳'), + ('𝑎', '𝑔'), + ('𝑖', '𝑧'), + ('𝒂', '𝒛'), + ('𝒶', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝓏'), + ('𝓪', '𝔃'), + ('𝔞', '𝔷'), + ('𝕒', '𝕫'), + ('𝖆', '𝖟'), + ('𝖺', '𝗓'), + ('𝗮', '𝘇'), + ('𝘢', '𝘻'), + ('𝙖', '𝙯'), + ('𝚊', '𝚥'), + ('𝛂', '𝛚'), + ('𝛜', '𝛡'), + ('𝛼', '𝜔'), + ('𝜖', '𝜛'), + ('𝜶', '𝝎'), + ('𝝐', '𝝕'), + ('𝝰', '𝞈'), + ('𝞊', '𝞏'), + ('𝞪', '𝟂'), + ('𝟄', '𝟉'), + ('𝟋', '𝟋'), + ('𝼀', '𝼉'), + ('𝼋', '𝼞'), + ('𝼥', '𝼪'), + ('𞤢', '𞥃'), +]; + +pub const MARK: &'static [(char, char)] = &[ + ('\u{300}', '\u{36f}'), + ('\u{483}', '\u{489}'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('\u{610}', '\u{61a}'), + ('\u{64b}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dc}'), + ('\u{6df}', '\u{6e4}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{6ea}', '\u{6ed}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', '\u{7f3}'), + ('\u{7fd}', '\u{7fd}'), + ('\u{816}', '\u{819}'), + ('\u{81b}', '\u{823}'), + ('\u{825}', '\u{827}'), + ('\u{829}', '\u{82d}'), + ('\u{859}', '\u{85b}'), + ('\u{897}', '\u{89f}'), + ('\u{8ca}', '\u{8e1}'), + ('\u{8e3}', 'ः'), + ('\u{93a}', '\u{93c}'), + ('ा', 'ॏ'), + ('\u{951}', '\u{957}'), + ('\u{962}', '\u{963}'), + ('\u{981}', 'ঃ'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9be}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', '\u{9cd}'), + ('\u{9d7}', '\u{9d7}'), + ('\u{9e2}', '\u{9e3}'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', 'ਃ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('\u{abc}', '\u{abc}'), + ('ા', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('\u{ae2}', '\u{ae3}'), + ('\u{afa}', '\u{aff}'), + ('\u{b01}', 'ଃ'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b3e}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('\u{b62}', '\u{b63}'), + ('\u{b82}', '\u{b82}'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('\u{bd7}', '\u{bd7}'), + ('\u{c00}', '\u{c04}'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c3e}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', 'ಃ'), + ('\u{cbc}', '\u{cbc}'), + ('ಾ', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('\u{ce2}', '\u{ce3}'), + ('ೳ', 'ೳ'), + ('\u{d00}', 'ഃ'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d3e}', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', '\u{d4d}'), + ('\u{d57}', '\u{d57}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', 'ඃ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('ෲ', 'ෳ'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('\u{e47}', '\u{e4e}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{ebc}'), + ('\u{ec8}', '\u{ece}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('༾', '༿'), + ('\u{f71}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('ါ', '\u{103e}'), + ('ၖ', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('ၢ', 'ၤ'), + ('ၧ', 'ၭ'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{108d}'), + ('ႏ', 'ႏ'), + ('ႚ', '\u{109d}'), + ('\u{135d}', '\u{135f}'), + ('\u{1712}', '\u{1715}'), + ('\u{1732}', '\u{1734}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('\u{17b4}', '\u{17d3}'), + ('\u{17dd}', '\u{17dd}'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '\u{180f}'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('\u{1a17}', '\u{1a1b}'), + ('ᩕ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', 'ᬄ'), + ('\u{1b34}', '\u{1b44}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', 'ᮂ'), + ('ᮡ', '\u{1bad}'), + ('\u{1be6}', '\u{1bf3}'), + ('ᰤ', '\u{1c37}'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('᳷', '\u{1cf9}'), + ('\u{1dc0}', '\u{1dff}'), + ('\u{20d0}', '\u{20f0}'), + ('\u{2cef}', '\u{2cf1}'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{2de0}', '\u{2dff}'), + ('\u{302a}', '\u{302f}'), + ('\u{3099}', '\u{309a}'), + ('\u{a66f}', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('\u{a69e}', '\u{a69f}'), + ('\u{a6f0}', '\u{a6f1}'), + ('\u{a802}', '\u{a802}'), + ('\u{a806}', '\u{a806}'), + ('\u{a80b}', '\u{a80b}'), + ('ꠣ', 'ꠧ'), + ('\u{a82c}', '\u{a82c}'), + ('ꢀ', 'ꢁ'), + ('ꢴ', '\u{a8c5}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92d}'), + ('\u{a947}', '\u{a953}'), + ('\u{a980}', 'ꦃ'), + ('\u{a9b3}', '\u{a9c0}'), + ('\u{a9e5}', '\u{a9e5}'), + ('\u{aa29}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', 'ꩍ'), + ('ꩻ', 'ꩽ'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabf}'), + ('\u{aac1}', '\u{aac1}'), + ('ꫫ', 'ꫯ'), + ('ꫵ', '\u{aaf6}'), + ('ꯣ', 'ꯪ'), + ('꯬', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{10376}', '\u{1037a}'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('\u{10d24}', '\u{10d27}'), + ('\u{10d69}', '\u{10d6d}'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('𑀀', '𑀂'), + ('\u{11038}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{11073}', '\u{11074}'), + ('\u{1107f}', '𑂂'), + ('𑂰', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{11134}'), + ('𑅅', '𑅆'), + ('\u{11173}', '\u{11173}'), + ('\u{11180}', '𑆂'), + ('𑆳', '\u{111c0}'), + ('\u{111c9}', '\u{111cc}'), + ('𑇎', '\u{111cf}'), + ('𑈬', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112ea}'), + ('\u{11300}', '𑌃'), + ('\u{1133b}', '\u{1133c}'), + ('\u{1133e}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('𑍢', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113b8}', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '\u{113d0}'), + ('\u{113d2}', '\u{113d2}'), + ('\u{113e1}', '\u{113e2}'), + ('𑐵', '\u{11446}'), + ('\u{1145e}', '\u{1145e}'), + ('\u{114b0}', '\u{114c3}'), + ('\u{115af}', '\u{115b5}'), + ('𑖸', '\u{115c0}'), + ('\u{115dc}', '\u{115dd}'), + ('𑘰', '\u{11640}'), + ('\u{116ab}', '\u{116b7}'), + ('\u{1171d}', '\u{1172b}'), + ('𑠬', '\u{1183a}'), + ('\u{11930}', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{1193e}'), + ('𑥀', '𑥀'), + ('𑥂', '\u{11943}'), + ('𑧑', '\u{119d7}'), + ('\u{119da}', '\u{119e0}'), + ('𑧤', '𑧤'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a33}', '𑨹'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a51}', '\u{11a5b}'), + ('\u{11a8a}', '\u{11a99}'), + ('𑰯', '\u{11c36}'), + ('\u{11c38}', '\u{11c3f}'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d45}'), + ('\u{11d47}', '\u{11d47}'), + ('𑶊', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '\u{11d97}'), + ('\u{11ef3}', '𑻶'), + ('\u{11f00}', '\u{11f01}'), + ('𑼃', '𑼃'), + ('𑼴', '\u{11f3a}'), + ('𑼾', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13440}', '\u{13440}'), + ('\u{13447}', '\u{13455}'), + ('\u{1611e}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('\u{16f4f}', '\u{16f4f}'), + ('𖽑', '𖾇'), + ('\u{16f8f}', '\u{16f92}'), + ('\u{16fe4}', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e130}', '\u{1e136}'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('\u{1e4ec}', '\u{1e4ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '\u{1e94a}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const MATH_SYMBOL: &'static [(char, char)] = &[ + ('+', '+'), + ('<', '>'), + ('|', '|'), + ('~', '~'), + ('¬', '¬'), + ('±', '±'), + ('×', '×'), + ('÷', '÷'), + ('϶', '϶'), + ('؆', '؈'), + ('⁄', '⁄'), + ('⁒', '⁒'), + ('⁺', '⁼'), + ('₊', '₌'), + ('℘', '℘'), + ('⅀', '⅄'), + ('⅋', '⅋'), + ('←', '↔'), + ('↚', '↛'), + ('↠', '↠'), + ('↣', '↣'), + ('↦', '↦'), + ('↮', '↮'), + ('⇎', '⇏'), + ('⇒', '⇒'), + ('⇔', '⇔'), + ('⇴', '⋿'), + ('⌠', '⌡'), + ('⍼', '⍼'), + ('⎛', '⎳'), + ('⏜', '⏡'), + ('▷', '▷'), + ('◁', '◁'), + ('◸', '◿'), + ('♯', '♯'), + ('⟀', '⟄'), + ('⟇', '⟥'), + ('⟰', '⟿'), + ('⤀', '⦂'), + ('⦙', '⧗'), + ('⧜', '⧻'), + ('⧾', '⫿'), + ('⬰', '⭄'), + ('⭇', '⭌'), + ('﬩', '﬩'), + ('﹢', '﹢'), + ('﹤', '﹦'), + ('＋', '＋'), + ('＜', '＞'), + ('｜', '｜'), + ('～', '～'), + ('￢', '￢'), + ('￩', '￬'), + ('𐶎', '𐶏'), + ('𝛁', '𝛁'), + ('𝛛', '𝛛'), + ('𝛻', '𝛻'), + ('𝜕', '𝜕'), + ('𝜵', '𝜵'), + ('𝝏', '𝝏'), + ('𝝯', '𝝯'), + ('𝞉', '𝞉'), + ('𝞩', '𝞩'), + ('𝟃', '𝟃'), + ('𞻰', '𞻱'), +]; + +pub const MODIFIER_LETTER: &'static [(char, char)] = &[ + ('ʰ', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('ʹ', 'ʹ'), + ('ͺ', 'ͺ'), + ('ՙ', 'ՙ'), + ('ـ', 'ـ'), + ('ۥ', 'ۦ'), + ('ߴ', 'ߵ'), + ('ߺ', 'ߺ'), + ('ࠚ', 'ࠚ'), + ('ࠤ', 'ࠤ'), + ('ࠨ', 'ࠨ'), + ('ࣉ', 'ࣉ'), + ('ॱ', 'ॱ'), + ('ๆ', 'ๆ'), + ('ໆ', 'ໆ'), + ('ჼ', 'ჼ'), + ('ៗ', 'ៗ'), + ('ᡃ', 'ᡃ'), + ('ᪧ', 'ᪧ'), + ('ᱸ', 'ᱽ'), + ('ᴬ', 'ᵪ'), + ('ᵸ', 'ᵸ'), + ('ᶛ', 'ᶿ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ⱼ', 'ⱽ'), + ('ⵯ', 'ⵯ'), + ('ⸯ', 'ⸯ'), + ('々', '々'), + ('〱', '〵'), + ('〻', '〻'), + ('ゝ', 'ゞ'), + ('ー', 'ヾ'), + ('ꀕ', 'ꀕ'), + ('ꓸ', 'ꓽ'), + ('ꘌ', 'ꘌ'), + ('ꙿ', 'ꙿ'), + ('ꚜ', 'ꚝ'), + ('ꜗ', 'ꜟ'), + ('ꝰ', 'ꝰ'), + ('ꞈ', 'ꞈ'), + ('ꟲ', 'ꟴ'), + ('ꟸ', 'ꟹ'), + ('ꧏ', 'ꧏ'), + ('ꧦ', 'ꧦ'), + ('ꩰ', 'ꩰ'), + ('ꫝ', 'ꫝ'), + ('ꫳ', 'ꫴ'), + ('ꭜ', 'ꭟ'), + ('ꭩ', 'ꭩ'), + ('ｰ', 'ｰ'), + ('\u{ff9e}', '\u{ff9f}'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐵎', '𐵎'), + ('𐵯', '𐵯'), + ('𖭀', '𖭃'), + ('𖵀', '𖵂'), + ('𖵫', '𖵬'), + ('𖾓', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '𖿣'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𞀰', '𞁭'), + ('𞄷', '𞄽'), + ('𞓫', '𞓫'), + ('𞥋', '𞥋'), +]; + +pub const MODIFIER_SYMBOL: &'static [(char, char)] = &[ + ('^', '^'), + ('`', '`'), + ('¨', '¨'), + ('¯', '¯'), + ('´', '´'), + ('¸', '¸'), + ('˂', '˅'), + ('˒', '˟'), + ('˥', '˫'), + ('˭', '˭'), + ('˯', '˿'), + ('͵', '͵'), + ('΄', '΅'), + ('࢈', '࢈'), + ('᾽', '᾽'), + ('᾿', '῁'), + ('῍', '῏'), + ('῝', '῟'), + ('῭', '`'), + ('´', '῾'), + ('゛', '゜'), + ('꜀', '꜖'), + ('꜠', '꜡'), + ('꞉', '꞊'), + ('꭛', '꭛'), + ('꭪', '꭫'), + ('﮲', '﯂'), + ('＾', '＾'), + ('｀', '｀'), + ('￣', '￣'), + ('🏻', '🏿'), +]; + +pub const NONSPACING_MARK: &'static [(char, char)] = &[ + ('\u{300}', '\u{36f}'), + ('\u{483}', '\u{487}'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('\u{610}', '\u{61a}'), + ('\u{64b}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dc}'), + ('\u{6df}', '\u{6e4}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{6ea}', '\u{6ed}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', '\u{7f3}'), + ('\u{7fd}', '\u{7fd}'), + ('\u{816}', '\u{819}'), + ('\u{81b}', '\u{823}'), + ('\u{825}', '\u{827}'), + ('\u{829}', '\u{82d}'), + ('\u{859}', '\u{85b}'), + ('\u{897}', '\u{89f}'), + ('\u{8ca}', '\u{8e1}'), + ('\u{8e3}', '\u{902}'), + ('\u{93a}', '\u{93a}'), + ('\u{93c}', '\u{93c}'), + ('\u{941}', '\u{948}'), + ('\u{94d}', '\u{94d}'), + ('\u{951}', '\u{957}'), + ('\u{962}', '\u{963}'), + ('\u{981}', '\u{981}'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9c1}', '\u{9c4}'), + ('\u{9cd}', '\u{9cd}'), + ('\u{9e2}', '\u{9e3}'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', '\u{a02}'), + ('\u{a3c}', '\u{a3c}'), + ('\u{a41}', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', '\u{a82}'), + ('\u{abc}', '\u{abc}'), + ('\u{ac1}', '\u{ac5}'), + ('\u{ac7}', '\u{ac8}'), + ('\u{acd}', '\u{acd}'), + ('\u{ae2}', '\u{ae3}'), + ('\u{afa}', '\u{aff}'), + ('\u{b01}', '\u{b01}'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b3f}', '\u{b3f}'), + ('\u{b41}', '\u{b44}'), + ('\u{b4d}', '\u{b4d}'), + ('\u{b55}', '\u{b56}'), + ('\u{b62}', '\u{b63}'), + ('\u{b82}', '\u{b82}'), + ('\u{bc0}', '\u{bc0}'), + ('\u{bcd}', '\u{bcd}'), + ('\u{c00}', '\u{c00}'), + ('\u{c04}', '\u{c04}'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c3e}', '\u{c40}'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', '\u{c81}'), + ('\u{cbc}', '\u{cbc}'), + ('\u{cbf}', '\u{cbf}'), + ('\u{cc6}', '\u{cc6}'), + ('\u{ccc}', '\u{ccd}'), + ('\u{ce2}', '\u{ce3}'), + ('\u{d00}', '\u{d01}'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d41}', '\u{d44}'), + ('\u{d4d}', '\u{d4d}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', '\u{d81}'), + ('\u{dca}', '\u{dca}'), + ('\u{dd2}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('\u{e47}', '\u{e4e}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{ebc}'), + ('\u{ec8}', '\u{ece}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('\u{f71}', '\u{f7e}'), + ('\u{f80}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('\u{102d}', '\u{1030}'), + ('\u{1032}', '\u{1037}'), + ('\u{1039}', '\u{103a}'), + ('\u{103d}', '\u{103e}'), + ('\u{1058}', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{1082}'), + ('\u{1085}', '\u{1086}'), + ('\u{108d}', '\u{108d}'), + ('\u{109d}', '\u{109d}'), + ('\u{135d}', '\u{135f}'), + ('\u{1712}', '\u{1714}'), + ('\u{1732}', '\u{1733}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('\u{17b4}', '\u{17b5}'), + ('\u{17b7}', '\u{17bd}'), + ('\u{17c6}', '\u{17c6}'), + ('\u{17c9}', '\u{17d3}'), + ('\u{17dd}', '\u{17dd}'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '\u{180f}'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', '\u{1922}'), + ('\u{1927}', '\u{1928}'), + ('\u{1932}', '\u{1932}'), + ('\u{1939}', '\u{193b}'), + ('\u{1a17}', '\u{1a18}'), + ('\u{1a1b}', '\u{1a1b}'), + ('\u{1a56}', '\u{1a56}'), + ('\u{1a58}', '\u{1a5e}'), + ('\u{1a60}', '\u{1a60}'), + ('\u{1a62}', '\u{1a62}'), + ('\u{1a65}', '\u{1a6c}'), + ('\u{1a73}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('\u{1ab0}', '\u{1abd}'), + ('\u{1abf}', '\u{1ace}'), + ('\u{1b00}', '\u{1b03}'), + ('\u{1b34}', '\u{1b34}'), + ('\u{1b36}', '\u{1b3a}'), + ('\u{1b3c}', '\u{1b3c}'), + ('\u{1b42}', '\u{1b42}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1b81}'), + ('\u{1ba2}', '\u{1ba5}'), + ('\u{1ba8}', '\u{1ba9}'), + ('\u{1bab}', '\u{1bad}'), + ('\u{1be6}', '\u{1be6}'), + ('\u{1be8}', '\u{1be9}'), + ('\u{1bed}', '\u{1bed}'), + ('\u{1bef}', '\u{1bf1}'), + ('\u{1c2c}', '\u{1c33}'), + ('\u{1c36}', '\u{1c37}'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce0}'), + ('\u{1ce2}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('\u{1cf8}', '\u{1cf9}'), + ('\u{1dc0}', '\u{1dff}'), + ('\u{20d0}', '\u{20dc}'), + ('\u{20e1}', '\u{20e1}'), + ('\u{20e5}', '\u{20f0}'), + ('\u{2cef}', '\u{2cf1}'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{2de0}', '\u{2dff}'), + ('\u{302a}', '\u{302d}'), + ('\u{3099}', '\u{309a}'), + ('\u{a66f}', '\u{a66f}'), + ('\u{a674}', '\u{a67d}'), + ('\u{a69e}', '\u{a69f}'), + ('\u{a6f0}', '\u{a6f1}'), + ('\u{a802}', '\u{a802}'), + ('\u{a806}', '\u{a806}'), + ('\u{a80b}', '\u{a80b}'), + ('\u{a825}', '\u{a826}'), + ('\u{a82c}', '\u{a82c}'), + ('\u{a8c4}', '\u{a8c5}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92d}'), + ('\u{a947}', '\u{a951}'), + ('\u{a980}', '\u{a982}'), + ('\u{a9b3}', '\u{a9b3}'), + ('\u{a9b6}', '\u{a9b9}'), + ('\u{a9bc}', '\u{a9bd}'), + ('\u{a9e5}', '\u{a9e5}'), + ('\u{aa29}', '\u{aa2e}'), + ('\u{aa31}', '\u{aa32}'), + ('\u{aa35}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', '\u{aa4c}'), + ('\u{aa7c}', '\u{aa7c}'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabf}'), + ('\u{aac1}', '\u{aac1}'), + ('\u{aaec}', '\u{aaed}'), + ('\u{aaf6}', '\u{aaf6}'), + ('\u{abe5}', '\u{abe5}'), + ('\u{abe8}', '\u{abe8}'), + ('\u{abed}', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{10376}', '\u{1037a}'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('\u{10d24}', '\u{10d27}'), + ('\u{10d69}', '\u{10d6d}'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('\u{11001}', '\u{11001}'), + ('\u{11038}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{11073}', '\u{11074}'), + ('\u{1107f}', '\u{11081}'), + ('\u{110b3}', '\u{110b6}'), + ('\u{110b9}', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{1112b}'), + ('\u{1112d}', '\u{11134}'), + ('\u{11173}', '\u{11173}'), + ('\u{11180}', '\u{11181}'), + ('\u{111b6}', '\u{111be}'), + ('\u{111c9}', '\u{111cc}'), + ('\u{111cf}', '\u{111cf}'), + ('\u{1122f}', '\u{11231}'), + ('\u{11234}', '\u{11234}'), + ('\u{11236}', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112df}'), + ('\u{112e3}', '\u{112ea}'), + ('\u{11300}', '\u{11301}'), + ('\u{1133b}', '\u{1133c}'), + ('\u{11340}', '\u{11340}'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113bb}', '\u{113c0}'), + ('\u{113ce}', '\u{113ce}'), + ('\u{113d0}', '\u{113d0}'), + ('\u{113d2}', '\u{113d2}'), + ('\u{113e1}', '\u{113e2}'), + ('\u{11438}', '\u{1143f}'), + ('\u{11442}', '\u{11444}'), + ('\u{11446}', '\u{11446}'), + ('\u{1145e}', '\u{1145e}'), + ('\u{114b3}', '\u{114b8}'), + ('\u{114ba}', '\u{114ba}'), + ('\u{114bf}', '\u{114c0}'), + ('\u{114c2}', '\u{114c3}'), + ('\u{115b2}', '\u{115b5}'), + ('\u{115bc}', '\u{115bd}'), + ('\u{115bf}', '\u{115c0}'), + ('\u{115dc}', '\u{115dd}'), + ('\u{11633}', '\u{1163a}'), + ('\u{1163d}', '\u{1163d}'), + ('\u{1163f}', '\u{11640}'), + ('\u{116ab}', '\u{116ab}'), + ('\u{116ad}', '\u{116ad}'), + ('\u{116b0}', '\u{116b5}'), + ('\u{116b7}', '\u{116b7}'), + ('\u{1171d}', '\u{1171d}'), + ('\u{1171f}', '\u{1171f}'), + ('\u{11722}', '\u{11725}'), + ('\u{11727}', '\u{1172b}'), + ('\u{1182f}', '\u{11837}'), + ('\u{11839}', '\u{1183a}'), + ('\u{1193b}', '\u{1193c}'), + ('\u{1193e}', '\u{1193e}'), + ('\u{11943}', '\u{11943}'), + ('\u{119d4}', '\u{119d7}'), + ('\u{119da}', '\u{119db}'), + ('\u{119e0}', '\u{119e0}'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a33}', '\u{11a38}'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a51}', '\u{11a56}'), + ('\u{11a59}', '\u{11a5b}'), + ('\u{11a8a}', '\u{11a96}'), + ('\u{11a98}', '\u{11a99}'), + ('\u{11c30}', '\u{11c36}'), + ('\u{11c38}', '\u{11c3d}'), + ('\u{11c3f}', '\u{11c3f}'), + ('\u{11c92}', '\u{11ca7}'), + ('\u{11caa}', '\u{11cb0}'), + ('\u{11cb2}', '\u{11cb3}'), + ('\u{11cb5}', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d45}'), + ('\u{11d47}', '\u{11d47}'), + ('\u{11d90}', '\u{11d91}'), + ('\u{11d95}', '\u{11d95}'), + ('\u{11d97}', '\u{11d97}'), + ('\u{11ef3}', '\u{11ef4}'), + ('\u{11f00}', '\u{11f01}'), + ('\u{11f36}', '\u{11f3a}'), + ('\u{11f40}', '\u{11f40}'), + ('\u{11f42}', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13440}', '\u{13440}'), + ('\u{13447}', '\u{13455}'), + ('\u{1611e}', '\u{16129}'), + ('\u{1612d}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('\u{16f4f}', '\u{16f4f}'), + ('\u{16f8f}', '\u{16f92}'), + ('\u{16fe4}', '\u{16fe4}'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d167}', '\u{1d169}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e130}', '\u{1e136}'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('\u{1e4ec}', '\u{1e4ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '\u{1e94a}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const NUMBER: &'static [(char, char)] = &[ + ('0', '9'), + ('²', '³'), + ('¹', '¹'), + ('¼', '¾'), + ('٠', '٩'), + ('۰', '۹'), + ('߀', '߉'), + ('०', '९'), + ('০', '৯'), + ('৴', '৹'), + ('੦', '੯'), + ('૦', '૯'), + ('୦', '୯'), + ('୲', '୷'), + ('௦', '௲'), + ('౦', '౯'), + ('౸', '౾'), + ('೦', '೯'), + ('൘', '൞'), + ('൦', '൸'), + ('෦', '෯'), + ('๐', '๙'), + ('໐', '໙'), + ('༠', '༳'), + ('၀', '၉'), + ('႐', '႙'), + ('፩', '፼'), + ('ᛮ', 'ᛰ'), + ('០', '៩'), + ('៰', '៹'), + ('᠐', '᠙'), + ('᥆', '᥏'), + ('᧐', '᧚'), + ('᪀', '᪉'), + ('᪐', '᪙'), + ('᭐', '᭙'), + ('᮰', '᮹'), + ('᱀', '᱉'), + ('᱐', '᱙'), + ('⁰', '⁰'), + ('⁴', '⁹'), + ('₀', '₉'), + ('⅐', 'ↂ'), + ('ↅ', '↉'), + ('①', '⒛'), + ('⓪', '⓿'), + ('❶', '➓'), + ('⳽', '⳽'), + ('〇', '〇'), + ('〡', '〩'), + ('〸', '〺'), + ('㆒', '㆕'), + ('㈠', '㈩'), + ('㉈', '㉏'), + ('㉑', '㉟'), + ('㊀', '㊉'), + ('㊱', '㊿'), + ('꘠', '꘩'), + ('ꛦ', 'ꛯ'), + ('꠰', '꠵'), + ('꣐', '꣙'), + ('꤀', '꤉'), + ('꧐', '꧙'), + ('꧰', '꧹'), + ('꩐', '꩙'), + ('꯰', '꯹'), + ('０', '９'), + ('𐄇', '𐄳'), + ('𐅀', '𐅸'), + ('𐆊', '𐆋'), + ('𐋡', '𐋻'), + ('𐌠', '𐌣'), + ('𐍁', '𐍁'), + ('𐍊', '𐍊'), + ('𐏑', '𐏕'), + ('𐒠', '𐒩'), + ('𐡘', '𐡟'), + ('𐡹', '𐡿'), + ('𐢧', '𐢯'), + ('𐣻', '𐣿'), + ('𐤖', '𐤛'), + ('𐦼', '𐦽'), + ('𐧀', '𐧏'), + ('𐧒', '𐧿'), + ('𐩀', '𐩈'), + ('𐩽', '𐩾'), + ('𐪝', '𐪟'), + ('𐫫', '𐫯'), + ('𐭘', '𐭟'), + ('𐭸', '𐭿'), + ('𐮩', '𐮯'), + ('𐳺', '𐳿'), + ('𐴰', '𐴹'), + ('𐵀', '𐵉'), + ('𐹠', '𐹾'), + ('𐼝', '𐼦'), + ('𐽑', '𐽔'), + ('𐿅', '𐿋'), + ('𑁒', '𑁯'), + ('𑃰', '𑃹'), + ('𑄶', '𑄿'), + ('𑇐', '𑇙'), + ('𑇡', '𑇴'), + ('𑋰', '𑋹'), + ('𑑐', '𑑙'), + ('𑓐', '𑓙'), + ('𑙐', '𑙙'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜰', '𑜻'), + ('𑣠', '𑣲'), + ('𑥐', '𑥙'), + ('𑯰', '𑯹'), + ('𑱐', '𑱬'), + ('𑵐', '𑵙'), + ('𑶠', '𑶩'), + ('𑽐', '𑽙'), + ('𑿀', '𑿔'), + ('𒐀', '𒑮'), + ('𖄰', '𖄹'), + ('𖩠', '𖩩'), + ('𖫀', '𖫉'), + ('𖭐', '𖭙'), + ('𖭛', '𖭡'), + ('𖵰', '𖵹'), + ('𖺀', '𖺖'), + ('𜳰', '𜳹'), + ('𝋀', '𝋓'), + ('𝋠', '𝋳'), + ('𝍠', '𝍸'), + ('𝟎', '𝟿'), + ('𞅀', '𞅉'), + ('𞋰', '𞋹'), + ('𞓰', '𞓹'), + ('𞗱', '𞗺'), + ('𞣇', '𞣏'), + ('𞥐', '𞥙'), + ('𞱱', '𞲫'), + ('𞲭', '𞲯'), + ('𞲱', '𞲴'), + ('𞴁', '𞴭'), + ('𞴯', '𞴽'), + ('🄀', '🄌'), + ('🯰', '🯹'), +]; + +pub const OPEN_PUNCTUATION: &'static [(char, char)] = &[ + ('(', '('), + ('[', '['), + ('{', '{'), + ('༺', '༺'), + ('༼', '༼'), + ('᚛', '᚛'), + ('‚', '‚'), + ('„', '„'), + ('⁅', '⁅'), + ('⁽', '⁽'), + ('₍', '₍'), + ('⌈', '⌈'), + ('⌊', '⌊'), + ('〈', '〈'), + ('❨', '❨'), + ('❪', '❪'), + ('❬', '❬'), + ('❮', '❮'), + ('❰', '❰'), + ('❲', '❲'), + ('❴', '❴'), + ('⟅', '⟅'), + ('⟦', '⟦'), + ('⟨', '⟨'), + ('⟪', '⟪'), + ('⟬', '⟬'), + ('⟮', '⟮'), + ('⦃', '⦃'), + ('⦅', '⦅'), + ('⦇', '⦇'), + ('⦉', '⦉'), + ('⦋', '⦋'), + ('⦍', '⦍'), + ('⦏', '⦏'), + ('⦑', '⦑'), + ('⦓', '⦓'), + ('⦕', '⦕'), + ('⦗', '⦗'), + ('⧘', '⧘'), + ('⧚', '⧚'), + ('⧼', '⧼'), + ('⸢', '⸢'), + ('⸤', '⸤'), + ('⸦', '⸦'), + ('⸨', '⸨'), + ('⹂', '⹂'), + ('⹕', '⹕'), + ('⹗', '⹗'), + ('⹙', '⹙'), + ('⹛', '⹛'), + ('〈', '〈'), + ('《', '《'), + ('「', '「'), + ('『', '『'), + ('【', '【'), + ('〔', '〔'), + ('〖', '〖'), + ('〘', '〘'), + ('〚', '〚'), + ('〝', '〝'), + ('﴿', '﴿'), + ('︗', '︗'), + ('︵', '︵'), + ('︷', '︷'), + ('︹', '︹'), + ('︻', '︻'), + ('︽', '︽'), + ('︿', '︿'), + ('﹁', '﹁'), + ('﹃', '﹃'), + ('﹇', '﹇'), + ('﹙', '﹙'), + ('﹛', '﹛'), + ('﹝', '﹝'), + ('（', '（'), + ('［', '［'), + ('｛', '｛'), + ('｟', '｟'), + ('｢', '｢'), +]; + +pub const OTHER: &'static [(char, char)] = &[ + ('\0', '\u{1f}'), + ('\u{7f}', '\u{9f}'), + ('\u{ad}', '\u{ad}'), + ('\u{378}', '\u{379}'), + ('\u{380}', '\u{383}'), + ('\u{38b}', '\u{38b}'), + ('\u{38d}', '\u{38d}'), + ('\u{3a2}', '\u{3a2}'), + ('\u{530}', '\u{530}'), + ('\u{557}', '\u{558}'), + ('\u{58b}', '\u{58c}'), + ('\u{590}', '\u{590}'), + ('\u{5c8}', '\u{5cf}'), + ('\u{5eb}', '\u{5ee}'), + ('\u{5f5}', '\u{605}'), + ('\u{61c}', '\u{61c}'), + ('\u{6dd}', '\u{6dd}'), + ('\u{70e}', '\u{70f}'), + ('\u{74b}', '\u{74c}'), + ('\u{7b2}', '\u{7bf}'), + ('\u{7fb}', '\u{7fc}'), + ('\u{82e}', '\u{82f}'), + ('\u{83f}', '\u{83f}'), + ('\u{85c}', '\u{85d}'), + ('\u{85f}', '\u{85f}'), + ('\u{86b}', '\u{86f}'), + ('\u{88f}', '\u{896}'), + ('\u{8e2}', '\u{8e2}'), + ('\u{984}', '\u{984}'), + ('\u{98d}', '\u{98e}'), + ('\u{991}', '\u{992}'), + ('\u{9a9}', '\u{9a9}'), + ('\u{9b1}', '\u{9b1}'), + ('\u{9b3}', '\u{9b5}'), + ('\u{9ba}', '\u{9bb}'), + ('\u{9c5}', '\u{9c6}'), + ('\u{9c9}', '\u{9ca}'), + ('\u{9cf}', '\u{9d6}'), + ('\u{9d8}', '\u{9db}'), + ('\u{9de}', '\u{9de}'), + ('\u{9e4}', '\u{9e5}'), + ('\u{9ff}', '\u{a00}'), + ('\u{a04}', '\u{a04}'), + ('\u{a0b}', '\u{a0e}'), + ('\u{a11}', '\u{a12}'), + ('\u{a29}', '\u{a29}'), + ('\u{a31}', '\u{a31}'), + ('\u{a34}', '\u{a34}'), + ('\u{a37}', '\u{a37}'), + ('\u{a3a}', '\u{a3b}'), + ('\u{a3d}', '\u{a3d}'), + ('\u{a43}', '\u{a46}'), + ('\u{a49}', '\u{a4a}'), + ('\u{a4e}', '\u{a50}'), + ('\u{a52}', '\u{a58}'), + ('\u{a5d}', '\u{a5d}'), + ('\u{a5f}', '\u{a65}'), + ('\u{a77}', '\u{a80}'), + ('\u{a84}', '\u{a84}'), + ('\u{a8e}', '\u{a8e}'), + ('\u{a92}', '\u{a92}'), + ('\u{aa9}', '\u{aa9}'), + ('\u{ab1}', '\u{ab1}'), + ('\u{ab4}', '\u{ab4}'), + ('\u{aba}', '\u{abb}'), + ('\u{ac6}', '\u{ac6}'), + ('\u{aca}', '\u{aca}'), + ('\u{ace}', '\u{acf}'), + ('\u{ad1}', '\u{adf}'), + ('\u{ae4}', '\u{ae5}'), + ('\u{af2}', '\u{af8}'), + ('\u{b00}', '\u{b00}'), + ('\u{b04}', '\u{b04}'), + ('\u{b0d}', '\u{b0e}'), + ('\u{b11}', '\u{b12}'), + ('\u{b29}', '\u{b29}'), + ('\u{b31}', '\u{b31}'), + ('\u{b34}', '\u{b34}'), + ('\u{b3a}', '\u{b3b}'), + ('\u{b45}', '\u{b46}'), + ('\u{b49}', '\u{b4a}'), + ('\u{b4e}', '\u{b54}'), + ('\u{b58}', '\u{b5b}'), + ('\u{b5e}', '\u{b5e}'), + ('\u{b64}', '\u{b65}'), + ('\u{b78}', '\u{b81}'), + ('\u{b84}', '\u{b84}'), + ('\u{b8b}', '\u{b8d}'), + ('\u{b91}', '\u{b91}'), + ('\u{b96}', '\u{b98}'), + ('\u{b9b}', '\u{b9b}'), + ('\u{b9d}', '\u{b9d}'), + ('\u{ba0}', '\u{ba2}'), + ('\u{ba5}', '\u{ba7}'), + ('\u{bab}', '\u{bad}'), + ('\u{bba}', '\u{bbd}'), + ('\u{bc3}', '\u{bc5}'), + ('\u{bc9}', '\u{bc9}'), + ('\u{bce}', '\u{bcf}'), + ('\u{bd1}', '\u{bd6}'), + ('\u{bd8}', '\u{be5}'), + ('\u{bfb}', '\u{bff}'), + ('\u{c0d}', '\u{c0d}'), + ('\u{c11}', '\u{c11}'), + ('\u{c29}', '\u{c29}'), + ('\u{c3a}', '\u{c3b}'), + ('\u{c45}', '\u{c45}'), + ('\u{c49}', '\u{c49}'), + ('\u{c4e}', '\u{c54}'), + ('\u{c57}', '\u{c57}'), + ('\u{c5b}', '\u{c5c}'), + ('\u{c5e}', '\u{c5f}'), + ('\u{c64}', '\u{c65}'), + ('\u{c70}', '\u{c76}'), + ('\u{c8d}', '\u{c8d}'), + ('\u{c91}', '\u{c91}'), + ('\u{ca9}', '\u{ca9}'), + ('\u{cb4}', '\u{cb4}'), + ('\u{cba}', '\u{cbb}'), + ('\u{cc5}', '\u{cc5}'), + ('\u{cc9}', '\u{cc9}'), + ('\u{cce}', '\u{cd4}'), + ('\u{cd7}', '\u{cdc}'), + ('\u{cdf}', '\u{cdf}'), + ('\u{ce4}', '\u{ce5}'), + ('\u{cf0}', '\u{cf0}'), + ('\u{cf4}', '\u{cff}'), + ('\u{d0d}', '\u{d0d}'), + ('\u{d11}', '\u{d11}'), + ('\u{d45}', '\u{d45}'), + ('\u{d49}', '\u{d49}'), + ('\u{d50}', '\u{d53}'), + ('\u{d64}', '\u{d65}'), + ('\u{d80}', '\u{d80}'), + ('\u{d84}', '\u{d84}'), + ('\u{d97}', '\u{d99}'), + ('\u{db2}', '\u{db2}'), + ('\u{dbc}', '\u{dbc}'), + ('\u{dbe}', '\u{dbf}'), + ('\u{dc7}', '\u{dc9}'), + ('\u{dcb}', '\u{dce}'), + ('\u{dd5}', '\u{dd5}'), + ('\u{dd7}', '\u{dd7}'), + ('\u{de0}', '\u{de5}'), + ('\u{df0}', '\u{df1}'), + ('\u{df5}', '\u{e00}'), + ('\u{e3b}', '\u{e3e}'), + ('\u{e5c}', '\u{e80}'), + ('\u{e83}', '\u{e83}'), + ('\u{e85}', '\u{e85}'), + ('\u{e8b}', '\u{e8b}'), + ('\u{ea4}', '\u{ea4}'), + ('\u{ea6}', '\u{ea6}'), + ('\u{ebe}', '\u{ebf}'), + ('\u{ec5}', '\u{ec5}'), + ('\u{ec7}', '\u{ec7}'), + ('\u{ecf}', '\u{ecf}'), + ('\u{eda}', '\u{edb}'), + ('\u{ee0}', '\u{eff}'), + ('\u{f48}', '\u{f48}'), + ('\u{f6d}', '\u{f70}'), + ('\u{f98}', '\u{f98}'), + ('\u{fbd}', '\u{fbd}'), + ('\u{fcd}', '\u{fcd}'), + ('\u{fdb}', '\u{fff}'), + ('\u{10c6}', '\u{10c6}'), + ('\u{10c8}', '\u{10cc}'), + ('\u{10ce}', '\u{10cf}'), + ('\u{1249}', '\u{1249}'), + ('\u{124e}', '\u{124f}'), + ('\u{1257}', '\u{1257}'), + ('\u{1259}', '\u{1259}'), + ('\u{125e}', '\u{125f}'), + ('\u{1289}', '\u{1289}'), + ('\u{128e}', '\u{128f}'), + ('\u{12b1}', '\u{12b1}'), + ('\u{12b6}', '\u{12b7}'), + ('\u{12bf}', '\u{12bf}'), + ('\u{12c1}', '\u{12c1}'), + ('\u{12c6}', '\u{12c7}'), + ('\u{12d7}', '\u{12d7}'), + ('\u{1311}', '\u{1311}'), + ('\u{1316}', '\u{1317}'), + ('\u{135b}', '\u{135c}'), + ('\u{137d}', '\u{137f}'), + ('\u{139a}', '\u{139f}'), + ('\u{13f6}', '\u{13f7}'), + ('\u{13fe}', '\u{13ff}'), + ('\u{169d}', '\u{169f}'), + ('\u{16f9}', '\u{16ff}'), + ('\u{1716}', '\u{171e}'), + ('\u{1737}', '\u{173f}'), + ('\u{1754}', '\u{175f}'), + ('\u{176d}', '\u{176d}'), + ('\u{1771}', '\u{1771}'), + ('\u{1774}', '\u{177f}'), + ('\u{17de}', '\u{17df}'), + ('\u{17ea}', '\u{17ef}'), + ('\u{17fa}', '\u{17ff}'), + ('\u{180e}', '\u{180e}'), + ('\u{181a}', '\u{181f}'), + ('\u{1879}', '\u{187f}'), + ('\u{18ab}', '\u{18af}'), + ('\u{18f6}', '\u{18ff}'), + ('\u{191f}', '\u{191f}'), + ('\u{192c}', '\u{192f}'), + ('\u{193c}', '\u{193f}'), + ('\u{1941}', '\u{1943}'), + ('\u{196e}', '\u{196f}'), + ('\u{1975}', '\u{197f}'), + ('\u{19ac}', '\u{19af}'), + ('\u{19ca}', '\u{19cf}'), + ('\u{19db}', '\u{19dd}'), + ('\u{1a1c}', '\u{1a1d}'), + ('\u{1a5f}', '\u{1a5f}'), + ('\u{1a7d}', '\u{1a7e}'), + ('\u{1a8a}', '\u{1a8f}'), + ('\u{1a9a}', '\u{1a9f}'), + ('\u{1aae}', '\u{1aaf}'), + ('\u{1acf}', '\u{1aff}'), + ('\u{1b4d}', '\u{1b4d}'), + ('\u{1bf4}', '\u{1bfb}'), + ('\u{1c38}', '\u{1c3a}'), + ('\u{1c4a}', '\u{1c4c}'), + ('\u{1c8b}', '\u{1c8f}'), + ('\u{1cbb}', '\u{1cbc}'), + ('\u{1cc8}', '\u{1ccf}'), + ('\u{1cfb}', '\u{1cff}'), + ('\u{1f16}', '\u{1f17}'), + ('\u{1f1e}', '\u{1f1f}'), + ('\u{1f46}', '\u{1f47}'), + ('\u{1f4e}', '\u{1f4f}'), + ('\u{1f58}', '\u{1f58}'), + ('\u{1f5a}', '\u{1f5a}'), + ('\u{1f5c}', '\u{1f5c}'), + ('\u{1f5e}', '\u{1f5e}'), + ('\u{1f7e}', '\u{1f7f}'), + ('\u{1fb5}', '\u{1fb5}'), + ('\u{1fc5}', '\u{1fc5}'), + ('\u{1fd4}', '\u{1fd5}'), + ('\u{1fdc}', '\u{1fdc}'), + ('\u{1ff0}', '\u{1ff1}'), + ('\u{1ff5}', '\u{1ff5}'), + ('\u{1fff}', '\u{1fff}'), + ('\u{200b}', '\u{200f}'), + ('\u{202a}', '\u{202e}'), + ('\u{2060}', '\u{206f}'), + ('\u{2072}', '\u{2073}'), + ('\u{208f}', '\u{208f}'), + ('\u{209d}', '\u{209f}'), + ('\u{20c1}', '\u{20cf}'), + ('\u{20f1}', '\u{20ff}'), + ('\u{218c}', '\u{218f}'), + ('\u{242a}', '\u{243f}'), + ('\u{244b}', '\u{245f}'), + ('\u{2b74}', '\u{2b75}'), + ('\u{2b96}', '\u{2b96}'), + ('\u{2cf4}', '\u{2cf8}'), + ('\u{2d26}', '\u{2d26}'), + ('\u{2d28}', '\u{2d2c}'), + ('\u{2d2e}', '\u{2d2f}'), + ('\u{2d68}', '\u{2d6e}'), + ('\u{2d71}', '\u{2d7e}'), + ('\u{2d97}', '\u{2d9f}'), + ('\u{2da7}', '\u{2da7}'), + ('\u{2daf}', '\u{2daf}'), + ('\u{2db7}', '\u{2db7}'), + ('\u{2dbf}', '\u{2dbf}'), + ('\u{2dc7}', '\u{2dc7}'), + ('\u{2dcf}', '\u{2dcf}'), + ('\u{2dd7}', '\u{2dd7}'), + ('\u{2ddf}', '\u{2ddf}'), + ('\u{2e5e}', '\u{2e7f}'), + ('\u{2e9a}', '\u{2e9a}'), + ('\u{2ef4}', '\u{2eff}'), + ('\u{2fd6}', '\u{2fef}'), + ('\u{3040}', '\u{3040}'), + ('\u{3097}', '\u{3098}'), + ('\u{3100}', '\u{3104}'), + ('\u{3130}', '\u{3130}'), + ('\u{318f}', '\u{318f}'), + ('\u{31e6}', '\u{31ee}'), + ('\u{321f}', '\u{321f}'), + ('\u{a48d}', '\u{a48f}'), + ('\u{a4c7}', '\u{a4cf}'), + ('\u{a62c}', '\u{a63f}'), + ('\u{a6f8}', '\u{a6ff}'), + ('\u{a7ce}', '\u{a7cf}'), + ('\u{a7d2}', '\u{a7d2}'), + ('\u{a7d4}', '\u{a7d4}'), + ('\u{a7dd}', '\u{a7f1}'), + ('\u{a82d}', '\u{a82f}'), + ('\u{a83a}', '\u{a83f}'), + ('\u{a878}', '\u{a87f}'), + ('\u{a8c6}', '\u{a8cd}'), + ('\u{a8da}', '\u{a8df}'), + ('\u{a954}', '\u{a95e}'), + ('\u{a97d}', '\u{a97f}'), + ('\u{a9ce}', '\u{a9ce}'), + ('\u{a9da}', '\u{a9dd}'), + ('\u{a9ff}', '\u{a9ff}'), + ('\u{aa37}', '\u{aa3f}'), + ('\u{aa4e}', '\u{aa4f}'), + ('\u{aa5a}', '\u{aa5b}'), + ('\u{aac3}', '\u{aada}'), + ('\u{aaf7}', '\u{ab00}'), + ('\u{ab07}', '\u{ab08}'), + ('\u{ab0f}', '\u{ab10}'), + ('\u{ab17}', '\u{ab1f}'), + ('\u{ab27}', '\u{ab27}'), + ('\u{ab2f}', '\u{ab2f}'), + ('\u{ab6c}', '\u{ab6f}'), + ('\u{abee}', '\u{abef}'), + ('\u{abfa}', '\u{abff}'), + ('\u{d7a4}', '\u{d7af}'), + ('\u{d7c7}', '\u{d7ca}'), + ('\u{d7fc}', '\u{f8ff}'), + ('\u{fa6e}', '\u{fa6f}'), + ('\u{fada}', '\u{faff}'), + ('\u{fb07}', '\u{fb12}'), + ('\u{fb18}', '\u{fb1c}'), + ('\u{fb37}', '\u{fb37}'), + ('\u{fb3d}', '\u{fb3d}'), + ('\u{fb3f}', '\u{fb3f}'), + ('\u{fb42}', '\u{fb42}'), + ('\u{fb45}', '\u{fb45}'), + ('\u{fbc3}', '\u{fbd2}'), + ('\u{fd90}', '\u{fd91}'), + ('\u{fdc8}', '\u{fdce}'), + ('\u{fdd0}', '\u{fdef}'), + ('\u{fe1a}', '\u{fe1f}'), + ('\u{fe53}', '\u{fe53}'), + ('\u{fe67}', '\u{fe67}'), + ('\u{fe6c}', '\u{fe6f}'), + ('\u{fe75}', '\u{fe75}'), + ('\u{fefd}', '\u{ff00}'), + ('\u{ffbf}', '\u{ffc1}'), + ('\u{ffc8}', '\u{ffc9}'), + ('\u{ffd0}', '\u{ffd1}'), + ('\u{ffd8}', '\u{ffd9}'), + ('\u{ffdd}', '\u{ffdf}'), + ('\u{ffe7}', '\u{ffe7}'), + ('\u{ffef}', '\u{fffb}'), + ('\u{fffe}', '\u{ffff}'), + ('\u{1000c}', '\u{1000c}'), + ('\u{10027}', '\u{10027}'), + ('\u{1003b}', '\u{1003b}'), + ('\u{1003e}', '\u{1003e}'), + ('\u{1004e}', '\u{1004f}'), + ('\u{1005e}', '\u{1007f}'), + ('\u{100fb}', '\u{100ff}'), + ('\u{10103}', '\u{10106}'), + ('\u{10134}', '\u{10136}'), + ('\u{1018f}', '\u{1018f}'), + ('\u{1019d}', '\u{1019f}'), + ('\u{101a1}', '\u{101cf}'), + ('\u{101fe}', '\u{1027f}'), + ('\u{1029d}', '\u{1029f}'), + ('\u{102d1}', '\u{102df}'), + ('\u{102fc}', '\u{102ff}'), + ('\u{10324}', '\u{1032c}'), + ('\u{1034b}', '\u{1034f}'), + ('\u{1037b}', '\u{1037f}'), + ('\u{1039e}', '\u{1039e}'), + ('\u{103c4}', '\u{103c7}'), + ('\u{103d6}', '\u{103ff}'), + ('\u{1049e}', '\u{1049f}'), + ('\u{104aa}', '\u{104af}'), + ('\u{104d4}', '\u{104d7}'), + ('\u{104fc}', '\u{104ff}'), + ('\u{10528}', '\u{1052f}'), + ('\u{10564}', '\u{1056e}'), + ('\u{1057b}', '\u{1057b}'), + ('\u{1058b}', '\u{1058b}'), + ('\u{10593}', '\u{10593}'), + ('\u{10596}', '\u{10596}'), + ('\u{105a2}', '\u{105a2}'), + ('\u{105b2}', '\u{105b2}'), + ('\u{105ba}', '\u{105ba}'), + ('\u{105bd}', '\u{105bf}'), + ('\u{105f4}', '\u{105ff}'), + ('\u{10737}', '\u{1073f}'), + ('\u{10756}', '\u{1075f}'), + ('\u{10768}', '\u{1077f}'), + ('\u{10786}', '\u{10786}'), + ('\u{107b1}', '\u{107b1}'), + ('\u{107bb}', '\u{107ff}'), + ('\u{10806}', '\u{10807}'), + ('\u{10809}', '\u{10809}'), + ('\u{10836}', '\u{10836}'), + ('\u{10839}', '\u{1083b}'), + ('\u{1083d}', '\u{1083e}'), + ('\u{10856}', '\u{10856}'), + ('\u{1089f}', '\u{108a6}'), + ('\u{108b0}', '\u{108df}'), + ('\u{108f3}', '\u{108f3}'), + ('\u{108f6}', '\u{108fa}'), + ('\u{1091c}', '\u{1091e}'), + ('\u{1093a}', '\u{1093e}'), + ('\u{10940}', '\u{1097f}'), + ('\u{109b8}', '\u{109bb}'), + ('\u{109d0}', '\u{109d1}'), + ('\u{10a04}', '\u{10a04}'), + ('\u{10a07}', '\u{10a0b}'), + ('\u{10a14}', '\u{10a14}'), + ('\u{10a18}', '\u{10a18}'), + ('\u{10a36}', '\u{10a37}'), + ('\u{10a3b}', '\u{10a3e}'), + ('\u{10a49}', '\u{10a4f}'), + ('\u{10a59}', '\u{10a5f}'), + ('\u{10aa0}', '\u{10abf}'), + ('\u{10ae7}', '\u{10aea}'), + ('\u{10af7}', '\u{10aff}'), + ('\u{10b36}', '\u{10b38}'), + ('\u{10b56}', '\u{10b57}'), + ('\u{10b73}', '\u{10b77}'), + ('\u{10b92}', '\u{10b98}'), + ('\u{10b9d}', '\u{10ba8}'), + ('\u{10bb0}', '\u{10bff}'), + ('\u{10c49}', '\u{10c7f}'), + ('\u{10cb3}', '\u{10cbf}'), + ('\u{10cf3}', '\u{10cf9}'), + ('\u{10d28}', '\u{10d2f}'), + ('\u{10d3a}', '\u{10d3f}'), + ('\u{10d66}', '\u{10d68}'), + ('\u{10d86}', '\u{10d8d}'), + ('\u{10d90}', '\u{10e5f}'), + ('\u{10e7f}', '\u{10e7f}'), + ('\u{10eaa}', '\u{10eaa}'), + ('\u{10eae}', '\u{10eaf}'), + ('\u{10eb2}', '\u{10ec1}'), + ('\u{10ec5}', '\u{10efb}'), + ('\u{10f28}', '\u{10f2f}'), + ('\u{10f5a}', '\u{10f6f}'), + ('\u{10f8a}', '\u{10faf}'), + ('\u{10fcc}', '\u{10fdf}'), + ('\u{10ff7}', '\u{10fff}'), + ('\u{1104e}', '\u{11051}'), + ('\u{11076}', '\u{1107e}'), + ('\u{110bd}', '\u{110bd}'), + ('\u{110c3}', '\u{110cf}'), + ('\u{110e9}', '\u{110ef}'), + ('\u{110fa}', '\u{110ff}'), + ('\u{11135}', '\u{11135}'), + ('\u{11148}', '\u{1114f}'), + ('\u{11177}', '\u{1117f}'), + ('\u{111e0}', '\u{111e0}'), + ('\u{111f5}', '\u{111ff}'), + ('\u{11212}', '\u{11212}'), + ('\u{11242}', '\u{1127f}'), + ('\u{11287}', '\u{11287}'), + ('\u{11289}', '\u{11289}'), + ('\u{1128e}', '\u{1128e}'), + ('\u{1129e}', '\u{1129e}'), + ('\u{112aa}', '\u{112af}'), + ('\u{112eb}', '\u{112ef}'), + ('\u{112fa}', '\u{112ff}'), + ('\u{11304}', '\u{11304}'), + ('\u{1130d}', '\u{1130e}'), + ('\u{11311}', '\u{11312}'), + ('\u{11329}', '\u{11329}'), + ('\u{11331}', '\u{11331}'), + ('\u{11334}', '\u{11334}'), + ('\u{1133a}', '\u{1133a}'), + ('\u{11345}', '\u{11346}'), + ('\u{11349}', '\u{1134a}'), + ('\u{1134e}', '\u{1134f}'), + ('\u{11351}', '\u{11356}'), + ('\u{11358}', '\u{1135c}'), + ('\u{11364}', '\u{11365}'), + ('\u{1136d}', '\u{1136f}'), + ('\u{11375}', '\u{1137f}'), + ('\u{1138a}', '\u{1138a}'), + ('\u{1138c}', '\u{1138d}'), + ('\u{1138f}', '\u{1138f}'), + ('\u{113b6}', '\u{113b6}'), + ('\u{113c1}', '\u{113c1}'), + ('\u{113c3}', '\u{113c4}'), + ('\u{113c6}', '\u{113c6}'), + ('\u{113cb}', '\u{113cb}'), + ('\u{113d6}', '\u{113d6}'), + ('\u{113d9}', '\u{113e0}'), + ('\u{113e3}', '\u{113ff}'), + ('\u{1145c}', '\u{1145c}'), + ('\u{11462}', '\u{1147f}'), + ('\u{114c8}', '\u{114cf}'), + ('\u{114da}', '\u{1157f}'), + ('\u{115b6}', '\u{115b7}'), + ('\u{115de}', '\u{115ff}'), + ('\u{11645}', '\u{1164f}'), + ('\u{1165a}', '\u{1165f}'), + ('\u{1166d}', '\u{1167f}'), + ('\u{116ba}', '\u{116bf}'), + ('\u{116ca}', '\u{116cf}'), + ('\u{116e4}', '\u{116ff}'), + ('\u{1171b}', '\u{1171c}'), + ('\u{1172c}', '\u{1172f}'), + ('\u{11747}', '\u{117ff}'), + ('\u{1183c}', '\u{1189f}'), + ('\u{118f3}', '\u{118fe}'), + ('\u{11907}', '\u{11908}'), + ('\u{1190a}', '\u{1190b}'), + ('\u{11914}', '\u{11914}'), + ('\u{11917}', '\u{11917}'), + ('\u{11936}', '\u{11936}'), + ('\u{11939}', '\u{1193a}'), + ('\u{11947}', '\u{1194f}'), + ('\u{1195a}', '\u{1199f}'), + ('\u{119a8}', '\u{119a9}'), + ('\u{119d8}', '\u{119d9}'), + ('\u{119e5}', '\u{119ff}'), + ('\u{11a48}', '\u{11a4f}'), + ('\u{11aa3}', '\u{11aaf}'), + ('\u{11af9}', '\u{11aff}'), + ('\u{11b0a}', '\u{11bbf}'), + ('\u{11be2}', '\u{11bef}'), + ('\u{11bfa}', '\u{11bff}'), + ('\u{11c09}', '\u{11c09}'), + ('\u{11c37}', '\u{11c37}'), + ('\u{11c46}', '\u{11c4f}'), + ('\u{11c6d}', '\u{11c6f}'), + ('\u{11c90}', '\u{11c91}'), + ('\u{11ca8}', '\u{11ca8}'), + ('\u{11cb7}', '\u{11cff}'), + ('\u{11d07}', '\u{11d07}'), + ('\u{11d0a}', '\u{11d0a}'), + ('\u{11d37}', '\u{11d39}'), + ('\u{11d3b}', '\u{11d3b}'), + ('\u{11d3e}', '\u{11d3e}'), + ('\u{11d48}', '\u{11d4f}'), + ('\u{11d5a}', '\u{11d5f}'), + ('\u{11d66}', '\u{11d66}'), + ('\u{11d69}', '\u{11d69}'), + ('\u{11d8f}', '\u{11d8f}'), + ('\u{11d92}', '\u{11d92}'), + ('\u{11d99}', '\u{11d9f}'), + ('\u{11daa}', '\u{11edf}'), + ('\u{11ef9}', '\u{11eff}'), + ('\u{11f11}', '\u{11f11}'), + ('\u{11f3b}', '\u{11f3d}'), + ('\u{11f5b}', '\u{11faf}'), + ('\u{11fb1}', '\u{11fbf}'), + ('\u{11ff2}', '\u{11ffe}'), + ('\u{1239a}', '\u{123ff}'), + ('\u{1246f}', '\u{1246f}'), + ('\u{12475}', '\u{1247f}'), + ('\u{12544}', '\u{12f8f}'), + ('\u{12ff3}', '\u{12fff}'), + ('\u{13430}', '\u{1343f}'), + ('\u{13456}', '\u{1345f}'), + ('\u{143fb}', '\u{143ff}'), + ('\u{14647}', '\u{160ff}'), + ('\u{1613a}', '\u{167ff}'), + ('\u{16a39}', '\u{16a3f}'), + ('\u{16a5f}', '\u{16a5f}'), + ('\u{16a6a}', '\u{16a6d}'), + ('\u{16abf}', '\u{16abf}'), + ('\u{16aca}', '\u{16acf}'), + ('\u{16aee}', '\u{16aef}'), + ('\u{16af6}', '\u{16aff}'), + ('\u{16b46}', '\u{16b4f}'), + ('\u{16b5a}', '\u{16b5a}'), + ('\u{16b62}', '\u{16b62}'), + ('\u{16b78}', '\u{16b7c}'), + ('\u{16b90}', '\u{16d3f}'), + ('\u{16d7a}', '\u{16e3f}'), + ('\u{16e9b}', '\u{16eff}'), + ('\u{16f4b}', '\u{16f4e}'), + ('\u{16f88}', '\u{16f8e}'), + ('\u{16fa0}', '\u{16fdf}'), + ('\u{16fe5}', '\u{16fef}'), + ('\u{16ff2}', '\u{16fff}'), + ('\u{187f8}', '\u{187ff}'), + ('\u{18cd6}', '\u{18cfe}'), + ('\u{18d09}', '\u{1afef}'), + ('\u{1aff4}', '\u{1aff4}'), + ('\u{1affc}', '\u{1affc}'), + ('\u{1afff}', '\u{1afff}'), + ('\u{1b123}', '\u{1b131}'), + ('\u{1b133}', '\u{1b14f}'), + ('\u{1b153}', '\u{1b154}'), + ('\u{1b156}', '\u{1b163}'), + ('\u{1b168}', '\u{1b16f}'), + ('\u{1b2fc}', '\u{1bbff}'), + ('\u{1bc6b}', '\u{1bc6f}'), + ('\u{1bc7d}', '\u{1bc7f}'), + ('\u{1bc89}', '\u{1bc8f}'), + ('\u{1bc9a}', '\u{1bc9b}'), + ('\u{1bca0}', '\u{1cbff}'), + ('\u{1ccfa}', '\u{1ccff}'), + ('\u{1ceb4}', '\u{1ceff}'), + ('\u{1cf2e}', '\u{1cf2f}'), + ('\u{1cf47}', '\u{1cf4f}'), + ('\u{1cfc4}', '\u{1cfff}'), + ('\u{1d0f6}', '\u{1d0ff}'), + ('\u{1d127}', '\u{1d128}'), + ('\u{1d173}', '\u{1d17a}'), + ('\u{1d1eb}', '\u{1d1ff}'), + ('\u{1d246}', '\u{1d2bf}'), + ('\u{1d2d4}', '\u{1d2df}'), + ('\u{1d2f4}', '\u{1d2ff}'), + ('\u{1d357}', '\u{1d35f}'), + ('\u{1d379}', '\u{1d3ff}'), + ('\u{1d455}', '\u{1d455}'), + ('\u{1d49d}', '\u{1d49d}'), + ('\u{1d4a0}', '\u{1d4a1}'), + ('\u{1d4a3}', '\u{1d4a4}'), + ('\u{1d4a7}', '\u{1d4a8}'), + ('\u{1d4ad}', '\u{1d4ad}'), + ('\u{1d4ba}', '\u{1d4ba}'), + ('\u{1d4bc}', '\u{1d4bc}'), + ('\u{1d4c4}', '\u{1d4c4}'), + ('\u{1d506}', '\u{1d506}'), + ('\u{1d50b}', '\u{1d50c}'), + ('\u{1d515}', '\u{1d515}'), + ('\u{1d51d}', '\u{1d51d}'), + ('\u{1d53a}', '\u{1d53a}'), + ('\u{1d53f}', '\u{1d53f}'), + ('\u{1d545}', '\u{1d545}'), + ('\u{1d547}', '\u{1d549}'), + ('\u{1d551}', '\u{1d551}'), + ('\u{1d6a6}', '\u{1d6a7}'), + ('\u{1d7cc}', '\u{1d7cd}'), + ('\u{1da8c}', '\u{1da9a}'), + ('\u{1daa0}', '\u{1daa0}'), + ('\u{1dab0}', '\u{1deff}'), + ('\u{1df1f}', '\u{1df24}'), + ('\u{1df2b}', '\u{1dfff}'), + ('\u{1e007}', '\u{1e007}'), + ('\u{1e019}', '\u{1e01a}'), + ('\u{1e022}', '\u{1e022}'), + ('\u{1e025}', '\u{1e025}'), + ('\u{1e02b}', '\u{1e02f}'), + ('\u{1e06e}', '\u{1e08e}'), + ('\u{1e090}', '\u{1e0ff}'), + ('\u{1e12d}', '\u{1e12f}'), + ('\u{1e13e}', '\u{1e13f}'), + ('\u{1e14a}', '\u{1e14d}'), + ('\u{1e150}', '\u{1e28f}'), + ('\u{1e2af}', '\u{1e2bf}'), + ('\u{1e2fa}', '\u{1e2fe}'), + ('\u{1e300}', '\u{1e4cf}'), + ('\u{1e4fa}', '\u{1e5cf}'), + ('\u{1e5fb}', '\u{1e5fe}'), + ('\u{1e600}', '\u{1e7df}'), + ('\u{1e7e7}', '\u{1e7e7}'), + ('\u{1e7ec}', '\u{1e7ec}'), + ('\u{1e7ef}', '\u{1e7ef}'), + ('\u{1e7ff}', '\u{1e7ff}'), + ('\u{1e8c5}', '\u{1e8c6}'), + ('\u{1e8d7}', '\u{1e8ff}'), + ('\u{1e94c}', '\u{1e94f}'), + ('\u{1e95a}', '\u{1e95d}'), + ('\u{1e960}', '\u{1ec70}'), + ('\u{1ecb5}', '\u{1ed00}'), + ('\u{1ed3e}', '\u{1edff}'), + ('\u{1ee04}', '\u{1ee04}'), + ('\u{1ee20}', '\u{1ee20}'), + ('\u{1ee23}', '\u{1ee23}'), + ('\u{1ee25}', '\u{1ee26}'), + ('\u{1ee28}', '\u{1ee28}'), + ('\u{1ee33}', '\u{1ee33}'), + ('\u{1ee38}', '\u{1ee38}'), + ('\u{1ee3a}', '\u{1ee3a}'), + ('\u{1ee3c}', '\u{1ee41}'), + ('\u{1ee43}', '\u{1ee46}'), + ('\u{1ee48}', '\u{1ee48}'), + ('\u{1ee4a}', '\u{1ee4a}'), + ('\u{1ee4c}', '\u{1ee4c}'), + ('\u{1ee50}', '\u{1ee50}'), + ('\u{1ee53}', '\u{1ee53}'), + ('\u{1ee55}', '\u{1ee56}'), + ('\u{1ee58}', '\u{1ee58}'), + ('\u{1ee5a}', '\u{1ee5a}'), + ('\u{1ee5c}', '\u{1ee5c}'), + ('\u{1ee5e}', '\u{1ee5e}'), + ('\u{1ee60}', '\u{1ee60}'), + ('\u{1ee63}', '\u{1ee63}'), + ('\u{1ee65}', '\u{1ee66}'), + ('\u{1ee6b}', '\u{1ee6b}'), + ('\u{1ee73}', '\u{1ee73}'), + ('\u{1ee78}', '\u{1ee78}'), + ('\u{1ee7d}', '\u{1ee7d}'), + ('\u{1ee7f}', '\u{1ee7f}'), + ('\u{1ee8a}', '\u{1ee8a}'), + ('\u{1ee9c}', '\u{1eea0}'), + ('\u{1eea4}', '\u{1eea4}'), + ('\u{1eeaa}', '\u{1eeaa}'), + ('\u{1eebc}', '\u{1eeef}'), + ('\u{1eef2}', '\u{1efff}'), + ('\u{1f02c}', '\u{1f02f}'), + ('\u{1f094}', '\u{1f09f}'), + ('\u{1f0af}', '\u{1f0b0}'), + ('\u{1f0c0}', '\u{1f0c0}'), + ('\u{1f0d0}', '\u{1f0d0}'), + ('\u{1f0f6}', '\u{1f0ff}'), + ('\u{1f1ae}', '\u{1f1e5}'), + ('\u{1f203}', '\u{1f20f}'), + ('\u{1f23c}', '\u{1f23f}'), + ('\u{1f249}', '\u{1f24f}'), + ('\u{1f252}', '\u{1f25f}'), + ('\u{1f266}', '\u{1f2ff}'), + ('\u{1f6d8}', '\u{1f6db}'), + ('\u{1f6ed}', '\u{1f6ef}'), + ('\u{1f6fd}', '\u{1f6ff}'), + ('\u{1f777}', '\u{1f77a}'), + ('\u{1f7da}', '\u{1f7df}'), + ('\u{1f7ec}', '\u{1f7ef}'), + ('\u{1f7f1}', '\u{1f7ff}'), + ('\u{1f80c}', '\u{1f80f}'), + ('\u{1f848}', '\u{1f84f}'), + ('\u{1f85a}', '\u{1f85f}'), + ('\u{1f888}', '\u{1f88f}'), + ('\u{1f8ae}', '\u{1f8af}'), + ('\u{1f8bc}', '\u{1f8bf}'), + ('\u{1f8c2}', '\u{1f8ff}'), + ('\u{1fa54}', '\u{1fa5f}'), + ('\u{1fa6e}', '\u{1fa6f}'), + ('\u{1fa7d}', '\u{1fa7f}'), + ('\u{1fa8a}', '\u{1fa8e}'), + ('\u{1fac7}', '\u{1facd}'), + ('\u{1fadd}', '\u{1fade}'), + ('\u{1faea}', '\u{1faef}'), + ('\u{1faf9}', '\u{1faff}'), + ('\u{1fb93}', '\u{1fb93}'), + ('\u{1fbfa}', '\u{1ffff}'), + ('\u{2a6e0}', '\u{2a6ff}'), + ('\u{2b73a}', '\u{2b73f}'), + ('\u{2b81e}', '\u{2b81f}'), + ('\u{2cea2}', '\u{2ceaf}'), + ('\u{2ebe1}', '\u{2ebef}'), + ('\u{2ee5e}', '\u{2f7ff}'), + ('\u{2fa1e}', '\u{2ffff}'), + ('\u{3134b}', '\u{3134f}'), + ('\u{323b0}', '\u{e00ff}'), + ('\u{e01f0}', '\u{10ffff}'), +]; + +pub const OTHER_LETTER: &'static [(char, char)] = &[ + ('ª', 'ª'), + ('º', 'º'), + ('ƻ', 'ƻ'), + ('ǀ', 'ǃ'), + ('ʔ', 'ʔ'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('ؠ', 'ؿ'), + ('ف', 'ي'), + ('ٮ', 'ٯ'), + ('ٱ', 'ۓ'), + ('ە', 'ە'), + ('ۮ', 'ۯ'), + ('ۺ', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', 'ܐ'), + ('ܒ', 'ܯ'), + ('ݍ', 'ޥ'), + ('ޱ', 'ޱ'), + ('ߊ', 'ߪ'), + ('ࠀ', 'ࠕ'), + ('ࡀ', 'ࡘ'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('ࢠ', 'ࣈ'), + ('ऄ', 'ह'), + ('ऽ', 'ऽ'), + ('ॐ', 'ॐ'), + ('क़', 'ॡ'), + ('ॲ', 'ঀ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('ঽ', 'ঽ'), + ('ৎ', 'ৎ'), + ('ড়', 'ঢ়'), + ('য়', 'ৡ'), + ('ৰ', 'ৱ'), + ('ৼ', 'ৼ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('ੲ', 'ੴ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('ઽ', 'ઽ'), + ('ૐ', 'ૐ'), + ('ૠ', 'ૡ'), + ('ૹ', 'ૹ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('ଽ', 'ଽ'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୡ'), + ('ୱ', 'ୱ'), + ('ஃ', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('ௐ', 'ௐ'), + ('అ', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('ఽ', 'ఽ'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', 'ౡ'), + ('ಀ', 'ಀ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಽ', 'ಽ'), + ('ೝ', 'ೞ'), + ('ೠ', 'ೡ'), + ('ೱ', 'ೲ'), + ('ഄ', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ഺ'), + ('ഽ', 'ഽ'), + ('ൎ', 'ൎ'), + ('ൔ', 'ൖ'), + ('ൟ', 'ൡ'), + ('ൺ', 'ൿ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('ก', 'ะ'), + ('า', 'ำ'), + ('เ', 'ๅ'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ະ'), + ('າ', 'ຳ'), + ('ຽ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('ཀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('ྈ', 'ྌ'), + ('က', 'ဪ'), + ('ဿ', 'ဿ'), + ('ၐ', 'ၕ'), + ('ၚ', 'ၝ'), + ('ၡ', 'ၡ'), + ('ၥ', 'ၦ'), + ('ၮ', 'ၰ'), + ('ၵ', 'ႁ'), + ('ႎ', 'ႎ'), + ('ᄀ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('ᎀ', 'ᎏ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛱ', 'ᛸ'), + ('ᜀ', 'ᜑ'), + ('ᜟ', 'ᜱ'), + ('ᝀ', 'ᝑ'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('ក', 'ឳ'), + ('ៜ', 'ៜ'), + ('ᠠ', 'ᡂ'), + ('ᡄ', 'ᡸ'), + ('ᢀ', 'ᢄ'), + ('ᢇ', 'ᢨ'), + ('ᢪ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('ᥐ', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('ᨀ', 'ᨖ'), + ('ᨠ', 'ᩔ'), + ('ᬅ', 'ᬳ'), + ('ᭅ', 'ᭌ'), + ('ᮃ', 'ᮠ'), + ('ᮮ', 'ᮯ'), + ('ᮺ', 'ᯥ'), + ('ᰀ', 'ᰣ'), + ('ᱍ', 'ᱏ'), + ('ᱚ', 'ᱷ'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', 'ᳶ'), + ('ᳺ', 'ᳺ'), + ('ℵ', 'ℸ'), + ('ⴰ', 'ⵧ'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('〆', '〆'), + ('〼', '〼'), + ('ぁ', 'ゖ'), + ('ゟ', 'ゟ'), + ('ァ', 'ヺ'), + ('ヿ', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꀔ'), + ('ꀖ', 'ꒌ'), + ('ꓐ', 'ꓷ'), + ('ꔀ', 'ꘋ'), + ('ꘐ', 'ꘟ'), + ('ꘪ', 'ꘫ'), + ('ꙮ', 'ꙮ'), + ('ꚠ', 'ꛥ'), + ('ꞏ', 'ꞏ'), + ('ꟷ', 'ꟷ'), + ('ꟻ', 'ꠁ'), + ('ꠃ', 'ꠅ'), + ('ꠇ', 'ꠊ'), + ('ꠌ', 'ꠢ'), + ('ꡀ', 'ꡳ'), + ('ꢂ', 'ꢳ'), + ('ꣲ', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', 'ꣾ'), + ('ꤊ', 'ꤥ'), + ('ꤰ', 'ꥆ'), + ('ꥠ', 'ꥼ'), + ('ꦄ', 'ꦲ'), + ('ꧠ', 'ꧤ'), + ('ꧧ', 'ꧯ'), + ('ꧺ', 'ꧾ'), + ('ꨀ', 'ꨨ'), + ('ꩀ', 'ꩂ'), + ('ꩄ', 'ꩋ'), + ('ꩠ', 'ꩯ'), + ('ꩱ', 'ꩶ'), + ('ꩺ', 'ꩺ'), + ('ꩾ', 'ꪯ'), + ('ꪱ', 'ꪱ'), + ('ꪵ', 'ꪶ'), + ('ꪹ', 'ꪽ'), + ('ꫀ', 'ꫀ'), + ('ꫂ', 'ꫂ'), + ('ꫛ', 'ꫜ'), + ('ꫠ', 'ꫪ'), + ('ꫲ', 'ꫲ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꯀ', 'ꯢ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('יִ', 'יִ'), + ('ײַ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('ｦ', 'ｯ'), + ('ｱ', 'ﾝ'), + ('ﾠ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐌀', '𐌟'), + ('𐌭', '𐍀'), + ('𐍂', '𐍉'), + ('𐍐', '𐍵'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐑐', '𐒝'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '𐨀'), + ('𐨐', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '𐫤'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐴀', '𐴣'), + ('𐵊', '𐵍'), + ('𐵏', '𐵏'), + ('𐺀', '𐺩'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('𐼀', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '𐽅'), + ('𐽰', '𐾁'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀃', '𑀷'), + ('𑁱', '𑁲'), + ('𑁵', '𑁵'), + ('𑂃', '𑂯'), + ('𑃐', '𑃨'), + ('𑄃', '𑄦'), + ('𑅄', '𑅄'), + ('𑅇', '𑅇'), + ('𑅐', '𑅲'), + ('𑅶', '𑅶'), + ('𑆃', '𑆲'), + ('𑇁', '𑇄'), + ('𑇚', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '𑈫'), + ('𑈿', '𑉀'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '𑋞'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('𑌽', '𑌽'), + ('𑍐', '𑍐'), + ('𑍝', '𑍡'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '𑎷'), + ('𑏑', '𑏑'), + ('𑏓', '𑏓'), + ('𑐀', '𑐴'), + ('𑑇', '𑑊'), + ('𑑟', '𑑡'), + ('𑒀', '𑒯'), + ('𑓄', '𑓅'), + ('𑓇', '𑓇'), + ('𑖀', '𑖮'), + ('𑗘', '𑗛'), + ('𑘀', '𑘯'), + ('𑙄', '𑙄'), + ('𑚀', '𑚪'), + ('𑚸', '𑚸'), + ('𑜀', '𑜚'), + ('𑝀', '𑝆'), + ('𑠀', '𑠫'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤯'), + ('𑤿', '𑤿'), + ('𑥁', '𑥁'), + ('𑦠', '𑦧'), + ('𑦪', '𑧐'), + ('𑧡', '𑧡'), + ('𑧣', '𑧣'), + ('𑨀', '𑨀'), + ('𑨋', '𑨲'), + ('𑨺', '𑨺'), + ('𑩐', '𑩐'), + ('𑩜', '𑪉'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑰀', '𑰈'), + ('𑰊', '𑰮'), + ('𑱀', '𑱀'), + ('𑱲', '𑲏'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '𑴰'), + ('𑵆', '𑵆'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶉'), + ('𑶘', '𑶘'), + ('𑻠', '𑻲'), + ('𑼂', '𑼂'), + ('𑼄', '𑼐'), + ('𑼒', '𑼳'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('𓑁', '𓑆'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄝'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩰', '𖪾'), + ('𖫐', '𖫭'), + ('𖬀', '𖬯'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵃', '𖵪'), + ('𖼀', '𖽊'), + ('𖽐', '𖽐'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𝼊', '𝼊'), + ('𞄀', '𞄬'), + ('𞅎', '𞅎'), + ('𞊐', '𞊭'), + ('𞋀', '𞋫'), + ('𞓐', '𞓪'), + ('𞗐', '𞗭'), + ('𞗰', '𞗰'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const OTHER_NUMBER: &'static [(char, char)] = &[ + ('²', '³'), + ('¹', '¹'), + ('¼', '¾'), + ('৴', '৹'), + ('୲', '୷'), + ('௰', '௲'), + ('౸', '౾'), + ('൘', '൞'), + ('൰', '൸'), + ('༪', '༳'), + ('፩', '፼'), + ('៰', '៹'), + ('᧚', '᧚'), + ('⁰', '⁰'), + ('⁴', '⁹'), + ('₀', '₉'), + ('⅐', '⅟'), + ('↉', '↉'), + ('①', '⒛'), + ('⓪', '⓿'), + ('❶', '➓'), + ('⳽', '⳽'), + ('㆒', '㆕'), + ('㈠', '㈩'), + ('㉈', '㉏'), + ('㉑', '㉟'), + ('㊀', '㊉'), + ('㊱', '㊿'), + ('꠰', '꠵'), + ('𐄇', '𐄳'), + ('𐅵', '𐅸'), + ('𐆊', '𐆋'), + ('𐋡', '𐋻'), + ('𐌠', '𐌣'), + ('𐡘', '𐡟'), + ('𐡹', '𐡿'), + ('𐢧', '𐢯'), + ('𐣻', '𐣿'), + ('𐤖', '𐤛'), + ('𐦼', '𐦽'), + ('𐧀', '𐧏'), + ('𐧒', '𐧿'), + ('𐩀', '𐩈'), + ('𐩽', '𐩾'), + ('𐪝', '𐪟'), + ('𐫫', '𐫯'), + ('𐭘', '𐭟'), + ('𐭸', '𐭿'), + ('𐮩', '𐮯'), + ('𐳺', '𐳿'), + ('𐹠', '𐹾'), + ('𐼝', '𐼦'), + ('𐽑', '𐽔'), + ('𐿅', '𐿋'), + ('𑁒', '𑁥'), + ('𑇡', '𑇴'), + ('𑜺', '𑜻'), + ('𑣪', '𑣲'), + ('𑱚', '𑱬'), + ('𑿀', '𑿔'), + ('𖭛', '𖭡'), + ('𖺀', '𖺖'), + ('𝋀', '𝋓'), + ('𝋠', '𝋳'), + ('𝍠', '𝍸'), + ('𞣇', '𞣏'), + ('𞱱', '𞲫'), + ('𞲭', '𞲯'), + ('𞲱', '𞲴'), + ('𞴁', '𞴭'), + ('𞴯', '𞴽'), + ('🄀', '🄌'), +]; + +pub const OTHER_PUNCTUATION: &'static [(char, char)] = &[ + ('!', '#'), + ('%', '\''), + ('*', '*'), + (',', ','), + ('.', '/'), + (':', ';'), + ('?', '@'), + ('\\', '\\'), + ('¡', '¡'), + ('§', '§'), + ('¶', '·'), + ('¿', '¿'), + (';', ';'), + ('·', '·'), + ('՚', '՟'), + ('։', '։'), + ('׀', '׀'), + ('׃', '׃'), + ('׆', '׆'), + ('׳', '״'), + ('؉', '؊'), + ('،', '؍'), + ('؛', '؛'), + ('؝', '؟'), + ('٪', '٭'), + ('۔', '۔'), + ('܀', '܍'), + ('߷', '߹'), + ('࠰', '࠾'), + ('࡞', '࡞'), + ('।', '॥'), + ('॰', '॰'), + ('৽', '৽'), + ('੶', '੶'), + ('૰', '૰'), + ('౷', '౷'), + ('಄', '಄'), + ('෴', '෴'), + ('๏', '๏'), + ('๚', '๛'), + ('༄', '༒'), + ('༔', '༔'), + ('྅', '྅'), + ('࿐', '࿔'), + ('࿙', '࿚'), + ('၊', '၏'), + ('჻', '჻'), + ('፠', '፨'), + ('᙮', '᙮'), + ('᛫', '᛭'), + ('᜵', '᜶'), + ('។', '៖'), + ('៘', '៚'), + ('᠀', '᠅'), + ('᠇', '᠊'), + ('᥄', '᥅'), + ('᨞', '᨟'), + ('᪠', '᪦'), + ('᪨', '᪭'), + ('᭎', '᭏'), + ('᭚', '᭠'), + ('᭽', '᭿'), + ('᯼', '᯿'), + ('᰻', '᰿'), + ('᱾', '᱿'), + ('᳀', '᳇'), + ('᳓', '᳓'), + ('‖', '‗'), + ('†', '‧'), + ('‰', '‸'), + ('※', '‾'), + ('⁁', '⁃'), + ('⁇', '⁑'), + ('⁓', '⁓'), + ('⁕', '⁞'), + ('⳹', '⳼'), + ('⳾', '⳿'), + ('⵰', '⵰'), + ('⸀', '⸁'), + ('⸆', '⸈'), + ('⸋', '⸋'), + ('⸎', '⸖'), + ('⸘', '⸙'), + ('⸛', '⸛'), + ('⸞', '⸟'), + ('⸪', '⸮'), + ('⸰', '⸹'), + ('⸼', '⸿'), + ('⹁', '⹁'), + ('⹃', '⹏'), + ('⹒', '⹔'), + ('、', '〃'), + ('〽', '〽'), + ('・', '・'), + ('꓾', '꓿'), + ('꘍', '꘏'), + ('꙳', '꙳'), + ('꙾', '꙾'), + ('꛲', '꛷'), + ('꡴', '꡷'), + ('꣎', '꣏'), + ('꣸', '꣺'), + ('꣼', '꣼'), + ('꤮', '꤯'), + ('꥟', '꥟'), + ('꧁', '꧍'), + ('꧞', '꧟'), + ('꩜', '꩟'), + ('꫞', '꫟'), + ('꫰', '꫱'), + ('꯫', '꯫'), + ('︐', '︖'), + ('︙', '︙'), + ('︰', '︰'), + ('﹅', '﹆'), + ('﹉', '﹌'), + ('﹐', '﹒'), + ('﹔', '﹗'), + ('﹟', '﹡'), + ('﹨', '﹨'), + ('﹪', '﹫'), + ('！', '＃'), + ('％', '＇'), + ('＊', '＊'), + ('，', '，'), + ('．', '／'), + ('：', '；'), + ('？', '＠'), + ('＼', '＼'), + ('｡', '｡'), + ('､', '･'), + ('𐄀', '𐄂'), + ('𐎟', '𐎟'), + ('𐏐', '𐏐'), + ('𐕯', '𐕯'), + ('𐡗', '𐡗'), + ('𐤟', '𐤟'), + ('𐤿', '𐤿'), + ('𐩐', '𐩘'), + ('𐩿', '𐩿'), + ('𐫰', '𐫶'), + ('𐬹', '𐬿'), + ('𐮙', '𐮜'), + ('𐽕', '𐽙'), + ('𐾆', '𐾉'), + ('𑁇', '𑁍'), + ('𑂻', '𑂼'), + ('𑂾', '𑃁'), + ('𑅀', '𑅃'), + ('𑅴', '𑅵'), + ('𑇅', '𑇈'), + ('𑇍', '𑇍'), + ('𑇛', '𑇛'), + ('𑇝', '𑇟'), + ('𑈸', '𑈽'), + ('𑊩', '𑊩'), + ('𑏔', '𑏕'), + ('𑏗', '𑏘'), + ('𑑋', '𑑏'), + ('𑑚', '𑑛'), + ('𑑝', '𑑝'), + ('𑓆', '𑓆'), + ('𑗁', '𑗗'), + ('𑙁', '𑙃'), + ('𑙠', '𑙬'), + ('𑚹', '𑚹'), + ('𑜼', '𑜾'), + ('𑠻', '𑠻'), + ('𑥄', '𑥆'), + ('𑧢', '𑧢'), + ('𑨿', '𑩆'), + ('𑪚', '𑪜'), + ('𑪞', '𑪢'), + ('𑬀', '𑬉'), + ('𑯡', '𑯡'), + ('𑱁', '𑱅'), + ('𑱰', '𑱱'), + ('𑻷', '𑻸'), + ('𑽃', '𑽏'), + ('𑿿', '𑿿'), + ('𒑰', '𒑴'), + ('𒿱', '𒿲'), + ('𖩮', '𖩯'), + ('𖫵', '𖫵'), + ('𖬷', '𖬻'), + ('𖭄', '𖭄'), + ('𖵭', '𖵯'), + ('𖺗', '𖺚'), + ('𖿢', '𖿢'), + ('𛲟', '𛲟'), + ('𝪇', '𝪋'), + ('𞗿', '𞗿'), + ('𞥞', '𞥟'), +]; + +pub const OTHER_SYMBOL: &'static [(char, char)] = &[ + ('¦', '¦'), + ('©', '©'), + ('®', '®'), + ('°', '°'), + ('҂', '҂'), + ('֍', '֎'), + ('؎', '؏'), + ('۞', '۞'), + ('۩', '۩'), + ('۽', '۾'), + ('߶', '߶'), + ('৺', '৺'), + ('୰', '୰'), + ('௳', '௸'), + ('௺', '௺'), + ('౿', '౿'), + ('൏', '൏'), + ('൹', '൹'), + ('༁', '༃'), + ('༓', '༓'), + ('༕', '༗'), + ('༚', '༟'), + ('༴', '༴'), + ('༶', '༶'), + ('༸', '༸'), + ('྾', '࿅'), + ('࿇', '࿌'), + ('࿎', '࿏'), + ('࿕', '࿘'), + ('႞', '႟'), + ('᎐', '᎙'), + ('᙭', '᙭'), + ('᥀', '᥀'), + ('᧞', '᧿'), + ('᭡', '᭪'), + ('᭴', '᭼'), + ('℀', '℁'), + ('℃', '℆'), + ('℈', '℉'), + ('℔', '℔'), + ('№', '℗'), + ('℞', '℣'), + ('℥', '℥'), + ('℧', '℧'), + ('℩', '℩'), + ('℮', '℮'), + ('℺', '℻'), + ('⅊', '⅊'), + ('⅌', '⅍'), + ('⅏', '⅏'), + ('↊', '↋'), + ('↕', '↙'), + ('↜', '↟'), + ('↡', '↢'), + ('↤', '↥'), + ('↧', '↭'), + ('↯', '⇍'), + ('⇐', '⇑'), + ('⇓', '⇓'), + ('⇕', '⇳'), + ('⌀', '⌇'), + ('⌌', '⌟'), + ('⌢', '⌨'), + ('⌫', '⍻'), + ('⍽', '⎚'), + ('⎴', '⏛'), + ('⏢', '␩'), + ('⑀', '⑊'), + ('⒜', 'ⓩ'), + ('─', '▶'), + ('▸', '◀'), + ('◂', '◷'), + ('☀', '♮'), + ('♰', '❧'), + ('➔', '➿'), + ('⠀', '⣿'), + ('⬀', '⬯'), + ('⭅', '⭆'), + ('⭍', '⭳'), + ('⭶', '⮕'), + ('⮗', '⯿'), + ('⳥', '⳪'), + ('⹐', '⹑'), + ('⺀', '⺙'), + ('⺛', '⻳'), + ('⼀', '⿕'), + ('⿰', '⿿'), + ('〄', '〄'), + ('〒', '〓'), + ('〠', '〠'), + ('〶', '〷'), + ('〾', '〿'), + ('㆐', '㆑'), + ('㆖', '㆟'), + ('㇀', '㇥'), + ('㇯', '㇯'), + ('㈀', '㈞'), + ('㈪', '㉇'), + ('㉐', '㉐'), + ('㉠', '㉿'), + ('㊊', '㊰'), + ('㋀', '㏿'), + ('䷀', '䷿'), + ('꒐', '꓆'), + ('꠨', '꠫'), + ('꠶', '꠷'), + ('꠹', '꠹'), + ('꩷', '꩹'), + ('﵀', '﵏'), + ('﷏', '﷏'), + ('﷽', '﷿'), + ('￤', '￤'), + ('￨', '￨'), + ('￭', '￮'), + ('', '�'), + ('𐄷', '𐄿'), + ('𐅹', '𐆉'), + ('𐆌', '𐆎'), + ('𐆐', '𐆜'), + ('𐆠', '𐆠'), + ('𐇐', '𐇼'), + ('𐡷', '𐡸'), + ('𐫈', '𐫈'), + ('𑜿', '𑜿'), + ('𑿕', '𑿜'), + ('𑿡', '𑿱'), + ('𖬼', '𖬿'), + ('𖭅', '𖭅'), + ('𛲜', '𛲜'), + ('𜰀', '𜳯'), + ('𜴀', '𜺳'), + ('𜽐', '𜿃'), + ('𝀀', '𝃵'), + ('𝄀', '𝄦'), + ('𝄩', '𝅘𝅥𝅲'), + ('𝅪', '𝅬'), + ('𝆃', '𝆄'), + ('𝆌', '𝆩'), + ('𝆮', '𝇪'), + ('𝈀', '𝉁'), + ('𝉅', '𝉅'), + ('𝌀', '𝍖'), + ('𝠀', '𝧿'), + ('𝨷', '𝨺'), + ('𝩭', '𝩴'), + ('𝩶', '𝪃'), + ('𝪅', '𝪆'), + ('𞅏', '𞅏'), + ('𞲬', '𞲬'), + ('𞴮', '𞴮'), + ('🀀', '🀫'), + ('🀰', '🂓'), + ('🂠', '🂮'), + ('🂱', '🂿'), + ('🃁', '🃏'), + ('🃑', '🃵'), + ('🄍', '🆭'), + ('🇦', '🈂'), + ('🈐', '🈻'), + ('🉀', '🉈'), + ('🉐', '🉑'), + ('🉠', '🉥'), + ('🌀', '🏺'), + ('🐀', '🛗'), + ('🛜', '🛬'), + ('🛰', '🛼'), + ('🜀', '🝶'), + ('🝻', '🟙'), + ('🟠', '🟫'), + ('🟰', '🟰'), + ('🠀', '🠋'), + ('🠐', '🡇'), + ('🡐', '🡙'), + ('🡠', '🢇'), + ('🢐', '🢭'), + ('🢰', '🢻'), + ('🣀', '🣁'), + ('🤀', '🩓'), + ('🩠', '🩭'), + ('🩰', '🩼'), + ('🪀', '🪉'), + ('🪏', '🫆'), + ('🫎', '🫜'), + ('🫟', '🫩'), + ('🫰', '🫸'), + ('🬀', '🮒'), + ('🮔', '🯯'), +]; + +pub const PARAGRAPH_SEPARATOR: &'static [(char, char)] = + &[('\u{2029}', '\u{2029}')]; + +pub const PRIVATE_USE: &'static [(char, char)] = &[ + ('\u{e000}', '\u{f8ff}'), + ('\u{f0000}', '\u{ffffd}'), + ('\u{100000}', '\u{10fffd}'), +]; + +pub const PUNCTUATION: &'static [(char, char)] = &[ + ('!', '#'), + ('%', '*'), + (',', '/'), + (':', ';'), + ('?', '@'), + ('[', ']'), + ('_', '_'), + ('{', '{'), + ('}', '}'), + ('¡', '¡'), + ('§', '§'), + ('«', '«'), + ('¶', '·'), + ('»', '»'), + ('¿', '¿'), + (';', ';'), + ('·', '·'), + ('՚', '՟'), + ('։', '֊'), + ('־', '־'), + ('׀', '׀'), + ('׃', '׃'), + ('׆', '׆'), + ('׳', '״'), + ('؉', '؊'), + ('،', '؍'), + ('؛', '؛'), + ('؝', '؟'), + ('٪', '٭'), + ('۔', '۔'), + ('܀', '܍'), + ('߷', '߹'), + ('࠰', '࠾'), + ('࡞', '࡞'), + ('।', '॥'), + ('॰', '॰'), + ('৽', '৽'), + ('੶', '੶'), + ('૰', '૰'), + ('౷', '౷'), + ('಄', '಄'), + ('෴', '෴'), + ('๏', '๏'), + ('๚', '๛'), + ('༄', '༒'), + ('༔', '༔'), + ('༺', '༽'), + ('྅', '྅'), + ('࿐', '࿔'), + ('࿙', '࿚'), + ('၊', '၏'), + ('჻', '჻'), + ('፠', '፨'), + ('᐀', '᐀'), + ('᙮', '᙮'), + ('᚛', '᚜'), + ('᛫', '᛭'), + ('᜵', '᜶'), + ('។', '៖'), + ('៘', '៚'), + ('᠀', '᠊'), + ('᥄', '᥅'), + ('᨞', '᨟'), + ('᪠', '᪦'), + ('᪨', '᪭'), + ('᭎', '᭏'), + ('᭚', '᭠'), + ('᭽', '᭿'), + ('᯼', '᯿'), + ('᰻', '᰿'), + ('᱾', '᱿'), + ('᳀', '᳇'), + ('᳓', '᳓'), + ('‐', '‧'), + ('‰', '⁃'), + ('⁅', '⁑'), + ('⁓', '⁞'), + ('⁽', '⁾'), + ('₍', '₎'), + ('⌈', '⌋'), + ('〈', '〉'), + ('❨', '❵'), + ('⟅', '⟆'), + ('⟦', '⟯'), + ('⦃', '⦘'), + ('⧘', '⧛'), + ('⧼', '⧽'), + ('⳹', '⳼'), + ('⳾', '⳿'), + ('⵰', '⵰'), + ('⸀', '⸮'), + ('⸰', '⹏'), + ('⹒', '⹝'), + ('、', '〃'), + ('〈', '】'), + ('〔', '〟'), + ('〰', '〰'), + ('〽', '〽'), + ('゠', '゠'), + ('・', '・'), + ('꓾', '꓿'), + ('꘍', '꘏'), + ('꙳', '꙳'), + ('꙾', '꙾'), + ('꛲', '꛷'), + ('꡴', '꡷'), + ('꣎', '꣏'), + ('꣸', '꣺'), + ('꣼', '꣼'), + ('꤮', '꤯'), + ('꥟', '꥟'), + ('꧁', '꧍'), + ('꧞', '꧟'), + ('꩜', '꩟'), + ('꫞', '꫟'), + ('꫰', '꫱'), + ('꯫', '꯫'), + ('﴾', '﴿'), + ('︐', '︙'), + ('︰', '﹒'), + ('﹔', '﹡'), + ('﹣', '﹣'), + ('﹨', '﹨'), + ('﹪', '﹫'), + ('！', '＃'), + ('％', '＊'), + ('，', '／'), + ('：', '；'), + ('？', '＠'), + ('［', '］'), + ('＿', '＿'), + ('｛', '｛'), + ('｝', '｝'), + ('｟', '･'), + ('𐄀', '𐄂'), + ('𐎟', '𐎟'), + ('𐏐', '𐏐'), + ('𐕯', '𐕯'), + ('𐡗', '𐡗'), + ('𐤟', '𐤟'), + ('𐤿', '𐤿'), + ('𐩐', '𐩘'), + ('𐩿', '𐩿'), + ('𐫰', '𐫶'), + ('𐬹', '𐬿'), + ('𐮙', '𐮜'), + ('𐵮', '𐵮'), + ('𐺭', '𐺭'), + ('𐽕', '𐽙'), + ('𐾆', '𐾉'), + ('𑁇', '𑁍'), + ('𑂻', '𑂼'), + ('𑂾', '𑃁'), + ('𑅀', '𑅃'), + ('𑅴', '𑅵'), + ('𑇅', '𑇈'), + ('𑇍', '𑇍'), + ('𑇛', '𑇛'), + ('𑇝', '𑇟'), + ('𑈸', '𑈽'), + ('𑊩', '𑊩'), + ('𑏔', '𑏕'), + ('𑏗', '𑏘'), + ('𑑋', '𑑏'), + ('𑑚', '𑑛'), + ('𑑝', '𑑝'), + ('𑓆', '𑓆'), + ('𑗁', '𑗗'), + ('𑙁', '𑙃'), + ('𑙠', '𑙬'), + ('𑚹', '𑚹'), + ('𑜼', '𑜾'), + ('𑠻', '𑠻'), + ('𑥄', '𑥆'), + ('𑧢', '𑧢'), + ('𑨿', '𑩆'), + ('𑪚', '𑪜'), + ('𑪞', '𑪢'), + ('𑬀', '𑬉'), + ('𑯡', '𑯡'), + ('𑱁', '𑱅'), + ('𑱰', '𑱱'), + ('𑻷', '𑻸'), + ('𑽃', '𑽏'), + ('𑿿', '𑿿'), + ('𒑰', '𒑴'), + ('𒿱', '𒿲'), + ('𖩮', '𖩯'), + ('𖫵', '𖫵'), + ('𖬷', '𖬻'), + ('𖭄', '𖭄'), + ('𖵭', '𖵯'), + ('𖺗', '𖺚'), + ('𖿢', '𖿢'), + ('𛲟', '𛲟'), + ('𝪇', '𝪋'), + ('𞗿', '𞗿'), + ('𞥞', '𞥟'), +]; + +pub const SEPARATOR: &'static [(char, char)] = &[ + (' ', ' '), + ('\u{a0}', '\u{a0}'), + ('\u{1680}', '\u{1680}'), + ('\u{2000}', '\u{200a}'), + ('\u{2028}', '\u{2029}'), + ('\u{202f}', '\u{202f}'), + ('\u{205f}', '\u{205f}'), + ('\u{3000}', '\u{3000}'), +]; + +pub const SPACE_SEPARATOR: &'static [(char, char)] = &[ + (' ', ' '), + ('\u{a0}', '\u{a0}'), + ('\u{1680}', '\u{1680}'), + ('\u{2000}', '\u{200a}'), + ('\u{202f}', '\u{202f}'), + ('\u{205f}', '\u{205f}'), + ('\u{3000}', '\u{3000}'), +]; + +pub const SPACING_MARK: &'static [(char, char)] = &[ + ('ः', 'ः'), + ('ऻ', 'ऻ'), + ('ा', 'ी'), + ('ॉ', 'ौ'), + ('ॎ', 'ॏ'), + ('ং', 'ঃ'), + ('\u{9be}', 'ী'), + ('ে', 'ৈ'), + ('ো', 'ৌ'), + ('\u{9d7}', '\u{9d7}'), + ('ਃ', 'ਃ'), + ('ਾ', 'ੀ'), + ('ઃ', 'ઃ'), + ('ા', 'ી'), + ('ૉ', 'ૉ'), + ('ો', 'ૌ'), + ('ଂ', 'ଃ'), + ('\u{b3e}', '\u{b3e}'), + ('ୀ', 'ୀ'), + ('େ', 'ୈ'), + ('ୋ', 'ୌ'), + ('\u{b57}', '\u{b57}'), + ('\u{bbe}', 'ி'), + ('ு', 'ூ'), + ('ெ', 'ை'), + ('ொ', 'ௌ'), + ('\u{bd7}', '\u{bd7}'), + ('ఁ', 'ః'), + ('ు', 'ౄ'), + ('ಂ', 'ಃ'), + ('ಾ', 'ಾ'), + ('\u{cc0}', 'ೄ'), + ('\u{cc7}', '\u{cc8}'), + ('\u{cca}', '\u{ccb}'), + ('\u{cd5}', '\u{cd6}'), + ('ೳ', 'ೳ'), + ('ം', 'ഃ'), + ('\u{d3e}', 'ീ'), + ('െ', 'ൈ'), + ('ൊ', 'ൌ'), + ('\u{d57}', '\u{d57}'), + ('ං', 'ඃ'), + ('\u{dcf}', 'ෑ'), + ('ෘ', '\u{ddf}'), + ('ෲ', 'ෳ'), + ('༾', '༿'), + ('ཿ', 'ཿ'), + ('ါ', 'ာ'), + ('ေ', 'ေ'), + ('း', 'း'), + ('ျ', 'ြ'), + ('ၖ', 'ၗ'), + ('ၢ', 'ၤ'), + ('ၧ', 'ၭ'), + ('ႃ', 'ႄ'), + ('ႇ', 'ႌ'), + ('ႏ', 'ႏ'), + ('ႚ', 'ႜ'), + ('\u{1715}', '\u{1715}'), + ('\u{1734}', '\u{1734}'), + ('ា', 'ា'), + ('ើ', 'ៅ'), + ('ះ', 'ៈ'), + ('ᤣ', 'ᤦ'), + ('ᤩ', 'ᤫ'), + ('ᤰ', 'ᤱ'), + ('ᤳ', 'ᤸ'), + ('ᨙ', 'ᨚ'), + ('ᩕ', 'ᩕ'), + ('ᩗ', 'ᩗ'), + ('ᩡ', 'ᩡ'), + ('ᩣ', 'ᩤ'), + ('ᩭ', 'ᩲ'), + ('ᬄ', 'ᬄ'), + ('\u{1b35}', '\u{1b35}'), + ('\u{1b3b}', '\u{1b3b}'), + ('\u{1b3d}', 'ᭁ'), + ('\u{1b43}', '\u{1b44}'), + ('ᮂ', 'ᮂ'), + ('ᮡ', 'ᮡ'), + ('ᮦ', 'ᮧ'), + ('\u{1baa}', '\u{1baa}'), + ('ᯧ', 'ᯧ'), + ('ᯪ', 'ᯬ'), + ('ᯮ', 'ᯮ'), + ('\u{1bf2}', '\u{1bf3}'), + ('ᰤ', 'ᰫ'), + ('ᰴ', 'ᰵ'), + ('᳡', '᳡'), + ('᳷', '᳷'), + ('\u{302e}', '\u{302f}'), + ('ꠣ', 'ꠤ'), + ('ꠧ', 'ꠧ'), + ('ꢀ', 'ꢁ'), + ('ꢴ', 'ꣃ'), + ('ꥒ', '\u{a953}'), + ('ꦃ', 'ꦃ'), + ('ꦴ', 'ꦵ'), + ('ꦺ', 'ꦻ'), + ('ꦾ', '\u{a9c0}'), + ('ꨯ', 'ꨰ'), + ('ꨳ', 'ꨴ'), + ('ꩍ', 'ꩍ'), + ('ꩻ', 'ꩻ'), + ('ꩽ', 'ꩽ'), + ('ꫫ', 'ꫫ'), + ('ꫮ', 'ꫯ'), + ('ꫵ', 'ꫵ'), + ('ꯣ', 'ꯤ'), + ('ꯦ', 'ꯧ'), + ('ꯩ', 'ꯪ'), + ('꯬', '꯬'), + ('𑀀', '𑀀'), + ('𑀂', '𑀂'), + ('𑂂', '𑂂'), + ('𑂰', '𑂲'), + ('𑂷', '𑂸'), + ('𑄬', '𑄬'), + ('𑅅', '𑅆'), + ('𑆂', '𑆂'), + ('𑆳', '𑆵'), + ('𑆿', '\u{111c0}'), + ('𑇎', '𑇎'), + ('𑈬', '𑈮'), + ('𑈲', '𑈳'), + ('\u{11235}', '\u{11235}'), + ('𑋠', '𑋢'), + ('𑌂', '𑌃'), + ('\u{1133e}', '𑌿'), + ('𑍁', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('𑍢', '𑍣'), + ('\u{113b8}', '𑎺'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏍'), + ('\u{113cf}', '\u{113cf}'), + ('𑐵', '𑐷'), + ('𑑀', '𑑁'), + ('𑑅', '𑑅'), + ('\u{114b0}', '𑒲'), + ('𑒹', '𑒹'), + ('𑒻', '𑒾'), + ('𑓁', '𑓁'), + ('\u{115af}', '𑖱'), + ('𑖸', '𑖻'), + ('𑖾', '𑖾'), + ('𑘰', '𑘲'), + ('𑘻', '𑘼'), + ('𑘾', '𑘾'), + ('𑚬', '𑚬'), + ('𑚮', '𑚯'), + ('\u{116b6}', '\u{116b6}'), + ('𑜞', '𑜞'), + ('𑜠', '𑜡'), + ('𑜦', '𑜦'), + ('𑠬', '𑠮'), + ('𑠸', '𑠸'), + ('\u{11930}', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193d}', '\u{1193d}'), + ('𑥀', '𑥀'), + ('𑥂', '𑥂'), + ('𑧑', '𑧓'), + ('𑧜', '𑧟'), + ('𑧤', '𑧤'), + ('𑨹', '𑨹'), + ('𑩗', '𑩘'), + ('𑪗', '𑪗'), + ('𑰯', '𑰯'), + ('𑰾', '𑰾'), + ('𑲩', '𑲩'), + ('𑲱', '𑲱'), + ('𑲴', '𑲴'), + ('𑶊', '𑶎'), + ('𑶓', '𑶔'), + ('𑶖', '𑶖'), + ('𑻵', '𑻶'), + ('𑼃', '𑼃'), + ('𑼴', '𑼵'), + ('𑼾', '𑼿'), + ('\u{11f41}', '\u{11f41}'), + ('𖄪', '𖄬'), + ('𖽑', '𖾇'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1d165}', '\u{1d166}'), + ('\u{1d16d}', '\u{1d172}'), +]; + +pub const SYMBOL: &'static [(char, char)] = &[ + ('$', '$'), + ('+', '+'), + ('<', '>'), + ('^', '^'), + ('`', '`'), + ('|', '|'), + ('~', '~'), + ('¢', '¦'), + ('¨', '©'), + ('¬', '¬'), + ('®', '±'), + ('´', '´'), + ('¸', '¸'), + ('×', '×'), + ('÷', '÷'), + ('˂', '˅'), + ('˒', '˟'), + ('˥', '˫'), + ('˭', '˭'), + ('˯', '˿'), + ('͵', '͵'), + ('΄', '΅'), + ('϶', '϶'), + ('҂', '҂'), + ('֍', '֏'), + ('؆', '؈'), + ('؋', '؋'), + ('؎', '؏'), + ('۞', '۞'), + ('۩', '۩'), + ('۽', '۾'), + ('߶', '߶'), + ('߾', '߿'), + ('࢈', '࢈'), + ('৲', '৳'), + ('৺', '৻'), + ('૱', '૱'), + ('୰', '୰'), + ('௳', '௺'), + ('౿', '౿'), + ('൏', '൏'), + ('൹', '൹'), + ('฿', '฿'), + ('༁', '༃'), + ('༓', '༓'), + ('༕', '༗'), + ('༚', '༟'), + ('༴', '༴'), + ('༶', '༶'), + ('༸', '༸'), + ('྾', '࿅'), + ('࿇', '࿌'), + ('࿎', '࿏'), + ('࿕', '࿘'), + ('႞', '႟'), + ('᎐', '᎙'), + ('᙭', '᙭'), + ('៛', '៛'), + ('᥀', '᥀'), + ('᧞', '᧿'), + ('᭡', '᭪'), + ('᭴', '᭼'), + ('᾽', '᾽'), + ('᾿', '῁'), + ('῍', '῏'), + ('῝', '῟'), + ('῭', '`'), + ('´', '῾'), + ('⁄', '⁄'), + ('⁒', '⁒'), + ('⁺', '⁼'), + ('₊', '₌'), + ('₠', '⃀'), + ('℀', '℁'), + ('℃', '℆'), + ('℈', '℉'), + ('℔', '℔'), + ('№', '℘'), + ('℞', '℣'), + ('℥', '℥'), + ('℧', '℧'), + ('℩', '℩'), + ('℮', '℮'), + ('℺', '℻'), + ('⅀', '⅄'), + ('⅊', '⅍'), + ('⅏', '⅏'), + ('↊', '↋'), + ('←', '⌇'), + ('⌌', '⌨'), + ('⌫', '␩'), + ('⑀', '⑊'), + ('⒜', 'ⓩ'), + ('─', '❧'), + ('➔', '⟄'), + ('⟇', '⟥'), + ('⟰', '⦂'), + ('⦙', '⧗'), + ('⧜', '⧻'), + ('⧾', '⭳'), + ('⭶', '⮕'), + ('⮗', '⯿'), + ('⳥', '⳪'), + ('⹐', '⹑'), + ('⺀', '⺙'), + ('⺛', '⻳'), + ('⼀', '⿕'), + ('⿰', '⿿'), + ('〄', '〄'), + ('〒', '〓'), + ('〠', '〠'), + ('〶', '〷'), + ('〾', '〿'), + ('゛', '゜'), + ('㆐', '㆑'), + ('㆖', '㆟'), + ('㇀', '㇥'), + ('㇯', '㇯'), + ('㈀', '㈞'), + ('㈪', '㉇'), + ('㉐', '㉐'), + ('㉠', '㉿'), + ('㊊', '㊰'), + ('㋀', '㏿'), + ('䷀', '䷿'), + ('꒐', '꓆'), + ('꜀', '꜖'), + ('꜠', '꜡'), + ('꞉', '꞊'), + ('꠨', '꠫'), + ('꠶', '꠹'), + ('꩷', '꩹'), + ('꭛', '꭛'), + ('꭪', '꭫'), + ('﬩', '﬩'), + ('﮲', '﯂'), + ('﵀', '﵏'), + ('﷏', '﷏'), + ('﷼', '﷿'), + ('﹢', '﹢'), + ('﹤', '﹦'), + ('﹩', '﹩'), + ('＄', '＄'), + ('＋', '＋'), + ('＜', '＞'), + ('＾', '＾'), + ('｀', '｀'), + ('｜', '｜'), + ('～', '～'), + ('￠', '￦'), + ('￨', '￮'), + ('', '�'), + ('𐄷', '𐄿'), + ('𐅹', '𐆉'), + ('𐆌', '𐆎'), + ('𐆐', '𐆜'), + ('𐆠', '𐆠'), + ('𐇐', '𐇼'), + ('𐡷', '𐡸'), + ('𐫈', '𐫈'), + ('𐶎', '𐶏'), + ('𑜿', '𑜿'), + ('𑿕', '𑿱'), + ('𖬼', '𖬿'), + ('𖭅', '𖭅'), + ('𛲜', '𛲜'), + ('𜰀', '𜳯'), + ('𜴀', '𜺳'), + ('𜽐', '𜿃'), + ('𝀀', '𝃵'), + ('𝄀', '𝄦'), + ('𝄩', '𝅘𝅥𝅲'), + ('𝅪', '𝅬'), + ('𝆃', '𝆄'), + ('𝆌', '𝆩'), + ('𝆮', '𝇪'), + ('𝈀', '𝉁'), + ('𝉅', '𝉅'), + ('𝌀', '𝍖'), + ('𝛁', '𝛁'), + ('𝛛', '𝛛'), + ('𝛻', '𝛻'), + ('𝜕', '𝜕'), + ('𝜵', '𝜵'), + ('𝝏', '𝝏'), + ('𝝯', '𝝯'), + ('𝞉', '𝞉'), + ('𝞩', '𝞩'), + ('𝟃', '𝟃'), + ('𝠀', '𝧿'), + ('𝨷', '𝨺'), + ('𝩭', '𝩴'), + ('𝩶', '𝪃'), + ('𝪅', '𝪆'), + ('𞅏', '𞅏'), + ('𞋿', '𞋿'), + ('𞲬', '𞲬'), + ('𞲰', '𞲰'), + ('𞴮', '𞴮'), + ('𞻰', '𞻱'), + ('🀀', '🀫'), + ('🀰', '🂓'), + ('🂠', '🂮'), + ('🂱', '🂿'), + ('🃁', '🃏'), + ('🃑', '🃵'), + ('🄍', '🆭'), + ('🇦', '🈂'), + ('🈐', '🈻'), + ('🉀', '🉈'), + ('🉐', '🉑'), + ('🉠', '🉥'), + ('🌀', '🛗'), + ('🛜', '🛬'), + ('🛰', '🛼'), + ('🜀', '🝶'), + ('🝻', '🟙'), + ('🟠', '🟫'), + ('🟰', '🟰'), + ('🠀', '🠋'), + ('🠐', '🡇'), + ('🡐', '🡙'), + ('🡠', '🢇'), + ('🢐', '🢭'), + ('🢰', '🢻'), + ('🣀', '🣁'), + ('🤀', '🩓'), + ('🩠', '🩭'), + ('🩰', '🩼'), + ('🪀', '🪉'), + ('🪏', '🫆'), + ('🫎', '🫜'), + ('🫟', '🫩'), + ('🫰', '🫸'), + ('🬀', '🮒'), + ('🮔', '🯯'), +]; + +pub const TITLECASE_LETTER: &'static [(char, char)] = &[ + ('Dž', 'Dž'), + ('Lj', 'Lj'), + ('Nj', 'Nj'), + ('Dz', 'Dz'), + ('ᾈ', 'ᾏ'), + ('ᾘ', 'ᾟ'), + ('ᾨ', 'ᾯ'), + ('ᾼ', 'ᾼ'), + ('ῌ', 'ῌ'), + ('ῼ', 'ῼ'), +]; + +pub const UNASSIGNED: &'static [(char, char)] = &[ + ('\u{378}', '\u{379}'), + ('\u{380}', '\u{383}'), + ('\u{38b}', '\u{38b}'), + ('\u{38d}', '\u{38d}'), + ('\u{3a2}', '\u{3a2}'), + ('\u{530}', '\u{530}'), + ('\u{557}', '\u{558}'), + ('\u{58b}', '\u{58c}'), + ('\u{590}', '\u{590}'), + ('\u{5c8}', '\u{5cf}'), + ('\u{5eb}', '\u{5ee}'), + ('\u{5f5}', '\u{5ff}'), + ('\u{70e}', '\u{70e}'), + ('\u{74b}', '\u{74c}'), + ('\u{7b2}', '\u{7bf}'), + ('\u{7fb}', '\u{7fc}'), + ('\u{82e}', '\u{82f}'), + ('\u{83f}', '\u{83f}'), + ('\u{85c}', '\u{85d}'), + ('\u{85f}', '\u{85f}'), + ('\u{86b}', '\u{86f}'), + ('\u{88f}', '\u{88f}'), + ('\u{892}', '\u{896}'), + ('\u{984}', '\u{984}'), + ('\u{98d}', '\u{98e}'), + ('\u{991}', '\u{992}'), + ('\u{9a9}', '\u{9a9}'), + ('\u{9b1}', '\u{9b1}'), + ('\u{9b3}', '\u{9b5}'), + ('\u{9ba}', '\u{9bb}'), + ('\u{9c5}', '\u{9c6}'), + ('\u{9c9}', '\u{9ca}'), + ('\u{9cf}', '\u{9d6}'), + ('\u{9d8}', '\u{9db}'), + ('\u{9de}', '\u{9de}'), + ('\u{9e4}', '\u{9e5}'), + ('\u{9ff}', '\u{a00}'), + ('\u{a04}', '\u{a04}'), + ('\u{a0b}', '\u{a0e}'), + ('\u{a11}', '\u{a12}'), + ('\u{a29}', '\u{a29}'), + ('\u{a31}', '\u{a31}'), + ('\u{a34}', '\u{a34}'), + ('\u{a37}', '\u{a37}'), + ('\u{a3a}', '\u{a3b}'), + ('\u{a3d}', '\u{a3d}'), + ('\u{a43}', '\u{a46}'), + ('\u{a49}', '\u{a4a}'), + ('\u{a4e}', '\u{a50}'), + ('\u{a52}', '\u{a58}'), + ('\u{a5d}', '\u{a5d}'), + ('\u{a5f}', '\u{a65}'), + ('\u{a77}', '\u{a80}'), + ('\u{a84}', '\u{a84}'), + ('\u{a8e}', '\u{a8e}'), + ('\u{a92}', '\u{a92}'), + ('\u{aa9}', '\u{aa9}'), + ('\u{ab1}', '\u{ab1}'), + ('\u{ab4}', '\u{ab4}'), + ('\u{aba}', '\u{abb}'), + ('\u{ac6}', '\u{ac6}'), + ('\u{aca}', '\u{aca}'), + ('\u{ace}', '\u{acf}'), + ('\u{ad1}', '\u{adf}'), + ('\u{ae4}', '\u{ae5}'), + ('\u{af2}', '\u{af8}'), + ('\u{b00}', '\u{b00}'), + ('\u{b04}', '\u{b04}'), + ('\u{b0d}', '\u{b0e}'), + ('\u{b11}', '\u{b12}'), + ('\u{b29}', '\u{b29}'), + ('\u{b31}', '\u{b31}'), + ('\u{b34}', '\u{b34}'), + ('\u{b3a}', '\u{b3b}'), + ('\u{b45}', '\u{b46}'), + ('\u{b49}', '\u{b4a}'), + ('\u{b4e}', '\u{b54}'), + ('\u{b58}', '\u{b5b}'), + ('\u{b5e}', '\u{b5e}'), + ('\u{b64}', '\u{b65}'), + ('\u{b78}', '\u{b81}'), + ('\u{b84}', '\u{b84}'), + ('\u{b8b}', '\u{b8d}'), + ('\u{b91}', '\u{b91}'), + ('\u{b96}', '\u{b98}'), + ('\u{b9b}', '\u{b9b}'), + ('\u{b9d}', '\u{b9d}'), + ('\u{ba0}', '\u{ba2}'), + ('\u{ba5}', '\u{ba7}'), + ('\u{bab}', '\u{bad}'), + ('\u{bba}', '\u{bbd}'), + ('\u{bc3}', '\u{bc5}'), + ('\u{bc9}', '\u{bc9}'), + ('\u{bce}', '\u{bcf}'), + ('\u{bd1}', '\u{bd6}'), + ('\u{bd8}', '\u{be5}'), + ('\u{bfb}', '\u{bff}'), + ('\u{c0d}', '\u{c0d}'), + ('\u{c11}', '\u{c11}'), + ('\u{c29}', '\u{c29}'), + ('\u{c3a}', '\u{c3b}'), + ('\u{c45}', '\u{c45}'), + ('\u{c49}', '\u{c49}'), + ('\u{c4e}', '\u{c54}'), + ('\u{c57}', '\u{c57}'), + ('\u{c5b}', '\u{c5c}'), + ('\u{c5e}', '\u{c5f}'), + ('\u{c64}', '\u{c65}'), + ('\u{c70}', '\u{c76}'), + ('\u{c8d}', '\u{c8d}'), + ('\u{c91}', '\u{c91}'), + ('\u{ca9}', '\u{ca9}'), + ('\u{cb4}', '\u{cb4}'), + ('\u{cba}', '\u{cbb}'), + ('\u{cc5}', '\u{cc5}'), + ('\u{cc9}', '\u{cc9}'), + ('\u{cce}', '\u{cd4}'), + ('\u{cd7}', '\u{cdc}'), + ('\u{cdf}', '\u{cdf}'), + ('\u{ce4}', '\u{ce5}'), + ('\u{cf0}', '\u{cf0}'), + ('\u{cf4}', '\u{cff}'), + ('\u{d0d}', '\u{d0d}'), + ('\u{d11}', '\u{d11}'), + ('\u{d45}', '\u{d45}'), + ('\u{d49}', '\u{d49}'), + ('\u{d50}', '\u{d53}'), + ('\u{d64}', '\u{d65}'), + ('\u{d80}', '\u{d80}'), + ('\u{d84}', '\u{d84}'), + ('\u{d97}', '\u{d99}'), + ('\u{db2}', '\u{db2}'), + ('\u{dbc}', '\u{dbc}'), + ('\u{dbe}', '\u{dbf}'), + ('\u{dc7}', '\u{dc9}'), + ('\u{dcb}', '\u{dce}'), + ('\u{dd5}', '\u{dd5}'), + ('\u{dd7}', '\u{dd7}'), + ('\u{de0}', '\u{de5}'), + ('\u{df0}', '\u{df1}'), + ('\u{df5}', '\u{e00}'), + ('\u{e3b}', '\u{e3e}'), + ('\u{e5c}', '\u{e80}'), + ('\u{e83}', '\u{e83}'), + ('\u{e85}', '\u{e85}'), + ('\u{e8b}', '\u{e8b}'), + ('\u{ea4}', '\u{ea4}'), + ('\u{ea6}', '\u{ea6}'), + ('\u{ebe}', '\u{ebf}'), + ('\u{ec5}', '\u{ec5}'), + ('\u{ec7}', '\u{ec7}'), + ('\u{ecf}', '\u{ecf}'), + ('\u{eda}', '\u{edb}'), + ('\u{ee0}', '\u{eff}'), + ('\u{f48}', '\u{f48}'), + ('\u{f6d}', '\u{f70}'), + ('\u{f98}', '\u{f98}'), + ('\u{fbd}', '\u{fbd}'), + ('\u{fcd}', '\u{fcd}'), + ('\u{fdb}', '\u{fff}'), + ('\u{10c6}', '\u{10c6}'), + ('\u{10c8}', '\u{10cc}'), + ('\u{10ce}', '\u{10cf}'), + ('\u{1249}', '\u{1249}'), + ('\u{124e}', '\u{124f}'), + ('\u{1257}', '\u{1257}'), + ('\u{1259}', '\u{1259}'), + ('\u{125e}', '\u{125f}'), + ('\u{1289}', '\u{1289}'), + ('\u{128e}', '\u{128f}'), + ('\u{12b1}', '\u{12b1}'), + ('\u{12b6}', '\u{12b7}'), + ('\u{12bf}', '\u{12bf}'), + ('\u{12c1}', '\u{12c1}'), + ('\u{12c6}', '\u{12c7}'), + ('\u{12d7}', '\u{12d7}'), + ('\u{1311}', '\u{1311}'), + ('\u{1316}', '\u{1317}'), + ('\u{135b}', '\u{135c}'), + ('\u{137d}', '\u{137f}'), + ('\u{139a}', '\u{139f}'), + ('\u{13f6}', '\u{13f7}'), + ('\u{13fe}', '\u{13ff}'), + ('\u{169d}', '\u{169f}'), + ('\u{16f9}', '\u{16ff}'), + ('\u{1716}', '\u{171e}'), + ('\u{1737}', '\u{173f}'), + ('\u{1754}', '\u{175f}'), + ('\u{176d}', '\u{176d}'), + ('\u{1771}', '\u{1771}'), + ('\u{1774}', '\u{177f}'), + ('\u{17de}', '\u{17df}'), + ('\u{17ea}', '\u{17ef}'), + ('\u{17fa}', '\u{17ff}'), + ('\u{181a}', '\u{181f}'), + ('\u{1879}', '\u{187f}'), + ('\u{18ab}', '\u{18af}'), + ('\u{18f6}', '\u{18ff}'), + ('\u{191f}', '\u{191f}'), + ('\u{192c}', '\u{192f}'), + ('\u{193c}', '\u{193f}'), + ('\u{1941}', '\u{1943}'), + ('\u{196e}', '\u{196f}'), + ('\u{1975}', '\u{197f}'), + ('\u{19ac}', '\u{19af}'), + ('\u{19ca}', '\u{19cf}'), + ('\u{19db}', '\u{19dd}'), + ('\u{1a1c}', '\u{1a1d}'), + ('\u{1a5f}', '\u{1a5f}'), + ('\u{1a7d}', '\u{1a7e}'), + ('\u{1a8a}', '\u{1a8f}'), + ('\u{1a9a}', '\u{1a9f}'), + ('\u{1aae}', '\u{1aaf}'), + ('\u{1acf}', '\u{1aff}'), + ('\u{1b4d}', '\u{1b4d}'), + ('\u{1bf4}', '\u{1bfb}'), + ('\u{1c38}', '\u{1c3a}'), + ('\u{1c4a}', '\u{1c4c}'), + ('\u{1c8b}', '\u{1c8f}'), + ('\u{1cbb}', '\u{1cbc}'), + ('\u{1cc8}', '\u{1ccf}'), + ('\u{1cfb}', '\u{1cff}'), + ('\u{1f16}', '\u{1f17}'), + ('\u{1f1e}', '\u{1f1f}'), + ('\u{1f46}', '\u{1f47}'), + ('\u{1f4e}', '\u{1f4f}'), + ('\u{1f58}', '\u{1f58}'), + ('\u{1f5a}', '\u{1f5a}'), + ('\u{1f5c}', '\u{1f5c}'), + ('\u{1f5e}', '\u{1f5e}'), + ('\u{1f7e}', '\u{1f7f}'), + ('\u{1fb5}', '\u{1fb5}'), + ('\u{1fc5}', '\u{1fc5}'), + ('\u{1fd4}', '\u{1fd5}'), + ('\u{1fdc}', '\u{1fdc}'), + ('\u{1ff0}', '\u{1ff1}'), + ('\u{1ff5}', '\u{1ff5}'), + ('\u{1fff}', '\u{1fff}'), + ('\u{2065}', '\u{2065}'), + ('\u{2072}', '\u{2073}'), + ('\u{208f}', '\u{208f}'), + ('\u{209d}', '\u{209f}'), + ('\u{20c1}', '\u{20cf}'), + ('\u{20f1}', '\u{20ff}'), + ('\u{218c}', '\u{218f}'), + ('\u{242a}', '\u{243f}'), + ('\u{244b}', '\u{245f}'), + ('\u{2b74}', '\u{2b75}'), + ('\u{2b96}', '\u{2b96}'), + ('\u{2cf4}', '\u{2cf8}'), + ('\u{2d26}', '\u{2d26}'), + ('\u{2d28}', '\u{2d2c}'), + ('\u{2d2e}', '\u{2d2f}'), + ('\u{2d68}', '\u{2d6e}'), + ('\u{2d71}', '\u{2d7e}'), + ('\u{2d97}', '\u{2d9f}'), + ('\u{2da7}', '\u{2da7}'), + ('\u{2daf}', '\u{2daf}'), + ('\u{2db7}', '\u{2db7}'), + ('\u{2dbf}', '\u{2dbf}'), + ('\u{2dc7}', '\u{2dc7}'), + ('\u{2dcf}', '\u{2dcf}'), + ('\u{2dd7}', '\u{2dd7}'), + ('\u{2ddf}', '\u{2ddf}'), + ('\u{2e5e}', '\u{2e7f}'), + ('\u{2e9a}', '\u{2e9a}'), + ('\u{2ef4}', '\u{2eff}'), + ('\u{2fd6}', '\u{2fef}'), + ('\u{3040}', '\u{3040}'), + ('\u{3097}', '\u{3098}'), + ('\u{3100}', '\u{3104}'), + ('\u{3130}', '\u{3130}'), + ('\u{318f}', '\u{318f}'), + ('\u{31e6}', '\u{31ee}'), + ('\u{321f}', '\u{321f}'), + ('\u{a48d}', '\u{a48f}'), + ('\u{a4c7}', '\u{a4cf}'), + ('\u{a62c}', '\u{a63f}'), + ('\u{a6f8}', '\u{a6ff}'), + ('\u{a7ce}', '\u{a7cf}'), + ('\u{a7d2}', '\u{a7d2}'), + ('\u{a7d4}', '\u{a7d4}'), + ('\u{a7dd}', '\u{a7f1}'), + ('\u{a82d}', '\u{a82f}'), + ('\u{a83a}', '\u{a83f}'), + ('\u{a878}', '\u{a87f}'), + ('\u{a8c6}', '\u{a8cd}'), + ('\u{a8da}', '\u{a8df}'), + ('\u{a954}', '\u{a95e}'), + ('\u{a97d}', '\u{a97f}'), + ('\u{a9ce}', '\u{a9ce}'), + ('\u{a9da}', '\u{a9dd}'), + ('\u{a9ff}', '\u{a9ff}'), + ('\u{aa37}', '\u{aa3f}'), + ('\u{aa4e}', '\u{aa4f}'), + ('\u{aa5a}', '\u{aa5b}'), + ('\u{aac3}', '\u{aada}'), + ('\u{aaf7}', '\u{ab00}'), + ('\u{ab07}', '\u{ab08}'), + ('\u{ab0f}', '\u{ab10}'), + ('\u{ab17}', '\u{ab1f}'), + ('\u{ab27}', '\u{ab27}'), + ('\u{ab2f}', '\u{ab2f}'), + ('\u{ab6c}', '\u{ab6f}'), + ('\u{abee}', '\u{abef}'), + ('\u{abfa}', '\u{abff}'), + ('\u{d7a4}', '\u{d7af}'), + ('\u{d7c7}', '\u{d7ca}'), + ('\u{d7fc}', '\u{d7ff}'), + ('\u{fa6e}', '\u{fa6f}'), + ('\u{fada}', '\u{faff}'), + ('\u{fb07}', '\u{fb12}'), + ('\u{fb18}', '\u{fb1c}'), + ('\u{fb37}', '\u{fb37}'), + ('\u{fb3d}', '\u{fb3d}'), + ('\u{fb3f}', '\u{fb3f}'), + ('\u{fb42}', '\u{fb42}'), + ('\u{fb45}', '\u{fb45}'), + ('\u{fbc3}', '\u{fbd2}'), + ('\u{fd90}', '\u{fd91}'), + ('\u{fdc8}', '\u{fdce}'), + ('\u{fdd0}', '\u{fdef}'), + ('\u{fe1a}', '\u{fe1f}'), + ('\u{fe53}', '\u{fe53}'), + ('\u{fe67}', '\u{fe67}'), + ('\u{fe6c}', '\u{fe6f}'), + ('\u{fe75}', '\u{fe75}'), + ('\u{fefd}', '\u{fefe}'), + ('\u{ff00}', '\u{ff00}'), + ('\u{ffbf}', '\u{ffc1}'), + ('\u{ffc8}', '\u{ffc9}'), + ('\u{ffd0}', '\u{ffd1}'), + ('\u{ffd8}', '\u{ffd9}'), + ('\u{ffdd}', '\u{ffdf}'), + ('\u{ffe7}', '\u{ffe7}'), + ('\u{ffef}', '\u{fff8}'), + ('\u{fffe}', '\u{ffff}'), + ('\u{1000c}', '\u{1000c}'), + ('\u{10027}', '\u{10027}'), + ('\u{1003b}', '\u{1003b}'), + ('\u{1003e}', '\u{1003e}'), + ('\u{1004e}', '\u{1004f}'), + ('\u{1005e}', '\u{1007f}'), + ('\u{100fb}', '\u{100ff}'), + ('\u{10103}', '\u{10106}'), + ('\u{10134}', '\u{10136}'), + ('\u{1018f}', '\u{1018f}'), + ('\u{1019d}', '\u{1019f}'), + ('\u{101a1}', '\u{101cf}'), + ('\u{101fe}', '\u{1027f}'), + ('\u{1029d}', '\u{1029f}'), + ('\u{102d1}', '\u{102df}'), + ('\u{102fc}', '\u{102ff}'), + ('\u{10324}', '\u{1032c}'), + ('\u{1034b}', '\u{1034f}'), + ('\u{1037b}', '\u{1037f}'), + ('\u{1039e}', '\u{1039e}'), + ('\u{103c4}', '\u{103c7}'), + ('\u{103d6}', '\u{103ff}'), + ('\u{1049e}', '\u{1049f}'), + ('\u{104aa}', '\u{104af}'), + ('\u{104d4}', '\u{104d7}'), + ('\u{104fc}', '\u{104ff}'), + ('\u{10528}', '\u{1052f}'), + ('\u{10564}', '\u{1056e}'), + ('\u{1057b}', '\u{1057b}'), + ('\u{1058b}', '\u{1058b}'), + ('\u{10593}', '\u{10593}'), + ('\u{10596}', '\u{10596}'), + ('\u{105a2}', '\u{105a2}'), + ('\u{105b2}', '\u{105b2}'), + ('\u{105ba}', '\u{105ba}'), + ('\u{105bd}', '\u{105bf}'), + ('\u{105f4}', '\u{105ff}'), + ('\u{10737}', '\u{1073f}'), + ('\u{10756}', '\u{1075f}'), + ('\u{10768}', '\u{1077f}'), + ('\u{10786}', '\u{10786}'), + ('\u{107b1}', '\u{107b1}'), + ('\u{107bb}', '\u{107ff}'), + ('\u{10806}', '\u{10807}'), + ('\u{10809}', '\u{10809}'), + ('\u{10836}', '\u{10836}'), + ('\u{10839}', '\u{1083b}'), + ('\u{1083d}', '\u{1083e}'), + ('\u{10856}', '\u{10856}'), + ('\u{1089f}', '\u{108a6}'), + ('\u{108b0}', '\u{108df}'), + ('\u{108f3}', '\u{108f3}'), + ('\u{108f6}', '\u{108fa}'), + ('\u{1091c}', '\u{1091e}'), + ('\u{1093a}', '\u{1093e}'), + ('\u{10940}', '\u{1097f}'), + ('\u{109b8}', '\u{109bb}'), + ('\u{109d0}', '\u{109d1}'), + ('\u{10a04}', '\u{10a04}'), + ('\u{10a07}', '\u{10a0b}'), + ('\u{10a14}', '\u{10a14}'), + ('\u{10a18}', '\u{10a18}'), + ('\u{10a36}', '\u{10a37}'), + ('\u{10a3b}', '\u{10a3e}'), + ('\u{10a49}', '\u{10a4f}'), + ('\u{10a59}', '\u{10a5f}'), + ('\u{10aa0}', '\u{10abf}'), + ('\u{10ae7}', '\u{10aea}'), + ('\u{10af7}', '\u{10aff}'), + ('\u{10b36}', '\u{10b38}'), + ('\u{10b56}', '\u{10b57}'), + ('\u{10b73}', '\u{10b77}'), + ('\u{10b92}', '\u{10b98}'), + ('\u{10b9d}', '\u{10ba8}'), + ('\u{10bb0}', '\u{10bff}'), + ('\u{10c49}', '\u{10c7f}'), + ('\u{10cb3}', '\u{10cbf}'), + ('\u{10cf3}', '\u{10cf9}'), + ('\u{10d28}', '\u{10d2f}'), + ('\u{10d3a}', '\u{10d3f}'), + ('\u{10d66}', '\u{10d68}'), + ('\u{10d86}', '\u{10d8d}'), + ('\u{10d90}', '\u{10e5f}'), + ('\u{10e7f}', '\u{10e7f}'), + ('\u{10eaa}', '\u{10eaa}'), + ('\u{10eae}', '\u{10eaf}'), + ('\u{10eb2}', '\u{10ec1}'), + ('\u{10ec5}', '\u{10efb}'), + ('\u{10f28}', '\u{10f2f}'), + ('\u{10f5a}', '\u{10f6f}'), + ('\u{10f8a}', '\u{10faf}'), + ('\u{10fcc}', '\u{10fdf}'), + ('\u{10ff7}', '\u{10fff}'), + ('\u{1104e}', '\u{11051}'), + ('\u{11076}', '\u{1107e}'), + ('\u{110c3}', '\u{110cc}'), + ('\u{110ce}', '\u{110cf}'), + ('\u{110e9}', '\u{110ef}'), + ('\u{110fa}', '\u{110ff}'), + ('\u{11135}', '\u{11135}'), + ('\u{11148}', '\u{1114f}'), + ('\u{11177}', '\u{1117f}'), + ('\u{111e0}', '\u{111e0}'), + ('\u{111f5}', '\u{111ff}'), + ('\u{11212}', '\u{11212}'), + ('\u{11242}', '\u{1127f}'), + ('\u{11287}', '\u{11287}'), + ('\u{11289}', '\u{11289}'), + ('\u{1128e}', '\u{1128e}'), + ('\u{1129e}', '\u{1129e}'), + ('\u{112aa}', '\u{112af}'), + ('\u{112eb}', '\u{112ef}'), + ('\u{112fa}', '\u{112ff}'), + ('\u{11304}', '\u{11304}'), + ('\u{1130d}', '\u{1130e}'), + ('\u{11311}', '\u{11312}'), + ('\u{11329}', '\u{11329}'), + ('\u{11331}', '\u{11331}'), + ('\u{11334}', '\u{11334}'), + ('\u{1133a}', '\u{1133a}'), + ('\u{11345}', '\u{11346}'), + ('\u{11349}', '\u{1134a}'), + ('\u{1134e}', '\u{1134f}'), + ('\u{11351}', '\u{11356}'), + ('\u{11358}', '\u{1135c}'), + ('\u{11364}', '\u{11365}'), + ('\u{1136d}', '\u{1136f}'), + ('\u{11375}', '\u{1137f}'), + ('\u{1138a}', '\u{1138a}'), + ('\u{1138c}', '\u{1138d}'), + ('\u{1138f}', '\u{1138f}'), + ('\u{113b6}', '\u{113b6}'), + ('\u{113c1}', '\u{113c1}'), + ('\u{113c3}', '\u{113c4}'), + ('\u{113c6}', '\u{113c6}'), + ('\u{113cb}', '\u{113cb}'), + ('\u{113d6}', '\u{113d6}'), + ('\u{113d9}', '\u{113e0}'), + ('\u{113e3}', '\u{113ff}'), + ('\u{1145c}', '\u{1145c}'), + ('\u{11462}', '\u{1147f}'), + ('\u{114c8}', '\u{114cf}'), + ('\u{114da}', '\u{1157f}'), + ('\u{115b6}', '\u{115b7}'), + ('\u{115de}', '\u{115ff}'), + ('\u{11645}', '\u{1164f}'), + ('\u{1165a}', '\u{1165f}'), + ('\u{1166d}', '\u{1167f}'), + ('\u{116ba}', '\u{116bf}'), + ('\u{116ca}', '\u{116cf}'), + ('\u{116e4}', '\u{116ff}'), + ('\u{1171b}', '\u{1171c}'), + ('\u{1172c}', '\u{1172f}'), + ('\u{11747}', '\u{117ff}'), + ('\u{1183c}', '\u{1189f}'), + ('\u{118f3}', '\u{118fe}'), + ('\u{11907}', '\u{11908}'), + ('\u{1190a}', '\u{1190b}'), + ('\u{11914}', '\u{11914}'), + ('\u{11917}', '\u{11917}'), + ('\u{11936}', '\u{11936}'), + ('\u{11939}', '\u{1193a}'), + ('\u{11947}', '\u{1194f}'), + ('\u{1195a}', '\u{1199f}'), + ('\u{119a8}', '\u{119a9}'), + ('\u{119d8}', '\u{119d9}'), + ('\u{119e5}', '\u{119ff}'), + ('\u{11a48}', '\u{11a4f}'), + ('\u{11aa3}', '\u{11aaf}'), + ('\u{11af9}', '\u{11aff}'), + ('\u{11b0a}', '\u{11bbf}'), + ('\u{11be2}', '\u{11bef}'), + ('\u{11bfa}', '\u{11bff}'), + ('\u{11c09}', '\u{11c09}'), + ('\u{11c37}', '\u{11c37}'), + ('\u{11c46}', '\u{11c4f}'), + ('\u{11c6d}', '\u{11c6f}'), + ('\u{11c90}', '\u{11c91}'), + ('\u{11ca8}', '\u{11ca8}'), + ('\u{11cb7}', '\u{11cff}'), + ('\u{11d07}', '\u{11d07}'), + ('\u{11d0a}', '\u{11d0a}'), + ('\u{11d37}', '\u{11d39}'), + ('\u{11d3b}', '\u{11d3b}'), + ('\u{11d3e}', '\u{11d3e}'), + ('\u{11d48}', '\u{11d4f}'), + ('\u{11d5a}', '\u{11d5f}'), + ('\u{11d66}', '\u{11d66}'), + ('\u{11d69}', '\u{11d69}'), + ('\u{11d8f}', '\u{11d8f}'), + ('\u{11d92}', '\u{11d92}'), + ('\u{11d99}', '\u{11d9f}'), + ('\u{11daa}', '\u{11edf}'), + ('\u{11ef9}', '\u{11eff}'), + ('\u{11f11}', '\u{11f11}'), + ('\u{11f3b}', '\u{11f3d}'), + ('\u{11f5b}', '\u{11faf}'), + ('\u{11fb1}', '\u{11fbf}'), + ('\u{11ff2}', '\u{11ffe}'), + ('\u{1239a}', '\u{123ff}'), + ('\u{1246f}', '\u{1246f}'), + ('\u{12475}', '\u{1247f}'), + ('\u{12544}', '\u{12f8f}'), + ('\u{12ff3}', '\u{12fff}'), + ('\u{13456}', '\u{1345f}'), + ('\u{143fb}', '\u{143ff}'), + ('\u{14647}', '\u{160ff}'), + ('\u{1613a}', '\u{167ff}'), + ('\u{16a39}', '\u{16a3f}'), + ('\u{16a5f}', '\u{16a5f}'), + ('\u{16a6a}', '\u{16a6d}'), + ('\u{16abf}', '\u{16abf}'), + ('\u{16aca}', '\u{16acf}'), + ('\u{16aee}', '\u{16aef}'), + ('\u{16af6}', '\u{16aff}'), + ('\u{16b46}', '\u{16b4f}'), + ('\u{16b5a}', '\u{16b5a}'), + ('\u{16b62}', '\u{16b62}'), + ('\u{16b78}', '\u{16b7c}'), + ('\u{16b90}', '\u{16d3f}'), + ('\u{16d7a}', '\u{16e3f}'), + ('\u{16e9b}', '\u{16eff}'), + ('\u{16f4b}', '\u{16f4e}'), + ('\u{16f88}', '\u{16f8e}'), + ('\u{16fa0}', '\u{16fdf}'), + ('\u{16fe5}', '\u{16fef}'), + ('\u{16ff2}', '\u{16fff}'), + ('\u{187f8}', '\u{187ff}'), + ('\u{18cd6}', '\u{18cfe}'), + ('\u{18d09}', '\u{1afef}'), + ('\u{1aff4}', '\u{1aff4}'), + ('\u{1affc}', '\u{1affc}'), + ('\u{1afff}', '\u{1afff}'), + ('\u{1b123}', '\u{1b131}'), + ('\u{1b133}', '\u{1b14f}'), + ('\u{1b153}', '\u{1b154}'), + ('\u{1b156}', '\u{1b163}'), + ('\u{1b168}', '\u{1b16f}'), + ('\u{1b2fc}', '\u{1bbff}'), + ('\u{1bc6b}', '\u{1bc6f}'), + ('\u{1bc7d}', '\u{1bc7f}'), + ('\u{1bc89}', '\u{1bc8f}'), + ('\u{1bc9a}', '\u{1bc9b}'), + ('\u{1bca4}', '\u{1cbff}'), + ('\u{1ccfa}', '\u{1ccff}'), + ('\u{1ceb4}', '\u{1ceff}'), + ('\u{1cf2e}', '\u{1cf2f}'), + ('\u{1cf47}', '\u{1cf4f}'), + ('\u{1cfc4}', '\u{1cfff}'), + ('\u{1d0f6}', '\u{1d0ff}'), + ('\u{1d127}', '\u{1d128}'), + ('\u{1d1eb}', '\u{1d1ff}'), + ('\u{1d246}', '\u{1d2bf}'), + ('\u{1d2d4}', '\u{1d2df}'), + ('\u{1d2f4}', '\u{1d2ff}'), + ('\u{1d357}', '\u{1d35f}'), + ('\u{1d379}', '\u{1d3ff}'), + ('\u{1d455}', '\u{1d455}'), + ('\u{1d49d}', '\u{1d49d}'), + ('\u{1d4a0}', '\u{1d4a1}'), + ('\u{1d4a3}', '\u{1d4a4}'), + ('\u{1d4a7}', '\u{1d4a8}'), + ('\u{1d4ad}', '\u{1d4ad}'), + ('\u{1d4ba}', '\u{1d4ba}'), + ('\u{1d4bc}', '\u{1d4bc}'), + ('\u{1d4c4}', '\u{1d4c4}'), + ('\u{1d506}', '\u{1d506}'), + ('\u{1d50b}', '\u{1d50c}'), + ('\u{1d515}', '\u{1d515}'), + ('\u{1d51d}', '\u{1d51d}'), + ('\u{1d53a}', '\u{1d53a}'), + ('\u{1d53f}', '\u{1d53f}'), + ('\u{1d545}', '\u{1d545}'), + ('\u{1d547}', '\u{1d549}'), + ('\u{1d551}', '\u{1d551}'), + ('\u{1d6a6}', '\u{1d6a7}'), + ('\u{1d7cc}', '\u{1d7cd}'), + ('\u{1da8c}', '\u{1da9a}'), + ('\u{1daa0}', '\u{1daa0}'), + ('\u{1dab0}', '\u{1deff}'), + ('\u{1df1f}', '\u{1df24}'), + ('\u{1df2b}', '\u{1dfff}'), + ('\u{1e007}', '\u{1e007}'), + ('\u{1e019}', '\u{1e01a}'), + ('\u{1e022}', '\u{1e022}'), + ('\u{1e025}', '\u{1e025}'), + ('\u{1e02b}', '\u{1e02f}'), + ('\u{1e06e}', '\u{1e08e}'), + ('\u{1e090}', '\u{1e0ff}'), + ('\u{1e12d}', '\u{1e12f}'), + ('\u{1e13e}', '\u{1e13f}'), + ('\u{1e14a}', '\u{1e14d}'), + ('\u{1e150}', '\u{1e28f}'), + ('\u{1e2af}', '\u{1e2bf}'), + ('\u{1e2fa}', '\u{1e2fe}'), + ('\u{1e300}', '\u{1e4cf}'), + ('\u{1e4fa}', '\u{1e5cf}'), + ('\u{1e5fb}', '\u{1e5fe}'), + ('\u{1e600}', '\u{1e7df}'), + ('\u{1e7e7}', '\u{1e7e7}'), + ('\u{1e7ec}', '\u{1e7ec}'), + ('\u{1e7ef}', '\u{1e7ef}'), + ('\u{1e7ff}', '\u{1e7ff}'), + ('\u{1e8c5}', '\u{1e8c6}'), + ('\u{1e8d7}', '\u{1e8ff}'), + ('\u{1e94c}', '\u{1e94f}'), + ('\u{1e95a}', '\u{1e95d}'), + ('\u{1e960}', '\u{1ec70}'), + ('\u{1ecb5}', '\u{1ed00}'), + ('\u{1ed3e}', '\u{1edff}'), + ('\u{1ee04}', '\u{1ee04}'), + ('\u{1ee20}', '\u{1ee20}'), + ('\u{1ee23}', '\u{1ee23}'), + ('\u{1ee25}', '\u{1ee26}'), + ('\u{1ee28}', '\u{1ee28}'), + ('\u{1ee33}', '\u{1ee33}'), + ('\u{1ee38}', '\u{1ee38}'), + ('\u{1ee3a}', '\u{1ee3a}'), + ('\u{1ee3c}', '\u{1ee41}'), + ('\u{1ee43}', '\u{1ee46}'), + ('\u{1ee48}', '\u{1ee48}'), + ('\u{1ee4a}', '\u{1ee4a}'), + ('\u{1ee4c}', '\u{1ee4c}'), + ('\u{1ee50}', '\u{1ee50}'), + ('\u{1ee53}', '\u{1ee53}'), + ('\u{1ee55}', '\u{1ee56}'), + ('\u{1ee58}', '\u{1ee58}'), + ('\u{1ee5a}', '\u{1ee5a}'), + ('\u{1ee5c}', '\u{1ee5c}'), + ('\u{1ee5e}', '\u{1ee5e}'), + ('\u{1ee60}', '\u{1ee60}'), + ('\u{1ee63}', '\u{1ee63}'), + ('\u{1ee65}', '\u{1ee66}'), + ('\u{1ee6b}', '\u{1ee6b}'), + ('\u{1ee73}', '\u{1ee73}'), + ('\u{1ee78}', '\u{1ee78}'), + ('\u{1ee7d}', '\u{1ee7d}'), + ('\u{1ee7f}', '\u{1ee7f}'), + ('\u{1ee8a}', '\u{1ee8a}'), + ('\u{1ee9c}', '\u{1eea0}'), + ('\u{1eea4}', '\u{1eea4}'), + ('\u{1eeaa}', '\u{1eeaa}'), + ('\u{1eebc}', '\u{1eeef}'), + ('\u{1eef2}', '\u{1efff}'), + ('\u{1f02c}', '\u{1f02f}'), + ('\u{1f094}', '\u{1f09f}'), + ('\u{1f0af}', '\u{1f0b0}'), + ('\u{1f0c0}', '\u{1f0c0}'), + ('\u{1f0d0}', '\u{1f0d0}'), + ('\u{1f0f6}', '\u{1f0ff}'), + ('\u{1f1ae}', '\u{1f1e5}'), + ('\u{1f203}', '\u{1f20f}'), + ('\u{1f23c}', '\u{1f23f}'), + ('\u{1f249}', '\u{1f24f}'), + ('\u{1f252}', '\u{1f25f}'), + ('\u{1f266}', '\u{1f2ff}'), + ('\u{1f6d8}', '\u{1f6db}'), + ('\u{1f6ed}', '\u{1f6ef}'), + ('\u{1f6fd}', '\u{1f6ff}'), + ('\u{1f777}', '\u{1f77a}'), + ('\u{1f7da}', '\u{1f7df}'), + ('\u{1f7ec}', '\u{1f7ef}'), + ('\u{1f7f1}', '\u{1f7ff}'), + ('\u{1f80c}', '\u{1f80f}'), + ('\u{1f848}', '\u{1f84f}'), + ('\u{1f85a}', '\u{1f85f}'), + ('\u{1f888}', '\u{1f88f}'), + ('\u{1f8ae}', '\u{1f8af}'), + ('\u{1f8bc}', '\u{1f8bf}'), + ('\u{1f8c2}', '\u{1f8ff}'), + ('\u{1fa54}', '\u{1fa5f}'), + ('\u{1fa6e}', '\u{1fa6f}'), + ('\u{1fa7d}', '\u{1fa7f}'), + ('\u{1fa8a}', '\u{1fa8e}'), + ('\u{1fac7}', '\u{1facd}'), + ('\u{1fadd}', '\u{1fade}'), + ('\u{1faea}', '\u{1faef}'), + ('\u{1faf9}', '\u{1faff}'), + ('\u{1fb93}', '\u{1fb93}'), + ('\u{1fbfa}', '\u{1ffff}'), + ('\u{2a6e0}', '\u{2a6ff}'), + ('\u{2b73a}', '\u{2b73f}'), + ('\u{2b81e}', '\u{2b81f}'), + ('\u{2cea2}', '\u{2ceaf}'), + ('\u{2ebe1}', '\u{2ebef}'), + ('\u{2ee5e}', '\u{2f7ff}'), + ('\u{2fa1e}', '\u{2ffff}'), + ('\u{3134b}', '\u{3134f}'), + ('\u{323b0}', '\u{e0000}'), + ('\u{e0002}', '\u{e001f}'), + ('\u{e0080}', '\u{e00ff}'), + ('\u{e01f0}', '\u{effff}'), + ('\u{ffffe}', '\u{fffff}'), + ('\u{10fffe}', '\u{10ffff}'), +]; + +pub const UPPERCASE_LETTER: &'static [(char, char)] = &[ + ('A', 'Z'), + ('À', 'Ö'), + ('Ø', 'Þ'), + ('Ā', 'Ā'), + ('Ă', 'Ă'), + ('Ą', 'Ą'), + ('Ć', 'Ć'), + ('Ĉ', 'Ĉ'), + ('Ċ', 'Ċ'), + ('Č', 'Č'), + ('Ď', 'Ď'), + ('Đ', 'Đ'), + ('Ē', 'Ē'), + ('Ĕ', 'Ĕ'), + ('Ė', 'Ė'), + ('Ę', 'Ę'), + ('Ě', 'Ě'), + ('Ĝ', 'Ĝ'), + ('Ğ', 'Ğ'), + ('Ġ', 'Ġ'), + ('Ģ', 'Ģ'), + ('Ĥ', 'Ĥ'), + ('Ħ', 'Ħ'), + ('Ĩ', 'Ĩ'), + ('Ī', 'Ī'), + ('Ĭ', 'Ĭ'), + ('Į', 'Į'), + ('İ', 'İ'), + ('IJ', 'IJ'), + ('Ĵ', 'Ĵ'), + ('Ķ', 'Ķ'), + ('Ĺ', 'Ĺ'), + ('Ļ', 'Ļ'), + ('Ľ', 'Ľ'), + ('Ŀ', 'Ŀ'), + ('Ł', 'Ł'), + ('Ń', 'Ń'), + ('Ņ', 'Ņ'), + ('Ň', 'Ň'), + ('Ŋ', 'Ŋ'), + ('Ō', 'Ō'), + ('Ŏ', 'Ŏ'), + ('Ő', 'Ő'), + ('Œ', 'Œ'), + ('Ŕ', 'Ŕ'), + ('Ŗ', 'Ŗ'), + ('Ř', 'Ř'), + ('Ś', 'Ś'), + ('Ŝ', 'Ŝ'), + ('Ş', 'Ş'), + ('Š', 'Š'), + ('Ţ', 'Ţ'), + ('Ť', 'Ť'), + ('Ŧ', 'Ŧ'), + ('Ũ', 'Ũ'), + ('Ū', 'Ū'), + ('Ŭ', 'Ŭ'), + ('Ů', 'Ů'), + ('Ű', 'Ű'), + ('Ų', 'Ų'), + ('Ŵ', 'Ŵ'), + ('Ŷ', 'Ŷ'), + ('Ÿ', 'Ź'), + ('Ż', 'Ż'), + ('Ž', 'Ž'), + ('Ɓ', 'Ƃ'), + ('Ƅ', 'Ƅ'), + ('Ɔ', 'Ƈ'), + ('Ɖ', 'Ƌ'), + ('Ǝ', 'Ƒ'), + ('Ɠ', 'Ɣ'), + ('Ɩ', 'Ƙ'), + ('Ɯ', 'Ɲ'), + ('Ɵ', 'Ơ'), + ('Ƣ', 'Ƣ'), + ('Ƥ', 'Ƥ'), + ('Ʀ', 'Ƨ'), + ('Ʃ', 'Ʃ'), + ('Ƭ', 'Ƭ'), + ('Ʈ', 'Ư'), + ('Ʊ', 'Ƴ'), + ('Ƶ', 'Ƶ'), + ('Ʒ', 'Ƹ'), + ('Ƽ', 'Ƽ'), + ('DŽ', 'DŽ'), + ('LJ', 'LJ'), + ('NJ', 'NJ'), + ('Ǎ', 'Ǎ'), + ('Ǐ', 'Ǐ'), + ('Ǒ', 'Ǒ'), + ('Ǔ', 'Ǔ'), + ('Ǖ', 'Ǖ'), + ('Ǘ', 'Ǘ'), + ('Ǚ', 'Ǚ'), + ('Ǜ', 'Ǜ'), + ('Ǟ', 'Ǟ'), + ('Ǡ', 'Ǡ'), + ('Ǣ', 'Ǣ'), + ('Ǥ', 'Ǥ'), + ('Ǧ', 'Ǧ'), + ('Ǩ', 'Ǩ'), + ('Ǫ', 'Ǫ'), + ('Ǭ', 'Ǭ'), + ('Ǯ', 'Ǯ'), + ('DZ', 'DZ'), + ('Ǵ', 'Ǵ'), + ('Ƕ', 'Ǹ'), + ('Ǻ', 'Ǻ'), + ('Ǽ', 'Ǽ'), + ('Ǿ', 'Ǿ'), + ('Ȁ', 'Ȁ'), + ('Ȃ', 'Ȃ'), + ('Ȅ', 'Ȅ'), + ('Ȇ', 'Ȇ'), + ('Ȉ', 'Ȉ'), + ('Ȋ', 'Ȋ'), + ('Ȍ', 'Ȍ'), + ('Ȏ', 'Ȏ'), + ('Ȑ', 'Ȑ'), + ('Ȓ', 'Ȓ'), + ('Ȕ', 'Ȕ'), + ('Ȗ', 'Ȗ'), + ('Ș', 'Ș'), + ('Ț', 'Ț'), + ('Ȝ', 'Ȝ'), + ('Ȟ', 'Ȟ'), + ('Ƞ', 'Ƞ'), + ('Ȣ', 'Ȣ'), + ('Ȥ', 'Ȥ'), + ('Ȧ', 'Ȧ'), + ('Ȩ', 'Ȩ'), + ('Ȫ', 'Ȫ'), + ('Ȭ', 'Ȭ'), + ('Ȯ', 'Ȯ'), + ('Ȱ', 'Ȱ'), + ('Ȳ', 'Ȳ'), + ('Ⱥ', 'Ȼ'), + ('Ƚ', 'Ⱦ'), + ('Ɂ', 'Ɂ'), + ('Ƀ', 'Ɇ'), + ('Ɉ', 'Ɉ'), + ('Ɋ', 'Ɋ'), + ('Ɍ', 'Ɍ'), + ('Ɏ', 'Ɏ'), + ('Ͱ', 'Ͱ'), + ('Ͳ', 'Ͳ'), + ('Ͷ', 'Ͷ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ώ'), + ('Α', 'Ρ'), + ('Σ', 'Ϋ'), + ('Ϗ', 'Ϗ'), + ('ϒ', 'ϔ'), + ('Ϙ', 'Ϙ'), + ('Ϛ', 'Ϛ'), + ('Ϝ', 'Ϝ'), + ('Ϟ', 'Ϟ'), + ('Ϡ', 'Ϡ'), + ('Ϣ', 'Ϣ'), + ('Ϥ', 'Ϥ'), + ('Ϧ', 'Ϧ'), + ('Ϩ', 'Ϩ'), + ('Ϫ', 'Ϫ'), + ('Ϭ', 'Ϭ'), + ('Ϯ', 'Ϯ'), + ('ϴ', 'ϴ'), + ('Ϸ', 'Ϸ'), + ('Ϲ', 'Ϻ'), + ('Ͻ', 'Я'), + ('Ѡ', 'Ѡ'), + ('Ѣ', 'Ѣ'), + ('Ѥ', 'Ѥ'), + ('Ѧ', 'Ѧ'), + ('Ѩ', 'Ѩ'), + ('Ѫ', 'Ѫ'), + ('Ѭ', 'Ѭ'), + ('Ѯ', 'Ѯ'), + ('Ѱ', 'Ѱ'), + ('Ѳ', 'Ѳ'), + ('Ѵ', 'Ѵ'), + ('Ѷ', 'Ѷ'), + ('Ѹ', 'Ѹ'), + ('Ѻ', 'Ѻ'), + ('Ѽ', 'Ѽ'), + ('Ѿ', 'Ѿ'), + ('Ҁ', 'Ҁ'), + ('Ҋ', 'Ҋ'), + ('Ҍ', 'Ҍ'), + ('Ҏ', 'Ҏ'), + ('Ґ', 'Ґ'), + ('Ғ', 'Ғ'), + ('Ҕ', 'Ҕ'), + ('Җ', 'Җ'), + ('Ҙ', 'Ҙ'), + ('Қ', 'Қ'), + ('Ҝ', 'Ҝ'), + ('Ҟ', 'Ҟ'), + ('Ҡ', 'Ҡ'), + ('Ң', 'Ң'), + ('Ҥ', 'Ҥ'), + ('Ҧ', 'Ҧ'), + ('Ҩ', 'Ҩ'), + ('Ҫ', 'Ҫ'), + ('Ҭ', 'Ҭ'), + ('Ү', 'Ү'), + ('Ұ', 'Ұ'), + ('Ҳ', 'Ҳ'), + ('Ҵ', 'Ҵ'), + ('Ҷ', 'Ҷ'), + ('Ҹ', 'Ҹ'), + ('Һ', 'Һ'), + ('Ҽ', 'Ҽ'), + ('Ҿ', 'Ҿ'), + ('Ӏ', 'Ӂ'), + ('Ӄ', 'Ӄ'), + ('Ӆ', 'Ӆ'), + ('Ӈ', 'Ӈ'), + ('Ӊ', 'Ӊ'), + ('Ӌ', 'Ӌ'), + ('Ӎ', 'Ӎ'), + ('Ӑ', 'Ӑ'), + ('Ӓ', 'Ӓ'), + ('Ӕ', 'Ӕ'), + ('Ӗ', 'Ӗ'), + ('Ә', 'Ә'), + ('Ӛ', 'Ӛ'), + ('Ӝ', 'Ӝ'), + ('Ӟ', 'Ӟ'), + ('Ӡ', 'Ӡ'), + ('Ӣ', 'Ӣ'), + ('Ӥ', 'Ӥ'), + ('Ӧ', 'Ӧ'), + ('Ө', 'Ө'), + ('Ӫ', 'Ӫ'), + ('Ӭ', 'Ӭ'), + ('Ӯ', 'Ӯ'), + ('Ӱ', 'Ӱ'), + ('Ӳ', 'Ӳ'), + ('Ӵ', 'Ӵ'), + ('Ӷ', 'Ӷ'), + ('Ӹ', 'Ӹ'), + ('Ӻ', 'Ӻ'), + ('Ӽ', 'Ӽ'), + ('Ӿ', 'Ӿ'), + ('Ԁ', 'Ԁ'), + ('Ԃ', 'Ԃ'), + ('Ԅ', 'Ԅ'), + ('Ԇ', 'Ԇ'), + ('Ԉ', 'Ԉ'), + ('Ԋ', 'Ԋ'), + ('Ԍ', 'Ԍ'), + ('Ԏ', 'Ԏ'), + ('Ԑ', 'Ԑ'), + ('Ԓ', 'Ԓ'), + ('Ԕ', 'Ԕ'), + ('Ԗ', 'Ԗ'), + ('Ԙ', 'Ԙ'), + ('Ԛ', 'Ԛ'), + ('Ԝ', 'Ԝ'), + ('Ԟ', 'Ԟ'), + ('Ԡ', 'Ԡ'), + ('Ԣ', 'Ԣ'), + ('Ԥ', 'Ԥ'), + ('Ԧ', 'Ԧ'), + ('Ԩ', 'Ԩ'), + ('Ԫ', 'Ԫ'), + ('Ԭ', 'Ԭ'), + ('Ԯ', 'Ԯ'), + ('Ա', 'Ֆ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('Ꭰ', 'Ᏽ'), + ('Ᲊ', 'Ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('Ḁ', 'Ḁ'), + ('Ḃ', 'Ḃ'), + ('Ḅ', 'Ḅ'), + ('Ḇ', 'Ḇ'), + ('Ḉ', 'Ḉ'), + ('Ḋ', 'Ḋ'), + ('Ḍ', 'Ḍ'), + ('Ḏ', 'Ḏ'), + ('Ḑ', 'Ḑ'), + ('Ḓ', 'Ḓ'), + ('Ḕ', 'Ḕ'), + ('Ḗ', 'Ḗ'), + ('Ḙ', 'Ḙ'), + ('Ḛ', 'Ḛ'), + ('Ḝ', 'Ḝ'), + ('Ḟ', 'Ḟ'), + ('Ḡ', 'Ḡ'), + ('Ḣ', 'Ḣ'), + ('Ḥ', 'Ḥ'), + ('Ḧ', 'Ḧ'), + ('Ḩ', 'Ḩ'), + ('Ḫ', 'Ḫ'), + ('Ḭ', 'Ḭ'), + ('Ḯ', 'Ḯ'), + ('Ḱ', 'Ḱ'), + ('Ḳ', 'Ḳ'), + ('Ḵ', 'Ḵ'), + ('Ḷ', 'Ḷ'), + ('Ḹ', 'Ḹ'), + ('Ḻ', 'Ḻ'), + ('Ḽ', 'Ḽ'), + ('Ḿ', 'Ḿ'), + ('Ṁ', 'Ṁ'), + ('Ṃ', 'Ṃ'), + ('Ṅ', 'Ṅ'), + ('Ṇ', 'Ṇ'), + ('Ṉ', 'Ṉ'), + ('Ṋ', 'Ṋ'), + ('Ṍ', 'Ṍ'), + ('Ṏ', 'Ṏ'), + ('Ṑ', 'Ṑ'), + ('Ṓ', 'Ṓ'), + ('Ṕ', 'Ṕ'), + ('Ṗ', 'Ṗ'), + ('Ṙ', 'Ṙ'), + ('Ṛ', 'Ṛ'), + ('Ṝ', 'Ṝ'), + ('Ṟ', 'Ṟ'), + ('Ṡ', 'Ṡ'), + ('Ṣ', 'Ṣ'), + ('Ṥ', 'Ṥ'), + ('Ṧ', 'Ṧ'), + ('Ṩ', 'Ṩ'), + ('Ṫ', 'Ṫ'), + ('Ṭ', 'Ṭ'), + ('Ṯ', 'Ṯ'), + ('Ṱ', 'Ṱ'), + ('Ṳ', 'Ṳ'), + ('Ṵ', 'Ṵ'), + ('Ṷ', 'Ṷ'), + ('Ṹ', 'Ṹ'), + ('Ṻ', 'Ṻ'), + ('Ṽ', 'Ṽ'), + ('Ṿ', 'Ṿ'), + ('Ẁ', 'Ẁ'), + ('Ẃ', 'Ẃ'), + ('Ẅ', 'Ẅ'), + ('Ẇ', 'Ẇ'), + ('Ẉ', 'Ẉ'), + ('Ẋ', 'Ẋ'), + ('Ẍ', 'Ẍ'), + ('Ẏ', 'Ẏ'), + ('Ẑ', 'Ẑ'), + ('Ẓ', 'Ẓ'), + ('Ẕ', 'Ẕ'), + ('ẞ', 'ẞ'), + ('Ạ', 'Ạ'), + ('Ả', 'Ả'), + ('Ấ', 'Ấ'), + ('Ầ', 'Ầ'), + ('Ẩ', 'Ẩ'), + ('Ẫ', 'Ẫ'), + ('Ậ', 'Ậ'), + ('Ắ', 'Ắ'), + ('Ằ', 'Ằ'), + ('Ẳ', 'Ẳ'), + ('Ẵ', 'Ẵ'), + ('Ặ', 'Ặ'), + ('Ẹ', 'Ẹ'), + ('Ẻ', 'Ẻ'), + ('Ẽ', 'Ẽ'), + ('Ế', 'Ế'), + ('Ề', 'Ề'), + ('Ể', 'Ể'), + ('Ễ', 'Ễ'), + ('Ệ', 'Ệ'), + ('Ỉ', 'Ỉ'), + ('Ị', 'Ị'), + ('Ọ', 'Ọ'), + ('Ỏ', 'Ỏ'), + ('Ố', 'Ố'), + ('Ồ', 'Ồ'), + ('Ổ', 'Ổ'), + ('Ỗ', 'Ỗ'), + ('Ộ', 'Ộ'), + ('Ớ', 'Ớ'), + ('Ờ', 'Ờ'), + ('Ở', 'Ở'), + ('Ỡ', 'Ỡ'), + ('Ợ', 'Ợ'), + ('Ụ', 'Ụ'), + ('Ủ', 'Ủ'), + ('Ứ', 'Ứ'), + ('Ừ', 'Ừ'), + ('Ử', 'Ử'), + ('Ữ', 'Ữ'), + ('Ự', 'Ự'), + ('Ỳ', 'Ỳ'), + ('Ỵ', 'Ỵ'), + ('Ỷ', 'Ỷ'), + ('Ỹ', 'Ỹ'), + ('Ỻ', 'Ỻ'), + ('Ỽ', 'Ỽ'), + ('Ỿ', 'Ỿ'), + ('Ἀ', 'Ἇ'), + ('Ἐ', 'Ἕ'), + ('Ἠ', 'Ἧ'), + ('Ἰ', 'Ἷ'), + ('Ὀ', 'Ὅ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'Ὗ'), + ('Ὠ', 'Ὧ'), + ('Ᾰ', 'Ά'), + ('Ὲ', 'Ή'), + ('Ῐ', 'Ί'), + ('Ῠ', 'Ῥ'), + ('Ὸ', 'Ώ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℋ', 'ℍ'), + ('ℐ', 'ℒ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℰ', 'ℳ'), + ('ℾ', 'ℿ'), + ('ⅅ', 'ⅅ'), + ('Ↄ', 'Ↄ'), + ('Ⰰ', 'Ⱟ'), + ('Ⱡ', 'Ⱡ'), + ('Ɫ', 'Ɽ'), + ('Ⱨ', 'Ⱨ'), + ('Ⱪ', 'Ⱪ'), + ('Ⱬ', 'Ⱬ'), + ('Ɑ', 'Ɒ'), + ('Ⱳ', 'Ⱳ'), + ('Ⱶ', 'Ⱶ'), + ('Ȿ', 'Ⲁ'), + ('Ⲃ', 'Ⲃ'), + ('Ⲅ', 'Ⲅ'), + ('Ⲇ', 'Ⲇ'), + ('Ⲉ', 'Ⲉ'), + ('Ⲋ', 'Ⲋ'), + ('Ⲍ', 'Ⲍ'), + ('Ⲏ', 'Ⲏ'), + ('Ⲑ', 'Ⲑ'), + ('Ⲓ', 'Ⲓ'), + ('Ⲕ', 'Ⲕ'), + ('Ⲗ', 'Ⲗ'), + ('Ⲙ', 'Ⲙ'), + ('Ⲛ', 'Ⲛ'), + ('Ⲝ', 'Ⲝ'), + ('Ⲟ', 'Ⲟ'), + ('Ⲡ', 'Ⲡ'), + ('Ⲣ', 'Ⲣ'), + ('Ⲥ', 'Ⲥ'), + ('Ⲧ', 'Ⲧ'), + ('Ⲩ', 'Ⲩ'), + ('Ⲫ', 'Ⲫ'), + ('Ⲭ', 'Ⲭ'), + ('Ⲯ', 'Ⲯ'), + ('Ⲱ', 'Ⲱ'), + ('Ⲳ', 'Ⲳ'), + ('Ⲵ', 'Ⲵ'), + ('Ⲷ', 'Ⲷ'), + ('Ⲹ', 'Ⲹ'), + ('Ⲻ', 'Ⲻ'), + ('Ⲽ', 'Ⲽ'), + ('Ⲿ', 'Ⲿ'), + ('Ⳁ', 'Ⳁ'), + ('Ⳃ', 'Ⳃ'), + ('Ⳅ', 'Ⳅ'), + ('Ⳇ', 'Ⳇ'), + ('Ⳉ', 'Ⳉ'), + ('Ⳋ', 'Ⳋ'), + ('Ⳍ', 'Ⳍ'), + ('Ⳏ', 'Ⳏ'), + ('Ⳑ', 'Ⳑ'), + ('Ⳓ', 'Ⳓ'), + ('Ⳕ', 'Ⳕ'), + ('Ⳗ', 'Ⳗ'), + ('Ⳙ', 'Ⳙ'), + ('Ⳛ', 'Ⳛ'), + ('Ⳝ', 'Ⳝ'), + ('Ⳟ', 'Ⳟ'), + ('Ⳡ', 'Ⳡ'), + ('Ⳣ', 'Ⳣ'), + ('Ⳬ', 'Ⳬ'), + ('Ⳮ', 'Ⳮ'), + ('Ⳳ', 'Ⳳ'), + ('Ꙁ', 'Ꙁ'), + ('Ꙃ', 'Ꙃ'), + ('Ꙅ', 'Ꙅ'), + ('Ꙇ', 'Ꙇ'), + ('Ꙉ', 'Ꙉ'), + ('Ꙋ', 'Ꙋ'), + ('Ꙍ', 'Ꙍ'), + ('Ꙏ', 'Ꙏ'), + ('Ꙑ', 'Ꙑ'), + ('Ꙓ', 'Ꙓ'), + ('Ꙕ', 'Ꙕ'), + ('Ꙗ', 'Ꙗ'), + ('Ꙙ', 'Ꙙ'), + ('Ꙛ', 'Ꙛ'), + ('Ꙝ', 'Ꙝ'), + ('Ꙟ', 'Ꙟ'), + ('Ꙡ', 'Ꙡ'), + ('Ꙣ', 'Ꙣ'), + ('Ꙥ', 'Ꙥ'), + ('Ꙧ', 'Ꙧ'), + ('Ꙩ', 'Ꙩ'), + ('Ꙫ', 'Ꙫ'), + ('Ꙭ', 'Ꙭ'), + ('Ꚁ', 'Ꚁ'), + ('Ꚃ', 'Ꚃ'), + ('Ꚅ', 'Ꚅ'), + ('Ꚇ', 'Ꚇ'), + ('Ꚉ', 'Ꚉ'), + ('Ꚋ', 'Ꚋ'), + ('Ꚍ', 'Ꚍ'), + ('Ꚏ', 'Ꚏ'), + ('Ꚑ', 'Ꚑ'), + ('Ꚓ', 'Ꚓ'), + ('Ꚕ', 'Ꚕ'), + ('Ꚗ', 'Ꚗ'), + ('Ꚙ', 'Ꚙ'), + ('Ꚛ', 'Ꚛ'), + ('Ꜣ', 'Ꜣ'), + ('Ꜥ', 'Ꜥ'), + ('Ꜧ', 'Ꜧ'), + ('Ꜩ', 'Ꜩ'), + ('Ꜫ', 'Ꜫ'), + ('Ꜭ', 'Ꜭ'), + ('Ꜯ', 'Ꜯ'), + ('Ꜳ', 'Ꜳ'), + ('Ꜵ', 'Ꜵ'), + ('Ꜷ', 'Ꜷ'), + ('Ꜹ', 'Ꜹ'), + ('Ꜻ', 'Ꜻ'), + ('Ꜽ', 'Ꜽ'), + ('Ꜿ', 'Ꜿ'), + ('Ꝁ', 'Ꝁ'), + ('Ꝃ', 'Ꝃ'), + ('Ꝅ', 'Ꝅ'), + ('Ꝇ', 'Ꝇ'), + ('Ꝉ', 'Ꝉ'), + ('Ꝋ', 'Ꝋ'), + ('Ꝍ', 'Ꝍ'), + ('Ꝏ', 'Ꝏ'), + ('Ꝑ', 'Ꝑ'), + ('Ꝓ', 'Ꝓ'), + ('Ꝕ', 'Ꝕ'), + ('Ꝗ', 'Ꝗ'), + ('Ꝙ', 'Ꝙ'), + ('Ꝛ', 'Ꝛ'), + ('Ꝝ', 'Ꝝ'), + ('Ꝟ', 'Ꝟ'), + ('Ꝡ', 'Ꝡ'), + ('Ꝣ', 'Ꝣ'), + ('Ꝥ', 'Ꝥ'), + ('Ꝧ', 'Ꝧ'), + ('Ꝩ', 'Ꝩ'), + ('Ꝫ', 'Ꝫ'), + ('Ꝭ', 'Ꝭ'), + ('Ꝯ', 'Ꝯ'), + ('Ꝺ', 'Ꝺ'), + ('Ꝼ', 'Ꝼ'), + ('Ᵹ', 'Ꝿ'), + ('Ꞁ', 'Ꞁ'), + ('Ꞃ', 'Ꞃ'), + ('Ꞅ', 'Ꞅ'), + ('Ꞇ', 'Ꞇ'), + ('Ꞌ', 'Ꞌ'), + ('Ɥ', 'Ɥ'), + ('Ꞑ', 'Ꞑ'), + ('Ꞓ', 'Ꞓ'), + ('Ꞗ', 'Ꞗ'), + ('Ꞙ', 'Ꞙ'), + ('Ꞛ', 'Ꞛ'), + ('Ꞝ', 'Ꞝ'), + ('Ꞟ', 'Ꞟ'), + ('Ꞡ', 'Ꞡ'), + ('Ꞣ', 'Ꞣ'), + ('Ꞥ', 'Ꞥ'), + ('Ꞧ', 'Ꞧ'), + ('Ꞩ', 'Ꞩ'), + ('Ɦ', 'Ɪ'), + ('Ʞ', 'Ꞵ'), + ('Ꞷ', 'Ꞷ'), + ('Ꞹ', 'Ꞹ'), + ('Ꞻ', 'Ꞻ'), + ('Ꞽ', 'Ꞽ'), + ('Ꞿ', 'Ꞿ'), + ('Ꟁ', 'Ꟁ'), + ('Ꟃ', 'Ꟃ'), + ('Ꞔ', 'Ꟈ'), + ('Ꟊ', 'Ꟊ'), + ('Ɤ', 'Ꟍ'), + ('Ꟑ', 'Ꟑ'), + ('Ꟗ', 'Ꟗ'), + ('Ꟙ', 'Ꟙ'), + ('Ꟛ', 'Ꟛ'), + ('Ƛ', 'Ƛ'), + ('Ꟶ', 'Ꟶ'), + ('Ａ', 'Ｚ'), + ('𐐀', '𐐧'), + ('𐒰', '𐓓'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐲀', '𐲲'), + ('𐵐', '𐵥'), + ('𑢠', '𑢿'), + ('𖹀', '𖹟'), + ('𝐀', '𝐙'), + ('𝐴', '𝑍'), + ('𝑨', '𝒁'), + ('𝒜', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒵'), + ('𝓐', '𝓩'), + ('𝔄', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔸', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕬', '𝖅'), + ('𝖠', '𝖹'), + ('𝗔', '𝗭'), + ('𝘈', '𝘡'), + ('𝘼', '𝙕'), + ('𝙰', '𝚉'), + ('𝚨', '𝛀'), + ('𝛢', '𝛺'), + ('𝜜', '𝜴'), + ('𝝖', '𝝮'), + ('𝞐', '𝞨'), + ('𝟊', '𝟊'), + ('𞤀', '𞤡'), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/grapheme_cluster_break.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/grapheme_cluster_break.rs new file mode 100644 index 0000000..6a6ec2a --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/grapheme_cluster_break.rs @@ -0,0 +1,1420 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate grapheme-cluster-break ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = &[ + ("CR", CR), + ("Control", CONTROL), + ("Extend", EXTEND), + ("L", L), + ("LF", LF), + ("LV", LV), + ("LVT", LVT), + ("Prepend", PREPEND), + ("Regional_Indicator", REGIONAL_INDICATOR), + ("SpacingMark", SPACINGMARK), + ("T", T), + ("V", V), + ("ZWJ", ZWJ), +]; + +pub const CR: &'static [(char, char)] = &[('\r', '\r')]; + +pub const CONTROL: &'static [(char, char)] = &[ + ('\0', '\t'), + ('\u{b}', '\u{c}'), + ('\u{e}', '\u{1f}'), + ('\u{7f}', '\u{9f}'), + ('\u{ad}', '\u{ad}'), + ('\u{61c}', '\u{61c}'), + ('\u{180e}', '\u{180e}'), + ('\u{200b}', '\u{200b}'), + ('\u{200e}', '\u{200f}'), + ('\u{2028}', '\u{202e}'), + ('\u{2060}', '\u{206f}'), + ('\u{feff}', '\u{feff}'), + ('\u{fff0}', '\u{fffb}'), + ('\u{13430}', '\u{1343f}'), + ('\u{1bca0}', '\u{1bca3}'), + ('\u{1d173}', '\u{1d17a}'), + ('\u{e0000}', '\u{e001f}'), + ('\u{e0080}', '\u{e00ff}'), + ('\u{e01f0}', '\u{e0fff}'), +]; + +pub const EXTEND: &'static [(char, char)] = &[ + ('\u{300}', '\u{36f}'), + ('\u{483}', '\u{489}'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('\u{610}', '\u{61a}'), + ('\u{64b}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dc}'), + ('\u{6df}', '\u{6e4}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{6ea}', '\u{6ed}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', '\u{7f3}'), + ('\u{7fd}', '\u{7fd}'), + ('\u{816}', '\u{819}'), + ('\u{81b}', '\u{823}'), + ('\u{825}', '\u{827}'), + ('\u{829}', '\u{82d}'), + ('\u{859}', '\u{85b}'), + ('\u{897}', '\u{89f}'), + ('\u{8ca}', '\u{8e1}'), + ('\u{8e3}', '\u{902}'), + ('\u{93a}', '\u{93a}'), + ('\u{93c}', '\u{93c}'), + ('\u{941}', '\u{948}'), + ('\u{94d}', '\u{94d}'), + ('\u{951}', '\u{957}'), + ('\u{962}', '\u{963}'), + ('\u{981}', '\u{981}'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9be}', '\u{9be}'), + ('\u{9c1}', '\u{9c4}'), + ('\u{9cd}', '\u{9cd}'), + ('\u{9d7}', '\u{9d7}'), + ('\u{9e2}', '\u{9e3}'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', '\u{a02}'), + ('\u{a3c}', '\u{a3c}'), + ('\u{a41}', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', '\u{a82}'), + ('\u{abc}', '\u{abc}'), + ('\u{ac1}', '\u{ac5}'), + ('\u{ac7}', '\u{ac8}'), + ('\u{acd}', '\u{acd}'), + ('\u{ae2}', '\u{ae3}'), + ('\u{afa}', '\u{aff}'), + ('\u{b01}', '\u{b01}'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b3e}', '\u{b3f}'), + ('\u{b41}', '\u{b44}'), + ('\u{b4d}', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('\u{b62}', '\u{b63}'), + ('\u{b82}', '\u{b82}'), + ('\u{bbe}', '\u{bbe}'), + ('\u{bc0}', '\u{bc0}'), + ('\u{bcd}', '\u{bcd}'), + ('\u{bd7}', '\u{bd7}'), + ('\u{c00}', '\u{c00}'), + ('\u{c04}', '\u{c04}'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c3e}', '\u{c40}'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', '\u{c81}'), + ('\u{cbc}', '\u{cbc}'), + ('\u{cbf}', '\u{cc0}'), + ('\u{cc2}', '\u{cc2}'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('\u{ce2}', '\u{ce3}'), + ('\u{d00}', '\u{d01}'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d3e}', '\u{d3e}'), + ('\u{d41}', '\u{d44}'), + ('\u{d4d}', '\u{d4d}'), + ('\u{d57}', '\u{d57}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', '\u{d81}'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dcf}'), + ('\u{dd2}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('\u{ddf}', '\u{ddf}'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('\u{e47}', '\u{e4e}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{ebc}'), + ('\u{ec8}', '\u{ece}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('\u{f71}', '\u{f7e}'), + ('\u{f80}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('\u{102d}', '\u{1030}'), + ('\u{1032}', '\u{1037}'), + ('\u{1039}', '\u{103a}'), + ('\u{103d}', '\u{103e}'), + ('\u{1058}', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{1082}'), + ('\u{1085}', '\u{1086}'), + ('\u{108d}', '\u{108d}'), + ('\u{109d}', '\u{109d}'), + ('\u{135d}', '\u{135f}'), + ('\u{1712}', '\u{1715}'), + ('\u{1732}', '\u{1734}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('\u{17b4}', '\u{17b5}'), + ('\u{17b7}', '\u{17bd}'), + ('\u{17c6}', '\u{17c6}'), + ('\u{17c9}', '\u{17d3}'), + ('\u{17dd}', '\u{17dd}'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '\u{180f}'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', '\u{1922}'), + ('\u{1927}', '\u{1928}'), + ('\u{1932}', '\u{1932}'), + ('\u{1939}', '\u{193b}'), + ('\u{1a17}', '\u{1a18}'), + ('\u{1a1b}', '\u{1a1b}'), + ('\u{1a56}', '\u{1a56}'), + ('\u{1a58}', '\u{1a5e}'), + ('\u{1a60}', '\u{1a60}'), + ('\u{1a62}', '\u{1a62}'), + ('\u{1a65}', '\u{1a6c}'), + ('\u{1a73}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', '\u{1b03}'), + ('\u{1b34}', '\u{1b3d}'), + ('\u{1b42}', '\u{1b44}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1b81}'), + ('\u{1ba2}', '\u{1ba5}'), + ('\u{1ba8}', '\u{1bad}'), + ('\u{1be6}', '\u{1be6}'), + ('\u{1be8}', '\u{1be9}'), + ('\u{1bed}', '\u{1bed}'), + ('\u{1bef}', '\u{1bf3}'), + ('\u{1c2c}', '\u{1c33}'), + ('\u{1c36}', '\u{1c37}'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce0}'), + ('\u{1ce2}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('\u{1cf8}', '\u{1cf9}'), + ('\u{1dc0}', '\u{1dff}'), + ('\u{200c}', '\u{200c}'), + ('\u{20d0}', '\u{20f0}'), + ('\u{2cef}', '\u{2cf1}'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{2de0}', '\u{2dff}'), + ('\u{302a}', '\u{302f}'), + ('\u{3099}', '\u{309a}'), + ('\u{a66f}', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('\u{a69e}', '\u{a69f}'), + ('\u{a6f0}', '\u{a6f1}'), + ('\u{a802}', '\u{a802}'), + ('\u{a806}', '\u{a806}'), + ('\u{a80b}', '\u{a80b}'), + ('\u{a825}', '\u{a826}'), + ('\u{a82c}', '\u{a82c}'), + ('\u{a8c4}', '\u{a8c5}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92d}'), + ('\u{a947}', '\u{a951}'), + ('\u{a953}', '\u{a953}'), + ('\u{a980}', '\u{a982}'), + ('\u{a9b3}', '\u{a9b3}'), + ('\u{a9b6}', '\u{a9b9}'), + ('\u{a9bc}', '\u{a9bd}'), + ('\u{a9c0}', '\u{a9c0}'), + ('\u{a9e5}', '\u{a9e5}'), + ('\u{aa29}', '\u{aa2e}'), + ('\u{aa31}', '\u{aa32}'), + ('\u{aa35}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', '\u{aa4c}'), + ('\u{aa7c}', '\u{aa7c}'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabf}'), + ('\u{aac1}', '\u{aac1}'), + ('\u{aaec}', '\u{aaed}'), + ('\u{aaf6}', '\u{aaf6}'), + ('\u{abe5}', '\u{abe5}'), + ('\u{abe8}', '\u{abe8}'), + ('\u{abed}', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('\u{ff9e}', '\u{ff9f}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{10376}', '\u{1037a}'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('\u{10d24}', '\u{10d27}'), + ('\u{10d69}', '\u{10d6d}'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('\u{11001}', '\u{11001}'), + ('\u{11038}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{11073}', '\u{11074}'), + ('\u{1107f}', '\u{11081}'), + ('\u{110b3}', '\u{110b6}'), + ('\u{110b9}', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{1112b}'), + ('\u{1112d}', '\u{11134}'), + ('\u{11173}', '\u{11173}'), + ('\u{11180}', '\u{11181}'), + ('\u{111b6}', '\u{111be}'), + ('\u{111c0}', '\u{111c0}'), + ('\u{111c9}', '\u{111cc}'), + ('\u{111cf}', '\u{111cf}'), + ('\u{1122f}', '\u{11231}'), + ('\u{11234}', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112df}'), + ('\u{112e3}', '\u{112ea}'), + ('\u{11300}', '\u{11301}'), + ('\u{1133b}', '\u{1133c}'), + ('\u{1133e}', '\u{1133e}'), + ('\u{11340}', '\u{11340}'), + ('\u{1134d}', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113b8}', '\u{113b8}'), + ('\u{113bb}', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '\u{113c9}'), + ('\u{113ce}', '\u{113d0}'), + ('\u{113d2}', '\u{113d2}'), + ('\u{113e1}', '\u{113e2}'), + ('\u{11438}', '\u{1143f}'), + ('\u{11442}', '\u{11444}'), + ('\u{11446}', '\u{11446}'), + ('\u{1145e}', '\u{1145e}'), + ('\u{114b0}', '\u{114b0}'), + ('\u{114b3}', '\u{114b8}'), + ('\u{114ba}', '\u{114ba}'), + ('\u{114bd}', '\u{114bd}'), + ('\u{114bf}', '\u{114c0}'), + ('\u{114c2}', '\u{114c3}'), + ('\u{115af}', '\u{115af}'), + ('\u{115b2}', '\u{115b5}'), + ('\u{115bc}', '\u{115bd}'), + ('\u{115bf}', '\u{115c0}'), + ('\u{115dc}', '\u{115dd}'), + ('\u{11633}', '\u{1163a}'), + ('\u{1163d}', '\u{1163d}'), + ('\u{1163f}', '\u{11640}'), + ('\u{116ab}', '\u{116ab}'), + ('\u{116ad}', '\u{116ad}'), + ('\u{116b0}', '\u{116b7}'), + ('\u{1171d}', '\u{1171d}'), + ('\u{1171f}', '\u{1171f}'), + ('\u{11722}', '\u{11725}'), + ('\u{11727}', '\u{1172b}'), + ('\u{1182f}', '\u{11837}'), + ('\u{11839}', '\u{1183a}'), + ('\u{11930}', '\u{11930}'), + ('\u{1193b}', '\u{1193e}'), + ('\u{11943}', '\u{11943}'), + ('\u{119d4}', '\u{119d7}'), + ('\u{119da}', '\u{119db}'), + ('\u{119e0}', '\u{119e0}'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a33}', '\u{11a38}'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a51}', '\u{11a56}'), + ('\u{11a59}', '\u{11a5b}'), + ('\u{11a8a}', '\u{11a96}'), + ('\u{11a98}', '\u{11a99}'), + ('\u{11c30}', '\u{11c36}'), + ('\u{11c38}', '\u{11c3d}'), + ('\u{11c3f}', '\u{11c3f}'), + ('\u{11c92}', '\u{11ca7}'), + ('\u{11caa}', '\u{11cb0}'), + ('\u{11cb2}', '\u{11cb3}'), + ('\u{11cb5}', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d45}'), + ('\u{11d47}', '\u{11d47}'), + ('\u{11d90}', '\u{11d91}'), + ('\u{11d95}', '\u{11d95}'), + ('\u{11d97}', '\u{11d97}'), + ('\u{11ef3}', '\u{11ef4}'), + ('\u{11f00}', '\u{11f01}'), + ('\u{11f36}', '\u{11f3a}'), + ('\u{11f40}', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13440}', '\u{13440}'), + ('\u{13447}', '\u{13455}'), + ('\u{1611e}', '\u{16129}'), + ('\u{1612d}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('\u{16f4f}', '\u{16f4f}'), + ('\u{16f8f}', '\u{16f92}'), + ('\u{16fe4}', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e130}', '\u{1e136}'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('\u{1e4ec}', '\u{1e4ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '\u{1e94a}'), + ('🏻', '🏿'), + ('\u{e0020}', '\u{e007f}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const L: &'static [(char, char)] = &[('ᄀ', 'ᅟ'), ('ꥠ', 'ꥼ')]; + +pub const LF: &'static [(char, char)] = &[('\n', '\n')]; + +pub const LV: &'static [(char, char)] = &[ + ('가', '가'), + ('개', '개'), + ('갸', '갸'), + ('걔', '걔'), + ('거', '거'), + ('게', '게'), + ('겨', '겨'), + ('계', '계'), + ('고', '고'), + ('과', '과'), + ('괘', '괘'), + ('괴', '괴'), + ('교', '교'), + ('구', '구'), + ('궈', '궈'), + ('궤', '궤'), + ('귀', '귀'), + ('규', '규'), + ('그', '그'), + ('긔', '긔'), + ('기', '기'), + ('까', '까'), + ('깨', '깨'), + ('꺄', '꺄'), + ('꺠', '꺠'), + ('꺼', '꺼'), + ('께', '께'), + ('껴', '껴'), + ('꼐', '꼐'), + ('꼬', '꼬'), + ('꽈', '꽈'), + ('꽤', '꽤'), + ('꾀', '꾀'), + ('꾜', '꾜'), + ('꾸', '꾸'), + ('꿔', '꿔'), + ('꿰', '꿰'), + ('뀌', '뀌'), + ('뀨', '뀨'), + ('끄', '끄'), + ('끠', '끠'), + ('끼', '끼'), + ('나', '나'), + ('내', '내'), + ('냐', '냐'), + ('냬', '냬'), + ('너', '너'), + ('네', '네'), + ('녀', '녀'), + ('녜', '녜'), + ('노', '노'), + ('놔', '놔'), + ('놰', '놰'), + ('뇌', '뇌'), + ('뇨', '뇨'), + ('누', '누'), + ('눠', '눠'), + ('눼', '눼'), + ('뉘', '뉘'), + ('뉴', '뉴'), + ('느', '느'), + ('늬', '늬'), + ('니', '니'), + ('다', '다'), + ('대', '대'), + ('댜', '댜'), + ('댸', '댸'), + ('더', '더'), + ('데', '데'), + ('뎌', '뎌'), + ('뎨', '뎨'), + ('도', '도'), + ('돠', '돠'), + ('돼', '돼'), + ('되', '되'), + ('됴', '됴'), + ('두', '두'), + ('둬', '둬'), + ('뒈', '뒈'), + ('뒤', '뒤'), + ('듀', '듀'), + ('드', '드'), + ('듸', '듸'), + ('디', '디'), + ('따', '따'), + ('때', '때'), + ('땨', '땨'), + ('떄', '떄'), + ('떠', '떠'), + ('떼', '떼'), + ('뗘', '뗘'), + ('뗴', '뗴'), + ('또', '또'), + ('똬', '똬'), + ('뙈', '뙈'), + ('뙤', '뙤'), + ('뚀', '뚀'), + ('뚜', '뚜'), + ('뚸', '뚸'), + ('뛔', '뛔'), + ('뛰', '뛰'), + ('뜌', '뜌'), + ('뜨', '뜨'), + ('띄', '띄'), + ('띠', '띠'), + ('라', '라'), + ('래', '래'), + ('랴', '랴'), + ('럐', '럐'), + ('러', '러'), + ('레', '레'), + ('려', '려'), + ('례', '례'), + ('로', '로'), + ('롸', '롸'), + ('뢔', '뢔'), + ('뢰', '뢰'), + ('료', '료'), + ('루', '루'), + ('뤄', '뤄'), + ('뤠', '뤠'), + ('뤼', '뤼'), + ('류', '류'), + ('르', '르'), + ('릐', '릐'), + ('리', '리'), + ('마', '마'), + ('매', '매'), + ('먀', '먀'), + ('먜', '먜'), + ('머', '머'), + ('메', '메'), + ('며', '며'), + ('몌', '몌'), + ('모', '모'), + ('뫄', '뫄'), + ('뫠', '뫠'), + ('뫼', '뫼'), + ('묘', '묘'), + ('무', '무'), + ('뭐', '뭐'), + ('뭬', '뭬'), + ('뮈', '뮈'), + ('뮤', '뮤'), + ('므', '므'), + ('믜', '믜'), + ('미', '미'), + ('바', '바'), + ('배', '배'), + ('뱌', '뱌'), + ('뱨', '뱨'), + ('버', '버'), + ('베', '베'), + ('벼', '벼'), + ('볘', '볘'), + ('보', '보'), + ('봐', '봐'), + ('봬', '봬'), + ('뵈', '뵈'), + ('뵤', '뵤'), + ('부', '부'), + ('붜', '붜'), + ('붸', '붸'), + ('뷔', '뷔'), + ('뷰', '뷰'), + ('브', '브'), + ('븨', '븨'), + ('비', '비'), + ('빠', '빠'), + ('빼', '빼'), + ('뺘', '뺘'), + ('뺴', '뺴'), + ('뻐', '뻐'), + ('뻬', '뻬'), + ('뼈', '뼈'), + ('뼤', '뼤'), + ('뽀', '뽀'), + ('뽜', '뽜'), + ('뽸', '뽸'), + ('뾔', '뾔'), + ('뾰', '뾰'), + ('뿌', '뿌'), + ('뿨', '뿨'), + ('쀄', '쀄'), + ('쀠', '쀠'), + ('쀼', '쀼'), + ('쁘', '쁘'), + ('쁴', '쁴'), + ('삐', '삐'), + ('사', '사'), + ('새', '새'), + ('샤', '샤'), + ('섀', '섀'), + ('서', '서'), + ('세', '세'), + ('셔', '셔'), + ('셰', '셰'), + ('소', '소'), + ('솨', '솨'), + ('쇄', '쇄'), + ('쇠', '쇠'), + ('쇼', '쇼'), + ('수', '수'), + ('숴', '숴'), + ('쉐', '쉐'), + ('쉬', '쉬'), + ('슈', '슈'), + ('스', '스'), + ('싀', '싀'), + ('시', '시'), + ('싸', '싸'), + ('쌔', '쌔'), + ('쌰', '쌰'), + ('썌', '썌'), + ('써', '써'), + ('쎄', '쎄'), + ('쎠', '쎠'), + ('쎼', '쎼'), + ('쏘', '쏘'), + ('쏴', '쏴'), + ('쐐', '쐐'), + ('쐬', '쐬'), + ('쑈', '쑈'), + ('쑤', '쑤'), + ('쒀', '쒀'), + ('쒜', '쒜'), + ('쒸', '쒸'), + ('쓔', '쓔'), + ('쓰', '쓰'), + ('씌', '씌'), + ('씨', '씨'), + ('아', '아'), + ('애', '애'), + ('야', '야'), + ('얘', '얘'), + ('어', '어'), + ('에', '에'), + ('여', '여'), + ('예', '예'), + ('오', '오'), + ('와', '와'), + ('왜', '왜'), + ('외', '외'), + ('요', '요'), + ('우', '우'), + ('워', '워'), + ('웨', '웨'), + ('위', '위'), + ('유', '유'), + ('으', '으'), + ('의', '의'), + ('이', '이'), + ('자', '자'), + ('재', '재'), + ('쟈', '쟈'), + ('쟤', '쟤'), + ('저', '저'), + ('제', '제'), + ('져', '져'), + ('졔', '졔'), + ('조', '조'), + ('좌', '좌'), + ('좨', '좨'), + ('죄', '죄'), + ('죠', '죠'), + ('주', '주'), + ('줘', '줘'), + ('줴', '줴'), + ('쥐', '쥐'), + ('쥬', '쥬'), + ('즈', '즈'), + ('즤', '즤'), + ('지', '지'), + ('짜', '짜'), + ('째', '째'), + ('쨔', '쨔'), + ('쨰', '쨰'), + ('쩌', '쩌'), + ('쩨', '쩨'), + ('쪄', '쪄'), + ('쪠', '쪠'), + ('쪼', '쪼'), + ('쫘', '쫘'), + ('쫴', '쫴'), + ('쬐', '쬐'), + ('쬬', '쬬'), + ('쭈', '쭈'), + ('쭤', '쭤'), + ('쮀', '쮀'), + ('쮜', '쮜'), + ('쮸', '쮸'), + ('쯔', '쯔'), + ('쯰', '쯰'), + ('찌', '찌'), + ('차', '차'), + ('채', '채'), + ('챠', '챠'), + ('챼', '챼'), + ('처', '처'), + ('체', '체'), + ('쳐', '쳐'), + ('쳬', '쳬'), + ('초', '초'), + ('촤', '촤'), + ('쵀', '쵀'), + ('최', '최'), + ('쵸', '쵸'), + ('추', '추'), + ('춰', '춰'), + ('췌', '췌'), + ('취', '취'), + ('츄', '츄'), + ('츠', '츠'), + ('츼', '츼'), + ('치', '치'), + ('카', '카'), + ('캐', '캐'), + ('캬', '캬'), + ('컈', '컈'), + ('커', '커'), + ('케', '케'), + ('켜', '켜'), + ('켸', '켸'), + ('코', '코'), + ('콰', '콰'), + ('쾌', '쾌'), + ('쾨', '쾨'), + ('쿄', '쿄'), + ('쿠', '쿠'), + ('쿼', '쿼'), + ('퀘', '퀘'), + ('퀴', '퀴'), + ('큐', '큐'), + ('크', '크'), + ('킈', '킈'), + ('키', '키'), + ('타', '타'), + ('태', '태'), + ('탸', '탸'), + ('턔', '턔'), + ('터', '터'), + ('테', '테'), + ('텨', '텨'), + ('톄', '톄'), + ('토', '토'), + ('톼', '톼'), + ('퇘', '퇘'), + ('퇴', '퇴'), + ('툐', '툐'), + ('투', '투'), + ('퉈', '퉈'), + ('퉤', '퉤'), + ('튀', '튀'), + ('튜', '튜'), + ('트', '트'), + ('틔', '틔'), + ('티', '티'), + ('파', '파'), + ('패', '패'), + ('퍄', '퍄'), + ('퍠', '퍠'), + ('퍼', '퍼'), + ('페', '페'), + ('펴', '펴'), + ('폐', '폐'), + ('포', '포'), + ('퐈', '퐈'), + ('퐤', '퐤'), + ('푀', '푀'), + ('표', '표'), + ('푸', '푸'), + ('풔', '풔'), + ('풰', '풰'), + ('퓌', '퓌'), + ('퓨', '퓨'), + ('프', '프'), + ('픠', '픠'), + ('피', '피'), + ('하', '하'), + ('해', '해'), + ('햐', '햐'), + ('햬', '햬'), + ('허', '허'), + ('헤', '헤'), + ('혀', '혀'), + ('혜', '혜'), + ('호', '호'), + ('화', '화'), + ('홰', '홰'), + ('회', '회'), + ('효', '효'), + ('후', '후'), + ('훠', '훠'), + ('훼', '훼'), + ('휘', '휘'), + ('휴', '휴'), + ('흐', '흐'), + ('희', '희'), + ('히', '히'), +]; + +pub const LVT: &'static [(char, char)] = &[ + ('각', '갛'), + ('객', '갷'), + ('갹', '걓'), + ('걕', '걯'), + ('걱', '겋'), + ('겍', '겧'), + ('격', '곃'), + ('곅', '곟'), + ('곡', '곻'), + ('곽', '괗'), + ('괙', '괳'), + ('괵', '굏'), + ('굑', '굫'), + ('국', '궇'), + ('궉', '궣'), + ('궥', '궿'), + ('귁', '귛'), + ('귝', '귷'), + ('극', '긓'), + ('긕', '긯'), + ('긱', '깋'), + ('깍', '깧'), + ('깩', '꺃'), + ('꺅', '꺟'), + ('꺡', '꺻'), + ('꺽', '껗'), + ('껙', '껳'), + ('껵', '꼏'), + ('꼑', '꼫'), + ('꼭', '꽇'), + ('꽉', '꽣'), + ('꽥', '꽿'), + ('꾁', '꾛'), + ('꾝', '꾷'), + ('꾹', '꿓'), + ('꿕', '꿯'), + ('꿱', '뀋'), + ('뀍', '뀧'), + ('뀩', '끃'), + ('끅', '끟'), + ('끡', '끻'), + ('끽', '낗'), + ('낙', '낳'), + ('낵', '냏'), + ('냑', '냫'), + ('냭', '넇'), + ('넉', '넣'), + ('넥', '넿'), + ('녁', '녛'), + ('녝', '녷'), + ('녹', '놓'), + ('놕', '놯'), + ('놱', '뇋'), + ('뇍', '뇧'), + ('뇩', '눃'), + ('눅', '눟'), + ('눡', '눻'), + ('눽', '뉗'), + ('뉙', '뉳'), + ('뉵', '늏'), + ('늑', '늫'), + ('늭', '닇'), + ('닉', '닣'), + ('닥', '닿'), + ('댁', '댛'), + ('댝', '댷'), + ('댹', '덓'), + ('덕', '덯'), + ('덱', '뎋'), + ('뎍', '뎧'), + ('뎩', '돃'), + ('독', '돟'), + ('돡', '돻'), + ('돽', '됗'), + ('됙', '됳'), + ('됵', '둏'), + ('둑', '둫'), + ('둭', '뒇'), + ('뒉', '뒣'), + ('뒥', '뒿'), + ('듁', '듛'), + ('득', '듷'), + ('듹', '딓'), + ('딕', '딯'), + ('딱', '땋'), + ('땍', '땧'), + ('땩', '떃'), + ('떅', '떟'), + ('떡', '떻'), + ('떽', '뗗'), + ('뗙', '뗳'), + ('뗵', '똏'), + ('똑', '똫'), + ('똭', '뙇'), + ('뙉', '뙣'), + ('뙥', '뙿'), + ('뚁', '뚛'), + ('뚝', '뚷'), + ('뚹', '뛓'), + ('뛕', '뛯'), + ('뛱', '뜋'), + ('뜍', '뜧'), + ('뜩', '띃'), + ('띅', '띟'), + ('띡', '띻'), + ('락', '랗'), + ('랙', '랳'), + ('략', '럏'), + ('럑', '럫'), + ('럭', '렇'), + ('렉', '렣'), + ('력', '렿'), + ('롁', '롛'), + ('록', '롷'), + ('롹', '뢓'), + ('뢕', '뢯'), + ('뢱', '룋'), + ('룍', '룧'), + ('룩', '뤃'), + ('뤅', '뤟'), + ('뤡', '뤻'), + ('뤽', '륗'), + ('륙', '륳'), + ('륵', '릏'), + ('릑', '릫'), + ('릭', '맇'), + ('막', '맣'), + ('맥', '맿'), + ('먁', '먛'), + ('먝', '먷'), + ('먹', '멓'), + ('멕', '멯'), + ('멱', '몋'), + ('몍', '몧'), + ('목', '뫃'), + ('뫅', '뫟'), + ('뫡', '뫻'), + ('뫽', '묗'), + ('묙', '묳'), + ('묵', '뭏'), + ('뭑', '뭫'), + ('뭭', '뮇'), + ('뮉', '뮣'), + ('뮥', '뮿'), + ('믁', '믛'), + ('믝', '믷'), + ('믹', '밓'), + ('박', '밯'), + ('백', '뱋'), + ('뱍', '뱧'), + ('뱩', '벃'), + ('벅', '벟'), + ('벡', '벻'), + ('벽', '볗'), + ('볙', '볳'), + ('복', '봏'), + ('봑', '봫'), + ('봭', '뵇'), + ('뵉', '뵣'), + ('뵥', '뵿'), + ('북', '붛'), + ('붝', '붷'), + ('붹', '뷓'), + ('뷕', '뷯'), + ('뷱', '븋'), + ('븍', '븧'), + ('븩', '빃'), + ('빅', '빟'), + ('빡', '빻'), + ('빽', '뺗'), + ('뺙', '뺳'), + ('뺵', '뻏'), + ('뻑', '뻫'), + ('뻭', '뼇'), + ('뼉', '뼣'), + ('뼥', '뼿'), + ('뽁', '뽛'), + ('뽝', '뽷'), + ('뽹', '뾓'), + ('뾕', '뾯'), + ('뾱', '뿋'), + ('뿍', '뿧'), + ('뿩', '쀃'), + ('쀅', '쀟'), + ('쀡', '쀻'), + ('쀽', '쁗'), + ('쁙', '쁳'), + ('쁵', '삏'), + ('삑', '삫'), + ('삭', '샇'), + ('색', '샣'), + ('샥', '샿'), + ('섁', '섛'), + ('석', '섷'), + ('섹', '셓'), + ('셕', '셯'), + ('셱', '솋'), + ('속', '솧'), + ('솩', '쇃'), + ('쇅', '쇟'), + ('쇡', '쇻'), + ('쇽', '숗'), + ('숙', '숳'), + ('숵', '쉏'), + ('쉑', '쉫'), + ('쉭', '슇'), + ('슉', '슣'), + ('슥', '슿'), + ('싁', '싛'), + ('식', '싷'), + ('싹', '쌓'), + ('쌕', '쌯'), + ('쌱', '썋'), + ('썍', '썧'), + ('썩', '쎃'), + ('쎅', '쎟'), + ('쎡', '쎻'), + ('쎽', '쏗'), + ('쏙', '쏳'), + ('쏵', '쐏'), + ('쐑', '쐫'), + ('쐭', '쑇'), + ('쑉', '쑣'), + ('쑥', '쑿'), + ('쒁', '쒛'), + ('쒝', '쒷'), + ('쒹', '쓓'), + ('쓕', '쓯'), + ('쓱', '씋'), + ('씍', '씧'), + ('씩', '앃'), + ('악', '앟'), + ('액', '앻'), + ('약', '얗'), + ('얙', '얳'), + ('억', '엏'), + ('엑', '엫'), + ('역', '옇'), + ('옉', '옣'), + ('옥', '옿'), + ('왁', '왛'), + ('왝', '왷'), + ('왹', '욓'), + ('욕', '욯'), + ('욱', '웋'), + ('웍', '웧'), + ('웩', '윃'), + ('윅', '윟'), + ('육', '윻'), + ('윽', '읗'), + ('읙', '읳'), + ('익', '잏'), + ('작', '잫'), + ('잭', '쟇'), + ('쟉', '쟣'), + ('쟥', '쟿'), + ('적', '젛'), + ('젝', '젷'), + ('젹', '졓'), + ('졕', '졯'), + ('족', '좋'), + ('좍', '좧'), + ('좩', '죃'), + ('죅', '죟'), + ('죡', '죻'), + ('죽', '줗'), + ('줙', '줳'), + ('줵', '쥏'), + ('쥑', '쥫'), + ('쥭', '즇'), + ('즉', '즣'), + ('즥', '즿'), + ('직', '짛'), + ('짝', '짷'), + ('짹', '쨓'), + ('쨕', '쨯'), + ('쨱', '쩋'), + ('쩍', '쩧'), + ('쩩', '쪃'), + ('쪅', '쪟'), + ('쪡', '쪻'), + ('쪽', '쫗'), + ('쫙', '쫳'), + ('쫵', '쬏'), + ('쬑', '쬫'), + ('쬭', '쭇'), + ('쭉', '쭣'), + ('쭥', '쭿'), + ('쮁', '쮛'), + ('쮝', '쮷'), + ('쮹', '쯓'), + ('쯕', '쯯'), + ('쯱', '찋'), + ('찍', '찧'), + ('착', '챃'), + ('책', '챟'), + ('챡', '챻'), + ('챽', '첗'), + ('척', '첳'), + ('첵', '쳏'), + ('쳑', '쳫'), + ('쳭', '촇'), + ('촉', '촣'), + ('촥', '촿'), + ('쵁', '쵛'), + ('쵝', '쵷'), + ('쵹', '춓'), + ('축', '춯'), + ('춱', '췋'), + ('췍', '췧'), + ('췩', '츃'), + ('츅', '츟'), + ('측', '츻'), + ('츽', '칗'), + ('칙', '칳'), + ('칵', '캏'), + ('캑', '캫'), + ('캭', '컇'), + ('컉', '컣'), + ('컥', '컿'), + ('켁', '켛'), + ('켝', '켷'), + ('켹', '콓'), + ('콕', '콯'), + ('콱', '쾋'), + ('쾍', '쾧'), + ('쾩', '쿃'), + ('쿅', '쿟'), + ('쿡', '쿻'), + ('쿽', '퀗'), + ('퀙', '퀳'), + ('퀵', '큏'), + ('큑', '큫'), + ('큭', '킇'), + ('킉', '킣'), + ('킥', '킿'), + ('탁', '탛'), + ('택', '탷'), + ('탹', '턓'), + ('턕', '턯'), + ('턱', '텋'), + ('텍', '텧'), + ('텩', '톃'), + ('톅', '톟'), + ('톡', '톻'), + ('톽', '퇗'), + ('퇙', '퇳'), + ('퇵', '툏'), + ('툑', '툫'), + ('툭', '퉇'), + ('퉉', '퉣'), + ('퉥', '퉿'), + ('튁', '튛'), + ('튝', '튷'), + ('특', '틓'), + ('틕', '틯'), + ('틱', '팋'), + ('팍', '팧'), + ('팩', '퍃'), + ('퍅', '퍟'), + ('퍡', '퍻'), + ('퍽', '펗'), + ('펙', '펳'), + ('펵', '폏'), + ('폑', '폫'), + ('폭', '퐇'), + ('퐉', '퐣'), + ('퐥', '퐿'), + ('푁', '푛'), + ('푝', '푷'), + ('푹', '풓'), + ('풕', '풯'), + ('풱', '퓋'), + ('퓍', '퓧'), + ('퓩', '픃'), + ('픅', '픟'), + ('픡', '픻'), + ('픽', '핗'), + ('학', '핳'), + ('핵', '햏'), + ('햑', '햫'), + ('햭', '헇'), + ('헉', '헣'), + ('헥', '헿'), + ('혁', '혛'), + ('혝', '혷'), + ('혹', '홓'), + ('확', '홯'), + ('홱', '횋'), + ('획', '횧'), + ('횩', '훃'), + ('훅', '훟'), + ('훡', '훻'), + ('훽', '휗'), + ('휙', '휳'), + ('휵', '흏'), + ('흑', '흫'), + ('흭', '힇'), + ('힉', '힣'), +]; + +pub const PREPEND: &'static [(char, char)] = &[ + ('\u{600}', '\u{605}'), + ('\u{6dd}', '\u{6dd}'), + ('\u{70f}', '\u{70f}'), + ('\u{890}', '\u{891}'), + ('\u{8e2}', '\u{8e2}'), + ('ൎ', 'ൎ'), + ('\u{110bd}', '\u{110bd}'), + ('\u{110cd}', '\u{110cd}'), + ('𑇂', '𑇃'), + ('𑏑', '𑏑'), + ('𑤿', '𑤿'), + ('𑥁', '𑥁'), + ('𑨺', '𑨺'), + ('𑪄', '𑪉'), + ('𑵆', '𑵆'), + ('𑼂', '𑼂'), +]; + +pub const REGIONAL_INDICATOR: &'static [(char, char)] = &[('🇦', '🇿')]; + +pub const SPACINGMARK: &'static [(char, char)] = &[ + ('ः', 'ः'), + ('ऻ', 'ऻ'), + ('ा', 'ी'), + ('ॉ', 'ौ'), + ('ॎ', 'ॏ'), + ('ং', 'ঃ'), + ('ি', 'ী'), + ('ে', 'ৈ'), + ('ো', 'ৌ'), + ('ਃ', 'ਃ'), + ('ਾ', 'ੀ'), + ('ઃ', 'ઃ'), + ('ા', 'ી'), + ('ૉ', 'ૉ'), + ('ો', 'ૌ'), + ('ଂ', 'ଃ'), + ('ୀ', 'ୀ'), + ('େ', 'ୈ'), + ('ୋ', 'ୌ'), + ('ி', 'ி'), + ('ு', 'ூ'), + ('ெ', 'ை'), + ('ொ', 'ௌ'), + ('ఁ', 'ః'), + ('ు', 'ౄ'), + ('ಂ', 'ಃ'), + ('ಾ', 'ಾ'), + ('ು', 'ು'), + ('ೃ', 'ೄ'), + ('ೳ', 'ೳ'), + ('ം', 'ഃ'), + ('ി', 'ീ'), + ('െ', 'ൈ'), + ('ൊ', 'ൌ'), + ('ං', 'ඃ'), + ('ැ', 'ෑ'), + ('ෘ', 'ෞ'), + ('ෲ', 'ෳ'), + ('ำ', 'ำ'), + ('ຳ', 'ຳ'), + ('༾', '༿'), + ('ཿ', 'ཿ'), + ('ေ', 'ေ'), + ('ျ', 'ြ'), + ('ၖ', 'ၗ'), + ('ႄ', 'ႄ'), + ('ា', 'ា'), + ('ើ', 'ៅ'), + ('ះ', 'ៈ'), + ('ᤣ', 'ᤦ'), + ('ᤩ', 'ᤫ'), + ('ᤰ', 'ᤱ'), + ('ᤳ', 'ᤸ'), + ('ᨙ', 'ᨚ'), + ('ᩕ', 'ᩕ'), + ('ᩗ', 'ᩗ'), + ('ᩭ', 'ᩲ'), + ('ᬄ', 'ᬄ'), + ('ᬾ', 'ᭁ'), + ('ᮂ', 'ᮂ'), + ('ᮡ', 'ᮡ'), + ('ᮦ', 'ᮧ'), + ('ᯧ', 'ᯧ'), + ('ᯪ', 'ᯬ'), + ('ᯮ', 'ᯮ'), + ('ᰤ', 'ᰫ'), + ('ᰴ', 'ᰵ'), + ('᳡', '᳡'), + ('᳷', '᳷'), + ('ꠣ', 'ꠤ'), + ('ꠧ', 'ꠧ'), + ('ꢀ', 'ꢁ'), + ('ꢴ', 'ꣃ'), + ('ꥒ', 'ꥒ'), + ('ꦃ', 'ꦃ'), + ('ꦴ', 'ꦵ'), + ('ꦺ', 'ꦻ'), + ('ꦾ', 'ꦿ'), + ('ꨯ', 'ꨰ'), + ('ꨳ', 'ꨴ'), + ('ꩍ', 'ꩍ'), + ('ꫫ', 'ꫫ'), + ('ꫮ', 'ꫯ'), + ('ꫵ', 'ꫵ'), + ('ꯣ', 'ꯤ'), + ('ꯦ', 'ꯧ'), + ('ꯩ', 'ꯪ'), + ('꯬', '꯬'), + ('𑀀', '𑀀'), + ('𑀂', '𑀂'), + ('𑂂', '𑂂'), + ('𑂰', '𑂲'), + ('𑂷', '𑂸'), + ('𑄬', '𑄬'), + ('𑅅', '𑅆'), + ('𑆂', '𑆂'), + ('𑆳', '𑆵'), + ('𑆿', '𑆿'), + ('𑇎', '𑇎'), + ('𑈬', '𑈮'), + ('𑈲', '𑈳'), + ('𑋠', '𑋢'), + ('𑌂', '𑌃'), + ('𑌿', '𑌿'), + ('𑍁', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '𑍌'), + ('𑍢', '𑍣'), + ('𑎹', '𑎺'), + ('𑏊', '𑏊'), + ('𑏌', '𑏍'), + ('𑐵', '𑐷'), + ('𑑀', '𑑁'), + ('𑑅', '𑑅'), + ('𑒱', '𑒲'), + ('𑒹', '𑒹'), + ('𑒻', '𑒼'), + ('𑒾', '𑒾'), + ('𑓁', '𑓁'), + ('𑖰', '𑖱'), + ('𑖸', '𑖻'), + ('𑖾', '𑖾'), + ('𑘰', '𑘲'), + ('𑘻', '𑘼'), + ('𑘾', '𑘾'), + ('𑚬', '𑚬'), + ('𑚮', '𑚯'), + ('𑜞', '𑜞'), + ('𑜦', '𑜦'), + ('𑠬', '𑠮'), + ('𑠸', '𑠸'), + ('𑤱', '𑤵'), + ('𑤷', '𑤸'), + ('𑥀', '𑥀'), + ('𑥂', '𑥂'), + ('𑧑', '𑧓'), + ('𑧜', '𑧟'), + ('𑧤', '𑧤'), + ('𑨹', '𑨹'), + ('𑩗', '𑩘'), + ('𑪗', '𑪗'), + ('𑰯', '𑰯'), + ('𑰾', '𑰾'), + ('𑲩', '𑲩'), + ('𑲱', '𑲱'), + ('𑲴', '𑲴'), + ('𑶊', '𑶎'), + ('𑶓', '𑶔'), + ('𑶖', '𑶖'), + ('𑻵', '𑻶'), + ('𑼃', '𑼃'), + ('𑼴', '𑼵'), + ('𑼾', '𑼿'), + ('𖄪', '𖄬'), + ('𖽑', '𖾇'), +]; + +pub const T: &'static [(char, char)] = &[('ᆨ', 'ᇿ'), ('ퟋ', 'ퟻ')]; + +pub const V: &'static [(char, char)] = + &[('ᅠ', 'ᆧ'), ('ힰ', 'ퟆ'), ('𖵣', '𖵣'), ('𖵧', '𖵪')]; + +pub const ZWJ: &'static [(char, char)] = &[('\u{200d}', '\u{200d}')]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/mod.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/mod.rs new file mode 100644 index 0000000..20736c7 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/mod.rs @@ -0,0 +1,57 @@ +#[cfg(feature = "unicode-age")] +pub mod age; + +#[cfg(feature = "unicode-case")] +pub mod case_folding_simple; + +#[cfg(feature = "unicode-gencat")] +pub mod general_category; + +#[cfg(feature = "unicode-segment")] +pub mod grapheme_cluster_break; + +#[cfg(all(feature = "unicode-perl", not(feature = "unicode-gencat")))] +#[allow(dead_code)] +pub mod perl_decimal; + +#[cfg(all(feature = "unicode-perl", not(feature = "unicode-bool")))] +#[allow(dead_code)] +pub mod perl_space; + +#[cfg(feature = "unicode-perl")] +pub mod perl_word; + +#[cfg(feature = "unicode-bool")] +pub mod property_bool; + +#[cfg(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", +))] +pub mod property_names; + +#[cfg(any( + feature = "unicode-age", + feature = "unicode-bool", + feature = "unicode-gencat", + feature = "unicode-perl", + feature = "unicode-script", + feature = "unicode-segment", +))] +pub mod property_values; + +#[cfg(feature = "unicode-script")] +pub mod script; + +#[cfg(feature = "unicode-script")] +pub mod script_extension; + +#[cfg(feature = "unicode-segment")] +pub mod sentence_break; + +#[cfg(feature = "unicode-segment")] +pub mod word_break; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_decimal.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_decimal.rs new file mode 100644 index 0000000..18996c2 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_decimal.rs @@ -0,0 +1,84 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate general-category ucd-16.0.0 --chars --include decimalnumber +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = + &[("Decimal_Number", DECIMAL_NUMBER)]; + +pub const DECIMAL_NUMBER: &'static [(char, char)] = &[ + ('0', '9'), + ('٠', '٩'), + ('۰', '۹'), + ('߀', '߉'), + ('०', '९'), + ('০', '৯'), + ('੦', '੯'), + ('૦', '૯'), + ('୦', '୯'), + ('௦', '௯'), + ('౦', '౯'), + ('೦', '೯'), + ('൦', '൯'), + ('෦', '෯'), + ('๐', '๙'), + ('໐', '໙'), + ('༠', '༩'), + ('၀', '၉'), + ('႐', '႙'), + ('០', '៩'), + ('᠐', '᠙'), + ('᥆', '᥏'), + ('᧐', '᧙'), + ('᪀', '᪉'), + ('᪐', '᪙'), + ('᭐', '᭙'), + ('᮰', '᮹'), + ('᱀', '᱉'), + ('᱐', '᱙'), + ('꘠', '꘩'), + ('꣐', '꣙'), + ('꤀', '꤉'), + ('꧐', '꧙'), + ('꧰', '꧹'), + ('꩐', '꩙'), + ('꯰', '꯹'), + ('０', '９'), + ('𐒠', '𐒩'), + ('𐴰', '𐴹'), + ('𐵀', '𐵉'), + ('𑁦', '𑁯'), + ('𑃰', '𑃹'), + ('𑄶', '𑄿'), + ('𑇐', '𑇙'), + ('𑋰', '𑋹'), + ('𑑐', '𑑙'), + ('𑓐', '𑓙'), + ('𑙐', '𑙙'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜰', '𑜹'), + ('𑣠', '𑣩'), + ('𑥐', '𑥙'), + ('𑯰', '𑯹'), + ('𑱐', '𑱙'), + ('𑵐', '𑵙'), + ('𑶠', '𑶩'), + ('𑽐', '𑽙'), + ('𖄰', '𖄹'), + ('𖩠', '𖩩'), + ('𖫀', '𖫉'), + ('𖭐', '𖭙'), + ('𖵰', '𖵹'), + ('𜳰', '𜳹'), + ('𝟎', '𝟿'), + ('𞅀', '𞅉'), + ('𞋰', '𞋹'), + ('𞓰', '𞓹'), + ('𞗱', '𞗺'), + ('𞥐', '𞥙'), + ('🯰', '🯹'), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_space.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_space.rs new file mode 100644 index 0000000..c969e37 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_space.rs @@ -0,0 +1,23 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate property-bool ucd-16.0.0 --chars --include whitespace +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = + &[("White_Space", WHITE_SPACE)]; + +pub const WHITE_SPACE: &'static [(char, char)] = &[ + ('\t', '\r'), + (' ', ' '), + ('\u{85}', '\u{85}'), + ('\u{a0}', '\u{a0}'), + ('\u{1680}', '\u{1680}'), + ('\u{2000}', '\u{200a}'), + ('\u{2028}', '\u{2029}'), + ('\u{202f}', '\u{202f}'), + ('\u{205f}', '\u{205f}'), + ('\u{3000}', '\u{3000}'), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_word.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_word.rs new file mode 100644 index 0000000..21c8c0f --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/perl_word.rs @@ -0,0 +1,806 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate perl-word ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const PERL_WORD: &'static [(char, char)] = &[ + ('0', '9'), + ('A', 'Z'), + ('_', '_'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('\u{300}', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('\u{483}', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', 'ՙ'), + ('ՠ', 'ֈ'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('\u{610}', '\u{61a}'), + ('ؠ', '٩'), + ('ٮ', 'ۓ'), + ('ە', '\u{6dc}'), + ('\u{6df}', '\u{6e8}'), + ('\u{6ea}', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', '\u{74a}'), + ('ݍ', 'ޱ'), + ('߀', 'ߵ'), + ('ߺ', 'ߺ'), + ('\u{7fd}', '\u{7fd}'), + ('ࠀ', '\u{82d}'), + ('ࡀ', '\u{85b}'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('\u{897}', '\u{8e1}'), + ('\u{8e3}', '\u{963}'), + ('०', '९'), + ('ॱ', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('\u{9bc}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', 'ৎ'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', '\u{9e3}'), + ('০', 'ৱ'), + ('ৼ', 'ৼ'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', 'ਃ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('੦', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('\u{abc}', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('ૐ', 'ૐ'), + ('ૠ', '\u{ae3}'), + ('૦', '૯'), + ('ૹ', '\u{aff}'), + ('\u{b01}', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('\u{b3c}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', '\u{b63}'), + ('୦', '୯'), + ('ୱ', 'ୱ'), + ('\u{b82}', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('ௐ', 'ௐ'), + ('\u{bd7}', '\u{bd7}'), + ('௦', '௯'), + ('\u{c00}', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('\u{c3c}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', '\u{c63}'), + ('౦', '౯'), + ('ಀ', 'ಃ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('\u{cbc}', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('ೝ', 'ೞ'), + ('ೠ', '\u{ce3}'), + ('೦', '೯'), + ('ೱ', 'ೳ'), + ('\u{d00}', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', 'ൎ'), + ('ൔ', '\u{d57}'), + ('ൟ', '\u{d63}'), + ('൦', '൯'), + ('ൺ', 'ൿ'), + ('\u{d81}', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('෦', '෯'), + ('ෲ', 'ෳ'), + ('ก', '\u{e3a}'), + ('เ', '\u{e4e}'), + ('๐', '๙'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('\u{ec8}', '\u{ece}'), + ('໐', '໙'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('\u{f18}', '\u{f19}'), + ('༠', '༩'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('༾', 'ཇ'), + ('ཉ', 'ཬ'), + ('\u{f71}', '\u{f84}'), + ('\u{f86}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('က', '၉'), + ('ၐ', '\u{109d}'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('\u{135d}', '\u{135f}'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', '\u{1715}'), + ('ᜟ', '\u{1734}'), + ('ᝀ', '\u{1753}'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('\u{1772}', '\u{1773}'), + ('ក', '\u{17d3}'), + ('ៗ', 'ៗ'), + ('ៜ', '\u{17dd}'), + ('០', '៩'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '᠙'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('᥆', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('᧐', '᧙'), + ('ᨀ', '\u{1a1b}'), + ('ᨠ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '᪉'), + ('᪐', '᪙'), + ('ᪧ', 'ᪧ'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', 'ᭌ'), + ('᭐', '᭙'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1bf3}'), + ('ᰀ', '\u{1c37}'), + ('᱀', '᱉'), + ('ᱍ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', 'ᳺ'), + ('ᴀ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('\u{200c}', '\u{200d}'), + ('‿', '⁀'), + ('⁔', '⁔'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('\u{20d0}', '\u{20f0}'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℯ', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⓐ', 'ⓩ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('\u{2d7f}', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('\u{2de0}', '\u{2dff}'), + ('ⸯ', 'ⸯ'), + ('々', '〇'), + ('〡', '\u{302f}'), + ('〱', '〵'), + ('〸', '〼'), + ('ぁ', 'ゖ'), + ('\u{3099}', '\u{309a}'), + ('ゝ', 'ゟ'), + ('ァ', 'ヺ'), + ('ー', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘫ'), + ('Ꙁ', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('ꙿ', '\u{a6f1}'), + ('ꜗ', 'ꜟ'), + ('Ꜣ', 'ꞈ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠧ'), + ('\u{a82c}', '\u{a82c}'), + ('ꡀ', 'ꡳ'), + ('ꢀ', '\u{a8c5}'), + ('꣐', '꣙'), + ('\u{a8e0}', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', '\u{a92d}'), + ('ꤰ', '\u{a953}'), + ('ꥠ', 'ꥼ'), + ('\u{a980}', '\u{a9c0}'), + ('ꧏ', '꧙'), + ('ꧠ', 'ꧾ'), + ('ꨀ', '\u{aa36}'), + ('ꩀ', 'ꩍ'), + ('꩐', '꩙'), + ('ꩠ', 'ꩶ'), + ('ꩺ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫯ'), + ('ꫲ', '\u{aaf6}'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꯪ'), + ('꯬', '\u{abed}'), + ('꯰', '꯹'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('︳', '︴'), + ('﹍', '﹏'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('０', '９'), + ('Ａ', 'Ｚ'), + ('＿', '＿'), + ('ａ', 'ｚ'), + ('ｦ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('\u{101fd}', '\u{101fd}'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('\u{102e0}', '\u{102e0}'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '\u{1037a}'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐐀', '𐒝'), + ('𐒠', '𐒩'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '\u{10ae6}'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '\u{10d27}'), + ('𐴰', '𐴹'), + ('𐵀', '𐵥'), + ('\u{10d69}', '\u{10d6d}'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('\u{10eab}', '\u{10eac}'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('\u{10efc}', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '\u{10f50}'), + ('𐽰', '\u{10f85}'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀀', '\u{11046}'), + ('𑁦', '𑁵'), + ('\u{1107f}', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('𑃐', '𑃨'), + ('𑃰', '𑃹'), + ('\u{11100}', '\u{11134}'), + ('𑄶', '𑄿'), + ('𑅄', '𑅇'), + ('𑅐', '\u{11173}'), + ('𑅶', '𑅶'), + ('\u{11180}', '𑇄'), + ('\u{111c9}', '\u{111cc}'), + ('𑇎', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '\u{11237}'), + ('\u{1123e}', '\u{11241}'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '\u{112ea}'), + ('𑋰', '𑋹'), + ('\u{11300}', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('\u{1133b}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('𑍐', '𑍐'), + ('\u{11357}', '\u{11357}'), + ('𑍝', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏓'), + ('\u{113e1}', '\u{113e2}'), + ('𑐀', '𑑊'), + ('𑑐', '𑑙'), + ('\u{1145e}', '𑑡'), + ('𑒀', '𑓅'), + ('𑓇', '𑓇'), + ('𑓐', '𑓙'), + ('𑖀', '\u{115b5}'), + ('𑖸', '\u{115c0}'), + ('𑗘', '\u{115dd}'), + ('𑘀', '\u{11640}'), + ('𑙄', '𑙄'), + ('𑙐', '𑙙'), + ('𑚀', '𑚸'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜀', '𑜚'), + ('\u{1171d}', '\u{1172b}'), + ('𑜰', '𑜹'), + ('𑝀', '𑝆'), + ('𑠀', '\u{1183a}'), + ('𑢠', '𑣩'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{11943}'), + ('𑥐', '𑥙'), + ('𑦠', '𑦧'), + ('𑦪', '\u{119d7}'), + ('\u{119da}', '𑧡'), + ('𑧣', '𑧤'), + ('𑨀', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('𑩐', '\u{11a99}'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑯰', '𑯹'), + ('𑰀', '𑰈'), + ('𑰊', '\u{11c36}'), + ('\u{11c38}', '𑱀'), + ('𑱐', '𑱙'), + ('𑱲', '𑲏'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d47}'), + ('𑵐', '𑵙'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶘'), + ('𑶠', '𑶩'), + ('𑻠', '𑻶'), + ('\u{11f00}', '𑼐'), + ('𑼒', '\u{11f3a}'), + ('𑼾', '\u{11f42}'), + ('𑽐', '\u{11f5a}'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('\u{13440}', '\u{13455}'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄹'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩠', '𖩩'), + ('𖩰', '𖪾'), + ('𖫀', '𖫉'), + ('𖫐', '𖫭'), + ('\u{16af0}', '\u{16af4}'), + ('𖬀', '\u{16b36}'), + ('𖭀', '𖭃'), + ('𖭐', '𖭙'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖵰', '𖵹'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('\u{16f4f}', '𖾇'), + ('\u{16f8f}', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('𜳰', '𜳹'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝟎', '𝟿'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), + ('𞄀', '𞄬'), + ('\u{1e130}', '𞄽'), + ('𞅀', '𞅉'), + ('𞅎', '𞅎'), + ('𞊐', '\u{1e2ae}'), + ('𞋀', '𞋹'), + ('𞓐', '𞓹'), + ('𞗐', '𞗺'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('𞤀', '𞥋'), + ('𞥐', '𞥙'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('🄰', '🅉'), + ('🅐', '🅩'), + ('🅰', '🆉'), + ('🯰', '🯹'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), + ('\u{e0100}', '\u{e01ef}'), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_bool.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_bool.rs new file mode 100644 index 0000000..3d62edc --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_bool.rs @@ -0,0 +1,12095 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate property-bool ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = &[ + ("ASCII_Hex_Digit", ASCII_HEX_DIGIT), + ("Alphabetic", ALPHABETIC), + ("Bidi_Control", BIDI_CONTROL), + ("Bidi_Mirrored", BIDI_MIRRORED), + ("Case_Ignorable", CASE_IGNORABLE), + ("Cased", CASED), + ("Changes_When_Casefolded", CHANGES_WHEN_CASEFOLDED), + ("Changes_When_Casemapped", CHANGES_WHEN_CASEMAPPED), + ("Changes_When_Lowercased", CHANGES_WHEN_LOWERCASED), + ("Changes_When_Titlecased", CHANGES_WHEN_TITLECASED), + ("Changes_When_Uppercased", CHANGES_WHEN_UPPERCASED), + ("Dash", DASH), + ("Default_Ignorable_Code_Point", DEFAULT_IGNORABLE_CODE_POINT), + ("Deprecated", DEPRECATED), + ("Diacritic", DIACRITIC), + ("Emoji", EMOJI), + ("Emoji_Component", EMOJI_COMPONENT), + ("Emoji_Modifier", EMOJI_MODIFIER), + ("Emoji_Modifier_Base", EMOJI_MODIFIER_BASE), + ("Emoji_Presentation", EMOJI_PRESENTATION), + ("Extended_Pictographic", EXTENDED_PICTOGRAPHIC), + ("Extender", EXTENDER), + ("Grapheme_Base", GRAPHEME_BASE), + ("Grapheme_Extend", GRAPHEME_EXTEND), + ("Grapheme_Link", GRAPHEME_LINK), + ("Hex_Digit", HEX_DIGIT), + ("Hyphen", HYPHEN), + ("IDS_Binary_Operator", IDS_BINARY_OPERATOR), + ("IDS_Trinary_Operator", IDS_TRINARY_OPERATOR), + ("IDS_Unary_Operator", IDS_UNARY_OPERATOR), + ("ID_Compat_Math_Continue", ID_COMPAT_MATH_CONTINUE), + ("ID_Compat_Math_Start", ID_COMPAT_MATH_START), + ("ID_Continue", ID_CONTINUE), + ("ID_Start", ID_START), + ("Ideographic", IDEOGRAPHIC), + ("InCB", INCB), + ("Join_Control", JOIN_CONTROL), + ("Logical_Order_Exception", LOGICAL_ORDER_EXCEPTION), + ("Lowercase", LOWERCASE), + ("Math", MATH), + ("Modifier_Combining_Mark", MODIFIER_COMBINING_MARK), + ("Noncharacter_Code_Point", NONCHARACTER_CODE_POINT), + ("Other_Alphabetic", OTHER_ALPHABETIC), + ("Other_Default_Ignorable_Code_Point", OTHER_DEFAULT_IGNORABLE_CODE_POINT), + ("Other_Grapheme_Extend", OTHER_GRAPHEME_EXTEND), + ("Other_ID_Continue", OTHER_ID_CONTINUE), + ("Other_ID_Start", OTHER_ID_START), + ("Other_Lowercase", OTHER_LOWERCASE), + ("Other_Math", OTHER_MATH), + ("Other_Uppercase", OTHER_UPPERCASE), + ("Pattern_Syntax", PATTERN_SYNTAX), + ("Pattern_White_Space", PATTERN_WHITE_SPACE), + ("Prepended_Concatenation_Mark", PREPENDED_CONCATENATION_MARK), + ("Quotation_Mark", QUOTATION_MARK), + ("Radical", RADICAL), + ("Regional_Indicator", REGIONAL_INDICATOR), + ("Sentence_Terminal", SENTENCE_TERMINAL), + ("Soft_Dotted", SOFT_DOTTED), + ("Terminal_Punctuation", TERMINAL_PUNCTUATION), + ("Unified_Ideograph", UNIFIED_IDEOGRAPH), + ("Uppercase", UPPERCASE), + ("Variation_Selector", VARIATION_SELECTOR), + ("White_Space", WHITE_SPACE), + ("XID_Continue", XID_CONTINUE), + ("XID_Start", XID_START), +]; + +pub const ASCII_HEX_DIGIT: &'static [(char, char)] = + &[('0', '9'), ('A', 'F'), ('a', 'f')]; + +pub const ALPHABETIC: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('\u{345}', '\u{345}'), + ('\u{363}', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', 'ՙ'), + ('ՠ', 'ֈ'), + ('\u{5b0}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('\u{610}', '\u{61a}'), + ('ؠ', '\u{657}'), + ('\u{659}', '\u{65f}'), + ('ٮ', 'ۓ'), + ('ە', '\u{6dc}'), + ('\u{6e1}', '\u{6e8}'), + ('\u{6ed}', 'ۯ'), + ('ۺ', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', '\u{73f}'), + ('ݍ', 'ޱ'), + ('ߊ', 'ߪ'), + ('ߴ', 'ߵ'), + ('ߺ', 'ߺ'), + ('ࠀ', '\u{817}'), + ('ࠚ', '\u{82c}'), + ('ࡀ', 'ࡘ'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('\u{897}', '\u{897}'), + ('ࢠ', 'ࣉ'), + ('\u{8d4}', '\u{8df}'), + ('\u{8e3}', '\u{8e9}'), + ('\u{8f0}', 'ऻ'), + ('ऽ', 'ौ'), + ('ॎ', 'ॐ'), + ('\u{955}', '\u{963}'), + ('ॱ', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('ঽ', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', 'ৌ'), + ('ৎ', 'ৎ'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', '\u{9e3}'), + ('ৰ', 'ৱ'), + ('ৼ', 'ৼ'), + ('\u{a01}', 'ਃ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4c}'), + ('\u{a51}', '\u{a51}'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('\u{a70}', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('ઽ', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', 'ૌ'), + ('ૐ', 'ૐ'), + ('ૠ', '\u{ae3}'), + ('ૹ', '\u{afc}'), + ('\u{b01}', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('ଽ', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', 'ୌ'), + ('\u{b56}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', '\u{b63}'), + ('ୱ', 'ୱ'), + ('\u{b82}', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', 'ௌ'), + ('ௐ', 'ௐ'), + ('\u{bd7}', '\u{bd7}'), + ('\u{c00}', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('ఽ', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4c}'), + ('\u{c55}', '\u{c56}'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', '\u{c63}'), + ('ಀ', 'ಃ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಽ', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccc}'), + ('\u{cd5}', '\u{cd6}'), + ('ೝ', 'ೞ'), + ('ೠ', '\u{ce3}'), + ('ೱ', 'ೳ'), + ('\u{d00}', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ഺ'), + ('ഽ', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', 'ൌ'), + ('ൎ', 'ൎ'), + ('ൔ', '\u{d57}'), + ('ൟ', '\u{d63}'), + ('ൺ', 'ൿ'), + ('\u{d81}', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('ෲ', 'ෳ'), + ('ก', '\u{e3a}'), + ('เ', 'ๆ'), + ('\u{e4d}', '\u{e4d}'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', '\u{eb9}'), + ('\u{ebb}', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('\u{ecd}', '\u{ecd}'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('ཀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('\u{f71}', '\u{f83}'), + ('ྈ', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('က', '\u{1036}'), + ('း', 'း'), + ('ျ', 'ဿ'), + ('ၐ', 'ႏ'), + ('ႚ', '\u{109d}'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', '\u{1713}'), + ('ᜟ', '\u{1733}'), + ('ᝀ', '\u{1753}'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('\u{1772}', '\u{1773}'), + ('ក', 'ឳ'), + ('ា', 'ៈ'), + ('ៗ', 'ៗ'), + ('ៜ', 'ៜ'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', 'ᤸ'), + ('ᥐ', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('ᨀ', '\u{1a1b}'), + ('ᨠ', '\u{1a5e}'), + ('ᩡ', '\u{1a74}'), + ('ᪧ', 'ᪧ'), + ('\u{1abf}', '\u{1ac0}'), + ('\u{1acc}', '\u{1ace}'), + ('\u{1b00}', 'ᬳ'), + ('\u{1b35}', '\u{1b43}'), + ('ᭅ', 'ᭌ'), + ('\u{1b80}', '\u{1ba9}'), + ('\u{1bac}', 'ᮯ'), + ('ᮺ', 'ᯥ'), + ('ᯧ', '\u{1bf1}'), + ('ᰀ', '\u{1c36}'), + ('ᱍ', 'ᱏ'), + ('ᱚ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', 'ᳶ'), + ('ᳺ', 'ᳺ'), + ('ᴀ', 'ᶿ'), + ('\u{1dd3}', '\u{1df4}'), + ('Ḁ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℯ', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⓐ', 'ⓩ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('\u{2de0}', '\u{2dff}'), + ('ⸯ', 'ⸯ'), + ('々', '〇'), + ('〡', '〩'), + ('〱', '〵'), + ('〸', '〼'), + ('ぁ', 'ゖ'), + ('ゝ', 'ゟ'), + ('ァ', 'ヺ'), + ('ー', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘟ'), + ('ꘪ', 'ꘫ'), + ('Ꙁ', 'ꙮ'), + ('\u{a674}', '\u{a67b}'), + ('ꙿ', 'ꛯ'), + ('ꜗ', 'ꜟ'), + ('Ꜣ', 'ꞈ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠅ'), + ('ꠇ', 'ꠧ'), + ('ꡀ', 'ꡳ'), + ('ꢀ', 'ꣃ'), + ('\u{a8c5}', '\u{a8c5}'), + ('ꣲ', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', '\u{a8ff}'), + ('ꤊ', '\u{a92a}'), + ('ꤰ', 'ꥒ'), + ('ꥠ', 'ꥼ'), + ('\u{a980}', 'ꦲ'), + ('ꦴ', 'ꦿ'), + ('ꧏ', 'ꧏ'), + ('ꧠ', 'ꧯ'), + ('ꧺ', 'ꧾ'), + ('ꨀ', '\u{aa36}'), + ('ꩀ', 'ꩍ'), + ('ꩠ', 'ꩶ'), + ('ꩺ', '\u{aabe}'), + ('ꫀ', 'ꫀ'), + ('ꫂ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫯ'), + ('ꫲ', 'ꫵ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꯪ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('ｦ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '\u{1037a}'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐐀', '𐒝'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '𐫤'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '\u{10d27}'), + ('𐵊', '𐵥'), + ('\u{10d69}', '\u{10d69}'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('\u{10eab}', '\u{10eac}'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('\u{10efc}', '\u{10efc}'), + ('𐼀', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '𐽅'), + ('𐽰', '𐾁'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀀', '\u{11045}'), + ('𑁱', '𑁵'), + ('\u{11080}', '𑂸'), + ('\u{110c2}', '\u{110c2}'), + ('𑃐', '𑃨'), + ('\u{11100}', '\u{11132}'), + ('𑅄', '𑅇'), + ('𑅐', '𑅲'), + ('𑅶', '𑅶'), + ('\u{11180}', '𑆿'), + ('𑇁', '𑇄'), + ('𑇎', '\u{111cf}'), + ('𑇚', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '\u{11234}'), + ('\u{11237}', '\u{11237}'), + ('\u{1123e}', '\u{11241}'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '\u{112e8}'), + ('\u{11300}', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('𑌽', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '𑍌'), + ('𑍐', '𑍐'), + ('\u{11357}', '\u{11357}'), + ('𑍝', '𑍣'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏍'), + ('𑏑', '𑏑'), + ('𑏓', '𑏓'), + ('𑐀', '𑑁'), + ('\u{11443}', '𑑅'), + ('𑑇', '𑑊'), + ('𑑟', '𑑡'), + ('𑒀', '𑓁'), + ('𑓄', '𑓅'), + ('𑓇', '𑓇'), + ('𑖀', '\u{115b5}'), + ('𑖸', '𑖾'), + ('𑗘', '\u{115dd}'), + ('𑘀', '𑘾'), + ('\u{11640}', '\u{11640}'), + ('𑙄', '𑙄'), + ('𑚀', '\u{116b5}'), + ('𑚸', '𑚸'), + ('𑜀', '𑜚'), + ('\u{1171d}', '\u{1172a}'), + ('𑝀', '𑝆'), + ('𑠀', '𑠸'), + ('𑢠', '𑣟'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{1193c}'), + ('𑤿', '𑥂'), + ('𑦠', '𑦧'), + ('𑦪', '\u{119d7}'), + ('\u{119da}', '𑧟'), + ('𑧡', '𑧡'), + ('𑧣', '𑧤'), + ('𑨀', '𑨲'), + ('\u{11a35}', '\u{11a3e}'), + ('𑩐', '𑪗'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑰀', '𑰈'), + ('𑰊', '\u{11c36}'), + ('\u{11c38}', '𑰾'), + ('𑱀', '𑱀'), + ('𑱲', '𑲏'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d41}'), + ('\u{11d43}', '\u{11d43}'), + ('𑵆', '\u{11d47}'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶖'), + ('𑶘', '𑶘'), + ('𑻠', '𑻶'), + ('\u{11f00}', '𑼐'), + ('𑼒', '\u{11f3a}'), + ('𑼾', '\u{11f40}'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('𓑁', '𓑆'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '\u{1612e}'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩰', '𖪾'), + ('𖫐', '𖫭'), + ('𖬀', '𖬯'), + ('𖭀', '𖭃'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('\u{16f4f}', '𖾇'), + ('\u{16f8f}', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '𖿣'), + ('\u{16ff0}', '\u{16ff1}'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('\u{1bc9e}', '\u{1bc9e}'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), + ('𞄀', '𞄬'), + ('𞄷', '𞄽'), + ('𞅎', '𞅎'), + ('𞊐', '𞊭'), + ('𞋀', '𞋫'), + ('𞓐', '𞓫'), + ('𞗐', '𞗭'), + ('𞗰', '𞗰'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('𞤀', '𞥃'), + ('\u{1e947}', '\u{1e947}'), + ('𞥋', '𞥋'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('🄰', '🅉'), + ('🅐', '🅩'), + ('🅰', '🆉'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const BIDI_CONTROL: &'static [(char, char)] = &[ + ('\u{61c}', '\u{61c}'), + ('\u{200e}', '\u{200f}'), + ('\u{202a}', '\u{202e}'), + ('\u{2066}', '\u{2069}'), +]; + +pub const BIDI_MIRRORED: &'static [(char, char)] = &[ + ('(', ')'), + ('<', '<'), + ('>', '>'), + ('[', '['), + (']', ']'), + ('{', '{'), + ('}', '}'), + ('«', '«'), + ('»', '»'), + ('༺', '༽'), + ('᚛', '᚜'), + ('‹', '›'), + ('⁅', '⁆'), + ('⁽', '⁾'), + ('₍', '₎'), + ('⅀', '⅀'), + ('∁', '∄'), + ('∈', '∍'), + ('∑', '∑'), + ('∕', '∖'), + ('√', '∝'), + ('∟', '∢'), + ('∤', '∤'), + ('∦', '∦'), + ('∫', '∳'), + ('∹', '∹'), + ('∻', '≌'), + ('≒', '≕'), + ('≟', '≠'), + ('≢', '≢'), + ('≤', '≫'), + ('≭', '⊌'), + ('⊏', '⊒'), + ('⊘', '⊘'), + ('⊢', '⊣'), + ('⊦', '⊸'), + ('⊾', '⊿'), + ('⋉', '⋍'), + ('⋐', '⋑'), + ('⋖', '⋭'), + ('⋰', '⋿'), + ('⌈', '⌋'), + ('⌠', '⌡'), + ('〈', '〉'), + ('❨', '❵'), + ('⟀', '⟀'), + ('⟃', '⟆'), + ('⟈', '⟉'), + ('⟋', '⟍'), + ('⟓', '⟖'), + ('⟜', '⟞'), + ('⟢', '⟯'), + ('⦃', '⦘'), + ('⦛', '⦠'), + ('⦢', '⦯'), + ('⦸', '⦸'), + ('⧀', '⧅'), + ('⧉', '⧉'), + ('⧎', '⧒'), + ('⧔', '⧕'), + ('⧘', '⧜'), + ('⧡', '⧡'), + ('⧣', '⧥'), + ('⧨', '⧩'), + ('⧴', '⧹'), + ('⧼', '⧽'), + ('⨊', '⨜'), + ('⨞', '⨡'), + ('⨤', '⨤'), + ('⨦', '⨦'), + ('⨩', '⨩'), + ('⨫', '⨮'), + ('⨴', '⨵'), + ('⨼', '⨾'), + ('⩗', '⩘'), + ('⩤', '⩥'), + ('⩪', '⩭'), + ('⩯', '⩰'), + ('⩳', '⩴'), + ('⩹', '⪣'), + ('⪦', '⪭'), + ('⪯', '⫖'), + ('⫝̸', '⫝̸'), + ('⫞', '⫞'), + ('⫢', '⫦'), + ('⫬', '⫮'), + ('⫳', '⫳'), + ('⫷', '⫻'), + ('⫽', '⫽'), + ('⯾', '⯾'), + ('⸂', '⸅'), + ('⸉', '⸊'), + ('⸌', '⸍'), + ('⸜', '⸝'), + ('⸠', '⸩'), + ('⹕', '⹜'), + ('〈', '】'), + ('〔', '〛'), + ('﹙', '﹞'), + ('﹤', '﹥'), + ('（', '）'), + ('＜', '＜'), + ('＞', '＞'), + ('［', '［'), + ('］', '］'), + ('｛', '｛'), + ('｝', '｝'), + ('｟', '｠'), + ('｢', '｣'), + ('𝛛', '𝛛'), + ('𝜕', '𝜕'), + ('𝝏', '𝝏'), + ('𝞉', '𝞉'), + ('𝟃', '𝟃'), +]; + +pub const CASE_IGNORABLE: &'static [(char, char)] = &[ + ('\'', '\''), + ('.', '.'), + (':', ':'), + ('^', '^'), + ('`', '`'), + ('¨', '¨'), + ('\u{ad}', '\u{ad}'), + ('¯', '¯'), + ('´', '´'), + ('·', '¸'), + ('ʰ', '\u{36f}'), + ('ʹ', '͵'), + ('ͺ', 'ͺ'), + ('΄', '΅'), + ('·', '·'), + ('\u{483}', '\u{489}'), + ('ՙ', 'ՙ'), + ('՟', '՟'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('״', '״'), + ('\u{600}', '\u{605}'), + ('\u{610}', '\u{61a}'), + ('\u{61c}', '\u{61c}'), + ('ـ', 'ـ'), + ('\u{64b}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dd}'), + ('\u{6df}', '\u{6e8}'), + ('\u{6ea}', '\u{6ed}'), + ('\u{70f}', '\u{70f}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', 'ߵ'), + ('ߺ', 'ߺ'), + ('\u{7fd}', '\u{7fd}'), + ('\u{816}', '\u{82d}'), + ('\u{859}', '\u{85b}'), + ('࢈', '࢈'), + ('\u{890}', '\u{891}'), + ('\u{897}', '\u{89f}'), + ('ࣉ', '\u{902}'), + ('\u{93a}', '\u{93a}'), + ('\u{93c}', '\u{93c}'), + ('\u{941}', '\u{948}'), + ('\u{94d}', '\u{94d}'), + ('\u{951}', '\u{957}'), + ('\u{962}', '\u{963}'), + ('ॱ', 'ॱ'), + ('\u{981}', '\u{981}'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9c1}', '\u{9c4}'), + ('\u{9cd}', '\u{9cd}'), + ('\u{9e2}', '\u{9e3}'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', '\u{a02}'), + ('\u{a3c}', '\u{a3c}'), + ('\u{a41}', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', '\u{a82}'), + ('\u{abc}', '\u{abc}'), + ('\u{ac1}', '\u{ac5}'), + ('\u{ac7}', '\u{ac8}'), + ('\u{acd}', '\u{acd}'), + ('\u{ae2}', '\u{ae3}'), + ('\u{afa}', '\u{aff}'), + ('\u{b01}', '\u{b01}'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b3f}', '\u{b3f}'), + ('\u{b41}', '\u{b44}'), + ('\u{b4d}', '\u{b4d}'), + ('\u{b55}', '\u{b56}'), + ('\u{b62}', '\u{b63}'), + ('\u{b82}', '\u{b82}'), + ('\u{bc0}', '\u{bc0}'), + ('\u{bcd}', '\u{bcd}'), + ('\u{c00}', '\u{c00}'), + ('\u{c04}', '\u{c04}'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c3e}', '\u{c40}'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', '\u{c81}'), + ('\u{cbc}', '\u{cbc}'), + ('\u{cbf}', '\u{cbf}'), + ('\u{cc6}', '\u{cc6}'), + ('\u{ccc}', '\u{ccd}'), + ('\u{ce2}', '\u{ce3}'), + ('\u{d00}', '\u{d01}'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d41}', '\u{d44}'), + ('\u{d4d}', '\u{d4d}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', '\u{d81}'), + ('\u{dca}', '\u{dca}'), + ('\u{dd2}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('ๆ', '\u{e4e}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{ebc}'), + ('ໆ', 'ໆ'), + ('\u{ec8}', '\u{ece}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('\u{f71}', '\u{f7e}'), + ('\u{f80}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('\u{102d}', '\u{1030}'), + ('\u{1032}', '\u{1037}'), + ('\u{1039}', '\u{103a}'), + ('\u{103d}', '\u{103e}'), + ('\u{1058}', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{1082}'), + ('\u{1085}', '\u{1086}'), + ('\u{108d}', '\u{108d}'), + ('\u{109d}', '\u{109d}'), + ('ჼ', 'ჼ'), + ('\u{135d}', '\u{135f}'), + ('\u{1712}', '\u{1714}'), + ('\u{1732}', '\u{1733}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('\u{17b4}', '\u{17b5}'), + ('\u{17b7}', '\u{17bd}'), + ('\u{17c6}', '\u{17c6}'), + ('\u{17c9}', '\u{17d3}'), + ('ៗ', 'ៗ'), + ('\u{17dd}', '\u{17dd}'), + ('\u{180b}', '\u{180f}'), + ('ᡃ', 'ᡃ'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', '\u{1922}'), + ('\u{1927}', '\u{1928}'), + ('\u{1932}', '\u{1932}'), + ('\u{1939}', '\u{193b}'), + ('\u{1a17}', '\u{1a18}'), + ('\u{1a1b}', '\u{1a1b}'), + ('\u{1a56}', '\u{1a56}'), + ('\u{1a58}', '\u{1a5e}'), + ('\u{1a60}', '\u{1a60}'), + ('\u{1a62}', '\u{1a62}'), + ('\u{1a65}', '\u{1a6c}'), + ('\u{1a73}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('ᪧ', 'ᪧ'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', '\u{1b03}'), + ('\u{1b34}', '\u{1b34}'), + ('\u{1b36}', '\u{1b3a}'), + ('\u{1b3c}', '\u{1b3c}'), + ('\u{1b42}', '\u{1b42}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1b81}'), + ('\u{1ba2}', '\u{1ba5}'), + ('\u{1ba8}', '\u{1ba9}'), + ('\u{1bab}', '\u{1bad}'), + ('\u{1be6}', '\u{1be6}'), + ('\u{1be8}', '\u{1be9}'), + ('\u{1bed}', '\u{1bed}'), + ('\u{1bef}', '\u{1bf1}'), + ('\u{1c2c}', '\u{1c33}'), + ('\u{1c36}', '\u{1c37}'), + ('ᱸ', 'ᱽ'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce0}'), + ('\u{1ce2}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('\u{1cf8}', '\u{1cf9}'), + ('ᴬ', 'ᵪ'), + ('ᵸ', 'ᵸ'), + ('ᶛ', '\u{1dff}'), + ('᾽', '᾽'), + ('᾿', '῁'), + ('῍', '῏'), + ('῝', '῟'), + ('῭', '`'), + ('´', '῾'), + ('\u{200b}', '\u{200f}'), + ('‘', '’'), + ('․', '․'), + ('‧', '‧'), + ('\u{202a}', '\u{202e}'), + ('\u{2060}', '\u{2064}'), + ('\u{2066}', '\u{206f}'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('\u{20d0}', '\u{20f0}'), + ('ⱼ', 'ⱽ'), + ('\u{2cef}', '\u{2cf1}'), + ('ⵯ', 'ⵯ'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{2de0}', '\u{2dff}'), + ('ⸯ', 'ⸯ'), + ('々', '々'), + ('\u{302a}', '\u{302d}'), + ('〱', '〵'), + ('〻', '〻'), + ('\u{3099}', 'ゞ'), + ('ー', 'ヾ'), + ('ꀕ', 'ꀕ'), + ('ꓸ', 'ꓽ'), + ('ꘌ', 'ꘌ'), + ('\u{a66f}', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('ꙿ', 'ꙿ'), + ('ꚜ', '\u{a69f}'), + ('\u{a6f0}', '\u{a6f1}'), + ('꜀', '꜡'), + ('ꝰ', 'ꝰ'), + ('ꞈ', '꞊'), + ('ꟲ', 'ꟴ'), + ('ꟸ', 'ꟹ'), + ('\u{a802}', '\u{a802}'), + ('\u{a806}', '\u{a806}'), + ('\u{a80b}', '\u{a80b}'), + ('\u{a825}', '\u{a826}'), + ('\u{a82c}', '\u{a82c}'), + ('\u{a8c4}', '\u{a8c5}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92d}'), + ('\u{a947}', '\u{a951}'), + ('\u{a980}', '\u{a982}'), + ('\u{a9b3}', '\u{a9b3}'), + ('\u{a9b6}', '\u{a9b9}'), + ('\u{a9bc}', '\u{a9bd}'), + ('ꧏ', 'ꧏ'), + ('\u{a9e5}', 'ꧦ'), + ('\u{aa29}', '\u{aa2e}'), + ('\u{aa31}', '\u{aa32}'), + ('\u{aa35}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', '\u{aa4c}'), + ('ꩰ', 'ꩰ'), + ('\u{aa7c}', '\u{aa7c}'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabf}'), + ('\u{aac1}', '\u{aac1}'), + ('ꫝ', 'ꫝ'), + ('\u{aaec}', '\u{aaed}'), + ('ꫳ', 'ꫴ'), + ('\u{aaf6}', '\u{aaf6}'), + ('꭛', 'ꭟ'), + ('ꭩ', '꭫'), + ('\u{abe5}', '\u{abe5}'), + ('\u{abe8}', '\u{abe8}'), + ('\u{abed}', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('﮲', '﯂'), + ('\u{fe00}', '\u{fe0f}'), + ('︓', '︓'), + ('\u{fe20}', '\u{fe2f}'), + ('﹒', '﹒'), + ('﹕', '﹕'), + ('\u{feff}', '\u{feff}'), + ('＇', '＇'), + ('．', '．'), + ('：', '：'), + ('＾', '＾'), + ('｀', '｀'), + ('ｰ', 'ｰ'), + ('\u{ff9e}', '\u{ff9f}'), + ('￣', '￣'), + ('\u{fff9}', '\u{fffb}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{10376}', '\u{1037a}'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('\u{10d24}', '\u{10d27}'), + ('𐵎', '𐵎'), + ('\u{10d69}', '\u{10d6d}'), + ('𐵯', '𐵯'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('\u{11001}', '\u{11001}'), + ('\u{11038}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{11073}', '\u{11074}'), + ('\u{1107f}', '\u{11081}'), + ('\u{110b3}', '\u{110b6}'), + ('\u{110b9}', '\u{110ba}'), + ('\u{110bd}', '\u{110bd}'), + ('\u{110c2}', '\u{110c2}'), + ('\u{110cd}', '\u{110cd}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{1112b}'), + ('\u{1112d}', '\u{11134}'), + ('\u{11173}', '\u{11173}'), + ('\u{11180}', '\u{11181}'), + ('\u{111b6}', '\u{111be}'), + ('\u{111c9}', '\u{111cc}'), + ('\u{111cf}', '\u{111cf}'), + ('\u{1122f}', '\u{11231}'), + ('\u{11234}', '\u{11234}'), + ('\u{11236}', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112df}'), + ('\u{112e3}', '\u{112ea}'), + ('\u{11300}', '\u{11301}'), + ('\u{1133b}', '\u{1133c}'), + ('\u{11340}', '\u{11340}'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113bb}', '\u{113c0}'), + ('\u{113ce}', '\u{113ce}'), + ('\u{113d0}', '\u{113d0}'), + ('\u{113d2}', '\u{113d2}'), + ('\u{113e1}', '\u{113e2}'), + ('\u{11438}', '\u{1143f}'), + ('\u{11442}', '\u{11444}'), + ('\u{11446}', '\u{11446}'), + ('\u{1145e}', '\u{1145e}'), + ('\u{114b3}', '\u{114b8}'), + ('\u{114ba}', '\u{114ba}'), + ('\u{114bf}', '\u{114c0}'), + ('\u{114c2}', '\u{114c3}'), + ('\u{115b2}', '\u{115b5}'), + ('\u{115bc}', '\u{115bd}'), + ('\u{115bf}', '\u{115c0}'), + ('\u{115dc}', '\u{115dd}'), + ('\u{11633}', '\u{1163a}'), + ('\u{1163d}', '\u{1163d}'), + ('\u{1163f}', '\u{11640}'), + ('\u{116ab}', '\u{116ab}'), + ('\u{116ad}', '\u{116ad}'), + ('\u{116b0}', '\u{116b5}'), + ('\u{116b7}', '\u{116b7}'), + ('\u{1171d}', '\u{1171d}'), + ('\u{1171f}', '\u{1171f}'), + ('\u{11722}', '\u{11725}'), + ('\u{11727}', '\u{1172b}'), + ('\u{1182f}', '\u{11837}'), + ('\u{11839}', '\u{1183a}'), + ('\u{1193b}', '\u{1193c}'), + ('\u{1193e}', '\u{1193e}'), + ('\u{11943}', '\u{11943}'), + ('\u{119d4}', '\u{119d7}'), + ('\u{119da}', '\u{119db}'), + ('\u{119e0}', '\u{119e0}'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a33}', '\u{11a38}'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a51}', '\u{11a56}'), + ('\u{11a59}', '\u{11a5b}'), + ('\u{11a8a}', '\u{11a96}'), + ('\u{11a98}', '\u{11a99}'), + ('\u{11c30}', '\u{11c36}'), + ('\u{11c38}', '\u{11c3d}'), + ('\u{11c3f}', '\u{11c3f}'), + ('\u{11c92}', '\u{11ca7}'), + ('\u{11caa}', '\u{11cb0}'), + ('\u{11cb2}', '\u{11cb3}'), + ('\u{11cb5}', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d45}'), + ('\u{11d47}', '\u{11d47}'), + ('\u{11d90}', '\u{11d91}'), + ('\u{11d95}', '\u{11d95}'), + ('\u{11d97}', '\u{11d97}'), + ('\u{11ef3}', '\u{11ef4}'), + ('\u{11f00}', '\u{11f01}'), + ('\u{11f36}', '\u{11f3a}'), + ('\u{11f40}', '\u{11f40}'), + ('\u{11f42}', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13430}', '\u{13440}'), + ('\u{13447}', '\u{13455}'), + ('\u{1611e}', '\u{16129}'), + ('\u{1612d}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('𖭀', '𖭃'), + ('𖵀', '𖵂'), + ('𖵫', '𖵬'), + ('\u{16f4f}', '\u{16f4f}'), + ('\u{16f8f}', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '\u{16fe4}'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('\u{1bca0}', '\u{1bca3}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d167}', '\u{1d169}'), + ('\u{1d173}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e130}', '𞄽'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('𞓫', '\u{1e4ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '𞥋'), + ('🏻', '🏿'), + ('\u{e0001}', '\u{e0001}'), + ('\u{e0020}', '\u{e007f}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const CASED: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ƺ'), + ('Ƽ', 'ƿ'), + ('DŽ', 'ʓ'), + ('ʕ', 'ʸ'), + ('ˀ', 'ˁ'), + ('ˠ', 'ˤ'), + ('\u{345}', '\u{345}'), + ('Ͱ', 'ͳ'), + ('Ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՠ', 'ֈ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ჿ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᴀ', 'ᶿ'), + ('Ḁ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℯ', 'ℴ'), + ('ℹ', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ⅿ'), + ('Ↄ', 'ↄ'), + ('Ⓐ', 'ⓩ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('Ꙁ', 'ꙭ'), + ('Ꚁ', 'ꚝ'), + ('Ꜣ', 'ꞇ'), + ('Ꞌ', 'ꞎ'), + ('Ꞑ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꟶ'), + ('ꟸ', 'ꟺ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('𐐀', '𐑏'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐞀', '𐞀'), + ('𐞃', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐵐', '𐵥'), + ('𐵰', '𐶅'), + ('𑢠', '𑣟'), + ('𖹀', '𖹿'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝼀', '𝼉'), + ('𝼋', '𝼞'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('𞤀', '𞥃'), + ('🄰', '🅉'), + ('🅐', '🅩'), + ('🅰', '🆉'), +]; + +pub const CHANGES_WHEN_CASEFOLDED: &'static [(char, char)] = &[ + ('A', 'Z'), + ('µ', 'µ'), + ('À', 'Ö'), + ('Ø', 'ß'), + ('Ā', 'Ā'), + ('Ă', 'Ă'), + ('Ą', 'Ą'), + ('Ć', 'Ć'), + ('Ĉ', 'Ĉ'), + ('Ċ', 'Ċ'), + ('Č', 'Č'), + ('Ď', 'Ď'), + ('Đ', 'Đ'), + ('Ē', 'Ē'), + ('Ĕ', 'Ĕ'), + ('Ė', 'Ė'), + ('Ę', 'Ę'), + ('Ě', 'Ě'), + ('Ĝ', 'Ĝ'), + ('Ğ', 'Ğ'), + ('Ġ', 'Ġ'), + ('Ģ', 'Ģ'), + ('Ĥ', 'Ĥ'), + ('Ħ', 'Ħ'), + ('Ĩ', 'Ĩ'), + ('Ī', 'Ī'), + ('Ĭ', 'Ĭ'), + ('Į', 'Į'), + ('İ', 'İ'), + ('IJ', 'IJ'), + ('Ĵ', 'Ĵ'), + ('Ķ', 'Ķ'), + ('Ĺ', 'Ĺ'), + ('Ļ', 'Ļ'), + ('Ľ', 'Ľ'), + ('Ŀ', 'Ŀ'), + ('Ł', 'Ł'), + ('Ń', 'Ń'), + ('Ņ', 'Ņ'), + ('Ň', 'Ň'), + ('ʼn', 'Ŋ'), + ('Ō', 'Ō'), + ('Ŏ', 'Ŏ'), + ('Ő', 'Ő'), + ('Œ', 'Œ'), + ('Ŕ', 'Ŕ'), + ('Ŗ', 'Ŗ'), + ('Ř', 'Ř'), + ('Ś', 'Ś'), + ('Ŝ', 'Ŝ'), + ('Ş', 'Ş'), + ('Š', 'Š'), + ('Ţ', 'Ţ'), + ('Ť', 'Ť'), + ('Ŧ', 'Ŧ'), + ('Ũ', 'Ũ'), + ('Ū', 'Ū'), + ('Ŭ', 'Ŭ'), + ('Ů', 'Ů'), + ('Ű', 'Ű'), + ('Ų', 'Ų'), + ('Ŵ', 'Ŵ'), + ('Ŷ', 'Ŷ'), + ('Ÿ', 'Ź'), + ('Ż', 'Ż'), + ('Ž', 'Ž'), + ('ſ', 'ſ'), + ('Ɓ', 'Ƃ'), + ('Ƅ', 'Ƅ'), + ('Ɔ', 'Ƈ'), + ('Ɖ', 'Ƌ'), + ('Ǝ', 'Ƒ'), + ('Ɠ', 'Ɣ'), + ('Ɩ', 'Ƙ'), + ('Ɯ', 'Ɲ'), + ('Ɵ', 'Ơ'), + ('Ƣ', 'Ƣ'), + ('Ƥ', 'Ƥ'), + ('Ʀ', 'Ƨ'), + ('Ʃ', 'Ʃ'), + ('Ƭ', 'Ƭ'), + ('Ʈ', 'Ư'), + ('Ʊ', 'Ƴ'), + ('Ƶ', 'Ƶ'), + ('Ʒ', 'Ƹ'), + ('Ƽ', 'Ƽ'), + ('DŽ', 'Dž'), + ('LJ', 'Lj'), + ('NJ', 'Nj'), + ('Ǎ', 'Ǎ'), + ('Ǐ', 'Ǐ'), + ('Ǒ', 'Ǒ'), + ('Ǔ', 'Ǔ'), + ('Ǖ', 'Ǖ'), + ('Ǘ', 'Ǘ'), + ('Ǚ', 'Ǚ'), + ('Ǜ', 'Ǜ'), + ('Ǟ', 'Ǟ'), + ('Ǡ', 'Ǡ'), + ('Ǣ', 'Ǣ'), + ('Ǥ', 'Ǥ'), + ('Ǧ', 'Ǧ'), + ('Ǩ', 'Ǩ'), + ('Ǫ', 'Ǫ'), + ('Ǭ', 'Ǭ'), + ('Ǯ', 'Ǯ'), + ('DZ', 'Dz'), + ('Ǵ', 'Ǵ'), + ('Ƕ', 'Ǹ'), + ('Ǻ', 'Ǻ'), + ('Ǽ', 'Ǽ'), + ('Ǿ', 'Ǿ'), + ('Ȁ', 'Ȁ'), + ('Ȃ', 'Ȃ'), + ('Ȅ', 'Ȅ'), + ('Ȇ', 'Ȇ'), + ('Ȉ', 'Ȉ'), + ('Ȋ', 'Ȋ'), + ('Ȍ', 'Ȍ'), + ('Ȏ', 'Ȏ'), + ('Ȑ', 'Ȑ'), + ('Ȓ', 'Ȓ'), + ('Ȕ', 'Ȕ'), + ('Ȗ', 'Ȗ'), + ('Ș', 'Ș'), + ('Ț', 'Ț'), + ('Ȝ', 'Ȝ'), + ('Ȟ', 'Ȟ'), + ('Ƞ', 'Ƞ'), + ('Ȣ', 'Ȣ'), + ('Ȥ', 'Ȥ'), + ('Ȧ', 'Ȧ'), + ('Ȩ', 'Ȩ'), + ('Ȫ', 'Ȫ'), + ('Ȭ', 'Ȭ'), + ('Ȯ', 'Ȯ'), + ('Ȱ', 'Ȱ'), + ('Ȳ', 'Ȳ'), + ('Ⱥ', 'Ȼ'), + ('Ƚ', 'Ⱦ'), + ('Ɂ', 'Ɂ'), + ('Ƀ', 'Ɇ'), + ('Ɉ', 'Ɉ'), + ('Ɋ', 'Ɋ'), + ('Ɍ', 'Ɍ'), + ('Ɏ', 'Ɏ'), + ('\u{345}', '\u{345}'), + ('Ͱ', 'Ͱ'), + ('Ͳ', 'Ͳ'), + ('Ͷ', 'Ͷ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ώ'), + ('Α', 'Ρ'), + ('Σ', 'Ϋ'), + ('ς', 'ς'), + ('Ϗ', 'ϑ'), + ('ϕ', 'ϖ'), + ('Ϙ', 'Ϙ'), + ('Ϛ', 'Ϛ'), + ('Ϝ', 'Ϝ'), + ('Ϟ', 'Ϟ'), + ('Ϡ', 'Ϡ'), + ('Ϣ', 'Ϣ'), + ('Ϥ', 'Ϥ'), + ('Ϧ', 'Ϧ'), + ('Ϩ', 'Ϩ'), + ('Ϫ', 'Ϫ'), + ('Ϭ', 'Ϭ'), + ('Ϯ', 'Ϯ'), + ('ϰ', 'ϱ'), + ('ϴ', 'ϵ'), + ('Ϸ', 'Ϸ'), + ('Ϲ', 'Ϻ'), + ('Ͻ', 'Я'), + ('Ѡ', 'Ѡ'), + ('Ѣ', 'Ѣ'), + ('Ѥ', 'Ѥ'), + ('Ѧ', 'Ѧ'), + ('Ѩ', 'Ѩ'), + ('Ѫ', 'Ѫ'), + ('Ѭ', 'Ѭ'), + ('Ѯ', 'Ѯ'), + ('Ѱ', 'Ѱ'), + ('Ѳ', 'Ѳ'), + ('Ѵ', 'Ѵ'), + ('Ѷ', 'Ѷ'), + ('Ѹ', 'Ѹ'), + ('Ѻ', 'Ѻ'), + ('Ѽ', 'Ѽ'), + ('Ѿ', 'Ѿ'), + ('Ҁ', 'Ҁ'), + ('Ҋ', 'Ҋ'), + ('Ҍ', 'Ҍ'), + ('Ҏ', 'Ҏ'), + ('Ґ', 'Ґ'), + ('Ғ', 'Ғ'), + ('Ҕ', 'Ҕ'), + ('Җ', 'Җ'), + ('Ҙ', 'Ҙ'), + ('Қ', 'Қ'), + ('Ҝ', 'Ҝ'), + ('Ҟ', 'Ҟ'), + ('Ҡ', 'Ҡ'), + ('Ң', 'Ң'), + ('Ҥ', 'Ҥ'), + ('Ҧ', 'Ҧ'), + ('Ҩ', 'Ҩ'), + ('Ҫ', 'Ҫ'), + ('Ҭ', 'Ҭ'), + ('Ү', 'Ү'), + ('Ұ', 'Ұ'), + ('Ҳ', 'Ҳ'), + ('Ҵ', 'Ҵ'), + ('Ҷ', 'Ҷ'), + ('Ҹ', 'Ҹ'), + ('Һ', 'Һ'), + ('Ҽ', 'Ҽ'), + ('Ҿ', 'Ҿ'), + ('Ӏ', 'Ӂ'), + ('Ӄ', 'Ӄ'), + ('Ӆ', 'Ӆ'), + ('Ӈ', 'Ӈ'), + ('Ӊ', 'Ӊ'), + ('Ӌ', 'Ӌ'), + ('Ӎ', 'Ӎ'), + ('Ӑ', 'Ӑ'), + ('Ӓ', 'Ӓ'), + ('Ӕ', 'Ӕ'), + ('Ӗ', 'Ӗ'), + ('Ә', 'Ә'), + ('Ӛ', 'Ӛ'), + ('Ӝ', 'Ӝ'), + ('Ӟ', 'Ӟ'), + ('Ӡ', 'Ӡ'), + ('Ӣ', 'Ӣ'), + ('Ӥ', 'Ӥ'), + ('Ӧ', 'Ӧ'), + ('Ө', 'Ө'), + ('Ӫ', 'Ӫ'), + ('Ӭ', 'Ӭ'), + ('Ӯ', 'Ӯ'), + ('Ӱ', 'Ӱ'), + ('Ӳ', 'Ӳ'), + ('Ӵ', 'Ӵ'), + ('Ӷ', 'Ӷ'), + ('Ӹ', 'Ӹ'), + ('Ӻ', 'Ӻ'), + ('Ӽ', 'Ӽ'), + ('Ӿ', 'Ӿ'), + ('Ԁ', 'Ԁ'), + ('Ԃ', 'Ԃ'), + ('Ԅ', 'Ԅ'), + ('Ԇ', 'Ԇ'), + ('Ԉ', 'Ԉ'), + ('Ԋ', 'Ԋ'), + ('Ԍ', 'Ԍ'), + ('Ԏ', 'Ԏ'), + ('Ԑ', 'Ԑ'), + ('Ԓ', 'Ԓ'), + ('Ԕ', 'Ԕ'), + ('Ԗ', 'Ԗ'), + ('Ԙ', 'Ԙ'), + ('Ԛ', 'Ԛ'), + ('Ԝ', 'Ԝ'), + ('Ԟ', 'Ԟ'), + ('Ԡ', 'Ԡ'), + ('Ԣ', 'Ԣ'), + ('Ԥ', 'Ԥ'), + ('Ԧ', 'Ԧ'), + ('Ԩ', 'Ԩ'), + ('Ԫ', 'Ԫ'), + ('Ԭ', 'Ԭ'), + ('Ԯ', 'Ԯ'), + ('Ա', 'Ֆ'), + ('և', 'և'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'Ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('Ḁ', 'Ḁ'), + ('Ḃ', 'Ḃ'), + ('Ḅ', 'Ḅ'), + ('Ḇ', 'Ḇ'), + ('Ḉ', 'Ḉ'), + ('Ḋ', 'Ḋ'), + ('Ḍ', 'Ḍ'), + ('Ḏ', 'Ḏ'), + ('Ḑ', 'Ḑ'), + ('Ḓ', 'Ḓ'), + ('Ḕ', 'Ḕ'), + ('Ḗ', 'Ḗ'), + ('Ḙ', 'Ḙ'), + ('Ḛ', 'Ḛ'), + ('Ḝ', 'Ḝ'), + ('Ḟ', 'Ḟ'), + ('Ḡ', 'Ḡ'), + ('Ḣ', 'Ḣ'), + ('Ḥ', 'Ḥ'), + ('Ḧ', 'Ḧ'), + ('Ḩ', 'Ḩ'), + ('Ḫ', 'Ḫ'), + ('Ḭ', 'Ḭ'), + ('Ḯ', 'Ḯ'), + ('Ḱ', 'Ḱ'), + ('Ḳ', 'Ḳ'), + ('Ḵ', 'Ḵ'), + ('Ḷ', 'Ḷ'), + ('Ḹ', 'Ḹ'), + ('Ḻ', 'Ḻ'), + ('Ḽ', 'Ḽ'), + ('Ḿ', 'Ḿ'), + ('Ṁ', 'Ṁ'), + ('Ṃ', 'Ṃ'), + ('Ṅ', 'Ṅ'), + ('Ṇ', 'Ṇ'), + ('Ṉ', 'Ṉ'), + ('Ṋ', 'Ṋ'), + ('Ṍ', 'Ṍ'), + ('Ṏ', 'Ṏ'), + ('Ṑ', 'Ṑ'), + ('Ṓ', 'Ṓ'), + ('Ṕ', 'Ṕ'), + ('Ṗ', 'Ṗ'), + ('Ṙ', 'Ṙ'), + ('Ṛ', 'Ṛ'), + ('Ṝ', 'Ṝ'), + ('Ṟ', 'Ṟ'), + ('Ṡ', 'Ṡ'), + ('Ṣ', 'Ṣ'), + ('Ṥ', 'Ṥ'), + ('Ṧ', 'Ṧ'), + ('Ṩ', 'Ṩ'), + ('Ṫ', 'Ṫ'), + ('Ṭ', 'Ṭ'), + ('Ṯ', 'Ṯ'), + ('Ṱ', 'Ṱ'), + ('Ṳ', 'Ṳ'), + ('Ṵ', 'Ṵ'), + ('Ṷ', 'Ṷ'), + ('Ṹ', 'Ṹ'), + ('Ṻ', 'Ṻ'), + ('Ṽ', 'Ṽ'), + ('Ṿ', 'Ṿ'), + ('Ẁ', 'Ẁ'), + ('Ẃ', 'Ẃ'), + ('Ẅ', 'Ẅ'), + ('Ẇ', 'Ẇ'), + ('Ẉ', 'Ẉ'), + ('Ẋ', 'Ẋ'), + ('Ẍ', 'Ẍ'), + ('Ẏ', 'Ẏ'), + ('Ẑ', 'Ẑ'), + ('Ẓ', 'Ẓ'), + ('Ẕ', 'Ẕ'), + ('ẚ', 'ẛ'), + ('ẞ', 'ẞ'), + ('Ạ', 'Ạ'), + ('Ả', 'Ả'), + ('Ấ', 'Ấ'), + ('Ầ', 'Ầ'), + ('Ẩ', 'Ẩ'), + ('Ẫ', 'Ẫ'), + ('Ậ', 'Ậ'), + ('Ắ', 'Ắ'), + ('Ằ', 'Ằ'), + ('Ẳ', 'Ẳ'), + ('Ẵ', 'Ẵ'), + ('Ặ', 'Ặ'), + ('Ẹ', 'Ẹ'), + ('Ẻ', 'Ẻ'), + ('Ẽ', 'Ẽ'), + ('Ế', 'Ế'), + ('Ề', 'Ề'), + ('Ể', 'Ể'), + ('Ễ', 'Ễ'), + ('Ệ', 'Ệ'), + ('Ỉ', 'Ỉ'), + ('Ị', 'Ị'), + ('Ọ', 'Ọ'), + ('Ỏ', 'Ỏ'), + ('Ố', 'Ố'), + ('Ồ', 'Ồ'), + ('Ổ', 'Ổ'), + ('Ỗ', 'Ỗ'), + ('Ộ', 'Ộ'), + ('Ớ', 'Ớ'), + ('Ờ', 'Ờ'), + ('Ở', 'Ở'), + ('Ỡ', 'Ỡ'), + ('Ợ', 'Ợ'), + ('Ụ', 'Ụ'), + ('Ủ', 'Ủ'), + ('Ứ', 'Ứ'), + ('Ừ', 'Ừ'), + ('Ử', 'Ử'), + ('Ữ', 'Ữ'), + ('Ự', 'Ự'), + ('Ỳ', 'Ỳ'), + ('Ỵ', 'Ỵ'), + ('Ỷ', 'Ỷ'), + ('Ỹ', 'Ỹ'), + ('Ỻ', 'Ỻ'), + ('Ỽ', 'Ỽ'), + ('Ỿ', 'Ỿ'), + ('Ἀ', 'Ἇ'), + ('Ἐ', 'Ἕ'), + ('Ἠ', 'Ἧ'), + ('Ἰ', 'Ἷ'), + ('Ὀ', 'Ὅ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'Ὗ'), + ('Ὠ', 'Ὧ'), + ('ᾀ', 'ᾯ'), + ('ᾲ', 'ᾴ'), + ('ᾷ', 'ᾼ'), + ('ῂ', 'ῄ'), + ('ῇ', 'ῌ'), + ('Ῐ', 'Ί'), + ('Ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῷ', 'ῼ'), + ('Ω', 'Ω'), + ('K', 'Å'), + ('Ⅎ', 'Ⅎ'), + ('Ⅰ', 'Ⅿ'), + ('Ↄ', 'Ↄ'), + ('Ⓐ', 'Ⓩ'), + ('Ⰰ', 'Ⱟ'), + ('Ⱡ', 'Ⱡ'), + ('Ɫ', 'Ɽ'), + ('Ⱨ', 'Ⱨ'), + ('Ⱪ', 'Ⱪ'), + ('Ⱬ', 'Ⱬ'), + ('Ɑ', 'Ɒ'), + ('Ⱳ', 'Ⱳ'), + ('Ⱶ', 'Ⱶ'), + ('Ȿ', 'Ⲁ'), + ('Ⲃ', 'Ⲃ'), + ('Ⲅ', 'Ⲅ'), + ('Ⲇ', 'Ⲇ'), + ('Ⲉ', 'Ⲉ'), + ('Ⲋ', 'Ⲋ'), + ('Ⲍ', 'Ⲍ'), + ('Ⲏ', 'Ⲏ'), + ('Ⲑ', 'Ⲑ'), + ('Ⲓ', 'Ⲓ'), + ('Ⲕ', 'Ⲕ'), + ('Ⲗ', 'Ⲗ'), + ('Ⲙ', 'Ⲙ'), + ('Ⲛ', 'Ⲛ'), + ('Ⲝ', 'Ⲝ'), + ('Ⲟ', 'Ⲟ'), + ('Ⲡ', 'Ⲡ'), + ('Ⲣ', 'Ⲣ'), + ('Ⲥ', 'Ⲥ'), + ('Ⲧ', 'Ⲧ'), + ('Ⲩ', 'Ⲩ'), + ('Ⲫ', 'Ⲫ'), + ('Ⲭ', 'Ⲭ'), + ('Ⲯ', 'Ⲯ'), + ('Ⲱ', 'Ⲱ'), + ('Ⲳ', 'Ⲳ'), + ('Ⲵ', 'Ⲵ'), + ('Ⲷ', 'Ⲷ'), + ('Ⲹ', 'Ⲹ'), + ('Ⲻ', 'Ⲻ'), + ('Ⲽ', 'Ⲽ'), + ('Ⲿ', 'Ⲿ'), + ('Ⳁ', 'Ⳁ'), + ('Ⳃ', 'Ⳃ'), + ('Ⳅ', 'Ⳅ'), + ('Ⳇ', 'Ⳇ'), + ('Ⳉ', 'Ⳉ'), + ('Ⳋ', 'Ⳋ'), + ('Ⳍ', 'Ⳍ'), + ('Ⳏ', 'Ⳏ'), + ('Ⳑ', 'Ⳑ'), + ('Ⳓ', 'Ⳓ'), + ('Ⳕ', 'Ⳕ'), + ('Ⳗ', 'Ⳗ'), + ('Ⳙ', 'Ⳙ'), + ('Ⳛ', 'Ⳛ'), + ('Ⳝ', 'Ⳝ'), + ('Ⳟ', 'Ⳟ'), + ('Ⳡ', 'Ⳡ'), + ('Ⳣ', 'Ⳣ'), + ('Ⳬ', 'Ⳬ'), + ('Ⳮ', 'Ⳮ'), + ('Ⳳ', 'Ⳳ'), + ('Ꙁ', 'Ꙁ'), + ('Ꙃ', 'Ꙃ'), + ('Ꙅ', 'Ꙅ'), + ('Ꙇ', 'Ꙇ'), + ('Ꙉ', 'Ꙉ'), + ('Ꙋ', 'Ꙋ'), + ('Ꙍ', 'Ꙍ'), + ('Ꙏ', 'Ꙏ'), + ('Ꙑ', 'Ꙑ'), + ('Ꙓ', 'Ꙓ'), + ('Ꙕ', 'Ꙕ'), + ('Ꙗ', 'Ꙗ'), + ('Ꙙ', 'Ꙙ'), + ('Ꙛ', 'Ꙛ'), + ('Ꙝ', 'Ꙝ'), + ('Ꙟ', 'Ꙟ'), + ('Ꙡ', 'Ꙡ'), + ('Ꙣ', 'Ꙣ'), + ('Ꙥ', 'Ꙥ'), + ('Ꙧ', 'Ꙧ'), + ('Ꙩ', 'Ꙩ'), + ('Ꙫ', 'Ꙫ'), + ('Ꙭ', 'Ꙭ'), + ('Ꚁ', 'Ꚁ'), + ('Ꚃ', 'Ꚃ'), + ('Ꚅ', 'Ꚅ'), + ('Ꚇ', 'Ꚇ'), + ('Ꚉ', 'Ꚉ'), + ('Ꚋ', 'Ꚋ'), + ('Ꚍ', 'Ꚍ'), + ('Ꚏ', 'Ꚏ'), + ('Ꚑ', 'Ꚑ'), + ('Ꚓ', 'Ꚓ'), + ('Ꚕ', 'Ꚕ'), + ('Ꚗ', 'Ꚗ'), + ('Ꚙ', 'Ꚙ'), + ('Ꚛ', 'Ꚛ'), + ('Ꜣ', 'Ꜣ'), + ('Ꜥ', 'Ꜥ'), + ('Ꜧ', 'Ꜧ'), + ('Ꜩ', 'Ꜩ'), + ('Ꜫ', 'Ꜫ'), + ('Ꜭ', 'Ꜭ'), + ('Ꜯ', 'Ꜯ'), + ('Ꜳ', 'Ꜳ'), + ('Ꜵ', 'Ꜵ'), + ('Ꜷ', 'Ꜷ'), + ('Ꜹ', 'Ꜹ'), + ('Ꜻ', 'Ꜻ'), + ('Ꜽ', 'Ꜽ'), + ('Ꜿ', 'Ꜿ'), + ('Ꝁ', 'Ꝁ'), + ('Ꝃ', 'Ꝃ'), + ('Ꝅ', 'Ꝅ'), + ('Ꝇ', 'Ꝇ'), + ('Ꝉ', 'Ꝉ'), + ('Ꝋ', 'Ꝋ'), + ('Ꝍ', 'Ꝍ'), + ('Ꝏ', 'Ꝏ'), + ('Ꝑ', 'Ꝑ'), + ('Ꝓ', 'Ꝓ'), + ('Ꝕ', 'Ꝕ'), + ('Ꝗ', 'Ꝗ'), + ('Ꝙ', 'Ꝙ'), + ('Ꝛ', 'Ꝛ'), + ('Ꝝ', 'Ꝝ'), + ('Ꝟ', 'Ꝟ'), + ('Ꝡ', 'Ꝡ'), + ('Ꝣ', 'Ꝣ'), + ('Ꝥ', 'Ꝥ'), + ('Ꝧ', 'Ꝧ'), + ('Ꝩ', 'Ꝩ'), + ('Ꝫ', 'Ꝫ'), + ('Ꝭ', 'Ꝭ'), + ('Ꝯ', 'Ꝯ'), + ('Ꝺ', 'Ꝺ'), + ('Ꝼ', 'Ꝼ'), + ('Ᵹ', 'Ꝿ'), + ('Ꞁ', 'Ꞁ'), + ('Ꞃ', 'Ꞃ'), + ('Ꞅ', 'Ꞅ'), + ('Ꞇ', 'Ꞇ'), + ('Ꞌ', 'Ꞌ'), + ('Ɥ', 'Ɥ'), + ('Ꞑ', 'Ꞑ'), + ('Ꞓ', 'Ꞓ'), + ('Ꞗ', 'Ꞗ'), + ('Ꞙ', 'Ꞙ'), + ('Ꞛ', 'Ꞛ'), + ('Ꞝ', 'Ꞝ'), + ('Ꞟ', 'Ꞟ'), + ('Ꞡ', 'Ꞡ'), + ('Ꞣ', 'Ꞣ'), + ('Ꞥ', 'Ꞥ'), + ('Ꞧ', 'Ꞧ'), + ('Ꞩ', 'Ꞩ'), + ('Ɦ', 'Ɪ'), + ('Ʞ', 'Ꞵ'), + ('Ꞷ', 'Ꞷ'), + ('Ꞹ', 'Ꞹ'), + ('Ꞻ', 'Ꞻ'), + ('Ꞽ', 'Ꞽ'), + ('Ꞿ', 'Ꞿ'), + ('Ꟁ', 'Ꟁ'), + ('Ꟃ', 'Ꟃ'), + ('Ꞔ', 'Ꟈ'), + ('Ꟊ', 'Ꟊ'), + ('Ɤ', 'Ꟍ'), + ('Ꟑ', 'Ꟑ'), + ('Ꟗ', 'Ꟗ'), + ('Ꟙ', 'Ꟙ'), + ('Ꟛ', 'Ꟛ'), + ('Ƛ', 'Ƛ'), + ('Ꟶ', 'Ꟶ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('Ａ', 'Ｚ'), + ('𐐀', '𐐧'), + ('𐒰', '𐓓'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐲀', '𐲲'), + ('𐵐', '𐵥'), + ('𑢠', '𑢿'), + ('𖹀', '𖹟'), + ('𞤀', '𞤡'), +]; + +pub const CHANGES_WHEN_CASEMAPPED: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('µ', 'µ'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ķ'), + ('Ĺ', 'ƌ'), + ('Ǝ', 'Ʃ'), + ('Ƭ', 'ƹ'), + ('Ƽ', 'ƽ'), + ('ƿ', 'ƿ'), + ('DŽ', 'Ƞ'), + ('Ȣ', 'ȳ'), + ('Ⱥ', 'ɔ'), + ('ɖ', 'ɗ'), + ('ə', 'ə'), + ('ɛ', 'ɜ'), + ('ɠ', 'ɡ'), + ('ɣ', 'ɦ'), + ('ɨ', 'ɬ'), + ('ɯ', 'ɯ'), + ('ɱ', 'ɲ'), + ('ɵ', 'ɵ'), + ('ɽ', 'ɽ'), + ('ʀ', 'ʀ'), + ('ʂ', 'ʃ'), + ('ʇ', 'ʌ'), + ('ʒ', 'ʒ'), + ('ʝ', 'ʞ'), + ('\u{345}', '\u{345}'), + ('Ͱ', 'ͳ'), + ('Ͷ', 'ͷ'), + ('ͻ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϑ'), + ('ϕ', 'ϵ'), + ('Ϸ', 'ϻ'), + ('Ͻ', 'ҁ'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ա', 'և'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჽ', 'ჿ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᵹ', 'ᵹ'), + ('ᵽ', 'ᵽ'), + ('ᶎ', 'ᶎ'), + ('Ḁ', 'ẛ'), + ('ẞ', 'ẞ'), + ('Ạ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('Ω', 'Ω'), + ('K', 'Å'), + ('Ⅎ', 'Ⅎ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ⅿ'), + ('Ↄ', 'ↄ'), + ('Ⓐ', 'ⓩ'), + ('Ⰰ', 'Ɒ'), + ('Ⱳ', 'ⱳ'), + ('Ⱶ', 'ⱶ'), + ('Ȿ', 'ⳣ'), + ('Ⳬ', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('Ꙁ', 'ꙭ'), + ('Ꚁ', 'ꚛ'), + ('Ꜣ', 'ꜯ'), + ('Ꜳ', 'ꝯ'), + ('Ꝺ', 'ꞇ'), + ('Ꞌ', 'Ɥ'), + ('Ꞑ', 'ꞔ'), + ('Ꞗ', 'Ɪ'), + ('Ʞ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('Ꟗ', 'Ƛ'), + ('Ꟶ', 'ꟶ'), + ('ꭓ', 'ꭓ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('𐐀', '𐑏'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐵐', '𐵥'), + ('𐵰', '𐶅'), + ('𑢠', '𑣟'), + ('𖹀', '𖹿'), + ('𞤀', '𞥃'), +]; + +pub const CHANGES_WHEN_LOWERCASED: &'static [(char, char)] = &[ + ('A', 'Z'), + ('À', 'Ö'), + ('Ø', 'Þ'), + ('Ā', 'Ā'), + ('Ă', 'Ă'), + ('Ą', 'Ą'), + ('Ć', 'Ć'), + ('Ĉ', 'Ĉ'), + ('Ċ', 'Ċ'), + ('Č', 'Č'), + ('Ď', 'Ď'), + ('Đ', 'Đ'), + ('Ē', 'Ē'), + ('Ĕ', 'Ĕ'), + ('Ė', 'Ė'), + ('Ę', 'Ę'), + ('Ě', 'Ě'), + ('Ĝ', 'Ĝ'), + ('Ğ', 'Ğ'), + ('Ġ', 'Ġ'), + ('Ģ', 'Ģ'), + ('Ĥ', 'Ĥ'), + ('Ħ', 'Ħ'), + ('Ĩ', 'Ĩ'), + ('Ī', 'Ī'), + ('Ĭ', 'Ĭ'), + ('Į', 'Į'), + ('İ', 'İ'), + ('IJ', 'IJ'), + ('Ĵ', 'Ĵ'), + ('Ķ', 'Ķ'), + ('Ĺ', 'Ĺ'), + ('Ļ', 'Ļ'), + ('Ľ', 'Ľ'), + ('Ŀ', 'Ŀ'), + ('Ł', 'Ł'), + ('Ń', 'Ń'), + ('Ņ', 'Ņ'), + ('Ň', 'Ň'), + ('Ŋ', 'Ŋ'), + ('Ō', 'Ō'), + ('Ŏ', 'Ŏ'), + ('Ő', 'Ő'), + ('Œ', 'Œ'), + ('Ŕ', 'Ŕ'), + ('Ŗ', 'Ŗ'), + ('Ř', 'Ř'), + ('Ś', 'Ś'), + ('Ŝ', 'Ŝ'), + ('Ş', 'Ş'), + ('Š', 'Š'), + ('Ţ', 'Ţ'), + ('Ť', 'Ť'), + ('Ŧ', 'Ŧ'), + ('Ũ', 'Ũ'), + ('Ū', 'Ū'), + ('Ŭ', 'Ŭ'), + ('Ů', 'Ů'), + ('Ű', 'Ű'), + ('Ų', 'Ų'), + ('Ŵ', 'Ŵ'), + ('Ŷ', 'Ŷ'), + ('Ÿ', 'Ź'), + ('Ż', 'Ż'), + ('Ž', 'Ž'), + ('Ɓ', 'Ƃ'), + ('Ƅ', 'Ƅ'), + ('Ɔ', 'Ƈ'), + ('Ɖ', 'Ƌ'), + ('Ǝ', 'Ƒ'), + ('Ɠ', 'Ɣ'), + ('Ɩ', 'Ƙ'), + ('Ɯ', 'Ɲ'), + ('Ɵ', 'Ơ'), + ('Ƣ', 'Ƣ'), + ('Ƥ', 'Ƥ'), + ('Ʀ', 'Ƨ'), + ('Ʃ', 'Ʃ'), + ('Ƭ', 'Ƭ'), + ('Ʈ', 'Ư'), + ('Ʊ', 'Ƴ'), + ('Ƶ', 'Ƶ'), + ('Ʒ', 'Ƹ'), + ('Ƽ', 'Ƽ'), + ('DŽ', 'Dž'), + ('LJ', 'Lj'), + ('NJ', 'Nj'), + ('Ǎ', 'Ǎ'), + ('Ǐ', 'Ǐ'), + ('Ǒ', 'Ǒ'), + ('Ǔ', 'Ǔ'), + ('Ǖ', 'Ǖ'), + ('Ǘ', 'Ǘ'), + ('Ǚ', 'Ǚ'), + ('Ǜ', 'Ǜ'), + ('Ǟ', 'Ǟ'), + ('Ǡ', 'Ǡ'), + ('Ǣ', 'Ǣ'), + ('Ǥ', 'Ǥ'), + ('Ǧ', 'Ǧ'), + ('Ǩ', 'Ǩ'), + ('Ǫ', 'Ǫ'), + ('Ǭ', 'Ǭ'), + ('Ǯ', 'Ǯ'), + ('DZ', 'Dz'), + ('Ǵ', 'Ǵ'), + ('Ƕ', 'Ǹ'), + ('Ǻ', 'Ǻ'), + ('Ǽ', 'Ǽ'), + ('Ǿ', 'Ǿ'), + ('Ȁ', 'Ȁ'), + ('Ȃ', 'Ȃ'), + ('Ȅ', 'Ȅ'), + ('Ȇ', 'Ȇ'), + ('Ȉ', 'Ȉ'), + ('Ȋ', 'Ȋ'), + ('Ȍ', 'Ȍ'), + ('Ȏ', 'Ȏ'), + ('Ȑ', 'Ȑ'), + ('Ȓ', 'Ȓ'), + ('Ȕ', 'Ȕ'), + ('Ȗ', 'Ȗ'), + ('Ș', 'Ș'), + ('Ț', 'Ț'), + ('Ȝ', 'Ȝ'), + ('Ȟ', 'Ȟ'), + ('Ƞ', 'Ƞ'), + ('Ȣ', 'Ȣ'), + ('Ȥ', 'Ȥ'), + ('Ȧ', 'Ȧ'), + ('Ȩ', 'Ȩ'), + ('Ȫ', 'Ȫ'), + ('Ȭ', 'Ȭ'), + ('Ȯ', 'Ȯ'), + ('Ȱ', 'Ȱ'), + ('Ȳ', 'Ȳ'), + ('Ⱥ', 'Ȼ'), + ('Ƚ', 'Ⱦ'), + ('Ɂ', 'Ɂ'), + ('Ƀ', 'Ɇ'), + ('Ɉ', 'Ɉ'), + ('Ɋ', 'Ɋ'), + ('Ɍ', 'Ɍ'), + ('Ɏ', 'Ɏ'), + ('Ͱ', 'Ͱ'), + ('Ͳ', 'Ͳ'), + ('Ͷ', 'Ͷ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ώ'), + ('Α', 'Ρ'), + ('Σ', 'Ϋ'), + ('Ϗ', 'Ϗ'), + ('Ϙ', 'Ϙ'), + ('Ϛ', 'Ϛ'), + ('Ϝ', 'Ϝ'), + ('Ϟ', 'Ϟ'), + ('Ϡ', 'Ϡ'), + ('Ϣ', 'Ϣ'), + ('Ϥ', 'Ϥ'), + ('Ϧ', 'Ϧ'), + ('Ϩ', 'Ϩ'), + ('Ϫ', 'Ϫ'), + ('Ϭ', 'Ϭ'), + ('Ϯ', 'Ϯ'), + ('ϴ', 'ϴ'), + ('Ϸ', 'Ϸ'), + ('Ϲ', 'Ϻ'), + ('Ͻ', 'Я'), + ('Ѡ', 'Ѡ'), + ('Ѣ', 'Ѣ'), + ('Ѥ', 'Ѥ'), + ('Ѧ', 'Ѧ'), + ('Ѩ', 'Ѩ'), + ('Ѫ', 'Ѫ'), + ('Ѭ', 'Ѭ'), + ('Ѯ', 'Ѯ'), + ('Ѱ', 'Ѱ'), + ('Ѳ', 'Ѳ'), + ('Ѵ', 'Ѵ'), + ('Ѷ', 'Ѷ'), + ('Ѹ', 'Ѹ'), + ('Ѻ', 'Ѻ'), + ('Ѽ', 'Ѽ'), + ('Ѿ', 'Ѿ'), + ('Ҁ', 'Ҁ'), + ('Ҋ', 'Ҋ'), + ('Ҍ', 'Ҍ'), + ('Ҏ', 'Ҏ'), + ('Ґ', 'Ґ'), + ('Ғ', 'Ғ'), + ('Ҕ', 'Ҕ'), + ('Җ', 'Җ'), + ('Ҙ', 'Ҙ'), + ('Қ', 'Қ'), + ('Ҝ', 'Ҝ'), + ('Ҟ', 'Ҟ'), + ('Ҡ', 'Ҡ'), + ('Ң', 'Ң'), + ('Ҥ', 'Ҥ'), + ('Ҧ', 'Ҧ'), + ('Ҩ', 'Ҩ'), + ('Ҫ', 'Ҫ'), + ('Ҭ', 'Ҭ'), + ('Ү', 'Ү'), + ('Ұ', 'Ұ'), + ('Ҳ', 'Ҳ'), + ('Ҵ', 'Ҵ'), + ('Ҷ', 'Ҷ'), + ('Ҹ', 'Ҹ'), + ('Һ', 'Һ'), + ('Ҽ', 'Ҽ'), + ('Ҿ', 'Ҿ'), + ('Ӏ', 'Ӂ'), + ('Ӄ', 'Ӄ'), + ('Ӆ', 'Ӆ'), + ('Ӈ', 'Ӈ'), + ('Ӊ', 'Ӊ'), + ('Ӌ', 'Ӌ'), + ('Ӎ', 'Ӎ'), + ('Ӑ', 'Ӑ'), + ('Ӓ', 'Ӓ'), + ('Ӕ', 'Ӕ'), + ('Ӗ', 'Ӗ'), + ('Ә', 'Ә'), + ('Ӛ', 'Ӛ'), + ('Ӝ', 'Ӝ'), + ('Ӟ', 'Ӟ'), + ('Ӡ', 'Ӡ'), + ('Ӣ', 'Ӣ'), + ('Ӥ', 'Ӥ'), + ('Ӧ', 'Ӧ'), + ('Ө', 'Ө'), + ('Ӫ', 'Ӫ'), + ('Ӭ', 'Ӭ'), + ('Ӯ', 'Ӯ'), + ('Ӱ', 'Ӱ'), + ('Ӳ', 'Ӳ'), + ('Ӵ', 'Ӵ'), + ('Ӷ', 'Ӷ'), + ('Ӹ', 'Ӹ'), + ('Ӻ', 'Ӻ'), + ('Ӽ', 'Ӽ'), + ('Ӿ', 'Ӿ'), + ('Ԁ', 'Ԁ'), + ('Ԃ', 'Ԃ'), + ('Ԅ', 'Ԅ'), + ('Ԇ', 'Ԇ'), + ('Ԉ', 'Ԉ'), + ('Ԋ', 'Ԋ'), + ('Ԍ', 'Ԍ'), + ('Ԏ', 'Ԏ'), + ('Ԑ', 'Ԑ'), + ('Ԓ', 'Ԓ'), + ('Ԕ', 'Ԕ'), + ('Ԗ', 'Ԗ'), + ('Ԙ', 'Ԙ'), + ('Ԛ', 'Ԛ'), + ('Ԝ', 'Ԝ'), + ('Ԟ', 'Ԟ'), + ('Ԡ', 'Ԡ'), + ('Ԣ', 'Ԣ'), + ('Ԥ', 'Ԥ'), + ('Ԧ', 'Ԧ'), + ('Ԩ', 'Ԩ'), + ('Ԫ', 'Ԫ'), + ('Ԭ', 'Ԭ'), + ('Ԯ', 'Ԯ'), + ('Ա', 'Ֆ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('Ꭰ', 'Ᏽ'), + ('Ᲊ', 'Ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('Ḁ', 'Ḁ'), + ('Ḃ', 'Ḃ'), + ('Ḅ', 'Ḅ'), + ('Ḇ', 'Ḇ'), + ('Ḉ', 'Ḉ'), + ('Ḋ', 'Ḋ'), + ('Ḍ', 'Ḍ'), + ('Ḏ', 'Ḏ'), + ('Ḑ', 'Ḑ'), + ('Ḓ', 'Ḓ'), + ('Ḕ', 'Ḕ'), + ('Ḗ', 'Ḗ'), + ('Ḙ', 'Ḙ'), + ('Ḛ', 'Ḛ'), + ('Ḝ', 'Ḝ'), + ('Ḟ', 'Ḟ'), + ('Ḡ', 'Ḡ'), + ('Ḣ', 'Ḣ'), + ('Ḥ', 'Ḥ'), + ('Ḧ', 'Ḧ'), + ('Ḩ', 'Ḩ'), + ('Ḫ', 'Ḫ'), + ('Ḭ', 'Ḭ'), + ('Ḯ', 'Ḯ'), + ('Ḱ', 'Ḱ'), + ('Ḳ', 'Ḳ'), + ('Ḵ', 'Ḵ'), + ('Ḷ', 'Ḷ'), + ('Ḹ', 'Ḹ'), + ('Ḻ', 'Ḻ'), + ('Ḽ', 'Ḽ'), + ('Ḿ', 'Ḿ'), + ('Ṁ', 'Ṁ'), + ('Ṃ', 'Ṃ'), + ('Ṅ', 'Ṅ'), + ('Ṇ', 'Ṇ'), + ('Ṉ', 'Ṉ'), + ('Ṋ', 'Ṋ'), + ('Ṍ', 'Ṍ'), + ('Ṏ', 'Ṏ'), + ('Ṑ', 'Ṑ'), + ('Ṓ', 'Ṓ'), + ('Ṕ', 'Ṕ'), + ('Ṗ', 'Ṗ'), + ('Ṙ', 'Ṙ'), + ('Ṛ', 'Ṛ'), + ('Ṝ', 'Ṝ'), + ('Ṟ', 'Ṟ'), + ('Ṡ', 'Ṡ'), + ('Ṣ', 'Ṣ'), + ('Ṥ', 'Ṥ'), + ('Ṧ', 'Ṧ'), + ('Ṩ', 'Ṩ'), + ('Ṫ', 'Ṫ'), + ('Ṭ', 'Ṭ'), + ('Ṯ', 'Ṯ'), + ('Ṱ', 'Ṱ'), + ('Ṳ', 'Ṳ'), + ('Ṵ', 'Ṵ'), + ('Ṷ', 'Ṷ'), + ('Ṹ', 'Ṹ'), + ('Ṻ', 'Ṻ'), + ('Ṽ', 'Ṽ'), + ('Ṿ', 'Ṿ'), + ('Ẁ', 'Ẁ'), + ('Ẃ', 'Ẃ'), + ('Ẅ', 'Ẅ'), + ('Ẇ', 'Ẇ'), + ('Ẉ', 'Ẉ'), + ('Ẋ', 'Ẋ'), + ('Ẍ', 'Ẍ'), + ('Ẏ', 'Ẏ'), + ('Ẑ', 'Ẑ'), + ('Ẓ', 'Ẓ'), + ('Ẕ', 'Ẕ'), + ('ẞ', 'ẞ'), + ('Ạ', 'Ạ'), + ('Ả', 'Ả'), + ('Ấ', 'Ấ'), + ('Ầ', 'Ầ'), + ('Ẩ', 'Ẩ'), + ('Ẫ', 'Ẫ'), + ('Ậ', 'Ậ'), + ('Ắ', 'Ắ'), + ('Ằ', 'Ằ'), + ('Ẳ', 'Ẳ'), + ('Ẵ', 'Ẵ'), + ('Ặ', 'Ặ'), + ('Ẹ', 'Ẹ'), + ('Ẻ', 'Ẻ'), + ('Ẽ', 'Ẽ'), + ('Ế', 'Ế'), + ('Ề', 'Ề'), + ('Ể', 'Ể'), + ('Ễ', 'Ễ'), + ('Ệ', 'Ệ'), + ('Ỉ', 'Ỉ'), + ('Ị', 'Ị'), + ('Ọ', 'Ọ'), + ('Ỏ', 'Ỏ'), + ('Ố', 'Ố'), + ('Ồ', 'Ồ'), + ('Ổ', 'Ổ'), + ('Ỗ', 'Ỗ'), + ('Ộ', 'Ộ'), + ('Ớ', 'Ớ'), + ('Ờ', 'Ờ'), + ('Ở', 'Ở'), + ('Ỡ', 'Ỡ'), + ('Ợ', 'Ợ'), + ('Ụ', 'Ụ'), + ('Ủ', 'Ủ'), + ('Ứ', 'Ứ'), + ('Ừ', 'Ừ'), + ('Ử', 'Ử'), + ('Ữ', 'Ữ'), + ('Ự', 'Ự'), + ('Ỳ', 'Ỳ'), + ('Ỵ', 'Ỵ'), + ('Ỷ', 'Ỷ'), + ('Ỹ', 'Ỹ'), + ('Ỻ', 'Ỻ'), + ('Ỽ', 'Ỽ'), + ('Ỿ', 'Ỿ'), + ('Ἀ', 'Ἇ'), + ('Ἐ', 'Ἕ'), + ('Ἠ', 'Ἧ'), + ('Ἰ', 'Ἷ'), + ('Ὀ', 'Ὅ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'Ὗ'), + ('Ὠ', 'Ὧ'), + ('ᾈ', 'ᾏ'), + ('ᾘ', 'ᾟ'), + ('ᾨ', 'ᾯ'), + ('Ᾰ', 'ᾼ'), + ('Ὲ', 'ῌ'), + ('Ῐ', 'Ί'), + ('Ῠ', 'Ῥ'), + ('Ὸ', 'ῼ'), + ('Ω', 'Ω'), + ('K', 'Å'), + ('Ⅎ', 'Ⅎ'), + ('Ⅰ', 'Ⅿ'), + ('Ↄ', 'Ↄ'), + ('Ⓐ', 'Ⓩ'), + ('Ⰰ', 'Ⱟ'), + ('Ⱡ', 'Ⱡ'), + ('Ɫ', 'Ɽ'), + ('Ⱨ', 'Ⱨ'), + ('Ⱪ', 'Ⱪ'), + ('Ⱬ', 'Ⱬ'), + ('Ɑ', 'Ɒ'), + ('Ⱳ', 'Ⱳ'), + ('Ⱶ', 'Ⱶ'), + ('Ȿ', 'Ⲁ'), + ('Ⲃ', 'Ⲃ'), + ('Ⲅ', 'Ⲅ'), + ('Ⲇ', 'Ⲇ'), + ('Ⲉ', 'Ⲉ'), + ('Ⲋ', 'Ⲋ'), + ('Ⲍ', 'Ⲍ'), + ('Ⲏ', 'Ⲏ'), + ('Ⲑ', 'Ⲑ'), + ('Ⲓ', 'Ⲓ'), + ('Ⲕ', 'Ⲕ'), + ('Ⲗ', 'Ⲗ'), + ('Ⲙ', 'Ⲙ'), + ('Ⲛ', 'Ⲛ'), + ('Ⲝ', 'Ⲝ'), + ('Ⲟ', 'Ⲟ'), + ('Ⲡ', 'Ⲡ'), + ('Ⲣ', 'Ⲣ'), + ('Ⲥ', 'Ⲥ'), + ('Ⲧ', 'Ⲧ'), + ('Ⲩ', 'Ⲩ'), + ('Ⲫ', 'Ⲫ'), + ('Ⲭ', 'Ⲭ'), + ('Ⲯ', 'Ⲯ'), + ('Ⲱ', 'Ⲱ'), + ('Ⲳ', 'Ⲳ'), + ('Ⲵ', 'Ⲵ'), + ('Ⲷ', 'Ⲷ'), + ('Ⲹ', 'Ⲹ'), + ('Ⲻ', 'Ⲻ'), + ('Ⲽ', 'Ⲽ'), + ('Ⲿ', 'Ⲿ'), + ('Ⳁ', 'Ⳁ'), + ('Ⳃ', 'Ⳃ'), + ('Ⳅ', 'Ⳅ'), + ('Ⳇ', 'Ⳇ'), + ('Ⳉ', 'Ⳉ'), + ('Ⳋ', 'Ⳋ'), + ('Ⳍ', 'Ⳍ'), + ('Ⳏ', 'Ⳏ'), + ('Ⳑ', 'Ⳑ'), + ('Ⳓ', 'Ⳓ'), + ('Ⳕ', 'Ⳕ'), + ('Ⳗ', 'Ⳗ'), + ('Ⳙ', 'Ⳙ'), + ('Ⳛ', 'Ⳛ'), + ('Ⳝ', 'Ⳝ'), + ('Ⳟ', 'Ⳟ'), + ('Ⳡ', 'Ⳡ'), + ('Ⳣ', 'Ⳣ'), + ('Ⳬ', 'Ⳬ'), + ('Ⳮ', 'Ⳮ'), + ('Ⳳ', 'Ⳳ'), + ('Ꙁ', 'Ꙁ'), + ('Ꙃ', 'Ꙃ'), + ('Ꙅ', 'Ꙅ'), + ('Ꙇ', 'Ꙇ'), + ('Ꙉ', 'Ꙉ'), + ('Ꙋ', 'Ꙋ'), + ('Ꙍ', 'Ꙍ'), + ('Ꙏ', 'Ꙏ'), + ('Ꙑ', 'Ꙑ'), + ('Ꙓ', 'Ꙓ'), + ('Ꙕ', 'Ꙕ'), + ('Ꙗ', 'Ꙗ'), + ('Ꙙ', 'Ꙙ'), + ('Ꙛ', 'Ꙛ'), + ('Ꙝ', 'Ꙝ'), + ('Ꙟ', 'Ꙟ'), + ('Ꙡ', 'Ꙡ'), + ('Ꙣ', 'Ꙣ'), + ('Ꙥ', 'Ꙥ'), + ('Ꙧ', 'Ꙧ'), + ('Ꙩ', 'Ꙩ'), + ('Ꙫ', 'Ꙫ'), + ('Ꙭ', 'Ꙭ'), + ('Ꚁ', 'Ꚁ'), + ('Ꚃ', 'Ꚃ'), + ('Ꚅ', 'Ꚅ'), + ('Ꚇ', 'Ꚇ'), + ('Ꚉ', 'Ꚉ'), + ('Ꚋ', 'Ꚋ'), + ('Ꚍ', 'Ꚍ'), + ('Ꚏ', 'Ꚏ'), + ('Ꚑ', 'Ꚑ'), + ('Ꚓ', 'Ꚓ'), + ('Ꚕ', 'Ꚕ'), + ('Ꚗ', 'Ꚗ'), + ('Ꚙ', 'Ꚙ'), + ('Ꚛ', 'Ꚛ'), + ('Ꜣ', 'Ꜣ'), + ('Ꜥ', 'Ꜥ'), + ('Ꜧ', 'Ꜧ'), + ('Ꜩ', 'Ꜩ'), + ('Ꜫ', 'Ꜫ'), + ('Ꜭ', 'Ꜭ'), + ('Ꜯ', 'Ꜯ'), + ('Ꜳ', 'Ꜳ'), + ('Ꜵ', 'Ꜵ'), + ('Ꜷ', 'Ꜷ'), + ('Ꜹ', 'Ꜹ'), + ('Ꜻ', 'Ꜻ'), + ('Ꜽ', 'Ꜽ'), + ('Ꜿ', 'Ꜿ'), + ('Ꝁ', 'Ꝁ'), + ('Ꝃ', 'Ꝃ'), + ('Ꝅ', 'Ꝅ'), + ('Ꝇ', 'Ꝇ'), + ('Ꝉ', 'Ꝉ'), + ('Ꝋ', 'Ꝋ'), + ('Ꝍ', 'Ꝍ'), + ('Ꝏ', 'Ꝏ'), + ('Ꝑ', 'Ꝑ'), + ('Ꝓ', 'Ꝓ'), + ('Ꝕ', 'Ꝕ'), + ('Ꝗ', 'Ꝗ'), + ('Ꝙ', 'Ꝙ'), + ('Ꝛ', 'Ꝛ'), + ('Ꝝ', 'Ꝝ'), + ('Ꝟ', 'Ꝟ'), + ('Ꝡ', 'Ꝡ'), + ('Ꝣ', 'Ꝣ'), + ('Ꝥ', 'Ꝥ'), + ('Ꝧ', 'Ꝧ'), + ('Ꝩ', 'Ꝩ'), + ('Ꝫ', 'Ꝫ'), + ('Ꝭ', 'Ꝭ'), + ('Ꝯ', 'Ꝯ'), + ('Ꝺ', 'Ꝺ'), + ('Ꝼ', 'Ꝼ'), + ('Ᵹ', 'Ꝿ'), + ('Ꞁ', 'Ꞁ'), + ('Ꞃ', 'Ꞃ'), + ('Ꞅ', 'Ꞅ'), + ('Ꞇ', 'Ꞇ'), + ('Ꞌ', 'Ꞌ'), + ('Ɥ', 'Ɥ'), + ('Ꞑ', 'Ꞑ'), + ('Ꞓ', 'Ꞓ'), + ('Ꞗ', 'Ꞗ'), + ('Ꞙ', 'Ꞙ'), + ('Ꞛ', 'Ꞛ'), + ('Ꞝ', 'Ꞝ'), + ('Ꞟ', 'Ꞟ'), + ('Ꞡ', 'Ꞡ'), + ('Ꞣ', 'Ꞣ'), + ('Ꞥ', 'Ꞥ'), + ('Ꞧ', 'Ꞧ'), + ('Ꞩ', 'Ꞩ'), + ('Ɦ', 'Ɪ'), + ('Ʞ', 'Ꞵ'), + ('Ꞷ', 'Ꞷ'), + ('Ꞹ', 'Ꞹ'), + ('Ꞻ', 'Ꞻ'), + ('Ꞽ', 'Ꞽ'), + ('Ꞿ', 'Ꞿ'), + ('Ꟁ', 'Ꟁ'), + ('Ꟃ', 'Ꟃ'), + ('Ꞔ', 'Ꟈ'), + ('Ꟊ', 'Ꟊ'), + ('Ɤ', 'Ꟍ'), + ('Ꟑ', 'Ꟑ'), + ('Ꟗ', 'Ꟗ'), + ('Ꟙ', 'Ꟙ'), + ('Ꟛ', 'Ꟛ'), + ('Ƛ', 'Ƛ'), + ('Ꟶ', 'Ꟶ'), + ('Ａ', 'Ｚ'), + ('𐐀', '𐐧'), + ('𐒰', '𐓓'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐲀', '𐲲'), + ('𐵐', '𐵥'), + ('𑢠', '𑢿'), + ('𖹀', '𖹟'), + ('𞤀', '𞤡'), +]; + +pub const CHANGES_WHEN_TITLECASED: &'static [(char, char)] = &[ + ('a', 'z'), + ('µ', 'µ'), + ('ß', 'ö'), + ('ø', 'ÿ'), + ('ā', 'ā'), + ('ă', 'ă'), + ('ą', 'ą'), + ('ć', 'ć'), + ('ĉ', 'ĉ'), + ('ċ', 'ċ'), + ('č', 'č'), + ('ď', 'ď'), + ('đ', 'đ'), + ('ē', 'ē'), + ('ĕ', 'ĕ'), + ('ė', 'ė'), + ('ę', 'ę'), + ('ě', 'ě'), + ('ĝ', 'ĝ'), + ('ğ', 'ğ'), + ('ġ', 'ġ'), + ('ģ', 'ģ'), + ('ĥ', 'ĥ'), + ('ħ', 'ħ'), + ('ĩ', 'ĩ'), + ('ī', 'ī'), + ('ĭ', 'ĭ'), + ('į', 'į'), + ('ı', 'ı'), + ('ij', 'ij'), + ('ĵ', 'ĵ'), + ('ķ', 'ķ'), + ('ĺ', 'ĺ'), + ('ļ', 'ļ'), + ('ľ', 'ľ'), + ('ŀ', 'ŀ'), + ('ł', 'ł'), + ('ń', 'ń'), + ('ņ', 'ņ'), + ('ň', 'ʼn'), + ('ŋ', 'ŋ'), + ('ō', 'ō'), + ('ŏ', 'ŏ'), + ('ő', 'ő'), + ('œ', 'œ'), + ('ŕ', 'ŕ'), + ('ŗ', 'ŗ'), + ('ř', 'ř'), + ('ś', 'ś'), + ('ŝ', 'ŝ'), + ('ş', 'ş'), + ('š', 'š'), + ('ţ', 'ţ'), + ('ť', 'ť'), + ('ŧ', 'ŧ'), + ('ũ', 'ũ'), + ('ū', 'ū'), + ('ŭ', 'ŭ'), + ('ů', 'ů'), + ('ű', 'ű'), + ('ų', 'ų'), + ('ŵ', 'ŵ'), + ('ŷ', 'ŷ'), + ('ź', 'ź'), + ('ż', 'ż'), + ('ž', 'ƀ'), + ('ƃ', 'ƃ'), + ('ƅ', 'ƅ'), + ('ƈ', 'ƈ'), + ('ƌ', 'ƌ'), + ('ƒ', 'ƒ'), + ('ƕ', 'ƕ'), + ('ƙ', 'ƛ'), + ('ƞ', 'ƞ'), + ('ơ', 'ơ'), + ('ƣ', 'ƣ'), + ('ƥ', 'ƥ'), + ('ƨ', 'ƨ'), + ('ƭ', 'ƭ'), + ('ư', 'ư'), + ('ƴ', 'ƴ'), + ('ƶ', 'ƶ'), + ('ƹ', 'ƹ'), + ('ƽ', 'ƽ'), + ('ƿ', 'ƿ'), + ('DŽ', 'DŽ'), + ('dž', 'LJ'), + ('lj', 'NJ'), + ('nj', 'nj'), + ('ǎ', 'ǎ'), + ('ǐ', 'ǐ'), + ('ǒ', 'ǒ'), + ('ǔ', 'ǔ'), + ('ǖ', 'ǖ'), + ('ǘ', 'ǘ'), + ('ǚ', 'ǚ'), + ('ǜ', 'ǝ'), + ('ǟ', 'ǟ'), + ('ǡ', 'ǡ'), + ('ǣ', 'ǣ'), + ('ǥ', 'ǥ'), + ('ǧ', 'ǧ'), + ('ǩ', 'ǩ'), + ('ǫ', 'ǫ'), + ('ǭ', 'ǭ'), + ('ǯ', 'DZ'), + ('dz', 'dz'), + ('ǵ', 'ǵ'), + ('ǹ', 'ǹ'), + ('ǻ', 'ǻ'), + ('ǽ', 'ǽ'), + ('ǿ', 'ǿ'), + ('ȁ', 'ȁ'), + ('ȃ', 'ȃ'), + ('ȅ', 'ȅ'), + ('ȇ', 'ȇ'), + ('ȉ', 'ȉ'), + ('ȋ', 'ȋ'), + ('ȍ', 'ȍ'), + ('ȏ', 'ȏ'), + ('ȑ', 'ȑ'), + ('ȓ', 'ȓ'), + ('ȕ', 'ȕ'), + ('ȗ', 'ȗ'), + ('ș', 'ș'), + ('ț', 'ț'), + ('ȝ', 'ȝ'), + ('ȟ', 'ȟ'), + ('ȣ', 'ȣ'), + ('ȥ', 'ȥ'), + ('ȧ', 'ȧ'), + ('ȩ', 'ȩ'), + ('ȫ', 'ȫ'), + ('ȭ', 'ȭ'), + ('ȯ', 'ȯ'), + ('ȱ', 'ȱ'), + ('ȳ', 'ȳ'), + ('ȼ', 'ȼ'), + ('ȿ', 'ɀ'), + ('ɂ', 'ɂ'), + ('ɇ', 'ɇ'), + ('ɉ', 'ɉ'), + ('ɋ', 'ɋ'), + ('ɍ', 'ɍ'), + ('ɏ', 'ɔ'), + ('ɖ', 'ɗ'), + ('ə', 'ə'), + ('ɛ', 'ɜ'), + ('ɠ', 'ɡ'), + ('ɣ', 'ɦ'), + ('ɨ', 'ɬ'), + ('ɯ', 'ɯ'), + ('ɱ', 'ɲ'), + ('ɵ', 'ɵ'), + ('ɽ', 'ɽ'), + ('ʀ', 'ʀ'), + ('ʂ', 'ʃ'), + ('ʇ', 'ʌ'), + ('ʒ', 'ʒ'), + ('ʝ', 'ʞ'), + ('\u{345}', '\u{345}'), + ('ͱ', 'ͱ'), + ('ͳ', 'ͳ'), + ('ͷ', 'ͷ'), + ('ͻ', 'ͽ'), + ('ΐ', 'ΐ'), + ('ά', 'ώ'), + ('ϐ', 'ϑ'), + ('ϕ', 'ϗ'), + ('ϙ', 'ϙ'), + ('ϛ', 'ϛ'), + ('ϝ', 'ϝ'), + ('ϟ', 'ϟ'), + ('ϡ', 'ϡ'), + ('ϣ', 'ϣ'), + ('ϥ', 'ϥ'), + ('ϧ', 'ϧ'), + ('ϩ', 'ϩ'), + ('ϫ', 'ϫ'), + ('ϭ', 'ϭ'), + ('ϯ', 'ϳ'), + ('ϵ', 'ϵ'), + ('ϸ', 'ϸ'), + ('ϻ', 'ϻ'), + ('а', 'џ'), + ('ѡ', 'ѡ'), + ('ѣ', 'ѣ'), + ('ѥ', 'ѥ'), + ('ѧ', 'ѧ'), + ('ѩ', 'ѩ'), + ('ѫ', 'ѫ'), + ('ѭ', 'ѭ'), + ('ѯ', 'ѯ'), + ('ѱ', 'ѱ'), + ('ѳ', 'ѳ'), + ('ѵ', 'ѵ'), + ('ѷ', 'ѷ'), + ('ѹ', 'ѹ'), + ('ѻ', 'ѻ'), + ('ѽ', 'ѽ'), + ('ѿ', 'ѿ'), + ('ҁ', 'ҁ'), + ('ҋ', 'ҋ'), + ('ҍ', 'ҍ'), + ('ҏ', 'ҏ'), + ('ґ', 'ґ'), + ('ғ', 'ғ'), + ('ҕ', 'ҕ'), + ('җ', 'җ'), + ('ҙ', 'ҙ'), + ('қ', 'қ'), + ('ҝ', 'ҝ'), + ('ҟ', 'ҟ'), + ('ҡ', 'ҡ'), + ('ң', 'ң'), + ('ҥ', 'ҥ'), + ('ҧ', 'ҧ'), + ('ҩ', 'ҩ'), + ('ҫ', 'ҫ'), + ('ҭ', 'ҭ'), + ('ү', 'ү'), + ('ұ', 'ұ'), + ('ҳ', 'ҳ'), + ('ҵ', 'ҵ'), + ('ҷ', 'ҷ'), + ('ҹ', 'ҹ'), + ('һ', 'һ'), + ('ҽ', 'ҽ'), + ('ҿ', 'ҿ'), + ('ӂ', 'ӂ'), + ('ӄ', 'ӄ'), + ('ӆ', 'ӆ'), + ('ӈ', 'ӈ'), + ('ӊ', 'ӊ'), + ('ӌ', 'ӌ'), + ('ӎ', 'ӏ'), + ('ӑ', 'ӑ'), + ('ӓ', 'ӓ'), + ('ӕ', 'ӕ'), + ('ӗ', 'ӗ'), + ('ә', 'ә'), + ('ӛ', 'ӛ'), + ('ӝ', 'ӝ'), + ('ӟ', 'ӟ'), + ('ӡ', 'ӡ'), + ('ӣ', 'ӣ'), + ('ӥ', 'ӥ'), + ('ӧ', 'ӧ'), + ('ө', 'ө'), + ('ӫ', 'ӫ'), + ('ӭ', 'ӭ'), + ('ӯ', 'ӯ'), + ('ӱ', 'ӱ'), + ('ӳ', 'ӳ'), + ('ӵ', 'ӵ'), + ('ӷ', 'ӷ'), + ('ӹ', 'ӹ'), + ('ӻ', 'ӻ'), + ('ӽ', 'ӽ'), + ('ӿ', 'ӿ'), + ('ԁ', 'ԁ'), + ('ԃ', 'ԃ'), + ('ԅ', 'ԅ'), + ('ԇ', 'ԇ'), + ('ԉ', 'ԉ'), + ('ԋ', 'ԋ'), + ('ԍ', 'ԍ'), + ('ԏ', 'ԏ'), + ('ԑ', 'ԑ'), + ('ԓ', 'ԓ'), + ('ԕ', 'ԕ'), + ('ԗ', 'ԗ'), + ('ԙ', 'ԙ'), + ('ԛ', 'ԛ'), + ('ԝ', 'ԝ'), + ('ԟ', 'ԟ'), + ('ԡ', 'ԡ'), + ('ԣ', 'ԣ'), + ('ԥ', 'ԥ'), + ('ԧ', 'ԧ'), + ('ԩ', 'ԩ'), + ('ԫ', 'ԫ'), + ('ԭ', 'ԭ'), + ('ԯ', 'ԯ'), + ('ա', 'և'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'ᲈ'), + ('ᲊ', 'ᲊ'), + ('ᵹ', 'ᵹ'), + ('ᵽ', 'ᵽ'), + ('ᶎ', 'ᶎ'), + ('ḁ', 'ḁ'), + ('ḃ', 'ḃ'), + ('ḅ', 'ḅ'), + ('ḇ', 'ḇ'), + ('ḉ', 'ḉ'), + ('ḋ', 'ḋ'), + ('ḍ', 'ḍ'), + ('ḏ', 'ḏ'), + ('ḑ', 'ḑ'), + ('ḓ', 'ḓ'), + ('ḕ', 'ḕ'), + ('ḗ', 'ḗ'), + ('ḙ', 'ḙ'), + ('ḛ', 'ḛ'), + ('ḝ', 'ḝ'), + ('ḟ', 'ḟ'), + ('ḡ', 'ḡ'), + ('ḣ', 'ḣ'), + ('ḥ', 'ḥ'), + ('ḧ', 'ḧ'), + ('ḩ', 'ḩ'), + ('ḫ', 'ḫ'), + ('ḭ', 'ḭ'), + ('ḯ', 'ḯ'), + ('ḱ', 'ḱ'), + ('ḳ', 'ḳ'), + ('ḵ', 'ḵ'), + ('ḷ', 'ḷ'), + ('ḹ', 'ḹ'), + ('ḻ', 'ḻ'), + ('ḽ', 'ḽ'), + ('ḿ', 'ḿ'), + ('ṁ', 'ṁ'), + ('ṃ', 'ṃ'), + ('ṅ', 'ṅ'), + ('ṇ', 'ṇ'), + ('ṉ', 'ṉ'), + ('ṋ', 'ṋ'), + ('ṍ', 'ṍ'), + ('ṏ', 'ṏ'), + ('ṑ', 'ṑ'), + ('ṓ', 'ṓ'), + ('ṕ', 'ṕ'), + ('ṗ', 'ṗ'), + ('ṙ', 'ṙ'), + ('ṛ', 'ṛ'), + ('ṝ', 'ṝ'), + ('ṟ', 'ṟ'), + ('ṡ', 'ṡ'), + ('ṣ', 'ṣ'), + ('ṥ', 'ṥ'), + ('ṧ', 'ṧ'), + ('ṩ', 'ṩ'), + ('ṫ', 'ṫ'), + ('ṭ', 'ṭ'), + ('ṯ', 'ṯ'), + ('ṱ', 'ṱ'), + ('ṳ', 'ṳ'), + ('ṵ', 'ṵ'), + ('ṷ', 'ṷ'), + ('ṹ', 'ṹ'), + ('ṻ', 'ṻ'), + ('ṽ', 'ṽ'), + ('ṿ', 'ṿ'), + ('ẁ', 'ẁ'), + ('ẃ', 'ẃ'), + ('ẅ', 'ẅ'), + ('ẇ', 'ẇ'), + ('ẉ', 'ẉ'), + ('ẋ', 'ẋ'), + ('ẍ', 'ẍ'), + ('ẏ', 'ẏ'), + ('ẑ', 'ẑ'), + ('ẓ', 'ẓ'), + ('ẕ', 'ẛ'), + ('ạ', 'ạ'), + ('ả', 'ả'), + ('ấ', 'ấ'), + ('ầ', 'ầ'), + ('ẩ', 'ẩ'), + ('ẫ', 'ẫ'), + ('ậ', 'ậ'), + ('ắ', 'ắ'), + ('ằ', 'ằ'), + ('ẳ', 'ẳ'), + ('ẵ', 'ẵ'), + ('ặ', 'ặ'), + ('ẹ', 'ẹ'), + ('ẻ', 'ẻ'), + ('ẽ', 'ẽ'), + ('ế', 'ế'), + ('ề', 'ề'), + ('ể', 'ể'), + ('ễ', 'ễ'), + ('ệ', 'ệ'), + ('ỉ', 'ỉ'), + ('ị', 'ị'), + ('ọ', 'ọ'), + ('ỏ', 'ỏ'), + ('ố', 'ố'), + ('ồ', 'ồ'), + ('ổ', 'ổ'), + ('ỗ', 'ỗ'), + ('ộ', 'ộ'), + ('ớ', 'ớ'), + ('ờ', 'ờ'), + ('ở', 'ở'), + ('ỡ', 'ỡ'), + ('ợ', 'ợ'), + ('ụ', 'ụ'), + ('ủ', 'ủ'), + ('ứ', 'ứ'), + ('ừ', 'ừ'), + ('ử', 'ử'), + ('ữ', 'ữ'), + ('ự', 'ự'), + ('ỳ', 'ỳ'), + ('ỵ', 'ỵ'), + ('ỷ', 'ỷ'), + ('ỹ', 'ỹ'), + ('ỻ', 'ỻ'), + ('ỽ', 'ỽ'), + ('ỿ', 'ἇ'), + ('ἐ', 'ἕ'), + ('ἠ', 'ἧ'), + ('ἰ', 'ἷ'), + ('ὀ', 'ὅ'), + ('ὐ', 'ὗ'), + ('ὠ', 'ὧ'), + ('ὰ', 'ώ'), + ('ᾀ', 'ᾇ'), + ('ᾐ', 'ᾗ'), + ('ᾠ', 'ᾧ'), + ('ᾰ', 'ᾴ'), + ('ᾶ', 'ᾷ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῇ'), + ('ῐ', 'ΐ'), + ('ῖ', 'ῗ'), + ('ῠ', 'ῧ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῷ'), + ('ⅎ', 'ⅎ'), + ('ⅰ', 'ⅿ'), + ('ↄ', 'ↄ'), + ('ⓐ', 'ⓩ'), + ('ⰰ', 'ⱟ'), + ('ⱡ', 'ⱡ'), + ('ⱥ', 'ⱦ'), + ('ⱨ', 'ⱨ'), + ('ⱪ', 'ⱪ'), + ('ⱬ', 'ⱬ'), + ('ⱳ', 'ⱳ'), + ('ⱶ', 'ⱶ'), + ('ⲁ', 'ⲁ'), + ('ⲃ', 'ⲃ'), + ('ⲅ', 'ⲅ'), + ('ⲇ', 'ⲇ'), + ('ⲉ', 'ⲉ'), + ('ⲋ', 'ⲋ'), + ('ⲍ', 'ⲍ'), + ('ⲏ', 'ⲏ'), + ('ⲑ', 'ⲑ'), + ('ⲓ', 'ⲓ'), + ('ⲕ', 'ⲕ'), + ('ⲗ', 'ⲗ'), + ('ⲙ', 'ⲙ'), + ('ⲛ', 'ⲛ'), + ('ⲝ', 'ⲝ'), + ('ⲟ', 'ⲟ'), + ('ⲡ', 'ⲡ'), + ('ⲣ', 'ⲣ'), + ('ⲥ', 'ⲥ'), + ('ⲧ', 'ⲧ'), + ('ⲩ', 'ⲩ'), + ('ⲫ', 'ⲫ'), + ('ⲭ', 'ⲭ'), + ('ⲯ', 'ⲯ'), + ('ⲱ', 'ⲱ'), + ('ⲳ', 'ⲳ'), + ('ⲵ', 'ⲵ'), + ('ⲷ', 'ⲷ'), + ('ⲹ', 'ⲹ'), + ('ⲻ', 'ⲻ'), + ('ⲽ', 'ⲽ'), + ('ⲿ', 'ⲿ'), + ('ⳁ', 'ⳁ'), + ('ⳃ', 'ⳃ'), + ('ⳅ', 'ⳅ'), + ('ⳇ', 'ⳇ'), + ('ⳉ', 'ⳉ'), + ('ⳋ', 'ⳋ'), + ('ⳍ', 'ⳍ'), + ('ⳏ', 'ⳏ'), + ('ⳑ', 'ⳑ'), + ('ⳓ', 'ⳓ'), + ('ⳕ', 'ⳕ'), + ('ⳗ', 'ⳗ'), + ('ⳙ', 'ⳙ'), + ('ⳛ', 'ⳛ'), + ('ⳝ', 'ⳝ'), + ('ⳟ', 'ⳟ'), + ('ⳡ', 'ⳡ'), + ('ⳣ', 'ⳣ'), + ('ⳬ', 'ⳬ'), + ('ⳮ', 'ⳮ'), + ('ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ꙁ', 'ꙁ'), + ('ꙃ', 'ꙃ'), + ('ꙅ', 'ꙅ'), + ('ꙇ', 'ꙇ'), + ('ꙉ', 'ꙉ'), + ('ꙋ', 'ꙋ'), + ('ꙍ', 'ꙍ'), + ('ꙏ', 'ꙏ'), + ('ꙑ', 'ꙑ'), + ('ꙓ', 'ꙓ'), + ('ꙕ', 'ꙕ'), + ('ꙗ', 'ꙗ'), + ('ꙙ', 'ꙙ'), + ('ꙛ', 'ꙛ'), + ('ꙝ', 'ꙝ'), + ('ꙟ', 'ꙟ'), + ('ꙡ', 'ꙡ'), + ('ꙣ', 'ꙣ'), + ('ꙥ', 'ꙥ'), + ('ꙧ', 'ꙧ'), + ('ꙩ', 'ꙩ'), + ('ꙫ', 'ꙫ'), + ('ꙭ', 'ꙭ'), + ('ꚁ', 'ꚁ'), + ('ꚃ', 'ꚃ'), + ('ꚅ', 'ꚅ'), + ('ꚇ', 'ꚇ'), + ('ꚉ', 'ꚉ'), + ('ꚋ', 'ꚋ'), + ('ꚍ', 'ꚍ'), + ('ꚏ', 'ꚏ'), + ('ꚑ', 'ꚑ'), + ('ꚓ', 'ꚓ'), + ('ꚕ', 'ꚕ'), + ('ꚗ', 'ꚗ'), + ('ꚙ', 'ꚙ'), + ('ꚛ', 'ꚛ'), + ('ꜣ', 'ꜣ'), + ('ꜥ', 'ꜥ'), + ('ꜧ', 'ꜧ'), + ('ꜩ', 'ꜩ'), + ('ꜫ', 'ꜫ'), + ('ꜭ', 'ꜭ'), + ('ꜯ', 'ꜯ'), + ('ꜳ', 'ꜳ'), + ('ꜵ', 'ꜵ'), + ('ꜷ', 'ꜷ'), + ('ꜹ', 'ꜹ'), + ('ꜻ', 'ꜻ'), + ('ꜽ', 'ꜽ'), + ('ꜿ', 'ꜿ'), + ('ꝁ', 'ꝁ'), + ('ꝃ', 'ꝃ'), + ('ꝅ', 'ꝅ'), + ('ꝇ', 'ꝇ'), + ('ꝉ', 'ꝉ'), + ('ꝋ', 'ꝋ'), + ('ꝍ', 'ꝍ'), + ('ꝏ', 'ꝏ'), + ('ꝑ', 'ꝑ'), + ('ꝓ', 'ꝓ'), + ('ꝕ', 'ꝕ'), + ('ꝗ', 'ꝗ'), + ('ꝙ', 'ꝙ'), + ('ꝛ', 'ꝛ'), + ('ꝝ', 'ꝝ'), + ('ꝟ', 'ꝟ'), + ('ꝡ', 'ꝡ'), + ('ꝣ', 'ꝣ'), + ('ꝥ', 'ꝥ'), + ('ꝧ', 'ꝧ'), + ('ꝩ', 'ꝩ'), + ('ꝫ', 'ꝫ'), + ('ꝭ', 'ꝭ'), + ('ꝯ', 'ꝯ'), + ('ꝺ', 'ꝺ'), + ('ꝼ', 'ꝼ'), + ('ꝿ', 'ꝿ'), + ('ꞁ', 'ꞁ'), + ('ꞃ', 'ꞃ'), + ('ꞅ', 'ꞅ'), + ('ꞇ', 'ꞇ'), + ('ꞌ', 'ꞌ'), + ('ꞑ', 'ꞑ'), + ('ꞓ', 'ꞔ'), + ('ꞗ', 'ꞗ'), + ('ꞙ', 'ꞙ'), + ('ꞛ', 'ꞛ'), + ('ꞝ', 'ꞝ'), + ('ꞟ', 'ꞟ'), + ('ꞡ', 'ꞡ'), + ('ꞣ', 'ꞣ'), + ('ꞥ', 'ꞥ'), + ('ꞧ', 'ꞧ'), + ('ꞩ', 'ꞩ'), + ('ꞵ', 'ꞵ'), + ('ꞷ', 'ꞷ'), + ('ꞹ', 'ꞹ'), + ('ꞻ', 'ꞻ'), + ('ꞽ', 'ꞽ'), + ('ꞿ', 'ꞿ'), + ('ꟁ', 'ꟁ'), + ('ꟃ', 'ꟃ'), + ('ꟈ', 'ꟈ'), + ('ꟊ', 'ꟊ'), + ('ꟍ', 'ꟍ'), + ('ꟑ', 'ꟑ'), + ('ꟗ', 'ꟗ'), + ('ꟙ', 'ꟙ'), + ('ꟛ', 'ꟛ'), + ('ꟶ', 'ꟶ'), + ('ꭓ', 'ꭓ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('ａ', 'ｚ'), + ('𐐨', '𐑏'), + ('𐓘', '𐓻'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐳀', '𐳲'), + ('𐵰', '𐶅'), + ('𑣀', '𑣟'), + ('𖹠', '𖹿'), + ('𞤢', '𞥃'), +]; + +pub const CHANGES_WHEN_UPPERCASED: &'static [(char, char)] = &[ + ('a', 'z'), + ('µ', 'µ'), + ('ß', 'ö'), + ('ø', 'ÿ'), + ('ā', 'ā'), + ('ă', 'ă'), + ('ą', 'ą'), + ('ć', 'ć'), + ('ĉ', 'ĉ'), + ('ċ', 'ċ'), + ('č', 'č'), + ('ď', 'ď'), + ('đ', 'đ'), + ('ē', 'ē'), + ('ĕ', 'ĕ'), + ('ė', 'ė'), + ('ę', 'ę'), + ('ě', 'ě'), + ('ĝ', 'ĝ'), + ('ğ', 'ğ'), + ('ġ', 'ġ'), + ('ģ', 'ģ'), + ('ĥ', 'ĥ'), + ('ħ', 'ħ'), + ('ĩ', 'ĩ'), + ('ī', 'ī'), + ('ĭ', 'ĭ'), + ('į', 'į'), + ('ı', 'ı'), + ('ij', 'ij'), + ('ĵ', 'ĵ'), + ('ķ', 'ķ'), + ('ĺ', 'ĺ'), + ('ļ', 'ļ'), + ('ľ', 'ľ'), + ('ŀ', 'ŀ'), + ('ł', 'ł'), + ('ń', 'ń'), + ('ņ', 'ņ'), + ('ň', 'ʼn'), + ('ŋ', 'ŋ'), + ('ō', 'ō'), + ('ŏ', 'ŏ'), + ('ő', 'ő'), + ('œ', 'œ'), + ('ŕ', 'ŕ'), + ('ŗ', 'ŗ'), + ('ř', 'ř'), + ('ś', 'ś'), + ('ŝ', 'ŝ'), + ('ş', 'ş'), + ('š', 'š'), + ('ţ', 'ţ'), + ('ť', 'ť'), + ('ŧ', 'ŧ'), + ('ũ', 'ũ'), + ('ū', 'ū'), + ('ŭ', 'ŭ'), + ('ů', 'ů'), + ('ű', 'ű'), + ('ų', 'ų'), + ('ŵ', 'ŵ'), + ('ŷ', 'ŷ'), + ('ź', 'ź'), + ('ż', 'ż'), + ('ž', 'ƀ'), + ('ƃ', 'ƃ'), + ('ƅ', 'ƅ'), + ('ƈ', 'ƈ'), + ('ƌ', 'ƌ'), + ('ƒ', 'ƒ'), + ('ƕ', 'ƕ'), + ('ƙ', 'ƛ'), + ('ƞ', 'ƞ'), + ('ơ', 'ơ'), + ('ƣ', 'ƣ'), + ('ƥ', 'ƥ'), + ('ƨ', 'ƨ'), + ('ƭ', 'ƭ'), + ('ư', 'ư'), + ('ƴ', 'ƴ'), + ('ƶ', 'ƶ'), + ('ƹ', 'ƹ'), + ('ƽ', 'ƽ'), + ('ƿ', 'ƿ'), + ('Dž', 'dž'), + ('Lj', 'lj'), + ('Nj', 'nj'), + ('ǎ', 'ǎ'), + ('ǐ', 'ǐ'), + ('ǒ', 'ǒ'), + ('ǔ', 'ǔ'), + ('ǖ', 'ǖ'), + ('ǘ', 'ǘ'), + ('ǚ', 'ǚ'), + ('ǜ', 'ǝ'), + ('ǟ', 'ǟ'), + ('ǡ', 'ǡ'), + ('ǣ', 'ǣ'), + ('ǥ', 'ǥ'), + ('ǧ', 'ǧ'), + ('ǩ', 'ǩ'), + ('ǫ', 'ǫ'), + ('ǭ', 'ǭ'), + ('ǯ', 'ǰ'), + ('Dz', 'dz'), + ('ǵ', 'ǵ'), + ('ǹ', 'ǹ'), + ('ǻ', 'ǻ'), + ('ǽ', 'ǽ'), + ('ǿ', 'ǿ'), + ('ȁ', 'ȁ'), + ('ȃ', 'ȃ'), + ('ȅ', 'ȅ'), + ('ȇ', 'ȇ'), + ('ȉ', 'ȉ'), + ('ȋ', 'ȋ'), + ('ȍ', 'ȍ'), + ('ȏ', 'ȏ'), + ('ȑ', 'ȑ'), + ('ȓ', 'ȓ'), + ('ȕ', 'ȕ'), + ('ȗ', 'ȗ'), + ('ș', 'ș'), + ('ț', 'ț'), + ('ȝ', 'ȝ'), + ('ȟ', 'ȟ'), + ('ȣ', 'ȣ'), + ('ȥ', 'ȥ'), + ('ȧ', 'ȧ'), + ('ȩ', 'ȩ'), + ('ȫ', 'ȫ'), + ('ȭ', 'ȭ'), + ('ȯ', 'ȯ'), + ('ȱ', 'ȱ'), + ('ȳ', 'ȳ'), + ('ȼ', 'ȼ'), + ('ȿ', 'ɀ'), + ('ɂ', 'ɂ'), + ('ɇ', 'ɇ'), + ('ɉ', 'ɉ'), + ('ɋ', 'ɋ'), + ('ɍ', 'ɍ'), + ('ɏ', 'ɔ'), + ('ɖ', 'ɗ'), + ('ə', 'ə'), + ('ɛ', 'ɜ'), + ('ɠ', 'ɡ'), + ('ɣ', 'ɦ'), + ('ɨ', 'ɬ'), + ('ɯ', 'ɯ'), + ('ɱ', 'ɲ'), + ('ɵ', 'ɵ'), + ('ɽ', 'ɽ'), + ('ʀ', 'ʀ'), + ('ʂ', 'ʃ'), + ('ʇ', 'ʌ'), + ('ʒ', 'ʒ'), + ('ʝ', 'ʞ'), + ('\u{345}', '\u{345}'), + ('ͱ', 'ͱ'), + ('ͳ', 'ͳ'), + ('ͷ', 'ͷ'), + ('ͻ', 'ͽ'), + ('ΐ', 'ΐ'), + ('ά', 'ώ'), + ('ϐ', 'ϑ'), + ('ϕ', 'ϗ'), + ('ϙ', 'ϙ'), + ('ϛ', 'ϛ'), + ('ϝ', 'ϝ'), + ('ϟ', 'ϟ'), + ('ϡ', 'ϡ'), + ('ϣ', 'ϣ'), + ('ϥ', 'ϥ'), + ('ϧ', 'ϧ'), + ('ϩ', 'ϩ'), + ('ϫ', 'ϫ'), + ('ϭ', 'ϭ'), + ('ϯ', 'ϳ'), + ('ϵ', 'ϵ'), + ('ϸ', 'ϸ'), + ('ϻ', 'ϻ'), + ('а', 'џ'), + ('ѡ', 'ѡ'), + ('ѣ', 'ѣ'), + ('ѥ', 'ѥ'), + ('ѧ', 'ѧ'), + ('ѩ', 'ѩ'), + ('ѫ', 'ѫ'), + ('ѭ', 'ѭ'), + ('ѯ', 'ѯ'), + ('ѱ', 'ѱ'), + ('ѳ', 'ѳ'), + ('ѵ', 'ѵ'), + ('ѷ', 'ѷ'), + ('ѹ', 'ѹ'), + ('ѻ', 'ѻ'), + ('ѽ', 'ѽ'), + ('ѿ', 'ѿ'), + ('ҁ', 'ҁ'), + ('ҋ', 'ҋ'), + ('ҍ', 'ҍ'), + ('ҏ', 'ҏ'), + ('ґ', 'ґ'), + ('ғ', 'ғ'), + ('ҕ', 'ҕ'), + ('җ', 'җ'), + ('ҙ', 'ҙ'), + ('қ', 'қ'), + ('ҝ', 'ҝ'), + ('ҟ', 'ҟ'), + ('ҡ', 'ҡ'), + ('ң', 'ң'), + ('ҥ', 'ҥ'), + ('ҧ', 'ҧ'), + ('ҩ', 'ҩ'), + ('ҫ', 'ҫ'), + ('ҭ', 'ҭ'), + ('ү', 'ү'), + ('ұ', 'ұ'), + ('ҳ', 'ҳ'), + ('ҵ', 'ҵ'), + ('ҷ', 'ҷ'), + ('ҹ', 'ҹ'), + ('һ', 'һ'), + ('ҽ', 'ҽ'), + ('ҿ', 'ҿ'), + ('ӂ', 'ӂ'), + ('ӄ', 'ӄ'), + ('ӆ', 'ӆ'), + ('ӈ', 'ӈ'), + ('ӊ', 'ӊ'), + ('ӌ', 'ӌ'), + ('ӎ', 'ӏ'), + ('ӑ', 'ӑ'), + ('ӓ', 'ӓ'), + ('ӕ', 'ӕ'), + ('ӗ', 'ӗ'), + ('ә', 'ә'), + ('ӛ', 'ӛ'), + ('ӝ', 'ӝ'), + ('ӟ', 'ӟ'), + ('ӡ', 'ӡ'), + ('ӣ', 'ӣ'), + ('ӥ', 'ӥ'), + ('ӧ', 'ӧ'), + ('ө', 'ө'), + ('ӫ', 'ӫ'), + ('ӭ', 'ӭ'), + ('ӯ', 'ӯ'), + ('ӱ', 'ӱ'), + ('ӳ', 'ӳ'), + ('ӵ', 'ӵ'), + ('ӷ', 'ӷ'), + ('ӹ', 'ӹ'), + ('ӻ', 'ӻ'), + ('ӽ', 'ӽ'), + ('ӿ', 'ӿ'), + ('ԁ', 'ԁ'), + ('ԃ', 'ԃ'), + ('ԅ', 'ԅ'), + ('ԇ', 'ԇ'), + ('ԉ', 'ԉ'), + ('ԋ', 'ԋ'), + ('ԍ', 'ԍ'), + ('ԏ', 'ԏ'), + ('ԑ', 'ԑ'), + ('ԓ', 'ԓ'), + ('ԕ', 'ԕ'), + ('ԗ', 'ԗ'), + ('ԙ', 'ԙ'), + ('ԛ', 'ԛ'), + ('ԝ', 'ԝ'), + ('ԟ', 'ԟ'), + ('ԡ', 'ԡ'), + ('ԣ', 'ԣ'), + ('ԥ', 'ԥ'), + ('ԧ', 'ԧ'), + ('ԩ', 'ԩ'), + ('ԫ', 'ԫ'), + ('ԭ', 'ԭ'), + ('ԯ', 'ԯ'), + ('ա', 'և'), + ('ა', 'ჺ'), + ('ჽ', 'ჿ'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'ᲈ'), + ('ᲊ', 'ᲊ'), + ('ᵹ', 'ᵹ'), + ('ᵽ', 'ᵽ'), + ('ᶎ', 'ᶎ'), + ('ḁ', 'ḁ'), + ('ḃ', 'ḃ'), + ('ḅ', 'ḅ'), + ('ḇ', 'ḇ'), + ('ḉ', 'ḉ'), + ('ḋ', 'ḋ'), + ('ḍ', 'ḍ'), + ('ḏ', 'ḏ'), + ('ḑ', 'ḑ'), + ('ḓ', 'ḓ'), + ('ḕ', 'ḕ'), + ('ḗ', 'ḗ'), + ('ḙ', 'ḙ'), + ('ḛ', 'ḛ'), + ('ḝ', 'ḝ'), + ('ḟ', 'ḟ'), + ('ḡ', 'ḡ'), + ('ḣ', 'ḣ'), + ('ḥ', 'ḥ'), + ('ḧ', 'ḧ'), + ('ḩ', 'ḩ'), + ('ḫ', 'ḫ'), + ('ḭ', 'ḭ'), + ('ḯ', 'ḯ'), + ('ḱ', 'ḱ'), + ('ḳ', 'ḳ'), + ('ḵ', 'ḵ'), + ('ḷ', 'ḷ'), + ('ḹ', 'ḹ'), + ('ḻ', 'ḻ'), + ('ḽ', 'ḽ'), + ('ḿ', 'ḿ'), + ('ṁ', 'ṁ'), + ('ṃ', 'ṃ'), + ('ṅ', 'ṅ'), + ('ṇ', 'ṇ'), + ('ṉ', 'ṉ'), + ('ṋ', 'ṋ'), + ('ṍ', 'ṍ'), + ('ṏ', 'ṏ'), + ('ṑ', 'ṑ'), + ('ṓ', 'ṓ'), + ('ṕ', 'ṕ'), + ('ṗ', 'ṗ'), + ('ṙ', 'ṙ'), + ('ṛ', 'ṛ'), + ('ṝ', 'ṝ'), + ('ṟ', 'ṟ'), + ('ṡ', 'ṡ'), + ('ṣ', 'ṣ'), + ('ṥ', 'ṥ'), + ('ṧ', 'ṧ'), + ('ṩ', 'ṩ'), + ('ṫ', 'ṫ'), + ('ṭ', 'ṭ'), + ('ṯ', 'ṯ'), + ('ṱ', 'ṱ'), + ('ṳ', 'ṳ'), + ('ṵ', 'ṵ'), + ('ṷ', 'ṷ'), + ('ṹ', 'ṹ'), + ('ṻ', 'ṻ'), + ('ṽ', 'ṽ'), + ('ṿ', 'ṿ'), + ('ẁ', 'ẁ'), + ('ẃ', 'ẃ'), + ('ẅ', 'ẅ'), + ('ẇ', 'ẇ'), + ('ẉ', 'ẉ'), + ('ẋ', 'ẋ'), + ('ẍ', 'ẍ'), + ('ẏ', 'ẏ'), + ('ẑ', 'ẑ'), + ('ẓ', 'ẓ'), + ('ẕ', 'ẛ'), + ('ạ', 'ạ'), + ('ả', 'ả'), + ('ấ', 'ấ'), + ('ầ', 'ầ'), + ('ẩ', 'ẩ'), + ('ẫ', 'ẫ'), + ('ậ', 'ậ'), + ('ắ', 'ắ'), + ('ằ', 'ằ'), + ('ẳ', 'ẳ'), + ('ẵ', 'ẵ'), + ('ặ', 'ặ'), + ('ẹ', 'ẹ'), + ('ẻ', 'ẻ'), + ('ẽ', 'ẽ'), + ('ế', 'ế'), + ('ề', 'ề'), + ('ể', 'ể'), + ('ễ', 'ễ'), + ('ệ', 'ệ'), + ('ỉ', 'ỉ'), + ('ị', 'ị'), + ('ọ', 'ọ'), + ('ỏ', 'ỏ'), + ('ố', 'ố'), + ('ồ', 'ồ'), + ('ổ', 'ổ'), + ('ỗ', 'ỗ'), + ('ộ', 'ộ'), + ('ớ', 'ớ'), + ('ờ', 'ờ'), + ('ở', 'ở'), + ('ỡ', 'ỡ'), + ('ợ', 'ợ'), + ('ụ', 'ụ'), + ('ủ', 'ủ'), + ('ứ', 'ứ'), + ('ừ', 'ừ'), + ('ử', 'ử'), + ('ữ', 'ữ'), + ('ự', 'ự'), + ('ỳ', 'ỳ'), + ('ỵ', 'ỵ'), + ('ỷ', 'ỷ'), + ('ỹ', 'ỹ'), + ('ỻ', 'ỻ'), + ('ỽ', 'ỽ'), + ('ỿ', 'ἇ'), + ('ἐ', 'ἕ'), + ('ἠ', 'ἧ'), + ('ἰ', 'ἷ'), + ('ὀ', 'ὅ'), + ('ὐ', 'ὗ'), + ('ὠ', 'ὧ'), + ('ὰ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾷ'), + ('ᾼ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῇ'), + ('ῌ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'ῗ'), + ('ῠ', 'ῧ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῷ'), + ('ῼ', 'ῼ'), + ('ⅎ', 'ⅎ'), + ('ⅰ', 'ⅿ'), + ('ↄ', 'ↄ'), + ('ⓐ', 'ⓩ'), + ('ⰰ', 'ⱟ'), + ('ⱡ', 'ⱡ'), + ('ⱥ', 'ⱦ'), + ('ⱨ', 'ⱨ'), + ('ⱪ', 'ⱪ'), + ('ⱬ', 'ⱬ'), + ('ⱳ', 'ⱳ'), + ('ⱶ', 'ⱶ'), + ('ⲁ', 'ⲁ'), + ('ⲃ', 'ⲃ'), + ('ⲅ', 'ⲅ'), + ('ⲇ', 'ⲇ'), + ('ⲉ', 'ⲉ'), + ('ⲋ', 'ⲋ'), + ('ⲍ', 'ⲍ'), + ('ⲏ', 'ⲏ'), + ('ⲑ', 'ⲑ'), + ('ⲓ', 'ⲓ'), + ('ⲕ', 'ⲕ'), + ('ⲗ', 'ⲗ'), + ('ⲙ', 'ⲙ'), + ('ⲛ', 'ⲛ'), + ('ⲝ', 'ⲝ'), + ('ⲟ', 'ⲟ'), + ('ⲡ', 'ⲡ'), + ('ⲣ', 'ⲣ'), + ('ⲥ', 'ⲥ'), + ('ⲧ', 'ⲧ'), + ('ⲩ', 'ⲩ'), + ('ⲫ', 'ⲫ'), + ('ⲭ', 'ⲭ'), + ('ⲯ', 'ⲯ'), + ('ⲱ', 'ⲱ'), + ('ⲳ', 'ⲳ'), + ('ⲵ', 'ⲵ'), + ('ⲷ', 'ⲷ'), + ('ⲹ', 'ⲹ'), + ('ⲻ', 'ⲻ'), + ('ⲽ', 'ⲽ'), + ('ⲿ', 'ⲿ'), + ('ⳁ', 'ⳁ'), + ('ⳃ', 'ⳃ'), + ('ⳅ', 'ⳅ'), + ('ⳇ', 'ⳇ'), + ('ⳉ', 'ⳉ'), + ('ⳋ', 'ⳋ'), + ('ⳍ', 'ⳍ'), + ('ⳏ', 'ⳏ'), + ('ⳑ', 'ⳑ'), + ('ⳓ', 'ⳓ'), + ('ⳕ', 'ⳕ'), + ('ⳗ', 'ⳗ'), + ('ⳙ', 'ⳙ'), + ('ⳛ', 'ⳛ'), + ('ⳝ', 'ⳝ'), + ('ⳟ', 'ⳟ'), + ('ⳡ', 'ⳡ'), + ('ⳣ', 'ⳣ'), + ('ⳬ', 'ⳬ'), + ('ⳮ', 'ⳮ'), + ('ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ꙁ', 'ꙁ'), + ('ꙃ', 'ꙃ'), + ('ꙅ', 'ꙅ'), + ('ꙇ', 'ꙇ'), + ('ꙉ', 'ꙉ'), + ('ꙋ', 'ꙋ'), + ('ꙍ', 'ꙍ'), + ('ꙏ', 'ꙏ'), + ('ꙑ', 'ꙑ'), + ('ꙓ', 'ꙓ'), + ('ꙕ', 'ꙕ'), + ('ꙗ', 'ꙗ'), + ('ꙙ', 'ꙙ'), + ('ꙛ', 'ꙛ'), + ('ꙝ', 'ꙝ'), + ('ꙟ', 'ꙟ'), + ('ꙡ', 'ꙡ'), + ('ꙣ', 'ꙣ'), + ('ꙥ', 'ꙥ'), + ('ꙧ', 'ꙧ'), + ('ꙩ', 'ꙩ'), + ('ꙫ', 'ꙫ'), + ('ꙭ', 'ꙭ'), + ('ꚁ', 'ꚁ'), + ('ꚃ', 'ꚃ'), + ('ꚅ', 'ꚅ'), + ('ꚇ', 'ꚇ'), + ('ꚉ', 'ꚉ'), + ('ꚋ', 'ꚋ'), + ('ꚍ', 'ꚍ'), + ('ꚏ', 'ꚏ'), + ('ꚑ', 'ꚑ'), + ('ꚓ', 'ꚓ'), + ('ꚕ', 'ꚕ'), + ('ꚗ', 'ꚗ'), + ('ꚙ', 'ꚙ'), + ('ꚛ', 'ꚛ'), + ('ꜣ', 'ꜣ'), + ('ꜥ', 'ꜥ'), + ('ꜧ', 'ꜧ'), + ('ꜩ', 'ꜩ'), + ('ꜫ', 'ꜫ'), + ('ꜭ', 'ꜭ'), + ('ꜯ', 'ꜯ'), + ('ꜳ', 'ꜳ'), + ('ꜵ', 'ꜵ'), + ('ꜷ', 'ꜷ'), + ('ꜹ', 'ꜹ'), + ('ꜻ', 'ꜻ'), + ('ꜽ', 'ꜽ'), + ('ꜿ', 'ꜿ'), + ('ꝁ', 'ꝁ'), + ('ꝃ', 'ꝃ'), + ('ꝅ', 'ꝅ'), + ('ꝇ', 'ꝇ'), + ('ꝉ', 'ꝉ'), + ('ꝋ', 'ꝋ'), + ('ꝍ', 'ꝍ'), + ('ꝏ', 'ꝏ'), + ('ꝑ', 'ꝑ'), + ('ꝓ', 'ꝓ'), + ('ꝕ', 'ꝕ'), + ('ꝗ', 'ꝗ'), + ('ꝙ', 'ꝙ'), + ('ꝛ', 'ꝛ'), + ('ꝝ', 'ꝝ'), + ('ꝟ', 'ꝟ'), + ('ꝡ', 'ꝡ'), + ('ꝣ', 'ꝣ'), + ('ꝥ', 'ꝥ'), + ('ꝧ', 'ꝧ'), + ('ꝩ', 'ꝩ'), + ('ꝫ', 'ꝫ'), + ('ꝭ', 'ꝭ'), + ('ꝯ', 'ꝯ'), + ('ꝺ', 'ꝺ'), + ('ꝼ', 'ꝼ'), + ('ꝿ', 'ꝿ'), + ('ꞁ', 'ꞁ'), + ('ꞃ', 'ꞃ'), + ('ꞅ', 'ꞅ'), + ('ꞇ', 'ꞇ'), + ('ꞌ', 'ꞌ'), + ('ꞑ', 'ꞑ'), + ('ꞓ', 'ꞔ'), + ('ꞗ', 'ꞗ'), + ('ꞙ', 'ꞙ'), + ('ꞛ', 'ꞛ'), + ('ꞝ', 'ꞝ'), + ('ꞟ', 'ꞟ'), + ('ꞡ', 'ꞡ'), + ('ꞣ', 'ꞣ'), + ('ꞥ', 'ꞥ'), + ('ꞧ', 'ꞧ'), + ('ꞩ', 'ꞩ'), + ('ꞵ', 'ꞵ'), + ('ꞷ', 'ꞷ'), + ('ꞹ', 'ꞹ'), + ('ꞻ', 'ꞻ'), + ('ꞽ', 'ꞽ'), + ('ꞿ', 'ꞿ'), + ('ꟁ', 'ꟁ'), + ('ꟃ', 'ꟃ'), + ('ꟈ', 'ꟈ'), + ('ꟊ', 'ꟊ'), + ('ꟍ', 'ꟍ'), + ('ꟑ', 'ꟑ'), + ('ꟗ', 'ꟗ'), + ('ꟙ', 'ꟙ'), + ('ꟛ', 'ꟛ'), + ('ꟶ', 'ꟶ'), + ('ꭓ', 'ꭓ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('ａ', 'ｚ'), + ('𐐨', '𐑏'), + ('𐓘', '𐓻'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐳀', '𐳲'), + ('𐵰', '𐶅'), + ('𑣀', '𑣟'), + ('𖹠', '𖹿'), + ('𞤢', '𞥃'), +]; + +pub const DASH: &'static [(char, char)] = &[ + ('-', '-'), + ('֊', '֊'), + ('־', '־'), + ('᐀', '᐀'), + ('᠆', '᠆'), + ('‐', '―'), + ('⁓', '⁓'), + ('⁻', '⁻'), + ('₋', '₋'), + ('−', '−'), + ('⸗', '⸗'), + ('⸚', '⸚'), + ('⸺', '⸻'), + ('⹀', '⹀'), + ('⹝', '⹝'), + ('〜', '〜'), + ('〰', '〰'), + ('゠', '゠'), + ('︱', '︲'), + ('﹘', '﹘'), + ('﹣', '﹣'), + ('－', '－'), + ('𐵮', '𐵮'), + ('𐺭', '𐺭'), +]; + +pub const DEFAULT_IGNORABLE_CODE_POINT: &'static [(char, char)] = &[ + ('\u{ad}', '\u{ad}'), + ('\u{34f}', '\u{34f}'), + ('\u{61c}', '\u{61c}'), + ('ᅟ', 'ᅠ'), + ('\u{17b4}', '\u{17b5}'), + ('\u{180b}', '\u{180f}'), + ('\u{200b}', '\u{200f}'), + ('\u{202a}', '\u{202e}'), + ('\u{2060}', '\u{206f}'), + ('ㅤ', 'ㅤ'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{feff}', '\u{feff}'), + ('ﾠ', 'ﾠ'), + ('\u{fff0}', '\u{fff8}'), + ('\u{1bca0}', '\u{1bca3}'), + ('\u{1d173}', '\u{1d17a}'), + ('\u{e0000}', '\u{e0fff}'), +]; + +pub const DEPRECATED: &'static [(char, char)] = &[ + ('ʼn', 'ʼn'), + ('ٳ', 'ٳ'), + ('\u{f77}', '\u{f77}'), + ('\u{f79}', '\u{f79}'), + ('ឣ', 'ឤ'), + ('\u{206a}', '\u{206f}'), + ('〈', '〉'), + ('\u{e0001}', '\u{e0001}'), +]; + +pub const DIACRITIC: &'static [(char, char)] = &[ + ('^', '^'), + ('`', '`'), + ('¨', '¨'), + ('¯', '¯'), + ('´', '´'), + ('·', '¸'), + ('ʰ', '\u{34e}'), + ('\u{350}', '\u{357}'), + ('\u{35d}', '\u{362}'), + ('ʹ', '͵'), + ('ͺ', 'ͺ'), + ('΄', '΅'), + ('\u{483}', '\u{487}'), + ('ՙ', 'ՙ'), + ('\u{591}', '\u{5a1}'), + ('\u{5a3}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c4}'), + ('\u{64b}', '\u{652}'), + ('\u{657}', '\u{658}'), + ('\u{6df}', '\u{6e0}'), + ('ۥ', 'ۦ'), + ('\u{6ea}', '\u{6ec}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', 'ߵ'), + ('\u{818}', '\u{819}'), + ('\u{898}', '\u{89f}'), + ('ࣉ', '\u{8d2}'), + ('\u{8e3}', '\u{8fe}'), + ('\u{93c}', '\u{93c}'), + ('\u{94d}', '\u{94d}'), + ('\u{951}', '\u{954}'), + ('ॱ', 'ॱ'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9cd}', '\u{9cd}'), + ('\u{a3c}', '\u{a3c}'), + ('\u{a4d}', '\u{a4d}'), + ('\u{abc}', '\u{abc}'), + ('\u{acd}', '\u{acd}'), + ('\u{afd}', '\u{aff}'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b4d}', '\u{b4d}'), + ('\u{b55}', '\u{b55}'), + ('\u{bcd}', '\u{bcd}'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c4d}', '\u{c4d}'), + ('\u{cbc}', '\u{cbc}'), + ('\u{ccd}', '\u{ccd}'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d4d}', '\u{d4d}'), + ('\u{dca}', '\u{dca}'), + ('\u{e3a}', '\u{e3a}'), + ('\u{e47}', '\u{e4c}'), + ('\u{e4e}', '\u{e4e}'), + ('\u{eba}', '\u{eba}'), + ('\u{ec8}', '\u{ecc}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('༾', '༿'), + ('\u{f82}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{fc6}', '\u{fc6}'), + ('\u{1037}', '\u{1037}'), + ('\u{1039}', '\u{103a}'), + ('ၣ', 'ၤ'), + ('ၩ', 'ၭ'), + ('ႇ', '\u{108d}'), + ('ႏ', 'ႏ'), + ('ႚ', 'ႛ'), + ('\u{135d}', '\u{135f}'), + ('\u{1714}', '\u{1715}'), + ('\u{1734}', '\u{1734}'), + ('\u{17c9}', '\u{17d3}'), + ('\u{17dd}', '\u{17dd}'), + ('\u{1939}', '\u{193b}'), + ('\u{1a60}', '\u{1a60}'), + ('\u{1a75}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('\u{1ab0}', '\u{1abe}'), + ('\u{1ac1}', '\u{1acb}'), + ('\u{1b34}', '\u{1b34}'), + ('\u{1b44}', '\u{1b44}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1baa}', '\u{1bab}'), + ('\u{1be6}', '\u{1be6}'), + ('\u{1bf2}', '\u{1bf3}'), + ('\u{1c36}', '\u{1c37}'), + ('ᱸ', 'ᱽ'), + ('\u{1cd0}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('᳷', '\u{1cf9}'), + ('ᴬ', 'ᵪ'), + ('\u{1dc4}', '\u{1dcf}'), + ('\u{1df5}', '\u{1dff}'), + ('᾽', '᾽'), + ('᾿', '῁'), + ('῍', '῏'), + ('῝', '῟'), + ('῭', '`'), + ('´', '῾'), + ('\u{2cef}', '\u{2cf1}'), + ('ⸯ', 'ⸯ'), + ('\u{302a}', '\u{302f}'), + ('\u{3099}', '゜'), + ('ー', 'ー'), + ('\u{a66f}', '\u{a66f}'), + ('\u{a67c}', '\u{a67d}'), + ('ꙿ', 'ꙿ'), + ('ꚜ', 'ꚝ'), + ('\u{a6f0}', '\u{a6f1}'), + ('꜀', '꜡'), + ('ꞈ', '꞊'), + ('ꟸ', 'ꟹ'), + ('\u{a806}', '\u{a806}'), + ('\u{a82c}', '\u{a82c}'), + ('\u{a8c4}', '\u{a8c4}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a92b}', '꤮'), + ('\u{a953}', '\u{a953}'), + ('\u{a9b3}', '\u{a9b3}'), + ('\u{a9c0}', '\u{a9c0}'), + ('\u{a9e5}', '\u{a9e5}'), + ('ꩻ', 'ꩽ'), + ('\u{aabf}', 'ꫂ'), + ('\u{aaf6}', '\u{aaf6}'), + ('꭛', 'ꭟ'), + ('ꭩ', '꭫'), + ('꯬', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{fe20}', '\u{fe2f}'), + ('＾', '＾'), + ('｀', '｀'), + ('ｰ', 'ｰ'), + ('\u{ff9e}', '\u{ff9f}'), + ('￣', '￣'), + ('\u{102e0}', '\u{102e0}'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('𐴢', '\u{10d27}'), + ('𐵎', '𐵎'), + ('\u{10d69}', '\u{10d6d}'), + ('\u{10efd}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('\u{11046}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{110b9}', '\u{110ba}'), + ('\u{11133}', '\u{11134}'), + ('\u{11173}', '\u{11173}'), + ('\u{111c0}', '\u{111c0}'), + ('\u{111ca}', '\u{111cc}'), + ('\u{11235}', '\u{11236}'), + ('\u{112e9}', '\u{112ea}'), + ('\u{1133b}', '\u{1133c}'), + ('\u{1134d}', '\u{1134d}'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113ce}', '\u{113d0}'), + ('\u{113d2}', '𑏓'), + ('\u{113e1}', '\u{113e2}'), + ('\u{11442}', '\u{11442}'), + ('\u{11446}', '\u{11446}'), + ('\u{114c2}', '\u{114c3}'), + ('\u{115bf}', '\u{115c0}'), + ('\u{1163f}', '\u{1163f}'), + ('\u{116b6}', '\u{116b7}'), + ('\u{1172b}', '\u{1172b}'), + ('\u{11839}', '\u{1183a}'), + ('\u{1193d}', '\u{1193e}'), + ('\u{11943}', '\u{11943}'), + ('\u{119e0}', '\u{119e0}'), + ('\u{11a34}', '\u{11a34}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a99}', '\u{11a99}'), + ('\u{11c3f}', '\u{11c3f}'), + ('\u{11d42}', '\u{11d42}'), + ('\u{11d44}', '\u{11d45}'), + ('\u{11d97}', '\u{11d97}'), + ('\u{11f41}', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13447}', '\u{13455}'), + ('\u{1612f}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('𖵫', '𖵬'), + ('\u{16f8f}', '𖾟'), + ('\u{16ff0}', '\u{16ff1}'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d167}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('𞀰', '𞁭'), + ('\u{1e130}', '\u{1e136}'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '\u{1e946}'), + ('\u{1e948}', '\u{1e94a}'), +]; + +pub const EMOJI: &'static [(char, char)] = &[ + ('#', '#'), + ('*', '*'), + ('0', '9'), + ('©', '©'), + ('®', '®'), + ('‼', '‼'), + ('⁉', '⁉'), + ('™', '™'), + ('ℹ', 'ℹ'), + ('↔', '↙'), + ('↩', '↪'), + ('⌚', '⌛'), + ('⌨', '⌨'), + ('⏏', '⏏'), + ('⏩', '⏳'), + ('⏸', '⏺'), + ('Ⓜ', 'Ⓜ'), + ('▪', '▫'), + ('▶', '▶'), + ('◀', '◀'), + ('◻', '◾'), + ('☀', '☄'), + ('☎', '☎'), + ('☑', '☑'), + ('☔', '☕'), + ('☘', '☘'), + ('☝', '☝'), + ('☠', '☠'), + ('☢', '☣'), + ('☦', '☦'), + ('☪', '☪'), + ('☮', '☯'), + ('☸', '☺'), + ('♀', '♀'), + ('♂', '♂'), + ('♈', '♓'), + ('♟', '♠'), + ('♣', '♣'), + ('♥', '♦'), + ('♨', '♨'), + ('♻', '♻'), + ('♾', '♿'), + ('⚒', '⚗'), + ('⚙', '⚙'), + ('⚛', '⚜'), + ('⚠', '⚡'), + ('⚧', '⚧'), + ('⚪', '⚫'), + ('⚰', '⚱'), + ('⚽', '⚾'), + ('⛄', '⛅'), + ('⛈', '⛈'), + ('⛎', '⛏'), + ('⛑', '⛑'), + ('⛓', '⛔'), + ('⛩', '⛪'), + ('⛰', '⛵'), + ('⛷', '⛺'), + ('⛽', '⛽'), + ('✂', '✂'), + ('✅', '✅'), + ('✈', '✍'), + ('✏', '✏'), + ('✒', '✒'), + ('✔', '✔'), + ('✖', '✖'), + ('✝', '✝'), + ('✡', '✡'), + ('✨', '✨'), + ('✳', '✴'), + ('❄', '❄'), + ('❇', '❇'), + ('❌', '❌'), + ('❎', '❎'), + ('❓', '❕'), + ('❗', '❗'), + ('❣', '❤'), + ('➕', '➗'), + ('➡', '➡'), + ('➰', '➰'), + ('➿', '➿'), + ('⤴', '⤵'), + ('⬅', '⬇'), + ('⬛', '⬜'), + ('⭐', '⭐'), + ('⭕', '⭕'), + ('〰', '〰'), + ('〽', '〽'), + ('㊗', '㊗'), + ('㊙', '㊙'), + ('🀄', '🀄'), + ('🃏', '🃏'), + ('🅰', '🅱'), + ('🅾', '🅿'), + ('🆎', '🆎'), + ('🆑', '🆚'), + ('🇦', '🇿'), + ('🈁', '🈂'), + ('🈚', '🈚'), + ('🈯', '🈯'), + ('🈲', '🈺'), + ('🉐', '🉑'), + ('🌀', '🌡'), + ('🌤', '🎓'), + ('🎖', '🎗'), + ('🎙', '🎛'), + ('🎞', '🏰'), + ('🏳', '🏵'), + ('🏷', '📽'), + ('📿', '🔽'), + ('🕉', '🕎'), + ('🕐', '🕧'), + ('🕯', '🕰'), + ('🕳', '🕺'), + ('🖇', '🖇'), + ('🖊', '🖍'), + ('🖐', '🖐'), + ('🖕', '🖖'), + ('🖤', '🖥'), + ('🖨', '🖨'), + ('🖱', '🖲'), + ('🖼', '🖼'), + ('🗂', '🗄'), + ('🗑', '🗓'), + ('🗜', '🗞'), + ('🗡', '🗡'), + ('🗣', '🗣'), + ('🗨', '🗨'), + ('🗯', '🗯'), + ('🗳', '🗳'), + ('🗺', '🙏'), + ('🚀', '🛅'), + ('🛋', '🛒'), + ('🛕', '🛗'), + ('🛜', '🛥'), + ('🛩', '🛩'), + ('🛫', '🛬'), + ('🛰', '🛰'), + ('🛳', '🛼'), + ('🟠', '🟫'), + ('🟰', '🟰'), + ('🤌', '🤺'), + ('🤼', '🥅'), + ('🥇', '🧿'), + ('🩰', '🩼'), + ('🪀', '🪉'), + ('🪏', '🫆'), + ('🫎', '🫜'), + ('🫟', '🫩'), + ('🫰', '🫸'), +]; + +pub const EMOJI_COMPONENT: &'static [(char, char)] = &[ + ('#', '#'), + ('*', '*'), + ('0', '9'), + ('\u{200d}', '\u{200d}'), + ('\u{20e3}', '\u{20e3}'), + ('\u{fe0f}', '\u{fe0f}'), + ('🇦', '🇿'), + ('🏻', '🏿'), + ('🦰', '🦳'), + ('\u{e0020}', '\u{e007f}'), +]; + +pub const EMOJI_MODIFIER: &'static [(char, char)] = &[('🏻', '🏿')]; + +pub const EMOJI_MODIFIER_BASE: &'static [(char, char)] = &[ + ('☝', '☝'), + ('⛹', '⛹'), + ('✊', '✍'), + ('🎅', '🎅'), + ('🏂', '🏄'), + ('🏇', '🏇'), + ('🏊', '🏌'), + ('👂', '👃'), + ('👆', '👐'), + ('👦', '👸'), + ('👼', '👼'), + ('💁', '💃'), + ('💅', '💇'), + ('💏', '💏'), + ('💑', '💑'), + ('💪', '💪'), + ('🕴', '🕵'), + ('🕺', '🕺'), + ('🖐', '🖐'), + ('🖕', '🖖'), + ('🙅', '🙇'), + ('🙋', '🙏'), + ('🚣', '🚣'), + ('🚴', '🚶'), + ('🛀', '🛀'), + ('🛌', '🛌'), + ('🤌', '🤌'), + ('🤏', '🤏'), + ('🤘', '🤟'), + ('🤦', '🤦'), + ('🤰', '🤹'), + ('🤼', '🤾'), + ('🥷', '🥷'), + ('🦵', '🦶'), + ('🦸', '🦹'), + ('🦻', '🦻'), + ('🧍', '🧏'), + ('🧑', '🧝'), + ('🫃', '🫅'), + ('🫰', '🫸'), +]; + +pub const EMOJI_PRESENTATION: &'static [(char, char)] = &[ + ('⌚', '⌛'), + ('⏩', '⏬'), + ('⏰', '⏰'), + ('⏳', '⏳'), + ('◽', '◾'), + ('☔', '☕'), + ('♈', '♓'), + ('♿', '♿'), + ('⚓', '⚓'), + ('⚡', '⚡'), + ('⚪', '⚫'), + ('⚽', '⚾'), + ('⛄', '⛅'), + ('⛎', '⛎'), + ('⛔', '⛔'), + ('⛪', '⛪'), + ('⛲', '⛳'), + ('⛵', '⛵'), + ('⛺', '⛺'), + ('⛽', '⛽'), + ('✅', '✅'), + ('✊', '✋'), + ('✨', '✨'), + ('❌', '❌'), + ('❎', '❎'), + ('❓', '❕'), + ('❗', '❗'), + ('➕', '➗'), + ('➰', '➰'), + ('➿', '➿'), + ('⬛', '⬜'), + ('⭐', '⭐'), + ('⭕', '⭕'), + ('🀄', '🀄'), + ('🃏', '🃏'), + ('🆎', '🆎'), + ('🆑', '🆚'), + ('🇦', '🇿'), + ('🈁', '🈁'), + ('🈚', '🈚'), + ('🈯', '🈯'), + ('🈲', '🈶'), + ('🈸', '🈺'), + ('🉐', '🉑'), + ('🌀', '🌠'), + ('🌭', '🌵'), + ('🌷', '🍼'), + ('🍾', '🎓'), + ('🎠', '🏊'), + ('🏏', '🏓'), + ('🏠', '🏰'), + ('🏴', '🏴'), + ('🏸', '🐾'), + ('👀', '👀'), + ('👂', '📼'), + ('📿', '🔽'), + ('🕋', '🕎'), + ('🕐', '🕧'), + ('🕺', '🕺'), + ('🖕', '🖖'), + ('🖤', '🖤'), + ('🗻', '🙏'), + ('🚀', '🛅'), + ('🛌', '🛌'), + ('🛐', '🛒'), + ('🛕', '🛗'), + ('🛜', '🛟'), + ('🛫', '🛬'), + ('🛴', '🛼'), + ('🟠', '🟫'), + ('🟰', '🟰'), + ('🤌', '🤺'), + ('🤼', '🥅'), + ('🥇', '🧿'), + ('🩰', '🩼'), + ('🪀', '🪉'), + ('🪏', '🫆'), + ('🫎', '🫜'), + ('🫟', '🫩'), + ('🫰', '🫸'), +]; + +pub const EXTENDED_PICTOGRAPHIC: &'static [(char, char)] = &[ + ('©', '©'), + ('®', '®'), + ('‼', '‼'), + ('⁉', '⁉'), + ('™', '™'), + ('ℹ', 'ℹ'), + ('↔', '↙'), + ('↩', '↪'), + ('⌚', '⌛'), + ('⌨', '⌨'), + ('⎈', '⎈'), + ('⏏', '⏏'), + ('⏩', '⏳'), + ('⏸', '⏺'), + ('Ⓜ', 'Ⓜ'), + ('▪', '▫'), + ('▶', '▶'), + ('◀', '◀'), + ('◻', '◾'), + ('☀', '★'), + ('☇', '☒'), + ('☔', '⚅'), + ('⚐', '✅'), + ('✈', '✒'), + ('✔', '✔'), + ('✖', '✖'), + ('✝', '✝'), + ('✡', '✡'), + ('✨', '✨'), + ('✳', '✴'), + ('❄', '❄'), + ('❇', '❇'), + ('❌', '❌'), + ('❎', '❎'), + ('❓', '❕'), + ('❗', '❗'), + ('❣', '❧'), + ('➕', '➗'), + ('➡', '➡'), + ('➰', '➰'), + ('➿', '➿'), + ('⤴', '⤵'), + ('⬅', '⬇'), + ('⬛', '⬜'), + ('⭐', '⭐'), + ('⭕', '⭕'), + ('〰', '〰'), + ('〽', '〽'), + ('㊗', '㊗'), + ('㊙', '㊙'), + ('🀀', '\u{1f0ff}'), + ('🄍', '🄏'), + ('🄯', '🄯'), + ('🅬', '🅱'), + ('🅾', '🅿'), + ('🆎', '🆎'), + ('🆑', '🆚'), + ('🆭', '\u{1f1e5}'), + ('🈁', '\u{1f20f}'), + ('🈚', '🈚'), + ('🈯', '🈯'), + ('🈲', '🈺'), + ('\u{1f23c}', '\u{1f23f}'), + ('\u{1f249}', '🏺'), + ('🐀', '🔽'), + ('🕆', '🙏'), + ('🚀', '\u{1f6ff}'), + ('🝴', '🝿'), + ('🟕', '\u{1f7ff}'), + ('\u{1f80c}', '\u{1f80f}'), + ('\u{1f848}', '\u{1f84f}'), + ('\u{1f85a}', '\u{1f85f}'), + ('\u{1f888}', '\u{1f88f}'), + ('\u{1f8ae}', '\u{1f8ff}'), + ('🤌', '🤺'), + ('🤼', '🥅'), + ('🥇', '\u{1faff}'), + ('\u{1fc00}', '\u{1fffd}'), +]; + +pub const EXTENDER: &'static [(char, char)] = &[ + ('·', '·'), + ('ː', 'ˑ'), + ('ـ', 'ـ'), + ('ߺ', 'ߺ'), + ('\u{a71}', '\u{a71}'), + ('\u{afb}', '\u{afb}'), + ('\u{b55}', '\u{b55}'), + ('ๆ', 'ๆ'), + ('ໆ', 'ໆ'), + ('᠊', '᠊'), + ('ᡃ', 'ᡃ'), + ('ᪧ', 'ᪧ'), + ('\u{1c36}', '\u{1c36}'), + ('ᱻ', 'ᱻ'), + ('々', '々'), + ('〱', '〵'), + ('ゝ', 'ゞ'), + ('ー', 'ヾ'), + ('ꀕ', 'ꀕ'), + ('ꘌ', 'ꘌ'), + ('ꧏ', 'ꧏ'), + ('ꧦ', 'ꧦ'), + ('ꩰ', 'ꩰ'), + ('ꫝ', 'ꫝ'), + ('ꫳ', 'ꫴ'), + ('ｰ', 'ｰ'), + ('𐞁', '𐞂'), + ('𐵎', '𐵎'), + ('\u{10d6a}', '\u{10d6a}'), + ('𐵯', '𐵯'), + ('\u{11237}', '\u{11237}'), + ('𑍝', '𑍝'), + ('\u{113d2}', '𑏓'), + ('𑗆', '𑗈'), + ('\u{11a98}', '\u{11a98}'), + ('𖭂', '𖭃'), + ('𖿠', '𖿡'), + ('𖿣', '𖿣'), + ('𞄼', '𞄽'), + ('\u{1e5ef}', '\u{1e5ef}'), + ('\u{1e944}', '\u{1e946}'), +]; + +pub const GRAPHEME_BASE: &'static [(char, char)] = &[ + (' ', '~'), + ('\u{a0}', '¬'), + ('®', '˿'), + ('Ͱ', 'ͷ'), + ('ͺ', 'Ϳ'), + ('΄', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', '҂'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', '֊'), + ('֍', '֏'), + ('־', '־'), + ('׀', '׀'), + ('׃', '׃'), + ('׆', '׆'), + ('א', 'ת'), + ('ׯ', '״'), + ('؆', '؏'), + ('؛', '؛'), + ('؝', 'ي'), + ('٠', 'ٯ'), + ('ٱ', 'ە'), + ('۞', '۞'), + ('ۥ', 'ۦ'), + ('۩', '۩'), + ('ۮ', '܍'), + ('ܐ', 'ܐ'), + ('ܒ', 'ܯ'), + ('ݍ', 'ޥ'), + ('ޱ', 'ޱ'), + ('߀', 'ߪ'), + ('ߴ', 'ߺ'), + ('߾', 'ࠕ'), + ('ࠚ', 'ࠚ'), + ('ࠤ', 'ࠤ'), + ('ࠨ', 'ࠨ'), + ('࠰', '࠾'), + ('ࡀ', 'ࡘ'), + ('࡞', '࡞'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢎ'), + ('ࢠ', 'ࣉ'), + ('ः', 'ह'), + ('ऻ', 'ऻ'), + ('ऽ', 'ी'), + ('ॉ', 'ौ'), + ('ॎ', 'ॐ'), + ('क़', 'ॡ'), + ('।', 'ঀ'), + ('ং', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('ঽ', 'ঽ'), + ('ি', 'ী'), + ('ে', 'ৈ'), + ('ো', 'ৌ'), + ('ৎ', 'ৎ'), + ('ড়', 'ঢ়'), + ('য়', 'ৡ'), + ('০', '৽'), + ('ਃ', 'ਃ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('ਾ', 'ੀ'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('੦', '੯'), + ('ੲ', 'ੴ'), + ('੶', '੶'), + ('ઃ', 'ઃ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('ઽ', 'ી'), + ('ૉ', 'ૉ'), + ('ો', 'ૌ'), + ('ૐ', 'ૐ'), + ('ૠ', 'ૡ'), + ('૦', '૱'), + ('ૹ', 'ૹ'), + ('ଂ', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('ଽ', 'ଽ'), + ('ୀ', 'ୀ'), + ('େ', 'ୈ'), + ('ୋ', 'ୌ'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୡ'), + ('୦', '୷'), + ('ஃ', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('ி', 'ி'), + ('ு', 'ூ'), + ('ெ', 'ை'), + ('ொ', 'ௌ'), + ('ௐ', 'ௐ'), + ('௦', '௺'), + ('ఁ', 'ః'), + ('అ', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('ఽ', 'ఽ'), + ('ు', 'ౄ'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', 'ౡ'), + ('౦', '౯'), + ('౷', 'ಀ'), + ('ಂ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಽ', 'ಾ'), + ('ು', 'ು'), + ('ೃ', 'ೄ'), + ('ೝ', 'ೞ'), + ('ೠ', 'ೡ'), + ('೦', '೯'), + ('ೱ', 'ೳ'), + ('ം', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ഺ'), + ('ഽ', 'ഽ'), + ('ി', 'ീ'), + ('െ', 'ൈ'), + ('ൊ', 'ൌ'), + ('ൎ', '൏'), + ('ൔ', 'ൖ'), + ('൘', 'ൡ'), + ('൦', 'ൿ'), + ('ං', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('ැ', 'ෑ'), + ('ෘ', 'ෞ'), + ('෦', '෯'), + ('ෲ', '෴'), + ('ก', 'ะ'), + ('า', 'ำ'), + ('฿', 'ๆ'), + ('๏', '๛'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ະ'), + ('າ', 'ຳ'), + ('ຽ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('໐', '໙'), + ('ໜ', 'ໟ'), + ('ༀ', '༗'), + ('༚', '༴'), + ('༶', '༶'), + ('༸', '༸'), + ('༺', 'ཇ'), + ('ཉ', 'ཬ'), + ('ཿ', 'ཿ'), + ('྅', '྅'), + ('ྈ', 'ྌ'), + ('྾', '࿅'), + ('࿇', '࿌'), + ('࿎', '࿚'), + ('က', 'ာ'), + ('ေ', 'ေ'), + ('း', 'း'), + ('ျ', 'ြ'), + ('ဿ', 'ၗ'), + ('ၚ', 'ၝ'), + ('ၡ', 'ၰ'), + ('ၵ', 'ႁ'), + ('ႃ', 'ႄ'), + ('ႇ', 'ႌ'), + ('ႎ', 'ႜ'), + ('႞', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('፠', '፼'), + ('ᎀ', '᎙'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('᐀', '᚜'), + ('ᚠ', 'ᛸ'), + ('ᜀ', 'ᜑ'), + ('ᜟ', 'ᜱ'), + ('᜵', '᜶'), + ('ᝀ', 'ᝑ'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('ក', 'ឳ'), + ('ា', 'ា'), + ('ើ', 'ៅ'), + ('ះ', 'ៈ'), + ('។', 'ៜ'), + ('០', '៩'), + ('៰', '៹'), + ('᠀', '᠊'), + ('᠐', '᠙'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢄ'), + ('ᢇ', 'ᢨ'), + ('ᢪ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('ᤣ', 'ᤦ'), + ('ᤩ', 'ᤫ'), + ('ᤰ', 'ᤱ'), + ('ᤳ', 'ᤸ'), + ('᥀', '᥀'), + ('᥄', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('᧐', '᧚'), + ('᧞', 'ᨖ'), + ('ᨙ', 'ᨚ'), + ('᨞', 'ᩕ'), + ('ᩗ', 'ᩗ'), + ('ᩡ', 'ᩡ'), + ('ᩣ', 'ᩤ'), + ('ᩭ', 'ᩲ'), + ('᪀', '᪉'), + ('᪐', '᪙'), + ('᪠', '᪭'), + ('ᬄ', 'ᬳ'), + ('ᬾ', 'ᭁ'), + ('ᭅ', 'ᭌ'), + ('᭎', '᭪'), + ('᭴', '᭿'), + ('ᮂ', 'ᮡ'), + ('ᮦ', 'ᮧ'), + ('ᮮ', 'ᯥ'), + ('ᯧ', 'ᯧ'), + ('ᯪ', 'ᯬ'), + ('ᯮ', 'ᯮ'), + ('᯼', 'ᰫ'), + ('ᰴ', 'ᰵ'), + ('᰻', '᱉'), + ('ᱍ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', '᳇'), + ('᳓', '᳓'), + ('᳡', '᳡'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', '᳷'), + ('ᳺ', 'ᳺ'), + ('ᴀ', 'ᶿ'), + ('Ḁ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ῄ'), + ('ῆ', 'ΐ'), + ('ῖ', 'Ί'), + ('῝', '`'), + ('ῲ', 'ῴ'), + ('ῶ', '῾'), + ('\u{2000}', '\u{200a}'), + ('‐', '‧'), + ('\u{202f}', '\u{205f}'), + ('⁰', 'ⁱ'), + ('⁴', '₎'), + ('ₐ', 'ₜ'), + ('₠', '⃀'), + ('℀', '↋'), + ('←', '␩'), + ('⑀', '⑊'), + ('①', '⭳'), + ('⭶', '⮕'), + ('⮗', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('⳹', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', '⵰'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('⸀', '⹝'), + ('⺀', '⺙'), + ('⺛', '⻳'), + ('⼀', '⿕'), + ('⿰', '〩'), + ('〰', '〿'), + ('ぁ', 'ゖ'), + ('゛', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('㆐', '㇥'), + ('㇯', '㈞'), + ('㈠', 'ꒌ'), + ('꒐', '꓆'), + ('ꓐ', 'ꘫ'), + ('Ꙁ', 'ꙮ'), + ('꙳', '꙳'), + ('꙾', 'ꚝ'), + ('ꚠ', 'ꛯ'), + ('꛲', '꛷'), + ('꜀', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠁ'), + ('ꠃ', 'ꠅ'), + ('ꠇ', 'ꠊ'), + ('ꠌ', 'ꠤ'), + ('ꠧ', '꠫'), + ('꠰', '꠹'), + ('ꡀ', '꡷'), + ('ꢀ', 'ꣃ'), + ('꣎', '꣙'), + ('ꣲ', 'ꣾ'), + ('꤀', 'ꤥ'), + ('꤮', 'ꥆ'), + ('ꥒ', 'ꥒ'), + ('꥟', 'ꥼ'), + ('ꦃ', 'ꦲ'), + ('ꦴ', 'ꦵ'), + ('ꦺ', 'ꦻ'), + ('ꦾ', 'ꦿ'), + ('꧁', '꧍'), + ('ꧏ', '꧙'), + ('꧞', 'ꧤ'), + ('ꧦ', 'ꧾ'), + ('ꨀ', 'ꨨ'), + ('ꨯ', 'ꨰ'), + ('ꨳ', 'ꨴ'), + ('ꩀ', 'ꩂ'), + ('ꩄ', 'ꩋ'), + ('ꩍ', 'ꩍ'), + ('꩐', '꩙'), + ('꩜', 'ꩻ'), + ('ꩽ', 'ꪯ'), + ('ꪱ', 'ꪱ'), + ('ꪵ', 'ꪶ'), + ('ꪹ', 'ꪽ'), + ('ꫀ', 'ꫀ'), + ('ꫂ', 'ꫂ'), + ('ꫛ', 'ꫫ'), + ('ꫮ', 'ꫵ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', '꭫'), + ('ꭰ', 'ꯤ'), + ('ꯦ', 'ꯧ'), + ('ꯩ', '꯬'), + ('꯰', '꯹'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'יִ'), + ('ײַ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', '﯂'), + ('ﯓ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('﷏', '﷏'), + ('ﷰ', '﷿'), + ('︐', '︙'), + ('︰', '﹒'), + ('﹔', '﹦'), + ('﹨', '﹫'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('！', 'ﾝ'), + ('ﾠ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('￠', '￦'), + ('￨', '￮'), + ('', '�'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐄀', '𐄂'), + ('𐄇', '𐄳'), + ('𐄷', '𐆎'), + ('𐆐', '𐆜'), + ('𐆠', '𐆠'), + ('𐇐', '𐇼'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐋡', '𐋻'), + ('𐌀', '𐌣'), + ('𐌭', '𐍊'), + ('𐍐', '𐍵'), + ('𐎀', '𐎝'), + ('𐎟', '𐏃'), + ('𐏈', '𐏕'), + ('𐐀', '𐒝'), + ('𐒠', '𐒩'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕯', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡗', '𐢞'), + ('𐢧', '𐢯'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐣻', '𐤛'), + ('𐤟', '𐤹'), + ('𐤿', '𐤿'), + ('𐦀', '𐦷'), + ('𐦼', '𐧏'), + ('𐧒', '𐨀'), + ('𐨐', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('𐩀', '𐩈'), + ('𐩐', '𐩘'), + ('𐩠', '𐪟'), + ('𐫀', '𐫤'), + ('𐫫', '𐫶'), + ('𐬀', '𐬵'), + ('𐬹', '𐭕'), + ('𐭘', '𐭲'), + ('𐭸', '𐮑'), + ('𐮙', '𐮜'), + ('𐮩', '𐮯'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐳺', '𐴣'), + ('𐴰', '𐴹'), + ('𐵀', '𐵥'), + ('𐵮', '𐶅'), + ('𐶎', '𐶏'), + ('𐹠', '𐹾'), + ('𐺀', '𐺩'), + ('𐺭', '𐺭'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('𐼀', '𐼧'), + ('𐼰', '𐽅'), + ('𐽑', '𐽙'), + ('𐽰', '𐾁'), + ('𐾆', '𐾉'), + ('𐾰', '𐿋'), + ('𐿠', '𐿶'), + ('𑀀', '𑀀'), + ('𑀂', '𑀷'), + ('𑁇', '𑁍'), + ('𑁒', '𑁯'), + ('𑁱', '𑁲'), + ('𑁵', '𑁵'), + ('𑂂', '𑂲'), + ('𑂷', '𑂸'), + ('𑂻', '𑂼'), + ('𑂾', '𑃁'), + ('𑃐', '𑃨'), + ('𑃰', '𑃹'), + ('𑄃', '𑄦'), + ('𑄬', '𑄬'), + ('𑄶', '𑅇'), + ('𑅐', '𑅲'), + ('𑅴', '𑅶'), + ('𑆂', '𑆵'), + ('𑆿', '𑆿'), + ('𑇁', '𑇈'), + ('𑇍', '𑇎'), + ('𑇐', '𑇟'), + ('𑇡', '𑇴'), + ('𑈀', '𑈑'), + ('𑈓', '𑈮'), + ('𑈲', '𑈳'), + ('𑈸', '𑈽'), + ('𑈿', '𑉀'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊩'), + ('𑊰', '𑋞'), + ('𑋠', '𑋢'), + ('𑋰', '𑋹'), + ('𑌂', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('𑌽', '𑌽'), + ('𑌿', '𑌿'), + ('𑍁', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '𑍌'), + ('𑍐', '𑍐'), + ('𑍝', '𑍣'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '𑎷'), + ('𑎹', '𑎺'), + ('𑏊', '𑏊'), + ('𑏌', '𑏍'), + ('𑏑', '𑏑'), + ('𑏓', '𑏕'), + ('𑏗', '𑏘'), + ('𑐀', '𑐷'), + ('𑑀', '𑑁'), + ('𑑅', '𑑅'), + ('𑑇', '𑑛'), + ('𑑝', '𑑝'), + ('𑑟', '𑑡'), + ('𑒀', '𑒯'), + ('𑒱', '𑒲'), + ('𑒹', '𑒹'), + ('𑒻', '𑒼'), + ('𑒾', '𑒾'), + ('𑓁', '𑓁'), + ('𑓄', '𑓇'), + ('𑓐', '𑓙'), + ('𑖀', '𑖮'), + ('𑖰', '𑖱'), + ('𑖸', '𑖻'), + ('𑖾', '𑖾'), + ('𑗁', '𑗛'), + ('𑘀', '𑘲'), + ('𑘻', '𑘼'), + ('𑘾', '𑘾'), + ('𑙁', '𑙄'), + ('𑙐', '𑙙'), + ('𑙠', '𑙬'), + ('𑚀', '𑚪'), + ('𑚬', '𑚬'), + ('𑚮', '𑚯'), + ('𑚸', '𑚹'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜀', '𑜚'), + ('𑜞', '𑜞'), + ('𑜠', '𑜡'), + ('𑜦', '𑜦'), + ('𑜰', '𑝆'), + ('𑠀', '𑠮'), + ('𑠸', '𑠸'), + ('𑠻', '𑠻'), + ('𑢠', '𑣲'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤯'), + ('𑤱', '𑤵'), + ('𑤷', '𑤸'), + ('𑤿', '𑥂'), + ('𑥄', '𑥆'), + ('𑥐', '𑥙'), + ('𑦠', '𑦧'), + ('𑦪', '𑧓'), + ('𑧜', '𑧟'), + ('𑧡', '𑧤'), + ('𑨀', '𑨀'), + ('𑨋', '𑨲'), + ('𑨹', '𑨺'), + ('𑨿', '𑩆'), + ('𑩐', '𑩐'), + ('𑩗', '𑩘'), + ('𑩜', '𑪉'), + ('𑪗', '𑪗'), + ('𑪚', '𑪢'), + ('𑪰', '𑫸'), + ('𑬀', '𑬉'), + ('𑯀', '𑯡'), + ('𑯰', '𑯹'), + ('𑰀', '𑰈'), + ('𑰊', '𑰯'), + ('𑰾', '𑰾'), + ('𑱀', '𑱅'), + ('𑱐', '𑱬'), + ('𑱰', '𑲏'), + ('𑲩', '𑲩'), + ('𑲱', '𑲱'), + ('𑲴', '𑲴'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '𑴰'), + ('𑵆', '𑵆'), + ('𑵐', '𑵙'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('𑶓', '𑶔'), + ('𑶖', '𑶖'), + ('𑶘', '𑶘'), + ('𑶠', '𑶩'), + ('𑻠', '𑻲'), + ('𑻵', '𑻸'), + ('𑼂', '𑼐'), + ('𑼒', '𑼵'), + ('𑼾', '𑼿'), + ('𑽃', '𑽙'), + ('𑾰', '𑾰'), + ('𑿀', '𑿱'), + ('𑿿', '𒎙'), + ('𒐀', '𒑮'), + ('𒑰', '𒑴'), + ('𒒀', '𒕃'), + ('𒾐', '𒿲'), + ('𓀀', '𓐯'), + ('𓑁', '𓑆'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄝'), + ('𖄪', '𖄬'), + ('𖄰', '𖄹'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩠', '𖩩'), + ('𖩮', '𖪾'), + ('𖫀', '𖫉'), + ('𖫐', '𖫭'), + ('𖫵', '𖫵'), + ('𖬀', '𖬯'), + ('𖬷', '𖭅'), + ('𖭐', '𖭙'), + ('𖭛', '𖭡'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵹'), + ('𖹀', '𖺚'), + ('𖼀', '𖽊'), + ('𖽐', '𖾇'), + ('𖾓', '𖾟'), + ('𖿠', '𖿣'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𛲜', '𛲜'), + ('𛲟', '𛲟'), + ('𜰀', '𜳹'), + ('𜴀', '𜺳'), + ('𜽐', '𜿃'), + ('𝀀', '𝃵'), + ('𝄀', '𝄦'), + ('𝄩', '𝅘𝅥𝅲'), + ('𝅪', '𝅬'), + ('𝆃', '𝆄'), + ('𝆌', '𝆩'), + ('𝆮', '𝇪'), + ('𝈀', '𝉁'), + ('𝉅', '𝉅'), + ('𝋀', '𝋓'), + ('𝋠', '𝋳'), + ('𝌀', '𝍖'), + ('𝍠', '𝍸'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝟋'), + ('𝟎', '𝧿'), + ('𝨷', '𝨺'), + ('𝩭', '𝩴'), + ('𝩶', '𝪃'), + ('𝪅', '𝪋'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('𞄀', '𞄬'), + ('𞄷', '𞄽'), + ('𞅀', '𞅉'), + ('𞅎', '𞅏'), + ('𞊐', '𞊭'), + ('𞋀', '𞋫'), + ('𞋰', '𞋹'), + ('𞋿', '𞋿'), + ('𞓐', '𞓫'), + ('𞓰', '𞓹'), + ('𞗐', '𞗭'), + ('𞗰', '𞗺'), + ('𞗿', '𞗿'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('𞣇', '𞣏'), + ('𞤀', '𞥃'), + ('𞥋', '𞥋'), + ('𞥐', '𞥙'), + ('𞥞', '𞥟'), + ('𞱱', '𞲴'), + ('𞴁', '𞴽'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𞻰', '𞻱'), + ('🀀', '🀫'), + ('🀰', '🂓'), + ('🂠', '🂮'), + ('🂱', '🂿'), + ('🃁', '🃏'), + ('🃑', '🃵'), + ('🄀', '🆭'), + ('🇦', '🈂'), + ('🈐', '🈻'), + ('🉀', '🉈'), + ('🉐', '🉑'), + ('🉠', '🉥'), + ('🌀', '🛗'), + ('🛜', '🛬'), + ('🛰', '🛼'), + ('🜀', '🝶'), + ('🝻', '🟙'), + ('🟠', '🟫'), + ('🟰', '🟰'), + ('🠀', '🠋'), + ('🠐', '🡇'), + ('🡐', '🡙'), + ('🡠', '🢇'), + ('🢐', '🢭'), + ('🢰', '🢻'), + ('🣀', '🣁'), + ('🤀', '🩓'), + ('🩠', '🩭'), + ('🩰', '🩼'), + ('🪀', '🪉'), + ('🪏', '🫆'), + ('🫎', '🫜'), + ('🫟', '🫩'), + ('🫰', '🫸'), + ('🬀', '🮒'), + ('🮔', '🯹'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const GRAPHEME_EXTEND: &'static [(char, char)] = &[ + ('\u{300}', '\u{36f}'), + ('\u{483}', '\u{489}'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('\u{610}', '\u{61a}'), + ('\u{64b}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dc}'), + ('\u{6df}', '\u{6e4}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{6ea}', '\u{6ed}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', '\u{7f3}'), + ('\u{7fd}', '\u{7fd}'), + ('\u{816}', '\u{819}'), + ('\u{81b}', '\u{823}'), + ('\u{825}', '\u{827}'), + ('\u{829}', '\u{82d}'), + ('\u{859}', '\u{85b}'), + ('\u{897}', '\u{89f}'), + ('\u{8ca}', '\u{8e1}'), + ('\u{8e3}', '\u{902}'), + ('\u{93a}', '\u{93a}'), + ('\u{93c}', '\u{93c}'), + ('\u{941}', '\u{948}'), + ('\u{94d}', '\u{94d}'), + ('\u{951}', '\u{957}'), + ('\u{962}', '\u{963}'), + ('\u{981}', '\u{981}'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9be}', '\u{9be}'), + ('\u{9c1}', '\u{9c4}'), + ('\u{9cd}', '\u{9cd}'), + ('\u{9d7}', '\u{9d7}'), + ('\u{9e2}', '\u{9e3}'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', '\u{a02}'), + ('\u{a3c}', '\u{a3c}'), + ('\u{a41}', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', '\u{a82}'), + ('\u{abc}', '\u{abc}'), + ('\u{ac1}', '\u{ac5}'), + ('\u{ac7}', '\u{ac8}'), + ('\u{acd}', '\u{acd}'), + ('\u{ae2}', '\u{ae3}'), + ('\u{afa}', '\u{aff}'), + ('\u{b01}', '\u{b01}'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b3e}', '\u{b3f}'), + ('\u{b41}', '\u{b44}'), + ('\u{b4d}', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('\u{b62}', '\u{b63}'), + ('\u{b82}', '\u{b82}'), + ('\u{bbe}', '\u{bbe}'), + ('\u{bc0}', '\u{bc0}'), + ('\u{bcd}', '\u{bcd}'), + ('\u{bd7}', '\u{bd7}'), + ('\u{c00}', '\u{c00}'), + ('\u{c04}', '\u{c04}'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c3e}', '\u{c40}'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', '\u{c81}'), + ('\u{cbc}', '\u{cbc}'), + ('\u{cbf}', '\u{cc0}'), + ('\u{cc2}', '\u{cc2}'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('\u{ce2}', '\u{ce3}'), + ('\u{d00}', '\u{d01}'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d3e}', '\u{d3e}'), + ('\u{d41}', '\u{d44}'), + ('\u{d4d}', '\u{d4d}'), + ('\u{d57}', '\u{d57}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', '\u{d81}'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dcf}'), + ('\u{dd2}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('\u{ddf}', '\u{ddf}'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('\u{e47}', '\u{e4e}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{ebc}'), + ('\u{ec8}', '\u{ece}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('\u{f71}', '\u{f7e}'), + ('\u{f80}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('\u{102d}', '\u{1030}'), + ('\u{1032}', '\u{1037}'), + ('\u{1039}', '\u{103a}'), + ('\u{103d}', '\u{103e}'), + ('\u{1058}', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{1082}'), + ('\u{1085}', '\u{1086}'), + ('\u{108d}', '\u{108d}'), + ('\u{109d}', '\u{109d}'), + ('\u{135d}', '\u{135f}'), + ('\u{1712}', '\u{1715}'), + ('\u{1732}', '\u{1734}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('\u{17b4}', '\u{17b5}'), + ('\u{17b7}', '\u{17bd}'), + ('\u{17c6}', '\u{17c6}'), + ('\u{17c9}', '\u{17d3}'), + ('\u{17dd}', '\u{17dd}'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '\u{180f}'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', '\u{1922}'), + ('\u{1927}', '\u{1928}'), + ('\u{1932}', '\u{1932}'), + ('\u{1939}', '\u{193b}'), + ('\u{1a17}', '\u{1a18}'), + ('\u{1a1b}', '\u{1a1b}'), + ('\u{1a56}', '\u{1a56}'), + ('\u{1a58}', '\u{1a5e}'), + ('\u{1a60}', '\u{1a60}'), + ('\u{1a62}', '\u{1a62}'), + ('\u{1a65}', '\u{1a6c}'), + ('\u{1a73}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', '\u{1b03}'), + ('\u{1b34}', '\u{1b3d}'), + ('\u{1b42}', '\u{1b44}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1b81}'), + ('\u{1ba2}', '\u{1ba5}'), + ('\u{1ba8}', '\u{1bad}'), + ('\u{1be6}', '\u{1be6}'), + ('\u{1be8}', '\u{1be9}'), + ('\u{1bed}', '\u{1bed}'), + ('\u{1bef}', '\u{1bf3}'), + ('\u{1c2c}', '\u{1c33}'), + ('\u{1c36}', '\u{1c37}'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce0}'), + ('\u{1ce2}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('\u{1cf8}', '\u{1cf9}'), + ('\u{1dc0}', '\u{1dff}'), + ('\u{200c}', '\u{200c}'), + ('\u{20d0}', '\u{20f0}'), + ('\u{2cef}', '\u{2cf1}'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{2de0}', '\u{2dff}'), + ('\u{302a}', '\u{302f}'), + ('\u{3099}', '\u{309a}'), + ('\u{a66f}', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('\u{a69e}', '\u{a69f}'), + ('\u{a6f0}', '\u{a6f1}'), + ('\u{a802}', '\u{a802}'), + ('\u{a806}', '\u{a806}'), + ('\u{a80b}', '\u{a80b}'), + ('\u{a825}', '\u{a826}'), + ('\u{a82c}', '\u{a82c}'), + ('\u{a8c4}', '\u{a8c5}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92d}'), + ('\u{a947}', '\u{a951}'), + ('\u{a953}', '\u{a953}'), + ('\u{a980}', '\u{a982}'), + ('\u{a9b3}', '\u{a9b3}'), + ('\u{a9b6}', '\u{a9b9}'), + ('\u{a9bc}', '\u{a9bd}'), + ('\u{a9c0}', '\u{a9c0}'), + ('\u{a9e5}', '\u{a9e5}'), + ('\u{aa29}', '\u{aa2e}'), + ('\u{aa31}', '\u{aa32}'), + ('\u{aa35}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', '\u{aa4c}'), + ('\u{aa7c}', '\u{aa7c}'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabf}'), + ('\u{aac1}', '\u{aac1}'), + ('\u{aaec}', '\u{aaed}'), + ('\u{aaf6}', '\u{aaf6}'), + ('\u{abe5}', '\u{abe5}'), + ('\u{abe8}', '\u{abe8}'), + ('\u{abed}', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('\u{ff9e}', '\u{ff9f}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{10376}', '\u{1037a}'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('\u{10d24}', '\u{10d27}'), + ('\u{10d69}', '\u{10d6d}'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('\u{11001}', '\u{11001}'), + ('\u{11038}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{11073}', '\u{11074}'), + ('\u{1107f}', '\u{11081}'), + ('\u{110b3}', '\u{110b6}'), + ('\u{110b9}', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{1112b}'), + ('\u{1112d}', '\u{11134}'), + ('\u{11173}', '\u{11173}'), + ('\u{11180}', '\u{11181}'), + ('\u{111b6}', '\u{111be}'), + ('\u{111c0}', '\u{111c0}'), + ('\u{111c9}', '\u{111cc}'), + ('\u{111cf}', '\u{111cf}'), + ('\u{1122f}', '\u{11231}'), + ('\u{11234}', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112df}'), + ('\u{112e3}', '\u{112ea}'), + ('\u{11300}', '\u{11301}'), + ('\u{1133b}', '\u{1133c}'), + ('\u{1133e}', '\u{1133e}'), + ('\u{11340}', '\u{11340}'), + ('\u{1134d}', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113b8}', '\u{113b8}'), + ('\u{113bb}', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '\u{113c9}'), + ('\u{113ce}', '\u{113d0}'), + ('\u{113d2}', '\u{113d2}'), + ('\u{113e1}', '\u{113e2}'), + ('\u{11438}', '\u{1143f}'), + ('\u{11442}', '\u{11444}'), + ('\u{11446}', '\u{11446}'), + ('\u{1145e}', '\u{1145e}'), + ('\u{114b0}', '\u{114b0}'), + ('\u{114b3}', '\u{114b8}'), + ('\u{114ba}', '\u{114ba}'), + ('\u{114bd}', '\u{114bd}'), + ('\u{114bf}', '\u{114c0}'), + ('\u{114c2}', '\u{114c3}'), + ('\u{115af}', '\u{115af}'), + ('\u{115b2}', '\u{115b5}'), + ('\u{115bc}', '\u{115bd}'), + ('\u{115bf}', '\u{115c0}'), + ('\u{115dc}', '\u{115dd}'), + ('\u{11633}', '\u{1163a}'), + ('\u{1163d}', '\u{1163d}'), + ('\u{1163f}', '\u{11640}'), + ('\u{116ab}', '\u{116ab}'), + ('\u{116ad}', '\u{116ad}'), + ('\u{116b0}', '\u{116b7}'), + ('\u{1171d}', '\u{1171d}'), + ('\u{1171f}', '\u{1171f}'), + ('\u{11722}', '\u{11725}'), + ('\u{11727}', '\u{1172b}'), + ('\u{1182f}', '\u{11837}'), + ('\u{11839}', '\u{1183a}'), + ('\u{11930}', '\u{11930}'), + ('\u{1193b}', '\u{1193e}'), + ('\u{11943}', '\u{11943}'), + ('\u{119d4}', '\u{119d7}'), + ('\u{119da}', '\u{119db}'), + ('\u{119e0}', '\u{119e0}'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a33}', '\u{11a38}'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a51}', '\u{11a56}'), + ('\u{11a59}', '\u{11a5b}'), + ('\u{11a8a}', '\u{11a96}'), + ('\u{11a98}', '\u{11a99}'), + ('\u{11c30}', '\u{11c36}'), + ('\u{11c38}', '\u{11c3d}'), + ('\u{11c3f}', '\u{11c3f}'), + ('\u{11c92}', '\u{11ca7}'), + ('\u{11caa}', '\u{11cb0}'), + ('\u{11cb2}', '\u{11cb3}'), + ('\u{11cb5}', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d45}'), + ('\u{11d47}', '\u{11d47}'), + ('\u{11d90}', '\u{11d91}'), + ('\u{11d95}', '\u{11d95}'), + ('\u{11d97}', '\u{11d97}'), + ('\u{11ef3}', '\u{11ef4}'), + ('\u{11f00}', '\u{11f01}'), + ('\u{11f36}', '\u{11f3a}'), + ('\u{11f40}', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13440}', '\u{13440}'), + ('\u{13447}', '\u{13455}'), + ('\u{1611e}', '\u{16129}'), + ('\u{1612d}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('\u{16f4f}', '\u{16f4f}'), + ('\u{16f8f}', '\u{16f92}'), + ('\u{16fe4}', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e130}', '\u{1e136}'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('\u{1e4ec}', '\u{1e4ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '\u{1e94a}'), + ('\u{e0020}', '\u{e007f}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const GRAPHEME_LINK: &'static [(char, char)] = &[ + ('\u{94d}', '\u{94d}'), + ('\u{9cd}', '\u{9cd}'), + ('\u{a4d}', '\u{a4d}'), + ('\u{acd}', '\u{acd}'), + ('\u{b4d}', '\u{b4d}'), + ('\u{bcd}', '\u{bcd}'), + ('\u{c4d}', '\u{c4d}'), + ('\u{ccd}', '\u{ccd}'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d4d}', '\u{d4d}'), + ('\u{dca}', '\u{dca}'), + ('\u{e3a}', '\u{e3a}'), + ('\u{eba}', '\u{eba}'), + ('\u{f84}', '\u{f84}'), + ('\u{1039}', '\u{103a}'), + ('\u{1714}', '\u{1715}'), + ('\u{1734}', '\u{1734}'), + ('\u{17d2}', '\u{17d2}'), + ('\u{1a60}', '\u{1a60}'), + ('\u{1b44}', '\u{1b44}'), + ('\u{1baa}', '\u{1bab}'), + ('\u{1bf2}', '\u{1bf3}'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{a806}', '\u{a806}'), + ('\u{a82c}', '\u{a82c}'), + ('\u{a8c4}', '\u{a8c4}'), + ('\u{a953}', '\u{a953}'), + ('\u{a9c0}', '\u{a9c0}'), + ('\u{aaf6}', '\u{aaf6}'), + ('\u{abed}', '\u{abed}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{11046}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{1107f}', '\u{1107f}'), + ('\u{110b9}', '\u{110b9}'), + ('\u{11133}', '\u{11134}'), + ('\u{111c0}', '\u{111c0}'), + ('\u{11235}', '\u{11235}'), + ('\u{112ea}', '\u{112ea}'), + ('\u{1134d}', '\u{1134d}'), + ('\u{113ce}', '\u{113d0}'), + ('\u{11442}', '\u{11442}'), + ('\u{114c2}', '\u{114c2}'), + ('\u{115bf}', '\u{115bf}'), + ('\u{1163f}', '\u{1163f}'), + ('\u{116b6}', '\u{116b6}'), + ('\u{1172b}', '\u{1172b}'), + ('\u{11839}', '\u{11839}'), + ('\u{1193d}', '\u{1193e}'), + ('\u{119e0}', '\u{119e0}'), + ('\u{11a34}', '\u{11a34}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a99}', '\u{11a99}'), + ('\u{11c3f}', '\u{11c3f}'), + ('\u{11d44}', '\u{11d45}'), + ('\u{11d97}', '\u{11d97}'), + ('\u{11f41}', '\u{11f42}'), + ('\u{1612f}', '\u{1612f}'), +]; + +pub const HEX_DIGIT: &'static [(char, char)] = &[ + ('0', '9'), + ('A', 'F'), + ('a', 'f'), + ('０', '９'), + ('Ａ', 'Ｆ'), + ('ａ', 'ｆ'), +]; + +pub const HYPHEN: &'static [(char, char)] = &[ + ('-', '-'), + ('\u{ad}', '\u{ad}'), + ('֊', '֊'), + ('᠆', '᠆'), + ('‐', '‑'), + ('⸗', '⸗'), + ('・', '・'), + ('﹣', '﹣'), + ('－', '－'), + ('･', '･'), +]; + +pub const IDS_BINARY_OPERATOR: &'static [(char, char)] = + &[('⿰', '⿱'), ('⿴', '⿽'), ('㇯', '㇯')]; + +pub const IDS_TRINARY_OPERATOR: &'static [(char, char)] = &[('⿲', '⿳')]; + +pub const IDS_UNARY_OPERATOR: &'static [(char, char)] = &[('⿾', '⿿')]; + +pub const ID_COMPAT_MATH_CONTINUE: &'static [(char, char)] = &[ + ('²', '³'), + ('¹', '¹'), + ('⁰', '⁰'), + ('⁴', '⁾'), + ('₀', '₎'), + ('∂', '∂'), + ('∇', '∇'), + ('∞', '∞'), + ('𝛁', '𝛁'), + ('𝛛', '𝛛'), + ('𝛻', '𝛻'), + ('𝜕', '𝜕'), + ('𝜵', '𝜵'), + ('𝝏', '𝝏'), + ('𝝯', '𝝯'), + ('𝞉', '𝞉'), + ('𝞩', '𝞩'), + ('𝟃', '𝟃'), +]; + +pub const ID_COMPAT_MATH_START: &'static [(char, char)] = &[ + ('∂', '∂'), + ('∇', '∇'), + ('∞', '∞'), + ('𝛁', '𝛁'), + ('𝛛', '𝛛'), + ('𝛻', '𝛻'), + ('𝜕', '𝜕'), + ('𝜵', '𝜵'), + ('𝝏', '𝝏'), + ('𝝯', '𝝯'), + ('𝞉', '𝞉'), + ('𝞩', '𝞩'), + ('𝟃', '𝟃'), +]; + +pub const ID_CONTINUE: &'static [(char, char)] = &[ + ('0', '9'), + ('A', 'Z'), + ('_', '_'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('·', '·'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('\u{300}', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('\u{483}', '\u{487}'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', 'ՙ'), + ('ՠ', 'ֈ'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('\u{610}', '\u{61a}'), + ('ؠ', '٩'), + ('ٮ', 'ۓ'), + ('ە', '\u{6dc}'), + ('\u{6df}', '\u{6e8}'), + ('\u{6ea}', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', '\u{74a}'), + ('ݍ', 'ޱ'), + ('߀', 'ߵ'), + ('ߺ', 'ߺ'), + ('\u{7fd}', '\u{7fd}'), + ('ࠀ', '\u{82d}'), + ('ࡀ', '\u{85b}'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('\u{897}', '\u{8e1}'), + ('\u{8e3}', '\u{963}'), + ('०', '९'), + ('ॱ', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('\u{9bc}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', 'ৎ'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', '\u{9e3}'), + ('০', 'ৱ'), + ('ৼ', 'ৼ'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', 'ਃ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('੦', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('\u{abc}', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('ૐ', 'ૐ'), + ('ૠ', '\u{ae3}'), + ('૦', '૯'), + ('ૹ', '\u{aff}'), + ('\u{b01}', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('\u{b3c}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', '\u{b63}'), + ('୦', '୯'), + ('ୱ', 'ୱ'), + ('\u{b82}', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('ௐ', 'ௐ'), + ('\u{bd7}', '\u{bd7}'), + ('௦', '௯'), + ('\u{c00}', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('\u{c3c}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', '\u{c63}'), + ('౦', '౯'), + ('ಀ', 'ಃ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('\u{cbc}', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('ೝ', 'ೞ'), + ('ೠ', '\u{ce3}'), + ('೦', '೯'), + ('ೱ', 'ೳ'), + ('\u{d00}', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', 'ൎ'), + ('ൔ', '\u{d57}'), + ('ൟ', '\u{d63}'), + ('൦', '൯'), + ('ൺ', 'ൿ'), + ('\u{d81}', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('෦', '෯'), + ('ෲ', 'ෳ'), + ('ก', '\u{e3a}'), + ('เ', '\u{e4e}'), + ('๐', '๙'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('\u{ec8}', '\u{ece}'), + ('໐', '໙'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('\u{f18}', '\u{f19}'), + ('༠', '༩'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('༾', 'ཇ'), + ('ཉ', 'ཬ'), + ('\u{f71}', '\u{f84}'), + ('\u{f86}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('က', '၉'), + ('ၐ', '\u{109d}'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('\u{135d}', '\u{135f}'), + ('፩', '፱'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', '\u{1715}'), + ('ᜟ', '\u{1734}'), + ('ᝀ', '\u{1753}'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('\u{1772}', '\u{1773}'), + ('ក', '\u{17d3}'), + ('ៗ', 'ៗ'), + ('ៜ', '\u{17dd}'), + ('០', '៩'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '᠙'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('᥆', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('᧐', '᧚'), + ('ᨀ', '\u{1a1b}'), + ('ᨠ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '᪉'), + ('᪐', '᪙'), + ('ᪧ', 'ᪧ'), + ('\u{1ab0}', '\u{1abd}'), + ('\u{1abf}', '\u{1ace}'), + ('\u{1b00}', 'ᭌ'), + ('᭐', '᭙'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1bf3}'), + ('ᰀ', '\u{1c37}'), + ('᱀', '᱉'), + ('ᱍ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', 'ᳺ'), + ('ᴀ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('\u{200c}', '\u{200d}'), + ('‿', '⁀'), + ('⁔', '⁔'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('\u{20d0}', '\u{20dc}'), + ('\u{20e1}', '\u{20e1}'), + ('\u{20e5}', '\u{20f0}'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('℘', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('\u{2d7f}', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('\u{2de0}', '\u{2dff}'), + ('々', '〇'), + ('〡', '\u{302f}'), + ('〱', '〵'), + ('〸', '〼'), + ('ぁ', 'ゖ'), + ('\u{3099}', 'ゟ'), + ('ァ', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘫ'), + ('Ꙁ', '\u{a66f}'), + ('\u{a674}', '\u{a67d}'), + ('ꙿ', '\u{a6f1}'), + ('ꜗ', 'ꜟ'), + ('Ꜣ', 'ꞈ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠧ'), + ('\u{a82c}', '\u{a82c}'), + ('ꡀ', 'ꡳ'), + ('ꢀ', '\u{a8c5}'), + ('꣐', '꣙'), + ('\u{a8e0}', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', '\u{a92d}'), + ('ꤰ', '\u{a953}'), + ('ꥠ', 'ꥼ'), + ('\u{a980}', '\u{a9c0}'), + ('ꧏ', '꧙'), + ('ꧠ', 'ꧾ'), + ('ꨀ', '\u{aa36}'), + ('ꩀ', 'ꩍ'), + ('꩐', '꩙'), + ('ꩠ', 'ꩶ'), + ('ꩺ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫯ'), + ('ꫲ', '\u{aaf6}'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꯪ'), + ('꯬', '\u{abed}'), + ('꯰', '꯹'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('︳', '︴'), + ('﹍', '﹏'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('０', '９'), + ('Ａ', 'Ｚ'), + ('＿', '＿'), + ('ａ', 'ｚ'), + ('･', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('\u{101fd}', '\u{101fd}'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('\u{102e0}', '\u{102e0}'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '\u{1037a}'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐐀', '𐒝'), + ('𐒠', '𐒩'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '\u{10ae6}'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '\u{10d27}'), + ('𐴰', '𐴹'), + ('𐵀', '𐵥'), + ('\u{10d69}', '\u{10d6d}'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('\u{10eab}', '\u{10eac}'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('\u{10efc}', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '\u{10f50}'), + ('𐽰', '\u{10f85}'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀀', '\u{11046}'), + ('𑁦', '𑁵'), + ('\u{1107f}', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('𑃐', '𑃨'), + ('𑃰', '𑃹'), + ('\u{11100}', '\u{11134}'), + ('𑄶', '𑄿'), + ('𑅄', '𑅇'), + ('𑅐', '\u{11173}'), + ('𑅶', '𑅶'), + ('\u{11180}', '𑇄'), + ('\u{111c9}', '\u{111cc}'), + ('𑇎', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '\u{11237}'), + ('\u{1123e}', '\u{11241}'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '\u{112ea}'), + ('𑋰', '𑋹'), + ('\u{11300}', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('\u{1133b}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('𑍐', '𑍐'), + ('\u{11357}', '\u{11357}'), + ('𑍝', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏓'), + ('\u{113e1}', '\u{113e2}'), + ('𑐀', '𑑊'), + ('𑑐', '𑑙'), + ('\u{1145e}', '𑑡'), + ('𑒀', '𑓅'), + ('𑓇', '𑓇'), + ('𑓐', '𑓙'), + ('𑖀', '\u{115b5}'), + ('𑖸', '\u{115c0}'), + ('𑗘', '\u{115dd}'), + ('𑘀', '\u{11640}'), + ('𑙄', '𑙄'), + ('𑙐', '𑙙'), + ('𑚀', '𑚸'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜀', '𑜚'), + ('\u{1171d}', '\u{1172b}'), + ('𑜰', '𑜹'), + ('𑝀', '𑝆'), + ('𑠀', '\u{1183a}'), + ('𑢠', '𑣩'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{11943}'), + ('𑥐', '𑥙'), + ('𑦠', '𑦧'), + ('𑦪', '\u{119d7}'), + ('\u{119da}', '𑧡'), + ('𑧣', '𑧤'), + ('𑨀', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('𑩐', '\u{11a99}'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑯰', '𑯹'), + ('𑰀', '𑰈'), + ('𑰊', '\u{11c36}'), + ('\u{11c38}', '𑱀'), + ('𑱐', '𑱙'), + ('𑱲', '𑲏'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d47}'), + ('𑵐', '𑵙'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶘'), + ('𑶠', '𑶩'), + ('𑻠', '𑻶'), + ('\u{11f00}', '𑼐'), + ('𑼒', '\u{11f3a}'), + ('𑼾', '\u{11f42}'), + ('𑽐', '\u{11f5a}'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('\u{13440}', '\u{13455}'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄹'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩠', '𖩩'), + ('𖩰', '𖪾'), + ('𖫀', '𖫉'), + ('𖫐', '𖫭'), + ('\u{16af0}', '\u{16af4}'), + ('𖬀', '\u{16b36}'), + ('𖭀', '𖭃'), + ('𖭐', '𖭙'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖵰', '𖵹'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('\u{16f4f}', '𖾇'), + ('\u{16f8f}', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('𜳰', '𜳹'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝟎', '𝟿'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), + ('𞄀', '𞄬'), + ('\u{1e130}', '𞄽'), + ('𞅀', '𞅉'), + ('𞅎', '𞅎'), + ('𞊐', '\u{1e2ae}'), + ('𞋀', '𞋹'), + ('𞓐', '𞓹'), + ('𞗐', '𞗺'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('𞤀', '𞥋'), + ('𞥐', '𞥙'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('🯰', '🯹'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const ID_START: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('Ͱ', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', 'ՙ'), + ('ՠ', 'ֈ'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('ؠ', 'ي'), + ('ٮ', 'ٯ'), + ('ٱ', 'ۓ'), + ('ە', 'ە'), + ('ۥ', 'ۦ'), + ('ۮ', 'ۯ'), + ('ۺ', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', 'ܐ'), + ('ܒ', 'ܯ'), + ('ݍ', 'ޥ'), + ('ޱ', 'ޱ'), + ('ߊ', 'ߪ'), + ('ߴ', 'ߵ'), + ('ߺ', 'ߺ'), + ('ࠀ', 'ࠕ'), + ('ࠚ', 'ࠚ'), + ('ࠤ', 'ࠤ'), + ('ࠨ', 'ࠨ'), + ('ࡀ', 'ࡘ'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('ࢠ', 'ࣉ'), + ('ऄ', 'ह'), + ('ऽ', 'ऽ'), + ('ॐ', 'ॐ'), + ('क़', 'ॡ'), + ('ॱ', 'ঀ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('ঽ', 'ঽ'), + ('ৎ', 'ৎ'), + ('ড়', 'ঢ়'), + ('য়', 'ৡ'), + ('ৰ', 'ৱ'), + ('ৼ', 'ৼ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('ੲ', 'ੴ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('ઽ', 'ઽ'), + ('ૐ', 'ૐ'), + ('ૠ', 'ૡ'), + ('ૹ', 'ૹ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('ଽ', 'ଽ'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୡ'), + ('ୱ', 'ୱ'), + ('ஃ', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('ௐ', 'ௐ'), + ('అ', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('ఽ', 'ఽ'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', 'ౡ'), + ('ಀ', 'ಀ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಽ', 'ಽ'), + ('ೝ', 'ೞ'), + ('ೠ', 'ೡ'), + ('ೱ', 'ೲ'), + ('ഄ', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ഺ'), + ('ഽ', 'ഽ'), + ('ൎ', 'ൎ'), + ('ൔ', 'ൖ'), + ('ൟ', 'ൡ'), + ('ൺ', 'ൿ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('ก', 'ะ'), + ('า', 'ำ'), + ('เ', 'ๆ'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ະ'), + ('າ', 'ຳ'), + ('ຽ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('ཀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('ྈ', 'ྌ'), + ('က', 'ဪ'), + ('ဿ', 'ဿ'), + ('ၐ', 'ၕ'), + ('ၚ', 'ၝ'), + ('ၡ', 'ၡ'), + ('ၥ', 'ၦ'), + ('ၮ', 'ၰ'), + ('ၵ', 'ႁ'), + ('ႎ', 'ႎ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', 'ᜑ'), + ('ᜟ', 'ᜱ'), + ('ᝀ', 'ᝑ'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('ក', 'ឳ'), + ('ៗ', 'ៗ'), + ('ៜ', 'ៜ'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢨ'), + ('ᢪ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('ᥐ', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('ᨀ', 'ᨖ'), + ('ᨠ', 'ᩔ'), + ('ᪧ', 'ᪧ'), + ('ᬅ', 'ᬳ'), + ('ᭅ', 'ᭌ'), + ('ᮃ', 'ᮠ'), + ('ᮮ', 'ᮯ'), + ('ᮺ', 'ᯥ'), + ('ᰀ', 'ᰣ'), + ('ᱍ', 'ᱏ'), + ('ᱚ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', 'ᳶ'), + ('ᳺ', 'ᳺ'), + ('ᴀ', 'ᶿ'), + ('Ḁ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('℘', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('々', '〇'), + ('〡', '〩'), + ('〱', '〵'), + ('〸', '〼'), + ('ぁ', 'ゖ'), + ('゛', 'ゟ'), + ('ァ', 'ヺ'), + ('ー', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘟ'), + ('ꘪ', 'ꘫ'), + ('Ꙁ', 'ꙮ'), + ('ꙿ', 'ꚝ'), + ('ꚠ', 'ꛯ'), + ('ꜗ', 'ꜟ'), + ('Ꜣ', 'ꞈ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠁ'), + ('ꠃ', 'ꠅ'), + ('ꠇ', 'ꠊ'), + ('ꠌ', 'ꠢ'), + ('ꡀ', 'ꡳ'), + ('ꢂ', 'ꢳ'), + ('ꣲ', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', 'ꣾ'), + ('ꤊ', 'ꤥ'), + ('ꤰ', 'ꥆ'), + ('ꥠ', 'ꥼ'), + ('ꦄ', 'ꦲ'), + ('ꧏ', 'ꧏ'), + ('ꧠ', 'ꧤ'), + ('ꧦ', 'ꧯ'), + ('ꧺ', 'ꧾ'), + ('ꨀ', 'ꨨ'), + ('ꩀ', 'ꩂ'), + ('ꩄ', 'ꩋ'), + ('ꩠ', 'ꩶ'), + ('ꩺ', 'ꩺ'), + ('ꩾ', 'ꪯ'), + ('ꪱ', 'ꪱ'), + ('ꪵ', 'ꪶ'), + ('ꪹ', 'ꪽ'), + ('ꫀ', 'ꫀ'), + ('ꫂ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫪ'), + ('ꫲ', 'ꫴ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꯢ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'יִ'), + ('ײַ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('ｦ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '𐍵'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐐀', '𐒝'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '𐨀'), + ('𐨐', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '𐫤'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '𐴣'), + ('𐵊', '𐵥'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('𐼀', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '𐽅'), + ('𐽰', '𐾁'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀃', '𑀷'), + ('𑁱', '𑁲'), + ('𑁵', '𑁵'), + ('𑂃', '𑂯'), + ('𑃐', '𑃨'), + ('𑄃', '𑄦'), + ('𑅄', '𑅄'), + ('𑅇', '𑅇'), + ('𑅐', '𑅲'), + ('𑅶', '𑅶'), + ('𑆃', '𑆲'), + ('𑇁', '𑇄'), + ('𑇚', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '𑈫'), + ('𑈿', '𑉀'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '𑋞'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('𑌽', '𑌽'), + ('𑍐', '𑍐'), + ('𑍝', '𑍡'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '𑎷'), + ('𑏑', '𑏑'), + ('𑏓', '𑏓'), + ('𑐀', '𑐴'), + ('𑑇', '𑑊'), + ('𑑟', '𑑡'), + ('𑒀', '𑒯'), + ('𑓄', '𑓅'), + ('𑓇', '𑓇'), + ('𑖀', '𑖮'), + ('𑗘', '𑗛'), + ('𑘀', '𑘯'), + ('𑙄', '𑙄'), + ('𑚀', '𑚪'), + ('𑚸', '𑚸'), + ('𑜀', '𑜚'), + ('𑝀', '𑝆'), + ('𑠀', '𑠫'), + ('𑢠', '𑣟'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤯'), + ('𑤿', '𑤿'), + ('𑥁', '𑥁'), + ('𑦠', '𑦧'), + ('𑦪', '𑧐'), + ('𑧡', '𑧡'), + ('𑧣', '𑧣'), + ('𑨀', '𑨀'), + ('𑨋', '𑨲'), + ('𑨺', '𑨺'), + ('𑩐', '𑩐'), + ('𑩜', '𑪉'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑰀', '𑰈'), + ('𑰊', '𑰮'), + ('𑱀', '𑱀'), + ('𑱲', '𑲏'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '𑴰'), + ('𑵆', '𑵆'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶉'), + ('𑶘', '𑶘'), + ('𑻠', '𑻲'), + ('𑼂', '𑼂'), + ('𑼄', '𑼐'), + ('𑼒', '𑼳'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('𓑁', '𓑆'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄝'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩰', '𖪾'), + ('𖫐', '𖫭'), + ('𖬀', '𖬯'), + ('𖭀', '𖭃'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('𖽐', '𖽐'), + ('𖾓', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '𖿣'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('𞄀', '𞄬'), + ('𞄷', '𞄽'), + ('𞅎', '𞅎'), + ('𞊐', '𞊭'), + ('𞋀', '𞋫'), + ('𞓐', '𞓫'), + ('𞗐', '𞗭'), + ('𞗰', '𞗰'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('𞤀', '𞥃'), + ('𞥋', '𞥋'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const IDEOGRAPHIC: &'static [(char, char)] = &[ + ('〆', '〇'), + ('〡', '〩'), + ('〸', '〺'), + ('㐀', '䶿'), + ('一', '鿿'), + ('豈', '舘'), + ('並', '龎'), + ('\u{16fe4}', '\u{16fe4}'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𛅰', '𛋻'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const INCB: &'static [(char, char)] = &[ + ('\u{300}', '\u{36f}'), + ('\u{483}', '\u{489}'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('\u{610}', '\u{61a}'), + ('\u{64b}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dc}'), + ('\u{6df}', '\u{6e4}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{6ea}', '\u{6ed}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', '\u{7f3}'), + ('\u{7fd}', '\u{7fd}'), + ('\u{816}', '\u{819}'), + ('\u{81b}', '\u{823}'), + ('\u{825}', '\u{827}'), + ('\u{829}', '\u{82d}'), + ('\u{859}', '\u{85b}'), + ('\u{897}', '\u{89f}'), + ('\u{8ca}', '\u{8e1}'), + ('\u{8e3}', '\u{902}'), + ('क', '\u{93a}'), + ('\u{93c}', '\u{93c}'), + ('\u{941}', '\u{948}'), + ('\u{94d}', '\u{94d}'), + ('\u{951}', 'य़'), + ('\u{962}', '\u{963}'), + ('ॸ', 'ॿ'), + ('\u{981}', '\u{981}'), + ('ক', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9be}', '\u{9be}'), + ('\u{9c1}', '\u{9c4}'), + ('\u{9cd}', '\u{9cd}'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', 'য়'), + ('\u{9e2}', '\u{9e3}'), + ('ৰ', 'ৱ'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', '\u{a02}'), + ('\u{a3c}', '\u{a3c}'), + ('\u{a41}', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', '\u{a82}'), + ('ક', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('\u{abc}', '\u{abc}'), + ('\u{ac1}', '\u{ac5}'), + ('\u{ac7}', '\u{ac8}'), + ('\u{acd}', '\u{acd}'), + ('\u{ae2}', '\u{ae3}'), + ('ૹ', '\u{aff}'), + ('\u{b01}', '\u{b01}'), + ('କ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b3e}', '\u{b3f}'), + ('\u{b41}', '\u{b44}'), + ('\u{b4d}', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୟ'), + ('\u{b62}', '\u{b63}'), + ('ୱ', 'ୱ'), + ('\u{b82}', '\u{b82}'), + ('\u{bbe}', '\u{bbe}'), + ('\u{bc0}', '\u{bc0}'), + ('\u{bcd}', '\u{bcd}'), + ('\u{bd7}', '\u{bd7}'), + ('\u{c00}', '\u{c00}'), + ('\u{c04}', '\u{c04}'), + ('క', 'న'), + ('ప', 'హ'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c3e}', '\u{c40}'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('ౘ', 'ౚ'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', '\u{c81}'), + ('\u{cbc}', '\u{cbc}'), + ('\u{cbf}', '\u{cc0}'), + ('\u{cc2}', '\u{cc2}'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('\u{ce2}', '\u{ce3}'), + ('\u{d00}', '\u{d01}'), + ('ക', '\u{d3c}'), + ('\u{d3e}', '\u{d3e}'), + ('\u{d41}', '\u{d44}'), + ('\u{d4d}', '\u{d4d}'), + ('\u{d57}', '\u{d57}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', '\u{d81}'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dcf}'), + ('\u{dd2}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('\u{ddf}', '\u{ddf}'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('\u{e47}', '\u{e4e}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{ebc}'), + ('\u{ec8}', '\u{ece}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('\u{f71}', '\u{f7e}'), + ('\u{f80}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('\u{102d}', '\u{1030}'), + ('\u{1032}', '\u{1037}'), + ('\u{1039}', '\u{103a}'), + ('\u{103d}', '\u{103e}'), + ('\u{1058}', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{1082}'), + ('\u{1085}', '\u{1086}'), + ('\u{108d}', '\u{108d}'), + ('\u{109d}', '\u{109d}'), + ('\u{135d}', '\u{135f}'), + ('\u{1712}', '\u{1715}'), + ('\u{1732}', '\u{1734}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('\u{17b4}', '\u{17b5}'), + ('\u{17b7}', '\u{17bd}'), + ('\u{17c6}', '\u{17c6}'), + ('\u{17c9}', '\u{17d3}'), + ('\u{17dd}', '\u{17dd}'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '\u{180f}'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', '\u{1922}'), + ('\u{1927}', '\u{1928}'), + ('\u{1932}', '\u{1932}'), + ('\u{1939}', '\u{193b}'), + ('\u{1a17}', '\u{1a18}'), + ('\u{1a1b}', '\u{1a1b}'), + ('\u{1a56}', '\u{1a56}'), + ('\u{1a58}', '\u{1a5e}'), + ('\u{1a60}', '\u{1a60}'), + ('\u{1a62}', '\u{1a62}'), + ('\u{1a65}', '\u{1a6c}'), + ('\u{1a73}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', '\u{1b03}'), + ('\u{1b34}', '\u{1b3d}'), + ('\u{1b42}', '\u{1b44}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1b81}'), + ('\u{1ba2}', '\u{1ba5}'), + ('\u{1ba8}', '\u{1bad}'), + ('\u{1be6}', '\u{1be6}'), + ('\u{1be8}', '\u{1be9}'), + ('\u{1bed}', '\u{1bed}'), + ('\u{1bef}', '\u{1bf3}'), + ('\u{1c2c}', '\u{1c33}'), + ('\u{1c36}', '\u{1c37}'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce0}'), + ('\u{1ce2}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('\u{1cf8}', '\u{1cf9}'), + ('\u{1dc0}', '\u{1dff}'), + ('\u{200d}', '\u{200d}'), + ('\u{20d0}', '\u{20f0}'), + ('\u{2cef}', '\u{2cf1}'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{2de0}', '\u{2dff}'), + ('\u{302a}', '\u{302f}'), + ('\u{3099}', '\u{309a}'), + ('\u{a66f}', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('\u{a69e}', '\u{a69f}'), + ('\u{a6f0}', '\u{a6f1}'), + ('\u{a802}', '\u{a802}'), + ('\u{a806}', '\u{a806}'), + ('\u{a80b}', '\u{a80b}'), + ('\u{a825}', '\u{a826}'), + ('\u{a82c}', '\u{a82c}'), + ('\u{a8c4}', '\u{a8c5}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92d}'), + ('\u{a947}', '\u{a951}'), + ('\u{a953}', '\u{a953}'), + ('\u{a980}', '\u{a982}'), + ('\u{a9b3}', '\u{a9b3}'), + ('\u{a9b6}', '\u{a9b9}'), + ('\u{a9bc}', '\u{a9bd}'), + ('\u{a9c0}', '\u{a9c0}'), + ('\u{a9e5}', '\u{a9e5}'), + ('\u{aa29}', '\u{aa2e}'), + ('\u{aa31}', '\u{aa32}'), + ('\u{aa35}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', '\u{aa4c}'), + ('\u{aa7c}', '\u{aa7c}'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabf}'), + ('\u{aac1}', '\u{aac1}'), + ('\u{aaec}', '\u{aaed}'), + ('\u{aaf6}', '\u{aaf6}'), + ('\u{abe5}', '\u{abe5}'), + ('\u{abe8}', '\u{abe8}'), + ('\u{abed}', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('\u{ff9e}', '\u{ff9f}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{10376}', '\u{1037a}'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('\u{10d24}', '\u{10d27}'), + ('\u{10d69}', '\u{10d6d}'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('\u{11001}', '\u{11001}'), + ('\u{11038}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{11073}', '\u{11074}'), + ('\u{1107f}', '\u{11081}'), + ('\u{110b3}', '\u{110b6}'), + ('\u{110b9}', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{1112b}'), + ('\u{1112d}', '\u{11134}'), + ('\u{11173}', '\u{11173}'), + ('\u{11180}', '\u{11181}'), + ('\u{111b6}', '\u{111be}'), + ('\u{111c0}', '\u{111c0}'), + ('\u{111c9}', '\u{111cc}'), + ('\u{111cf}', '\u{111cf}'), + ('\u{1122f}', '\u{11231}'), + ('\u{11234}', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112df}'), + ('\u{112e3}', '\u{112ea}'), + ('\u{11300}', '\u{11301}'), + ('\u{1133b}', '\u{1133c}'), + ('\u{1133e}', '\u{1133e}'), + ('\u{11340}', '\u{11340}'), + ('\u{1134d}', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113b8}', '\u{113b8}'), + ('\u{113bb}', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '\u{113c9}'), + ('\u{113ce}', '\u{113d0}'), + ('\u{113d2}', '\u{113d2}'), + ('\u{113e1}', '\u{113e2}'), + ('\u{11438}', '\u{1143f}'), + ('\u{11442}', '\u{11444}'), + ('\u{11446}', '\u{11446}'), + ('\u{1145e}', '\u{1145e}'), + ('\u{114b0}', '\u{114b0}'), + ('\u{114b3}', '\u{114b8}'), + ('\u{114ba}', '\u{114ba}'), + ('\u{114bd}', '\u{114bd}'), + ('\u{114bf}', '\u{114c0}'), + ('\u{114c2}', '\u{114c3}'), + ('\u{115af}', '\u{115af}'), + ('\u{115b2}', '\u{115b5}'), + ('\u{115bc}', '\u{115bd}'), + ('\u{115bf}', '\u{115c0}'), + ('\u{115dc}', '\u{115dd}'), + ('\u{11633}', '\u{1163a}'), + ('\u{1163d}', '\u{1163d}'), + ('\u{1163f}', '\u{11640}'), + ('\u{116ab}', '\u{116ab}'), + ('\u{116ad}', '\u{116ad}'), + ('\u{116b0}', '\u{116b7}'), + ('\u{1171d}', '\u{1171d}'), + ('\u{1171f}', '\u{1171f}'), + ('\u{11722}', '\u{11725}'), + ('\u{11727}', '\u{1172b}'), + ('\u{1182f}', '\u{11837}'), + ('\u{11839}', '\u{1183a}'), + ('\u{11930}', '\u{11930}'), + ('\u{1193b}', '\u{1193e}'), + ('\u{11943}', '\u{11943}'), + ('\u{119d4}', '\u{119d7}'), + ('\u{119da}', '\u{119db}'), + ('\u{119e0}', '\u{119e0}'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a33}', '\u{11a38}'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a51}', '\u{11a56}'), + ('\u{11a59}', '\u{11a5b}'), + ('\u{11a8a}', '\u{11a96}'), + ('\u{11a98}', '\u{11a99}'), + ('\u{11c30}', '\u{11c36}'), + ('\u{11c38}', '\u{11c3d}'), + ('\u{11c3f}', '\u{11c3f}'), + ('\u{11c92}', '\u{11ca7}'), + ('\u{11caa}', '\u{11cb0}'), + ('\u{11cb2}', '\u{11cb3}'), + ('\u{11cb5}', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d45}'), + ('\u{11d47}', '\u{11d47}'), + ('\u{11d90}', '\u{11d91}'), + ('\u{11d95}', '\u{11d95}'), + ('\u{11d97}', '\u{11d97}'), + ('\u{11ef3}', '\u{11ef4}'), + ('\u{11f00}', '\u{11f01}'), + ('\u{11f36}', '\u{11f3a}'), + ('\u{11f40}', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13440}', '\u{13440}'), + ('\u{13447}', '\u{13455}'), + ('\u{1611e}', '\u{16129}'), + ('\u{1612d}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('\u{16f4f}', '\u{16f4f}'), + ('\u{16f8f}', '\u{16f92}'), + ('\u{16fe4}', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e130}', '\u{1e136}'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('\u{1e4ec}', '\u{1e4ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '\u{1e94a}'), + ('🏻', '🏿'), + ('\u{e0020}', '\u{e007f}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const JOIN_CONTROL: &'static [(char, char)] = &[('\u{200c}', '\u{200d}')]; + +pub const LOGICAL_ORDER_EXCEPTION: &'static [(char, char)] = &[ + ('เ', 'ไ'), + ('ເ', 'ໄ'), + ('ᦵ', 'ᦷ'), + ('ᦺ', 'ᦺ'), + ('ꪵ', 'ꪶ'), + ('ꪹ', 'ꪹ'), + ('ꪻ', 'ꪼ'), +]; + +pub const LOWERCASE: &'static [(char, char)] = &[ + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('ß', 'ö'), + ('ø', 'ÿ'), + ('ā', 'ā'), + ('ă', 'ă'), + ('ą', 'ą'), + ('ć', 'ć'), + ('ĉ', 'ĉ'), + ('ċ', 'ċ'), + ('č', 'č'), + ('ď', 'ď'), + ('đ', 'đ'), + ('ē', 'ē'), + ('ĕ', 'ĕ'), + ('ė', 'ė'), + ('ę', 'ę'), + ('ě', 'ě'), + ('ĝ', 'ĝ'), + ('ğ', 'ğ'), + ('ġ', 'ġ'), + ('ģ', 'ģ'), + ('ĥ', 'ĥ'), + ('ħ', 'ħ'), + ('ĩ', 'ĩ'), + ('ī', 'ī'), + ('ĭ', 'ĭ'), + ('į', 'į'), + ('ı', 'ı'), + ('ij', 'ij'), + ('ĵ', 'ĵ'), + ('ķ', 'ĸ'), + ('ĺ', 'ĺ'), + ('ļ', 'ļ'), + ('ľ', 'ľ'), + ('ŀ', 'ŀ'), + ('ł', 'ł'), + ('ń', 'ń'), + ('ņ', 'ņ'), + ('ň', 'ʼn'), + ('ŋ', 'ŋ'), + ('ō', 'ō'), + ('ŏ', 'ŏ'), + ('ő', 'ő'), + ('œ', 'œ'), + ('ŕ', 'ŕ'), + ('ŗ', 'ŗ'), + ('ř', 'ř'), + ('ś', 'ś'), + ('ŝ', 'ŝ'), + ('ş', 'ş'), + ('š', 'š'), + ('ţ', 'ţ'), + ('ť', 'ť'), + ('ŧ', 'ŧ'), + ('ũ', 'ũ'), + ('ū', 'ū'), + ('ŭ', 'ŭ'), + ('ů', 'ů'), + ('ű', 'ű'), + ('ų', 'ų'), + ('ŵ', 'ŵ'), + ('ŷ', 'ŷ'), + ('ź', 'ź'), + ('ż', 'ż'), + ('ž', 'ƀ'), + ('ƃ', 'ƃ'), + ('ƅ', 'ƅ'), + ('ƈ', 'ƈ'), + ('ƌ', 'ƍ'), + ('ƒ', 'ƒ'), + ('ƕ', 'ƕ'), + ('ƙ', 'ƛ'), + ('ƞ', 'ƞ'), + ('ơ', 'ơ'), + ('ƣ', 'ƣ'), + ('ƥ', 'ƥ'), + ('ƨ', 'ƨ'), + ('ƪ', 'ƫ'), + ('ƭ', 'ƭ'), + ('ư', 'ư'), + ('ƴ', 'ƴ'), + ('ƶ', 'ƶ'), + ('ƹ', 'ƺ'), + ('ƽ', 'ƿ'), + ('dž', 'dž'), + ('lj', 'lj'), + ('nj', 'nj'), + ('ǎ', 'ǎ'), + ('ǐ', 'ǐ'), + ('ǒ', 'ǒ'), + ('ǔ', 'ǔ'), + ('ǖ', 'ǖ'), + ('ǘ', 'ǘ'), + ('ǚ', 'ǚ'), + ('ǜ', 'ǝ'), + ('ǟ', 'ǟ'), + ('ǡ', 'ǡ'), + ('ǣ', 'ǣ'), + ('ǥ', 'ǥ'), + ('ǧ', 'ǧ'), + ('ǩ', 'ǩ'), + ('ǫ', 'ǫ'), + ('ǭ', 'ǭ'), + ('ǯ', 'ǰ'), + ('dz', 'dz'), + ('ǵ', 'ǵ'), + ('ǹ', 'ǹ'), + ('ǻ', 'ǻ'), + ('ǽ', 'ǽ'), + ('ǿ', 'ǿ'), + ('ȁ', 'ȁ'), + ('ȃ', 'ȃ'), + ('ȅ', 'ȅ'), + ('ȇ', 'ȇ'), + ('ȉ', 'ȉ'), + ('ȋ', 'ȋ'), + ('ȍ', 'ȍ'), + ('ȏ', 'ȏ'), + ('ȑ', 'ȑ'), + ('ȓ', 'ȓ'), + ('ȕ', 'ȕ'), + ('ȗ', 'ȗ'), + ('ș', 'ș'), + ('ț', 'ț'), + ('ȝ', 'ȝ'), + ('ȟ', 'ȟ'), + ('ȡ', 'ȡ'), + ('ȣ', 'ȣ'), + ('ȥ', 'ȥ'), + ('ȧ', 'ȧ'), + ('ȩ', 'ȩ'), + ('ȫ', 'ȫ'), + ('ȭ', 'ȭ'), + ('ȯ', 'ȯ'), + ('ȱ', 'ȱ'), + ('ȳ', 'ȹ'), + ('ȼ', 'ȼ'), + ('ȿ', 'ɀ'), + ('ɂ', 'ɂ'), + ('ɇ', 'ɇ'), + ('ɉ', 'ɉ'), + ('ɋ', 'ɋ'), + ('ɍ', 'ɍ'), + ('ɏ', 'ʓ'), + ('ʕ', 'ʸ'), + ('ˀ', 'ˁ'), + ('ˠ', 'ˤ'), + ('\u{345}', '\u{345}'), + ('ͱ', 'ͱ'), + ('ͳ', 'ͳ'), + ('ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('ΐ', 'ΐ'), + ('ά', 'ώ'), + ('ϐ', 'ϑ'), + ('ϕ', 'ϗ'), + ('ϙ', 'ϙ'), + ('ϛ', 'ϛ'), + ('ϝ', 'ϝ'), + ('ϟ', 'ϟ'), + ('ϡ', 'ϡ'), + ('ϣ', 'ϣ'), + ('ϥ', 'ϥ'), + ('ϧ', 'ϧ'), + ('ϩ', 'ϩ'), + ('ϫ', 'ϫ'), + ('ϭ', 'ϭ'), + ('ϯ', 'ϳ'), + ('ϵ', 'ϵ'), + ('ϸ', 'ϸ'), + ('ϻ', 'ϼ'), + ('а', 'џ'), + ('ѡ', 'ѡ'), + ('ѣ', 'ѣ'), + ('ѥ', 'ѥ'), + ('ѧ', 'ѧ'), + ('ѩ', 'ѩ'), + ('ѫ', 'ѫ'), + ('ѭ', 'ѭ'), + ('ѯ', 'ѯ'), + ('ѱ', 'ѱ'), + ('ѳ', 'ѳ'), + ('ѵ', 'ѵ'), + ('ѷ', 'ѷ'), + ('ѹ', 'ѹ'), + ('ѻ', 'ѻ'), + ('ѽ', 'ѽ'), + ('ѿ', 'ѿ'), + ('ҁ', 'ҁ'), + ('ҋ', 'ҋ'), + ('ҍ', 'ҍ'), + ('ҏ', 'ҏ'), + ('ґ', 'ґ'), + ('ғ', 'ғ'), + ('ҕ', 'ҕ'), + ('җ', 'җ'), + ('ҙ', 'ҙ'), + ('қ', 'қ'), + ('ҝ', 'ҝ'), + ('ҟ', 'ҟ'), + ('ҡ', 'ҡ'), + ('ң', 'ң'), + ('ҥ', 'ҥ'), + ('ҧ', 'ҧ'), + ('ҩ', 'ҩ'), + ('ҫ', 'ҫ'), + ('ҭ', 'ҭ'), + ('ү', 'ү'), + ('ұ', 'ұ'), + ('ҳ', 'ҳ'), + ('ҵ', 'ҵ'), + ('ҷ', 'ҷ'), + ('ҹ', 'ҹ'), + ('һ', 'һ'), + ('ҽ', 'ҽ'), + ('ҿ', 'ҿ'), + ('ӂ', 'ӂ'), + ('ӄ', 'ӄ'), + ('ӆ', 'ӆ'), + ('ӈ', 'ӈ'), + ('ӊ', 'ӊ'), + ('ӌ', 'ӌ'), + ('ӎ', 'ӏ'), + ('ӑ', 'ӑ'), + ('ӓ', 'ӓ'), + ('ӕ', 'ӕ'), + ('ӗ', 'ӗ'), + ('ә', 'ә'), + ('ӛ', 'ӛ'), + ('ӝ', 'ӝ'), + ('ӟ', 'ӟ'), + ('ӡ', 'ӡ'), + ('ӣ', 'ӣ'), + ('ӥ', 'ӥ'), + ('ӧ', 'ӧ'), + ('ө', 'ө'), + ('ӫ', 'ӫ'), + ('ӭ', 'ӭ'), + ('ӯ', 'ӯ'), + ('ӱ', 'ӱ'), + ('ӳ', 'ӳ'), + ('ӵ', 'ӵ'), + ('ӷ', 'ӷ'), + ('ӹ', 'ӹ'), + ('ӻ', 'ӻ'), + ('ӽ', 'ӽ'), + ('ӿ', 'ӿ'), + ('ԁ', 'ԁ'), + ('ԃ', 'ԃ'), + ('ԅ', 'ԅ'), + ('ԇ', 'ԇ'), + ('ԉ', 'ԉ'), + ('ԋ', 'ԋ'), + ('ԍ', 'ԍ'), + ('ԏ', 'ԏ'), + ('ԑ', 'ԑ'), + ('ԓ', 'ԓ'), + ('ԕ', 'ԕ'), + ('ԗ', 'ԗ'), + ('ԙ', 'ԙ'), + ('ԛ', 'ԛ'), + ('ԝ', 'ԝ'), + ('ԟ', 'ԟ'), + ('ԡ', 'ԡ'), + ('ԣ', 'ԣ'), + ('ԥ', 'ԥ'), + ('ԧ', 'ԧ'), + ('ԩ', 'ԩ'), + ('ԫ', 'ԫ'), + ('ԭ', 'ԭ'), + ('ԯ', 'ԯ'), + ('ՠ', 'ֈ'), + ('ა', 'ჺ'), + ('ჼ', 'ჿ'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'ᲈ'), + ('ᲊ', 'ᲊ'), + ('ᴀ', 'ᶿ'), + ('ḁ', 'ḁ'), + ('ḃ', 'ḃ'), + ('ḅ', 'ḅ'), + ('ḇ', 'ḇ'), + ('ḉ', 'ḉ'), + ('ḋ', 'ḋ'), + ('ḍ', 'ḍ'), + ('ḏ', 'ḏ'), + ('ḑ', 'ḑ'), + ('ḓ', 'ḓ'), + ('ḕ', 'ḕ'), + ('ḗ', 'ḗ'), + ('ḙ', 'ḙ'), + ('ḛ', 'ḛ'), + ('ḝ', 'ḝ'), + ('ḟ', 'ḟ'), + ('ḡ', 'ḡ'), + ('ḣ', 'ḣ'), + ('ḥ', 'ḥ'), + ('ḧ', 'ḧ'), + ('ḩ', 'ḩ'), + ('ḫ', 'ḫ'), + ('ḭ', 'ḭ'), + ('ḯ', 'ḯ'), + ('ḱ', 'ḱ'), + ('ḳ', 'ḳ'), + ('ḵ', 'ḵ'), + ('ḷ', 'ḷ'), + ('ḹ', 'ḹ'), + ('ḻ', 'ḻ'), + ('ḽ', 'ḽ'), + ('ḿ', 'ḿ'), + ('ṁ', 'ṁ'), + ('ṃ', 'ṃ'), + ('ṅ', 'ṅ'), + ('ṇ', 'ṇ'), + ('ṉ', 'ṉ'), + ('ṋ', 'ṋ'), + ('ṍ', 'ṍ'), + ('ṏ', 'ṏ'), + ('ṑ', 'ṑ'), + ('ṓ', 'ṓ'), + ('ṕ', 'ṕ'), + ('ṗ', 'ṗ'), + ('ṙ', 'ṙ'), + ('ṛ', 'ṛ'), + ('ṝ', 'ṝ'), + ('ṟ', 'ṟ'), + ('ṡ', 'ṡ'), + ('ṣ', 'ṣ'), + ('ṥ', 'ṥ'), + ('ṧ', 'ṧ'), + ('ṩ', 'ṩ'), + ('ṫ', 'ṫ'), + ('ṭ', 'ṭ'), + ('ṯ', 'ṯ'), + ('ṱ', 'ṱ'), + ('ṳ', 'ṳ'), + ('ṵ', 'ṵ'), + ('ṷ', 'ṷ'), + ('ṹ', 'ṹ'), + ('ṻ', 'ṻ'), + ('ṽ', 'ṽ'), + ('ṿ', 'ṿ'), + ('ẁ', 'ẁ'), + ('ẃ', 'ẃ'), + ('ẅ', 'ẅ'), + ('ẇ', 'ẇ'), + ('ẉ', 'ẉ'), + ('ẋ', 'ẋ'), + ('ẍ', 'ẍ'), + ('ẏ', 'ẏ'), + ('ẑ', 'ẑ'), + ('ẓ', 'ẓ'), + ('ẕ', 'ẝ'), + ('ẟ', 'ẟ'), + ('ạ', 'ạ'), + ('ả', 'ả'), + ('ấ', 'ấ'), + ('ầ', 'ầ'), + ('ẩ', 'ẩ'), + ('ẫ', 'ẫ'), + ('ậ', 'ậ'), + ('ắ', 'ắ'), + ('ằ', 'ằ'), + ('ẳ', 'ẳ'), + ('ẵ', 'ẵ'), + ('ặ', 'ặ'), + ('ẹ', 'ẹ'), + ('ẻ', 'ẻ'), + ('ẽ', 'ẽ'), + ('ế', 'ế'), + ('ề', 'ề'), + ('ể', 'ể'), + ('ễ', 'ễ'), + ('ệ', 'ệ'), + ('ỉ', 'ỉ'), + ('ị', 'ị'), + ('ọ', 'ọ'), + ('ỏ', 'ỏ'), + ('ố', 'ố'), + ('ồ', 'ồ'), + ('ổ', 'ổ'), + ('ỗ', 'ỗ'), + ('ộ', 'ộ'), + ('ớ', 'ớ'), + ('ờ', 'ờ'), + ('ở', 'ở'), + ('ỡ', 'ỡ'), + ('ợ', 'ợ'), + ('ụ', 'ụ'), + ('ủ', 'ủ'), + ('ứ', 'ứ'), + ('ừ', 'ừ'), + ('ử', 'ử'), + ('ữ', 'ữ'), + ('ự', 'ự'), + ('ỳ', 'ỳ'), + ('ỵ', 'ỵ'), + ('ỷ', 'ỷ'), + ('ỹ', 'ỹ'), + ('ỻ', 'ỻ'), + ('ỽ', 'ỽ'), + ('ỿ', 'ἇ'), + ('ἐ', 'ἕ'), + ('ἠ', 'ἧ'), + ('ἰ', 'ἷ'), + ('ὀ', 'ὅ'), + ('ὐ', 'ὗ'), + ('ὠ', 'ὧ'), + ('ὰ', 'ώ'), + ('ᾀ', 'ᾇ'), + ('ᾐ', 'ᾗ'), + ('ᾠ', 'ᾧ'), + ('ᾰ', 'ᾴ'), + ('ᾶ', 'ᾷ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῇ'), + ('ῐ', 'ΐ'), + ('ῖ', 'ῗ'), + ('ῠ', 'ῧ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῷ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ℊ', 'ℊ'), + ('ℎ', 'ℏ'), + ('ℓ', 'ℓ'), + ('ℯ', 'ℯ'), + ('ℴ', 'ℴ'), + ('ℹ', 'ℹ'), + ('ℼ', 'ℽ'), + ('ⅆ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('ⅰ', 'ⅿ'), + ('ↄ', 'ↄ'), + ('ⓐ', 'ⓩ'), + ('ⰰ', 'ⱟ'), + ('ⱡ', 'ⱡ'), + ('ⱥ', 'ⱦ'), + ('ⱨ', 'ⱨ'), + ('ⱪ', 'ⱪ'), + ('ⱬ', 'ⱬ'), + ('ⱱ', 'ⱱ'), + ('ⱳ', 'ⱴ'), + ('ⱶ', 'ⱽ'), + ('ⲁ', 'ⲁ'), + ('ⲃ', 'ⲃ'), + ('ⲅ', 'ⲅ'), + ('ⲇ', 'ⲇ'), + ('ⲉ', 'ⲉ'), + ('ⲋ', 'ⲋ'), + ('ⲍ', 'ⲍ'), + ('ⲏ', 'ⲏ'), + ('ⲑ', 'ⲑ'), + ('ⲓ', 'ⲓ'), + ('ⲕ', 'ⲕ'), + ('ⲗ', 'ⲗ'), + ('ⲙ', 'ⲙ'), + ('ⲛ', 'ⲛ'), + ('ⲝ', 'ⲝ'), + ('ⲟ', 'ⲟ'), + ('ⲡ', 'ⲡ'), + ('ⲣ', 'ⲣ'), + ('ⲥ', 'ⲥ'), + ('ⲧ', 'ⲧ'), + ('ⲩ', 'ⲩ'), + ('ⲫ', 'ⲫ'), + ('ⲭ', 'ⲭ'), + ('ⲯ', 'ⲯ'), + ('ⲱ', 'ⲱ'), + ('ⲳ', 'ⲳ'), + ('ⲵ', 'ⲵ'), + ('ⲷ', 'ⲷ'), + ('ⲹ', 'ⲹ'), + ('ⲻ', 'ⲻ'), + ('ⲽ', 'ⲽ'), + ('ⲿ', 'ⲿ'), + ('ⳁ', 'ⳁ'), + ('ⳃ', 'ⳃ'), + ('ⳅ', 'ⳅ'), + ('ⳇ', 'ⳇ'), + ('ⳉ', 'ⳉ'), + ('ⳋ', 'ⳋ'), + ('ⳍ', 'ⳍ'), + ('ⳏ', 'ⳏ'), + ('ⳑ', 'ⳑ'), + ('ⳓ', 'ⳓ'), + ('ⳕ', 'ⳕ'), + ('ⳗ', 'ⳗ'), + ('ⳙ', 'ⳙ'), + ('ⳛ', 'ⳛ'), + ('ⳝ', 'ⳝ'), + ('ⳟ', 'ⳟ'), + ('ⳡ', 'ⳡ'), + ('ⳣ', 'ⳤ'), + ('ⳬ', 'ⳬ'), + ('ⳮ', 'ⳮ'), + ('ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ꙁ', 'ꙁ'), + ('ꙃ', 'ꙃ'), + ('ꙅ', 'ꙅ'), + ('ꙇ', 'ꙇ'), + ('ꙉ', 'ꙉ'), + ('ꙋ', 'ꙋ'), + ('ꙍ', 'ꙍ'), + ('ꙏ', 'ꙏ'), + ('ꙑ', 'ꙑ'), + ('ꙓ', 'ꙓ'), + ('ꙕ', 'ꙕ'), + ('ꙗ', 'ꙗ'), + ('ꙙ', 'ꙙ'), + ('ꙛ', 'ꙛ'), + ('ꙝ', 'ꙝ'), + ('ꙟ', 'ꙟ'), + ('ꙡ', 'ꙡ'), + ('ꙣ', 'ꙣ'), + ('ꙥ', 'ꙥ'), + ('ꙧ', 'ꙧ'), + ('ꙩ', 'ꙩ'), + ('ꙫ', 'ꙫ'), + ('ꙭ', 'ꙭ'), + ('ꚁ', 'ꚁ'), + ('ꚃ', 'ꚃ'), + ('ꚅ', 'ꚅ'), + ('ꚇ', 'ꚇ'), + ('ꚉ', 'ꚉ'), + ('ꚋ', 'ꚋ'), + ('ꚍ', 'ꚍ'), + ('ꚏ', 'ꚏ'), + ('ꚑ', 'ꚑ'), + ('ꚓ', 'ꚓ'), + ('ꚕ', 'ꚕ'), + ('ꚗ', 'ꚗ'), + ('ꚙ', 'ꚙ'), + ('ꚛ', 'ꚝ'), + ('ꜣ', 'ꜣ'), + ('ꜥ', 'ꜥ'), + ('ꜧ', 'ꜧ'), + ('ꜩ', 'ꜩ'), + ('ꜫ', 'ꜫ'), + ('ꜭ', 'ꜭ'), + ('ꜯ', 'ꜱ'), + ('ꜳ', 'ꜳ'), + ('ꜵ', 'ꜵ'), + ('ꜷ', 'ꜷ'), + ('ꜹ', 'ꜹ'), + ('ꜻ', 'ꜻ'), + ('ꜽ', 'ꜽ'), + ('ꜿ', 'ꜿ'), + ('ꝁ', 'ꝁ'), + ('ꝃ', 'ꝃ'), + ('ꝅ', 'ꝅ'), + ('ꝇ', 'ꝇ'), + ('ꝉ', 'ꝉ'), + ('ꝋ', 'ꝋ'), + ('ꝍ', 'ꝍ'), + ('ꝏ', 'ꝏ'), + ('ꝑ', 'ꝑ'), + ('ꝓ', 'ꝓ'), + ('ꝕ', 'ꝕ'), + ('ꝗ', 'ꝗ'), + ('ꝙ', 'ꝙ'), + ('ꝛ', 'ꝛ'), + ('ꝝ', 'ꝝ'), + ('ꝟ', 'ꝟ'), + ('ꝡ', 'ꝡ'), + ('ꝣ', 'ꝣ'), + ('ꝥ', 'ꝥ'), + ('ꝧ', 'ꝧ'), + ('ꝩ', 'ꝩ'), + ('ꝫ', 'ꝫ'), + ('ꝭ', 'ꝭ'), + ('ꝯ', 'ꝸ'), + ('ꝺ', 'ꝺ'), + ('ꝼ', 'ꝼ'), + ('ꝿ', 'ꝿ'), + ('ꞁ', 'ꞁ'), + ('ꞃ', 'ꞃ'), + ('ꞅ', 'ꞅ'), + ('ꞇ', 'ꞇ'), + ('ꞌ', 'ꞌ'), + ('ꞎ', 'ꞎ'), + ('ꞑ', 'ꞑ'), + ('ꞓ', 'ꞕ'), + ('ꞗ', 'ꞗ'), + ('ꞙ', 'ꞙ'), + ('ꞛ', 'ꞛ'), + ('ꞝ', 'ꞝ'), + ('ꞟ', 'ꞟ'), + ('ꞡ', 'ꞡ'), + ('ꞣ', 'ꞣ'), + ('ꞥ', 'ꞥ'), + ('ꞧ', 'ꞧ'), + ('ꞩ', 'ꞩ'), + ('ꞯ', 'ꞯ'), + ('ꞵ', 'ꞵ'), + ('ꞷ', 'ꞷ'), + ('ꞹ', 'ꞹ'), + ('ꞻ', 'ꞻ'), + ('ꞽ', 'ꞽ'), + ('ꞿ', 'ꞿ'), + ('ꟁ', 'ꟁ'), + ('ꟃ', 'ꟃ'), + ('ꟈ', 'ꟈ'), + ('ꟊ', 'ꟊ'), + ('ꟍ', 'ꟍ'), + ('ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'ꟕ'), + ('ꟗ', 'ꟗ'), + ('ꟙ', 'ꟙ'), + ('ꟛ', 'ꟛ'), + ('ꟲ', 'ꟴ'), + ('ꟶ', 'ꟶ'), + ('ꟸ', 'ꟺ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('ａ', 'ｚ'), + ('𐐨', '𐑏'), + ('𐓘', '𐓻'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐞀', '𐞀'), + ('𐞃', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐳀', '𐳲'), + ('𐵰', '𐶅'), + ('𑣀', '𑣟'), + ('𖹠', '𖹿'), + ('𝐚', '𝐳'), + ('𝑎', '𝑔'), + ('𝑖', '𝑧'), + ('𝒂', '𝒛'), + ('𝒶', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝓏'), + ('𝓪', '𝔃'), + ('𝔞', '𝔷'), + ('𝕒', '𝕫'), + ('𝖆', '𝖟'), + ('𝖺', '𝗓'), + ('𝗮', '𝘇'), + ('𝘢', '𝘻'), + ('𝙖', '𝙯'), + ('𝚊', '𝚥'), + ('𝛂', '𝛚'), + ('𝛜', '𝛡'), + ('𝛼', '𝜔'), + ('𝜖', '𝜛'), + ('𝜶', '𝝎'), + ('𝝐', '𝝕'), + ('𝝰', '𝞈'), + ('𝞊', '𝞏'), + ('𝞪', '𝟂'), + ('𝟄', '𝟉'), + ('𝟋', '𝟋'), + ('𝼀', '𝼉'), + ('𝼋', '𝼞'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('𞤢', '𞥃'), +]; + +pub const MATH: &'static [(char, char)] = &[ + ('+', '+'), + ('<', '>'), + ('^', '^'), + ('|', '|'), + ('~', '~'), + ('¬', '¬'), + ('±', '±'), + ('×', '×'), + ('÷', '÷'), + ('ϐ', 'ϒ'), + ('ϕ', 'ϕ'), + ('ϰ', 'ϱ'), + ('ϴ', '϶'), + ('؆', '؈'), + ('‖', '‖'), + ('′', '‴'), + ('⁀', '⁀'), + ('⁄', '⁄'), + ('⁒', '⁒'), + ('\u{2061}', '\u{2064}'), + ('⁺', '⁾'), + ('₊', '₎'), + ('\u{20d0}', '\u{20dc}'), + ('\u{20e1}', '\u{20e1}'), + ('\u{20e5}', '\u{20e6}'), + ('\u{20eb}', '\u{20ef}'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('℘', 'ℝ'), + ('ℤ', 'ℤ'), + ('ℨ', '℩'), + ('ℬ', 'ℭ'), + ('ℯ', 'ℱ'), + ('ℳ', 'ℸ'), + ('ℼ', 'ⅉ'), + ('⅋', '⅋'), + ('←', '↧'), + ('↩', '↮'), + ('↰', '↱'), + ('↶', '↷'), + ('↼', '⇛'), + ('⇝', '⇝'), + ('⇤', '⇥'), + ('⇴', '⋿'), + ('⌈', '⌋'), + ('⌠', '⌡'), + ('⍼', '⍼'), + ('⎛', '⎵'), + ('⎷', '⎷'), + ('⏐', '⏐'), + ('⏜', '⏢'), + ('■', '□'), + ('▮', '▷'), + ('▼', '◁'), + ('◆', '◇'), + ('◊', '○'), + ('●', '◓'), + ('◢', '◢'), + ('◤', '◤'), + ('◧', '◬'), + ('◸', '◿'), + ('★', '☆'), + ('♀', '♀'), + ('♂', '♂'), + ('♠', '♣'), + ('♭', '♯'), + ('⟀', '⟿'), + ('⤀', '⫿'), + ('⬰', '⭄'), + ('⭇', '⭌'), + ('﬩', '﬩'), + ('﹡', '﹦'), + ('﹨', '﹨'), + ('＋', '＋'), + ('＜', '＞'), + ('＼', '＼'), + ('＾', '＾'), + ('｜', '｜'), + ('～', '～'), + ('￢', '￢'), + ('￩', '￬'), + ('𐶎', '𐶏'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝟋'), + ('𝟎', '𝟿'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𞻰', '𞻱'), +]; + +pub const MODIFIER_COMBINING_MARK: &'static [(char, char)] = &[ + ('\u{654}', '\u{655}'), + ('\u{658}', '\u{658}'), + ('\u{6dc}', '\u{6dc}'), + ('\u{6e3}', '\u{6e3}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{8ca}', '\u{8cb}'), + ('\u{8cd}', '\u{8cf}'), + ('\u{8d3}', '\u{8d3}'), + ('\u{8f3}', '\u{8f3}'), +]; + +pub const NONCHARACTER_CODE_POINT: &'static [(char, char)] = &[ + ('\u{fdd0}', '\u{fdef}'), + ('\u{fffe}', '\u{ffff}'), + ('\u{1fffe}', '\u{1ffff}'), + ('\u{2fffe}', '\u{2ffff}'), + ('\u{3fffe}', '\u{3ffff}'), + ('\u{4fffe}', '\u{4ffff}'), + ('\u{5fffe}', '\u{5ffff}'), + ('\u{6fffe}', '\u{6ffff}'), + ('\u{7fffe}', '\u{7ffff}'), + ('\u{8fffe}', '\u{8ffff}'), + ('\u{9fffe}', '\u{9ffff}'), + ('\u{afffe}', '\u{affff}'), + ('\u{bfffe}', '\u{bffff}'), + ('\u{cfffe}', '\u{cffff}'), + ('\u{dfffe}', '\u{dffff}'), + ('\u{efffe}', '\u{effff}'), + ('\u{ffffe}', '\u{fffff}'), + ('\u{10fffe}', '\u{10ffff}'), +]; + +pub const OTHER_ALPHABETIC: &'static [(char, char)] = &[ + ('\u{345}', '\u{345}'), + ('\u{363}', '\u{36f}'), + ('\u{5b0}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('\u{610}', '\u{61a}'), + ('\u{64b}', '\u{657}'), + ('\u{659}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dc}'), + ('\u{6e1}', '\u{6e4}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{6ed}', '\u{6ed}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{73f}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{816}', '\u{817}'), + ('\u{81b}', '\u{823}'), + ('\u{825}', '\u{827}'), + ('\u{829}', '\u{82c}'), + ('\u{897}', '\u{897}'), + ('\u{8d4}', '\u{8df}'), + ('\u{8e3}', '\u{8e9}'), + ('\u{8f0}', 'ः'), + ('\u{93a}', 'ऻ'), + ('ा', 'ौ'), + ('ॎ', 'ॏ'), + ('\u{955}', '\u{957}'), + ('\u{962}', '\u{963}'), + ('\u{981}', 'ঃ'), + ('\u{9be}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', 'ৌ'), + ('\u{9d7}', '\u{9d7}'), + ('\u{9e2}', '\u{9e3}'), + ('\u{a01}', 'ਃ'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4c}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('ા', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', 'ૌ'), + ('\u{ae2}', '\u{ae3}'), + ('\u{afa}', '\u{afc}'), + ('\u{b01}', 'ଃ'), + ('\u{b3e}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', 'ୌ'), + ('\u{b56}', '\u{b57}'), + ('\u{b62}', '\u{b63}'), + ('\u{b82}', '\u{b82}'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', 'ௌ'), + ('\u{bd7}', '\u{bd7}'), + ('\u{c00}', '\u{c04}'), + ('\u{c3e}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4c}'), + ('\u{c55}', '\u{c56}'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', 'ಃ'), + ('ಾ', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccc}'), + ('\u{cd5}', '\u{cd6}'), + ('\u{ce2}', '\u{ce3}'), + ('ೳ', 'ೳ'), + ('\u{d00}', 'ഃ'), + ('\u{d3e}', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', 'ൌ'), + ('\u{d57}', '\u{d57}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', 'ඃ'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('ෲ', 'ෳ'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('\u{e4d}', '\u{e4d}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{eb9}'), + ('\u{ebb}', '\u{ebc}'), + ('\u{ecd}', '\u{ecd}'), + ('\u{f71}', '\u{f83}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('ါ', '\u{1036}'), + ('း', 'း'), + ('ျ', '\u{103e}'), + ('ၖ', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('ၢ', 'ၤ'), + ('ၧ', 'ၭ'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{108d}'), + ('ႏ', 'ႏ'), + ('ႚ', '\u{109d}'), + ('\u{1712}', '\u{1713}'), + ('\u{1732}', '\u{1733}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('ា', 'ៈ'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', 'ᤸ'), + ('\u{1a17}', '\u{1a1b}'), + ('ᩕ', '\u{1a5e}'), + ('ᩡ', '\u{1a74}'), + ('\u{1abf}', '\u{1ac0}'), + ('\u{1acc}', '\u{1ace}'), + ('\u{1b00}', 'ᬄ'), + ('\u{1b35}', '\u{1b43}'), + ('\u{1b80}', 'ᮂ'), + ('ᮡ', '\u{1ba9}'), + ('\u{1bac}', '\u{1bad}'), + ('ᯧ', '\u{1bf1}'), + ('ᰤ', '\u{1c36}'), + ('\u{1dd3}', '\u{1df4}'), + ('Ⓐ', 'ⓩ'), + ('\u{2de0}', '\u{2dff}'), + ('\u{a674}', '\u{a67b}'), + ('\u{a69e}', '\u{a69f}'), + ('\u{a802}', '\u{a802}'), + ('\u{a80b}', '\u{a80b}'), + ('ꠣ', 'ꠧ'), + ('ꢀ', 'ꢁ'), + ('ꢴ', 'ꣃ'), + ('\u{a8c5}', '\u{a8c5}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92a}'), + ('\u{a947}', 'ꥒ'), + ('\u{a980}', 'ꦃ'), + ('ꦴ', 'ꦿ'), + ('\u{a9e5}', '\u{a9e5}'), + ('\u{aa29}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', 'ꩍ'), + ('ꩻ', 'ꩽ'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabe}'), + ('ꫫ', 'ꫯ'), + ('ꫵ', 'ꫵ'), + ('ꯣ', 'ꯪ'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{10376}', '\u{1037a}'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10d24}', '\u{10d27}'), + ('\u{10d69}', '\u{10d69}'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10efc}'), + ('𑀀', '𑀂'), + ('\u{11038}', '\u{11045}'), + ('\u{11073}', '\u{11074}'), + ('\u{11080}', '𑂂'), + ('𑂰', '𑂸'), + ('\u{110c2}', '\u{110c2}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{11132}'), + ('𑅅', '𑅆'), + ('\u{11180}', '𑆂'), + ('𑆳', '𑆿'), + ('𑇎', '\u{111cf}'), + ('𑈬', '\u{11234}'), + ('\u{11237}', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112e8}'), + ('\u{11300}', '𑌃'), + ('\u{1133e}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '𑍌'), + ('\u{11357}', '\u{11357}'), + ('𑍢', '𑍣'), + ('\u{113b8}', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏍'), + ('𑐵', '𑑁'), + ('\u{11443}', '𑑅'), + ('\u{114b0}', '𑓁'), + ('\u{115af}', '\u{115b5}'), + ('𑖸', '𑖾'), + ('\u{115dc}', '\u{115dd}'), + ('𑘰', '𑘾'), + ('\u{11640}', '\u{11640}'), + ('\u{116ab}', '\u{116b5}'), + ('\u{1171d}', '\u{1172a}'), + ('𑠬', '𑠸'), + ('\u{11930}', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{1193c}'), + ('𑥀', '𑥀'), + ('𑥂', '𑥂'), + ('𑧑', '\u{119d7}'), + ('\u{119da}', '𑧟'), + ('𑧤', '𑧤'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a35}', '𑨹'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a51}', '\u{11a5b}'), + ('\u{11a8a}', '𑪗'), + ('𑰯', '\u{11c36}'), + ('\u{11c38}', '𑰾'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d41}'), + ('\u{11d43}', '\u{11d43}'), + ('\u{11d47}', '\u{11d47}'), + ('𑶊', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶖'), + ('\u{11ef3}', '𑻶'), + ('\u{11f00}', '\u{11f01}'), + ('𑼃', '𑼃'), + ('𑼴', '\u{11f3a}'), + ('𑼾', '\u{11f40}'), + ('\u{1611e}', '\u{1612e}'), + ('\u{16f4f}', '\u{16f4f}'), + ('𖽑', '𖾇'), + ('\u{16f8f}', '\u{16f92}'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1bc9e}', '\u{1bc9e}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e947}', '\u{1e947}'), + ('🄰', '🅉'), + ('🅐', '🅩'), + ('🅰', '🆉'), +]; + +pub const OTHER_DEFAULT_IGNORABLE_CODE_POINT: &'static [(char, char)] = &[ + ('\u{34f}', '\u{34f}'), + ('ᅟ', 'ᅠ'), + ('\u{17b4}', '\u{17b5}'), + ('\u{2065}', '\u{2065}'), + ('ㅤ', 'ㅤ'), + ('ﾠ', 'ﾠ'), + ('\u{fff0}', '\u{fff8}'), + ('\u{e0000}', '\u{e0000}'), + ('\u{e0002}', '\u{e001f}'), + ('\u{e0080}', '\u{e00ff}'), + ('\u{e01f0}', '\u{e0fff}'), +]; + +pub const OTHER_GRAPHEME_EXTEND: &'static [(char, char)] = &[ + ('\u{9be}', '\u{9be}'), + ('\u{9d7}', '\u{9d7}'), + ('\u{b3e}', '\u{b3e}'), + ('\u{b57}', '\u{b57}'), + ('\u{bbe}', '\u{bbe}'), + ('\u{bd7}', '\u{bd7}'), + ('\u{cc0}', '\u{cc0}'), + ('\u{cc2}', '\u{cc2}'), + ('\u{cc7}', '\u{cc8}'), + ('\u{cca}', '\u{ccb}'), + ('\u{cd5}', '\u{cd6}'), + ('\u{d3e}', '\u{d3e}'), + ('\u{d57}', '\u{d57}'), + ('\u{dcf}', '\u{dcf}'), + ('\u{ddf}', '\u{ddf}'), + ('\u{1715}', '\u{1715}'), + ('\u{1734}', '\u{1734}'), + ('\u{1b35}', '\u{1b35}'), + ('\u{1b3b}', '\u{1b3b}'), + ('\u{1b3d}', '\u{1b3d}'), + ('\u{1b43}', '\u{1b44}'), + ('\u{1baa}', '\u{1baa}'), + ('\u{1bf2}', '\u{1bf3}'), + ('\u{200c}', '\u{200c}'), + ('\u{302e}', '\u{302f}'), + ('\u{a953}', '\u{a953}'), + ('\u{a9c0}', '\u{a9c0}'), + ('\u{ff9e}', '\u{ff9f}'), + ('\u{111c0}', '\u{111c0}'), + ('\u{11235}', '\u{11235}'), + ('\u{1133e}', '\u{1133e}'), + ('\u{1134d}', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('\u{113b8}', '\u{113b8}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '\u{113c9}'), + ('\u{113cf}', '\u{113cf}'), + ('\u{114b0}', '\u{114b0}'), + ('\u{114bd}', '\u{114bd}'), + ('\u{115af}', '\u{115af}'), + ('\u{116b6}', '\u{116b6}'), + ('\u{11930}', '\u{11930}'), + ('\u{1193d}', '\u{1193d}'), + ('\u{11f41}', '\u{11f41}'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1d165}', '\u{1d166}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{e0020}', '\u{e007f}'), +]; + +pub const OTHER_ID_CONTINUE: &'static [(char, char)] = &[ + ('·', '·'), + ('·', '·'), + ('፩', '፱'), + ('᧚', '᧚'), + ('\u{200c}', '\u{200d}'), + ('・', '・'), + ('･', '･'), +]; + +pub const OTHER_ID_START: &'static [(char, char)] = + &[('\u{1885}', '\u{1886}'), ('℘', '℘'), ('℮', '℮'), ('゛', '゜')]; + +pub const OTHER_LOWERCASE: &'static [(char, char)] = &[ + ('ª', 'ª'), + ('º', 'º'), + ('ʰ', 'ʸ'), + ('ˀ', 'ˁ'), + ('ˠ', 'ˤ'), + ('\u{345}', '\u{345}'), + ('ͺ', 'ͺ'), + ('ჼ', 'ჼ'), + ('ᴬ', 'ᵪ'), + ('ᵸ', 'ᵸ'), + ('ᶛ', 'ᶿ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ⅰ', 'ⅿ'), + ('ⓐ', 'ⓩ'), + ('ⱼ', 'ⱽ'), + ('ꚜ', 'ꚝ'), + ('ꝰ', 'ꝰ'), + ('ꟲ', 'ꟴ'), + ('ꟸ', 'ꟹ'), + ('ꭜ', 'ꭟ'), + ('ꭩ', 'ꭩ'), + ('𐞀', '𐞀'), + ('𐞃', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𞀰', '𞁭'), +]; + +pub const OTHER_MATH: &'static [(char, char)] = &[ + ('^', '^'), + ('ϐ', 'ϒ'), + ('ϕ', 'ϕ'), + ('ϰ', 'ϱ'), + ('ϴ', 'ϵ'), + ('‖', '‖'), + ('′', '‴'), + ('⁀', '⁀'), + ('\u{2061}', '\u{2064}'), + ('⁽', '⁾'), + ('₍', '₎'), + ('\u{20d0}', '\u{20dc}'), + ('\u{20e1}', '\u{20e1}'), + ('\u{20e5}', '\u{20e6}'), + ('\u{20eb}', '\u{20ef}'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('ℨ', '℩'), + ('ℬ', 'ℭ'), + ('ℯ', 'ℱ'), + ('ℳ', 'ℸ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('↕', '↙'), + ('↜', '↟'), + ('↡', '↢'), + ('↤', '↥'), + ('↧', '↧'), + ('↩', '↭'), + ('↰', '↱'), + ('↶', '↷'), + ('↼', '⇍'), + ('⇐', '⇑'), + ('⇓', '⇓'), + ('⇕', '⇛'), + ('⇝', '⇝'), + ('⇤', '⇥'), + ('⌈', '⌋'), + ('⎴', '⎵'), + ('⎷', '⎷'), + ('⏐', '⏐'), + ('⏢', '⏢'), + ('■', '□'), + ('▮', '▶'), + ('▼', '◀'), + ('◆', '◇'), + ('◊', '○'), + ('●', '◓'), + ('◢', '◢'), + ('◤', '◤'), + ('◧', '◬'), + ('★', '☆'), + ('♀', '♀'), + ('♂', '♂'), + ('♠', '♣'), + ('♭', '♮'), + ('⟅', '⟆'), + ('⟦', '⟯'), + ('⦃', '⦘'), + ('⧘', '⧛'), + ('⧼', '⧽'), + ('﹡', '﹡'), + ('﹣', '﹣'), + ('﹨', '﹨'), + ('＼', '＼'), + ('＾', '＾'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝟎', '𝟿'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), +]; + +pub const OTHER_UPPERCASE: &'static [(char, char)] = + &[('Ⅰ', 'Ⅿ'), ('Ⓐ', 'Ⓩ'), ('🄰', '🅉'), ('🅐', '🅩'), ('🅰', '🆉')]; + +pub const PATTERN_SYNTAX: &'static [(char, char)] = &[ + ('!', '/'), + (':', '@'), + ('[', '^'), + ('`', '`'), + ('{', '~'), + ('¡', '§'), + ('©', '©'), + ('«', '¬'), + ('®', '®'), + ('°', '±'), + ('¶', '¶'), + ('»', '»'), + ('¿', '¿'), + ('×', '×'), + ('÷', '÷'), + ('‐', '‧'), + ('‰', '‾'), + ('⁁', '⁓'), + ('⁕', '⁞'), + ('←', '\u{245f}'), + ('─', '❵'), + ('➔', '⯿'), + ('⸀', '\u{2e7f}'), + ('、', '〃'), + ('〈', '〠'), + ('〰', '〰'), + ('﴾', '﴿'), + ('﹅', '﹆'), +]; + +pub const PATTERN_WHITE_SPACE: &'static [(char, char)] = &[ + ('\t', '\r'), + (' ', ' '), + ('\u{85}', '\u{85}'), + ('\u{200e}', '\u{200f}'), + ('\u{2028}', '\u{2029}'), +]; + +pub const PREPENDED_CONCATENATION_MARK: &'static [(char, char)] = &[ + ('\u{600}', '\u{605}'), + ('\u{6dd}', '\u{6dd}'), + ('\u{70f}', '\u{70f}'), + ('\u{890}', '\u{891}'), + ('\u{8e2}', '\u{8e2}'), + ('\u{110bd}', '\u{110bd}'), + ('\u{110cd}', '\u{110cd}'), +]; + +pub const QUOTATION_MARK: &'static [(char, char)] = &[ + ('"', '"'), + ('\'', '\''), + ('«', '«'), + ('»', '»'), + ('‘', '‟'), + ('‹', '›'), + ('⹂', '⹂'), + ('「', '』'), + ('〝', '〟'), + ('﹁', '﹄'), + ('＂', '＂'), + ('＇', '＇'), + ('｢', '｣'), +]; + +pub const RADICAL: &'static [(char, char)] = + &[('⺀', '⺙'), ('⺛', '⻳'), ('⼀', '⿕')]; + +pub const REGIONAL_INDICATOR: &'static [(char, char)] = &[('🇦', '🇿')]; + +pub const SENTENCE_TERMINAL: &'static [(char, char)] = &[ + ('!', '!'), + ('.', '.'), + ('?', '?'), + ('։', '։'), + ('؝', '؟'), + ('۔', '۔'), + ('܀', '܂'), + ('߹', '߹'), + ('࠷', '࠷'), + ('࠹', '࠹'), + ('࠽', '࠾'), + ('।', '॥'), + ('၊', '။'), + ('።', '።'), + ('፧', '፨'), + ('᙮', '᙮'), + ('᜵', '᜶'), + ('។', '៕'), + ('᠃', '᠃'), + ('᠉', '᠉'), + ('᥄', '᥅'), + ('᪨', '᪫'), + ('᭎', '᭏'), + ('᭚', '᭛'), + ('᭞', '᭟'), + ('᭽', '᭿'), + ('᰻', '᰼'), + ('᱾', '᱿'), + ('․', '․'), + ('‼', '‽'), + ('⁇', '⁉'), + ('⳹', '⳻'), + ('⸮', '⸮'), + ('⸼', '⸼'), + ('⹓', '⹔'), + ('。', '。'), + ('꓿', '꓿'), + ('꘎', '꘏'), + ('꛳', '꛳'), + ('꛷', '꛷'), + ('꡶', '꡷'), + ('꣎', '꣏'), + ('꤯', '꤯'), + ('꧈', '꧉'), + ('꩝', '꩟'), + ('꫰', '꫱'), + ('꯫', '꯫'), + ('︒', '︒'), + ('︕', '︖'), + ('﹒', '﹒'), + ('﹖', '﹗'), + ('！', '！'), + ('．', '．'), + ('？', '？'), + ('｡', '｡'), + ('𐩖', '𐩗'), + ('𐽕', '𐽙'), + ('𐾆', '𐾉'), + ('𑁇', '𑁈'), + ('𑂾', '𑃁'), + ('𑅁', '𑅃'), + ('𑇅', '𑇆'), + ('𑇍', '𑇍'), + ('𑇞', '𑇟'), + ('𑈸', '𑈹'), + ('𑈻', '𑈼'), + ('𑊩', '𑊩'), + ('𑏔', '𑏕'), + ('𑑋', '𑑌'), + ('𑗂', '𑗃'), + ('𑗉', '𑗗'), + ('𑙁', '𑙂'), + ('𑜼', '𑜾'), + ('𑥄', '𑥄'), + ('𑥆', '𑥆'), + ('𑩂', '𑩃'), + ('𑪛', '𑪜'), + ('𑱁', '𑱂'), + ('𑻷', '𑻸'), + ('𑽃', '𑽄'), + ('𖩮', '𖩯'), + ('𖫵', '𖫵'), + ('𖬷', '𖬸'), + ('𖭄', '𖭄'), + ('𖵮', '𖵯'), + ('𖺘', '𖺘'), + ('𛲟', '𛲟'), + ('𝪈', '𝪈'), +]; + +pub const SOFT_DOTTED: &'static [(char, char)] = &[ + ('i', 'j'), + ('į', 'į'), + ('ɉ', 'ɉ'), + ('ɨ', 'ɨ'), + ('ʝ', 'ʝ'), + ('ʲ', 'ʲ'), + ('ϳ', 'ϳ'), + ('і', 'і'), + ('ј', 'ј'), + ('ᵢ', 'ᵢ'), + ('ᶖ', 'ᶖ'), + ('ᶤ', 'ᶤ'), + ('ᶨ', 'ᶨ'), + ('ḭ', 'ḭ'), + ('ị', 'ị'), + ('ⁱ', 'ⁱ'), + ('ⅈ', 'ⅉ'), + ('ⱼ', 'ⱼ'), + ('𝐢', '𝐣'), + ('𝑖', '𝑗'), + ('𝒊', '𝒋'), + ('𝒾', '𝒿'), + ('𝓲', '𝓳'), + ('𝔦', '𝔧'), + ('𝕚', '𝕛'), + ('𝖎', '𝖏'), + ('𝗂', '𝗃'), + ('𝗶', '𝗷'), + ('𝘪', '𝘫'), + ('𝙞', '𝙟'), + ('𝚒', '𝚓'), + ('𝼚', '𝼚'), + ('𞁌', '𞁍'), + ('𞁨', '𞁨'), +]; + +pub const TERMINAL_PUNCTUATION: &'static [(char, char)] = &[ + ('!', '!'), + (',', ','), + ('.', '.'), + (':', ';'), + ('?', '?'), + (';', ';'), + ('·', '·'), + ('։', '։'), + ('׃', '׃'), + ('،', '،'), + ('؛', '؛'), + ('؝', '؟'), + ('۔', '۔'), + ('܀', '܊'), + ('܌', '܌'), + ('߸', '߹'), + ('࠰', '࠵'), + ('࠷', '࠾'), + ('࡞', '࡞'), + ('।', '॥'), + ('๚', '๛'), + ('༈', '༈'), + ('།', '༒'), + ('၊', '။'), + ('፡', '፨'), + ('᙮', '᙮'), + ('᛫', '᛭'), + ('᜵', '᜶'), + ('។', '៖'), + ('៚', '៚'), + ('᠂', '᠅'), + ('᠈', '᠉'), + ('᥄', '᥅'), + ('᪨', '᪫'), + ('᭎', '᭏'), + ('᭚', '᭛'), + ('᭝', '᭟'), + ('᭽', '᭿'), + ('᰻', '᰿'), + ('᱾', '᱿'), + ('․', '․'), + ('‼', '‽'), + ('⁇', '⁉'), + ('⳹', '⳻'), + ('⸮', '⸮'), + ('⸼', '⸼'), + ('⹁', '⹁'), + ('⹌', '⹌'), + ('⹎', '⹏'), + ('⹓', '⹔'), + ('、', '。'), + ('꓾', '꓿'), + ('꘍', '꘏'), + ('꛳', '꛷'), + ('꡶', '꡷'), + ('꣎', '꣏'), + ('꤯', '꤯'), + ('꧇', '꧉'), + ('꩝', '꩟'), + ('꫟', '꫟'), + ('꫰', '꫱'), + ('꯫', '꯫'), + ('︒', '︒'), + ('︕', '︖'), + ('﹐', '﹒'), + ('﹔', '﹗'), + ('！', '！'), + ('，', '，'), + ('．', '．'), + ('：', '；'), + ('？', '？'), + ('｡', '｡'), + ('､', '､'), + ('𐎟', '𐎟'), + ('𐏐', '𐏐'), + ('𐡗', '𐡗'), + ('𐤟', '𐤟'), + ('𐩖', '𐩗'), + ('𐫰', '𐫵'), + ('𐬺', '𐬿'), + ('𐮙', '𐮜'), + ('𐽕', '𐽙'), + ('𐾆', '𐾉'), + ('𑁇', '𑁍'), + ('𑂾', '𑃁'), + ('𑅁', '𑅃'), + ('𑇅', '𑇆'), + ('𑇍', '𑇍'), + ('𑇞', '𑇟'), + ('𑈸', '𑈼'), + ('𑊩', '𑊩'), + ('𑏔', '𑏕'), + ('𑑋', '𑑍'), + ('𑑚', '𑑛'), + ('𑗂', '𑗅'), + ('𑗉', '𑗗'), + ('𑙁', '𑙂'), + ('𑜼', '𑜾'), + ('𑥄', '𑥄'), + ('𑥆', '𑥆'), + ('𑩂', '𑩃'), + ('𑪛', '𑪜'), + ('𑪡', '𑪢'), + ('𑱁', '𑱃'), + ('𑱱', '𑱱'), + ('𑻷', '𑻸'), + ('𑽃', '𑽄'), + ('𒑰', '𒑴'), + ('𖩮', '𖩯'), + ('𖫵', '𖫵'), + ('𖬷', '𖬹'), + ('𖭄', '𖭄'), + ('𖵮', '𖵯'), + ('𖺗', '𖺘'), + ('𛲟', '𛲟'), + ('𝪇', '𝪊'), +]; + +pub const UNIFIED_IDEOGRAPH: &'static [(char, char)] = &[ + ('㐀', '䶿'), + ('一', '鿿'), + ('﨎', '﨏'), + ('﨑', '﨑'), + ('﨓', '﨔'), + ('﨟', '﨟'), + ('﨡', '﨡'), + ('﨣', '﨤'), + ('﨧', '﨩'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const UPPERCASE: &'static [(char, char)] = &[ + ('A', 'Z'), + ('À', 'Ö'), + ('Ø', 'Þ'), + ('Ā', 'Ā'), + ('Ă', 'Ă'), + ('Ą', 'Ą'), + ('Ć', 'Ć'), + ('Ĉ', 'Ĉ'), + ('Ċ', 'Ċ'), + ('Č', 'Č'), + ('Ď', 'Ď'), + ('Đ', 'Đ'), + ('Ē', 'Ē'), + ('Ĕ', 'Ĕ'), + ('Ė', 'Ė'), + ('Ę', 'Ę'), + ('Ě', 'Ě'), + ('Ĝ', 'Ĝ'), + ('Ğ', 'Ğ'), + ('Ġ', 'Ġ'), + ('Ģ', 'Ģ'), + ('Ĥ', 'Ĥ'), + ('Ħ', 'Ħ'), + ('Ĩ', 'Ĩ'), + ('Ī', 'Ī'), + ('Ĭ', 'Ĭ'), + ('Į', 'Į'), + ('İ', 'İ'), + ('IJ', 'IJ'), + ('Ĵ', 'Ĵ'), + ('Ķ', 'Ķ'), + ('Ĺ', 'Ĺ'), + ('Ļ', 'Ļ'), + ('Ľ', 'Ľ'), + ('Ŀ', 'Ŀ'), + ('Ł', 'Ł'), + ('Ń', 'Ń'), + ('Ņ', 'Ņ'), + ('Ň', 'Ň'), + ('Ŋ', 'Ŋ'), + ('Ō', 'Ō'), + ('Ŏ', 'Ŏ'), + ('Ő', 'Ő'), + ('Œ', 'Œ'), + ('Ŕ', 'Ŕ'), + ('Ŗ', 'Ŗ'), + ('Ř', 'Ř'), + ('Ś', 'Ś'), + ('Ŝ', 'Ŝ'), + ('Ş', 'Ş'), + ('Š', 'Š'), + ('Ţ', 'Ţ'), + ('Ť', 'Ť'), + ('Ŧ', 'Ŧ'), + ('Ũ', 'Ũ'), + ('Ū', 'Ū'), + ('Ŭ', 'Ŭ'), + ('Ů', 'Ů'), + ('Ű', 'Ű'), + ('Ų', 'Ų'), + ('Ŵ', 'Ŵ'), + ('Ŷ', 'Ŷ'), + ('Ÿ', 'Ź'), + ('Ż', 'Ż'), + ('Ž', 'Ž'), + ('Ɓ', 'Ƃ'), + ('Ƅ', 'Ƅ'), + ('Ɔ', 'Ƈ'), + ('Ɖ', 'Ƌ'), + ('Ǝ', 'Ƒ'), + ('Ɠ', 'Ɣ'), + ('Ɩ', 'Ƙ'), + ('Ɯ', 'Ɲ'), + ('Ɵ', 'Ơ'), + ('Ƣ', 'Ƣ'), + ('Ƥ', 'Ƥ'), + ('Ʀ', 'Ƨ'), + ('Ʃ', 'Ʃ'), + ('Ƭ', 'Ƭ'), + ('Ʈ', 'Ư'), + ('Ʊ', 'Ƴ'), + ('Ƶ', 'Ƶ'), + ('Ʒ', 'Ƹ'), + ('Ƽ', 'Ƽ'), + ('DŽ', 'DŽ'), + ('LJ', 'LJ'), + ('NJ', 'NJ'), + ('Ǎ', 'Ǎ'), + ('Ǐ', 'Ǐ'), + ('Ǒ', 'Ǒ'), + ('Ǔ', 'Ǔ'), + ('Ǖ', 'Ǖ'), + ('Ǘ', 'Ǘ'), + ('Ǚ', 'Ǚ'), + ('Ǜ', 'Ǜ'), + ('Ǟ', 'Ǟ'), + ('Ǡ', 'Ǡ'), + ('Ǣ', 'Ǣ'), + ('Ǥ', 'Ǥ'), + ('Ǧ', 'Ǧ'), + ('Ǩ', 'Ǩ'), + ('Ǫ', 'Ǫ'), + ('Ǭ', 'Ǭ'), + ('Ǯ', 'Ǯ'), + ('DZ', 'DZ'), + ('Ǵ', 'Ǵ'), + ('Ƕ', 'Ǹ'), + ('Ǻ', 'Ǻ'), + ('Ǽ', 'Ǽ'), + ('Ǿ', 'Ǿ'), + ('Ȁ', 'Ȁ'), + ('Ȃ', 'Ȃ'), + ('Ȅ', 'Ȅ'), + ('Ȇ', 'Ȇ'), + ('Ȉ', 'Ȉ'), + ('Ȋ', 'Ȋ'), + ('Ȍ', 'Ȍ'), + ('Ȏ', 'Ȏ'), + ('Ȑ', 'Ȑ'), + ('Ȓ', 'Ȓ'), + ('Ȕ', 'Ȕ'), + ('Ȗ', 'Ȗ'), + ('Ș', 'Ș'), + ('Ț', 'Ț'), + ('Ȝ', 'Ȝ'), + ('Ȟ', 'Ȟ'), + ('Ƞ', 'Ƞ'), + ('Ȣ', 'Ȣ'), + ('Ȥ', 'Ȥ'), + ('Ȧ', 'Ȧ'), + ('Ȩ', 'Ȩ'), + ('Ȫ', 'Ȫ'), + ('Ȭ', 'Ȭ'), + ('Ȯ', 'Ȯ'), + ('Ȱ', 'Ȱ'), + ('Ȳ', 'Ȳ'), + ('Ⱥ', 'Ȼ'), + ('Ƚ', 'Ⱦ'), + ('Ɂ', 'Ɂ'), + ('Ƀ', 'Ɇ'), + ('Ɉ', 'Ɉ'), + ('Ɋ', 'Ɋ'), + ('Ɍ', 'Ɍ'), + ('Ɏ', 'Ɏ'), + ('Ͱ', 'Ͱ'), + ('Ͳ', 'Ͳ'), + ('Ͷ', 'Ͷ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ώ'), + ('Α', 'Ρ'), + ('Σ', 'Ϋ'), + ('Ϗ', 'Ϗ'), + ('ϒ', 'ϔ'), + ('Ϙ', 'Ϙ'), + ('Ϛ', 'Ϛ'), + ('Ϝ', 'Ϝ'), + ('Ϟ', 'Ϟ'), + ('Ϡ', 'Ϡ'), + ('Ϣ', 'Ϣ'), + ('Ϥ', 'Ϥ'), + ('Ϧ', 'Ϧ'), + ('Ϩ', 'Ϩ'), + ('Ϫ', 'Ϫ'), + ('Ϭ', 'Ϭ'), + ('Ϯ', 'Ϯ'), + ('ϴ', 'ϴ'), + ('Ϸ', 'Ϸ'), + ('Ϲ', 'Ϻ'), + ('Ͻ', 'Я'), + ('Ѡ', 'Ѡ'), + ('Ѣ', 'Ѣ'), + ('Ѥ', 'Ѥ'), + ('Ѧ', 'Ѧ'), + ('Ѩ', 'Ѩ'), + ('Ѫ', 'Ѫ'), + ('Ѭ', 'Ѭ'), + ('Ѯ', 'Ѯ'), + ('Ѱ', 'Ѱ'), + ('Ѳ', 'Ѳ'), + ('Ѵ', 'Ѵ'), + ('Ѷ', 'Ѷ'), + ('Ѹ', 'Ѹ'), + ('Ѻ', 'Ѻ'), + ('Ѽ', 'Ѽ'), + ('Ѿ', 'Ѿ'), + ('Ҁ', 'Ҁ'), + ('Ҋ', 'Ҋ'), + ('Ҍ', 'Ҍ'), + ('Ҏ', 'Ҏ'), + ('Ґ', 'Ґ'), + ('Ғ', 'Ғ'), + ('Ҕ', 'Ҕ'), + ('Җ', 'Җ'), + ('Ҙ', 'Ҙ'), + ('Қ', 'Қ'), + ('Ҝ', 'Ҝ'), + ('Ҟ', 'Ҟ'), + ('Ҡ', 'Ҡ'), + ('Ң', 'Ң'), + ('Ҥ', 'Ҥ'), + ('Ҧ', 'Ҧ'), + ('Ҩ', 'Ҩ'), + ('Ҫ', 'Ҫ'), + ('Ҭ', 'Ҭ'), + ('Ү', 'Ү'), + ('Ұ', 'Ұ'), + ('Ҳ', 'Ҳ'), + ('Ҵ', 'Ҵ'), + ('Ҷ', 'Ҷ'), + ('Ҹ', 'Ҹ'), + ('Һ', 'Һ'), + ('Ҽ', 'Ҽ'), + ('Ҿ', 'Ҿ'), + ('Ӏ', 'Ӂ'), + ('Ӄ', 'Ӄ'), + ('Ӆ', 'Ӆ'), + ('Ӈ', 'Ӈ'), + ('Ӊ', 'Ӊ'), + ('Ӌ', 'Ӌ'), + ('Ӎ', 'Ӎ'), + ('Ӑ', 'Ӑ'), + ('Ӓ', 'Ӓ'), + ('Ӕ', 'Ӕ'), + ('Ӗ', 'Ӗ'), + ('Ә', 'Ә'), + ('Ӛ', 'Ӛ'), + ('Ӝ', 'Ӝ'), + ('Ӟ', 'Ӟ'), + ('Ӡ', 'Ӡ'), + ('Ӣ', 'Ӣ'), + ('Ӥ', 'Ӥ'), + ('Ӧ', 'Ӧ'), + ('Ө', 'Ө'), + ('Ӫ', 'Ӫ'), + ('Ӭ', 'Ӭ'), + ('Ӯ', 'Ӯ'), + ('Ӱ', 'Ӱ'), + ('Ӳ', 'Ӳ'), + ('Ӵ', 'Ӵ'), + ('Ӷ', 'Ӷ'), + ('Ӹ', 'Ӹ'), + ('Ӻ', 'Ӻ'), + ('Ӽ', 'Ӽ'), + ('Ӿ', 'Ӿ'), + ('Ԁ', 'Ԁ'), + ('Ԃ', 'Ԃ'), + ('Ԅ', 'Ԅ'), + ('Ԇ', 'Ԇ'), + ('Ԉ', 'Ԉ'), + ('Ԋ', 'Ԋ'), + ('Ԍ', 'Ԍ'), + ('Ԏ', 'Ԏ'), + ('Ԑ', 'Ԑ'), + ('Ԓ', 'Ԓ'), + ('Ԕ', 'Ԕ'), + ('Ԗ', 'Ԗ'), + ('Ԙ', 'Ԙ'), + ('Ԛ', 'Ԛ'), + ('Ԝ', 'Ԝ'), + ('Ԟ', 'Ԟ'), + ('Ԡ', 'Ԡ'), + ('Ԣ', 'Ԣ'), + ('Ԥ', 'Ԥ'), + ('Ԧ', 'Ԧ'), + ('Ԩ', 'Ԩ'), + ('Ԫ', 'Ԫ'), + ('Ԭ', 'Ԭ'), + ('Ԯ', 'Ԯ'), + ('Ա', 'Ֆ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('Ꭰ', 'Ᏽ'), + ('Ᲊ', 'Ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('Ḁ', 'Ḁ'), + ('Ḃ', 'Ḃ'), + ('Ḅ', 'Ḅ'), + ('Ḇ', 'Ḇ'), + ('Ḉ', 'Ḉ'), + ('Ḋ', 'Ḋ'), + ('Ḍ', 'Ḍ'), + ('Ḏ', 'Ḏ'), + ('Ḑ', 'Ḑ'), + ('Ḓ', 'Ḓ'), + ('Ḕ', 'Ḕ'), + ('Ḗ', 'Ḗ'), + ('Ḙ', 'Ḙ'), + ('Ḛ', 'Ḛ'), + ('Ḝ', 'Ḝ'), + ('Ḟ', 'Ḟ'), + ('Ḡ', 'Ḡ'), + ('Ḣ', 'Ḣ'), + ('Ḥ', 'Ḥ'), + ('Ḧ', 'Ḧ'), + ('Ḩ', 'Ḩ'), + ('Ḫ', 'Ḫ'), + ('Ḭ', 'Ḭ'), + ('Ḯ', 'Ḯ'), + ('Ḱ', 'Ḱ'), + ('Ḳ', 'Ḳ'), + ('Ḵ', 'Ḵ'), + ('Ḷ', 'Ḷ'), + ('Ḹ', 'Ḹ'), + ('Ḻ', 'Ḻ'), + ('Ḽ', 'Ḽ'), + ('Ḿ', 'Ḿ'), + ('Ṁ', 'Ṁ'), + ('Ṃ', 'Ṃ'), + ('Ṅ', 'Ṅ'), + ('Ṇ', 'Ṇ'), + ('Ṉ', 'Ṉ'), + ('Ṋ', 'Ṋ'), + ('Ṍ', 'Ṍ'), + ('Ṏ', 'Ṏ'), + ('Ṑ', 'Ṑ'), + ('Ṓ', 'Ṓ'), + ('Ṕ', 'Ṕ'), + ('Ṗ', 'Ṗ'), + ('Ṙ', 'Ṙ'), + ('Ṛ', 'Ṛ'), + ('Ṝ', 'Ṝ'), + ('Ṟ', 'Ṟ'), + ('Ṡ', 'Ṡ'), + ('Ṣ', 'Ṣ'), + ('Ṥ', 'Ṥ'), + ('Ṧ', 'Ṧ'), + ('Ṩ', 'Ṩ'), + ('Ṫ', 'Ṫ'), + ('Ṭ', 'Ṭ'), + ('Ṯ', 'Ṯ'), + ('Ṱ', 'Ṱ'), + ('Ṳ', 'Ṳ'), + ('Ṵ', 'Ṵ'), + ('Ṷ', 'Ṷ'), + ('Ṹ', 'Ṹ'), + ('Ṻ', 'Ṻ'), + ('Ṽ', 'Ṽ'), + ('Ṿ', 'Ṿ'), + ('Ẁ', 'Ẁ'), + ('Ẃ', 'Ẃ'), + ('Ẅ', 'Ẅ'), + ('Ẇ', 'Ẇ'), + ('Ẉ', 'Ẉ'), + ('Ẋ', 'Ẋ'), + ('Ẍ', 'Ẍ'), + ('Ẏ', 'Ẏ'), + ('Ẑ', 'Ẑ'), + ('Ẓ', 'Ẓ'), + ('Ẕ', 'Ẕ'), + ('ẞ', 'ẞ'), + ('Ạ', 'Ạ'), + ('Ả', 'Ả'), + ('Ấ', 'Ấ'), + ('Ầ', 'Ầ'), + ('Ẩ', 'Ẩ'), + ('Ẫ', 'Ẫ'), + ('Ậ', 'Ậ'), + ('Ắ', 'Ắ'), + ('Ằ', 'Ằ'), + ('Ẳ', 'Ẳ'), + ('Ẵ', 'Ẵ'), + ('Ặ', 'Ặ'), + ('Ẹ', 'Ẹ'), + ('Ẻ', 'Ẻ'), + ('Ẽ', 'Ẽ'), + ('Ế', 'Ế'), + ('Ề', 'Ề'), + ('Ể', 'Ể'), + ('Ễ', 'Ễ'), + ('Ệ', 'Ệ'), + ('Ỉ', 'Ỉ'), + ('Ị', 'Ị'), + ('Ọ', 'Ọ'), + ('Ỏ', 'Ỏ'), + ('Ố', 'Ố'), + ('Ồ', 'Ồ'), + ('Ổ', 'Ổ'), + ('Ỗ', 'Ỗ'), + ('Ộ', 'Ộ'), + ('Ớ', 'Ớ'), + ('Ờ', 'Ờ'), + ('Ở', 'Ở'), + ('Ỡ', 'Ỡ'), + ('Ợ', 'Ợ'), + ('Ụ', 'Ụ'), + ('Ủ', 'Ủ'), + ('Ứ', 'Ứ'), + ('Ừ', 'Ừ'), + ('Ử', 'Ử'), + ('Ữ', 'Ữ'), + ('Ự', 'Ự'), + ('Ỳ', 'Ỳ'), + ('Ỵ', 'Ỵ'), + ('Ỷ', 'Ỷ'), + ('Ỹ', 'Ỹ'), + ('Ỻ', 'Ỻ'), + ('Ỽ', 'Ỽ'), + ('Ỿ', 'Ỿ'), + ('Ἀ', 'Ἇ'), + ('Ἐ', 'Ἕ'), + ('Ἠ', 'Ἧ'), + ('Ἰ', 'Ἷ'), + ('Ὀ', 'Ὅ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'Ὗ'), + ('Ὠ', 'Ὧ'), + ('Ᾰ', 'Ά'), + ('Ὲ', 'Ή'), + ('Ῐ', 'Ί'), + ('Ῠ', 'Ῥ'), + ('Ὸ', 'Ώ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℋ', 'ℍ'), + ('ℐ', 'ℒ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℰ', 'ℳ'), + ('ℾ', 'ℿ'), + ('ⅅ', 'ⅅ'), + ('Ⅰ', 'Ⅿ'), + ('Ↄ', 'Ↄ'), + ('Ⓐ', 'Ⓩ'), + ('Ⰰ', 'Ⱟ'), + ('Ⱡ', 'Ⱡ'), + ('Ɫ', 'Ɽ'), + ('Ⱨ', 'Ⱨ'), + ('Ⱪ', 'Ⱪ'), + ('Ⱬ', 'Ⱬ'), + ('Ɑ', 'Ɒ'), + ('Ⱳ', 'Ⱳ'), + ('Ⱶ', 'Ⱶ'), + ('Ȿ', 'Ⲁ'), + ('Ⲃ', 'Ⲃ'), + ('Ⲅ', 'Ⲅ'), + ('Ⲇ', 'Ⲇ'), + ('Ⲉ', 'Ⲉ'), + ('Ⲋ', 'Ⲋ'), + ('Ⲍ', 'Ⲍ'), + ('Ⲏ', 'Ⲏ'), + ('Ⲑ', 'Ⲑ'), + ('Ⲓ', 'Ⲓ'), + ('Ⲕ', 'Ⲕ'), + ('Ⲗ', 'Ⲗ'), + ('Ⲙ', 'Ⲙ'), + ('Ⲛ', 'Ⲛ'), + ('Ⲝ', 'Ⲝ'), + ('Ⲟ', 'Ⲟ'), + ('Ⲡ', 'Ⲡ'), + ('Ⲣ', 'Ⲣ'), + ('Ⲥ', 'Ⲥ'), + ('Ⲧ', 'Ⲧ'), + ('Ⲩ', 'Ⲩ'), + ('Ⲫ', 'Ⲫ'), + ('Ⲭ', 'Ⲭ'), + ('Ⲯ', 'Ⲯ'), + ('Ⲱ', 'Ⲱ'), + ('Ⲳ', 'Ⲳ'), + ('Ⲵ', 'Ⲵ'), + ('Ⲷ', 'Ⲷ'), + ('Ⲹ', 'Ⲹ'), + ('Ⲻ', 'Ⲻ'), + ('Ⲽ', 'Ⲽ'), + ('Ⲿ', 'Ⲿ'), + ('Ⳁ', 'Ⳁ'), + ('Ⳃ', 'Ⳃ'), + ('Ⳅ', 'Ⳅ'), + ('Ⳇ', 'Ⳇ'), + ('Ⳉ', 'Ⳉ'), + ('Ⳋ', 'Ⳋ'), + ('Ⳍ', 'Ⳍ'), + ('Ⳏ', 'Ⳏ'), + ('Ⳑ', 'Ⳑ'), + ('Ⳓ', 'Ⳓ'), + ('Ⳕ', 'Ⳕ'), + ('Ⳗ', 'Ⳗ'), + ('Ⳙ', 'Ⳙ'), + ('Ⳛ', 'Ⳛ'), + ('Ⳝ', 'Ⳝ'), + ('Ⳟ', 'Ⳟ'), + ('Ⳡ', 'Ⳡ'), + ('Ⳣ', 'Ⳣ'), + ('Ⳬ', 'Ⳬ'), + ('Ⳮ', 'Ⳮ'), + ('Ⳳ', 'Ⳳ'), + ('Ꙁ', 'Ꙁ'), + ('Ꙃ', 'Ꙃ'), + ('Ꙅ', 'Ꙅ'), + ('Ꙇ', 'Ꙇ'), + ('Ꙉ', 'Ꙉ'), + ('Ꙋ', 'Ꙋ'), + ('Ꙍ', 'Ꙍ'), + ('Ꙏ', 'Ꙏ'), + ('Ꙑ', 'Ꙑ'), + ('Ꙓ', 'Ꙓ'), + ('Ꙕ', 'Ꙕ'), + ('Ꙗ', 'Ꙗ'), + ('Ꙙ', 'Ꙙ'), + ('Ꙛ', 'Ꙛ'), + ('Ꙝ', 'Ꙝ'), + ('Ꙟ', 'Ꙟ'), + ('Ꙡ', 'Ꙡ'), + ('Ꙣ', 'Ꙣ'), + ('Ꙥ', 'Ꙥ'), + ('Ꙧ', 'Ꙧ'), + ('Ꙩ', 'Ꙩ'), + ('Ꙫ', 'Ꙫ'), + ('Ꙭ', 'Ꙭ'), + ('Ꚁ', 'Ꚁ'), + ('Ꚃ', 'Ꚃ'), + ('Ꚅ', 'Ꚅ'), + ('Ꚇ', 'Ꚇ'), + ('Ꚉ', 'Ꚉ'), + ('Ꚋ', 'Ꚋ'), + ('Ꚍ', 'Ꚍ'), + ('Ꚏ', 'Ꚏ'), + ('Ꚑ', 'Ꚑ'), + ('Ꚓ', 'Ꚓ'), + ('Ꚕ', 'Ꚕ'), + ('Ꚗ', 'Ꚗ'), + ('Ꚙ', 'Ꚙ'), + ('Ꚛ', 'Ꚛ'), + ('Ꜣ', 'Ꜣ'), + ('Ꜥ', 'Ꜥ'), + ('Ꜧ', 'Ꜧ'), + ('Ꜩ', 'Ꜩ'), + ('Ꜫ', 'Ꜫ'), + ('Ꜭ', 'Ꜭ'), + ('Ꜯ', 'Ꜯ'), + ('Ꜳ', 'Ꜳ'), + ('Ꜵ', 'Ꜵ'), + ('Ꜷ', 'Ꜷ'), + ('Ꜹ', 'Ꜹ'), + ('Ꜻ', 'Ꜻ'), + ('Ꜽ', 'Ꜽ'), + ('Ꜿ', 'Ꜿ'), + ('Ꝁ', 'Ꝁ'), + ('Ꝃ', 'Ꝃ'), + ('Ꝅ', 'Ꝅ'), + ('Ꝇ', 'Ꝇ'), + ('Ꝉ', 'Ꝉ'), + ('Ꝋ', 'Ꝋ'), + ('Ꝍ', 'Ꝍ'), + ('Ꝏ', 'Ꝏ'), + ('Ꝑ', 'Ꝑ'), + ('Ꝓ', 'Ꝓ'), + ('Ꝕ', 'Ꝕ'), + ('Ꝗ', 'Ꝗ'), + ('Ꝙ', 'Ꝙ'), + ('Ꝛ', 'Ꝛ'), + ('Ꝝ', 'Ꝝ'), + ('Ꝟ', 'Ꝟ'), + ('Ꝡ', 'Ꝡ'), + ('Ꝣ', 'Ꝣ'), + ('Ꝥ', 'Ꝥ'), + ('Ꝧ', 'Ꝧ'), + ('Ꝩ', 'Ꝩ'), + ('Ꝫ', 'Ꝫ'), + ('Ꝭ', 'Ꝭ'), + ('Ꝯ', 'Ꝯ'), + ('Ꝺ', 'Ꝺ'), + ('Ꝼ', 'Ꝼ'), + ('Ᵹ', 'Ꝿ'), + ('Ꞁ', 'Ꞁ'), + ('Ꞃ', 'Ꞃ'), + ('Ꞅ', 'Ꞅ'), + ('Ꞇ', 'Ꞇ'), + ('Ꞌ', 'Ꞌ'), + ('Ɥ', 'Ɥ'), + ('Ꞑ', 'Ꞑ'), + ('Ꞓ', 'Ꞓ'), + ('Ꞗ', 'Ꞗ'), + ('Ꞙ', 'Ꞙ'), + ('Ꞛ', 'Ꞛ'), + ('Ꞝ', 'Ꞝ'), + ('Ꞟ', 'Ꞟ'), + ('Ꞡ', 'Ꞡ'), + ('Ꞣ', 'Ꞣ'), + ('Ꞥ', 'Ꞥ'), + ('Ꞧ', 'Ꞧ'), + ('Ꞩ', 'Ꞩ'), + ('Ɦ', 'Ɪ'), + ('Ʞ', 'Ꞵ'), + ('Ꞷ', 'Ꞷ'), + ('Ꞹ', 'Ꞹ'), + ('Ꞻ', 'Ꞻ'), + ('Ꞽ', 'Ꞽ'), + ('Ꞿ', 'Ꞿ'), + ('Ꟁ', 'Ꟁ'), + ('Ꟃ', 'Ꟃ'), + ('Ꞔ', 'Ꟈ'), + ('Ꟊ', 'Ꟊ'), + ('Ɤ', 'Ꟍ'), + ('Ꟑ', 'Ꟑ'), + ('Ꟗ', 'Ꟗ'), + ('Ꟙ', 'Ꟙ'), + ('Ꟛ', 'Ꟛ'), + ('Ƛ', 'Ƛ'), + ('Ꟶ', 'Ꟶ'), + ('Ａ', 'Ｚ'), + ('𐐀', '𐐧'), + ('𐒰', '𐓓'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐲀', '𐲲'), + ('𐵐', '𐵥'), + ('𑢠', '𑢿'), + ('𖹀', '𖹟'), + ('𝐀', '𝐙'), + ('𝐴', '𝑍'), + ('𝑨', '𝒁'), + ('𝒜', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒵'), + ('𝓐', '𝓩'), + ('𝔄', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔸', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕬', '𝖅'), + ('𝖠', '𝖹'), + ('𝗔', '𝗭'), + ('𝘈', '𝘡'), + ('𝘼', '𝙕'), + ('𝙰', '𝚉'), + ('𝚨', '𝛀'), + ('𝛢', '𝛺'), + ('𝜜', '𝜴'), + ('𝝖', '𝝮'), + ('𝞐', '𝞨'), + ('𝟊', '𝟊'), + ('𞤀', '𞤡'), + ('🄰', '🅉'), + ('🅐', '🅩'), + ('🅰', '🆉'), +]; + +pub const VARIATION_SELECTOR: &'static [(char, char)] = &[ + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '\u{180f}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const WHITE_SPACE: &'static [(char, char)] = &[ + ('\t', '\r'), + (' ', ' '), + ('\u{85}', '\u{85}'), + ('\u{a0}', '\u{a0}'), + ('\u{1680}', '\u{1680}'), + ('\u{2000}', '\u{200a}'), + ('\u{2028}', '\u{2029}'), + ('\u{202f}', '\u{202f}'), + ('\u{205f}', '\u{205f}'), + ('\u{3000}', '\u{3000}'), +]; + +pub const XID_CONTINUE: &'static [(char, char)] = &[ + ('0', '9'), + ('A', 'Z'), + ('_', '_'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('·', '·'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('\u{300}', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͻ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('\u{483}', '\u{487}'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', 'ՙ'), + ('ՠ', 'ֈ'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('\u{610}', '\u{61a}'), + ('ؠ', '٩'), + ('ٮ', 'ۓ'), + ('ە', '\u{6dc}'), + ('\u{6df}', '\u{6e8}'), + ('\u{6ea}', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', '\u{74a}'), + ('ݍ', 'ޱ'), + ('߀', 'ߵ'), + ('ߺ', 'ߺ'), + ('\u{7fd}', '\u{7fd}'), + ('ࠀ', '\u{82d}'), + ('ࡀ', '\u{85b}'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('\u{897}', '\u{8e1}'), + ('\u{8e3}', '\u{963}'), + ('०', '९'), + ('ॱ', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('\u{9bc}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', 'ৎ'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', '\u{9e3}'), + ('০', 'ৱ'), + ('ৼ', 'ৼ'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', 'ਃ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('੦', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('\u{abc}', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('ૐ', 'ૐ'), + ('ૠ', '\u{ae3}'), + ('૦', '૯'), + ('ૹ', '\u{aff}'), + ('\u{b01}', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('\u{b3c}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', '\u{b63}'), + ('୦', '୯'), + ('ୱ', 'ୱ'), + ('\u{b82}', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('ௐ', 'ௐ'), + ('\u{bd7}', '\u{bd7}'), + ('௦', '௯'), + ('\u{c00}', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('\u{c3c}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', '\u{c63}'), + ('౦', '౯'), + ('ಀ', 'ಃ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('\u{cbc}', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('ೝ', 'ೞ'), + ('ೠ', '\u{ce3}'), + ('೦', '೯'), + ('ೱ', 'ೳ'), + ('\u{d00}', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', 'ൎ'), + ('ൔ', '\u{d57}'), + ('ൟ', '\u{d63}'), + ('൦', '൯'), + ('ൺ', 'ൿ'), + ('\u{d81}', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('෦', '෯'), + ('ෲ', 'ෳ'), + ('ก', '\u{e3a}'), + ('เ', '\u{e4e}'), + ('๐', '๙'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('\u{ec8}', '\u{ece}'), + ('໐', '໙'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('\u{f18}', '\u{f19}'), + ('༠', '༩'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('༾', 'ཇ'), + ('ཉ', 'ཬ'), + ('\u{f71}', '\u{f84}'), + ('\u{f86}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('က', '၉'), + ('ၐ', '\u{109d}'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('\u{135d}', '\u{135f}'), + ('፩', '፱'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', '\u{1715}'), + ('ᜟ', '\u{1734}'), + ('ᝀ', '\u{1753}'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('\u{1772}', '\u{1773}'), + ('ក', '\u{17d3}'), + ('ៗ', 'ៗ'), + ('ៜ', '\u{17dd}'), + ('០', '៩'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '᠙'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('᥆', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('᧐', '᧚'), + ('ᨀ', '\u{1a1b}'), + ('ᨠ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '᪉'), + ('᪐', '᪙'), + ('ᪧ', 'ᪧ'), + ('\u{1ab0}', '\u{1abd}'), + ('\u{1abf}', '\u{1ace}'), + ('\u{1b00}', 'ᭌ'), + ('᭐', '᭙'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', '\u{1bf3}'), + ('ᰀ', '\u{1c37}'), + ('᱀', '᱉'), + ('ᱍ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', 'ᳺ'), + ('ᴀ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('\u{200c}', '\u{200d}'), + ('‿', '⁀'), + ('⁔', '⁔'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('\u{20d0}', '\u{20dc}'), + ('\u{20e1}', '\u{20e1}'), + ('\u{20e5}', '\u{20f0}'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('℘', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('\u{2d7f}', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('\u{2de0}', '\u{2dff}'), + ('々', '〇'), + ('〡', '\u{302f}'), + ('〱', '〵'), + ('〸', '〼'), + ('ぁ', 'ゖ'), + ('\u{3099}', '\u{309a}'), + ('ゝ', 'ゟ'), + ('ァ', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘫ'), + ('Ꙁ', '\u{a66f}'), + ('\u{a674}', '\u{a67d}'), + ('ꙿ', '\u{a6f1}'), + ('ꜗ', 'ꜟ'), + ('Ꜣ', 'ꞈ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠧ'), + ('\u{a82c}', '\u{a82c}'), + ('ꡀ', 'ꡳ'), + ('ꢀ', '\u{a8c5}'), + ('꣐', '꣙'), + ('\u{a8e0}', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', '\u{a92d}'), + ('ꤰ', '\u{a953}'), + ('ꥠ', 'ꥼ'), + ('\u{a980}', '\u{a9c0}'), + ('ꧏ', '꧙'), + ('ꧠ', 'ꧾ'), + ('ꨀ', '\u{aa36}'), + ('ꩀ', 'ꩍ'), + ('꩐', '꩙'), + ('ꩠ', 'ꩶ'), + ('ꩺ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫯ'), + ('ꫲ', '\u{aaf6}'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꯪ'), + ('꯬', '\u{abed}'), + ('꯰', '꯹'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﱝ'), + ('ﱤ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷹ'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('︳', '︴'), + ('﹍', '﹏'), + ('ﹱ', 'ﹱ'), + ('ﹳ', 'ﹳ'), + ('ﹷ', 'ﹷ'), + ('ﹹ', 'ﹹ'), + ('ﹻ', 'ﹻ'), + ('ﹽ', 'ﹽ'), + ('ﹿ', 'ﻼ'), + ('０', '９'), + ('Ａ', 'Ｚ'), + ('＿', '＿'), + ('ａ', 'ｚ'), + ('･', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('\u{101fd}', '\u{101fd}'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('\u{102e0}', '\u{102e0}'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '\u{1037a}'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐐀', '𐒝'), + ('𐒠', '𐒩'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '\u{10ae6}'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '\u{10d27}'), + ('𐴰', '𐴹'), + ('𐵀', '𐵥'), + ('\u{10d69}', '\u{10d6d}'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('\u{10eab}', '\u{10eac}'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('\u{10efc}', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '\u{10f50}'), + ('𐽰', '\u{10f85}'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀀', '\u{11046}'), + ('𑁦', '𑁵'), + ('\u{1107f}', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('𑃐', '𑃨'), + ('𑃰', '𑃹'), + ('\u{11100}', '\u{11134}'), + ('𑄶', '𑄿'), + ('𑅄', '𑅇'), + ('𑅐', '\u{11173}'), + ('𑅶', '𑅶'), + ('\u{11180}', '𑇄'), + ('\u{111c9}', '\u{111cc}'), + ('𑇎', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '\u{11237}'), + ('\u{1123e}', '\u{11241}'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '\u{112ea}'), + ('𑋰', '𑋹'), + ('\u{11300}', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('\u{1133b}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('𑍐', '𑍐'), + ('\u{11357}', '\u{11357}'), + ('𑍝', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏓'), + ('\u{113e1}', '\u{113e2}'), + ('𑐀', '𑑊'), + ('𑑐', '𑑙'), + ('\u{1145e}', '𑑡'), + ('𑒀', '𑓅'), + ('𑓇', '𑓇'), + ('𑓐', '𑓙'), + ('𑖀', '\u{115b5}'), + ('𑖸', '\u{115c0}'), + ('𑗘', '\u{115dd}'), + ('𑘀', '\u{11640}'), + ('𑙄', '𑙄'), + ('𑙐', '𑙙'), + ('𑚀', '𑚸'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜀', '𑜚'), + ('\u{1171d}', '\u{1172b}'), + ('𑜰', '𑜹'), + ('𑝀', '𑝆'), + ('𑠀', '\u{1183a}'), + ('𑢠', '𑣩'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{11943}'), + ('𑥐', '𑥙'), + ('𑦠', '𑦧'), + ('𑦪', '\u{119d7}'), + ('\u{119da}', '𑧡'), + ('𑧣', '𑧤'), + ('𑨀', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('𑩐', '\u{11a99}'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑯰', '𑯹'), + ('𑰀', '𑰈'), + ('𑰊', '\u{11c36}'), + ('\u{11c38}', '𑱀'), + ('𑱐', '𑱙'), + ('𑱲', '𑲏'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d47}'), + ('𑵐', '𑵙'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶘'), + ('𑶠', '𑶩'), + ('𑻠', '𑻶'), + ('\u{11f00}', '𑼐'), + ('𑼒', '\u{11f3a}'), + ('𑼾', '\u{11f42}'), + ('𑽐', '\u{11f5a}'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('\u{13440}', '\u{13455}'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄹'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩠', '𖩩'), + ('𖩰', '𖪾'), + ('𖫀', '𖫉'), + ('𖫐', '𖫭'), + ('\u{16af0}', '\u{16af4}'), + ('𖬀', '\u{16b36}'), + ('𖭀', '𖭃'), + ('𖭐', '𖭙'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖵰', '𖵹'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('\u{16f4f}', '𖾇'), + ('\u{16f8f}', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('𜳰', '𜳹'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝟎', '𝟿'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), + ('𞄀', '𞄬'), + ('\u{1e130}', '𞄽'), + ('𞅀', '𞅉'), + ('𞅎', '𞅎'), + ('𞊐', '\u{1e2ae}'), + ('𞋀', '𞋹'), + ('𞓐', '𞓹'), + ('𞗐', '𞗺'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('𞤀', '𞥋'), + ('𞥐', '𞥙'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('🯰', '🯹'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const XID_START: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ˁ'), + ('ˆ', 'ˑ'), + ('ˠ', 'ˤ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('Ͱ', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͻ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', 'ՙ'), + ('ՠ', 'ֈ'), + ('א', 'ת'), + ('ׯ', 'ײ'), + ('ؠ', 'ي'), + ('ٮ', 'ٯ'), + ('ٱ', 'ۓ'), + ('ە', 'ە'), + ('ۥ', 'ۦ'), + ('ۮ', 'ۯ'), + ('ۺ', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', 'ܐ'), + ('ܒ', 'ܯ'), + ('ݍ', 'ޥ'), + ('ޱ', 'ޱ'), + ('ߊ', 'ߪ'), + ('ߴ', 'ߵ'), + ('ߺ', 'ߺ'), + ('ࠀ', 'ࠕ'), + ('ࠚ', 'ࠚ'), + ('ࠤ', 'ࠤ'), + ('ࠨ', 'ࠨ'), + ('ࡀ', 'ࡘ'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('ࢠ', 'ࣉ'), + ('ऄ', 'ह'), + ('ऽ', 'ऽ'), + ('ॐ', 'ॐ'), + ('क़', 'ॡ'), + ('ॱ', 'ঀ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('ঽ', 'ঽ'), + ('ৎ', 'ৎ'), + ('ড়', 'ঢ়'), + ('য়', 'ৡ'), + ('ৰ', 'ৱ'), + ('ৼ', 'ৼ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('ੲ', 'ੴ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('ઽ', 'ઽ'), + ('ૐ', 'ૐ'), + ('ૠ', 'ૡ'), + ('ૹ', 'ૹ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('ଽ', 'ଽ'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୡ'), + ('ୱ', 'ୱ'), + ('ஃ', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('ௐ', 'ௐ'), + ('అ', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('ఽ', 'ఽ'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', 'ౡ'), + ('ಀ', 'ಀ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಽ', 'ಽ'), + ('ೝ', 'ೞ'), + ('ೠ', 'ೡ'), + ('ೱ', 'ೲ'), + ('ഄ', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ഺ'), + ('ഽ', 'ഽ'), + ('ൎ', 'ൎ'), + ('ൔ', 'ൖ'), + ('ൟ', 'ൡ'), + ('ൺ', 'ൿ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('ก', 'ะ'), + ('า', 'า'), + ('เ', 'ๆ'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ະ'), + ('າ', 'າ'), + ('ຽ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('ཀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('ྈ', 'ྌ'), + ('က', 'ဪ'), + ('ဿ', 'ဿ'), + ('ၐ', 'ၕ'), + ('ၚ', 'ၝ'), + ('ၡ', 'ၡ'), + ('ၥ', 'ၦ'), + ('ၮ', 'ၰ'), + ('ၵ', 'ႁ'), + ('ႎ', 'ႎ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', 'ᜑ'), + ('ᜟ', 'ᜱ'), + ('ᝀ', 'ᝑ'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('ក', 'ឳ'), + ('ៗ', 'ៗ'), + ('ៜ', 'ៜ'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢨ'), + ('ᢪ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('ᥐ', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('ᨀ', 'ᨖ'), + ('ᨠ', 'ᩔ'), + ('ᪧ', 'ᪧ'), + ('ᬅ', 'ᬳ'), + ('ᭅ', 'ᭌ'), + ('ᮃ', 'ᮠ'), + ('ᮮ', 'ᮯ'), + ('ᮺ', 'ᯥ'), + ('ᰀ', 'ᰣ'), + ('ᱍ', 'ᱏ'), + ('ᱚ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', 'ᳶ'), + ('ᳺ', 'ᳺ'), + ('ᴀ', 'ᶿ'), + ('Ḁ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('℘', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('々', '〇'), + ('〡', '〩'), + ('〱', '〵'), + ('〸', '〼'), + ('ぁ', 'ゖ'), + ('ゝ', 'ゟ'), + ('ァ', 'ヺ'), + ('ー', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘟ'), + ('ꘪ', 'ꘫ'), + ('Ꙁ', 'ꙮ'), + ('ꙿ', 'ꚝ'), + ('ꚠ', 'ꛯ'), + ('ꜗ', 'ꜟ'), + ('Ꜣ', 'ꞈ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠁ'), + ('ꠃ', 'ꠅ'), + ('ꠇ', 'ꠊ'), + ('ꠌ', 'ꠢ'), + ('ꡀ', 'ꡳ'), + ('ꢂ', 'ꢳ'), + ('ꣲ', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', 'ꣾ'), + ('ꤊ', 'ꤥ'), + ('ꤰ', 'ꥆ'), + ('ꥠ', 'ꥼ'), + ('ꦄ', 'ꦲ'), + ('ꧏ', 'ꧏ'), + ('ꧠ', 'ꧤ'), + ('ꧦ', 'ꧯ'), + ('ꧺ', 'ꧾ'), + ('ꨀ', 'ꨨ'), + ('ꩀ', 'ꩂ'), + ('ꩄ', 'ꩋ'), + ('ꩠ', 'ꩶ'), + ('ꩺ', 'ꩺ'), + ('ꩾ', 'ꪯ'), + ('ꪱ', 'ꪱ'), + ('ꪵ', 'ꪶ'), + ('ꪹ', 'ꪽ'), + ('ꫀ', 'ꫀ'), + ('ꫂ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫪ'), + ('ꫲ', 'ꫴ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꯢ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('יִ', 'יִ'), + ('ײַ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﱝ'), + ('ﱤ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷹ'), + ('ﹱ', 'ﹱ'), + ('ﹳ', 'ﹳ'), + ('ﹷ', 'ﹷ'), + ('ﹹ', 'ﹹ'), + ('ﹻ', 'ﹻ'), + ('ﹽ', 'ﹽ'), + ('ﹿ', 'ﻼ'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('ｦ', 'ﾝ'), + ('ﾠ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '𐍵'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐐀', '𐒝'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '𐨀'), + ('𐨐', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '𐫤'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '𐴣'), + ('𐵊', '𐵥'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('𐼀', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '𐽅'), + ('𐽰', '𐾁'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀃', '𑀷'), + ('𑁱', '𑁲'), + ('𑁵', '𑁵'), + ('𑂃', '𑂯'), + ('𑃐', '𑃨'), + ('𑄃', '𑄦'), + ('𑅄', '𑅄'), + ('𑅇', '𑅇'), + ('𑅐', '𑅲'), + ('𑅶', '𑅶'), + ('𑆃', '𑆲'), + ('𑇁', '𑇄'), + ('𑇚', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '𑈫'), + ('𑈿', '𑉀'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '𑋞'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('𑌽', '𑌽'), + ('𑍐', '𑍐'), + ('𑍝', '𑍡'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '𑎷'), + ('𑏑', '𑏑'), + ('𑏓', '𑏓'), + ('𑐀', '𑐴'), + ('𑑇', '𑑊'), + ('𑑟', '𑑡'), + ('𑒀', '𑒯'), + ('𑓄', '𑓅'), + ('𑓇', '𑓇'), + ('𑖀', '𑖮'), + ('𑗘', '𑗛'), + ('𑘀', '𑘯'), + ('𑙄', '𑙄'), + ('𑚀', '𑚪'), + ('𑚸', '𑚸'), + ('𑜀', '𑜚'), + ('𑝀', '𑝆'), + ('𑠀', '𑠫'), + ('𑢠', '𑣟'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤯'), + ('𑤿', '𑤿'), + ('𑥁', '𑥁'), + ('𑦠', '𑦧'), + ('𑦪', '𑧐'), + ('𑧡', '𑧡'), + ('𑧣', '𑧣'), + ('𑨀', '𑨀'), + ('𑨋', '𑨲'), + ('𑨺', '𑨺'), + ('𑩐', '𑩐'), + ('𑩜', '𑪉'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑰀', '𑰈'), + ('𑰊', '𑰮'), + ('𑱀', '𑱀'), + ('𑱲', '𑲏'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '𑴰'), + ('𑵆', '𑵆'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶉'), + ('𑶘', '𑶘'), + ('𑻠', '𑻲'), + ('𑼂', '𑼂'), + ('𑼄', '𑼐'), + ('𑼒', '𑼳'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('𓑁', '𓑆'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄝'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩰', '𖪾'), + ('𖫐', '𖫭'), + ('𖬀', '𖬯'), + ('𖭀', '𖭃'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('𖽐', '𖽐'), + ('𖾓', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '𖿣'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('𞄀', '𞄬'), + ('𞄷', '𞄽'), + ('𞅎', '𞅎'), + ('𞊐', '𞊭'), + ('𞋀', '𞋫'), + ('𞓐', '𞓫'), + ('𞗐', '𞗭'), + ('𞗰', '𞗰'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('𞤀', '𞥃'), + ('𞥋', '𞥋'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_names.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_names.rs new file mode 100644 index 0000000..a27b491 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_names.rs @@ -0,0 +1,281 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate property-names ucd-16.0.0 +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const PROPERTY_NAMES: &'static [(&'static str, &'static str)] = &[ + ("age", "Age"), + ("ahex", "ASCII_Hex_Digit"), + ("alpha", "Alphabetic"), + ("alphabetic", "Alphabetic"), + ("asciihexdigit", "ASCII_Hex_Digit"), + ("bc", "Bidi_Class"), + ("bidic", "Bidi_Control"), + ("bidiclass", "Bidi_Class"), + ("bidicontrol", "Bidi_Control"), + ("bidim", "Bidi_Mirrored"), + ("bidimirrored", "Bidi_Mirrored"), + ("bidimirroringglyph", "Bidi_Mirroring_Glyph"), + ("bidipairedbracket", "Bidi_Paired_Bracket"), + ("bidipairedbrackettype", "Bidi_Paired_Bracket_Type"), + ("blk", "Block"), + ("block", "Block"), + ("bmg", "Bidi_Mirroring_Glyph"), + ("bpb", "Bidi_Paired_Bracket"), + ("bpt", "Bidi_Paired_Bracket_Type"), + ("canonicalcombiningclass", "Canonical_Combining_Class"), + ("cased", "Cased"), + ("casefolding", "Case_Folding"), + ("caseignorable", "Case_Ignorable"), + ("ccc", "Canonical_Combining_Class"), + ("ce", "Composition_Exclusion"), + ("cf", "Case_Folding"), + ("changeswhencasefolded", "Changes_When_Casefolded"), + ("changeswhencasemapped", "Changes_When_Casemapped"), + ("changeswhenlowercased", "Changes_When_Lowercased"), + ("changeswhennfkccasefolded", "Changes_When_NFKC_Casefolded"), + ("changeswhentitlecased", "Changes_When_Titlecased"), + ("changeswhenuppercased", "Changes_When_Uppercased"), + ("ci", "Case_Ignorable"), + ("cjkaccountingnumeric", "kAccountingNumeric"), + ("cjkcompatibilityvariant", "kCompatibilityVariant"), + ("cjkiicore", "kIICore"), + ("cjkirggsource", "kIRG_GSource"), + ("cjkirghsource", "kIRG_HSource"), + ("cjkirgjsource", "kIRG_JSource"), + ("cjkirgkpsource", "kIRG_KPSource"), + ("cjkirgksource", "kIRG_KSource"), + ("cjkirgmsource", "kIRG_MSource"), + ("cjkirgssource", "kIRG_SSource"), + ("cjkirgtsource", "kIRG_TSource"), + ("cjkirguksource", "kIRG_UKSource"), + ("cjkirgusource", "kIRG_USource"), + ("cjkirgvsource", "kIRG_VSource"), + ("cjkothernumeric", "kOtherNumeric"), + ("cjkprimarynumeric", "kPrimaryNumeric"), + ("cjkrsunicode", "kRSUnicode"), + ("compex", "Full_Composition_Exclusion"), + ("compositionexclusion", "Composition_Exclusion"), + ("cwcf", "Changes_When_Casefolded"), + ("cwcm", "Changes_When_Casemapped"), + ("cwkcf", "Changes_When_NFKC_Casefolded"), + ("cwl", "Changes_When_Lowercased"), + ("cwt", "Changes_When_Titlecased"), + ("cwu", "Changes_When_Uppercased"), + ("dash", "Dash"), + ("decompositionmapping", "Decomposition_Mapping"), + ("decompositiontype", "Decomposition_Type"), + ("defaultignorablecodepoint", "Default_Ignorable_Code_Point"), + ("dep", "Deprecated"), + ("deprecated", "Deprecated"), + ("di", "Default_Ignorable_Code_Point"), + ("dia", "Diacritic"), + ("diacritic", "Diacritic"), + ("dm", "Decomposition_Mapping"), + ("dt", "Decomposition_Type"), + ("ea", "East_Asian_Width"), + ("eastasianwidth", "East_Asian_Width"), + ("ebase", "Emoji_Modifier_Base"), + ("ecomp", "Emoji_Component"), + ("emod", "Emoji_Modifier"), + ("emoji", "Emoji"), + ("emojicomponent", "Emoji_Component"), + ("emojimodifier", "Emoji_Modifier"), + ("emojimodifierbase", "Emoji_Modifier_Base"), + ("emojipresentation", "Emoji_Presentation"), + ("epres", "Emoji_Presentation"), + ("equideo", "Equivalent_Unified_Ideograph"), + ("equivalentunifiedideograph", "Equivalent_Unified_Ideograph"), + ("expandsonnfc", "Expands_On_NFC"), + ("expandsonnfd", "Expands_On_NFD"), + ("expandsonnfkc", "Expands_On_NFKC"), + ("expandsonnfkd", "Expands_On_NFKD"), + ("ext", "Extender"), + ("extendedpictographic", "Extended_Pictographic"), + ("extender", "Extender"), + ("extpict", "Extended_Pictographic"), + ("fcnfkc", "FC_NFKC_Closure"), + ("fcnfkcclosure", "FC_NFKC_Closure"), + ("fullcompositionexclusion", "Full_Composition_Exclusion"), + ("gc", "General_Category"), + ("gcb", "Grapheme_Cluster_Break"), + ("generalcategory", "General_Category"), + ("graphemebase", "Grapheme_Base"), + ("graphemeclusterbreak", "Grapheme_Cluster_Break"), + ("graphemeextend", "Grapheme_Extend"), + ("graphemelink", "Grapheme_Link"), + ("grbase", "Grapheme_Base"), + ("grext", "Grapheme_Extend"), + ("grlink", "Grapheme_Link"), + ("hangulsyllabletype", "Hangul_Syllable_Type"), + ("hex", "Hex_Digit"), + ("hexdigit", "Hex_Digit"), + ("hst", "Hangul_Syllable_Type"), + ("hyphen", "Hyphen"), + ("idc", "ID_Continue"), + ("idcompatmathcontinue", "ID_Compat_Math_Continue"), + ("idcompatmathstart", "ID_Compat_Math_Start"), + ("idcontinue", "ID_Continue"), + ("ideo", "Ideographic"), + ("ideographic", "Ideographic"), + ("ids", "ID_Start"), + ("idsb", "IDS_Binary_Operator"), + ("idsbinaryoperator", "IDS_Binary_Operator"), + ("idst", "IDS_Trinary_Operator"), + ("idstart", "ID_Start"), + ("idstrinaryoperator", "IDS_Trinary_Operator"), + ("idsu", "IDS_Unary_Operator"), + ("idsunaryoperator", "IDS_Unary_Operator"), + ("incb", "Indic_Conjunct_Break"), + ("indicconjunctbreak", "Indic_Conjunct_Break"), + ("indicpositionalcategory", "Indic_Positional_Category"), + ("indicsyllabiccategory", "Indic_Syllabic_Category"), + ("inpc", "Indic_Positional_Category"), + ("insc", "Indic_Syllabic_Category"), + ("isc", "ISO_Comment"), + ("jamoshortname", "Jamo_Short_Name"), + ("jg", "Joining_Group"), + ("joinc", "Join_Control"), + ("joincontrol", "Join_Control"), + ("joininggroup", "Joining_Group"), + ("joiningtype", "Joining_Type"), + ("jsn", "Jamo_Short_Name"), + ("jt", "Joining_Type"), + ("kaccountingnumeric", "kAccountingNumeric"), + ("kcompatibilityvariant", "kCompatibilityVariant"), + ("kehcat", "kEH_Cat"), + ("kehdesc", "kEH_Desc"), + ("kehhg", "kEH_HG"), + ("kehifao", "kEH_IFAO"), + ("kehjsesh", "kEH_JSesh"), + ("kehnomirror", "kEH_NoMirror"), + ("kehnorotate", "kEH_NoRotate"), + ("kiicore", "kIICore"), + ("kirggsource", "kIRG_GSource"), + ("kirghsource", "kIRG_HSource"), + ("kirgjsource", "kIRG_JSource"), + ("kirgkpsource", "kIRG_KPSource"), + ("kirgksource", "kIRG_KSource"), + ("kirgmsource", "kIRG_MSource"), + ("kirgssource", "kIRG_SSource"), + ("kirgtsource", "kIRG_TSource"), + ("kirguksource", "kIRG_UKSource"), + ("kirgusource", "kIRG_USource"), + ("kirgvsource", "kIRG_VSource"), + ("kothernumeric", "kOtherNumeric"), + ("kprimarynumeric", "kPrimaryNumeric"), + ("krsunicode", "kRSUnicode"), + ("lb", "Line_Break"), + ("lc", "Lowercase_Mapping"), + ("linebreak", "Line_Break"), + ("loe", "Logical_Order_Exception"), + ("logicalorderexception", "Logical_Order_Exception"), + ("lower", "Lowercase"), + ("lowercase", "Lowercase"), + ("lowercasemapping", "Lowercase_Mapping"), + ("math", "Math"), + ("mcm", "Modifier_Combining_Mark"), + ("modifiercombiningmark", "Modifier_Combining_Mark"), + ("na", "Name"), + ("na1", "Unicode_1_Name"), + ("name", "Name"), + ("namealias", "Name_Alias"), + ("nchar", "Noncharacter_Code_Point"), + ("nfcqc", "NFC_Quick_Check"), + ("nfcquickcheck", "NFC_Quick_Check"), + ("nfdqc", "NFD_Quick_Check"), + ("nfdquickcheck", "NFD_Quick_Check"), + ("nfkccasefold", "NFKC_Casefold"), + ("nfkccf", "NFKC_Casefold"), + ("nfkcqc", "NFKC_Quick_Check"), + ("nfkcquickcheck", "NFKC_Quick_Check"), + ("nfkcscf", "NFKC_Simple_Casefold"), + ("nfkcsimplecasefold", "NFKC_Simple_Casefold"), + ("nfkdqc", "NFKD_Quick_Check"), + ("nfkdquickcheck", "NFKD_Quick_Check"), + ("noncharactercodepoint", "Noncharacter_Code_Point"), + ("nt", "Numeric_Type"), + ("numerictype", "Numeric_Type"), + ("numericvalue", "Numeric_Value"), + ("nv", "Numeric_Value"), + ("oalpha", "Other_Alphabetic"), + ("ocomment", "ISO_Comment"), + ("odi", "Other_Default_Ignorable_Code_Point"), + ("ogrext", "Other_Grapheme_Extend"), + ("oidc", "Other_ID_Continue"), + ("oids", "Other_ID_Start"), + ("olower", "Other_Lowercase"), + ("omath", "Other_Math"), + ("otheralphabetic", "Other_Alphabetic"), + ("otherdefaultignorablecodepoint", "Other_Default_Ignorable_Code_Point"), + ("othergraphemeextend", "Other_Grapheme_Extend"), + ("otheridcontinue", "Other_ID_Continue"), + ("otheridstart", "Other_ID_Start"), + ("otherlowercase", "Other_Lowercase"), + ("othermath", "Other_Math"), + ("otheruppercase", "Other_Uppercase"), + ("oupper", "Other_Uppercase"), + ("patsyn", "Pattern_Syntax"), + ("patternsyntax", "Pattern_Syntax"), + ("patternwhitespace", "Pattern_White_Space"), + ("patws", "Pattern_White_Space"), + ("pcm", "Prepended_Concatenation_Mark"), + ("prependedconcatenationmark", "Prepended_Concatenation_Mark"), + ("qmark", "Quotation_Mark"), + ("quotationmark", "Quotation_Mark"), + ("radical", "Radical"), + ("regionalindicator", "Regional_Indicator"), + ("ri", "Regional_Indicator"), + ("sb", "Sentence_Break"), + ("sc", "Script"), + ("scf", "Simple_Case_Folding"), + ("script", "Script"), + ("scriptextensions", "Script_Extensions"), + ("scx", "Script_Extensions"), + ("sd", "Soft_Dotted"), + ("sentencebreak", "Sentence_Break"), + ("sentenceterminal", "Sentence_Terminal"), + ("sfc", "Simple_Case_Folding"), + ("simplecasefolding", "Simple_Case_Folding"), + ("simplelowercasemapping", "Simple_Lowercase_Mapping"), + ("simpletitlecasemapping", "Simple_Titlecase_Mapping"), + ("simpleuppercasemapping", "Simple_Uppercase_Mapping"), + ("slc", "Simple_Lowercase_Mapping"), + ("softdotted", "Soft_Dotted"), + ("space", "White_Space"), + ("stc", "Simple_Titlecase_Mapping"), + ("sterm", "Sentence_Terminal"), + ("suc", "Simple_Uppercase_Mapping"), + ("tc", "Titlecase_Mapping"), + ("term", "Terminal_Punctuation"), + ("terminalpunctuation", "Terminal_Punctuation"), + ("titlecasemapping", "Titlecase_Mapping"), + ("uc", "Uppercase_Mapping"), + ("uideo", "Unified_Ideograph"), + ("unicode1name", "Unicode_1_Name"), + ("unicoderadicalstroke", "kRSUnicode"), + ("unifiedideograph", "Unified_Ideograph"), + ("upper", "Uppercase"), + ("uppercase", "Uppercase"), + ("uppercasemapping", "Uppercase_Mapping"), + ("urs", "kRSUnicode"), + ("variationselector", "Variation_Selector"), + ("verticalorientation", "Vertical_Orientation"), + ("vo", "Vertical_Orientation"), + ("vs", "Variation_Selector"), + ("wb", "Word_Break"), + ("whitespace", "White_Space"), + ("wordbreak", "Word_Break"), + ("wspace", "White_Space"), + ("xidc", "XID_Continue"), + ("xidcontinue", "XID_Continue"), + ("xids", "XID_Start"), + ("xidstart", "XID_Start"), + ("xonfc", "Expands_On_NFC"), + ("xonfd", "Expands_On_NFD"), + ("xonfkc", "Expands_On_NFKC"), + ("xonfkd", "Expands_On_NFKD"), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_values.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_values.rs new file mode 100644 index 0000000..2270d66 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/property_values.rs @@ -0,0 +1,956 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate property-values ucd-16.0.0 --include gc,script,scx,age,gcb,wb,sb +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const PROPERTY_VALUES: &'static [( + &'static str, + &'static [(&'static str, &'static str)], +)] = &[ + ( + "Age", + &[ + ("1.1", "V1_1"), + ("10.0", "V10_0"), + ("11.0", "V11_0"), + ("12.0", "V12_0"), + ("12.1", "V12_1"), + ("13.0", "V13_0"), + ("14.0", "V14_0"), + ("15.0", "V15_0"), + ("15.1", "V15_1"), + ("16.0", "V16_0"), + ("2.0", "V2_0"), + ("2.1", "V2_1"), + ("3.0", "V3_0"), + ("3.1", "V3_1"), + ("3.2", "V3_2"), + ("4.0", "V4_0"), + ("4.1", "V4_1"), + ("5.0", "V5_0"), + ("5.1", "V5_1"), + ("5.2", "V5_2"), + ("6.0", "V6_0"), + ("6.1", "V6_1"), + ("6.2", "V6_2"), + ("6.3", "V6_3"), + ("7.0", "V7_0"), + ("8.0", "V8_0"), + ("9.0", "V9_0"), + ("na", "Unassigned"), + ("unassigned", "Unassigned"), + ("v100", "V10_0"), + ("v11", "V1_1"), + ("v110", "V11_0"), + ("v120", "V12_0"), + ("v121", "V12_1"), + ("v130", "V13_0"), + ("v140", "V14_0"), + ("v150", "V15_0"), + ("v151", "V15_1"), + ("v160", "V16_0"), + ("v20", "V2_0"), + ("v21", "V2_1"), + ("v30", "V3_0"), + ("v31", "V3_1"), + ("v32", "V3_2"), + ("v40", "V4_0"), + ("v41", "V4_1"), + ("v50", "V5_0"), + ("v51", "V5_1"), + ("v52", "V5_2"), + ("v60", "V6_0"), + ("v61", "V6_1"), + ("v62", "V6_2"), + ("v63", "V6_3"), + ("v70", "V7_0"), + ("v80", "V8_0"), + ("v90", "V9_0"), + ], + ), + ( + "General_Category", + &[ + ("c", "Other"), + ("casedletter", "Cased_Letter"), + ("cc", "Control"), + ("cf", "Format"), + ("closepunctuation", "Close_Punctuation"), + ("cn", "Unassigned"), + ("cntrl", "Control"), + ("co", "Private_Use"), + ("combiningmark", "Mark"), + ("connectorpunctuation", "Connector_Punctuation"), + ("control", "Control"), + ("cs", "Surrogate"), + ("currencysymbol", "Currency_Symbol"), + ("dashpunctuation", "Dash_Punctuation"), + ("decimalnumber", "Decimal_Number"), + ("digit", "Decimal_Number"), + ("enclosingmark", "Enclosing_Mark"), + ("finalpunctuation", "Final_Punctuation"), + ("format", "Format"), + ("initialpunctuation", "Initial_Punctuation"), + ("l", "Letter"), + ("lc", "Cased_Letter"), + ("letter", "Letter"), + ("letternumber", "Letter_Number"), + ("lineseparator", "Line_Separator"), + ("ll", "Lowercase_Letter"), + ("lm", "Modifier_Letter"), + ("lo", "Other_Letter"), + ("lowercaseletter", "Lowercase_Letter"), + ("lt", "Titlecase_Letter"), + ("lu", "Uppercase_Letter"), + ("m", "Mark"), + ("mark", "Mark"), + ("mathsymbol", "Math_Symbol"), + ("mc", "Spacing_Mark"), + ("me", "Enclosing_Mark"), + ("mn", "Nonspacing_Mark"), + ("modifierletter", "Modifier_Letter"), + ("modifiersymbol", "Modifier_Symbol"), + ("n", "Number"), + ("nd", "Decimal_Number"), + ("nl", "Letter_Number"), + ("no", "Other_Number"), + ("nonspacingmark", "Nonspacing_Mark"), + ("number", "Number"), + ("openpunctuation", "Open_Punctuation"), + ("other", "Other"), + ("otherletter", "Other_Letter"), + ("othernumber", "Other_Number"), + ("otherpunctuation", "Other_Punctuation"), + ("othersymbol", "Other_Symbol"), + ("p", "Punctuation"), + ("paragraphseparator", "Paragraph_Separator"), + ("pc", "Connector_Punctuation"), + ("pd", "Dash_Punctuation"), + ("pe", "Close_Punctuation"), + ("pf", "Final_Punctuation"), + ("pi", "Initial_Punctuation"), + ("po", "Other_Punctuation"), + ("privateuse", "Private_Use"), + ("ps", "Open_Punctuation"), + ("punct", "Punctuation"), + ("punctuation", "Punctuation"), + ("s", "Symbol"), + ("sc", "Currency_Symbol"), + ("separator", "Separator"), + ("sk", "Modifier_Symbol"), + ("sm", "Math_Symbol"), + ("so", "Other_Symbol"), + ("spaceseparator", "Space_Separator"), + ("spacingmark", "Spacing_Mark"), + ("surrogate", "Surrogate"), + ("symbol", "Symbol"), + ("titlecaseletter", "Titlecase_Letter"), + ("unassigned", "Unassigned"), + ("uppercaseletter", "Uppercase_Letter"), + ("z", "Separator"), + ("zl", "Line_Separator"), + ("zp", "Paragraph_Separator"), + ("zs", "Space_Separator"), + ], + ), + ( + "Grapheme_Cluster_Break", + &[ + ("cn", "Control"), + ("control", "Control"), + ("cr", "CR"), + ("eb", "E_Base"), + ("ebase", "E_Base"), + ("ebasegaz", "E_Base_GAZ"), + ("ebg", "E_Base_GAZ"), + ("em", "E_Modifier"), + ("emodifier", "E_Modifier"), + ("ex", "Extend"), + ("extend", "Extend"), + ("gaz", "Glue_After_Zwj"), + ("glueafterzwj", "Glue_After_Zwj"), + ("l", "L"), + ("lf", "LF"), + ("lv", "LV"), + ("lvt", "LVT"), + ("other", "Other"), + ("pp", "Prepend"), + ("prepend", "Prepend"), + ("regionalindicator", "Regional_Indicator"), + ("ri", "Regional_Indicator"), + ("sm", "SpacingMark"), + ("spacingmark", "SpacingMark"), + ("t", "T"), + ("v", "V"), + ("xx", "Other"), + ("zwj", "ZWJ"), + ], + ), + ( + "Script", + &[ + ("adlam", "Adlam"), + ("adlm", "Adlam"), + ("aghb", "Caucasian_Albanian"), + ("ahom", "Ahom"), + ("anatolianhieroglyphs", "Anatolian_Hieroglyphs"), + ("arab", "Arabic"), + ("arabic", "Arabic"), + ("armenian", "Armenian"), + ("armi", "Imperial_Aramaic"), + ("armn", "Armenian"), + ("avestan", "Avestan"), + ("avst", "Avestan"), + ("bali", "Balinese"), + ("balinese", "Balinese"), + ("bamu", "Bamum"), + ("bamum", "Bamum"), + ("bass", "Bassa_Vah"), + ("bassavah", "Bassa_Vah"), + ("batak", "Batak"), + ("batk", "Batak"), + ("beng", "Bengali"), + ("bengali", "Bengali"), + ("bhaiksuki", "Bhaiksuki"), + ("bhks", "Bhaiksuki"), + ("bopo", "Bopomofo"), + ("bopomofo", "Bopomofo"), + ("brah", "Brahmi"), + ("brahmi", "Brahmi"), + ("brai", "Braille"), + ("braille", "Braille"), + ("bugi", "Buginese"), + ("buginese", "Buginese"), + ("buhd", "Buhid"), + ("buhid", "Buhid"), + ("cakm", "Chakma"), + ("canadianaboriginal", "Canadian_Aboriginal"), + ("cans", "Canadian_Aboriginal"), + ("cari", "Carian"), + ("carian", "Carian"), + ("caucasianalbanian", "Caucasian_Albanian"), + ("chakma", "Chakma"), + ("cham", "Cham"), + ("cher", "Cherokee"), + ("cherokee", "Cherokee"), + ("chorasmian", "Chorasmian"), + ("chrs", "Chorasmian"), + ("common", "Common"), + ("copt", "Coptic"), + ("coptic", "Coptic"), + ("cpmn", "Cypro_Minoan"), + ("cprt", "Cypriot"), + ("cuneiform", "Cuneiform"), + ("cypriot", "Cypriot"), + ("cyprominoan", "Cypro_Minoan"), + ("cyrillic", "Cyrillic"), + ("cyrl", "Cyrillic"), + ("deseret", "Deseret"), + ("deva", "Devanagari"), + ("devanagari", "Devanagari"), + ("diak", "Dives_Akuru"), + ("divesakuru", "Dives_Akuru"), + ("dogr", "Dogra"), + ("dogra", "Dogra"), + ("dsrt", "Deseret"), + ("dupl", "Duployan"), + ("duployan", "Duployan"), + ("egyp", "Egyptian_Hieroglyphs"), + ("egyptianhieroglyphs", "Egyptian_Hieroglyphs"), + ("elba", "Elbasan"), + ("elbasan", "Elbasan"), + ("elym", "Elymaic"), + ("elymaic", "Elymaic"), + ("ethi", "Ethiopic"), + ("ethiopic", "Ethiopic"), + ("gara", "Garay"), + ("garay", "Garay"), + ("geor", "Georgian"), + ("georgian", "Georgian"), + ("glag", "Glagolitic"), + ("glagolitic", "Glagolitic"), + ("gong", "Gunjala_Gondi"), + ("gonm", "Masaram_Gondi"), + ("goth", "Gothic"), + ("gothic", "Gothic"), + ("gran", "Grantha"), + ("grantha", "Grantha"), + ("greek", "Greek"), + ("grek", "Greek"), + ("gujarati", "Gujarati"), + ("gujr", "Gujarati"), + ("gukh", "Gurung_Khema"), + ("gunjalagondi", "Gunjala_Gondi"), + ("gurmukhi", "Gurmukhi"), + ("guru", "Gurmukhi"), + ("gurungkhema", "Gurung_Khema"), + ("han", "Han"), + ("hang", "Hangul"), + ("hangul", "Hangul"), + ("hani", "Han"), + ("hanifirohingya", "Hanifi_Rohingya"), + ("hano", "Hanunoo"), + ("hanunoo", "Hanunoo"), + ("hatr", "Hatran"), + ("hatran", "Hatran"), + ("hebr", "Hebrew"), + ("hebrew", "Hebrew"), + ("hira", "Hiragana"), + ("hiragana", "Hiragana"), + ("hluw", "Anatolian_Hieroglyphs"), + ("hmng", "Pahawh_Hmong"), + ("hmnp", "Nyiakeng_Puachue_Hmong"), + ("hrkt", "Katakana_Or_Hiragana"), + ("hung", "Old_Hungarian"), + ("imperialaramaic", "Imperial_Aramaic"), + ("inherited", "Inherited"), + ("inscriptionalpahlavi", "Inscriptional_Pahlavi"), + ("inscriptionalparthian", "Inscriptional_Parthian"), + ("ital", "Old_Italic"), + ("java", "Javanese"), + ("javanese", "Javanese"), + ("kaithi", "Kaithi"), + ("kali", "Kayah_Li"), + ("kana", "Katakana"), + ("kannada", "Kannada"), + ("katakana", "Katakana"), + ("katakanaorhiragana", "Katakana_Or_Hiragana"), + ("kawi", "Kawi"), + ("kayahli", "Kayah_Li"), + ("khar", "Kharoshthi"), + ("kharoshthi", "Kharoshthi"), + ("khitansmallscript", "Khitan_Small_Script"), + ("khmer", "Khmer"), + ("khmr", "Khmer"), + ("khoj", "Khojki"), + ("khojki", "Khojki"), + ("khudawadi", "Khudawadi"), + ("kiratrai", "Kirat_Rai"), + ("kits", "Khitan_Small_Script"), + ("knda", "Kannada"), + ("krai", "Kirat_Rai"), + ("kthi", "Kaithi"), + ("lana", "Tai_Tham"), + ("lao", "Lao"), + ("laoo", "Lao"), + ("latin", "Latin"), + ("latn", "Latin"), + ("lepc", "Lepcha"), + ("lepcha", "Lepcha"), + ("limb", "Limbu"), + ("limbu", "Limbu"), + ("lina", "Linear_A"), + ("linb", "Linear_B"), + ("lineara", "Linear_A"), + ("linearb", "Linear_B"), + ("lisu", "Lisu"), + ("lyci", "Lycian"), + ("lycian", "Lycian"), + ("lydi", "Lydian"), + ("lydian", "Lydian"), + ("mahajani", "Mahajani"), + ("mahj", "Mahajani"), + ("maka", "Makasar"), + ("makasar", "Makasar"), + ("malayalam", "Malayalam"), + ("mand", "Mandaic"), + ("mandaic", "Mandaic"), + ("mani", "Manichaean"), + ("manichaean", "Manichaean"), + ("marc", "Marchen"), + ("marchen", "Marchen"), + ("masaramgondi", "Masaram_Gondi"), + ("medefaidrin", "Medefaidrin"), + ("medf", "Medefaidrin"), + ("meeteimayek", "Meetei_Mayek"), + ("mend", "Mende_Kikakui"), + ("mendekikakui", "Mende_Kikakui"), + ("merc", "Meroitic_Cursive"), + ("mero", "Meroitic_Hieroglyphs"), + ("meroiticcursive", "Meroitic_Cursive"), + ("meroitichieroglyphs", "Meroitic_Hieroglyphs"), + ("miao", "Miao"), + ("mlym", "Malayalam"), + ("modi", "Modi"), + ("mong", "Mongolian"), + ("mongolian", "Mongolian"), + ("mro", "Mro"), + ("mroo", "Mro"), + ("mtei", "Meetei_Mayek"), + ("mult", "Multani"), + ("multani", "Multani"), + ("myanmar", "Myanmar"), + ("mymr", "Myanmar"), + ("nabataean", "Nabataean"), + ("nagm", "Nag_Mundari"), + ("nagmundari", "Nag_Mundari"), + ("nand", "Nandinagari"), + ("nandinagari", "Nandinagari"), + ("narb", "Old_North_Arabian"), + ("nbat", "Nabataean"), + ("newa", "Newa"), + ("newtailue", "New_Tai_Lue"), + ("nko", "Nko"), + ("nkoo", "Nko"), + ("nshu", "Nushu"), + ("nushu", "Nushu"), + ("nyiakengpuachuehmong", "Nyiakeng_Puachue_Hmong"), + ("ogam", "Ogham"), + ("ogham", "Ogham"), + ("olchiki", "Ol_Chiki"), + ("olck", "Ol_Chiki"), + ("oldhungarian", "Old_Hungarian"), + ("olditalic", "Old_Italic"), + ("oldnortharabian", "Old_North_Arabian"), + ("oldpermic", "Old_Permic"), + ("oldpersian", "Old_Persian"), + ("oldsogdian", "Old_Sogdian"), + ("oldsoutharabian", "Old_South_Arabian"), + ("oldturkic", "Old_Turkic"), + ("olduyghur", "Old_Uyghur"), + ("olonal", "Ol_Onal"), + ("onao", "Ol_Onal"), + ("oriya", "Oriya"), + ("orkh", "Old_Turkic"), + ("orya", "Oriya"), + ("osage", "Osage"), + ("osge", "Osage"), + ("osma", "Osmanya"), + ("osmanya", "Osmanya"), + ("ougr", "Old_Uyghur"), + ("pahawhhmong", "Pahawh_Hmong"), + ("palm", "Palmyrene"), + ("palmyrene", "Palmyrene"), + ("pauc", "Pau_Cin_Hau"), + ("paucinhau", "Pau_Cin_Hau"), + ("perm", "Old_Permic"), + ("phag", "Phags_Pa"), + ("phagspa", "Phags_Pa"), + ("phli", "Inscriptional_Pahlavi"), + ("phlp", "Psalter_Pahlavi"), + ("phnx", "Phoenician"), + ("phoenician", "Phoenician"), + ("plrd", "Miao"), + ("prti", "Inscriptional_Parthian"), + ("psalterpahlavi", "Psalter_Pahlavi"), + ("qaac", "Coptic"), + ("qaai", "Inherited"), + ("rejang", "Rejang"), + ("rjng", "Rejang"), + ("rohg", "Hanifi_Rohingya"), + ("runic", "Runic"), + ("runr", "Runic"), + ("samaritan", "Samaritan"), + ("samr", "Samaritan"), + ("sarb", "Old_South_Arabian"), + ("saur", "Saurashtra"), + ("saurashtra", "Saurashtra"), + ("sgnw", "SignWriting"), + ("sharada", "Sharada"), + ("shavian", "Shavian"), + ("shaw", "Shavian"), + ("shrd", "Sharada"), + ("sidd", "Siddham"), + ("siddham", "Siddham"), + ("signwriting", "SignWriting"), + ("sind", "Khudawadi"), + ("sinh", "Sinhala"), + ("sinhala", "Sinhala"), + ("sogd", "Sogdian"), + ("sogdian", "Sogdian"), + ("sogo", "Old_Sogdian"), + ("sora", "Sora_Sompeng"), + ("sorasompeng", "Sora_Sompeng"), + ("soyo", "Soyombo"), + ("soyombo", "Soyombo"), + ("sund", "Sundanese"), + ("sundanese", "Sundanese"), + ("sunu", "Sunuwar"), + ("sunuwar", "Sunuwar"), + ("sylo", "Syloti_Nagri"), + ("sylotinagri", "Syloti_Nagri"), + ("syrc", "Syriac"), + ("syriac", "Syriac"), + ("tagalog", "Tagalog"), + ("tagb", "Tagbanwa"), + ("tagbanwa", "Tagbanwa"), + ("taile", "Tai_Le"), + ("taitham", "Tai_Tham"), + ("taiviet", "Tai_Viet"), + ("takr", "Takri"), + ("takri", "Takri"), + ("tale", "Tai_Le"), + ("talu", "New_Tai_Lue"), + ("tamil", "Tamil"), + ("taml", "Tamil"), + ("tang", "Tangut"), + ("tangsa", "Tangsa"), + ("tangut", "Tangut"), + ("tavt", "Tai_Viet"), + ("telu", "Telugu"), + ("telugu", "Telugu"), + ("tfng", "Tifinagh"), + ("tglg", "Tagalog"), + ("thaa", "Thaana"), + ("thaana", "Thaana"), + ("thai", "Thai"), + ("tibetan", "Tibetan"), + ("tibt", "Tibetan"), + ("tifinagh", "Tifinagh"), + ("tirh", "Tirhuta"), + ("tirhuta", "Tirhuta"), + ("tnsa", "Tangsa"), + ("todhri", "Todhri"), + ("todr", "Todhri"), + ("toto", "Toto"), + ("tulutigalari", "Tulu_Tigalari"), + ("tutg", "Tulu_Tigalari"), + ("ugar", "Ugaritic"), + ("ugaritic", "Ugaritic"), + ("unknown", "Unknown"), + ("vai", "Vai"), + ("vaii", "Vai"), + ("vith", "Vithkuqi"), + ("vithkuqi", "Vithkuqi"), + ("wancho", "Wancho"), + ("wara", "Warang_Citi"), + ("warangciti", "Warang_Citi"), + ("wcho", "Wancho"), + ("xpeo", "Old_Persian"), + ("xsux", "Cuneiform"), + ("yezi", "Yezidi"), + ("yezidi", "Yezidi"), + ("yi", "Yi"), + ("yiii", "Yi"), + ("zanabazarsquare", "Zanabazar_Square"), + ("zanb", "Zanabazar_Square"), + ("zinh", "Inherited"), + ("zyyy", "Common"), + ("zzzz", "Unknown"), + ], + ), + ( + "Script_Extensions", + &[ + ("adlam", "Adlam"), + ("adlm", "Adlam"), + ("aghb", "Caucasian_Albanian"), + ("ahom", "Ahom"), + ("anatolianhieroglyphs", "Anatolian_Hieroglyphs"), + ("arab", "Arabic"), + ("arabic", "Arabic"), + ("armenian", "Armenian"), + ("armi", "Imperial_Aramaic"), + ("armn", "Armenian"), + ("avestan", "Avestan"), + ("avst", "Avestan"), + ("bali", "Balinese"), + ("balinese", "Balinese"), + ("bamu", "Bamum"), + ("bamum", "Bamum"), + ("bass", "Bassa_Vah"), + ("bassavah", "Bassa_Vah"), + ("batak", "Batak"), + ("batk", "Batak"), + ("beng", "Bengali"), + ("bengali", "Bengali"), + ("bhaiksuki", "Bhaiksuki"), + ("bhks", "Bhaiksuki"), + ("bopo", "Bopomofo"), + ("bopomofo", "Bopomofo"), + ("brah", "Brahmi"), + ("brahmi", "Brahmi"), + ("brai", "Braille"), + ("braille", "Braille"), + ("bugi", "Buginese"), + ("buginese", "Buginese"), + ("buhd", "Buhid"), + ("buhid", "Buhid"), + ("cakm", "Chakma"), + ("canadianaboriginal", "Canadian_Aboriginal"), + ("cans", "Canadian_Aboriginal"), + ("cari", "Carian"), + ("carian", "Carian"), + ("caucasianalbanian", "Caucasian_Albanian"), + ("chakma", "Chakma"), + ("cham", "Cham"), + ("cher", "Cherokee"), + ("cherokee", "Cherokee"), + ("chorasmian", "Chorasmian"), + ("chrs", "Chorasmian"), + ("common", "Common"), + ("copt", "Coptic"), + ("coptic", "Coptic"), + ("cpmn", "Cypro_Minoan"), + ("cprt", "Cypriot"), + ("cuneiform", "Cuneiform"), + ("cypriot", "Cypriot"), + ("cyprominoan", "Cypro_Minoan"), + ("cyrillic", "Cyrillic"), + ("cyrl", "Cyrillic"), + ("deseret", "Deseret"), + ("deva", "Devanagari"), + ("devanagari", "Devanagari"), + ("diak", "Dives_Akuru"), + ("divesakuru", "Dives_Akuru"), + ("dogr", "Dogra"), + ("dogra", "Dogra"), + ("dsrt", "Deseret"), + ("dupl", "Duployan"), + ("duployan", "Duployan"), + ("egyp", "Egyptian_Hieroglyphs"), + ("egyptianhieroglyphs", "Egyptian_Hieroglyphs"), + ("elba", "Elbasan"), + ("elbasan", "Elbasan"), + ("elym", "Elymaic"), + ("elymaic", "Elymaic"), + ("ethi", "Ethiopic"), + ("ethiopic", "Ethiopic"), + ("gara", "Garay"), + ("garay", "Garay"), + ("geor", "Georgian"), + ("georgian", "Georgian"), + ("glag", "Glagolitic"), + ("glagolitic", "Glagolitic"), + ("gong", "Gunjala_Gondi"), + ("gonm", "Masaram_Gondi"), + ("goth", "Gothic"), + ("gothic", "Gothic"), + ("gran", "Grantha"), + ("grantha", "Grantha"), + ("greek", "Greek"), + ("grek", "Greek"), + ("gujarati", "Gujarati"), + ("gujr", "Gujarati"), + ("gukh", "Gurung_Khema"), + ("gunjalagondi", "Gunjala_Gondi"), + ("gurmukhi", "Gurmukhi"), + ("guru", "Gurmukhi"), + ("gurungkhema", "Gurung_Khema"), + ("han", "Han"), + ("hang", "Hangul"), + ("hangul", "Hangul"), + ("hani", "Han"), + ("hanifirohingya", "Hanifi_Rohingya"), + ("hano", "Hanunoo"), + ("hanunoo", "Hanunoo"), + ("hatr", "Hatran"), + ("hatran", "Hatran"), + ("hebr", "Hebrew"), + ("hebrew", "Hebrew"), + ("hira", "Hiragana"), + ("hiragana", "Hiragana"), + ("hluw", "Anatolian_Hieroglyphs"), + ("hmng", "Pahawh_Hmong"), + ("hmnp", "Nyiakeng_Puachue_Hmong"), + ("hrkt", "Katakana_Or_Hiragana"), + ("hung", "Old_Hungarian"), + ("imperialaramaic", "Imperial_Aramaic"), + ("inherited", "Inherited"), + ("inscriptionalpahlavi", "Inscriptional_Pahlavi"), + ("inscriptionalparthian", "Inscriptional_Parthian"), + ("ital", "Old_Italic"), + ("java", "Javanese"), + ("javanese", "Javanese"), + ("kaithi", "Kaithi"), + ("kali", "Kayah_Li"), + ("kana", "Katakana"), + ("kannada", "Kannada"), + ("katakana", "Katakana"), + ("katakanaorhiragana", "Katakana_Or_Hiragana"), + ("kawi", "Kawi"), + ("kayahli", "Kayah_Li"), + ("khar", "Kharoshthi"), + ("kharoshthi", "Kharoshthi"), + ("khitansmallscript", "Khitan_Small_Script"), + ("khmer", "Khmer"), + ("khmr", "Khmer"), + ("khoj", "Khojki"), + ("khojki", "Khojki"), + ("khudawadi", "Khudawadi"), + ("kiratrai", "Kirat_Rai"), + ("kits", "Khitan_Small_Script"), + ("knda", "Kannada"), + ("krai", "Kirat_Rai"), + ("kthi", "Kaithi"), + ("lana", "Tai_Tham"), + ("lao", "Lao"), + ("laoo", "Lao"), + ("latin", "Latin"), + ("latn", "Latin"), + ("lepc", "Lepcha"), + ("lepcha", "Lepcha"), + ("limb", "Limbu"), + ("limbu", "Limbu"), + ("lina", "Linear_A"), + ("linb", "Linear_B"), + ("lineara", "Linear_A"), + ("linearb", "Linear_B"), + ("lisu", "Lisu"), + ("lyci", "Lycian"), + ("lycian", "Lycian"), + ("lydi", "Lydian"), + ("lydian", "Lydian"), + ("mahajani", "Mahajani"), + ("mahj", "Mahajani"), + ("maka", "Makasar"), + ("makasar", "Makasar"), + ("malayalam", "Malayalam"), + ("mand", "Mandaic"), + ("mandaic", "Mandaic"), + ("mani", "Manichaean"), + ("manichaean", "Manichaean"), + ("marc", "Marchen"), + ("marchen", "Marchen"), + ("masaramgondi", "Masaram_Gondi"), + ("medefaidrin", "Medefaidrin"), + ("medf", "Medefaidrin"), + ("meeteimayek", "Meetei_Mayek"), + ("mend", "Mende_Kikakui"), + ("mendekikakui", "Mende_Kikakui"), + ("merc", "Meroitic_Cursive"), + ("mero", "Meroitic_Hieroglyphs"), + ("meroiticcursive", "Meroitic_Cursive"), + ("meroitichieroglyphs", "Meroitic_Hieroglyphs"), + ("miao", "Miao"), + ("mlym", "Malayalam"), + ("modi", "Modi"), + ("mong", "Mongolian"), + ("mongolian", "Mongolian"), + ("mro", "Mro"), + ("mroo", "Mro"), + ("mtei", "Meetei_Mayek"), + ("mult", "Multani"), + ("multani", "Multani"), + ("myanmar", "Myanmar"), + ("mymr", "Myanmar"), + ("nabataean", "Nabataean"), + ("nagm", "Nag_Mundari"), + ("nagmundari", "Nag_Mundari"), + ("nand", "Nandinagari"), + ("nandinagari", "Nandinagari"), + ("narb", "Old_North_Arabian"), + ("nbat", "Nabataean"), + ("newa", "Newa"), + ("newtailue", "New_Tai_Lue"), + ("nko", "Nko"), + ("nkoo", "Nko"), + ("nshu", "Nushu"), + ("nushu", "Nushu"), + ("nyiakengpuachuehmong", "Nyiakeng_Puachue_Hmong"), + ("ogam", "Ogham"), + ("ogham", "Ogham"), + ("olchiki", "Ol_Chiki"), + ("olck", "Ol_Chiki"), + ("oldhungarian", "Old_Hungarian"), + ("olditalic", "Old_Italic"), + ("oldnortharabian", "Old_North_Arabian"), + ("oldpermic", "Old_Permic"), + ("oldpersian", "Old_Persian"), + ("oldsogdian", "Old_Sogdian"), + ("oldsoutharabian", "Old_South_Arabian"), + ("oldturkic", "Old_Turkic"), + ("olduyghur", "Old_Uyghur"), + ("olonal", "Ol_Onal"), + ("onao", "Ol_Onal"), + ("oriya", "Oriya"), + ("orkh", "Old_Turkic"), + ("orya", "Oriya"), + ("osage", "Osage"), + ("osge", "Osage"), + ("osma", "Osmanya"), + ("osmanya", "Osmanya"), + ("ougr", "Old_Uyghur"), + ("pahawhhmong", "Pahawh_Hmong"), + ("palm", "Palmyrene"), + ("palmyrene", "Palmyrene"), + ("pauc", "Pau_Cin_Hau"), + ("paucinhau", "Pau_Cin_Hau"), + ("perm", "Old_Permic"), + ("phag", "Phags_Pa"), + ("phagspa", "Phags_Pa"), + ("phli", "Inscriptional_Pahlavi"), + ("phlp", "Psalter_Pahlavi"), + ("phnx", "Phoenician"), + ("phoenician", "Phoenician"), + ("plrd", "Miao"), + ("prti", "Inscriptional_Parthian"), + ("psalterpahlavi", "Psalter_Pahlavi"), + ("qaac", "Coptic"), + ("qaai", "Inherited"), + ("rejang", "Rejang"), + ("rjng", "Rejang"), + ("rohg", "Hanifi_Rohingya"), + ("runic", "Runic"), + ("runr", "Runic"), + ("samaritan", "Samaritan"), + ("samr", "Samaritan"), + ("sarb", "Old_South_Arabian"), + ("saur", "Saurashtra"), + ("saurashtra", "Saurashtra"), + ("sgnw", "SignWriting"), + ("sharada", "Sharada"), + ("shavian", "Shavian"), + ("shaw", "Shavian"), + ("shrd", "Sharada"), + ("sidd", "Siddham"), + ("siddham", "Siddham"), + ("signwriting", "SignWriting"), + ("sind", "Khudawadi"), + ("sinh", "Sinhala"), + ("sinhala", "Sinhala"), + ("sogd", "Sogdian"), + ("sogdian", "Sogdian"), + ("sogo", "Old_Sogdian"), + ("sora", "Sora_Sompeng"), + ("sorasompeng", "Sora_Sompeng"), + ("soyo", "Soyombo"), + ("soyombo", "Soyombo"), + ("sund", "Sundanese"), + ("sundanese", "Sundanese"), + ("sunu", "Sunuwar"), + ("sunuwar", "Sunuwar"), + ("sylo", "Syloti_Nagri"), + ("sylotinagri", "Syloti_Nagri"), + ("syrc", "Syriac"), + ("syriac", "Syriac"), + ("tagalog", "Tagalog"), + ("tagb", "Tagbanwa"), + ("tagbanwa", "Tagbanwa"), + ("taile", "Tai_Le"), + ("taitham", "Tai_Tham"), + ("taiviet", "Tai_Viet"), + ("takr", "Takri"), + ("takri", "Takri"), + ("tale", "Tai_Le"), + ("talu", "New_Tai_Lue"), + ("tamil", "Tamil"), + ("taml", "Tamil"), + ("tang", "Tangut"), + ("tangsa", "Tangsa"), + ("tangut", "Tangut"), + ("tavt", "Tai_Viet"), + ("telu", "Telugu"), + ("telugu", "Telugu"), + ("tfng", "Tifinagh"), + ("tglg", "Tagalog"), + ("thaa", "Thaana"), + ("thaana", "Thaana"), + ("thai", "Thai"), + ("tibetan", "Tibetan"), + ("tibt", "Tibetan"), + ("tifinagh", "Tifinagh"), + ("tirh", "Tirhuta"), + ("tirhuta", "Tirhuta"), + ("tnsa", "Tangsa"), + ("todhri", "Todhri"), + ("todr", "Todhri"), + ("toto", "Toto"), + ("tulutigalari", "Tulu_Tigalari"), + ("tutg", "Tulu_Tigalari"), + ("ugar", "Ugaritic"), + ("ugaritic", "Ugaritic"), + ("unknown", "Unknown"), + ("vai", "Vai"), + ("vaii", "Vai"), + ("vith", "Vithkuqi"), + ("vithkuqi", "Vithkuqi"), + ("wancho", "Wancho"), + ("wara", "Warang_Citi"), + ("warangciti", "Warang_Citi"), + ("wcho", "Wancho"), + ("xpeo", "Old_Persian"), + ("xsux", "Cuneiform"), + ("yezi", "Yezidi"), + ("yezidi", "Yezidi"), + ("yi", "Yi"), + ("yiii", "Yi"), + ("zanabazarsquare", "Zanabazar_Square"), + ("zanb", "Zanabazar_Square"), + ("zinh", "Inherited"), + ("zyyy", "Common"), + ("zzzz", "Unknown"), + ], + ), + ( + "Sentence_Break", + &[ + ("at", "ATerm"), + ("aterm", "ATerm"), + ("cl", "Close"), + ("close", "Close"), + ("cr", "CR"), + ("ex", "Extend"), + ("extend", "Extend"), + ("fo", "Format"), + ("format", "Format"), + ("le", "OLetter"), + ("lf", "LF"), + ("lo", "Lower"), + ("lower", "Lower"), + ("nu", "Numeric"), + ("numeric", "Numeric"), + ("oletter", "OLetter"), + ("other", "Other"), + ("sc", "SContinue"), + ("scontinue", "SContinue"), + ("se", "Sep"), + ("sep", "Sep"), + ("sp", "Sp"), + ("st", "STerm"), + ("sterm", "STerm"), + ("up", "Upper"), + ("upper", "Upper"), + ("xx", "Other"), + ], + ), + ( + "Word_Break", + &[ + ("aletter", "ALetter"), + ("cr", "CR"), + ("doublequote", "Double_Quote"), + ("dq", "Double_Quote"), + ("eb", "E_Base"), + ("ebase", "E_Base"), + ("ebasegaz", "E_Base_GAZ"), + ("ebg", "E_Base_GAZ"), + ("em", "E_Modifier"), + ("emodifier", "E_Modifier"), + ("ex", "ExtendNumLet"), + ("extend", "Extend"), + ("extendnumlet", "ExtendNumLet"), + ("fo", "Format"), + ("format", "Format"), + ("gaz", "Glue_After_Zwj"), + ("glueafterzwj", "Glue_After_Zwj"), + ("hebrewletter", "Hebrew_Letter"), + ("hl", "Hebrew_Letter"), + ("ka", "Katakana"), + ("katakana", "Katakana"), + ("le", "ALetter"), + ("lf", "LF"), + ("mb", "MidNumLet"), + ("midletter", "MidLetter"), + ("midnum", "MidNum"), + ("midnumlet", "MidNumLet"), + ("ml", "MidLetter"), + ("mn", "MidNum"), + ("newline", "Newline"), + ("nl", "Newline"), + ("nu", "Numeric"), + ("numeric", "Numeric"), + ("other", "Other"), + ("regionalindicator", "Regional_Indicator"), + ("ri", "Regional_Indicator"), + ("singlequote", "Single_Quote"), + ("sq", "Single_Quote"), + ("wsegspace", "WSegSpace"), + ("xx", "Other"), + ("zwj", "ZWJ"), + ], + ), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/script.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/script.rs new file mode 100644 index 0000000..3e437ca --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/script.rs @@ -0,0 +1,1300 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate script ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = &[ + ("Adlam", ADLAM), + ("Ahom", AHOM), + ("Anatolian_Hieroglyphs", ANATOLIAN_HIEROGLYPHS), + ("Arabic", ARABIC), + ("Armenian", ARMENIAN), + ("Avestan", AVESTAN), + ("Balinese", BALINESE), + ("Bamum", BAMUM), + ("Bassa_Vah", BASSA_VAH), + ("Batak", BATAK), + ("Bengali", BENGALI), + ("Bhaiksuki", BHAIKSUKI), + ("Bopomofo", BOPOMOFO), + ("Brahmi", BRAHMI), + ("Braille", BRAILLE), + ("Buginese", BUGINESE), + ("Buhid", BUHID), + ("Canadian_Aboriginal", CANADIAN_ABORIGINAL), + ("Carian", CARIAN), + ("Caucasian_Albanian", CAUCASIAN_ALBANIAN), + ("Chakma", CHAKMA), + ("Cham", CHAM), + ("Cherokee", CHEROKEE), + ("Chorasmian", CHORASMIAN), + ("Common", COMMON), + ("Coptic", COPTIC), + ("Cuneiform", CUNEIFORM), + ("Cypriot", CYPRIOT), + ("Cypro_Minoan", CYPRO_MINOAN), + ("Cyrillic", CYRILLIC), + ("Deseret", DESERET), + ("Devanagari", DEVANAGARI), + ("Dives_Akuru", DIVES_AKURU), + ("Dogra", DOGRA), + ("Duployan", DUPLOYAN), + ("Egyptian_Hieroglyphs", EGYPTIAN_HIEROGLYPHS), + ("Elbasan", ELBASAN), + ("Elymaic", ELYMAIC), + ("Ethiopic", ETHIOPIC), + ("Garay", GARAY), + ("Georgian", GEORGIAN), + ("Glagolitic", GLAGOLITIC), + ("Gothic", GOTHIC), + ("Grantha", GRANTHA), + ("Greek", GREEK), + ("Gujarati", GUJARATI), + ("Gunjala_Gondi", GUNJALA_GONDI), + ("Gurmukhi", GURMUKHI), + ("Gurung_Khema", GURUNG_KHEMA), + ("Han", HAN), + ("Hangul", HANGUL), + ("Hanifi_Rohingya", HANIFI_ROHINGYA), + ("Hanunoo", HANUNOO), + ("Hatran", HATRAN), + ("Hebrew", HEBREW), + ("Hiragana", HIRAGANA), + ("Imperial_Aramaic", IMPERIAL_ARAMAIC), + ("Inherited", INHERITED), + ("Inscriptional_Pahlavi", INSCRIPTIONAL_PAHLAVI), + ("Inscriptional_Parthian", INSCRIPTIONAL_PARTHIAN), + ("Javanese", JAVANESE), + ("Kaithi", KAITHI), + ("Kannada", KANNADA), + ("Katakana", KATAKANA), + ("Kawi", KAWI), + ("Kayah_Li", KAYAH_LI), + ("Kharoshthi", KHAROSHTHI), + ("Khitan_Small_Script", KHITAN_SMALL_SCRIPT), + ("Khmer", KHMER), + ("Khojki", KHOJKI), + ("Khudawadi", KHUDAWADI), + ("Kirat_Rai", KIRAT_RAI), + ("Lao", LAO), + ("Latin", LATIN), + ("Lepcha", LEPCHA), + ("Limbu", LIMBU), + ("Linear_A", LINEAR_A), + ("Linear_B", LINEAR_B), + ("Lisu", LISU), + ("Lycian", LYCIAN), + ("Lydian", LYDIAN), + ("Mahajani", MAHAJANI), + ("Makasar", MAKASAR), + ("Malayalam", MALAYALAM), + ("Mandaic", MANDAIC), + ("Manichaean", MANICHAEAN), + ("Marchen", MARCHEN), + ("Masaram_Gondi", MASARAM_GONDI), + ("Medefaidrin", MEDEFAIDRIN), + ("Meetei_Mayek", MEETEI_MAYEK), + ("Mende_Kikakui", MENDE_KIKAKUI), + ("Meroitic_Cursive", MEROITIC_CURSIVE), + ("Meroitic_Hieroglyphs", MEROITIC_HIEROGLYPHS), + ("Miao", MIAO), + ("Modi", MODI), + ("Mongolian", MONGOLIAN), + ("Mro", MRO), + ("Multani", MULTANI), + ("Myanmar", MYANMAR), + ("Nabataean", NABATAEAN), + ("Nag_Mundari", NAG_MUNDARI), + ("Nandinagari", NANDINAGARI), + ("New_Tai_Lue", NEW_TAI_LUE), + ("Newa", NEWA), + ("Nko", NKO), + ("Nushu", NUSHU), + ("Nyiakeng_Puachue_Hmong", NYIAKENG_PUACHUE_HMONG), + ("Ogham", OGHAM), + ("Ol_Chiki", OL_CHIKI), + ("Ol_Onal", OL_ONAL), + ("Old_Hungarian", OLD_HUNGARIAN), + ("Old_Italic", OLD_ITALIC), + ("Old_North_Arabian", OLD_NORTH_ARABIAN), + ("Old_Permic", OLD_PERMIC), + ("Old_Persian", OLD_PERSIAN), + ("Old_Sogdian", OLD_SOGDIAN), + ("Old_South_Arabian", OLD_SOUTH_ARABIAN), + ("Old_Turkic", OLD_TURKIC), + ("Old_Uyghur", OLD_UYGHUR), + ("Oriya", ORIYA), + ("Osage", OSAGE), + ("Osmanya", OSMANYA), + ("Pahawh_Hmong", PAHAWH_HMONG), + ("Palmyrene", PALMYRENE), + ("Pau_Cin_Hau", PAU_CIN_HAU), + ("Phags_Pa", PHAGS_PA), + ("Phoenician", PHOENICIAN), + ("Psalter_Pahlavi", PSALTER_PAHLAVI), + ("Rejang", REJANG), + ("Runic", RUNIC), + ("Samaritan", SAMARITAN), + ("Saurashtra", SAURASHTRA), + ("Sharada", SHARADA), + ("Shavian", SHAVIAN), + ("Siddham", SIDDHAM), + ("SignWriting", SIGNWRITING), + ("Sinhala", SINHALA), + ("Sogdian", SOGDIAN), + ("Sora_Sompeng", SORA_SOMPENG), + ("Soyombo", SOYOMBO), + ("Sundanese", SUNDANESE), + ("Sunuwar", SUNUWAR), + ("Syloti_Nagri", SYLOTI_NAGRI), + ("Syriac", SYRIAC), + ("Tagalog", TAGALOG), + ("Tagbanwa", TAGBANWA), + ("Tai_Le", TAI_LE), + ("Tai_Tham", TAI_THAM), + ("Tai_Viet", TAI_VIET), + ("Takri", TAKRI), + ("Tamil", TAMIL), + ("Tangsa", TANGSA), + ("Tangut", TANGUT), + ("Telugu", TELUGU), + ("Thaana", THAANA), + ("Thai", THAI), + ("Tibetan", TIBETAN), + ("Tifinagh", TIFINAGH), + ("Tirhuta", TIRHUTA), + ("Todhri", TODHRI), + ("Toto", TOTO), + ("Tulu_Tigalari", TULU_TIGALARI), + ("Ugaritic", UGARITIC), + ("Vai", VAI), + ("Vithkuqi", VITHKUQI), + ("Wancho", WANCHO), + ("Warang_Citi", WARANG_CITI), + ("Yezidi", YEZIDI), + ("Yi", YI), + ("Zanabazar_Square", ZANABAZAR_SQUARE), +]; + +pub const ADLAM: &'static [(char, char)] = + &[('𞤀', '𞥋'), ('𞥐', '𞥙'), ('𞥞', '𞥟')]; + +pub const AHOM: &'static [(char, char)] = + &[('𑜀', '𑜚'), ('\u{1171d}', '\u{1172b}'), ('𑜰', '𑝆')]; + +pub const ANATOLIAN_HIEROGLYPHS: &'static [(char, char)] = &[('𔐀', '𔙆')]; + +pub const ARABIC: &'static [(char, char)] = &[ + ('\u{600}', '\u{604}'), + ('؆', '؋'), + ('؍', '\u{61a}'), + ('\u{61c}', '؞'), + ('ؠ', 'ؿ'), + ('ف', 'ي'), + ('\u{656}', 'ٯ'), + ('ٱ', '\u{6dc}'), + ('۞', 'ۿ'), + ('ݐ', 'ݿ'), + ('ࡰ', 'ࢎ'), + ('\u{890}', '\u{891}'), + ('\u{897}', '\u{8e1}'), + ('\u{8e3}', '\u{8ff}'), + ('ﭐ', '﯂'), + ('ﯓ', 'ﴽ'), + ('﵀', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('﷏', '﷏'), + ('ﷰ', '﷿'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('𐹠', '𐹾'), + ('𐻂', '𐻄'), + ('\u{10efc}', '\u{10eff}'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𞻰', '𞻱'), +]; + +pub const ARMENIAN: &'static [(char, char)] = + &[('Ա', 'Ֆ'), ('ՙ', '֊'), ('֍', '֏'), ('ﬓ', 'ﬗ')]; + +pub const AVESTAN: &'static [(char, char)] = &[('𐬀', '𐬵'), ('𐬹', '𐬿')]; + +pub const BALINESE: &'static [(char, char)] = &[('\u{1b00}', 'ᭌ'), ('᭎', '᭿')]; + +pub const BAMUM: &'static [(char, char)] = &[('ꚠ', '꛷'), ('𖠀', '𖨸')]; + +pub const BASSA_VAH: &'static [(char, char)] = + &[('𖫐', '𖫭'), ('\u{16af0}', '𖫵')]; + +pub const BATAK: &'static [(char, char)] = &[('ᯀ', '\u{1bf3}'), ('᯼', '᯿')]; + +pub const BENGALI: &'static [(char, char)] = &[ + ('ঀ', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('\u{9bc}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', 'ৎ'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', '\u{9e3}'), + ('০', '\u{9fe}'), +]; + +pub const BHAIKSUKI: &'static [(char, char)] = + &[('𑰀', '𑰈'), ('𑰊', '\u{11c36}'), ('\u{11c38}', '𑱅'), ('𑱐', '𑱬')]; + +pub const BOPOMOFO: &'static [(char, char)] = + &[('˪', '˫'), ('ㄅ', 'ㄯ'), ('ㆠ', 'ㆿ')]; + +pub const BRAHMI: &'static [(char, char)] = + &[('𑀀', '𑁍'), ('𑁒', '𑁵'), ('\u{1107f}', '\u{1107f}')]; + +pub const BRAILLE: &'static [(char, char)] = &[('⠀', '⣿')]; + +pub const BUGINESE: &'static [(char, char)] = &[('ᨀ', '\u{1a1b}'), ('᨞', '᨟')]; + +pub const BUHID: &'static [(char, char)] = &[('ᝀ', '\u{1753}')]; + +pub const CANADIAN_ABORIGINAL: &'static [(char, char)] = + &[('᐀', 'ᙿ'), ('ᢰ', 'ᣵ'), ('𑪰', '𑪿')]; + +pub const CARIAN: &'static [(char, char)] = &[('𐊠', '𐋐')]; + +pub const CAUCASIAN_ALBANIAN: &'static [(char, char)] = + &[('𐔰', '𐕣'), ('𐕯', '𐕯')]; + +pub const CHAKMA: &'static [(char, char)] = + &[('\u{11100}', '\u{11134}'), ('𑄶', '𑅇')]; + +pub const CHAM: &'static [(char, char)] = + &[('ꨀ', '\u{aa36}'), ('ꩀ', 'ꩍ'), ('꩐', '꩙'), ('꩜', '꩟')]; + +pub const CHEROKEE: &'static [(char, char)] = + &[('Ꭰ', 'Ᏽ'), ('ᏸ', 'ᏽ'), ('ꭰ', 'ꮿ')]; + +pub const CHORASMIAN: &'static [(char, char)] = &[('𐾰', '𐿋')]; + +pub const COMMON: &'static [(char, char)] = &[ + ('\0', '@'), + ('[', '`'), + ('{', '©'), + ('«', '¹'), + ('»', '¿'), + ('×', '×'), + ('÷', '÷'), + ('ʹ', '˟'), + ('˥', '˩'), + ('ˬ', '˿'), + ('ʹ', 'ʹ'), + (';', ';'), + ('΅', '΅'), + ('·', '·'), + ('\u{605}', '\u{605}'), + ('،', '،'), + ('؛', '؛'), + ('؟', '؟'), + ('ـ', 'ـ'), + ('\u{6dd}', '\u{6dd}'), + ('\u{8e2}', '\u{8e2}'), + ('।', '॥'), + ('฿', '฿'), + ('࿕', '࿘'), + ('჻', '჻'), + ('᛫', '᛭'), + ('᜵', '᜶'), + ('᠂', '᠃'), + ('᠅', '᠅'), + ('᳓', '᳓'), + ('᳡', '᳡'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', '᳷'), + ('ᳺ', 'ᳺ'), + ('\u{2000}', '\u{200b}'), + ('\u{200e}', '\u{2064}'), + ('\u{2066}', '⁰'), + ('⁴', '⁾'), + ('₀', '₎'), + ('₠', '⃀'), + ('℀', '℥'), + ('℧', '℩'), + ('ℬ', 'ℱ'), + ('ℳ', '⅍'), + ('⅏', '⅟'), + ('↉', '↋'), + ('←', '␩'), + ('⑀', '⑊'), + ('①', '⟿'), + ('⤀', '⭳'), + ('⭶', '⮕'), + ('⮗', '⯿'), + ('⸀', '⹝'), + ('⿰', '〄'), + ('〆', '〆'), + ('〈', '〠'), + ('〰', '〷'), + ('〼', '〿'), + ('゛', '゜'), + ('゠', '゠'), + ('・', 'ー'), + ('㆐', '㆟'), + ('㇀', '㇥'), + ('㇯', '㇯'), + ('㈠', '㉟'), + ('㉿', '㋏'), + ('㋿', '㋿'), + ('㍘', '㏿'), + ('䷀', '䷿'), + ('꜀', '꜡'), + ('ꞈ', '꞊'), + ('꠰', '꠹'), + ('꤮', '꤮'), + ('ꧏ', 'ꧏ'), + ('꭛', '꭛'), + ('꭪', '꭫'), + ('﴾', '﴿'), + ('︐', '︙'), + ('︰', '﹒'), + ('﹔', '﹦'), + ('﹨', '﹫'), + ('\u{feff}', '\u{feff}'), + ('！', '＠'), + ('［', '｀'), + ('｛', '･'), + ('ｰ', 'ｰ'), + ('\u{ff9e}', '\u{ff9f}'), + ('￠', '￦'), + ('￨', '￮'), + ('\u{fff9}', '�'), + ('𐄀', '𐄂'), + ('𐄇', '𐄳'), + ('𐄷', '𐄿'), + ('𐆐', '𐆜'), + ('𐇐', '𐇼'), + ('𐋡', '𐋻'), + ('\u{1bca0}', '\u{1bca3}'), + ('𜰀', '𜳹'), + ('𜴀', '𜺳'), + ('𜽐', '𜿃'), + ('𝀀', '𝃵'), + ('𝄀', '𝄦'), + ('𝄩', '\u{1d166}'), + ('𝅪', '\u{1d17a}'), + ('𝆃', '𝆄'), + ('𝆌', '𝆩'), + ('𝆮', '𝇪'), + ('𝋀', '𝋓'), + ('𝋠', '𝋳'), + ('𝌀', '𝍖'), + ('𝍠', '𝍸'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝟋'), + ('𝟎', '𝟿'), + ('𞱱', '𞲴'), + ('𞴁', '𞴽'), + ('🀀', '🀫'), + ('🀰', '🂓'), + ('🂠', '🂮'), + ('🂱', '🂿'), + ('🃁', '🃏'), + ('🃑', '🃵'), + ('🄀', '🆭'), + ('🇦', '🇿'), + ('🈁', '🈂'), + ('🈐', '🈻'), + ('🉀', '🉈'), + ('🉐', '🉑'), + ('🉠', '🉥'), + ('🌀', '🛗'), + ('🛜', '🛬'), + ('🛰', '🛼'), + ('🜀', '🝶'), + ('🝻', '🟙'), + ('🟠', '🟫'), + ('🟰', '🟰'), + ('🠀', '🠋'), + ('🠐', '🡇'), + ('🡐', '🡙'), + ('🡠', '🢇'), + ('🢐', '🢭'), + ('🢰', '🢻'), + ('🣀', '🣁'), + ('🤀', '🩓'), + ('🩠', '🩭'), + ('🩰', '🩼'), + ('🪀', '🪉'), + ('🪏', '🫆'), + ('🫎', '🫜'), + ('🫟', '🫩'), + ('🫰', '🫸'), + ('🬀', '🮒'), + ('🮔', '🯹'), + ('\u{e0001}', '\u{e0001}'), + ('\u{e0020}', '\u{e007f}'), +]; + +pub const COPTIC: &'static [(char, char)] = + &[('Ϣ', 'ϯ'), ('Ⲁ', 'ⳳ'), ('⳹', '⳿')]; + +pub const CUNEIFORM: &'static [(char, char)] = + &[('𒀀', '𒎙'), ('𒐀', '𒑮'), ('𒑰', '𒑴'), ('𒒀', '𒕃')]; + +pub const CYPRIOT: &'static [(char, char)] = + &[('𐠀', '𐠅'), ('𐠈', '𐠈'), ('𐠊', '𐠵'), ('𐠷', '𐠸'), ('𐠼', '𐠼'), ('𐠿', '𐠿')]; + +pub const CYPRO_MINOAN: &'static [(char, char)] = &[('𒾐', '𒿲')]; + +pub const CYRILLIC: &'static [(char, char)] = &[ + ('Ѐ', '\u{484}'), + ('\u{487}', 'ԯ'), + ('ᲀ', 'ᲊ'), + ('ᴫ', 'ᴫ'), + ('ᵸ', 'ᵸ'), + ('\u{2de0}', '\u{2dff}'), + ('Ꙁ', '\u{a69f}'), + ('\u{fe2e}', '\u{fe2f}'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), +]; + +pub const DESERET: &'static [(char, char)] = &[('𐐀', '𐑏')]; + +pub const DEVANAGARI: &'static [(char, char)] = &[ + ('\u{900}', 'ॐ'), + ('\u{955}', '\u{963}'), + ('०', 'ॿ'), + ('\u{a8e0}', '\u{a8ff}'), + ('𑬀', '𑬉'), +]; + +pub const DIVES_AKURU: &'static [(char, char)] = &[ + ('𑤀', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '𑥆'), + ('𑥐', '𑥙'), +]; + +pub const DOGRA: &'static [(char, char)] = &[('𑠀', '𑠻')]; + +pub const DUPLOYAN: &'static [(char, char)] = + &[('𛰀', '𛱪'), ('𛱰', '𛱼'), ('𛲀', '𛲈'), ('𛲐', '𛲙'), ('𛲜', '𛲟')]; + +pub const EGYPTIAN_HIEROGLYPHS: &'static [(char, char)] = + &[('𓀀', '\u{13455}'), ('𓑠', '𔏺')]; + +pub const ELBASAN: &'static [(char, char)] = &[('𐔀', '𐔧')]; + +pub const ELYMAIC: &'static [(char, char)] = &[('𐿠', '𐿶')]; + +pub const ETHIOPIC: &'static [(char, char)] = &[ + ('ሀ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('\u{135d}', '፼'), + ('ᎀ', '᎙'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), +]; + +pub const GARAY: &'static [(char, char)] = + &[('𐵀', '𐵥'), ('\u{10d69}', '𐶅'), ('𐶎', '𐶏')]; + +pub const GEORGIAN: &'static [(char, char)] = &[ + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ჿ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), +]; + +pub const GLAGOLITIC: &'static [(char, char)] = &[ + ('Ⰰ', 'ⱟ'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), +]; + +pub const GOTHIC: &'static [(char, char)] = &[('𐌰', '𐍊')]; + +pub const GRANTHA: &'static [(char, char)] = &[ + ('\u{11300}', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('\u{1133c}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('𑍐', '𑍐'), + ('\u{11357}', '\u{11357}'), + ('𑍝', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), +]; + +pub const GREEK: &'static [(char, char)] = &[ + ('Ͱ', 'ͳ'), + ('͵', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('΄', '΄'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϡ'), + ('ϰ', 'Ͽ'), + ('ᴦ', 'ᴪ'), + ('ᵝ', 'ᵡ'), + ('ᵦ', 'ᵪ'), + ('ᶿ', 'ᶿ'), + ('ἀ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ῄ'), + ('ῆ', 'ΐ'), + ('ῖ', 'Ί'), + ('῝', '`'), + ('ῲ', 'ῴ'), + ('ῶ', '῾'), + ('Ω', 'Ω'), + ('ꭥ', 'ꭥ'), + ('𐅀', '𐆎'), + ('𐆠', '𐆠'), + ('𝈀', '𝉅'), +]; + +pub const GUJARATI: &'static [(char, char)] = &[ + ('\u{a81}', 'ઃ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('\u{abc}', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('ૐ', 'ૐ'), + ('ૠ', '\u{ae3}'), + ('૦', '૱'), + ('ૹ', '\u{aff}'), +]; + +pub const GUNJALA_GONDI: &'static [(char, char)] = &[ + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶘'), + ('𑶠', '𑶩'), +]; + +pub const GURMUKHI: &'static [(char, char)] = &[ + ('\u{a01}', 'ਃ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('੦', '੶'), +]; + +pub const GURUNG_KHEMA: &'static [(char, char)] = &[('𖄀', '𖄹')]; + +pub const HAN: &'static [(char, char)] = &[ + ('⺀', '⺙'), + ('⺛', '⻳'), + ('⼀', '⿕'), + ('々', '々'), + ('〇', '〇'), + ('〡', '〩'), + ('〸', '〻'), + ('㐀', '䶿'), + ('一', '鿿'), + ('豈', '舘'), + ('並', '龎'), + ('𖿢', '𖿣'), + ('\u{16ff0}', '\u{16ff1}'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const HANGUL: &'static [(char, char)] = &[ + ('ᄀ', 'ᇿ'), + ('\u{302e}', '\u{302f}'), + ('ㄱ', 'ㆎ'), + ('㈀', '㈞'), + ('㉠', '㉾'), + ('ꥠ', 'ꥼ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('ﾠ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), +]; + +pub const HANIFI_ROHINGYA: &'static [(char, char)] = + &[('𐴀', '\u{10d27}'), ('𐴰', '𐴹')]; + +pub const HANUNOO: &'static [(char, char)] = &[('ᜠ', '\u{1734}')]; + +pub const HATRAN: &'static [(char, char)] = + &[('𐣠', '𐣲'), ('𐣴', '𐣵'), ('𐣻', '𐣿')]; + +pub const HEBREW: &'static [(char, char)] = &[ + ('\u{591}', '\u{5c7}'), + ('א', 'ת'), + ('ׯ', '״'), + ('יִ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﭏ'), +]; + +pub const HIRAGANA: &'static [(char, char)] = &[ + ('ぁ', 'ゖ'), + ('ゝ', 'ゟ'), + ('𛀁', '𛄟'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('🈀', '🈀'), +]; + +pub const IMPERIAL_ARAMAIC: &'static [(char, char)] = + &[('𐡀', '𐡕'), ('𐡗', '𐡟')]; + +pub const INHERITED: &'static [(char, char)] = &[ + ('\u{300}', '\u{36f}'), + ('\u{485}', '\u{486}'), + ('\u{64b}', '\u{655}'), + ('\u{670}', '\u{670}'), + ('\u{951}', '\u{954}'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce0}'), + ('\u{1ce2}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('\u{1cf8}', '\u{1cf9}'), + ('\u{1dc0}', '\u{1dff}'), + ('\u{200c}', '\u{200d}'), + ('\u{20d0}', '\u{20f0}'), + ('\u{302a}', '\u{302d}'), + ('\u{3099}', '\u{309a}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2d}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{1133b}', '\u{1133b}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d167}', '\u{1d169}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const INSCRIPTIONAL_PAHLAVI: &'static [(char, char)] = + &[('𐭠', '𐭲'), ('𐭸', '𐭿')]; + +pub const INSCRIPTIONAL_PARTHIAN: &'static [(char, char)] = + &[('𐭀', '𐭕'), ('𐭘', '𐭟')]; + +pub const JAVANESE: &'static [(char, char)] = + &[('\u{a980}', '꧍'), ('꧐', '꧙'), ('꧞', '꧟')]; + +pub const KAITHI: &'static [(char, char)] = + &[('\u{11080}', '\u{110c2}'), ('\u{110cd}', '\u{110cd}')]; + +pub const KANNADA: &'static [(char, char)] = &[ + ('ಀ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('\u{cbc}', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('ೝ', 'ೞ'), + ('ೠ', '\u{ce3}'), + ('೦', '೯'), + ('ೱ', 'ೳ'), +]; + +pub const KATAKANA: &'static [(char, char)] = &[ + ('ァ', 'ヺ'), + ('ヽ', 'ヿ'), + ('ㇰ', 'ㇿ'), + ('㋐', '㋾'), + ('㌀', '㍗'), + ('ｦ', 'ｯ'), + ('ｱ', 'ﾝ'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛀀'), + ('𛄠', '𛄢'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), +]; + +pub const KAWI: &'static [(char, char)] = + &[('\u{11f00}', '𑼐'), ('𑼒', '\u{11f3a}'), ('𑼾', '\u{11f5a}')]; + +pub const KAYAH_LI: &'static [(char, char)] = &[('꤀', '\u{a92d}'), ('꤯', '꤯')]; + +pub const KHAROSHTHI: &'static [(char, char)] = &[ + ('𐨀', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '𐩈'), + ('𐩐', '𐩘'), +]; + +pub const KHITAN_SMALL_SCRIPT: &'static [(char, char)] = + &[('\u{16fe4}', '\u{16fe4}'), ('𘬀', '𘳕'), ('𘳿', '𘳿')]; + +pub const KHMER: &'static [(char, char)] = + &[('ក', '\u{17dd}'), ('០', '៩'), ('៰', '៹'), ('᧠', '᧿')]; + +pub const KHOJKI: &'static [(char, char)] = &[('𑈀', '𑈑'), ('𑈓', '\u{11241}')]; + +pub const KHUDAWADI: &'static [(char, char)] = + &[('𑊰', '\u{112ea}'), ('𑋰', '𑋹')]; + +pub const KIRAT_RAI: &'static [(char, char)] = &[('𖵀', '𖵹')]; + +pub const LAO: &'static [(char, char)] = &[ + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('\u{ec8}', '\u{ece}'), + ('໐', '໙'), + ('ໜ', 'ໟ'), +]; + +pub const LATIN: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('ª', 'ª'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ʸ'), + ('ˠ', 'ˤ'), + ('ᴀ', 'ᴥ'), + ('ᴬ', 'ᵜ'), + ('ᵢ', 'ᵥ'), + ('ᵫ', 'ᵷ'), + ('ᵹ', 'ᶾ'), + ('Ḁ', 'ỿ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('K', 'Å'), + ('Ⅎ', 'Ⅎ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⱡ', 'Ɀ'), + ('Ꜣ', 'ꞇ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꟿ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭤ'), + ('ꭦ', 'ꭩ'), + ('ff', 'st'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), +]; + +pub const LEPCHA: &'static [(char, char)] = + &[('ᰀ', '\u{1c37}'), ('᰻', '᱉'), ('ᱍ', 'ᱏ')]; + +pub const LIMBU: &'static [(char, char)] = &[ + ('ᤀ', 'ᤞ'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('᥀', '᥀'), + ('᥄', '᥏'), +]; + +pub const LINEAR_A: &'static [(char, char)] = + &[('𐘀', '𐜶'), ('𐝀', '𐝕'), ('𐝠', '𐝧')]; + +pub const LINEAR_B: &'static [(char, char)] = &[ + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), +]; + +pub const LISU: &'static [(char, char)] = &[('ꓐ', '꓿'), ('𑾰', '𑾰')]; + +pub const LYCIAN: &'static [(char, char)] = &[('𐊀', '𐊜')]; + +pub const LYDIAN: &'static [(char, char)] = &[('𐤠', '𐤹'), ('𐤿', '𐤿')]; + +pub const MAHAJANI: &'static [(char, char)] = &[('𑅐', '𑅶')]; + +pub const MAKASAR: &'static [(char, char)] = &[('𑻠', '𑻸')]; + +pub const MALAYALAM: &'static [(char, char)] = &[ + ('\u{d00}', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', '൏'), + ('ൔ', '\u{d63}'), + ('൦', 'ൿ'), +]; + +pub const MANDAIC: &'static [(char, char)] = &[('ࡀ', '\u{85b}'), ('࡞', '࡞')]; + +pub const MANICHAEAN: &'static [(char, char)] = + &[('𐫀', '\u{10ae6}'), ('𐫫', '𐫶')]; + +pub const MARCHEN: &'static [(char, char)] = + &[('𑱰', '𑲏'), ('\u{11c92}', '\u{11ca7}'), ('𑲩', '\u{11cb6}')]; + +pub const MASARAM_GONDI: &'static [(char, char)] = &[ + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d47}'), + ('𑵐', '𑵙'), +]; + +pub const MEDEFAIDRIN: &'static [(char, char)] = &[('𖹀', '𖺚')]; + +pub const MEETEI_MAYEK: &'static [(char, char)] = + &[('ꫠ', '\u{aaf6}'), ('ꯀ', '\u{abed}'), ('꯰', '꯹')]; + +pub const MENDE_KIKAKUI: &'static [(char, char)] = + &[('𞠀', '𞣄'), ('𞣇', '\u{1e8d6}')]; + +pub const MEROITIC_CURSIVE: &'static [(char, char)] = + &[('𐦠', '𐦷'), ('𐦼', '𐧏'), ('𐧒', '𐧿')]; + +pub const MEROITIC_HIEROGLYPHS: &'static [(char, char)] = &[('𐦀', '𐦟')]; + +pub const MIAO: &'static [(char, char)] = + &[('𖼀', '𖽊'), ('\u{16f4f}', '𖾇'), ('\u{16f8f}', '𖾟')]; + +pub const MODI: &'static [(char, char)] = &[('𑘀', '𑙄'), ('𑙐', '𑙙')]; + +pub const MONGOLIAN: &'static [(char, char)] = + &[('᠀', '᠁'), ('᠄', '᠄'), ('᠆', '᠙'), ('ᠠ', 'ᡸ'), ('ᢀ', 'ᢪ'), ('𑙠', '𑙬')]; + +pub const MRO: &'static [(char, char)] = &[('𖩀', '𖩞'), ('𖩠', '𖩩'), ('𖩮', '𖩯')]; + +pub const MULTANI: &'static [(char, char)] = + &[('𑊀', '𑊆'), ('𑊈', '𑊈'), ('𑊊', '𑊍'), ('𑊏', '𑊝'), ('𑊟', '𑊩')]; + +pub const MYANMAR: &'static [(char, char)] = + &[('က', '႟'), ('ꧠ', 'ꧾ'), ('ꩠ', 'ꩿ'), ('𑛐', '𑛣')]; + +pub const NABATAEAN: &'static [(char, char)] = &[('𐢀', '𐢞'), ('𐢧', '𐢯')]; + +pub const NAG_MUNDARI: &'static [(char, char)] = &[('𞓐', '𞓹')]; + +pub const NANDINAGARI: &'static [(char, char)] = + &[('𑦠', '𑦧'), ('𑦪', '\u{119d7}'), ('\u{119da}', '𑧤')]; + +pub const NEW_TAI_LUE: &'static [(char, char)] = + &[('ᦀ', 'ᦫ'), ('ᦰ', 'ᧉ'), ('᧐', '᧚'), ('᧞', '᧟')]; + +pub const NEWA: &'static [(char, char)] = &[('𑐀', '𑑛'), ('𑑝', '𑑡')]; + +pub const NKO: &'static [(char, char)] = &[('߀', 'ߺ'), ('\u{7fd}', '߿')]; + +pub const NUSHU: &'static [(char, char)] = &[('𖿡', '𖿡'), ('𛅰', '𛋻')]; + +pub const NYIAKENG_PUACHUE_HMONG: &'static [(char, char)] = + &[('𞄀', '𞄬'), ('\u{1e130}', '𞄽'), ('𞅀', '𞅉'), ('𞅎', '𞅏')]; + +pub const OGHAM: &'static [(char, char)] = &[('\u{1680}', '᚜')]; + +pub const OL_CHIKI: &'static [(char, char)] = &[('᱐', '᱿')]; + +pub const OL_ONAL: &'static [(char, char)] = &[('𞗐', '𞗺'), ('𞗿', '𞗿')]; + +pub const OLD_HUNGARIAN: &'static [(char, char)] = + &[('𐲀', '𐲲'), ('𐳀', '𐳲'), ('𐳺', '𐳿')]; + +pub const OLD_ITALIC: &'static [(char, char)] = &[('𐌀', '𐌣'), ('𐌭', '𐌯')]; + +pub const OLD_NORTH_ARABIAN: &'static [(char, char)] = &[('𐪀', '𐪟')]; + +pub const OLD_PERMIC: &'static [(char, char)] = &[('𐍐', '\u{1037a}')]; + +pub const OLD_PERSIAN: &'static [(char, char)] = &[('𐎠', '𐏃'), ('𐏈', '𐏕')]; + +pub const OLD_SOGDIAN: &'static [(char, char)] = &[('𐼀', '𐼧')]; + +pub const OLD_SOUTH_ARABIAN: &'static [(char, char)] = &[('𐩠', '𐩿')]; + +pub const OLD_TURKIC: &'static [(char, char)] = &[('𐰀', '𐱈')]; + +pub const OLD_UYGHUR: &'static [(char, char)] = &[('𐽰', '𐾉')]; + +pub const ORIYA: &'static [(char, char)] = &[ + ('\u{b01}', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('\u{b3c}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', '\u{b63}'), + ('୦', '୷'), +]; + +pub const OSAGE: &'static [(char, char)] = &[('𐒰', '𐓓'), ('𐓘', '𐓻')]; + +pub const OSMANYA: &'static [(char, char)] = &[('𐒀', '𐒝'), ('𐒠', '𐒩')]; + +pub const PAHAWH_HMONG: &'static [(char, char)] = + &[('𖬀', '𖭅'), ('𖭐', '𖭙'), ('𖭛', '𖭡'), ('𖭣', '𖭷'), ('𖭽', '𖮏')]; + +pub const PALMYRENE: &'static [(char, char)] = &[('𐡠', '𐡿')]; + +pub const PAU_CIN_HAU: &'static [(char, char)] = &[('𑫀', '𑫸')]; + +pub const PHAGS_PA: &'static [(char, char)] = &[('ꡀ', '꡷')]; + +pub const PHOENICIAN: &'static [(char, char)] = &[('𐤀', '𐤛'), ('𐤟', '𐤟')]; + +pub const PSALTER_PAHLAVI: &'static [(char, char)] = + &[('𐮀', '𐮑'), ('𐮙', '𐮜'), ('𐮩', '𐮯')]; + +pub const REJANG: &'static [(char, char)] = &[('ꤰ', '\u{a953}'), ('꥟', '꥟')]; + +pub const RUNIC: &'static [(char, char)] = &[('ᚠ', 'ᛪ'), ('ᛮ', 'ᛸ')]; + +pub const SAMARITAN: &'static [(char, char)] = &[('ࠀ', '\u{82d}'), ('࠰', '࠾')]; + +pub const SAURASHTRA: &'static [(char, char)] = + &[('ꢀ', '\u{a8c5}'), ('꣎', '꣙')]; + +pub const SHARADA: &'static [(char, char)] = &[('\u{11180}', '𑇟')]; + +pub const SHAVIAN: &'static [(char, char)] = &[('𐑐', '𐑿')]; + +pub const SIDDHAM: &'static [(char, char)] = + &[('𑖀', '\u{115b5}'), ('𑖸', '\u{115dd}')]; + +pub const SIGNWRITING: &'static [(char, char)] = + &[('𝠀', '𝪋'), ('\u{1da9b}', '\u{1da9f}'), ('\u{1daa1}', '\u{1daaf}')]; + +pub const SINHALA: &'static [(char, char)] = &[ + ('\u{d81}', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('෦', '෯'), + ('ෲ', '෴'), + ('𑇡', '𑇴'), +]; + +pub const SOGDIAN: &'static [(char, char)] = &[('𐼰', '𐽙')]; + +pub const SORA_SOMPENG: &'static [(char, char)] = &[('𑃐', '𑃨'), ('𑃰', '𑃹')]; + +pub const SOYOMBO: &'static [(char, char)] = &[('𑩐', '𑪢')]; + +pub const SUNDANESE: &'static [(char, char)] = + &[('\u{1b80}', 'ᮿ'), ('᳀', '᳇')]; + +pub const SUNUWAR: &'static [(char, char)] = &[('𑯀', '𑯡'), ('𑯰', '𑯹')]; + +pub const SYLOTI_NAGRI: &'static [(char, char)] = &[('ꠀ', '\u{a82c}')]; + +pub const SYRIAC: &'static [(char, char)] = + &[('܀', '܍'), ('\u{70f}', '\u{74a}'), ('ݍ', 'ݏ'), ('ࡠ', 'ࡪ')]; + +pub const TAGALOG: &'static [(char, char)] = &[('ᜀ', '\u{1715}'), ('ᜟ', 'ᜟ')]; + +pub const TAGBANWA: &'static [(char, char)] = + &[('ᝠ', 'ᝬ'), ('ᝮ', 'ᝰ'), ('\u{1772}', '\u{1773}')]; + +pub const TAI_LE: &'static [(char, char)] = &[('ᥐ', 'ᥭ'), ('ᥰ', 'ᥴ')]; + +pub const TAI_THAM: &'static [(char, char)] = &[ + ('ᨠ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '᪉'), + ('᪐', '᪙'), + ('᪠', '᪭'), +]; + +pub const TAI_VIET: &'static [(char, char)] = &[('ꪀ', 'ꫂ'), ('ꫛ', '꫟')]; + +pub const TAKRI: &'static [(char, char)] = &[('𑚀', '𑚹'), ('𑛀', '𑛉')]; + +pub const TAMIL: &'static [(char, char)] = &[ + ('\u{b82}', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('ௐ', 'ௐ'), + ('\u{bd7}', '\u{bd7}'), + ('௦', '௺'), + ('𑿀', '𑿱'), + ('𑿿', '𑿿'), +]; + +pub const TANGSA: &'static [(char, char)] = &[('𖩰', '𖪾'), ('𖫀', '𖫉')]; + +pub const TANGUT: &'static [(char, char)] = + &[('𖿠', '𖿠'), ('𗀀', '𘟷'), ('𘠀', '𘫿'), ('𘴀', '𘴈')]; + +pub const TELUGU: &'static [(char, char)] = &[ + ('\u{c00}', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('\u{c3c}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', '\u{c63}'), + ('౦', '౯'), + ('౷', '౿'), +]; + +pub const THAANA: &'static [(char, char)] = &[('ހ', 'ޱ')]; + +pub const THAI: &'static [(char, char)] = &[('ก', '\u{e3a}'), ('เ', '๛')]; + +pub const TIBETAN: &'static [(char, char)] = &[ + ('ༀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('\u{f71}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('྾', '࿌'), + ('࿎', '࿔'), + ('࿙', '࿚'), +]; + +pub const TIFINAGH: &'static [(char, char)] = + &[('ⴰ', 'ⵧ'), ('ⵯ', '⵰'), ('\u{2d7f}', '\u{2d7f}')]; + +pub const TIRHUTA: &'static [(char, char)] = &[('𑒀', '𑓇'), ('𑓐', '𑓙')]; + +pub const TODHRI: &'static [(char, char)] = &[('𐗀', '𐗳')]; + +pub const TOTO: &'static [(char, char)] = &[('𞊐', '\u{1e2ae}')]; + +pub const TULU_TIGALARI: &'static [(char, char)] = &[ + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏕'), + ('𑏗', '𑏘'), + ('\u{113e1}', '\u{113e2}'), +]; + +pub const UGARITIC: &'static [(char, char)] = &[('𐎀', '𐎝'), ('𐎟', '𐎟')]; + +pub const VAI: &'static [(char, char)] = &[('ꔀ', 'ꘫ')]; + +pub const VITHKUQI: &'static [(char, char)] = &[ + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), +]; + +pub const WANCHO: &'static [(char, char)] = &[('𞋀', '𞋹'), ('𞋿', '𞋿')]; + +pub const WARANG_CITI: &'static [(char, char)] = &[('𑢠', '𑣲'), ('𑣿', '𑣿')]; + +pub const YEZIDI: &'static [(char, char)] = + &[('𐺀', '𐺩'), ('\u{10eab}', '𐺭'), ('𐺰', '𐺱')]; + +pub const YI: &'static [(char, char)] = &[('ꀀ', 'ꒌ'), ('꒐', '꓆')]; + +pub const ZANABAZAR_SQUARE: &'static [(char, char)] = &[('𑨀', '\u{11a47}')]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/script_extension.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/script_extension.rs new file mode 100644 index 0000000..e3f492e --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/script_extension.rs @@ -0,0 +1,1718 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate script-extension ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = &[ + ("Adlam", ADLAM), + ("Ahom", AHOM), + ("Anatolian_Hieroglyphs", ANATOLIAN_HIEROGLYPHS), + ("Arabic", ARABIC), + ("Armenian", ARMENIAN), + ("Avestan", AVESTAN), + ("Balinese", BALINESE), + ("Bamum", BAMUM), + ("Bassa_Vah", BASSA_VAH), + ("Batak", BATAK), + ("Bengali", BENGALI), + ("Bhaiksuki", BHAIKSUKI), + ("Bopomofo", BOPOMOFO), + ("Brahmi", BRAHMI), + ("Braille", BRAILLE), + ("Buginese", BUGINESE), + ("Buhid", BUHID), + ("Canadian_Aboriginal", CANADIAN_ABORIGINAL), + ("Carian", CARIAN), + ("Caucasian_Albanian", CAUCASIAN_ALBANIAN), + ("Chakma", CHAKMA), + ("Cham", CHAM), + ("Cherokee", CHEROKEE), + ("Chorasmian", CHORASMIAN), + ("Common", COMMON), + ("Coptic", COPTIC), + ("Cuneiform", CUNEIFORM), + ("Cypriot", CYPRIOT), + ("Cypro_Minoan", CYPRO_MINOAN), + ("Cyrillic", CYRILLIC), + ("Deseret", DESERET), + ("Devanagari", DEVANAGARI), + ("Dives_Akuru", DIVES_AKURU), + ("Dogra", DOGRA), + ("Duployan", DUPLOYAN), + ("Egyptian_Hieroglyphs", EGYPTIAN_HIEROGLYPHS), + ("Elbasan", ELBASAN), + ("Elymaic", ELYMAIC), + ("Ethiopic", ETHIOPIC), + ("Garay", GARAY), + ("Georgian", GEORGIAN), + ("Glagolitic", GLAGOLITIC), + ("Gothic", GOTHIC), + ("Grantha", GRANTHA), + ("Greek", GREEK), + ("Gujarati", GUJARATI), + ("Gunjala_Gondi", GUNJALA_GONDI), + ("Gurmukhi", GURMUKHI), + ("Gurung_Khema", GURUNG_KHEMA), + ("Han", HAN), + ("Hangul", HANGUL), + ("Hanifi_Rohingya", HANIFI_ROHINGYA), + ("Hanunoo", HANUNOO), + ("Hatran", HATRAN), + ("Hebrew", HEBREW), + ("Hiragana", HIRAGANA), + ("Imperial_Aramaic", IMPERIAL_ARAMAIC), + ("Inherited", INHERITED), + ("Inscriptional_Pahlavi", INSCRIPTIONAL_PAHLAVI), + ("Inscriptional_Parthian", INSCRIPTIONAL_PARTHIAN), + ("Javanese", JAVANESE), + ("Kaithi", KAITHI), + ("Kannada", KANNADA), + ("Katakana", KATAKANA), + ("Kawi", KAWI), + ("Kayah_Li", KAYAH_LI), + ("Kharoshthi", KHAROSHTHI), + ("Khitan_Small_Script", KHITAN_SMALL_SCRIPT), + ("Khmer", KHMER), + ("Khojki", KHOJKI), + ("Khudawadi", KHUDAWADI), + ("Kirat_Rai", KIRAT_RAI), + ("Lao", LAO), + ("Latin", LATIN), + ("Lepcha", LEPCHA), + ("Limbu", LIMBU), + ("Linear_A", LINEAR_A), + ("Linear_B", LINEAR_B), + ("Lisu", LISU), + ("Lycian", LYCIAN), + ("Lydian", LYDIAN), + ("Mahajani", MAHAJANI), + ("Makasar", MAKASAR), + ("Malayalam", MALAYALAM), + ("Mandaic", MANDAIC), + ("Manichaean", MANICHAEAN), + ("Marchen", MARCHEN), + ("Masaram_Gondi", MASARAM_GONDI), + ("Medefaidrin", MEDEFAIDRIN), + ("Meetei_Mayek", MEETEI_MAYEK), + ("Mende_Kikakui", MENDE_KIKAKUI), + ("Meroitic_Cursive", MEROITIC_CURSIVE), + ("Meroitic_Hieroglyphs", MEROITIC_HIEROGLYPHS), + ("Miao", MIAO), + ("Modi", MODI), + ("Mongolian", MONGOLIAN), + ("Mro", MRO), + ("Multani", MULTANI), + ("Myanmar", MYANMAR), + ("Nabataean", NABATAEAN), + ("Nag_Mundari", NAG_MUNDARI), + ("Nandinagari", NANDINAGARI), + ("New_Tai_Lue", NEW_TAI_LUE), + ("Newa", NEWA), + ("Nko", NKO), + ("Nushu", NUSHU), + ("Nyiakeng_Puachue_Hmong", NYIAKENG_PUACHUE_HMONG), + ("Ogham", OGHAM), + ("Ol_Chiki", OL_CHIKI), + ("Ol_Onal", OL_ONAL), + ("Old_Hungarian", OLD_HUNGARIAN), + ("Old_Italic", OLD_ITALIC), + ("Old_North_Arabian", OLD_NORTH_ARABIAN), + ("Old_Permic", OLD_PERMIC), + ("Old_Persian", OLD_PERSIAN), + ("Old_Sogdian", OLD_SOGDIAN), + ("Old_South_Arabian", OLD_SOUTH_ARABIAN), + ("Old_Turkic", OLD_TURKIC), + ("Old_Uyghur", OLD_UYGHUR), + ("Oriya", ORIYA), + ("Osage", OSAGE), + ("Osmanya", OSMANYA), + ("Pahawh_Hmong", PAHAWH_HMONG), + ("Palmyrene", PALMYRENE), + ("Pau_Cin_Hau", PAU_CIN_HAU), + ("Phags_Pa", PHAGS_PA), + ("Phoenician", PHOENICIAN), + ("Psalter_Pahlavi", PSALTER_PAHLAVI), + ("Rejang", REJANG), + ("Runic", RUNIC), + ("Samaritan", SAMARITAN), + ("Saurashtra", SAURASHTRA), + ("Sharada", SHARADA), + ("Shavian", SHAVIAN), + ("Siddham", SIDDHAM), + ("SignWriting", SIGNWRITING), + ("Sinhala", SINHALA), + ("Sogdian", SOGDIAN), + ("Sora_Sompeng", SORA_SOMPENG), + ("Soyombo", SOYOMBO), + ("Sundanese", SUNDANESE), + ("Sunuwar", SUNUWAR), + ("Syloti_Nagri", SYLOTI_NAGRI), + ("Syriac", SYRIAC), + ("Tagalog", TAGALOG), + ("Tagbanwa", TAGBANWA), + ("Tai_Le", TAI_LE), + ("Tai_Tham", TAI_THAM), + ("Tai_Viet", TAI_VIET), + ("Takri", TAKRI), + ("Tamil", TAMIL), + ("Tangsa", TANGSA), + ("Tangut", TANGUT), + ("Telugu", TELUGU), + ("Thaana", THAANA), + ("Thai", THAI), + ("Tibetan", TIBETAN), + ("Tifinagh", TIFINAGH), + ("Tirhuta", TIRHUTA), + ("Todhri", TODHRI), + ("Toto", TOTO), + ("Tulu_Tigalari", TULU_TIGALARI), + ("Ugaritic", UGARITIC), + ("Vai", VAI), + ("Vithkuqi", VITHKUQI), + ("Wancho", WANCHO), + ("Warang_Citi", WARANG_CITI), + ("Yezidi", YEZIDI), + ("Yi", YI), + ("Zanabazar_Square", ZANABAZAR_SQUARE), +]; + +pub const ADLAM: &'static [(char, char)] = &[ + ('؟', '؟'), + ('ـ', 'ـ'), + ('⁏', '⁏'), + ('⹁', '⹁'), + ('𞤀', '𞥋'), + ('𞥐', '𞥙'), + ('𞥞', '𞥟'), +]; + +pub const AHOM: &'static [(char, char)] = + &[('𑜀', '𑜚'), ('\u{1171d}', '\u{1172b}'), ('𑜰', '𑝆')]; + +pub const ANATOLIAN_HIEROGLYPHS: &'static [(char, char)] = &[('𔐀', '𔙆')]; + +pub const ARABIC: &'static [(char, char)] = &[ + ('\u{600}', '\u{604}'), + ('؆', '\u{6dc}'), + ('۞', 'ۿ'), + ('ݐ', 'ݿ'), + ('ࡰ', 'ࢎ'), + ('\u{890}', '\u{891}'), + ('\u{897}', '\u{8e1}'), + ('\u{8e3}', '\u{8ff}'), + ('⁏', '⁏'), + ('⹁', '⹁'), + ('ﭐ', '﯂'), + ('ﯓ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('﷏', '﷏'), + ('ﷰ', '﷿'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('\u{102e0}', '𐋻'), + ('𐹠', '𐹾'), + ('𐻂', '𐻄'), + ('\u{10efc}', '\u{10eff}'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𞻰', '𞻱'), +]; + +pub const ARMENIAN: &'static [(char, char)] = + &[('\u{308}', '\u{308}'), ('Ա', 'Ֆ'), ('ՙ', '֊'), ('֍', '֏'), ('ﬓ', 'ﬗ')]; + +pub const AVESTAN: &'static [(char, char)] = + &[('·', '·'), ('⸰', '⸱'), ('𐬀', '𐬵'), ('𐬹', '𐬿')]; + +pub const BALINESE: &'static [(char, char)] = &[('\u{1b00}', 'ᭌ'), ('᭎', '᭿')]; + +pub const BAMUM: &'static [(char, char)] = &[('ꚠ', '꛷'), ('𖠀', '𖨸')]; + +pub const BASSA_VAH: &'static [(char, char)] = + &[('𖫐', '𖫭'), ('\u{16af0}', '𖫵')]; + +pub const BATAK: &'static [(char, char)] = &[('ᯀ', '\u{1bf3}'), ('᯼', '᯿')]; + +pub const BENGALI: &'static [(char, char)] = &[ + ('ʼ', 'ʼ'), + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('ঀ', 'ঃ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('\u{9bc}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', 'ৎ'), + ('\u{9d7}', '\u{9d7}'), + ('ড়', 'ঢ়'), + ('য়', '\u{9e3}'), + ('০', '\u{9fe}'), + ('\u{1cd0}', '\u{1cd0}'), + ('\u{1cd2}', '\u{1cd2}'), + ('\u{1cd5}', '\u{1cd6}'), + ('\u{1cd8}', '\u{1cd8}'), + ('᳡', '᳡'), + ('ᳪ', 'ᳪ'), + ('\u{1ced}', '\u{1ced}'), + ('ᳲ', 'ᳲ'), + ('ᳵ', '᳷'), + ('\u{a8f1}', '\u{a8f1}'), +]; + +pub const BHAIKSUKI: &'static [(char, char)] = + &[('𑰀', '𑰈'), ('𑰊', '\u{11c36}'), ('\u{11c38}', '𑱅'), ('𑱐', '𑱬')]; + +pub const BOPOMOFO: &'static [(char, char)] = &[ + ('ˇ', 'ˇ'), + ('ˉ', 'ˋ'), + ('˙', '˙'), + ('˪', '˫'), + ('、', '〃'), + ('〈', '】'), + ('〓', '〟'), + ('\u{302a}', '\u{302d}'), + ('〰', '〰'), + ('〷', '〷'), + ('・', '・'), + ('ㄅ', 'ㄯ'), + ('ㆠ', 'ㆿ'), + ('﹅', '﹆'), + ('｡', '･'), +]; + +pub const BRAHMI: &'static [(char, char)] = + &[('𑀀', '𑁍'), ('𑁒', '𑁵'), ('\u{1107f}', '\u{1107f}')]; + +pub const BRAILLE: &'static [(char, char)] = &[('⠀', '⣿')]; + +pub const BUGINESE: &'static [(char, char)] = + &[('ᨀ', '\u{1a1b}'), ('᨞', '᨟'), ('ꧏ', 'ꧏ')]; + +pub const BUHID: &'static [(char, char)] = &[('᜵', '᜶'), ('ᝀ', '\u{1753}')]; + +pub const CANADIAN_ABORIGINAL: &'static [(char, char)] = + &[('᐀', 'ᙿ'), ('ᢰ', 'ᣵ'), ('𑪰', '𑪿')]; + +pub const CARIAN: &'static [(char, char)] = + &[('·', '·'), ('⁚', '⁚'), ('⁝', '⁝'), ('⸱', '⸱'), ('𐊠', '𐋐')]; + +pub const CAUCASIAN_ALBANIAN: &'static [(char, char)] = &[ + ('\u{304}', '\u{304}'), + ('\u{331}', '\u{331}'), + ('\u{35e}', '\u{35e}'), + ('𐔰', '𐕣'), + ('𐕯', '𐕯'), +]; + +pub const CHAKMA: &'static [(char, char)] = + &[('০', '৯'), ('၀', '၉'), ('\u{11100}', '\u{11134}'), ('𑄶', '𑅇')]; + +pub const CHAM: &'static [(char, char)] = + &[('ꨀ', '\u{aa36}'), ('ꩀ', 'ꩍ'), ('꩐', '꩙'), ('꩜', '꩟')]; + +pub const CHEROKEE: &'static [(char, char)] = &[ + ('\u{300}', '\u{302}'), + ('\u{304}', '\u{304}'), + ('\u{30b}', '\u{30c}'), + ('\u{323}', '\u{324}'), + ('\u{330}', '\u{331}'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ꭰ', 'ꮿ'), +]; + +pub const CHORASMIAN: &'static [(char, char)] = &[('𐾰', '𐿋')]; + +pub const COMMON: &'static [(char, char)] = &[ + ('\0', '@'), + ('[', '`'), + ('{', '©'), + ('«', '¶'), + ('¸', '¹'), + ('»', '¿'), + ('×', '×'), + ('÷', '÷'), + ('ʹ', 'ʻ'), + ('ʽ', 'ˆ'), + ('ˈ', 'ˈ'), + ('ˌ', 'ˌ'), + ('ˎ', '˖'), + ('˘', '˘'), + ('˚', '˟'), + ('˥', '˩'), + ('ˬ', '˿'), + (';', ';'), + ('΅', '΅'), + ('·', '·'), + ('\u{605}', '\u{605}'), + ('\u{6dd}', '\u{6dd}'), + ('\u{8e2}', '\u{8e2}'), + ('฿', '฿'), + ('࿕', '࿘'), + ('\u{2000}', '\u{200b}'), + ('\u{200e}', '\u{202e}'), + ('‰', '⁎'), + ('⁐', '⁙'), + ('⁛', '⁜'), + ('⁞', '\u{2064}'), + ('\u{2066}', '⁰'), + ('⁴', '⁾'), + ('₀', '₎'), + ('₠', '⃀'), + ('℀', '℥'), + ('℧', '℩'), + ('ℬ', 'ℱ'), + ('ℳ', '⅍'), + ('⅏', '⅟'), + ('↉', '↋'), + ('←', '␩'), + ('⑀', '⑊'), + ('①', '⟿'), + ('⤀', '⭳'), + ('⭶', '⮕'), + ('⮗', '⯿'), + ('⸀', '⸖'), + ('⸘', 'ⸯ'), + ('⸲', '⸻'), + ('⸽', '⹀'), + ('⹂', '⹂'), + ('⹄', '⹝'), + ('\u{3000}', '\u{3000}'), + ('〄', '〄'), + ('〒', '〒'), + ('〠', '〠'), + ('〶', '〶'), + ('㉈', '㉟'), + ('㉿', '㉿'), + ('㊱', '㊿'), + ('㋌', '㋏'), + ('㍱', '㍺'), + ('㎀', '㏟'), + ('㏿', '㏿'), + ('䷀', '䷿'), + ('꜈', '꜡'), + ('ꞈ', '꞊'), + ('꭛', '꭛'), + ('꭪', '꭫'), + ('︐', '︙'), + ('︰', '﹄'), + ('﹇', '﹒'), + ('﹔', '﹦'), + ('﹨', '﹫'), + ('\u{feff}', '\u{feff}'), + ('！', '＠'), + ('［', '｀'), + ('｛', '｠'), + ('￠', '￦'), + ('￨', '￮'), + ('\u{fff9}', '�'), + ('𐆐', '𐆜'), + ('𐇐', '𐇼'), + ('𜰀', '𜳹'), + ('𜴀', '𜺳'), + ('𜽐', '𜿃'), + ('𝀀', '𝃵'), + ('𝄀', '𝄦'), + ('𝄩', '\u{1d166}'), + ('𝅪', '\u{1d17a}'), + ('𝆃', '𝆄'), + ('𝆌', '𝆩'), + ('𝆮', '𝇪'), + ('𝋀', '𝋓'), + ('𝋠', '𝋳'), + ('𝌀', '𝍖'), + ('𝍲', '𝍸'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝟋'), + ('𝟎', '𝟿'), + ('𞱱', '𞲴'), + ('𞴁', '𞴽'), + ('🀀', '🀫'), + ('🀰', '🂓'), + ('🂠', '🂮'), + ('🂱', '🂿'), + ('🃁', '🃏'), + ('🃑', '🃵'), + ('🄀', '🆭'), + ('🇦', '🇿'), + ('🈁', '🈂'), + ('🈐', '🈻'), + ('🉀', '🉈'), + ('🉠', '🉥'), + ('🌀', '🛗'), + ('🛜', '🛬'), + ('🛰', '🛼'), + ('🜀', '🝶'), + ('🝻', '🟙'), + ('🟠', '🟫'), + ('🟰', '🟰'), + ('🠀', '🠋'), + ('🠐', '🡇'), + ('🡐', '🡙'), + ('🡠', '🢇'), + ('🢐', '🢭'), + ('🢰', '🢻'), + ('🣀', '🣁'), + ('🤀', '🩓'), + ('🩠', '🩭'), + ('🩰', '🩼'), + ('🪀', '🪉'), + ('🪏', '🫆'), + ('🫎', '🫜'), + ('🫟', '🫩'), + ('🫰', '🫸'), + ('🬀', '🮒'), + ('🮔', '🯹'), + ('\u{e0001}', '\u{e0001}'), + ('\u{e0020}', '\u{e007f}'), +]; + +pub const COPTIC: &'static [(char, char)] = &[ + ('·', '·'), + ('\u{300}', '\u{300}'), + ('\u{304}', '\u{305}'), + ('\u{307}', '\u{307}'), + ('ʹ', '͵'), + ('Ϣ', 'ϯ'), + ('Ⲁ', 'ⳳ'), + ('⳹', '⳿'), + ('⸗', '⸗'), + ('\u{102e0}', '𐋻'), +]; + +pub const CUNEIFORM: &'static [(char, char)] = + &[('𒀀', '𒎙'), ('𒐀', '𒑮'), ('𒑰', '𒑴'), ('𒒀', '𒕃')]; + +pub const CYPRIOT: &'static [(char, char)] = &[ + ('𐄀', '𐄂'), + ('𐄇', '𐄳'), + ('𐄷', '𐄿'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐠿'), +]; + +pub const CYPRO_MINOAN: &'static [(char, char)] = &[('𐄀', '𐄁'), ('𒾐', '𒿲')]; + +pub const CYRILLIC: &'static [(char, char)] = &[ + ('ʼ', 'ʼ'), + ('\u{300}', '\u{302}'), + ('\u{304}', '\u{304}'), + ('\u{306}', '\u{306}'), + ('\u{308}', '\u{308}'), + ('\u{30b}', '\u{30b}'), + ('\u{311}', '\u{311}'), + ('Ѐ', 'ԯ'), + ('ᲀ', 'ᲊ'), + ('ᴫ', 'ᴫ'), + ('ᵸ', 'ᵸ'), + ('\u{1df8}', '\u{1df8}'), + ('\u{2de0}', '\u{2dff}'), + ('⹃', '⹃'), + ('Ꙁ', '\u{a69f}'), + ('\u{fe2e}', '\u{fe2f}'), + ('𞀰', '𞁭'), + ('\u{1e08f}', '\u{1e08f}'), +]; + +pub const DESERET: &'static [(char, char)] = &[('𐐀', '𐑏')]; + +pub const DEVANAGARI: &'static [(char, char)] = &[ + ('ʼ', 'ʼ'), + ('\u{900}', '\u{952}'), + ('\u{955}', 'ॿ'), + ('\u{1cd0}', 'ᳶ'), + ('\u{1cf8}', '\u{1cf9}'), + ('\u{20f0}', '\u{20f0}'), + ('꠰', '꠹'), + ('\u{a8e0}', '\u{a8ff}'), + ('𑬀', '𑬉'), +]; + +pub const DIVES_AKURU: &'static [(char, char)] = &[ + ('𑤀', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '𑥆'), + ('𑥐', '𑥙'), +]; + +pub const DOGRA: &'static [(char, char)] = + &[('।', '९'), ('꠰', '꠹'), ('𑠀', '𑠻')]; + +pub const DUPLOYAN: &'static [(char, char)] = &[ + ('·', '·'), + ('\u{307}', '\u{308}'), + ('\u{30a}', '\u{30a}'), + ('\u{323}', '\u{324}'), + ('⸼', '⸼'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𛲜', '\u{1bca3}'), +]; + +pub const EGYPTIAN_HIEROGLYPHS: &'static [(char, char)] = + &[('𓀀', '\u{13455}'), ('𓑠', '𔏺')]; + +pub const ELBASAN: &'static [(char, char)] = + &[('·', '·'), ('\u{305}', '\u{305}'), ('𐔀', '𐔧')]; + +pub const ELYMAIC: &'static [(char, char)] = &[('𐿠', '𐿶')]; + +pub const ETHIOPIC: &'static [(char, char)] = &[ + ('\u{30e}', '\u{30e}'), + ('ሀ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('\u{135d}', '፼'), + ('ᎀ', '᎙'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), +]; + +pub const GARAY: &'static [(char, char)] = &[ + ('،', '،'), + ('؛', '؛'), + ('؟', '؟'), + ('𐵀', '𐵥'), + ('\u{10d69}', '𐶅'), + ('𐶎', '𐶏'), +]; + +pub const GEORGIAN: &'static [(char, char)] = &[ + ('·', '·'), + ('։', '։'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჿ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('⁚', '⁚'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('⸱', '⸱'), +]; + +pub const GLAGOLITIC: &'static [(char, char)] = &[ + ('·', '·'), + ('\u{303}', '\u{303}'), + ('\u{305}', '\u{305}'), + ('\u{484}', '\u{484}'), + ('\u{487}', '\u{487}'), + ('։', '։'), + ('჻', '჻'), + ('⁚', '⁚'), + ('Ⰰ', 'ⱟ'), + ('⹃', '⹃'), + ('\u{a66f}', '\u{a66f}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), +]; + +pub const GOTHIC: &'static [(char, char)] = &[ + ('·', '·'), + ('\u{304}', '\u{305}'), + ('\u{308}', '\u{308}'), + ('\u{331}', '\u{331}'), + ('𐌰', '𐍊'), +]; + +pub const GRANTHA: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('௦', '௳'), + ('\u{1cd0}', '\u{1cd0}'), + ('\u{1cd2}', '᳓'), + ('ᳲ', '\u{1cf4}'), + ('\u{1cf8}', '\u{1cf9}'), + ('\u{20f0}', '\u{20f0}'), + ('\u{11300}', '𑌃'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('\u{1133b}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('𑍐', '𑍐'), + ('\u{11357}', '\u{11357}'), + ('𑍝', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('𑿐', '𑿑'), + ('𑿓', '𑿓'), +]; + +pub const GREEK: &'static [(char, char)] = &[ + ('·', '·'), + ('\u{300}', '\u{301}'), + ('\u{304}', '\u{304}'), + ('\u{306}', '\u{306}'), + ('\u{308}', '\u{308}'), + ('\u{313}', '\u{313}'), + ('\u{342}', '\u{342}'), + ('\u{345}', '\u{345}'), + ('Ͱ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('΄', '΄'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϡ'), + ('ϰ', 'Ͽ'), + ('ᴦ', 'ᴪ'), + ('ᵝ', 'ᵡ'), + ('ᵦ', 'ᵪ'), + ('ᶿ', '\u{1dc1}'), + ('ἀ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ῄ'), + ('ῆ', 'ΐ'), + ('ῖ', 'Ί'), + ('῝', '`'), + ('ῲ', 'ῴ'), + ('ῶ', '῾'), + ('⁝', '⁝'), + ('Ω', 'Ω'), + ('ꭥ', 'ꭥ'), + ('𐅀', '𐆎'), + ('𐆠', '𐆠'), + ('𝈀', '𝉅'), +]; + +pub const GUJARATI: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('\u{a81}', 'ઃ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('\u{abc}', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('ૐ', 'ૐ'), + ('ૠ', '\u{ae3}'), + ('૦', '૱'), + ('ૹ', '\u{aff}'), + ('꠰', '꠹'), +]; + +pub const GUNJALA_GONDI: &'static [(char, char)] = &[ + ('·', '·'), + ('।', '॥'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '𑶘'), + ('𑶠', '𑶩'), +]; + +pub const GURMUKHI: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('\u{a01}', 'ਃ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('੦', '੶'), + ('꠰', '꠹'), +]; + +pub const GURUNG_KHEMA: &'static [(char, char)] = &[('॥', '॥'), ('𖄀', '𖄹')]; + +pub const HAN: &'static [(char, char)] = &[ + ('·', '·'), + ('⺀', '⺙'), + ('⺛', '⻳'), + ('⼀', '⿕'), + ('⿰', '⿿'), + ('、', '〃'), + ('々', '】'), + ('〓', '〟'), + ('〡', '\u{302d}'), + ('〰', '〰'), + ('〷', '〿'), + ('・', '・'), + ('㆐', '㆟'), + ('㇀', '㇥'), + ('㇯', '㇯'), + ('㈠', '㉇'), + ('㊀', '㊰'), + ('㋀', '㋋'), + ('㋿', '㋿'), + ('㍘', '㍰'), + ('㍻', '㍿'), + ('㏠', '㏾'), + ('㐀', '䶿'), + ('一', '鿿'), + ('꜀', '꜇'), + ('豈', '舘'), + ('並', '龎'), + ('﹅', '﹆'), + ('｡', '･'), + ('𖿢', '𖿣'), + ('\u{16ff0}', '\u{16ff1}'), + ('𝍠', '𝍱'), + ('🉐', '🉑'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const HANGUL: &'static [(char, char)] = &[ + ('ᄀ', 'ᇿ'), + ('、', '〃'), + ('〈', '】'), + ('〓', '〟'), + ('\u{302e}', '〰'), + ('〷', '〷'), + ('・', '・'), + ('ㄱ', 'ㆎ'), + ('㈀', '㈞'), + ('㉠', '㉾'), + ('ꥠ', 'ꥼ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('﹅', '﹆'), + ('｡', '･'), + ('ﾠ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), +]; + +pub const HANIFI_ROHINGYA: &'static [(char, char)] = &[ + ('،', '،'), + ('؛', '؛'), + ('؟', '؟'), + ('ـ', 'ـ'), + ('۔', '۔'), + ('𐴀', '\u{10d27}'), + ('𐴰', '𐴹'), +]; + +pub const HANUNOO: &'static [(char, char)] = &[('ᜠ', '᜶')]; + +pub const HATRAN: &'static [(char, char)] = + &[('𐣠', '𐣲'), ('𐣴', '𐣵'), ('𐣻', '𐣿')]; + +pub const HEBREW: &'static [(char, char)] = &[ + ('\u{307}', '\u{308}'), + ('\u{591}', '\u{5c7}'), + ('א', 'ת'), + ('ׯ', '״'), + ('יִ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﭏ'), +]; + +pub const HIRAGANA: &'static [(char, char)] = &[ + ('、', '〃'), + ('〈', '】'), + ('〓', '〟'), + ('〰', '〵'), + ('〷', '〷'), + ('〼', '〽'), + ('ぁ', 'ゖ'), + ('\u{3099}', '゠'), + ('・', 'ー'), + ('﹅', '﹆'), + ('｡', '･'), + ('ｰ', 'ｰ'), + ('\u{ff9e}', '\u{ff9f}'), + ('𛀁', '𛄟'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('🈀', '🈀'), +]; + +pub const IMPERIAL_ARAMAIC: &'static [(char, char)] = + &[('𐡀', '𐡕'), ('𐡗', '𐡟')]; + +pub const INHERITED: &'static [(char, char)] = &[ + ('\u{30f}', '\u{30f}'), + ('\u{312}', '\u{312}'), + ('\u{314}', '\u{31f}'), + ('\u{321}', '\u{322}'), + ('\u{326}', '\u{32c}'), + ('\u{32f}', '\u{32f}'), + ('\u{332}', '\u{341}'), + ('\u{343}', '\u{344}'), + ('\u{346}', '\u{357}'), + ('\u{359}', '\u{35d}'), + ('\u{35f}', '\u{362}'), + ('\u{953}', '\u{954}'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1dc2}', '\u{1df7}'), + ('\u{1df9}', '\u{1df9}'), + ('\u{1dfb}', '\u{1dff}'), + ('\u{200c}', '\u{200d}'), + ('\u{20d0}', '\u{20ef}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2d}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d167}', '\u{1d169}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const INSCRIPTIONAL_PAHLAVI: &'static [(char, char)] = + &[('𐭠', '𐭲'), ('𐭸', '𐭿')]; + +pub const INSCRIPTIONAL_PARTHIAN: &'static [(char, char)] = + &[('𐭀', '𐭕'), ('𐭘', '𐭟')]; + +pub const JAVANESE: &'static [(char, char)] = + &[('\u{a980}', '꧍'), ('ꧏ', '꧙'), ('꧞', '꧟')]; + +pub const KAITHI: &'static [(char, char)] = &[ + ('०', '९'), + ('⸱', '⸱'), + ('꠰', '꠹'), + ('\u{11080}', '\u{110c2}'), + ('\u{110cd}', '\u{110cd}'), +]; + +pub const KANNADA: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('ಀ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('\u{cbc}', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('ೝ', 'ೞ'), + ('ೠ', '\u{ce3}'), + ('೦', '೯'), + ('ೱ', 'ೳ'), + ('\u{1cd0}', '\u{1cd0}'), + ('\u{1cd2}', '᳓'), + ('\u{1cda}', '\u{1cda}'), + ('ᳲ', 'ᳲ'), + ('\u{1cf4}', '\u{1cf4}'), + ('꠰', '꠵'), +]; + +pub const KATAKANA: &'static [(char, char)] = &[ + ('\u{305}', '\u{305}'), + ('\u{323}', '\u{323}'), + ('、', '〃'), + ('〈', '】'), + ('〓', '〟'), + ('〰', '〵'), + ('〷', '〷'), + ('〼', '〽'), + ('\u{3099}', '゜'), + ('゠', 'ヿ'), + ('ㇰ', 'ㇿ'), + ('㋐', '㋾'), + ('㌀', '㍗'), + ('﹅', '﹆'), + ('｡', '\u{ff9f}'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛀀'), + ('𛄠', '𛄢'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), +]; + +pub const KAWI: &'static [(char, char)] = + &[('\u{11f00}', '𑼐'), ('𑼒', '\u{11f3a}'), ('𑼾', '\u{11f5a}')]; + +pub const KAYAH_LI: &'static [(char, char)] = &[('꤀', '꤯')]; + +pub const KHAROSHTHI: &'static [(char, char)] = &[ + ('𐨀', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '𐩈'), + ('𐩐', '𐩘'), +]; + +pub const KHITAN_SMALL_SCRIPT: &'static [(char, char)] = + &[('\u{16fe4}', '\u{16fe4}'), ('𘬀', '𘳕'), ('𘳿', '𘳿')]; + +pub const KHMER: &'static [(char, char)] = + &[('ក', '\u{17dd}'), ('០', '៩'), ('៰', '៹'), ('᧠', '᧿')]; + +pub const KHOJKI: &'static [(char, char)] = + &[('૦', '૯'), ('꠰', '꠹'), ('𑈀', '𑈑'), ('𑈓', '\u{11241}')]; + +pub const KHUDAWADI: &'static [(char, char)] = + &[('।', '॥'), ('꠰', '꠹'), ('𑊰', '\u{112ea}'), ('𑋰', '𑋹')]; + +pub const KIRAT_RAI: &'static [(char, char)] = &[('𖵀', '𖵹')]; + +pub const LAO: &'static [(char, char)] = &[ + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('\u{ec8}', '\u{ece}'), + ('໐', '໙'), + ('ໜ', 'ໟ'), +]; + +pub const LATIN: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('ª', 'ª'), + ('·', '·'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', 'ʸ'), + ('ʼ', 'ʼ'), + ('ˇ', 'ˇ'), + ('ˉ', 'ˋ'), + ('ˍ', 'ˍ'), + ('˗', '˗'), + ('˙', '˙'), + ('ˠ', 'ˤ'), + ('\u{300}', '\u{30e}'), + ('\u{310}', '\u{311}'), + ('\u{313}', '\u{313}'), + ('\u{320}', '\u{320}'), + ('\u{323}', '\u{325}'), + ('\u{32d}', '\u{32e}'), + ('\u{330}', '\u{331}'), + ('\u{358}', '\u{358}'), + ('\u{35e}', '\u{35e}'), + ('\u{363}', '\u{36f}'), + ('\u{485}', '\u{486}'), + ('\u{951}', '\u{952}'), + ('჻', '჻'), + ('ᴀ', 'ᴥ'), + ('ᴬ', 'ᵜ'), + ('ᵢ', 'ᵥ'), + ('ᵫ', 'ᵷ'), + ('ᵹ', 'ᶾ'), + ('\u{1df8}', '\u{1df8}'), + ('Ḁ', 'ỿ'), + ('\u{202f}', '\u{202f}'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('\u{20f0}', '\u{20f0}'), + ('K', 'Å'), + ('Ⅎ', 'Ⅎ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⱡ', 'Ɀ'), + ('⸗', '⸗'), + ('꜀', '꜇'), + ('Ꜣ', 'ꞇ'), + ('Ꞌ', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꟿ'), + ('꤮', '꤮'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭤ'), + ('ꭦ', 'ꭩ'), + ('ff', 'st'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), +]; + +pub const LEPCHA: &'static [(char, char)] = + &[('ᰀ', '\u{1c37}'), ('᰻', '᱉'), ('ᱍ', 'ᱏ')]; + +pub const LIMBU: &'static [(char, char)] = &[ + ('॥', '॥'), + ('ᤀ', 'ᤞ'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('᥀', '᥀'), + ('᥄', '᥏'), +]; + +pub const LINEAR_A: &'static [(char, char)] = + &[('𐄇', '𐄳'), ('𐘀', '𐜶'), ('𐝀', '𐝕'), ('𐝠', '𐝧')]; + +pub const LINEAR_B: &'static [(char, char)] = &[ + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐄀', '𐄂'), + ('𐄇', '𐄳'), + ('𐄷', '𐄿'), +]; + +pub const LISU: &'static [(char, char)] = + &[('ʼ', 'ʼ'), ('ˍ', 'ˍ'), ('《', '》'), ('ꓐ', '꓿'), ('𑾰', '𑾰')]; + +pub const LYCIAN: &'static [(char, char)] = &[('⁚', '⁚'), ('𐊀', '𐊜')]; + +pub const LYDIAN: &'static [(char, char)] = + &[('·', '·'), ('⸱', '⸱'), ('𐤠', '𐤹'), ('𐤿', '𐤿')]; + +pub const MAHAJANI: &'static [(char, char)] = + &[('·', '·'), ('।', '९'), ('꠰', '꠹'), ('𑅐', '𑅶')]; + +pub const MAKASAR: &'static [(char, char)] = &[('𑻠', '𑻸')]; + +pub const MALAYALAM: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('\u{d00}', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', '൏'), + ('ൔ', '\u{d63}'), + ('൦', 'ൿ'), + ('\u{1cda}', '\u{1cda}'), + ('ᳲ', 'ᳲ'), + ('꠰', '꠲'), +]; + +pub const MANDAIC: &'static [(char, char)] = + &[('ـ', 'ـ'), ('ࡀ', '\u{85b}'), ('࡞', '࡞')]; + +pub const MANICHAEAN: &'static [(char, char)] = + &[('ـ', 'ـ'), ('𐫀', '\u{10ae6}'), ('𐫫', '𐫶')]; + +pub const MARCHEN: &'static [(char, char)] = + &[('𑱰', '𑲏'), ('\u{11c92}', '\u{11ca7}'), ('𑲩', '\u{11cb6}')]; + +pub const MASARAM_GONDI: &'static [(char, char)] = &[ + ('।', '॥'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d47}'), + ('𑵐', '𑵙'), +]; + +pub const MEDEFAIDRIN: &'static [(char, char)] = &[('𖹀', '𖺚')]; + +pub const MEETEI_MAYEK: &'static [(char, char)] = + &[('ꫠ', '\u{aaf6}'), ('ꯀ', '\u{abed}'), ('꯰', '꯹')]; + +pub const MENDE_KIKAKUI: &'static [(char, char)] = + &[('𞠀', '𞣄'), ('𞣇', '\u{1e8d6}')]; + +pub const MEROITIC_CURSIVE: &'static [(char, char)] = + &[('𐦠', '𐦷'), ('𐦼', '𐧏'), ('𐧒', '𐧿')]; + +pub const MEROITIC_HIEROGLYPHS: &'static [(char, char)] = + &[('⁝', '⁝'), ('𐦀', '𐦟')]; + +pub const MIAO: &'static [(char, char)] = + &[('𖼀', '𖽊'), ('\u{16f4f}', '𖾇'), ('\u{16f8f}', '𖾟')]; + +pub const MODI: &'static [(char, char)] = + &[('꠰', '꠹'), ('𑘀', '𑙄'), ('𑙐', '𑙙')]; + +pub const MONGOLIAN: &'static [(char, char)] = &[ + ('᠀', '᠙'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢪ'), + ('\u{202f}', '\u{202f}'), + ('、', '。'), + ('〈', '》'), + ('𑙠', '𑙬'), +]; + +pub const MRO: &'static [(char, char)] = &[('𖩀', '𖩞'), ('𖩠', '𖩩'), ('𖩮', '𖩯')]; + +pub const MULTANI: &'static [(char, char)] = + &[('੦', '੯'), ('𑊀', '𑊆'), ('𑊈', '𑊈'), ('𑊊', '𑊍'), ('𑊏', '𑊝'), ('𑊟', '𑊩')]; + +pub const MYANMAR: &'static [(char, char)] = + &[('က', '႟'), ('꤮', '꤮'), ('ꧠ', 'ꧾ'), ('ꩠ', 'ꩿ'), ('𑛐', '𑛣')]; + +pub const NABATAEAN: &'static [(char, char)] = &[('𐢀', '𐢞'), ('𐢧', '𐢯')]; + +pub const NAG_MUNDARI: &'static [(char, char)] = &[('𞓐', '𞓹')]; + +pub const NANDINAGARI: &'static [(char, char)] = &[ + ('।', '॥'), + ('೦', '೯'), + ('ᳩ', 'ᳩ'), + ('ᳲ', 'ᳲ'), + ('ᳺ', 'ᳺ'), + ('꠰', '꠵'), + ('𑦠', '𑦧'), + ('𑦪', '\u{119d7}'), + ('\u{119da}', '𑧤'), +]; + +pub const NEW_TAI_LUE: &'static [(char, char)] = + &[('ᦀ', 'ᦫ'), ('ᦰ', 'ᧉ'), ('᧐', '᧚'), ('᧞', '᧟')]; + +pub const NEWA: &'static [(char, char)] = &[('𑐀', '𑑛'), ('𑑝', '𑑡')]; + +pub const NKO: &'static [(char, char)] = &[ + ('،', '،'), + ('؛', '؛'), + ('؟', '؟'), + ('߀', 'ߺ'), + ('\u{7fd}', '߿'), + ('﴾', '﴿'), +]; + +pub const NUSHU: &'static [(char, char)] = &[('𖿡', '𖿡'), ('𛅰', '𛋻')]; + +pub const NYIAKENG_PUACHUE_HMONG: &'static [(char, char)] = + &[('𞄀', '𞄬'), ('\u{1e130}', '𞄽'), ('𞅀', '𞅉'), ('𞅎', '𞅏')]; + +pub const OGHAM: &'static [(char, char)] = &[('\u{1680}', '᚜')]; + +pub const OL_CHIKI: &'static [(char, char)] = &[('᱐', '᱿')]; + +pub const OL_ONAL: &'static [(char, char)] = + &[('।', '॥'), ('𞗐', '𞗺'), ('𞗿', '𞗿')]; + +pub const OLD_HUNGARIAN: &'static [(char, char)] = &[ + ('⁚', '⁚'), + ('⁝', '⁝'), + ('⸱', '⸱'), + ('⹁', '⹁'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐳺', '𐳿'), +]; + +pub const OLD_ITALIC: &'static [(char, char)] = &[('𐌀', '𐌣'), ('𐌭', '𐌯')]; + +pub const OLD_NORTH_ARABIAN: &'static [(char, char)] = &[('𐪀', '𐪟')]; + +pub const OLD_PERMIC: &'static [(char, char)] = &[ + ('·', '·'), + ('\u{300}', '\u{300}'), + ('\u{306}', '\u{308}'), + ('\u{313}', '\u{313}'), + ('\u{483}', '\u{483}'), + ('𐍐', '\u{1037a}'), +]; + +pub const OLD_PERSIAN: &'static [(char, char)] = &[('𐎠', '𐏃'), ('𐏈', '𐏕')]; + +pub const OLD_SOGDIAN: &'static [(char, char)] = &[('𐼀', '𐼧')]; + +pub const OLD_SOUTH_ARABIAN: &'static [(char, char)] = &[('𐩠', '𐩿')]; + +pub const OLD_TURKIC: &'static [(char, char)] = + &[('⁚', '⁚'), ('⸰', '⸰'), ('𐰀', '𐱈')]; + +pub const OLD_UYGHUR: &'static [(char, char)] = + &[('ـ', 'ـ'), ('𐫲', '𐫲'), ('𐽰', '𐾉')]; + +pub const ORIYA: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('\u{b01}', 'ଃ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('\u{b3c}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', '\u{b63}'), + ('୦', '୷'), + ('\u{1cda}', '\u{1cda}'), + ('ᳲ', 'ᳲ'), +]; + +pub const OSAGE: &'static [(char, char)] = &[ + ('\u{301}', '\u{301}'), + ('\u{304}', '\u{304}'), + ('\u{30b}', '\u{30b}'), + ('\u{358}', '\u{358}'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), +]; + +pub const OSMANYA: &'static [(char, char)] = &[('𐒀', '𐒝'), ('𐒠', '𐒩')]; + +pub const PAHAWH_HMONG: &'static [(char, char)] = + &[('𖬀', '𖭅'), ('𖭐', '𖭙'), ('𖭛', '𖭡'), ('𖭣', '𖭷'), ('𖭽', '𖮏')]; + +pub const PALMYRENE: &'static [(char, char)] = &[('𐡠', '𐡿')]; + +pub const PAU_CIN_HAU: &'static [(char, char)] = &[('𑫀', '𑫸')]; + +pub const PHAGS_PA: &'static [(char, char)] = &[ + ('᠂', '᠃'), + ('᠅', '᠅'), + ('\u{202f}', '\u{202f}'), + ('。', '。'), + ('ꡀ', '꡷'), +]; + +pub const PHOENICIAN: &'static [(char, char)] = &[('𐤀', '𐤛'), ('𐤟', '𐤟')]; + +pub const PSALTER_PAHLAVI: &'static [(char, char)] = + &[('ـ', 'ـ'), ('𐮀', '𐮑'), ('𐮙', '𐮜'), ('𐮩', '𐮯')]; + +pub const REJANG: &'static [(char, char)] = &[('ꤰ', '\u{a953}'), ('꥟', '꥟')]; + +pub const RUNIC: &'static [(char, char)] = &[('ᚠ', 'ᛸ')]; + +pub const SAMARITAN: &'static [(char, char)] = + &[('ࠀ', '\u{82d}'), ('࠰', '࠾'), ('⸱', '⸱')]; + +pub const SAURASHTRA: &'static [(char, char)] = + &[('ꢀ', '\u{a8c5}'), ('꣎', '꣙')]; + +pub const SHARADA: &'static [(char, char)] = &[ + ('\u{951}', '\u{951}'), + ('\u{1cd7}', '\u{1cd7}'), + ('\u{1cd9}', '\u{1cd9}'), + ('\u{1cdc}', '\u{1cdd}'), + ('\u{1ce0}', '\u{1ce0}'), + ('꠰', '꠵'), + ('꠸', '꠸'), + ('\u{11180}', '𑇟'), +]; + +pub const SHAVIAN: &'static [(char, char)] = &[('·', '·'), ('𐑐', '𐑿')]; + +pub const SIDDHAM: &'static [(char, char)] = + &[('𑖀', '\u{115b5}'), ('𑖸', '\u{115dd}')]; + +pub const SIGNWRITING: &'static [(char, char)] = + &[('𝠀', '𝪋'), ('\u{1da9b}', '\u{1da9f}'), ('\u{1daa1}', '\u{1daaf}')]; + +pub const SINHALA: &'static [(char, char)] = &[ + ('।', '॥'), + ('\u{d81}', 'ඃ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('෦', '෯'), + ('ෲ', '෴'), + ('ᳲ', 'ᳲ'), + ('𑇡', '𑇴'), +]; + +pub const SOGDIAN: &'static [(char, char)] = &[('ـ', 'ـ'), ('𐼰', '𐽙')]; + +pub const SORA_SOMPENG: &'static [(char, char)] = &[('𑃐', '𑃨'), ('𑃰', '𑃹')]; + +pub const SOYOMBO: &'static [(char, char)] = &[('𑩐', '𑪢')]; + +pub const SUNDANESE: &'static [(char, char)] = + &[('\u{1b80}', 'ᮿ'), ('᳀', '᳇')]; + +pub const SUNUWAR: &'static [(char, char)] = &[ + ('\u{300}', '\u{301}'), + ('\u{303}', '\u{303}'), + ('\u{30d}', '\u{30d}'), + ('\u{310}', '\u{310}'), + ('\u{32d}', '\u{32d}'), + ('\u{331}', '\u{331}'), + ('𑯀', '𑯡'), + ('𑯰', '𑯹'), +]; + +pub const SYLOTI_NAGRI: &'static [(char, char)] = + &[('।', '॥'), ('০', '৯'), ('ꠀ', '\u{a82c}')]; + +pub const SYRIAC: &'static [(char, char)] = &[ + ('\u{303}', '\u{304}'), + ('\u{307}', '\u{308}'), + ('\u{30a}', '\u{30a}'), + ('\u{320}', '\u{320}'), + ('\u{323}', '\u{325}'), + ('\u{32d}', '\u{32e}'), + ('\u{330}', '\u{330}'), + ('،', '،'), + ('؛', '\u{61c}'), + ('؟', '؟'), + ('ـ', 'ـ'), + ('\u{64b}', '\u{655}'), + ('\u{670}', '\u{670}'), + ('܀', '܍'), + ('\u{70f}', '\u{74a}'), + ('ݍ', 'ݏ'), + ('ࡠ', 'ࡪ'), + ('\u{1df8}', '\u{1df8}'), + ('\u{1dfa}', '\u{1dfa}'), +]; + +pub const TAGALOG: &'static [(char, char)] = + &[('ᜀ', '\u{1715}'), ('ᜟ', 'ᜟ'), ('᜵', '᜶')]; + +pub const TAGBANWA: &'static [(char, char)] = + &[('᜵', '᜶'), ('ᝠ', 'ᝬ'), ('ᝮ', 'ᝰ'), ('\u{1772}', '\u{1773}')]; + +pub const TAI_LE: &'static [(char, char)] = &[ + ('\u{300}', '\u{301}'), + ('\u{307}', '\u{308}'), + ('\u{30c}', '\u{30c}'), + ('၀', '၉'), + ('ᥐ', 'ᥭ'), + ('ᥰ', 'ᥴ'), +]; + +pub const TAI_THAM: &'static [(char, char)] = &[ + ('ᨠ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '᪉'), + ('᪐', '᪙'), + ('᪠', '᪭'), +]; + +pub const TAI_VIET: &'static [(char, char)] = &[('ꪀ', 'ꫂ'), ('ꫛ', '꫟')]; + +pub const TAKRI: &'static [(char, char)] = + &[('।', '॥'), ('꠰', '꠹'), ('𑚀', '𑚹'), ('𑛀', '𑛉')]; + +pub const TAMIL: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('\u{b82}', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('ௐ', 'ௐ'), + ('\u{bd7}', '\u{bd7}'), + ('௦', '௺'), + ('\u{1cda}', '\u{1cda}'), + ('ꣳ', 'ꣳ'), + ('\u{11301}', '\u{11301}'), + ('𑌃', '𑌃'), + ('\u{1133b}', '\u{1133c}'), + ('𑿀', '𑿱'), + ('𑿿', '𑿿'), +]; + +pub const TANGSA: &'static [(char, char)] = &[('𖩰', '𖪾'), ('𖫀', '𖫉')]; + +pub const TANGUT: &'static [(char, char)] = &[ + ('⿰', '⿿'), + ('㇯', '㇯'), + ('𖿠', '𖿠'), + ('𗀀', '𘟷'), + ('𘠀', '𘫿'), + ('𘴀', '𘴈'), +]; + +pub const TELUGU: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('\u{c00}', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('\u{c3c}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', '\u{c63}'), + ('౦', '౯'), + ('౷', '౿'), + ('\u{1cda}', '\u{1cda}'), + ('ᳲ', 'ᳲ'), +]; + +pub const THAANA: &'static [(char, char)] = &[ + ('،', '،'), + ('؛', '\u{61c}'), + ('؟', '؟'), + ('٠', '٩'), + ('ހ', 'ޱ'), + ('ﷲ', 'ﷲ'), + ('﷽', '﷽'), +]; + +pub const THAI: &'static [(char, char)] = &[ + ('ʼ', 'ʼ'), + ('˗', '˗'), + ('\u{303}', '\u{303}'), + ('\u{331}', '\u{331}'), + ('ก', '\u{e3a}'), + ('เ', '๛'), +]; + +pub const TIBETAN: &'static [(char, char)] = &[ + ('ༀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('\u{f71}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('྾', '࿌'), + ('࿎', '࿔'), + ('࿙', '࿚'), + ('〈', '》'), +]; + +pub const TIFINAGH: &'static [(char, char)] = &[ + ('\u{302}', '\u{302}'), + ('\u{304}', '\u{304}'), + ('\u{307}', '\u{307}'), + ('\u{309}', '\u{309}'), + ('ⴰ', 'ⵧ'), + ('ⵯ', '⵰'), + ('\u{2d7f}', '\u{2d7f}'), +]; + +pub const TIRHUTA: &'static [(char, char)] = &[ + ('\u{951}', '\u{952}'), + ('।', '॥'), + ('ᳲ', 'ᳲ'), + ('꠰', '꠹'), + ('𑒀', '𑓇'), + ('𑓐', '𑓙'), +]; + +pub const TODHRI: &'static [(char, char)] = &[ + ('\u{301}', '\u{301}'), + ('\u{304}', '\u{304}'), + ('\u{307}', '\u{307}'), + ('\u{311}', '\u{311}'), + ('\u{313}', '\u{313}'), + ('\u{35e}', '\u{35e}'), + ('𐗀', '𐗳'), +]; + +pub const TOTO: &'static [(char, char)] = &[('ʼ', 'ʼ'), ('𞊐', '\u{1e2ae}')]; + +pub const TULU_TIGALARI: &'static [(char, char)] = &[ + ('೦', '೯'), + ('ᳲ', 'ᳲ'), + ('\u{1cf4}', '\u{1cf4}'), + ('꠰', '꠵'), + ('\u{a8f1}', '\u{a8f1}'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '𑏕'), + ('𑏗', '𑏘'), + ('\u{113e1}', '\u{113e2}'), +]; + +pub const UGARITIC: &'static [(char, char)] = &[('𐎀', '𐎝'), ('𐎟', '𐎟')]; + +pub const VAI: &'static [(char, char)] = &[('ꔀ', 'ꘫ')]; + +pub const VITHKUQI: &'static [(char, char)] = &[ + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), +]; + +pub const WANCHO: &'static [(char, char)] = &[('𞋀', '𞋹'), ('𞋿', '𞋿')]; + +pub const WARANG_CITI: &'static [(char, char)] = &[('𑢠', '𑣲'), ('𑣿', '𑣿')]; + +pub const YEZIDI: &'static [(char, char)] = &[ + ('،', '،'), + ('؛', '؛'), + ('؟', '؟'), + ('٠', '٩'), + ('𐺀', '𐺩'), + ('\u{10eab}', '𐺭'), + ('𐺰', '𐺱'), +]; + +pub const YI: &'static [(char, char)] = &[ + ('、', '。'), + ('〈', '】'), + ('〔', '〛'), + ('・', '・'), + ('ꀀ', 'ꒌ'), + ('꒐', '꓆'), + ('｡', '･'), +]; + +pub const ZANABAZAR_SQUARE: &'static [(char, char)] = &[('𑨀', '\u{11a47}')]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/sentence_break.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/sentence_break.rs new file mode 100644 index 0000000..af1c5be --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/sentence_break.rs @@ -0,0 +1,2530 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate sentence-break ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = &[ + ("ATerm", ATERM), + ("CR", CR), + ("Close", CLOSE), + ("Extend", EXTEND), + ("Format", FORMAT), + ("LF", LF), + ("Lower", LOWER), + ("Numeric", NUMERIC), + ("OLetter", OLETTER), + ("SContinue", SCONTINUE), + ("STerm", STERM), + ("Sep", SEP), + ("Sp", SP), + ("Upper", UPPER), +]; + +pub const ATERM: &'static [(char, char)] = + &[('.', '.'), ('․', '․'), ('﹒', '﹒'), ('．', '．')]; + +pub const CR: &'static [(char, char)] = &[('\r', '\r')]; + +pub const CLOSE: &'static [(char, char)] = &[ + ('"', '"'), + ('\'', ')'), + ('[', '['), + (']', ']'), + ('{', '{'), + ('}', '}'), + ('«', '«'), + ('»', '»'), + ('༺', '༽'), + ('᚛', '᚜'), + ('‘', '‟'), + ('‹', '›'), + ('⁅', '⁆'), + ('⁽', '⁾'), + ('₍', '₎'), + ('⌈', '⌋'), + ('〈', '〉'), + ('❛', '❠'), + ('❨', '❵'), + ('⟅', '⟆'), + ('⟦', '⟯'), + ('⦃', '⦘'), + ('⧘', '⧛'), + ('⧼', '⧽'), + ('⸀', '⸍'), + ('⸜', '⸝'), + ('⸠', '⸩'), + ('⹂', '⹂'), + ('⹕', '⹜'), + ('〈', '】'), + ('〔', '〛'), + ('〝', '〟'), + ('﴾', '﴿'), + ('︗', '︘'), + ('︵', '﹄'), + ('﹇', '﹈'), + ('﹙', '﹞'), + ('（', '）'), + ('［', '［'), + ('］', '］'), + ('｛', '｛'), + ('｝', '｝'), + ('｟', '｠'), + ('｢', '｣'), + ('🙶', '🙸'), +]; + +pub const EXTEND: &'static [(char, char)] = &[ + ('\u{300}', '\u{36f}'), + ('\u{483}', '\u{489}'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('\u{610}', '\u{61a}'), + ('\u{64b}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dc}'), + ('\u{6df}', '\u{6e4}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{6ea}', '\u{6ed}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', '\u{7f3}'), + ('\u{7fd}', '\u{7fd}'), + ('\u{816}', '\u{819}'), + ('\u{81b}', '\u{823}'), + ('\u{825}', '\u{827}'), + ('\u{829}', '\u{82d}'), + ('\u{859}', '\u{85b}'), + ('\u{897}', '\u{89f}'), + ('\u{8ca}', '\u{8e1}'), + ('\u{8e3}', 'ः'), + ('\u{93a}', '\u{93c}'), + ('ा', 'ॏ'), + ('\u{951}', '\u{957}'), + ('\u{962}', '\u{963}'), + ('\u{981}', 'ঃ'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9be}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', '\u{9cd}'), + ('\u{9d7}', '\u{9d7}'), + ('\u{9e2}', '\u{9e3}'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', 'ਃ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('\u{abc}', '\u{abc}'), + ('ા', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('\u{ae2}', '\u{ae3}'), + ('\u{afa}', '\u{aff}'), + ('\u{b01}', 'ଃ'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b3e}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('\u{b62}', '\u{b63}'), + ('\u{b82}', '\u{b82}'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('\u{bd7}', '\u{bd7}'), + ('\u{c00}', '\u{c04}'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c3e}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', 'ಃ'), + ('\u{cbc}', '\u{cbc}'), + ('ಾ', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('\u{ce2}', '\u{ce3}'), + ('ೳ', 'ೳ'), + ('\u{d00}', 'ഃ'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d3e}', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', '\u{d4d}'), + ('\u{d57}', '\u{d57}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', 'ඃ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('ෲ', 'ෳ'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('\u{e47}', '\u{e4e}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{ebc}'), + ('\u{ec8}', '\u{ece}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('༾', '༿'), + ('\u{f71}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('ါ', '\u{103e}'), + ('ၖ', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('ၢ', 'ၤ'), + ('ၧ', 'ၭ'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{108d}'), + ('ႏ', 'ႏ'), + ('ႚ', '\u{109d}'), + ('\u{135d}', '\u{135f}'), + ('\u{1712}', '\u{1715}'), + ('\u{1732}', '\u{1734}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('\u{17b4}', '\u{17d3}'), + ('\u{17dd}', '\u{17dd}'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '\u{180f}'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('\u{1a17}', '\u{1a1b}'), + ('ᩕ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', 'ᬄ'), + ('\u{1b34}', '\u{1b44}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', 'ᮂ'), + ('ᮡ', '\u{1bad}'), + ('\u{1be6}', '\u{1bf3}'), + ('ᰤ', '\u{1c37}'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('᳷', '\u{1cf9}'), + ('\u{1dc0}', '\u{1dff}'), + ('\u{200c}', '\u{200d}'), + ('\u{20d0}', '\u{20f0}'), + ('\u{2cef}', '\u{2cf1}'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{2de0}', '\u{2dff}'), + ('\u{302a}', '\u{302f}'), + ('\u{3099}', '\u{309a}'), + ('\u{a66f}', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('\u{a69e}', '\u{a69f}'), + ('\u{a6f0}', '\u{a6f1}'), + ('\u{a802}', '\u{a802}'), + ('\u{a806}', '\u{a806}'), + ('\u{a80b}', '\u{a80b}'), + ('ꠣ', 'ꠧ'), + ('\u{a82c}', '\u{a82c}'), + ('ꢀ', 'ꢁ'), + ('ꢴ', '\u{a8c5}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92d}'), + ('\u{a947}', '\u{a953}'), + ('\u{a980}', 'ꦃ'), + ('\u{a9b3}', '\u{a9c0}'), + ('\u{a9e5}', '\u{a9e5}'), + ('\u{aa29}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', 'ꩍ'), + ('ꩻ', 'ꩽ'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabf}'), + ('\u{aac1}', '\u{aac1}'), + ('ꫫ', 'ꫯ'), + ('ꫵ', '\u{aaf6}'), + ('ꯣ', 'ꯪ'), + ('꯬', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('\u{ff9e}', '\u{ff9f}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{10376}', '\u{1037a}'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('\u{10d24}', '\u{10d27}'), + ('\u{10d69}', '\u{10d6d}'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('𑀀', '𑀂'), + ('\u{11038}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{11073}', '\u{11074}'), + ('\u{1107f}', '𑂂'), + ('𑂰', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{11134}'), + ('𑅅', '𑅆'), + ('\u{11173}', '\u{11173}'), + ('\u{11180}', '𑆂'), + ('𑆳', '\u{111c0}'), + ('\u{111c9}', '\u{111cc}'), + ('𑇎', '\u{111cf}'), + ('𑈬', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112ea}'), + ('\u{11300}', '𑌃'), + ('\u{1133b}', '\u{1133c}'), + ('\u{1133e}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('𑍢', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113b8}', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '\u{113d0}'), + ('\u{113d2}', '\u{113d2}'), + ('\u{113e1}', '\u{113e2}'), + ('𑐵', '\u{11446}'), + ('\u{1145e}', '\u{1145e}'), + ('\u{114b0}', '\u{114c3}'), + ('\u{115af}', '\u{115b5}'), + ('𑖸', '\u{115c0}'), + ('\u{115dc}', '\u{115dd}'), + ('𑘰', '\u{11640}'), + ('\u{116ab}', '\u{116b7}'), + ('\u{1171d}', '\u{1172b}'), + ('𑠬', '\u{1183a}'), + ('\u{11930}', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{1193e}'), + ('𑥀', '𑥀'), + ('𑥂', '\u{11943}'), + ('𑧑', '\u{119d7}'), + ('\u{119da}', '\u{119e0}'), + ('𑧤', '𑧤'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a33}', '𑨹'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a51}', '\u{11a5b}'), + ('\u{11a8a}', '\u{11a99}'), + ('𑰯', '\u{11c36}'), + ('\u{11c38}', '\u{11c3f}'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d45}'), + ('\u{11d47}', '\u{11d47}'), + ('𑶊', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '\u{11d97}'), + ('\u{11ef3}', '𑻶'), + ('\u{11f00}', '\u{11f01}'), + ('𑼃', '𑼃'), + ('𑼴', '\u{11f3a}'), + ('𑼾', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13440}', '\u{13440}'), + ('\u{13447}', '\u{13455}'), + ('\u{1611e}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('\u{16f4f}', '\u{16f4f}'), + ('𖽑', '𖾇'), + ('\u{16f8f}', '\u{16f92}'), + ('\u{16fe4}', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e130}', '\u{1e136}'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('\u{1e4ec}', '\u{1e4ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '\u{1e94a}'), + ('\u{e0020}', '\u{e007f}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const FORMAT: &'static [(char, char)] = &[ + ('\u{ad}', '\u{ad}'), + ('\u{61c}', '\u{61c}'), + ('\u{70f}', '\u{70f}'), + ('\u{180e}', '\u{180e}'), + ('\u{200b}', '\u{200b}'), + ('\u{200e}', '\u{200f}'), + ('\u{202a}', '\u{202e}'), + ('\u{2060}', '\u{2064}'), + ('\u{2066}', '\u{206f}'), + ('\u{feff}', '\u{feff}'), + ('\u{fff9}', '\u{fffb}'), + ('\u{13430}', '\u{1343f}'), + ('\u{1bca0}', '\u{1bca3}'), + ('\u{1d173}', '\u{1d17a}'), + ('\u{e0001}', '\u{e0001}'), +]; + +pub const LF: &'static [(char, char)] = &[('\n', '\n')]; + +pub const LOWER: &'static [(char, char)] = &[ + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('ß', 'ö'), + ('ø', 'ÿ'), + ('ā', 'ā'), + ('ă', 'ă'), + ('ą', 'ą'), + ('ć', 'ć'), + ('ĉ', 'ĉ'), + ('ċ', 'ċ'), + ('č', 'č'), + ('ď', 'ď'), + ('đ', 'đ'), + ('ē', 'ē'), + ('ĕ', 'ĕ'), + ('ė', 'ė'), + ('ę', 'ę'), + ('ě', 'ě'), + ('ĝ', 'ĝ'), + ('ğ', 'ğ'), + ('ġ', 'ġ'), + ('ģ', 'ģ'), + ('ĥ', 'ĥ'), + ('ħ', 'ħ'), + ('ĩ', 'ĩ'), + ('ī', 'ī'), + ('ĭ', 'ĭ'), + ('į', 'į'), + ('ı', 'ı'), + ('ij', 'ij'), + ('ĵ', 'ĵ'), + ('ķ', 'ĸ'), + ('ĺ', 'ĺ'), + ('ļ', 'ļ'), + ('ľ', 'ľ'), + ('ŀ', 'ŀ'), + ('ł', 'ł'), + ('ń', 'ń'), + ('ņ', 'ņ'), + ('ň', 'ʼn'), + ('ŋ', 'ŋ'), + ('ō', 'ō'), + ('ŏ', 'ŏ'), + ('ő', 'ő'), + ('œ', 'œ'), + ('ŕ', 'ŕ'), + ('ŗ', 'ŗ'), + ('ř', 'ř'), + ('ś', 'ś'), + ('ŝ', 'ŝ'), + ('ş', 'ş'), + ('š', 'š'), + ('ţ', 'ţ'), + ('ť', 'ť'), + ('ŧ', 'ŧ'), + ('ũ', 'ũ'), + ('ū', 'ū'), + ('ŭ', 'ŭ'), + ('ů', 'ů'), + ('ű', 'ű'), + ('ų', 'ų'), + ('ŵ', 'ŵ'), + ('ŷ', 'ŷ'), + ('ź', 'ź'), + ('ż', 'ż'), + ('ž', 'ƀ'), + ('ƃ', 'ƃ'), + ('ƅ', 'ƅ'), + ('ƈ', 'ƈ'), + ('ƌ', 'ƍ'), + ('ƒ', 'ƒ'), + ('ƕ', 'ƕ'), + ('ƙ', 'ƛ'), + ('ƞ', 'ƞ'), + ('ơ', 'ơ'), + ('ƣ', 'ƣ'), + ('ƥ', 'ƥ'), + ('ƨ', 'ƨ'), + ('ƪ', 'ƫ'), + ('ƭ', 'ƭ'), + ('ư', 'ư'), + ('ƴ', 'ƴ'), + ('ƶ', 'ƶ'), + ('ƹ', 'ƺ'), + ('ƽ', 'ƿ'), + ('dž', 'dž'), + ('lj', 'lj'), + ('nj', 'nj'), + ('ǎ', 'ǎ'), + ('ǐ', 'ǐ'), + ('ǒ', 'ǒ'), + ('ǔ', 'ǔ'), + ('ǖ', 'ǖ'), + ('ǘ', 'ǘ'), + ('ǚ', 'ǚ'), + ('ǜ', 'ǝ'), + ('ǟ', 'ǟ'), + ('ǡ', 'ǡ'), + ('ǣ', 'ǣ'), + ('ǥ', 'ǥ'), + ('ǧ', 'ǧ'), + ('ǩ', 'ǩ'), + ('ǫ', 'ǫ'), + ('ǭ', 'ǭ'), + ('ǯ', 'ǰ'), + ('dz', 'dz'), + ('ǵ', 'ǵ'), + ('ǹ', 'ǹ'), + ('ǻ', 'ǻ'), + ('ǽ', 'ǽ'), + ('ǿ', 'ǿ'), + ('ȁ', 'ȁ'), + ('ȃ', 'ȃ'), + ('ȅ', 'ȅ'), + ('ȇ', 'ȇ'), + ('ȉ', 'ȉ'), + ('ȋ', 'ȋ'), + ('ȍ', 'ȍ'), + ('ȏ', 'ȏ'), + ('ȑ', 'ȑ'), + ('ȓ', 'ȓ'), + ('ȕ', 'ȕ'), + ('ȗ', 'ȗ'), + ('ș', 'ș'), + ('ț', 'ț'), + ('ȝ', 'ȝ'), + ('ȟ', 'ȟ'), + ('ȡ', 'ȡ'), + ('ȣ', 'ȣ'), + ('ȥ', 'ȥ'), + ('ȧ', 'ȧ'), + ('ȩ', 'ȩ'), + ('ȫ', 'ȫ'), + ('ȭ', 'ȭ'), + ('ȯ', 'ȯ'), + ('ȱ', 'ȱ'), + ('ȳ', 'ȹ'), + ('ȼ', 'ȼ'), + ('ȿ', 'ɀ'), + ('ɂ', 'ɂ'), + ('ɇ', 'ɇ'), + ('ɉ', 'ɉ'), + ('ɋ', 'ɋ'), + ('ɍ', 'ɍ'), + ('ɏ', 'ʓ'), + ('ʕ', 'ʸ'), + ('ˀ', 'ˁ'), + ('ˠ', 'ˤ'), + ('ͱ', 'ͱ'), + ('ͳ', 'ͳ'), + ('ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('ΐ', 'ΐ'), + ('ά', 'ώ'), + ('ϐ', 'ϑ'), + ('ϕ', 'ϗ'), + ('ϙ', 'ϙ'), + ('ϛ', 'ϛ'), + ('ϝ', 'ϝ'), + ('ϟ', 'ϟ'), + ('ϡ', 'ϡ'), + ('ϣ', 'ϣ'), + ('ϥ', 'ϥ'), + ('ϧ', 'ϧ'), + ('ϩ', 'ϩ'), + ('ϫ', 'ϫ'), + ('ϭ', 'ϭ'), + ('ϯ', 'ϳ'), + ('ϵ', 'ϵ'), + ('ϸ', 'ϸ'), + ('ϻ', 'ϼ'), + ('а', 'џ'), + ('ѡ', 'ѡ'), + ('ѣ', 'ѣ'), + ('ѥ', 'ѥ'), + ('ѧ', 'ѧ'), + ('ѩ', 'ѩ'), + ('ѫ', 'ѫ'), + ('ѭ', 'ѭ'), + ('ѯ', 'ѯ'), + ('ѱ', 'ѱ'), + ('ѳ', 'ѳ'), + ('ѵ', 'ѵ'), + ('ѷ', 'ѷ'), + ('ѹ', 'ѹ'), + ('ѻ', 'ѻ'), + ('ѽ', 'ѽ'), + ('ѿ', 'ѿ'), + ('ҁ', 'ҁ'), + ('ҋ', 'ҋ'), + ('ҍ', 'ҍ'), + ('ҏ', 'ҏ'), + ('ґ', 'ґ'), + ('ғ', 'ғ'), + ('ҕ', 'ҕ'), + ('җ', 'җ'), + ('ҙ', 'ҙ'), + ('қ', 'қ'), + ('ҝ', 'ҝ'), + ('ҟ', 'ҟ'), + ('ҡ', 'ҡ'), + ('ң', 'ң'), + ('ҥ', 'ҥ'), + ('ҧ', 'ҧ'), + ('ҩ', 'ҩ'), + ('ҫ', 'ҫ'), + ('ҭ', 'ҭ'), + ('ү', 'ү'), + ('ұ', 'ұ'), + ('ҳ', 'ҳ'), + ('ҵ', 'ҵ'), + ('ҷ', 'ҷ'), + ('ҹ', 'ҹ'), + ('һ', 'һ'), + ('ҽ', 'ҽ'), + ('ҿ', 'ҿ'), + ('ӂ', 'ӂ'), + ('ӄ', 'ӄ'), + ('ӆ', 'ӆ'), + ('ӈ', 'ӈ'), + ('ӊ', 'ӊ'), + ('ӌ', 'ӌ'), + ('ӎ', 'ӏ'), + ('ӑ', 'ӑ'), + ('ӓ', 'ӓ'), + ('ӕ', 'ӕ'), + ('ӗ', 'ӗ'), + ('ә', 'ә'), + ('ӛ', 'ӛ'), + ('ӝ', 'ӝ'), + ('ӟ', 'ӟ'), + ('ӡ', 'ӡ'), + ('ӣ', 'ӣ'), + ('ӥ', 'ӥ'), + ('ӧ', 'ӧ'), + ('ө', 'ө'), + ('ӫ', 'ӫ'), + ('ӭ', 'ӭ'), + ('ӯ', 'ӯ'), + ('ӱ', 'ӱ'), + ('ӳ', 'ӳ'), + ('ӵ', 'ӵ'), + ('ӷ', 'ӷ'), + ('ӹ', 'ӹ'), + ('ӻ', 'ӻ'), + ('ӽ', 'ӽ'), + ('ӿ', 'ӿ'), + ('ԁ', 'ԁ'), + ('ԃ', 'ԃ'), + ('ԅ', 'ԅ'), + ('ԇ', 'ԇ'), + ('ԉ', 'ԉ'), + ('ԋ', 'ԋ'), + ('ԍ', 'ԍ'), + ('ԏ', 'ԏ'), + ('ԑ', 'ԑ'), + ('ԓ', 'ԓ'), + ('ԕ', 'ԕ'), + ('ԗ', 'ԗ'), + ('ԙ', 'ԙ'), + ('ԛ', 'ԛ'), + ('ԝ', 'ԝ'), + ('ԟ', 'ԟ'), + ('ԡ', 'ԡ'), + ('ԣ', 'ԣ'), + ('ԥ', 'ԥ'), + ('ԧ', 'ԧ'), + ('ԩ', 'ԩ'), + ('ԫ', 'ԫ'), + ('ԭ', 'ԭ'), + ('ԯ', 'ԯ'), + ('ՠ', 'ֈ'), + ('ჼ', 'ჼ'), + ('ᏸ', 'ᏽ'), + ('ᲀ', 'ᲈ'), + ('ᲊ', 'ᲊ'), + ('ᴀ', 'ᶿ'), + ('ḁ', 'ḁ'), + ('ḃ', 'ḃ'), + ('ḅ', 'ḅ'), + ('ḇ', 'ḇ'), + ('ḉ', 'ḉ'), + ('ḋ', 'ḋ'), + ('ḍ', 'ḍ'), + ('ḏ', 'ḏ'), + ('ḑ', 'ḑ'), + ('ḓ', 'ḓ'), + ('ḕ', 'ḕ'), + ('ḗ', 'ḗ'), + ('ḙ', 'ḙ'), + ('ḛ', 'ḛ'), + ('ḝ', 'ḝ'), + ('ḟ', 'ḟ'), + ('ḡ', 'ḡ'), + ('ḣ', 'ḣ'), + ('ḥ', 'ḥ'), + ('ḧ', 'ḧ'), + ('ḩ', 'ḩ'), + ('ḫ', 'ḫ'), + ('ḭ', 'ḭ'), + ('ḯ', 'ḯ'), + ('ḱ', 'ḱ'), + ('ḳ', 'ḳ'), + ('ḵ', 'ḵ'), + ('ḷ', 'ḷ'), + ('ḹ', 'ḹ'), + ('ḻ', 'ḻ'), + ('ḽ', 'ḽ'), + ('ḿ', 'ḿ'), + ('ṁ', 'ṁ'), + ('ṃ', 'ṃ'), + ('ṅ', 'ṅ'), + ('ṇ', 'ṇ'), + ('ṉ', 'ṉ'), + ('ṋ', 'ṋ'), + ('ṍ', 'ṍ'), + ('ṏ', 'ṏ'), + ('ṑ', 'ṑ'), + ('ṓ', 'ṓ'), + ('ṕ', 'ṕ'), + ('ṗ', 'ṗ'), + ('ṙ', 'ṙ'), + ('ṛ', 'ṛ'), + ('ṝ', 'ṝ'), + ('ṟ', 'ṟ'), + ('ṡ', 'ṡ'), + ('ṣ', 'ṣ'), + ('ṥ', 'ṥ'), + ('ṧ', 'ṧ'), + ('ṩ', 'ṩ'), + ('ṫ', 'ṫ'), + ('ṭ', 'ṭ'), + ('ṯ', 'ṯ'), + ('ṱ', 'ṱ'), + ('ṳ', 'ṳ'), + ('ṵ', 'ṵ'), + ('ṷ', 'ṷ'), + ('ṹ', 'ṹ'), + ('ṻ', 'ṻ'), + ('ṽ', 'ṽ'), + ('ṿ', 'ṿ'), + ('ẁ', 'ẁ'), + ('ẃ', 'ẃ'), + ('ẅ', 'ẅ'), + ('ẇ', 'ẇ'), + ('ẉ', 'ẉ'), + ('ẋ', 'ẋ'), + ('ẍ', 'ẍ'), + ('ẏ', 'ẏ'), + ('ẑ', 'ẑ'), + ('ẓ', 'ẓ'), + ('ẕ', 'ẝ'), + ('ẟ', 'ẟ'), + ('ạ', 'ạ'), + ('ả', 'ả'), + ('ấ', 'ấ'), + ('ầ', 'ầ'), + ('ẩ', 'ẩ'), + ('ẫ', 'ẫ'), + ('ậ', 'ậ'), + ('ắ', 'ắ'), + ('ằ', 'ằ'), + ('ẳ', 'ẳ'), + ('ẵ', 'ẵ'), + ('ặ', 'ặ'), + ('ẹ', 'ẹ'), + ('ẻ', 'ẻ'), + ('ẽ', 'ẽ'), + ('ế', 'ế'), + ('ề', 'ề'), + ('ể', 'ể'), + ('ễ', 'ễ'), + ('ệ', 'ệ'), + ('ỉ', 'ỉ'), + ('ị', 'ị'), + ('ọ', 'ọ'), + ('ỏ', 'ỏ'), + ('ố', 'ố'), + ('ồ', 'ồ'), + ('ổ', 'ổ'), + ('ỗ', 'ỗ'), + ('ộ', 'ộ'), + ('ớ', 'ớ'), + ('ờ', 'ờ'), + ('ở', 'ở'), + ('ỡ', 'ỡ'), + ('ợ', 'ợ'), + ('ụ', 'ụ'), + ('ủ', 'ủ'), + ('ứ', 'ứ'), + ('ừ', 'ừ'), + ('ử', 'ử'), + ('ữ', 'ữ'), + ('ự', 'ự'), + ('ỳ', 'ỳ'), + ('ỵ', 'ỵ'), + ('ỷ', 'ỷ'), + ('ỹ', 'ỹ'), + ('ỻ', 'ỻ'), + ('ỽ', 'ỽ'), + ('ỿ', 'ἇ'), + ('ἐ', 'ἕ'), + ('ἠ', 'ἧ'), + ('ἰ', 'ἷ'), + ('ὀ', 'ὅ'), + ('ὐ', 'ὗ'), + ('ὠ', 'ὧ'), + ('ὰ', 'ώ'), + ('ᾀ', 'ᾇ'), + ('ᾐ', 'ᾗ'), + ('ᾠ', 'ᾧ'), + ('ᾰ', 'ᾴ'), + ('ᾶ', 'ᾷ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῇ'), + ('ῐ', 'ΐ'), + ('ῖ', 'ῗ'), + ('ῠ', 'ῧ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῷ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ℊ', 'ℊ'), + ('ℎ', 'ℏ'), + ('ℓ', 'ℓ'), + ('ℯ', 'ℯ'), + ('ℴ', 'ℴ'), + ('ℹ', 'ℹ'), + ('ℼ', 'ℽ'), + ('ⅆ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('ⅰ', 'ⅿ'), + ('ↄ', 'ↄ'), + ('ⓐ', 'ⓩ'), + ('ⰰ', 'ⱟ'), + ('ⱡ', 'ⱡ'), + ('ⱥ', 'ⱦ'), + ('ⱨ', 'ⱨ'), + ('ⱪ', 'ⱪ'), + ('ⱬ', 'ⱬ'), + ('ⱱ', 'ⱱ'), + ('ⱳ', 'ⱴ'), + ('ⱶ', 'ⱽ'), + ('ⲁ', 'ⲁ'), + ('ⲃ', 'ⲃ'), + ('ⲅ', 'ⲅ'), + ('ⲇ', 'ⲇ'), + ('ⲉ', 'ⲉ'), + ('ⲋ', 'ⲋ'), + ('ⲍ', 'ⲍ'), + ('ⲏ', 'ⲏ'), + ('ⲑ', 'ⲑ'), + ('ⲓ', 'ⲓ'), + ('ⲕ', 'ⲕ'), + ('ⲗ', 'ⲗ'), + ('ⲙ', 'ⲙ'), + ('ⲛ', 'ⲛ'), + ('ⲝ', 'ⲝ'), + ('ⲟ', 'ⲟ'), + ('ⲡ', 'ⲡ'), + ('ⲣ', 'ⲣ'), + ('ⲥ', 'ⲥ'), + ('ⲧ', 'ⲧ'), + ('ⲩ', 'ⲩ'), + ('ⲫ', 'ⲫ'), + ('ⲭ', 'ⲭ'), + ('ⲯ', 'ⲯ'), + ('ⲱ', 'ⲱ'), + ('ⲳ', 'ⲳ'), + ('ⲵ', 'ⲵ'), + ('ⲷ', 'ⲷ'), + ('ⲹ', 'ⲹ'), + ('ⲻ', 'ⲻ'), + ('ⲽ', 'ⲽ'), + ('ⲿ', 'ⲿ'), + ('ⳁ', 'ⳁ'), + ('ⳃ', 'ⳃ'), + ('ⳅ', 'ⳅ'), + ('ⳇ', 'ⳇ'), + ('ⳉ', 'ⳉ'), + ('ⳋ', 'ⳋ'), + ('ⳍ', 'ⳍ'), + ('ⳏ', 'ⳏ'), + ('ⳑ', 'ⳑ'), + ('ⳓ', 'ⳓ'), + ('ⳕ', 'ⳕ'), + ('ⳗ', 'ⳗ'), + ('ⳙ', 'ⳙ'), + ('ⳛ', 'ⳛ'), + ('ⳝ', 'ⳝ'), + ('ⳟ', 'ⳟ'), + ('ⳡ', 'ⳡ'), + ('ⳣ', 'ⳤ'), + ('ⳬ', 'ⳬ'), + ('ⳮ', 'ⳮ'), + ('ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ꙁ', 'ꙁ'), + ('ꙃ', 'ꙃ'), + ('ꙅ', 'ꙅ'), + ('ꙇ', 'ꙇ'), + ('ꙉ', 'ꙉ'), + ('ꙋ', 'ꙋ'), + ('ꙍ', 'ꙍ'), + ('ꙏ', 'ꙏ'), + ('ꙑ', 'ꙑ'), + ('ꙓ', 'ꙓ'), + ('ꙕ', 'ꙕ'), + ('ꙗ', 'ꙗ'), + ('ꙙ', 'ꙙ'), + ('ꙛ', 'ꙛ'), + ('ꙝ', 'ꙝ'), + ('ꙟ', 'ꙟ'), + ('ꙡ', 'ꙡ'), + ('ꙣ', 'ꙣ'), + ('ꙥ', 'ꙥ'), + ('ꙧ', 'ꙧ'), + ('ꙩ', 'ꙩ'), + ('ꙫ', 'ꙫ'), + ('ꙭ', 'ꙭ'), + ('ꚁ', 'ꚁ'), + ('ꚃ', 'ꚃ'), + ('ꚅ', 'ꚅ'), + ('ꚇ', 'ꚇ'), + ('ꚉ', 'ꚉ'), + ('ꚋ', 'ꚋ'), + ('ꚍ', 'ꚍ'), + ('ꚏ', 'ꚏ'), + ('ꚑ', 'ꚑ'), + ('ꚓ', 'ꚓ'), + ('ꚕ', 'ꚕ'), + ('ꚗ', 'ꚗ'), + ('ꚙ', 'ꚙ'), + ('ꚛ', 'ꚝ'), + ('ꜣ', 'ꜣ'), + ('ꜥ', 'ꜥ'), + ('ꜧ', 'ꜧ'), + ('ꜩ', 'ꜩ'), + ('ꜫ', 'ꜫ'), + ('ꜭ', 'ꜭ'), + ('ꜯ', 'ꜱ'), + ('ꜳ', 'ꜳ'), + ('ꜵ', 'ꜵ'), + ('ꜷ', 'ꜷ'), + ('ꜹ', 'ꜹ'), + ('ꜻ', 'ꜻ'), + ('ꜽ', 'ꜽ'), + ('ꜿ', 'ꜿ'), + ('ꝁ', 'ꝁ'), + ('ꝃ', 'ꝃ'), + ('ꝅ', 'ꝅ'), + ('ꝇ', 'ꝇ'), + ('ꝉ', 'ꝉ'), + ('ꝋ', 'ꝋ'), + ('ꝍ', 'ꝍ'), + ('ꝏ', 'ꝏ'), + ('ꝑ', 'ꝑ'), + ('ꝓ', 'ꝓ'), + ('ꝕ', 'ꝕ'), + ('ꝗ', 'ꝗ'), + ('ꝙ', 'ꝙ'), + ('ꝛ', 'ꝛ'), + ('ꝝ', 'ꝝ'), + ('ꝟ', 'ꝟ'), + ('ꝡ', 'ꝡ'), + ('ꝣ', 'ꝣ'), + ('ꝥ', 'ꝥ'), + ('ꝧ', 'ꝧ'), + ('ꝩ', 'ꝩ'), + ('ꝫ', 'ꝫ'), + ('ꝭ', 'ꝭ'), + ('ꝯ', 'ꝸ'), + ('ꝺ', 'ꝺ'), + ('ꝼ', 'ꝼ'), + ('ꝿ', 'ꝿ'), + ('ꞁ', 'ꞁ'), + ('ꞃ', 'ꞃ'), + ('ꞅ', 'ꞅ'), + ('ꞇ', 'ꞇ'), + ('ꞌ', 'ꞌ'), + ('ꞎ', 'ꞎ'), + ('ꞑ', 'ꞑ'), + ('ꞓ', 'ꞕ'), + ('ꞗ', 'ꞗ'), + ('ꞙ', 'ꞙ'), + ('ꞛ', 'ꞛ'), + ('ꞝ', 'ꞝ'), + ('ꞟ', 'ꞟ'), + ('ꞡ', 'ꞡ'), + ('ꞣ', 'ꞣ'), + ('ꞥ', 'ꞥ'), + ('ꞧ', 'ꞧ'), + ('ꞩ', 'ꞩ'), + ('ꞯ', 'ꞯ'), + ('ꞵ', 'ꞵ'), + ('ꞷ', 'ꞷ'), + ('ꞹ', 'ꞹ'), + ('ꞻ', 'ꞻ'), + ('ꞽ', 'ꞽ'), + ('ꞿ', 'ꞿ'), + ('ꟁ', 'ꟁ'), + ('ꟃ', 'ꟃ'), + ('ꟈ', 'ꟈ'), + ('ꟊ', 'ꟊ'), + ('ꟍ', 'ꟍ'), + ('ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'ꟕ'), + ('ꟗ', 'ꟗ'), + ('ꟙ', 'ꟙ'), + ('ꟛ', 'ꟛ'), + ('ꟲ', 'ꟴ'), + ('ꟶ', 'ꟶ'), + ('ꟸ', 'ꟺ'), + ('ꬰ', 'ꭚ'), + ('ꭜ', 'ꭩ'), + ('ꭰ', 'ꮿ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('ａ', 'ｚ'), + ('𐐨', '𐑏'), + ('𐓘', '𐓻'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐞀', '𐞀'), + ('𐞃', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐳀', '𐳲'), + ('𐵰', '𐶅'), + ('𑣀', '𑣟'), + ('𖹠', '𖹿'), + ('𝐚', '𝐳'), + ('𝑎', '𝑔'), + ('𝑖', '𝑧'), + ('𝒂', '𝒛'), + ('𝒶', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝓏'), + ('𝓪', '𝔃'), + ('𝔞', '𝔷'), + ('𝕒', '𝕫'), + ('𝖆', '𝖟'), + ('𝖺', '𝗓'), + ('𝗮', '𝘇'), + ('𝘢', '𝘻'), + ('𝙖', '𝙯'), + ('𝚊', '𝚥'), + ('𝛂', '𝛚'), + ('𝛜', '𝛡'), + ('𝛼', '𝜔'), + ('𝜖', '𝜛'), + ('𝜶', '𝝎'), + ('𝝐', '𝝕'), + ('𝝰', '𝞈'), + ('𝞊', '𝞏'), + ('𝞪', '𝟂'), + ('𝟄', '𝟉'), + ('𝟋', '𝟋'), + ('𝼀', '𝼉'), + ('𝼋', '𝼞'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('𞤢', '𞥃'), +]; + +pub const NUMERIC: &'static [(char, char)] = &[ + ('0', '9'), + ('\u{600}', '\u{605}'), + ('٠', '٩'), + ('٫', '٬'), + ('\u{6dd}', '\u{6dd}'), + ('۰', '۹'), + ('߀', '߉'), + ('\u{890}', '\u{891}'), + ('\u{8e2}', '\u{8e2}'), + ('०', '९'), + ('০', '৯'), + ('੦', '੯'), + ('૦', '૯'), + ('୦', '୯'), + ('௦', '௯'), + ('౦', '౯'), + ('೦', '೯'), + ('൦', '൯'), + ('෦', '෯'), + ('๐', '๙'), + ('໐', '໙'), + ('༠', '༩'), + ('၀', '၉'), + ('႐', '႙'), + ('០', '៩'), + ('᠐', '᠙'), + ('᥆', '᥏'), + ('᧐', '᧚'), + ('᪀', '᪉'), + ('᪐', '᪙'), + ('᭐', '᭙'), + ('᮰', '᮹'), + ('᱀', '᱉'), + ('᱐', '᱙'), + ('꘠', '꘩'), + ('꣐', '꣙'), + ('꤀', '꤉'), + ('꧐', '꧙'), + ('꧰', '꧹'), + ('꩐', '꩙'), + ('꯰', '꯹'), + ('０', '９'), + ('𐒠', '𐒩'), + ('𐴰', '𐴹'), + ('𐵀', '𐵉'), + ('𑁦', '𑁯'), + ('\u{110bd}', '\u{110bd}'), + ('\u{110cd}', '\u{110cd}'), + ('𑃰', '𑃹'), + ('𑄶', '𑄿'), + ('𑇐', '𑇙'), + ('𑋰', '𑋹'), + ('𑑐', '𑑙'), + ('𑓐', '𑓙'), + ('𑙐', '𑙙'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜰', '𑜹'), + ('𑣠', '𑣩'), + ('𑥐', '𑥙'), + ('𑯰', '𑯹'), + ('𑱐', '𑱙'), + ('𑵐', '𑵙'), + ('𑶠', '𑶩'), + ('𑽐', '𑽙'), + ('𖄰', '𖄹'), + ('𖩠', '𖩩'), + ('𖫀', '𖫉'), + ('𖭐', '𖭙'), + ('𖵰', '𖵹'), + ('𜳰', '𜳹'), + ('𝟎', '𝟿'), + ('𞅀', '𞅉'), + ('𞋰', '𞋹'), + ('𞓰', '𞓹'), + ('𞗱', '𞗺'), + ('𞥐', '𞥙'), + ('🯰', '🯹'), +]; + +pub const OLETTER: &'static [(char, char)] = &[ + ('ƻ', 'ƻ'), + ('ǀ', 'ǃ'), + ('ʔ', 'ʔ'), + ('ʹ', 'ʿ'), + ('ˆ', 'ˑ'), + ('ˬ', 'ˬ'), + ('ˮ', 'ˮ'), + ('ʹ', 'ʹ'), + ('ՙ', 'ՙ'), + ('א', 'ת'), + ('ׯ', '׳'), + ('ؠ', 'ي'), + ('ٮ', 'ٯ'), + ('ٱ', 'ۓ'), + ('ە', 'ە'), + ('ۥ', 'ۦ'), + ('ۮ', 'ۯ'), + ('ۺ', 'ۼ'), + ('ۿ', 'ۿ'), + ('ܐ', 'ܐ'), + ('ܒ', 'ܯ'), + ('ݍ', 'ޥ'), + ('ޱ', 'ޱ'), + ('ߊ', 'ߪ'), + ('ߴ', 'ߵ'), + ('ߺ', 'ߺ'), + ('ࠀ', 'ࠕ'), + ('ࠚ', 'ࠚ'), + ('ࠤ', 'ࠤ'), + ('ࠨ', 'ࠨ'), + ('ࡀ', 'ࡘ'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('ࢠ', 'ࣉ'), + ('ऄ', 'ह'), + ('ऽ', 'ऽ'), + ('ॐ', 'ॐ'), + ('क़', 'ॡ'), + ('ॱ', 'ঀ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('ঽ', 'ঽ'), + ('ৎ', 'ৎ'), + ('ড়', 'ঢ়'), + ('য়', 'ৡ'), + ('ৰ', 'ৱ'), + ('ৼ', 'ৼ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('ੲ', 'ੴ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('ઽ', 'ઽ'), + ('ૐ', 'ૐ'), + ('ૠ', 'ૡ'), + ('ૹ', 'ૹ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('ଽ', 'ଽ'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୡ'), + ('ୱ', 'ୱ'), + ('ஃ', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('ௐ', 'ௐ'), + ('అ', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('ఽ', 'ఽ'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', 'ౡ'), + ('ಀ', 'ಀ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಽ', 'ಽ'), + ('ೝ', 'ೞ'), + ('ೠ', 'ೡ'), + ('ೱ', 'ೲ'), + ('ഄ', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ഺ'), + ('ഽ', 'ഽ'), + ('ൎ', 'ൎ'), + ('ൔ', 'ൖ'), + ('ൟ', 'ൡ'), + ('ൺ', 'ൿ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('ก', 'ะ'), + ('า', 'ำ'), + ('เ', 'ๆ'), + ('ກ', 'ຂ'), + ('ຄ', 'ຄ'), + ('ຆ', 'ຊ'), + ('ຌ', 'ຣ'), + ('ລ', 'ລ'), + ('ວ', 'ະ'), + ('າ', 'ຳ'), + ('ຽ', 'ຽ'), + ('ເ', 'ໄ'), + ('ໆ', 'ໆ'), + ('ໜ', 'ໟ'), + ('ༀ', 'ༀ'), + ('ཀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('ྈ', 'ྌ'), + ('က', 'ဪ'), + ('ဿ', 'ဿ'), + ('ၐ', 'ၕ'), + ('ၚ', 'ၝ'), + ('ၡ', 'ၡ'), + ('ၥ', 'ၦ'), + ('ၮ', 'ၰ'), + ('ၵ', 'ႁ'), + ('ႎ', 'ႎ'), + ('ა', 'ჺ'), + ('ჽ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('ᎀ', 'ᎏ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', 'ᜑ'), + ('ᜟ', 'ᜱ'), + ('ᝀ', 'ᝑ'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('ក', 'ឳ'), + ('ៗ', 'ៗ'), + ('ៜ', 'ៜ'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢄ'), + ('ᢇ', 'ᢨ'), + ('ᢪ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('ᥐ', 'ᥭ'), + ('ᥰ', 'ᥴ'), + ('ᦀ', 'ᦫ'), + ('ᦰ', 'ᧉ'), + ('ᨀ', 'ᨖ'), + ('ᨠ', 'ᩔ'), + ('ᪧ', 'ᪧ'), + ('ᬅ', 'ᬳ'), + ('ᭅ', 'ᭌ'), + ('ᮃ', 'ᮠ'), + ('ᮮ', 'ᮯ'), + ('ᮺ', 'ᯥ'), + ('ᰀ', 'ᰣ'), + ('ᱍ', 'ᱏ'), + ('ᱚ', 'ᱽ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', 'ᳶ'), + ('ᳺ', 'ᳺ'), + ('ℵ', 'ℸ'), + ('ↀ', 'ↂ'), + ('ↅ', 'ↈ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('ⸯ', 'ⸯ'), + ('々', '〇'), + ('〡', '〩'), + ('〱', '〵'), + ('〸', '〼'), + ('ぁ', 'ゖ'), + ('ゝ', 'ゟ'), + ('ァ', 'ヺ'), + ('ー', 'ヿ'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ㇰ', 'ㇿ'), + ('㐀', '䶿'), + ('一', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘟ'), + ('ꘪ', 'ꘫ'), + ('ꙮ', 'ꙮ'), + ('ꙿ', 'ꙿ'), + ('ꚠ', 'ꛯ'), + ('ꜗ', 'ꜟ'), + ('ꞈ', 'ꞈ'), + ('ꞏ', 'ꞏ'), + ('ꟷ', 'ꟷ'), + ('ꟻ', 'ꠁ'), + ('ꠃ', 'ꠅ'), + ('ꠇ', 'ꠊ'), + ('ꠌ', 'ꠢ'), + ('ꡀ', 'ꡳ'), + ('ꢂ', 'ꢳ'), + ('ꣲ', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', 'ꣾ'), + ('ꤊ', 'ꤥ'), + ('ꤰ', 'ꥆ'), + ('ꥠ', 'ꥼ'), + ('ꦄ', 'ꦲ'), + ('ꧏ', 'ꧏ'), + ('ꧠ', 'ꧤ'), + ('ꧦ', 'ꧯ'), + ('ꧺ', 'ꧾ'), + ('ꨀ', 'ꨨ'), + ('ꩀ', 'ꩂ'), + ('ꩄ', 'ꩋ'), + ('ꩠ', 'ꩶ'), + ('ꩺ', 'ꩺ'), + ('ꩾ', 'ꪯ'), + ('ꪱ', 'ꪱ'), + ('ꪵ', 'ꪶ'), + ('ꪹ', 'ꪽ'), + ('ꫀ', 'ꫀ'), + ('ꫂ', 'ꫂ'), + ('ꫛ', 'ꫝ'), + ('ꫠ', 'ꫪ'), + ('ꫲ', 'ꫴ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꯀ', 'ꯢ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('豈', '舘'), + ('並', '龎'), + ('יִ', 'יִ'), + ('ײַ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('ｦ', 'ﾝ'), + ('ﾠ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '𐍵'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐑐', '𐒝'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞁', '𐞂'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '𐨀'), + ('𐨐', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '𐫤'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐴀', '𐴣'), + ('𐵊', '𐵏'), + ('𐵯', '𐵯'), + ('𐺀', '𐺩'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('𐼀', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '𐽅'), + ('𐽰', '𐾁'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀃', '𑀷'), + ('𑁱', '𑁲'), + ('𑁵', '𑁵'), + ('𑂃', '𑂯'), + ('𑃐', '𑃨'), + ('𑄃', '𑄦'), + ('𑅄', '𑅄'), + ('𑅇', '𑅇'), + ('𑅐', '𑅲'), + ('𑅶', '𑅶'), + ('𑆃', '𑆲'), + ('𑇁', '𑇄'), + ('𑇚', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '𑈫'), + ('𑈿', '𑉀'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '𑋞'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('𑌽', '𑌽'), + ('𑍐', '𑍐'), + ('𑍝', '𑍡'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '𑎷'), + ('𑏑', '𑏑'), + ('𑏓', '𑏓'), + ('𑐀', '𑐴'), + ('𑑇', '𑑊'), + ('𑑟', '𑑡'), + ('𑒀', '𑒯'), + ('𑓄', '𑓅'), + ('𑓇', '𑓇'), + ('𑖀', '𑖮'), + ('𑗘', '𑗛'), + ('𑘀', '𑘯'), + ('𑙄', '𑙄'), + ('𑚀', '𑚪'), + ('𑚸', '𑚸'), + ('𑜀', '𑜚'), + ('𑝀', '𑝆'), + ('𑠀', '𑠫'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤯'), + ('𑤿', '𑤿'), + ('𑥁', '𑥁'), + ('𑦠', '𑦧'), + ('𑦪', '𑧐'), + ('𑧡', '𑧡'), + ('𑧣', '𑧣'), + ('𑨀', '𑨀'), + ('𑨋', '𑨲'), + ('𑨺', '𑨺'), + ('𑩐', '𑩐'), + ('𑩜', '𑪉'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑰀', '𑰈'), + ('𑰊', '𑰮'), + ('𑱀', '𑱀'), + ('𑱲', '𑲏'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '𑴰'), + ('𑵆', '𑵆'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶉'), + ('𑶘', '𑶘'), + ('𑻠', '𑻲'), + ('𑼂', '𑼂'), + ('𑼄', '𑼐'), + ('𑼒', '𑼳'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('𓑁', '𓑆'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄝'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩰', '𖪾'), + ('𖫐', '𖫭'), + ('𖬀', '𖬯'), + ('𖭀', '𖭃'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖼀', '𖽊'), + ('𖽐', '𖽐'), + ('𖾓', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '𖿣'), + ('𗀀', '𘟷'), + ('𘠀', '𘳕'), + ('𘳿', '𘴈'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛄢'), + ('𛄲', '𛄲'), + ('𛅐', '𛅒'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), + ('𛅰', '𛋻'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𝼊', '𝼊'), + ('𞄀', '𞄬'), + ('𞄷', '𞄽'), + ('𞅎', '𞅎'), + ('𞊐', '𞊭'), + ('𞋀', '𞋫'), + ('𞓐', '𞓫'), + ('𞗐', '𞗭'), + ('𞗰', '𞗰'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('𞥋', '𞥋'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('𠀀', '𪛟'), + ('𪜀', '𫜹'), + ('𫝀', '𫠝'), + ('𫠠', '𬺡'), + ('𬺰', '𮯠'), + ('𮯰', '𮹝'), + ('丽', '𪘀'), + ('𰀀', '𱍊'), + ('𱍐', '𲎯'), +]; + +pub const SCONTINUE: &'static [(char, char)] = &[ + (',', '-'), + (':', ';'), + (';', ';'), + ('՝', '՝'), + ('،', '؍'), + ('߸', '߸'), + ('᠂', '᠂'), + ('᠈', '᠈'), + ('–', '—'), + ('、', '、'), + ('︐', '︑'), + ('︓', '︔'), + ('︱', '︲'), + ('﹐', '﹑'), + ('﹔', '﹕'), + ('﹘', '﹘'), + ('﹣', '﹣'), + ('，', '－'), + ('：', '；'), + ('､', '､'), +]; + +pub const STERM: &'static [(char, char)] = &[ + ('!', '!'), + ('?', '?'), + ('։', '։'), + ('؝', '؟'), + ('۔', '۔'), + ('܀', '܂'), + ('߹', '߹'), + ('࠷', '࠷'), + ('࠹', '࠹'), + ('࠽', '࠾'), + ('।', '॥'), + ('၊', '။'), + ('።', '።'), + ('፧', '፨'), + ('᙮', '᙮'), + ('᜵', '᜶'), + ('។', '៕'), + ('᠃', '᠃'), + ('᠉', '᠉'), + ('᥄', '᥅'), + ('᪨', '᪫'), + ('᭎', '᭏'), + ('᭚', '᭛'), + ('᭞', '᭟'), + ('᭽', '᭿'), + ('᰻', '᰼'), + ('᱾', '᱿'), + ('‼', '‽'), + ('⁇', '⁉'), + ('⳹', '⳻'), + ('⸮', '⸮'), + ('⸼', '⸼'), + ('⹓', '⹔'), + ('。', '。'), + ('꓿', '꓿'), + ('꘎', '꘏'), + ('꛳', '꛳'), + ('꛷', '꛷'), + ('꡶', '꡷'), + ('꣎', '꣏'), + ('꤯', '꤯'), + ('꧈', '꧉'), + ('꩝', '꩟'), + ('꫰', '꫱'), + ('꯫', '꯫'), + ('︒', '︒'), + ('︕', '︖'), + ('﹖', '﹗'), + ('！', '！'), + ('？', '？'), + ('｡', '｡'), + ('𐩖', '𐩗'), + ('𐽕', '𐽙'), + ('𐾆', '𐾉'), + ('𑁇', '𑁈'), + ('𑂾', '𑃁'), + ('𑅁', '𑅃'), + ('𑇅', '𑇆'), + ('𑇍', '𑇍'), + ('𑇞', '𑇟'), + ('𑈸', '𑈹'), + ('𑈻', '𑈼'), + ('𑊩', '𑊩'), + ('𑏔', '𑏕'), + ('𑑋', '𑑌'), + ('𑗂', '𑗃'), + ('𑗉', '𑗗'), + ('𑙁', '𑙂'), + ('𑜼', '𑜾'), + ('𑥄', '𑥄'), + ('𑥆', '𑥆'), + ('𑩂', '𑩃'), + ('𑪛', '𑪜'), + ('𑱁', '𑱂'), + ('𑻷', '𑻸'), + ('𑽃', '𑽄'), + ('𖩮', '𖩯'), + ('𖫵', '𖫵'), + ('𖬷', '𖬸'), + ('𖭄', '𖭄'), + ('𖵮', '𖵯'), + ('𖺘', '𖺘'), + ('𛲟', '𛲟'), + ('𝪈', '𝪈'), +]; + +pub const SEP: &'static [(char, char)] = + &[('\u{85}', '\u{85}'), ('\u{2028}', '\u{2029}')]; + +pub const SP: &'static [(char, char)] = &[ + ('\t', '\t'), + ('\u{b}', '\u{c}'), + (' ', ' '), + ('\u{a0}', '\u{a0}'), + ('\u{1680}', '\u{1680}'), + ('\u{2000}', '\u{200a}'), + ('\u{202f}', '\u{202f}'), + ('\u{205f}', '\u{205f}'), + ('\u{3000}', '\u{3000}'), +]; + +pub const UPPER: &'static [(char, char)] = &[ + ('A', 'Z'), + ('À', 'Ö'), + ('Ø', 'Þ'), + ('Ā', 'Ā'), + ('Ă', 'Ă'), + ('Ą', 'Ą'), + ('Ć', 'Ć'), + ('Ĉ', 'Ĉ'), + ('Ċ', 'Ċ'), + ('Č', 'Č'), + ('Ď', 'Ď'), + ('Đ', 'Đ'), + ('Ē', 'Ē'), + ('Ĕ', 'Ĕ'), + ('Ė', 'Ė'), + ('Ę', 'Ę'), + ('Ě', 'Ě'), + ('Ĝ', 'Ĝ'), + ('Ğ', 'Ğ'), + ('Ġ', 'Ġ'), + ('Ģ', 'Ģ'), + ('Ĥ', 'Ĥ'), + ('Ħ', 'Ħ'), + ('Ĩ', 'Ĩ'), + ('Ī', 'Ī'), + ('Ĭ', 'Ĭ'), + ('Į', 'Į'), + ('İ', 'İ'), + ('IJ', 'IJ'), + ('Ĵ', 'Ĵ'), + ('Ķ', 'Ķ'), + ('Ĺ', 'Ĺ'), + ('Ļ', 'Ļ'), + ('Ľ', 'Ľ'), + ('Ŀ', 'Ŀ'), + ('Ł', 'Ł'), + ('Ń', 'Ń'), + ('Ņ', 'Ņ'), + ('Ň', 'Ň'), + ('Ŋ', 'Ŋ'), + ('Ō', 'Ō'), + ('Ŏ', 'Ŏ'), + ('Ő', 'Ő'), + ('Œ', 'Œ'), + ('Ŕ', 'Ŕ'), + ('Ŗ', 'Ŗ'), + ('Ř', 'Ř'), + ('Ś', 'Ś'), + ('Ŝ', 'Ŝ'), + ('Ş', 'Ş'), + ('Š', 'Š'), + ('Ţ', 'Ţ'), + ('Ť', 'Ť'), + ('Ŧ', 'Ŧ'), + ('Ũ', 'Ũ'), + ('Ū', 'Ū'), + ('Ŭ', 'Ŭ'), + ('Ů', 'Ů'), + ('Ű', 'Ű'), + ('Ų', 'Ų'), + ('Ŵ', 'Ŵ'), + ('Ŷ', 'Ŷ'), + ('Ÿ', 'Ź'), + ('Ż', 'Ż'), + ('Ž', 'Ž'), + ('Ɓ', 'Ƃ'), + ('Ƅ', 'Ƅ'), + ('Ɔ', 'Ƈ'), + ('Ɖ', 'Ƌ'), + ('Ǝ', 'Ƒ'), + ('Ɠ', 'Ɣ'), + ('Ɩ', 'Ƙ'), + ('Ɯ', 'Ɲ'), + ('Ɵ', 'Ơ'), + ('Ƣ', 'Ƣ'), + ('Ƥ', 'Ƥ'), + ('Ʀ', 'Ƨ'), + ('Ʃ', 'Ʃ'), + ('Ƭ', 'Ƭ'), + ('Ʈ', 'Ư'), + ('Ʊ', 'Ƴ'), + ('Ƶ', 'Ƶ'), + ('Ʒ', 'Ƹ'), + ('Ƽ', 'Ƽ'), + ('DŽ', 'Dž'), + ('LJ', 'Lj'), + ('NJ', 'Nj'), + ('Ǎ', 'Ǎ'), + ('Ǐ', 'Ǐ'), + ('Ǒ', 'Ǒ'), + ('Ǔ', 'Ǔ'), + ('Ǖ', 'Ǖ'), + ('Ǘ', 'Ǘ'), + ('Ǚ', 'Ǚ'), + ('Ǜ', 'Ǜ'), + ('Ǟ', 'Ǟ'), + ('Ǡ', 'Ǡ'), + ('Ǣ', 'Ǣ'), + ('Ǥ', 'Ǥ'), + ('Ǧ', 'Ǧ'), + ('Ǩ', 'Ǩ'), + ('Ǫ', 'Ǫ'), + ('Ǭ', 'Ǭ'), + ('Ǯ', 'Ǯ'), + ('DZ', 'Dz'), + ('Ǵ', 'Ǵ'), + ('Ƕ', 'Ǹ'), + ('Ǻ', 'Ǻ'), + ('Ǽ', 'Ǽ'), + ('Ǿ', 'Ǿ'), + ('Ȁ', 'Ȁ'), + ('Ȃ', 'Ȃ'), + ('Ȅ', 'Ȅ'), + ('Ȇ', 'Ȇ'), + ('Ȉ', 'Ȉ'), + ('Ȋ', 'Ȋ'), + ('Ȍ', 'Ȍ'), + ('Ȏ', 'Ȏ'), + ('Ȑ', 'Ȑ'), + ('Ȓ', 'Ȓ'), + ('Ȕ', 'Ȕ'), + ('Ȗ', 'Ȗ'), + ('Ș', 'Ș'), + ('Ț', 'Ț'), + ('Ȝ', 'Ȝ'), + ('Ȟ', 'Ȟ'), + ('Ƞ', 'Ƞ'), + ('Ȣ', 'Ȣ'), + ('Ȥ', 'Ȥ'), + ('Ȧ', 'Ȧ'), + ('Ȩ', 'Ȩ'), + ('Ȫ', 'Ȫ'), + ('Ȭ', 'Ȭ'), + ('Ȯ', 'Ȯ'), + ('Ȱ', 'Ȱ'), + ('Ȳ', 'Ȳ'), + ('Ⱥ', 'Ȼ'), + ('Ƚ', 'Ⱦ'), + ('Ɂ', 'Ɂ'), + ('Ƀ', 'Ɇ'), + ('Ɉ', 'Ɉ'), + ('Ɋ', 'Ɋ'), + ('Ɍ', 'Ɍ'), + ('Ɏ', 'Ɏ'), + ('Ͱ', 'Ͱ'), + ('Ͳ', 'Ͳ'), + ('Ͷ', 'Ͷ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ώ'), + ('Α', 'Ρ'), + ('Σ', 'Ϋ'), + ('Ϗ', 'Ϗ'), + ('ϒ', 'ϔ'), + ('Ϙ', 'Ϙ'), + ('Ϛ', 'Ϛ'), + ('Ϝ', 'Ϝ'), + ('Ϟ', 'Ϟ'), + ('Ϡ', 'Ϡ'), + ('Ϣ', 'Ϣ'), + ('Ϥ', 'Ϥ'), + ('Ϧ', 'Ϧ'), + ('Ϩ', 'Ϩ'), + ('Ϫ', 'Ϫ'), + ('Ϭ', 'Ϭ'), + ('Ϯ', 'Ϯ'), + ('ϴ', 'ϴ'), + ('Ϸ', 'Ϸ'), + ('Ϲ', 'Ϻ'), + ('Ͻ', 'Я'), + ('Ѡ', 'Ѡ'), + ('Ѣ', 'Ѣ'), + ('Ѥ', 'Ѥ'), + ('Ѧ', 'Ѧ'), + ('Ѩ', 'Ѩ'), + ('Ѫ', 'Ѫ'), + ('Ѭ', 'Ѭ'), + ('Ѯ', 'Ѯ'), + ('Ѱ', 'Ѱ'), + ('Ѳ', 'Ѳ'), + ('Ѵ', 'Ѵ'), + ('Ѷ', 'Ѷ'), + ('Ѹ', 'Ѹ'), + ('Ѻ', 'Ѻ'), + ('Ѽ', 'Ѽ'), + ('Ѿ', 'Ѿ'), + ('Ҁ', 'Ҁ'), + ('Ҋ', 'Ҋ'), + ('Ҍ', 'Ҍ'), + ('Ҏ', 'Ҏ'), + ('Ґ', 'Ґ'), + ('Ғ', 'Ғ'), + ('Ҕ', 'Ҕ'), + ('Җ', 'Җ'), + ('Ҙ', 'Ҙ'), + ('Қ', 'Қ'), + ('Ҝ', 'Ҝ'), + ('Ҟ', 'Ҟ'), + ('Ҡ', 'Ҡ'), + ('Ң', 'Ң'), + ('Ҥ', 'Ҥ'), + ('Ҧ', 'Ҧ'), + ('Ҩ', 'Ҩ'), + ('Ҫ', 'Ҫ'), + ('Ҭ', 'Ҭ'), + ('Ү', 'Ү'), + ('Ұ', 'Ұ'), + ('Ҳ', 'Ҳ'), + ('Ҵ', 'Ҵ'), + ('Ҷ', 'Ҷ'), + ('Ҹ', 'Ҹ'), + ('Һ', 'Һ'), + ('Ҽ', 'Ҽ'), + ('Ҿ', 'Ҿ'), + ('Ӏ', 'Ӂ'), + ('Ӄ', 'Ӄ'), + ('Ӆ', 'Ӆ'), + ('Ӈ', 'Ӈ'), + ('Ӊ', 'Ӊ'), + ('Ӌ', 'Ӌ'), + ('Ӎ', 'Ӎ'), + ('Ӑ', 'Ӑ'), + ('Ӓ', 'Ӓ'), + ('Ӕ', 'Ӕ'), + ('Ӗ', 'Ӗ'), + ('Ә', 'Ә'), + ('Ӛ', 'Ӛ'), + ('Ӝ', 'Ӝ'), + ('Ӟ', 'Ӟ'), + ('Ӡ', 'Ӡ'), + ('Ӣ', 'Ӣ'), + ('Ӥ', 'Ӥ'), + ('Ӧ', 'Ӧ'), + ('Ө', 'Ө'), + ('Ӫ', 'Ӫ'), + ('Ӭ', 'Ӭ'), + ('Ӯ', 'Ӯ'), + ('Ӱ', 'Ӱ'), + ('Ӳ', 'Ӳ'), + ('Ӵ', 'Ӵ'), + ('Ӷ', 'Ӷ'), + ('Ӹ', 'Ӹ'), + ('Ӻ', 'Ӻ'), + ('Ӽ', 'Ӽ'), + ('Ӿ', 'Ӿ'), + ('Ԁ', 'Ԁ'), + ('Ԃ', 'Ԃ'), + ('Ԅ', 'Ԅ'), + ('Ԇ', 'Ԇ'), + ('Ԉ', 'Ԉ'), + ('Ԋ', 'Ԋ'), + ('Ԍ', 'Ԍ'), + ('Ԏ', 'Ԏ'), + ('Ԑ', 'Ԑ'), + ('Ԓ', 'Ԓ'), + ('Ԕ', 'Ԕ'), + ('Ԗ', 'Ԗ'), + ('Ԙ', 'Ԙ'), + ('Ԛ', 'Ԛ'), + ('Ԝ', 'Ԝ'), + ('Ԟ', 'Ԟ'), + ('Ԡ', 'Ԡ'), + ('Ԣ', 'Ԣ'), + ('Ԥ', 'Ԥ'), + ('Ԧ', 'Ԧ'), + ('Ԩ', 'Ԩ'), + ('Ԫ', 'Ԫ'), + ('Ԭ', 'Ԭ'), + ('Ԯ', 'Ԯ'), + ('Ա', 'Ֆ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('Ꭰ', 'Ᏽ'), + ('Ᲊ', 'Ᲊ'), + ('Ḁ', 'Ḁ'), + ('Ḃ', 'Ḃ'), + ('Ḅ', 'Ḅ'), + ('Ḇ', 'Ḇ'), + ('Ḉ', 'Ḉ'), + ('Ḋ', 'Ḋ'), + ('Ḍ', 'Ḍ'), + ('Ḏ', 'Ḏ'), + ('Ḑ', 'Ḑ'), + ('Ḓ', 'Ḓ'), + ('Ḕ', 'Ḕ'), + ('Ḗ', 'Ḗ'), + ('Ḙ', 'Ḙ'), + ('Ḛ', 'Ḛ'), + ('Ḝ', 'Ḝ'), + ('Ḟ', 'Ḟ'), + ('Ḡ', 'Ḡ'), + ('Ḣ', 'Ḣ'), + ('Ḥ', 'Ḥ'), + ('Ḧ', 'Ḧ'), + ('Ḩ', 'Ḩ'), + ('Ḫ', 'Ḫ'), + ('Ḭ', 'Ḭ'), + ('Ḯ', 'Ḯ'), + ('Ḱ', 'Ḱ'), + ('Ḳ', 'Ḳ'), + ('Ḵ', 'Ḵ'), + ('Ḷ', 'Ḷ'), + ('Ḹ', 'Ḹ'), + ('Ḻ', 'Ḻ'), + ('Ḽ', 'Ḽ'), + ('Ḿ', 'Ḿ'), + ('Ṁ', 'Ṁ'), + ('Ṃ', 'Ṃ'), + ('Ṅ', 'Ṅ'), + ('Ṇ', 'Ṇ'), + ('Ṉ', 'Ṉ'), + ('Ṋ', 'Ṋ'), + ('Ṍ', 'Ṍ'), + ('Ṏ', 'Ṏ'), + ('Ṑ', 'Ṑ'), + ('Ṓ', 'Ṓ'), + ('Ṕ', 'Ṕ'), + ('Ṗ', 'Ṗ'), + ('Ṙ', 'Ṙ'), + ('Ṛ', 'Ṛ'), + ('Ṝ', 'Ṝ'), + ('Ṟ', 'Ṟ'), + ('Ṡ', 'Ṡ'), + ('Ṣ', 'Ṣ'), + ('Ṥ', 'Ṥ'), + ('Ṧ', 'Ṧ'), + ('Ṩ', 'Ṩ'), + ('Ṫ', 'Ṫ'), + ('Ṭ', 'Ṭ'), + ('Ṯ', 'Ṯ'), + ('Ṱ', 'Ṱ'), + ('Ṳ', 'Ṳ'), + ('Ṵ', 'Ṵ'), + ('Ṷ', 'Ṷ'), + ('Ṹ', 'Ṹ'), + ('Ṻ', 'Ṻ'), + ('Ṽ', 'Ṽ'), + ('Ṿ', 'Ṿ'), + ('Ẁ', 'Ẁ'), + ('Ẃ', 'Ẃ'), + ('Ẅ', 'Ẅ'), + ('Ẇ', 'Ẇ'), + ('Ẉ', 'Ẉ'), + ('Ẋ', 'Ẋ'), + ('Ẍ', 'Ẍ'), + ('Ẏ', 'Ẏ'), + ('Ẑ', 'Ẑ'), + ('Ẓ', 'Ẓ'), + ('Ẕ', 'Ẕ'), + ('ẞ', 'ẞ'), + ('Ạ', 'Ạ'), + ('Ả', 'Ả'), + ('Ấ', 'Ấ'), + ('Ầ', 'Ầ'), + ('Ẩ', 'Ẩ'), + ('Ẫ', 'Ẫ'), + ('Ậ', 'Ậ'), + ('Ắ', 'Ắ'), + ('Ằ', 'Ằ'), + ('Ẳ', 'Ẳ'), + ('Ẵ', 'Ẵ'), + ('Ặ', 'Ặ'), + ('Ẹ', 'Ẹ'), + ('Ẻ', 'Ẻ'), + ('Ẽ', 'Ẽ'), + ('Ế', 'Ế'), + ('Ề', 'Ề'), + ('Ể', 'Ể'), + ('Ễ', 'Ễ'), + ('Ệ', 'Ệ'), + ('Ỉ', 'Ỉ'), + ('Ị', 'Ị'), + ('Ọ', 'Ọ'), + ('Ỏ', 'Ỏ'), + ('Ố', 'Ố'), + ('Ồ', 'Ồ'), + ('Ổ', 'Ổ'), + ('Ỗ', 'Ỗ'), + ('Ộ', 'Ộ'), + ('Ớ', 'Ớ'), + ('Ờ', 'Ờ'), + ('Ở', 'Ở'), + ('Ỡ', 'Ỡ'), + ('Ợ', 'Ợ'), + ('Ụ', 'Ụ'), + ('Ủ', 'Ủ'), + ('Ứ', 'Ứ'), + ('Ừ', 'Ừ'), + ('Ử', 'Ử'), + ('Ữ', 'Ữ'), + ('Ự', 'Ự'), + ('Ỳ', 'Ỳ'), + ('Ỵ', 'Ỵ'), + ('Ỷ', 'Ỷ'), + ('Ỹ', 'Ỹ'), + ('Ỻ', 'Ỻ'), + ('Ỽ', 'Ỽ'), + ('Ỿ', 'Ỿ'), + ('Ἀ', 'Ἇ'), + ('Ἐ', 'Ἕ'), + ('Ἠ', 'Ἧ'), + ('Ἰ', 'Ἷ'), + ('Ὀ', 'Ὅ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'Ὗ'), + ('Ὠ', 'Ὧ'), + ('ᾈ', 'ᾏ'), + ('ᾘ', 'ᾟ'), + ('ᾨ', 'ᾯ'), + ('Ᾰ', 'ᾼ'), + ('Ὲ', 'ῌ'), + ('Ῐ', 'Ί'), + ('Ῠ', 'Ῥ'), + ('Ὸ', 'ῼ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℋ', 'ℍ'), + ('ℐ', 'ℒ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℰ', 'ℳ'), + ('ℾ', 'ℿ'), + ('ⅅ', 'ⅅ'), + ('Ⅰ', 'Ⅿ'), + ('Ↄ', 'Ↄ'), + ('Ⓐ', 'Ⓩ'), + ('Ⰰ', 'Ⱟ'), + ('Ⱡ', 'Ⱡ'), + ('Ɫ', 'Ɽ'), + ('Ⱨ', 'Ⱨ'), + ('Ⱪ', 'Ⱪ'), + ('Ⱬ', 'Ⱬ'), + ('Ɑ', 'Ɒ'), + ('Ⱳ', 'Ⱳ'), + ('Ⱶ', 'Ⱶ'), + ('Ȿ', 'Ⲁ'), + ('Ⲃ', 'Ⲃ'), + ('Ⲅ', 'Ⲅ'), + ('Ⲇ', 'Ⲇ'), + ('Ⲉ', 'Ⲉ'), + ('Ⲋ', 'Ⲋ'), + ('Ⲍ', 'Ⲍ'), + ('Ⲏ', 'Ⲏ'), + ('Ⲑ', 'Ⲑ'), + ('Ⲓ', 'Ⲓ'), + ('Ⲕ', 'Ⲕ'), + ('Ⲗ', 'Ⲗ'), + ('Ⲙ', 'Ⲙ'), + ('Ⲛ', 'Ⲛ'), + ('Ⲝ', 'Ⲝ'), + ('Ⲟ', 'Ⲟ'), + ('Ⲡ', 'Ⲡ'), + ('Ⲣ', 'Ⲣ'), + ('Ⲥ', 'Ⲥ'), + ('Ⲧ', 'Ⲧ'), + ('Ⲩ', 'Ⲩ'), + ('Ⲫ', 'Ⲫ'), + ('Ⲭ', 'Ⲭ'), + ('Ⲯ', 'Ⲯ'), + ('Ⲱ', 'Ⲱ'), + ('Ⲳ', 'Ⲳ'), + ('Ⲵ', 'Ⲵ'), + ('Ⲷ', 'Ⲷ'), + ('Ⲹ', 'Ⲹ'), + ('Ⲻ', 'Ⲻ'), + ('Ⲽ', 'Ⲽ'), + ('Ⲿ', 'Ⲿ'), + ('Ⳁ', 'Ⳁ'), + ('Ⳃ', 'Ⳃ'), + ('Ⳅ', 'Ⳅ'), + ('Ⳇ', 'Ⳇ'), + ('Ⳉ', 'Ⳉ'), + ('Ⳋ', 'Ⳋ'), + ('Ⳍ', 'Ⳍ'), + ('Ⳏ', 'Ⳏ'), + ('Ⳑ', 'Ⳑ'), + ('Ⳓ', 'Ⳓ'), + ('Ⳕ', 'Ⳕ'), + ('Ⳗ', 'Ⳗ'), + ('Ⳙ', 'Ⳙ'), + ('Ⳛ', 'Ⳛ'), + ('Ⳝ', 'Ⳝ'), + ('Ⳟ', 'Ⳟ'), + ('Ⳡ', 'Ⳡ'), + ('Ⳣ', 'Ⳣ'), + ('Ⳬ', 'Ⳬ'), + ('Ⳮ', 'Ⳮ'), + ('Ⳳ', 'Ⳳ'), + ('Ꙁ', 'Ꙁ'), + ('Ꙃ', 'Ꙃ'), + ('Ꙅ', 'Ꙅ'), + ('Ꙇ', 'Ꙇ'), + ('Ꙉ', 'Ꙉ'), + ('Ꙋ', 'Ꙋ'), + ('Ꙍ', 'Ꙍ'), + ('Ꙏ', 'Ꙏ'), + ('Ꙑ', 'Ꙑ'), + ('Ꙓ', 'Ꙓ'), + ('Ꙕ', 'Ꙕ'), + ('Ꙗ', 'Ꙗ'), + ('Ꙙ', 'Ꙙ'), + ('Ꙛ', 'Ꙛ'), + ('Ꙝ', 'Ꙝ'), + ('Ꙟ', 'Ꙟ'), + ('Ꙡ', 'Ꙡ'), + ('Ꙣ', 'Ꙣ'), + ('Ꙥ', 'Ꙥ'), + ('Ꙧ', 'Ꙧ'), + ('Ꙩ', 'Ꙩ'), + ('Ꙫ', 'Ꙫ'), + ('Ꙭ', 'Ꙭ'), + ('Ꚁ', 'Ꚁ'), + ('Ꚃ', 'Ꚃ'), + ('Ꚅ', 'Ꚅ'), + ('Ꚇ', 'Ꚇ'), + ('Ꚉ', 'Ꚉ'), + ('Ꚋ', 'Ꚋ'), + ('Ꚍ', 'Ꚍ'), + ('Ꚏ', 'Ꚏ'), + ('Ꚑ', 'Ꚑ'), + ('Ꚓ', 'Ꚓ'), + ('Ꚕ', 'Ꚕ'), + ('Ꚗ', 'Ꚗ'), + ('Ꚙ', 'Ꚙ'), + ('Ꚛ', 'Ꚛ'), + ('Ꜣ', 'Ꜣ'), + ('Ꜥ', 'Ꜥ'), + ('Ꜧ', 'Ꜧ'), + ('Ꜩ', 'Ꜩ'), + ('Ꜫ', 'Ꜫ'), + ('Ꜭ', 'Ꜭ'), + ('Ꜯ', 'Ꜯ'), + ('Ꜳ', 'Ꜳ'), + ('Ꜵ', 'Ꜵ'), + ('Ꜷ', 'Ꜷ'), + ('Ꜹ', 'Ꜹ'), + ('Ꜻ', 'Ꜻ'), + ('Ꜽ', 'Ꜽ'), + ('Ꜿ', 'Ꜿ'), + ('Ꝁ', 'Ꝁ'), + ('Ꝃ', 'Ꝃ'), + ('Ꝅ', 'Ꝅ'), + ('Ꝇ', 'Ꝇ'), + ('Ꝉ', 'Ꝉ'), + ('Ꝋ', 'Ꝋ'), + ('Ꝍ', 'Ꝍ'), + ('Ꝏ', 'Ꝏ'), + ('Ꝑ', 'Ꝑ'), + ('Ꝓ', 'Ꝓ'), + ('Ꝕ', 'Ꝕ'), + ('Ꝗ', 'Ꝗ'), + ('Ꝙ', 'Ꝙ'), + ('Ꝛ', 'Ꝛ'), + ('Ꝝ', 'Ꝝ'), + ('Ꝟ', 'Ꝟ'), + ('Ꝡ', 'Ꝡ'), + ('Ꝣ', 'Ꝣ'), + ('Ꝥ', 'Ꝥ'), + ('Ꝧ', 'Ꝧ'), + ('Ꝩ', 'Ꝩ'), + ('Ꝫ', 'Ꝫ'), + ('Ꝭ', 'Ꝭ'), + ('Ꝯ', 'Ꝯ'), + ('Ꝺ', 'Ꝺ'), + ('Ꝼ', 'Ꝼ'), + ('Ᵹ', 'Ꝿ'), + ('Ꞁ', 'Ꞁ'), + ('Ꞃ', 'Ꞃ'), + ('Ꞅ', 'Ꞅ'), + ('Ꞇ', 'Ꞇ'), + ('Ꞌ', 'Ꞌ'), + ('Ɥ', 'Ɥ'), + ('Ꞑ', 'Ꞑ'), + ('Ꞓ', 'Ꞓ'), + ('Ꞗ', 'Ꞗ'), + ('Ꞙ', 'Ꞙ'), + ('Ꞛ', 'Ꞛ'), + ('Ꞝ', 'Ꞝ'), + ('Ꞟ', 'Ꞟ'), + ('Ꞡ', 'Ꞡ'), + ('Ꞣ', 'Ꞣ'), + ('Ꞥ', 'Ꞥ'), + ('Ꞧ', 'Ꞧ'), + ('Ꞩ', 'Ꞩ'), + ('Ɦ', 'Ɪ'), + ('Ʞ', 'Ꞵ'), + ('Ꞷ', 'Ꞷ'), + ('Ꞹ', 'Ꞹ'), + ('Ꞻ', 'Ꞻ'), + ('Ꞽ', 'Ꞽ'), + ('Ꞿ', 'Ꞿ'), + ('Ꟁ', 'Ꟁ'), + ('Ꟃ', 'Ꟃ'), + ('Ꞔ', 'Ꟈ'), + ('Ꟊ', 'Ꟊ'), + ('Ɤ', 'Ꟍ'), + ('Ꟑ', 'Ꟑ'), + ('Ꟗ', 'Ꟗ'), + ('Ꟙ', 'Ꟙ'), + ('Ꟛ', 'Ꟛ'), + ('Ƛ', 'Ƛ'), + ('Ꟶ', 'Ꟶ'), + ('Ａ', 'Ｚ'), + ('𐐀', '𐐧'), + ('𐒰', '𐓓'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐲀', '𐲲'), + ('𐵐', '𐵥'), + ('𑢠', '𑢿'), + ('𖹀', '𖹟'), + ('𝐀', '𝐙'), + ('𝐴', '𝑍'), + ('𝑨', '𝒁'), + ('𝒜', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒵'), + ('𝓐', '𝓩'), + ('𝔄', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔸', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕬', '𝖅'), + ('𝖠', '𝖹'), + ('𝗔', '𝗭'), + ('𝘈', '𝘡'), + ('𝘼', '𝙕'), + ('𝙰', '𝚉'), + ('𝚨', '𝛀'), + ('𝛢', '𝛺'), + ('𝜜', '𝜴'), + ('𝝖', '𝝮'), + ('𝞐', '𝞨'), + ('𝟊', '𝟊'), + ('𞤀', '𞤡'), + ('🄰', '🅉'), + ('🅐', '🅩'), + ('🅰', '🆉'), +]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/word_break.rs b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/word_break.rs new file mode 100644 index 0000000..b764d34 --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/unicode_tables/word_break.rs @@ -0,0 +1,1152 @@ +// DO NOT EDIT THIS FILE. IT WAS AUTOMATICALLY GENERATED BY: +// +// ucd-generate word-break ucd-16.0.0 --chars +// +// Unicode version: 16.0.0. +// +// ucd-generate 0.3.1 is available on crates.io. + +pub const BY_NAME: &'static [(&'static str, &'static [(char, char)])] = &[ + ("ALetter", ALETTER), + ("CR", CR), + ("Double_Quote", DOUBLE_QUOTE), + ("Extend", EXTEND), + ("ExtendNumLet", EXTENDNUMLET), + ("Format", FORMAT), + ("Hebrew_Letter", HEBREW_LETTER), + ("Katakana", KATAKANA), + ("LF", LF), + ("MidLetter", MIDLETTER), + ("MidNum", MIDNUM), + ("MidNumLet", MIDNUMLET), + ("Newline", NEWLINE), + ("Numeric", NUMERIC), + ("Regional_Indicator", REGIONAL_INDICATOR), + ("Single_Quote", SINGLE_QUOTE), + ("WSegSpace", WSEGSPACE), + ("ZWJ", ZWJ), +]; + +pub const ALETTER: &'static [(char, char)] = &[ + ('A', 'Z'), + ('a', 'z'), + ('ª', 'ª'), + ('µ', 'µ'), + ('º', 'º'), + ('À', 'Ö'), + ('Ø', 'ö'), + ('ø', '˗'), + ('˞', '˿'), + ('Ͱ', 'ʹ'), + ('Ͷ', 'ͷ'), + ('ͺ', 'ͽ'), + ('Ϳ', 'Ϳ'), + ('Ά', 'Ά'), + ('Έ', 'Ί'), + ('Ό', 'Ό'), + ('Ύ', 'Ρ'), + ('Σ', 'ϵ'), + ('Ϸ', 'ҁ'), + ('Ҋ', 'ԯ'), + ('Ա', 'Ֆ'), + ('ՙ', '՜'), + ('՞', '՞'), + ('ՠ', 'ֈ'), + ('֊', '֊'), + ('׳', '׳'), + ('ؠ', 'ي'), + ('ٮ', 'ٯ'), + ('ٱ', 'ۓ'), + ('ە', 'ە'), + ('ۥ', 'ۦ'), + ('ۮ', 'ۯ'), + ('ۺ', 'ۼ'), + ('ۿ', 'ۿ'), + ('\u{70f}', 'ܐ'), + ('ܒ', 'ܯ'), + ('ݍ', 'ޥ'), + ('ޱ', 'ޱ'), + ('ߊ', 'ߪ'), + ('ߴ', 'ߵ'), + ('ߺ', 'ߺ'), + ('ࠀ', 'ࠕ'), + ('ࠚ', 'ࠚ'), + ('ࠤ', 'ࠤ'), + ('ࠨ', 'ࠨ'), + ('ࡀ', 'ࡘ'), + ('ࡠ', 'ࡪ'), + ('ࡰ', 'ࢇ'), + ('ࢉ', 'ࢎ'), + ('ࢠ', 'ࣉ'), + ('ऄ', 'ह'), + ('ऽ', 'ऽ'), + ('ॐ', 'ॐ'), + ('क़', 'ॡ'), + ('ॱ', 'ঀ'), + ('অ', 'ঌ'), + ('এ', 'ঐ'), + ('ও', 'ন'), + ('প', 'র'), + ('ল', 'ল'), + ('শ', 'হ'), + ('ঽ', 'ঽ'), + ('ৎ', 'ৎ'), + ('ড়', 'ঢ়'), + ('য়', 'ৡ'), + ('ৰ', 'ৱ'), + ('ৼ', 'ৼ'), + ('ਅ', 'ਊ'), + ('ਏ', 'ਐ'), + ('ਓ', 'ਨ'), + ('ਪ', 'ਰ'), + ('ਲ', 'ਲ਼'), + ('ਵ', 'ਸ਼'), + ('ਸ', 'ਹ'), + ('ਖ਼', 'ੜ'), + ('ਫ਼', 'ਫ਼'), + ('ੲ', 'ੴ'), + ('અ', 'ઍ'), + ('એ', 'ઑ'), + ('ઓ', 'ન'), + ('પ', 'ર'), + ('લ', 'ળ'), + ('વ', 'હ'), + ('ઽ', 'ઽ'), + ('ૐ', 'ૐ'), + ('ૠ', 'ૡ'), + ('ૹ', 'ૹ'), + ('ଅ', 'ଌ'), + ('ଏ', 'ଐ'), + ('ଓ', 'ନ'), + ('ପ', 'ର'), + ('ଲ', 'ଳ'), + ('ଵ', 'ହ'), + ('ଽ', 'ଽ'), + ('ଡ଼', 'ଢ଼'), + ('ୟ', 'ୡ'), + ('ୱ', 'ୱ'), + ('ஃ', 'ஃ'), + ('அ', 'ஊ'), + ('எ', 'ஐ'), + ('ஒ', 'க'), + ('ங', 'ச'), + ('ஜ', 'ஜ'), + ('ஞ', 'ட'), + ('ண', 'த'), + ('ந', 'ப'), + ('ம', 'ஹ'), + ('ௐ', 'ௐ'), + ('అ', 'ఌ'), + ('ఎ', 'ఐ'), + ('ఒ', 'న'), + ('ప', 'హ'), + ('ఽ', 'ఽ'), + ('ౘ', 'ౚ'), + ('ౝ', 'ౝ'), + ('ౠ', 'ౡ'), + ('ಀ', 'ಀ'), + ('ಅ', 'ಌ'), + ('ಎ', 'ಐ'), + ('ಒ', 'ನ'), + ('ಪ', 'ಳ'), + ('ವ', 'ಹ'), + ('ಽ', 'ಽ'), + ('ೝ', 'ೞ'), + ('ೠ', 'ೡ'), + ('ೱ', 'ೲ'), + ('ഄ', 'ഌ'), + ('എ', 'ഐ'), + ('ഒ', 'ഺ'), + ('ഽ', 'ഽ'), + ('ൎ', 'ൎ'), + ('ൔ', 'ൖ'), + ('ൟ', 'ൡ'), + ('ൺ', 'ൿ'), + ('අ', 'ඖ'), + ('ක', 'න'), + ('ඳ', 'ර'), + ('ල', 'ල'), + ('ව', 'ෆ'), + ('ༀ', 'ༀ'), + ('ཀ', 'ཇ'), + ('ཉ', 'ཬ'), + ('ྈ', 'ྌ'), + ('Ⴀ', 'Ⴥ'), + ('Ⴧ', 'Ⴧ'), + ('Ⴭ', 'Ⴭ'), + ('ა', 'ჺ'), + ('ჼ', 'ቈ'), + ('ቊ', 'ቍ'), + ('ቐ', 'ቖ'), + ('ቘ', 'ቘ'), + ('ቚ', 'ቝ'), + ('በ', 'ኈ'), + ('ኊ', 'ኍ'), + ('ነ', 'ኰ'), + ('ኲ', 'ኵ'), + ('ኸ', 'ኾ'), + ('ዀ', 'ዀ'), + ('ዂ', 'ዅ'), + ('ወ', 'ዖ'), + ('ዘ', 'ጐ'), + ('ጒ', 'ጕ'), + ('ጘ', 'ፚ'), + ('ᎀ', 'ᎏ'), + ('Ꭰ', 'Ᏽ'), + ('ᏸ', 'ᏽ'), + ('ᐁ', 'ᙬ'), + ('ᙯ', 'ᙿ'), + ('ᚁ', 'ᚚ'), + ('ᚠ', 'ᛪ'), + ('ᛮ', 'ᛸ'), + ('ᜀ', 'ᜑ'), + ('ᜟ', 'ᜱ'), + ('ᝀ', 'ᝑ'), + ('ᝠ', 'ᝬ'), + ('ᝮ', 'ᝰ'), + ('ᠠ', 'ᡸ'), + ('ᢀ', 'ᢄ'), + ('ᢇ', 'ᢨ'), + ('ᢪ', 'ᢪ'), + ('ᢰ', 'ᣵ'), + ('ᤀ', 'ᤞ'), + ('ᨀ', 'ᨖ'), + ('ᬅ', 'ᬳ'), + ('ᭅ', 'ᭌ'), + ('ᮃ', 'ᮠ'), + ('ᮮ', 'ᮯ'), + ('ᮺ', 'ᯥ'), + ('ᰀ', 'ᰣ'), + ('ᱍ', 'ᱏ'), + ('ᱚ', 'ᱽ'), + ('ᲀ', 'ᲊ'), + ('Ა', 'Ჺ'), + ('Ჽ', 'Ჿ'), + ('ᳩ', 'ᳬ'), + ('ᳮ', 'ᳳ'), + ('ᳵ', 'ᳶ'), + ('ᳺ', 'ᳺ'), + ('ᴀ', 'ᶿ'), + ('Ḁ', 'ἕ'), + ('Ἐ', 'Ἕ'), + ('ἠ', 'ὅ'), + ('Ὀ', 'Ὅ'), + ('ὐ', 'ὗ'), + ('Ὑ', 'Ὑ'), + ('Ὓ', 'Ὓ'), + ('Ὕ', 'Ὕ'), + ('Ὗ', 'ώ'), + ('ᾀ', 'ᾴ'), + ('ᾶ', 'ᾼ'), + ('ι', 'ι'), + ('ῂ', 'ῄ'), + ('ῆ', 'ῌ'), + ('ῐ', 'ΐ'), + ('ῖ', 'Ί'), + ('ῠ', 'Ῥ'), + ('ῲ', 'ῴ'), + ('ῶ', 'ῼ'), + ('ⁱ', 'ⁱ'), + ('ⁿ', 'ⁿ'), + ('ₐ', 'ₜ'), + ('ℂ', 'ℂ'), + ('ℇ', 'ℇ'), + ('ℊ', 'ℓ'), + ('ℕ', 'ℕ'), + ('ℙ', 'ℝ'), + ('ℤ', 'ℤ'), + ('Ω', 'Ω'), + ('ℨ', 'ℨ'), + ('K', 'ℭ'), + ('ℯ', 'ℹ'), + ('ℼ', 'ℿ'), + ('ⅅ', 'ⅉ'), + ('ⅎ', 'ⅎ'), + ('Ⅰ', 'ↈ'), + ('Ⓐ', 'ⓩ'), + ('Ⰰ', 'ⳤ'), + ('Ⳬ', 'ⳮ'), + ('Ⳳ', 'ⳳ'), + ('ⴀ', 'ⴥ'), + ('ⴧ', 'ⴧ'), + ('ⴭ', 'ⴭ'), + ('ⴰ', 'ⵧ'), + ('ⵯ', 'ⵯ'), + ('ⶀ', 'ⶖ'), + ('ⶠ', 'ⶦ'), + ('ⶨ', 'ⶮ'), + ('ⶰ', 'ⶶ'), + ('ⶸ', 'ⶾ'), + ('ⷀ', 'ⷆ'), + ('ⷈ', 'ⷎ'), + ('ⷐ', 'ⷖ'), + ('ⷘ', 'ⷞ'), + ('ⸯ', 'ⸯ'), + ('々', '々'), + ('〻', '〼'), + ('ㄅ', 'ㄯ'), + ('ㄱ', 'ㆎ'), + ('ㆠ', 'ㆿ'), + ('ꀀ', 'ꒌ'), + ('ꓐ', 'ꓽ'), + ('ꔀ', 'ꘌ'), + ('ꘐ', 'ꘟ'), + ('ꘪ', 'ꘫ'), + ('Ꙁ', 'ꙮ'), + ('ꙿ', 'ꚝ'), + ('ꚠ', 'ꛯ'), + ('꜈', 'ꟍ'), + ('Ꟑ', 'ꟑ'), + ('ꟓ', 'ꟓ'), + ('ꟕ', 'Ƛ'), + ('ꟲ', 'ꠁ'), + ('ꠃ', 'ꠅ'), + ('ꠇ', 'ꠊ'), + ('ꠌ', 'ꠢ'), + ('ꡀ', 'ꡳ'), + ('ꢂ', 'ꢳ'), + ('ꣲ', 'ꣷ'), + ('ꣻ', 'ꣻ'), + ('ꣽ', 'ꣾ'), + ('ꤊ', 'ꤥ'), + ('ꤰ', 'ꥆ'), + ('ꥠ', 'ꥼ'), + ('ꦄ', 'ꦲ'), + ('ꧏ', 'ꧏ'), + ('ꨀ', 'ꨨ'), + ('ꩀ', 'ꩂ'), + ('ꩄ', 'ꩋ'), + ('ꫠ', 'ꫪ'), + ('ꫲ', 'ꫴ'), + ('ꬁ', 'ꬆ'), + ('ꬉ', 'ꬎ'), + ('ꬑ', 'ꬖ'), + ('ꬠ', 'ꬦ'), + ('ꬨ', 'ꬮ'), + ('ꬰ', 'ꭩ'), + ('ꭰ', 'ꯢ'), + ('가', '힣'), + ('ힰ', 'ퟆ'), + ('ퟋ', 'ퟻ'), + ('ff', 'st'), + ('ﬓ', 'ﬗ'), + ('ﭐ', 'ﮱ'), + ('ﯓ', 'ﴽ'), + ('ﵐ', 'ﶏ'), + ('ﶒ', 'ﷇ'), + ('ﷰ', 'ﷻ'), + ('ﹰ', 'ﹴ'), + ('ﹶ', 'ﻼ'), + ('Ａ', 'Ｚ'), + ('ａ', 'ｚ'), + ('ﾠ', 'ﾾ'), + ('ￂ', 'ￇ'), + ('ￊ', 'ￏ'), + ('ￒ', 'ￗ'), + ('ￚ', 'ￜ'), + ('𐀀', '𐀋'), + ('𐀍', '𐀦'), + ('𐀨', '𐀺'), + ('𐀼', '𐀽'), + ('𐀿', '𐁍'), + ('𐁐', '𐁝'), + ('𐂀', '𐃺'), + ('𐅀', '𐅴'), + ('𐊀', '𐊜'), + ('𐊠', '𐋐'), + ('𐌀', '𐌟'), + ('𐌭', '𐍊'), + ('𐍐', '𐍵'), + ('𐎀', '𐎝'), + ('𐎠', '𐏃'), + ('𐏈', '𐏏'), + ('𐏑', '𐏕'), + ('𐐀', '𐒝'), + ('𐒰', '𐓓'), + ('𐓘', '𐓻'), + ('𐔀', '𐔧'), + ('𐔰', '𐕣'), + ('𐕰', '𐕺'), + ('𐕼', '𐖊'), + ('𐖌', '𐖒'), + ('𐖔', '𐖕'), + ('𐖗', '𐖡'), + ('𐖣', '𐖱'), + ('𐖳', '𐖹'), + ('𐖻', '𐖼'), + ('𐗀', '𐗳'), + ('𐘀', '𐜶'), + ('𐝀', '𐝕'), + ('𐝠', '𐝧'), + ('𐞀', '𐞅'), + ('𐞇', '𐞰'), + ('𐞲', '𐞺'), + ('𐠀', '𐠅'), + ('𐠈', '𐠈'), + ('𐠊', '𐠵'), + ('𐠷', '𐠸'), + ('𐠼', '𐠼'), + ('𐠿', '𐡕'), + ('𐡠', '𐡶'), + ('𐢀', '𐢞'), + ('𐣠', '𐣲'), + ('𐣴', '𐣵'), + ('𐤀', '𐤕'), + ('𐤠', '𐤹'), + ('𐦀', '𐦷'), + ('𐦾', '𐦿'), + ('𐨀', '𐨀'), + ('𐨐', '𐨓'), + ('𐨕', '𐨗'), + ('𐨙', '𐨵'), + ('𐩠', '𐩼'), + ('𐪀', '𐪜'), + ('𐫀', '𐫇'), + ('𐫉', '𐫤'), + ('𐬀', '𐬵'), + ('𐭀', '𐭕'), + ('𐭠', '𐭲'), + ('𐮀', '𐮑'), + ('𐰀', '𐱈'), + ('𐲀', '𐲲'), + ('𐳀', '𐳲'), + ('𐴀', '𐴣'), + ('𐵊', '𐵥'), + ('𐵯', '𐶅'), + ('𐺀', '𐺩'), + ('𐺰', '𐺱'), + ('𐻂', '𐻄'), + ('𐼀', '𐼜'), + ('𐼧', '𐼧'), + ('𐼰', '𐽅'), + ('𐽰', '𐾁'), + ('𐾰', '𐿄'), + ('𐿠', '𐿶'), + ('𑀃', '𑀷'), + ('𑁱', '𑁲'), + ('𑁵', '𑁵'), + ('𑂃', '𑂯'), + ('𑃐', '𑃨'), + ('𑄃', '𑄦'), + ('𑅄', '𑅄'), + ('𑅇', '𑅇'), + ('𑅐', '𑅲'), + ('𑅶', '𑅶'), + ('𑆃', '𑆲'), + ('𑇁', '𑇄'), + ('𑇚', '𑇚'), + ('𑇜', '𑇜'), + ('𑈀', '𑈑'), + ('𑈓', '𑈫'), + ('𑈿', '𑉀'), + ('𑊀', '𑊆'), + ('𑊈', '𑊈'), + ('𑊊', '𑊍'), + ('𑊏', '𑊝'), + ('𑊟', '𑊨'), + ('𑊰', '𑋞'), + ('𑌅', '𑌌'), + ('𑌏', '𑌐'), + ('𑌓', '𑌨'), + ('𑌪', '𑌰'), + ('𑌲', '𑌳'), + ('𑌵', '𑌹'), + ('𑌽', '𑌽'), + ('𑍐', '𑍐'), + ('𑍝', '𑍡'), + ('𑎀', '𑎉'), + ('𑎋', '𑎋'), + ('𑎎', '𑎎'), + ('𑎐', '𑎵'), + ('𑎷', '𑎷'), + ('𑏑', '𑏑'), + ('𑏓', '𑏓'), + ('𑐀', '𑐴'), + ('𑑇', '𑑊'), + ('𑑟', '𑑡'), + ('𑒀', '𑒯'), + ('𑓄', '𑓅'), + ('𑓇', '𑓇'), + ('𑖀', '𑖮'), + ('𑗘', '𑗛'), + ('𑘀', '𑘯'), + ('𑙄', '𑙄'), + ('𑚀', '𑚪'), + ('𑚸', '𑚸'), + ('𑠀', '𑠫'), + ('𑢠', '𑣟'), + ('𑣿', '𑤆'), + ('𑤉', '𑤉'), + ('𑤌', '𑤓'), + ('𑤕', '𑤖'), + ('𑤘', '𑤯'), + ('𑤿', '𑤿'), + ('𑥁', '𑥁'), + ('𑦠', '𑦧'), + ('𑦪', '𑧐'), + ('𑧡', '𑧡'), + ('𑧣', '𑧣'), + ('𑨀', '𑨀'), + ('𑨋', '𑨲'), + ('𑨺', '𑨺'), + ('𑩐', '𑩐'), + ('𑩜', '𑪉'), + ('𑪝', '𑪝'), + ('𑪰', '𑫸'), + ('𑯀', '𑯠'), + ('𑰀', '𑰈'), + ('𑰊', '𑰮'), + ('𑱀', '𑱀'), + ('𑱲', '𑲏'), + ('𑴀', '𑴆'), + ('𑴈', '𑴉'), + ('𑴋', '𑴰'), + ('𑵆', '𑵆'), + ('𑵠', '𑵥'), + ('𑵧', '𑵨'), + ('𑵪', '𑶉'), + ('𑶘', '𑶘'), + ('𑻠', '𑻲'), + ('𑼂', '𑼂'), + ('𑼄', '𑼐'), + ('𑼒', '𑼳'), + ('𑾰', '𑾰'), + ('𒀀', '𒎙'), + ('𒐀', '𒑮'), + ('𒒀', '𒕃'), + ('𒾐', '𒿰'), + ('𓀀', '𓐯'), + ('𓑁', '𓑆'), + ('𓑠', '𔏺'), + ('𔐀', '𔙆'), + ('𖄀', '𖄝'), + ('𖠀', '𖨸'), + ('𖩀', '𖩞'), + ('𖩰', '𖪾'), + ('𖫐', '𖫭'), + ('𖬀', '𖬯'), + ('𖭀', '𖭃'), + ('𖭣', '𖭷'), + ('𖭽', '𖮏'), + ('𖵀', '𖵬'), + ('𖹀', '𖹿'), + ('𖼀', '𖽊'), + ('𖽐', '𖽐'), + ('𖾓', '𖾟'), + ('𖿠', '𖿡'), + ('𖿣', '𖿣'), + ('𛰀', '𛱪'), + ('𛱰', '𛱼'), + ('𛲀', '𛲈'), + ('𛲐', '𛲙'), + ('𝐀', '𝑔'), + ('𝑖', '𝒜'), + ('𝒞', '𝒟'), + ('𝒢', '𝒢'), + ('𝒥', '𝒦'), + ('𝒩', '𝒬'), + ('𝒮', '𝒹'), + ('𝒻', '𝒻'), + ('𝒽', '𝓃'), + ('𝓅', '𝔅'), + ('𝔇', '𝔊'), + ('𝔍', '𝔔'), + ('𝔖', '𝔜'), + ('𝔞', '𝔹'), + ('𝔻', '𝔾'), + ('𝕀', '𝕄'), + ('𝕆', '𝕆'), + ('𝕊', '𝕐'), + ('𝕒', '𝚥'), + ('𝚨', '𝛀'), + ('𝛂', '𝛚'), + ('𝛜', '𝛺'), + ('𝛼', '𝜔'), + ('𝜖', '𝜴'), + ('𝜶', '𝝎'), + ('𝝐', '𝝮'), + ('𝝰', '𝞈'), + ('𝞊', '𝞨'), + ('𝞪', '𝟂'), + ('𝟄', '𝟋'), + ('𝼀', '𝼞'), + ('𝼥', '𝼪'), + ('𞀰', '𞁭'), + ('𞄀', '𞄬'), + ('𞄷', '𞄽'), + ('𞅎', '𞅎'), + ('𞊐', '𞊭'), + ('𞋀', '𞋫'), + ('𞓐', '𞓫'), + ('𞗐', '𞗭'), + ('𞗰', '𞗰'), + ('𞟠', '𞟦'), + ('𞟨', '𞟫'), + ('𞟭', '𞟮'), + ('𞟰', '𞟾'), + ('𞠀', '𞣄'), + ('𞤀', '𞥃'), + ('𞥋', '𞥋'), + ('𞸀', '𞸃'), + ('𞸅', '𞸟'), + ('𞸡', '𞸢'), + ('𞸤', '𞸤'), + ('𞸧', '𞸧'), + ('𞸩', '𞸲'), + ('𞸴', '𞸷'), + ('𞸹', '𞸹'), + ('𞸻', '𞸻'), + ('𞹂', '𞹂'), + ('𞹇', '𞹇'), + ('𞹉', '𞹉'), + ('𞹋', '𞹋'), + ('𞹍', '𞹏'), + ('𞹑', '𞹒'), + ('𞹔', '𞹔'), + ('𞹗', '𞹗'), + ('𞹙', '𞹙'), + ('𞹛', '𞹛'), + ('𞹝', '𞹝'), + ('𞹟', '𞹟'), + ('𞹡', '𞹢'), + ('𞹤', '𞹤'), + ('𞹧', '𞹪'), + ('𞹬', '𞹲'), + ('𞹴', '𞹷'), + ('𞹹', '𞹼'), + ('𞹾', '𞹾'), + ('𞺀', '𞺉'), + ('𞺋', '𞺛'), + ('𞺡', '𞺣'), + ('𞺥', '𞺩'), + ('𞺫', '𞺻'), + ('🄰', '🅉'), + ('🅐', '🅩'), + ('🅰', '🆉'), +]; + +pub const CR: &'static [(char, char)] = &[('\r', '\r')]; + +pub const DOUBLE_QUOTE: &'static [(char, char)] = &[('"', '"')]; + +pub const EXTEND: &'static [(char, char)] = &[ + ('\u{300}', '\u{36f}'), + ('\u{483}', '\u{489}'), + ('\u{591}', '\u{5bd}'), + ('\u{5bf}', '\u{5bf}'), + ('\u{5c1}', '\u{5c2}'), + ('\u{5c4}', '\u{5c5}'), + ('\u{5c7}', '\u{5c7}'), + ('\u{610}', '\u{61a}'), + ('\u{64b}', '\u{65f}'), + ('\u{670}', '\u{670}'), + ('\u{6d6}', '\u{6dc}'), + ('\u{6df}', '\u{6e4}'), + ('\u{6e7}', '\u{6e8}'), + ('\u{6ea}', '\u{6ed}'), + ('\u{711}', '\u{711}'), + ('\u{730}', '\u{74a}'), + ('\u{7a6}', '\u{7b0}'), + ('\u{7eb}', '\u{7f3}'), + ('\u{7fd}', '\u{7fd}'), + ('\u{816}', '\u{819}'), + ('\u{81b}', '\u{823}'), + ('\u{825}', '\u{827}'), + ('\u{829}', '\u{82d}'), + ('\u{859}', '\u{85b}'), + ('\u{897}', '\u{89f}'), + ('\u{8ca}', '\u{8e1}'), + ('\u{8e3}', 'ः'), + ('\u{93a}', '\u{93c}'), + ('ा', 'ॏ'), + ('\u{951}', '\u{957}'), + ('\u{962}', '\u{963}'), + ('\u{981}', 'ঃ'), + ('\u{9bc}', '\u{9bc}'), + ('\u{9be}', '\u{9c4}'), + ('ে', 'ৈ'), + ('ো', '\u{9cd}'), + ('\u{9d7}', '\u{9d7}'), + ('\u{9e2}', '\u{9e3}'), + ('\u{9fe}', '\u{9fe}'), + ('\u{a01}', 'ਃ'), + ('\u{a3c}', '\u{a3c}'), + ('ਾ', '\u{a42}'), + ('\u{a47}', '\u{a48}'), + ('\u{a4b}', '\u{a4d}'), + ('\u{a51}', '\u{a51}'), + ('\u{a70}', '\u{a71}'), + ('\u{a75}', '\u{a75}'), + ('\u{a81}', 'ઃ'), + ('\u{abc}', '\u{abc}'), + ('ા', '\u{ac5}'), + ('\u{ac7}', 'ૉ'), + ('ો', '\u{acd}'), + ('\u{ae2}', '\u{ae3}'), + ('\u{afa}', '\u{aff}'), + ('\u{b01}', 'ଃ'), + ('\u{b3c}', '\u{b3c}'), + ('\u{b3e}', '\u{b44}'), + ('େ', 'ୈ'), + ('ୋ', '\u{b4d}'), + ('\u{b55}', '\u{b57}'), + ('\u{b62}', '\u{b63}'), + ('\u{b82}', '\u{b82}'), + ('\u{bbe}', 'ூ'), + ('ெ', 'ை'), + ('ொ', '\u{bcd}'), + ('\u{bd7}', '\u{bd7}'), + ('\u{c00}', '\u{c04}'), + ('\u{c3c}', '\u{c3c}'), + ('\u{c3e}', 'ౄ'), + ('\u{c46}', '\u{c48}'), + ('\u{c4a}', '\u{c4d}'), + ('\u{c55}', '\u{c56}'), + ('\u{c62}', '\u{c63}'), + ('\u{c81}', 'ಃ'), + ('\u{cbc}', '\u{cbc}'), + ('ಾ', 'ೄ'), + ('\u{cc6}', '\u{cc8}'), + ('\u{cca}', '\u{ccd}'), + ('\u{cd5}', '\u{cd6}'), + ('\u{ce2}', '\u{ce3}'), + ('ೳ', 'ೳ'), + ('\u{d00}', 'ഃ'), + ('\u{d3b}', '\u{d3c}'), + ('\u{d3e}', '\u{d44}'), + ('െ', 'ൈ'), + ('ൊ', '\u{d4d}'), + ('\u{d57}', '\u{d57}'), + ('\u{d62}', '\u{d63}'), + ('\u{d81}', 'ඃ'), + ('\u{dca}', '\u{dca}'), + ('\u{dcf}', '\u{dd4}'), + ('\u{dd6}', '\u{dd6}'), + ('ෘ', '\u{ddf}'), + ('ෲ', 'ෳ'), + ('\u{e31}', '\u{e31}'), + ('\u{e34}', '\u{e3a}'), + ('\u{e47}', '\u{e4e}'), + ('\u{eb1}', '\u{eb1}'), + ('\u{eb4}', '\u{ebc}'), + ('\u{ec8}', '\u{ece}'), + ('\u{f18}', '\u{f19}'), + ('\u{f35}', '\u{f35}'), + ('\u{f37}', '\u{f37}'), + ('\u{f39}', '\u{f39}'), + ('༾', '༿'), + ('\u{f71}', '\u{f84}'), + ('\u{f86}', '\u{f87}'), + ('\u{f8d}', '\u{f97}'), + ('\u{f99}', '\u{fbc}'), + ('\u{fc6}', '\u{fc6}'), + ('ါ', '\u{103e}'), + ('ၖ', '\u{1059}'), + ('\u{105e}', '\u{1060}'), + ('ၢ', 'ၤ'), + ('ၧ', 'ၭ'), + ('\u{1071}', '\u{1074}'), + ('\u{1082}', '\u{108d}'), + ('ႏ', 'ႏ'), + ('ႚ', '\u{109d}'), + ('\u{135d}', '\u{135f}'), + ('\u{1712}', '\u{1715}'), + ('\u{1732}', '\u{1734}'), + ('\u{1752}', '\u{1753}'), + ('\u{1772}', '\u{1773}'), + ('\u{17b4}', '\u{17d3}'), + ('\u{17dd}', '\u{17dd}'), + ('\u{180b}', '\u{180d}'), + ('\u{180f}', '\u{180f}'), + ('\u{1885}', '\u{1886}'), + ('\u{18a9}', '\u{18a9}'), + ('\u{1920}', 'ᤫ'), + ('ᤰ', '\u{193b}'), + ('\u{1a17}', '\u{1a1b}'), + ('ᩕ', '\u{1a5e}'), + ('\u{1a60}', '\u{1a7c}'), + ('\u{1a7f}', '\u{1a7f}'), + ('\u{1ab0}', '\u{1ace}'), + ('\u{1b00}', 'ᬄ'), + ('\u{1b34}', '\u{1b44}'), + ('\u{1b6b}', '\u{1b73}'), + ('\u{1b80}', 'ᮂ'), + ('ᮡ', '\u{1bad}'), + ('\u{1be6}', '\u{1bf3}'), + ('ᰤ', '\u{1c37}'), + ('\u{1cd0}', '\u{1cd2}'), + ('\u{1cd4}', '\u{1ce8}'), + ('\u{1ced}', '\u{1ced}'), + ('\u{1cf4}', '\u{1cf4}'), + ('᳷', '\u{1cf9}'), + ('\u{1dc0}', '\u{1dff}'), + ('\u{200c}', '\u{200c}'), + ('\u{20d0}', '\u{20f0}'), + ('\u{2cef}', '\u{2cf1}'), + ('\u{2d7f}', '\u{2d7f}'), + ('\u{2de0}', '\u{2dff}'), + ('\u{302a}', '\u{302f}'), + ('\u{3099}', '\u{309a}'), + ('\u{a66f}', '\u{a672}'), + ('\u{a674}', '\u{a67d}'), + ('\u{a69e}', '\u{a69f}'), + ('\u{a6f0}', '\u{a6f1}'), + ('\u{a802}', '\u{a802}'), + ('\u{a806}', '\u{a806}'), + ('\u{a80b}', '\u{a80b}'), + ('ꠣ', 'ꠧ'), + ('\u{a82c}', '\u{a82c}'), + ('ꢀ', 'ꢁ'), + ('ꢴ', '\u{a8c5}'), + ('\u{a8e0}', '\u{a8f1}'), + ('\u{a8ff}', '\u{a8ff}'), + ('\u{a926}', '\u{a92d}'), + ('\u{a947}', '\u{a953}'), + ('\u{a980}', 'ꦃ'), + ('\u{a9b3}', '\u{a9c0}'), + ('\u{a9e5}', '\u{a9e5}'), + ('\u{aa29}', '\u{aa36}'), + ('\u{aa43}', '\u{aa43}'), + ('\u{aa4c}', 'ꩍ'), + ('ꩻ', 'ꩽ'), + ('\u{aab0}', '\u{aab0}'), + ('\u{aab2}', '\u{aab4}'), + ('\u{aab7}', '\u{aab8}'), + ('\u{aabe}', '\u{aabf}'), + ('\u{aac1}', '\u{aac1}'), + ('ꫫ', 'ꫯ'), + ('ꫵ', '\u{aaf6}'), + ('ꯣ', 'ꯪ'), + ('꯬', '\u{abed}'), + ('\u{fb1e}', '\u{fb1e}'), + ('\u{fe00}', '\u{fe0f}'), + ('\u{fe20}', '\u{fe2f}'), + ('\u{ff9e}', '\u{ff9f}'), + ('\u{101fd}', '\u{101fd}'), + ('\u{102e0}', '\u{102e0}'), + ('\u{10376}', '\u{1037a}'), + ('\u{10a01}', '\u{10a03}'), + ('\u{10a05}', '\u{10a06}'), + ('\u{10a0c}', '\u{10a0f}'), + ('\u{10a38}', '\u{10a3a}'), + ('\u{10a3f}', '\u{10a3f}'), + ('\u{10ae5}', '\u{10ae6}'), + ('\u{10d24}', '\u{10d27}'), + ('\u{10d69}', '\u{10d6d}'), + ('\u{10eab}', '\u{10eac}'), + ('\u{10efc}', '\u{10eff}'), + ('\u{10f46}', '\u{10f50}'), + ('\u{10f82}', '\u{10f85}'), + ('𑀀', '𑀂'), + ('\u{11038}', '\u{11046}'), + ('\u{11070}', '\u{11070}'), + ('\u{11073}', '\u{11074}'), + ('\u{1107f}', '𑂂'), + ('𑂰', '\u{110ba}'), + ('\u{110c2}', '\u{110c2}'), + ('\u{11100}', '\u{11102}'), + ('\u{11127}', '\u{11134}'), + ('𑅅', '𑅆'), + ('\u{11173}', '\u{11173}'), + ('\u{11180}', '𑆂'), + ('𑆳', '\u{111c0}'), + ('\u{111c9}', '\u{111cc}'), + ('𑇎', '\u{111cf}'), + ('𑈬', '\u{11237}'), + ('\u{1123e}', '\u{1123e}'), + ('\u{11241}', '\u{11241}'), + ('\u{112df}', '\u{112ea}'), + ('\u{11300}', '𑌃'), + ('\u{1133b}', '\u{1133c}'), + ('\u{1133e}', '𑍄'), + ('𑍇', '𑍈'), + ('𑍋', '\u{1134d}'), + ('\u{11357}', '\u{11357}'), + ('𑍢', '𑍣'), + ('\u{11366}', '\u{1136c}'), + ('\u{11370}', '\u{11374}'), + ('\u{113b8}', '\u{113c0}'), + ('\u{113c2}', '\u{113c2}'), + ('\u{113c5}', '\u{113c5}'), + ('\u{113c7}', '𑏊'), + ('𑏌', '\u{113d0}'), + ('\u{113d2}', '\u{113d2}'), + ('\u{113e1}', '\u{113e2}'), + ('𑐵', '\u{11446}'), + ('\u{1145e}', '\u{1145e}'), + ('\u{114b0}', '\u{114c3}'), + ('\u{115af}', '\u{115b5}'), + ('𑖸', '\u{115c0}'), + ('\u{115dc}', '\u{115dd}'), + ('𑘰', '\u{11640}'), + ('\u{116ab}', '\u{116b7}'), + ('\u{1171d}', '\u{1172b}'), + ('𑠬', '\u{1183a}'), + ('\u{11930}', '𑤵'), + ('𑤷', '𑤸'), + ('\u{1193b}', '\u{1193e}'), + ('𑥀', '𑥀'), + ('𑥂', '\u{11943}'), + ('𑧑', '\u{119d7}'), + ('\u{119da}', '\u{119e0}'), + ('𑧤', '𑧤'), + ('\u{11a01}', '\u{11a0a}'), + ('\u{11a33}', '𑨹'), + ('\u{11a3b}', '\u{11a3e}'), + ('\u{11a47}', '\u{11a47}'), + ('\u{11a51}', '\u{11a5b}'), + ('\u{11a8a}', '\u{11a99}'), + ('𑰯', '\u{11c36}'), + ('\u{11c38}', '\u{11c3f}'), + ('\u{11c92}', '\u{11ca7}'), + ('𑲩', '\u{11cb6}'), + ('\u{11d31}', '\u{11d36}'), + ('\u{11d3a}', '\u{11d3a}'), + ('\u{11d3c}', '\u{11d3d}'), + ('\u{11d3f}', '\u{11d45}'), + ('\u{11d47}', '\u{11d47}'), + ('𑶊', '𑶎'), + ('\u{11d90}', '\u{11d91}'), + ('𑶓', '\u{11d97}'), + ('\u{11ef3}', '𑻶'), + ('\u{11f00}', '\u{11f01}'), + ('𑼃', '𑼃'), + ('𑼴', '\u{11f3a}'), + ('𑼾', '\u{11f42}'), + ('\u{11f5a}', '\u{11f5a}'), + ('\u{13440}', '\u{13440}'), + ('\u{13447}', '\u{13455}'), + ('\u{1611e}', '\u{1612f}'), + ('\u{16af0}', '\u{16af4}'), + ('\u{16b30}', '\u{16b36}'), + ('\u{16f4f}', '\u{16f4f}'), + ('𖽑', '𖾇'), + ('\u{16f8f}', '\u{16f92}'), + ('\u{16fe4}', '\u{16fe4}'), + ('\u{16ff0}', '\u{16ff1}'), + ('\u{1bc9d}', '\u{1bc9e}'), + ('\u{1cf00}', '\u{1cf2d}'), + ('\u{1cf30}', '\u{1cf46}'), + ('\u{1d165}', '\u{1d169}'), + ('\u{1d16d}', '\u{1d172}'), + ('\u{1d17b}', '\u{1d182}'), + ('\u{1d185}', '\u{1d18b}'), + ('\u{1d1aa}', '\u{1d1ad}'), + ('\u{1d242}', '\u{1d244}'), + ('\u{1da00}', '\u{1da36}'), + ('\u{1da3b}', '\u{1da6c}'), + ('\u{1da75}', '\u{1da75}'), + ('\u{1da84}', '\u{1da84}'), + ('\u{1da9b}', '\u{1da9f}'), + ('\u{1daa1}', '\u{1daaf}'), + ('\u{1e000}', '\u{1e006}'), + ('\u{1e008}', '\u{1e018}'), + ('\u{1e01b}', '\u{1e021}'), + ('\u{1e023}', '\u{1e024}'), + ('\u{1e026}', '\u{1e02a}'), + ('\u{1e08f}', '\u{1e08f}'), + ('\u{1e130}', '\u{1e136}'), + ('\u{1e2ae}', '\u{1e2ae}'), + ('\u{1e2ec}', '\u{1e2ef}'), + ('\u{1e4ec}', '\u{1e4ef}'), + ('\u{1e5ee}', '\u{1e5ef}'), + ('\u{1e8d0}', '\u{1e8d6}'), + ('\u{1e944}', '\u{1e94a}'), + ('🏻', '🏿'), + ('\u{e0020}', '\u{e007f}'), + ('\u{e0100}', '\u{e01ef}'), +]; + +pub const EXTENDNUMLET: &'static [(char, char)] = &[ + ('_', '_'), + ('\u{202f}', '\u{202f}'), + ('‿', '⁀'), + ('⁔', '⁔'), + ('︳', '︴'), + ('﹍', '﹏'), + ('＿', '＿'), +]; + +pub const FORMAT: &'static [(char, char)] = &[ + ('\u{ad}', '\u{ad}'), + ('\u{61c}', '\u{61c}'), + ('\u{180e}', '\u{180e}'), + ('\u{200e}', '\u{200f}'), + ('\u{202a}', '\u{202e}'), + ('\u{2060}', '\u{2064}'), + ('\u{2066}', '\u{206f}'), + ('\u{feff}', '\u{feff}'), + ('\u{fff9}', '\u{fffb}'), + ('\u{13430}', '\u{1343f}'), + ('\u{1bca0}', '\u{1bca3}'), + ('\u{1d173}', '\u{1d17a}'), + ('\u{e0001}', '\u{e0001}'), +]; + +pub const HEBREW_LETTER: &'static [(char, char)] = &[ + ('א', 'ת'), + ('ׯ', 'ײ'), + ('יִ', 'יִ'), + ('ײַ', 'ﬨ'), + ('שׁ', 'זּ'), + ('טּ', 'לּ'), + ('מּ', 'מּ'), + ('נּ', 'סּ'), + ('ףּ', 'פּ'), + ('צּ', 'ﭏ'), +]; + +pub const KATAKANA: &'static [(char, char)] = &[ + ('〱', '〵'), + ('゛', '゜'), + ('゠', 'ヺ'), + ('ー', 'ヿ'), + ('ㇰ', 'ㇿ'), + ('㋐', '㋾'), + ('㌀', '㍗'), + ('ｦ', 'ﾝ'), + ('𚿰', '𚿳'), + ('𚿵', '𚿻'), + ('𚿽', '𚿾'), + ('𛀀', '𛀀'), + ('𛄠', '𛄢'), + ('𛅕', '𛅕'), + ('𛅤', '𛅧'), +]; + +pub const LF: &'static [(char, char)] = &[('\n', '\n')]; + +pub const MIDLETTER: &'static [(char, char)] = &[ + (':', ':'), + ('·', '·'), + ('·', '·'), + ('՟', '՟'), + ('״', '״'), + ('‧', '‧'), + ('︓', '︓'), + ('﹕', '﹕'), + ('：', '：'), +]; + +pub const MIDNUM: &'static [(char, char)] = &[ + (',', ','), + (';', ';'), + (';', ';'), + ('։', '։'), + ('،', '؍'), + ('٬', '٬'), + ('߸', '߸'), + ('⁄', '⁄'), + ('﹐', '﹐'), + ('﹔', '﹔'), + ('，', '，'), + ('；', '；'), +]; + +pub const MIDNUMLET: &'static [(char, char)] = &[ + ('.', '.'), + ('‘', '’'), + ('․', '․'), + ('﹒', '﹒'), + ('＇', '＇'), + ('．', '．'), +]; + +pub const NEWLINE: &'static [(char, char)] = + &[('\u{b}', '\u{c}'), ('\u{85}', '\u{85}'), ('\u{2028}', '\u{2029}')]; + +pub const NUMERIC: &'static [(char, char)] = &[ + ('0', '9'), + ('\u{600}', '\u{605}'), + ('٠', '٩'), + ('٫', '٫'), + ('\u{6dd}', '\u{6dd}'), + ('۰', '۹'), + ('߀', '߉'), + ('\u{890}', '\u{891}'), + ('\u{8e2}', '\u{8e2}'), + ('०', '९'), + ('০', '৯'), + ('੦', '੯'), + ('૦', '૯'), + ('୦', '୯'), + ('௦', '௯'), + ('౦', '౯'), + ('೦', '೯'), + ('൦', '൯'), + ('෦', '෯'), + ('๐', '๙'), + ('໐', '໙'), + ('༠', '༩'), + ('၀', '၉'), + ('႐', '႙'), + ('០', '៩'), + ('᠐', '᠙'), + ('᥆', '᥏'), + ('᧐', '᧚'), + ('᪀', '᪉'), + ('᪐', '᪙'), + ('᭐', '᭙'), + ('᮰', '᮹'), + ('᱀', '᱉'), + ('᱐', '᱙'), + ('꘠', '꘩'), + ('꣐', '꣙'), + ('꤀', '꤉'), + ('꧐', '꧙'), + ('꧰', '꧹'), + ('꩐', '꩙'), + ('꯰', '꯹'), + ('０', '９'), + ('𐒠', '𐒩'), + ('𐴰', '𐴹'), + ('𐵀', '𐵉'), + ('𑁦', '𑁯'), + ('\u{110bd}', '\u{110bd}'), + ('\u{110cd}', '\u{110cd}'), + ('𑃰', '𑃹'), + ('𑄶', '𑄿'), + ('𑇐', '𑇙'), + ('𑋰', '𑋹'), + ('𑑐', '𑑙'), + ('𑓐', '𑓙'), + ('𑙐', '𑙙'), + ('𑛀', '𑛉'), + ('𑛐', '𑛣'), + ('𑜰', '𑜹'), + ('𑣠', '𑣩'), + ('𑥐', '𑥙'), + ('𑯰', '𑯹'), + ('𑱐', '𑱙'), + ('𑵐', '𑵙'), + ('𑶠', '𑶩'), + ('𑽐', '𑽙'), + ('𖄰', '𖄹'), + ('𖩠', '𖩩'), + ('𖫀', '𖫉'), + ('𖭐', '𖭙'), + ('𖵰', '𖵹'), + ('𜳰', '𜳹'), + ('𝟎', '𝟿'), + ('𞅀', '𞅉'), + ('𞋰', '𞋹'), + ('𞓰', '𞓹'), + ('𞗱', '𞗺'), + ('𞥐', '𞥙'), + ('🯰', '🯹'), +]; + +pub const REGIONAL_INDICATOR: &'static [(char, char)] = &[('🇦', '🇿')]; + +pub const SINGLE_QUOTE: &'static [(char, char)] = &[('\'', '\'')]; + +pub const WSEGSPACE: &'static [(char, char)] = &[ + (' ', ' '), + ('\u{1680}', '\u{1680}'), + ('\u{2000}', '\u{2006}'), + ('\u{2008}', '\u{200a}'), + ('\u{205f}', '\u{205f}'), + ('\u{3000}', '\u{3000}'), +]; + +pub const ZWJ: &'static [(char, char)] = &[('\u{200d}', '\u{200d}')]; diff --git a/third_party/crates/regex-syntax-0.8.11/src/utf8.rs b/third_party/crates/regex-syntax-0.8.11/src/utf8.rs new file mode 100644 index 0000000..537035e --- /dev/null +++ b/third_party/crates/regex-syntax-0.8.11/src/utf8.rs @@ -0,0 +1,592 @@ +/*! +Converts ranges of Unicode scalar values to equivalent ranges of UTF-8 bytes. + +This is sub-module is useful for constructing byte based automatons that need +to embed UTF-8 decoding. The most common use of this module is in conjunction +with the [`hir::ClassUnicodeRange`](crate::hir::ClassUnicodeRange) type. + +See the documentation on the `Utf8Sequences` iterator for more details and +an example. + +# Wait, what is this? + +This is simplest to explain with an example. Let's say you wanted to test +whether a particular byte sequence was a Cyrillic character. One possible +scalar value range is `[0400-04FF]`. The set of allowed bytes for this +range can be expressed as a sequence of byte ranges: + +```text +[D0-D3][80-BF] +``` + +This is simple enough: simply encode the boundaries, `0400` encodes to +`D0 80` and `04FF` encodes to `D3 BF`, and create ranges from each +corresponding pair of bytes: `D0` to `D3` and `80` to `BF`. + +However, what if you wanted to add the Cyrillic Supplementary characters to +your range? Your range might then become `[0400-052F]`. The same procedure +as above doesn't quite work because `052F` encodes to `D4 AF`. The byte ranges +you'd get from the previous transformation would be `[D0-D4][80-AF]`. However, +this isn't quite correct because this range doesn't capture many characters, +for example, `04FF` (because its last byte, `BF` isn't in the range `80-AF`). + +Instead, you need multiple sequences of byte ranges: + +```text +[D0-D3][80-BF] # matches codepoints 0400-04FF +[D4][80-AF] # matches codepoints 0500-052F +``` + +This gets even more complicated if you want bigger ranges, particularly if +they naively contain surrogate codepoints. For example, the sequence of byte +ranges for the basic multilingual plane (`[0000-FFFF]`) look like this: + +```text +[0-7F] +[C2-DF][80-BF] +[E0][A0-BF][80-BF] +[E1-EC][80-BF][80-BF] +[ED][80-9F][80-BF] +[EE-EF][80-BF][80-BF] +``` + +Note that the byte ranges above will *not* match any erroneous encoding of +UTF-8, including encodings of surrogate codepoints. + +And, of course, for all of Unicode (`[000000-10FFFF]`): + +```text +[0-7F] +[C2-DF][80-BF] +[E0][A0-BF][80-BF] +[E1-EC][80-BF][80-BF] +[ED][80-9F][80-BF] +[EE-EF][80-BF][80-BF] +[F0][90-BF][80-BF][80-BF] +[F1-F3][80-BF][80-BF][80-BF] +[F4][80-8F][80-BF][80-BF] +``` + +This module automates the process of creating these byte ranges from ranges of +Unicode scalar values. + +# Lineage + +I got the idea and general implementation strategy from Russ Cox in his +[article on regexps](https://web.archive.org/web/20160404141123/https://swtch.com/~rsc/regexp/regexp3.html) and RE2. +Russ Cox got it from Ken Thompson's `grep` (no source, folk lore?). +I also got the idea from +[Lucene](https://github.com/apache/lucene-solr/blob/ae93f4e7ac6a3908046391de35d4f50a0d3c59ca/lucene/core/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java), +which uses it for executing automata on their term index. +*/ + +use core::{char, fmt, iter::FusedIterator, slice}; + +use alloc::{vec, vec::Vec}; + +const MAX_UTF8_BYTES: usize = 4; + +/// Utf8Sequence represents a sequence of byte ranges. +/// +/// To match a Utf8Sequence, a candidate byte sequence must match each +/// successive range. +/// +/// For example, if there are two ranges, `[C2-DF][80-BF]`, then the byte +/// sequence `\xDD\x61` would not match because `0x61 < 0x80`. +#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord)] +pub enum Utf8Sequence { + /// One byte range. + One(Utf8Range), + /// Two successive byte ranges. + Two([Utf8Range; 2]), + /// Three successive byte ranges. + Three([Utf8Range; 3]), + /// Four successive byte ranges. + Four([Utf8Range; 4]), +} + +impl Utf8Sequence { + /// Creates a new UTF-8 sequence from the encoded bytes of a scalar value + /// range. + /// + /// This assumes that `start` and `end` have the same length. + fn from_encoded_range(start: &[u8], end: &[u8]) -> Self { + assert_eq!(start.len(), end.len()); + match start.len() { + 2 => Utf8Sequence::Two([ + Utf8Range::new(start[0], end[0]), + Utf8Range::new(start[1], end[1]), + ]), + 3 => Utf8Sequence::Three([ + Utf8Range::new(start[0], end[0]), + Utf8Range::new(start[1], end[1]), + Utf8Range::new(start[2], end[2]), + ]), + 4 => Utf8Sequence::Four([ + Utf8Range::new(start[0], end[0]), + Utf8Range::new(start[1], end[1]), + Utf8Range::new(start[2], end[2]), + Utf8Range::new(start[3], end[3]), + ]), + n => unreachable!("invalid encoded length: {n}"), + } + } + + /// Returns the underlying sequence of byte ranges as a slice. + pub fn as_slice(&self) -> &[Utf8Range] { + use self::Utf8Sequence::*; + match *self { + One(ref r) => slice::from_ref(r), + Two(ref r) => &r[..], + Three(ref r) => &r[..], + Four(ref r) => &r[..], + } + } + + /// Returns the number of byte ranges in this sequence. + /// + /// The length is guaranteed to be in the closed interval `[1, 4]`. + pub fn len(&self) -> usize { + self.as_slice().len() + } + + /// Reverses the ranges in this sequence. + /// + /// For example, if this corresponds to the following sequence: + /// + /// ```text + /// [D0-D3][80-BF] + /// ``` + /// + /// Then after reversal, it will be + /// + /// ```text + /// [80-BF][D0-D3] + /// ``` + /// + /// This is useful when one is constructing a UTF-8 automaton to match + /// character classes in reverse. + pub fn reverse(&mut self) { + match *self { + Utf8Sequence::One(_) => {} + Utf8Sequence::Two(ref mut x) => x.reverse(), + Utf8Sequence::Three(ref mut x) => x.reverse(), + Utf8Sequence::Four(ref mut x) => x.reverse(), + } + } + + /// Returns true if and only if a prefix of `bytes` matches this sequence + /// of byte ranges. + pub fn matches(&self, bytes: &[u8]) -> bool { + if bytes.len() < self.len() { + return false; + } + for (&b, r) in bytes.iter().zip(self) { + if !r.matches(b) { + return false; + } + } + true + } +} + +impl<'a> IntoIterator for &'a Utf8Sequence { + type IntoIter = slice::Iter<'a, Utf8Range>; + type Item = &'a Utf8Range; + + fn into_iter(self) -> Self::IntoIter { + self.as_slice().iter() + } +} + +impl fmt::Debug for Utf8Sequence { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use self::Utf8Sequence::*; + match *self { + One(ref r) => write!(f, "{r:?}"), + Two(ref r) => write!(f, "{:?}{:?}", r[0], r[1]), + Three(ref r) => write!(f, "{:?}{:?}{:?}", r[0], r[1], r[2]), + Four(ref r) => { + write!(f, "{:?}{:?}{:?}{:?}", r[0], r[1], r[2], r[3]) + } + } + } +} + +/// A single inclusive range of UTF-8 bytes. +#[derive(Clone, Copy, Eq, PartialEq, PartialOrd, Ord)] +pub struct Utf8Range { + /// Start of byte range (inclusive). + pub start: u8, + /// End of byte range (inclusive). + pub end: u8, +} + +impl Utf8Range { + fn new(start: u8, end: u8) -> Self { + Utf8Range { start, end } + } + + /// Returns true if and only if the given byte is in this range. + pub fn matches(&self, b: u8) -> bool { + self.start <= b && b <= self.end + } +} + +impl fmt::Debug for Utf8Range { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.start == self.end { + write!(f, "[{:X}]", self.start) + } else { + write!(f, "[{:X}-{:X}]", self.start, self.end) + } + } +} + +/// An iterator over ranges of matching UTF-8 byte sequences. +/// +/// The iteration represents an alternation of comprehensive byte sequences +/// that match precisely the set of UTF-8 encoded scalar values. +/// +/// A byte sequence corresponds to one of the scalar values in the range given +/// if and only if it completely matches exactly one of the sequences of byte +/// ranges produced by this iterator. +/// +/// Each sequence of byte ranges matches a unique set of bytes. That is, no two +/// sequences will match the same bytes. +/// +/// # Example +/// +/// This shows how to match an arbitrary byte sequence against a range of +/// scalar values. +/// +/// ```rust +/// use regex_syntax::utf8::{Utf8Sequences, Utf8Sequence}; +/// +/// fn matches(seqs: &[Utf8Sequence], bytes: &[u8]) -> bool { +/// for range in seqs { +/// if range.matches(bytes) { +/// return true; +/// } +/// } +/// false +/// } +/// +/// // Test the basic multilingual plane. +/// let seqs: Vec<_> = Utf8Sequences::new('\u{0}', '\u{FFFF}').collect(); +/// +/// // UTF-8 encoding of 'a'. +/// assert!(matches(&seqs, &[0x61])); +/// // UTF-8 encoding of '☃' (`\u{2603}`). +/// assert!(matches(&seqs, &[0xE2, 0x98, 0x83])); +/// // UTF-8 encoding of `\u{10348}` (outside the BMP). +/// assert!(!matches(&seqs, &[0xF0, 0x90, 0x8D, 0x88])); +/// // Tries to match against a UTF-8 encoding of a surrogate codepoint, +/// // which is invalid UTF-8, and therefore fails, despite the fact that +/// // the corresponding codepoint (0xD800) falls in the range given. +/// assert!(!matches(&seqs, &[0xED, 0xA0, 0x80])); +/// // And fails against plain old invalid UTF-8. +/// assert!(!matches(&seqs, &[0xFF, 0xFF])); +/// ``` +/// +/// If this example seems circuitous, that's because it is! It's meant to be +/// illustrative. In practice, you could just try to decode your byte sequence +/// and compare it with the scalar value range directly. However, this is not +/// always possible (for example, in a byte based automaton). +#[derive(Debug)] +pub struct Utf8Sequences { + range_stack: Vec<ScalarRange>, +} + +impl Utf8Sequences { + /// Create a new iterator over UTF-8 byte ranges for the scalar value range + /// given. + pub fn new(start: char, end: char) -> Self { + let range = + ScalarRange { start: u32::from(start), end: u32::from(end) }; + Utf8Sequences { range_stack: vec![range] } + } + + /// reset resets the scalar value range. + /// Any existing state is cleared, but resources may be reused. + /// + /// N.B. Benchmarks say that this method is dubious. + #[doc(hidden)] + pub fn reset(&mut self, start: char, end: char) { + self.range_stack.clear(); + self.push(u32::from(start), u32::from(end)); + } + + fn push(&mut self, start: u32, end: u32) { + self.range_stack.push(ScalarRange { start, end }); + } +} + +struct ScalarRange { + start: u32, + end: u32, +} + +impl fmt::Debug for ScalarRange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "ScalarRange({:X}, {:X})", self.start, self.end) + } +} + +impl Iterator for Utf8Sequences { + type Item = Utf8Sequence; + + fn next(&mut self) -> Option<Self::Item> { + 'TOP: while let Some(mut r) = self.range_stack.pop() { + 'INNER: loop { + if let Some((r1, r2)) = r.split() { + self.push(r2.start, r2.end); + r.start = r1.start; + r.end = r1.end; + continue 'INNER; + } + if !r.is_valid() { + continue 'TOP; + } + for i in 1..MAX_UTF8_BYTES { + let max = max_scalar_value(i); + if r.start <= max && max < r.end { + self.push(max + 1, r.end); + r.end = max; + continue 'INNER; + } + } + if let Some(ascii_range) = r.as_ascii() { + return Some(Utf8Sequence::One(ascii_range)); + } + for i in 1..MAX_UTF8_BYTES { + let m = (1 << (6 * i)) - 1; + if (r.start & !m) != (r.end & !m) { + if (r.start & m) != 0 { + self.push((r.start | m) + 1, r.end); + r.end = r.start | m; + continue 'INNER; + } + if (r.end & m) != m { + self.push(r.end & !m, r.end); + r.end = (r.end & !m) - 1; + continue 'INNER; + } + } + } + let mut start = [0; MAX_UTF8_BYTES]; + let mut end = [0; MAX_UTF8_BYTES]; + let n = r.encode(&mut start, &mut end); + return Some(Utf8Sequence::from_encoded_range( + &start[0..n], + &end[0..n], + )); + } + } + None + } +} + +impl FusedIterator for Utf8Sequences {} + +impl ScalarRange { + /// split splits this range if it overlaps with a surrogate codepoint. + /// + /// Either or both ranges may be invalid. + fn split(&self) -> Option<(ScalarRange, ScalarRange)> { + if self.start < 0xE000 && self.end > 0xD7FF { + Some(( + ScalarRange { start: self.start, end: 0xD7FF }, + ScalarRange { start: 0xE000, end: self.end }, + )) + } else { + None + } + } + + /// is_valid returns true if and only if start <= end. + fn is_valid(&self) -> bool { + self.start <= self.end + } + + /// as_ascii returns this range as a Utf8Range if and only if all scalar + /// values in this range can be encoded as a single byte. + fn as_ascii(&self) -> Option<Utf8Range> { + if self.is_ascii() { + let start = u8::try_from(self.start).unwrap(); + let end = u8::try_from(self.end).unwrap(); + Some(Utf8Range::new(start, end)) + } else { + None + } + } + + /// is_ascii returns true if the range is ASCII only (i.e., takes a single + /// byte to encode any scalar value). + fn is_ascii(&self) -> bool { + self.is_valid() && self.end <= 0x7f + } + + /// encode writes the UTF-8 encoding of the start and end of this range + /// to the corresponding destination slices, and returns the number of + /// bytes written. + /// + /// The slices should have room for at least `MAX_UTF8_BYTES`. + fn encode(&self, start: &mut [u8], end: &mut [u8]) -> usize { + let cs = char::from_u32(self.start).unwrap(); + let ce = char::from_u32(self.end).unwrap(); + let ss = cs.encode_utf8(start); + let se = ce.encode_utf8(end); + assert_eq!(ss.len(), se.len()); + ss.len() + } +} + +fn max_scalar_value(nbytes: usize) -> u32 { + match nbytes { + 1 => 0x007F, + 2 => 0x07FF, + 3 => 0xFFFF, + 4 => 0x0010_FFFF, + _ => unreachable!("invalid UTF-8 byte sequence size"), + } +} + +#[cfg(test)] +mod tests { + use core::char; + + use alloc::{vec, vec::Vec}; + + use crate::utf8::{Utf8Range, Utf8Sequences}; + + fn rutf8(s: u8, e: u8) -> Utf8Range { + Utf8Range::new(s, e) + } + + fn never_accepts_surrogate_codepoints(start: char, end: char) { + for cp in 0xD800..0xE000 { + let buf = encode_surrogate(cp); + for r in Utf8Sequences::new(start, end) { + if r.matches(&buf) { + panic!( + "Sequence ({:X}, {:X}) contains range {:?}, \ + which matches surrogate code point {:X} \ + with encoded bytes {:?}", + u32::from(start), + u32::from(end), + r, + cp, + buf, + ); + } + } + } + } + + #[test] + fn codepoints_no_surrogates() { + never_accepts_surrogate_codepoints('\u{0}', '\u{FFFF}'); + never_accepts_surrogate_codepoints('\u{0}', '\u{10FFFF}'); + never_accepts_surrogate_codepoints('\u{0}', '\u{10FFFE}'); + never_accepts_surrogate_codepoints('\u{80}', '\u{10FFFF}'); + never_accepts_surrogate_codepoints('\u{D7FF}', '\u{E000}'); + } + + #[test] + fn single_codepoint_one_sequence() { + // Tests that every range of scalar values that contains a single + // scalar value is recognized by one sequence of byte ranges. + for i in 0x0..=0x0010_FFFF { + let c = match char::from_u32(i) { + None => continue, + Some(c) => c, + }; + let seqs: Vec<_> = Utf8Sequences::new(c, c).collect(); + assert_eq!(seqs.len(), 1); + } + } + + #[test] + fn bmp() { + use crate::utf8::Utf8Sequence::*; + + let seqs = Utf8Sequences::new('\u{0}', '\u{FFFF}').collect::<Vec<_>>(); + assert_eq!( + seqs, + vec![ + One(rutf8(0x0, 0x7F)), + Two([rutf8(0xC2, 0xDF), rutf8(0x80, 0xBF)]), + Three([ + rutf8(0xE0, 0xE0), + rutf8(0xA0, 0xBF), + rutf8(0x80, 0xBF) + ]), + Three([ + rutf8(0xE1, 0xEC), + rutf8(0x80, 0xBF), + rutf8(0x80, 0xBF) + ]), + Three([ + rutf8(0xED, 0xED), + rutf8(0x80, 0x9F), + rutf8(0x80, 0xBF) + ]), + Three([ + rutf8(0xEE, 0xEF), + rutf8(0x80, 0xBF), + rutf8(0x80, 0xBF) + ]), + ] + ); + } + + #[test] + fn reverse() { + use crate::utf8::Utf8Sequence::*; + + let mut s = One(rutf8(0xA, 0xB)); + s.reverse(); + assert_eq!(s.as_slice(), &[rutf8(0xA, 0xB)]); + + let mut s = Two([rutf8(0xA, 0xB), rutf8(0xB, 0xC)]); + s.reverse(); + assert_eq!(s.as_slice(), &[rutf8(0xB, 0xC), rutf8(0xA, 0xB)]); + + let mut s = Three([rutf8(0xA, 0xB), rutf8(0xB, 0xC), rutf8(0xC, 0xD)]); + s.reverse(); + assert_eq!( + s.as_slice(), + &[rutf8(0xC, 0xD), rutf8(0xB, 0xC), rutf8(0xA, 0xB)] + ); + + let mut s = Four([ + rutf8(0xA, 0xB), + rutf8(0xB, 0xC), + rutf8(0xC, 0xD), + rutf8(0xD, 0xE), + ]); + s.reverse(); + assert_eq!( + s.as_slice(), + &[ + rutf8(0xD, 0xE), + rutf8(0xC, 0xD), + rutf8(0xB, 0xC), + rutf8(0xA, 0xB) + ] + ); + } + + fn encode_surrogate(cp: u32) -> [u8; 3] { + const TAG_CONT: u8 = 0b1000_0000; + const TAG_THREE_B: u8 = 0b1110_0000; + + assert!(0xD800 <= cp && cp < 0xE000); + let mut dst = [0; 3]; + dst[0] = u8::try_from(cp >> 12 & 0x0F).unwrap() | TAG_THREE_B; + dst[1] = u8::try_from(cp >> 6 & 0x3F).unwrap() | TAG_CONT; + dst[2] = u8::try_from(cp & 0x3F).unwrap() | TAG_CONT; + dst + } +} diff --git a/third_party/crates/rustversion-1.0.23/.cargo-checksum.json b/third_party/crates/rustversion-1.0.23/.cargo-checksum.json new file mode 100644 index 0000000..11c4c77 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"50e3a4f0cc0e0deff35877f5e4a7cdb4a294b78e154b047bcdd3d08124485e94",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"73ae72c51294eb07f1ab8cd2aff4163687b0f2bb15063d8de6ae436aef6961c0","Cargo.lock":"1112abcf0225a742208d7b0337d470a5b6cef7e887100bda96d6b32105ddec5e","Cargo.toml":"fc211ccd64d3ad59a0715c1a6f8cbfbbd83045f644194394595d16646eee9947","Cargo.toml.orig":"aea407b728196dfc09352d170f2e65204da583a9899c3b6377a6c63daa056ecf","LICENSE-APACHE":"62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"913d04d158843566991845cc9d76e24610bf31f5a99e55115023ccc0b16ff1e7","build/build.rs":"4438b108d5b198ca43ef0a78509da89861020674abd05166addd1c3058afbfa6","build/rustc.rs":"faf67033ec831d968406f5a66c475df337f7221e9d1ab3477fed3a629e1ec5bc","src/attr.rs":"9301cd4aff5a9648c057d5d8de9eb66921f0c3a715c51ada4459576bd49c8b19","src/bound.rs":"13c35302bd5d96ebe54509477e736a8b720e18cb2474e5805480c4517a20e5b0","src/constfn.rs":"613b8f53b21cc06b4f619fce9000993d3e7873b650701ca01cef1e53bed5b40a","src/date.rs":"f79c55ddcd18e124610cd5c9867b1ebaae1feeb6dfc37e2b7f94e3137aa883b6","src/error.rs":"cb37102f03ebbaca313d80f9714fe08dfef92fe956789ee87d93eb6121705f4f","src/expand.rs":"9c93525a8c4a0e620369ed256a1ee40a0b48f984e0fad8db8c0e2998bc8ecb23","src/expr.rs":"8e8ca76f4f5838436d9d7273f499c698bb41f6c15bc07d32ec5c1cb8bd3dd731","src/iter.rs":"8d4b817b9abc4e817105b673e15f29ef9bb8284a010ce01ac2d83387fe136947","src/lib.rs":"77c732493550b7e7a46ae2d25e89c363a851ba25a9e9a89f1aad67bad4afbb04","src/release.rs":"e0755ed1889b4c6f0faaa5adee5f60f27f9926becff875dbb41b6a65ae756591","src/time.rs":"6fcf3a1ed238a97bd7a39b187c88bdcef0246878424e06aa7eb779f841529b34","src/token.rs":"824ce765f692db73afa02d3ebb0281c750748035efc98fa547be29d3072665ce","src/version.rs":"afdb048bba95bbb885945eba5527b6bf0eca0105642bfc304c2f82a8b7d556df","tests/compiletest.rs":"4e381aa8ca3eabb7ac14d1e0c3700b3223e47640547a6988cfa13ad68255f60f","tests/test_const.rs":"cb1e01479bd579b1b77312a9063ab456fe44abad2243e81873c2bf7ef5f205eb","tests/test_eval.rs":"6f0ee3f49c9a0d0c374a4d0e9a9dce753cd9fc2ca7725e000a435dbd5f4a9ce3","tests/test_parse.rs":"f450e0a7f2391561484abb1b851124b5747755b76f945e121370adc710f2099d","tests/ui/bad-bound.rs":"25bde278fcaabf62868417148a5e5f2006bf589d7ebd7bf6004fb8d78e47594f","tests/ui/bad-bound.stderr":"23b4ca0aa1a7df75b5fb34e553a31faa6a3ccc029ecd7d68b056897c2c9cff9f","tests/ui/bad-date.rs":"6e23714dae8b6346fefe50dacd4abba3265248bbadfdd60c739138aa8a0037ba","tests/ui/bad-date.stderr":"f4aec72ca91ba743cadcdaf5a874c701949add5efce490aaf2bf1248ef578768","tests/ui/bad-not.rs":"f003df8bd245e9dd8edc3a6d94078ee5162fac7a98db881271f0f5b6db98d45d","tests/ui/bad-not.stderr":"1aebb3121d067d432a32dd95ff380ceac15a0531c34f90cee950582558841c56","tests/ui/bad-version.rs":"f4ea2cd038e6c63deb9c2e3ceffce93dbf179d9ce18c16d88f3b6cd7138a8c8e","tests/ui/bad-version.stderr":"e9421586d59cb9135123bd97710655014f95cc5f55b0f9611e1d2e8ce9e29a5d","tests/ui/const-not-fn.rs":"10bbe38f0d89391fff0698756e4cfd4e72a41090360393a0c951b67df14d1c35","tests/ui/const-not-fn.stderr":"1180662fd3b8c4426d46923918a2e58dd9b6259f1a364469ae13d5fc3d69ce6c"},"package":"cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"} \ No newline at end of file diff --git a/third_party/crates/rustversion-1.0.23/.cargo_vcs_info.json b/third_party/crates/rustversion-1.0.23/.cargo_vcs_info.json new file mode 100644 index 0000000..53d72fc --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "3a7c76605450b9a7299c6502a421909de9126a59" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/third_party/crates/rustversion-1.0.23/.github/FUNDING.yml b/third_party/crates/rustversion-1.0.23/.github/FUNDING.yml new file mode 100644 index 0000000..7507077 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/.github/FUNDING.yml @@ -0,0 +1 @@ +github: dtolnay diff --git a/third_party/crates/rustversion-1.0.23/.github/workflows/ci.yml b/third_party/crates/rustversion-1.0.23/.github/workflows/ci.yml new file mode 100644 index 0000000..3874eeb --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/.github/workflows/ci.yml @@ -0,0 +1,117 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + schedule: [cron: "40 1 * * *"] + +permissions: + contents: read + +env: + RUSTFLAGS: -Dwarnings + +jobs: + pre_ci: + uses: dtolnay/.github/.github/workflows/pre_ci.yml@master + + test: + name: Rust ${{matrix.rust}} + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + rust: [nightly, beta, stable, 1.85.0] + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{matrix.rust}} + - name: Enable type layout randomization + run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout >> $GITHUB_ENV + if: matrix.rust == 'nightly' + - run: cargo test + - name: RUSTFLAGS=-Zfmt-debug=none cargo test + run: RUSTFLAGS=${RUSTFLAGS}\ -Zfmt-debug=none cargo test + if: matrix.rust == 'nightly' + - uses: actions/upload-artifact@v7 + if: matrix.rust == 'nightly' && always() + with: + name: Cargo.lock + path: Cargo.lock + continue-on-error: true + + windows: + name: Windows + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: windows-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - run: cargo test + + msrv: + name: Rust 1.31.0 + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@1.31.0 + - run: cargo check + + doc: + name: Documentation + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + RUSTDOCFLAGS: -Dwarnings + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/install@cargo-docs-rs + - run: cargo docs-rs + + clippy: + name: Clippy + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@clippy + - run: cargo clippy --tests -- -Dclippy::all -Dclippy::pedantic + + miri: + name: Miri + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@miri + - run: cargo miri setup + - run: cargo miri test + env: + MIRIFLAGS: -Zmiri-strict-provenance + + outdated: + name: Outdated + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/install@cargo-outdated + - run: cargo outdated --workspace --exit-code 1 diff --git a/third_party/crates/rustversion-1.0.23/BUILD.bazel b/third_party/crates/rustversion-1.0.23/BUILD.bazel new file mode 100644 index 0000000..b8ec667 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/BUILD.bazel @@ -0,0 +1,179 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load( + "@rules_rust//cargo:defs.bzl", + "cargo_build_script", + "cargo_toml_env_vars", +) +load("@rules_rust//rust:defs.bzl", "rust_proc_macro") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_proc_macro( + name = "rustversion", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_root = "src/lib.rs", + edition = "2018", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=rustversion", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "1.0.23", + deps = [ + ":build_script_build", + ], +) + +cargo_build_script( + name = "_bs", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + build_script_env_files = [ + ":cargo_toml_env_vars", + ], + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + "**/*.rs", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_name = "build_script_build", + crate_root = "build/build.rs", + data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + edition = "2018", + emit_warnings = False, + pkg_name = "rustversion", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=rustversion", + "manual", + "noclippy", + "norustfmt", + ], + version = "1.0.23", + visibility = ["//visibility:private"], +) + +alias( + name = "build_script_build", + actual = ":_bs", + tags = ["manual"], +) diff --git a/third_party/crates/rustversion-1.0.23/Cargo.lock b/third_party/crates/rustversion-1.0.23/Cargo.lock new file mode 100644 index 0000000..af1ce7c --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/Cargo.lock @@ -0,0 +1,248 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "dissimilar" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "equivalent 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-ident 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +dependencies = [ + "trybuild 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_core 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.46 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 2.0.118 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_core 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", + "zmij 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_core 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.46 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-ident 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "target-triple" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-util 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "indexmap 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_core 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_spanned 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "toml_datetime 1.1.1+spec-1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "toml_parser 1.1.2+spec-1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "toml_writer 1.1.1+spec-1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winnow 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_core 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winnow 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "trybuild" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dissimilar 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "glob 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.150 (registry+https://github.com/rust-lang/crates.io-index)", + "target-triple 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 1.1.2+spec-1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "windows-sys 0.61.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "windows-link 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum dissimilar 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" +"checksum equivalent 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +"checksum glob 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +"checksum hashbrown 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +"checksum indexmap 2.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +"checksum itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +"checksum memchr 2.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +"checksum proc-macro2 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)" = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +"checksum quote 1.0.46 (registry+https://github.com/rust-lang/crates.io-index)" = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +"checksum serde 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)" = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +"checksum serde_core 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)" = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +"checksum serde_derive 1.0.228 (registry+https://github.com/rust-lang/crates.io-index)" = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +"checksum serde_json 1.0.150 (registry+https://github.com/rust-lang/crates.io-index)" = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +"checksum serde_spanned 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +"checksum syn 2.0.118 (registry+https://github.com/rust-lang/crates.io-index)" = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +"checksum target-triple 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "591ef38edfb78ca4771ee32cf494cb8771944bee237a9b91fc9c1424ac4b777b" +"checksum termcolor 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +"checksum toml 1.1.2+spec-1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +"checksum toml_datetime 1.1.1+spec-1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +"checksum toml_parser 1.1.2+spec-1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +"checksum toml_writer 1.1.1+spec-1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" +"checksum trybuild 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)" = "0710d4dfbeae4f9c390baa784c49858a7468fa433f3fe5d0ec5ebef651cf59f9" +"checksum unicode-ident 1.0.24 (registry+https://github.com/rust-lang/crates.io-index)" = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +"checksum winapi-util 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +"checksum windows-link 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +"checksum windows-sys 0.61.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +"checksum winnow 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +"checksum zmij 1.0.21 (registry+https://github.com/rust-lang/crates.io-index)" = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/third_party/crates/rustversion-1.0.23/Cargo.toml b/third_party/crates/rustversion-1.0.23/Cargo.toml new file mode 100644 index 0000000..0b906bc --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/Cargo.toml @@ -0,0 +1,69 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +rust-version = "1.31" +name = "rustversion" +version = "1.0.23" +authors = ["David Tolnay <dtolnay@gmail.com>"] +build = "build/build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Conditional compilation according to rustc compiler version" +documentation = "https://docs.rs/rustversion" +readme = "README.md" +categories = [ + "development-tools::build-utils", + "no-std", + "no-std::no-alloc", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/rustversion" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +rustdoc-args = [ + "--generate-link-to-definition", + "--generate-macro-expansion", + "--extern-html-root-url=core=https://doc.rust-lang.org", + "--extern-html-root-url=alloc=https://doc.rust-lang.org", + "--extern-html-root-url=std=https://doc.rust-lang.org", + "--extern-html-root-url=proc_macro=https://doc.rust-lang.org", +] + +[lib] +name = "rustversion" +path = "src/lib.rs" +proc-macro = true + +[[test]] +name = "compiletest" +path = "tests/compiletest.rs" + +[[test]] +name = "test_const" +path = "tests/test_const.rs" + +[[test]] +name = "test_eval" +path = "tests/test_eval.rs" + +[[test]] +name = "test_parse" +path = "tests/test_parse.rs" + +[dev-dependencies.trybuild] +version = "1.0.49" +features = ["diff"] diff --git a/third_party/crates/rustversion-1.0.23/Cargo.toml.orig b/third_party/crates/rustversion-1.0.23/Cargo.toml.orig new file mode 100644 index 0000000..a2a4d66 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/Cargo.toml.orig @@ -0,0 +1,29 @@ +[package] +name = "rustversion" +version = "1.0.23" +authors = ["David Tolnay <dtolnay@gmail.com>"] +build = "build/build.rs" +categories = ["development-tools::build-utils", "no-std", "no-std::no-alloc"] +description = "Conditional compilation according to rustc compiler version" +documentation = "https://docs.rs/rustversion" +edition = "2018" +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/rustversion" +rust-version = "1.31" + +[lib] +proc-macro = true + +[dev-dependencies] +trybuild = { version = "1.0.49", features = ["diff"] } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +rustdoc-args = [ + "--generate-link-to-definition", + "--generate-macro-expansion", + "--extern-html-root-url=core=https://doc.rust-lang.org", + "--extern-html-root-url=alloc=https://doc.rust-lang.org", + "--extern-html-root-url=std=https://doc.rust-lang.org", + "--extern-html-root-url=proc_macro=https://doc.rust-lang.org", +] diff --git a/third_party/crates/rustversion-1.0.23/LICENSE-APACHE b/third_party/crates/rustversion-1.0.23/LICENSE-APACHE new file mode 100644 index 0000000..1b5ec8b --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/LICENSE-APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/third_party/crates/rustversion-1.0.23/LICENSE-MIT b/third_party/crates/rustversion-1.0.23/LICENSE-MIT new file mode 100644 index 0000000..31aa793 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third_party/crates/rustversion-1.0.23/README.md b/third_party/crates/rustversion-1.0.23/README.md new file mode 100644 index 0000000..d9b0eb6 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/README.md @@ -0,0 +1,167 @@ +Compiler version cfg +==================== + +[<img alt="github" src="https://img.shields.io/badge/github-dtolnay/rustversion-8da0cb?style=for-the-badge&labelColor=555555&logo=github" height="20">](https://github.com/dtolnay/rustversion) +[<img alt="crates.io" src="https://img.shields.io/crates/v/rustversion.svg?style=for-the-badge&color=fc8d62&logo=rust" height="20">](https://crates.io/crates/rustversion) +[<img alt="docs.rs" src="https://img.shields.io/badge/docs.rs-rustversion-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs" height="20">](https://docs.rs/rustversion) +[<img alt="build status" src="https://img.shields.io/github/actions/workflow/status/dtolnay/rustversion/ci.yml?branch=master&style=for-the-badge" height="20">](https://github.com/dtolnay/rustversion/actions?query=branch%3Amaster) + +This crate provides macros for conditional compilation according to rustc +compiler version, analogous to [`#[cfg(...)]`][cfg] and +[`#[cfg_attr(...)]`][cfg_attr]. + +[cfg]: https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg-attribute +[cfg_attr]: https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute + +```toml +[dependencies] +rustversion = "1.0" +``` + +<br> + +## Selectors + +- <b>`#[rustversion::stable]`</b> + —<br> + True on any stable compiler. + +- <b>`#[rustversion::stable(1.34)]`</b> + —<br> + True on exactly the specified stable compiler. + +- <b>`#[rustversion::beta]`</b> + —<br> + True on any beta compiler. + +- <b>`#[rustversion::nightly]`</b> + —<br> + True on any nightly compiler or dev build. + +- <b>`#[rustversion::nightly(2025-01-01)]`</b> + —<br> + True on exactly one nightly. + +- <b>`#[rustversion::since(1.34)]`</b> + —<br> + True on that stable release and any later compiler, including beta and + nightly. + +- <b>`#[rustversion::since(2025-01-01)]`</b> + —<br> + True on that nightly and all newer ones. + +- <b>`#[rustversion::before(`</b><i>version or date</i><b>`)]`</b> + —<br> + Negative of *#[rustversion::since(...)]*. + +- <b>`#[rustversion::not(`</b><i>selector</i><b>`)]`</b> + —<br> + Negative of any selector; for example *#[rustversion::not(nightly)]*. + +- <b>`#[rustversion::any(`</b><i>selectors...</i><b>`)]`</b> + —<br> + True if any of the comma-separated selectors is true; for example + *#[rustversion::any(stable, beta)]*. + +- <b>`#[rustversion::all(`</b><i>selectors...</i><b>`)]`</b> + —<br> + True if all of the comma-separated selectors are true; for example + *#[rustversion::all(since(1.31), before(1.34))]*. + +- <b>`#[rustversion::attr(`</b><i>selector</i><b>`, `</b><i>attribute</i><b>`)]`</b> + —<br> + For conditional inclusion of attributes; analogous to `cfg_attr`. + +- <b>`rustversion::cfg!(`</b><i>selector</i><b>`)`</b> + —<br> + An expression form of any of the above attributes; for example + *if rustversion::cfg!(any(stable, beta)) { ... }*. + +<br> + +## Use cases + +Providing additional trait impls as types are stabilized in the standard library +without breaking compatibility with older compilers; in this case Pin\<P\> +stabilized in [Rust 1.33][pin]: + +[pin]: https://blog.rust-lang.org/2019/02/28/Rust-1.33.0.html#pinning + +```rust +#[rustversion::since(1.33)] +use std::pin::Pin; + +#[rustversion::since(1.33)] +impl<P: MyTrait> MyTrait for Pin<P> { + /* ... */ +} +``` + +Similar but for language features; the ability to control alignment greater than +1 of packed structs was stabilized in [Rust 1.33][packed]. + +[packed]: https://github.com/rust-lang/rust/blob/master/RELEASES.md#version-1330-2019-02-28 + +```rust +#[rustversion::attr(before(1.33), repr(packed))] +#[rustversion::attr(since(1.33), repr(packed(2)))] +struct Six(i16, i32); + +fn main() { + println!("{}", std::mem::align_of::<Six>()); +} +``` + +Augmenting code with `const` as const impls are stabilized in the standard +library. This use of `const` as an attribute is recognized as a special case by +the rustversion::attr macro. + +```rust +use std::time::Duration; + +#[rustversion::attr(since(1.32), const)] +fn duration_as_days(dur: Duration) -> u64 { + dur.as_secs() / 60 / 60 / 24 +} +``` + +Emitting Cargo cfg directives from a build script. Note that this requires +listing `rustversion` under `[build-dependencies]` in Cargo.toml, not +`[dependencies]`. + +```rust +// build.rs + +fn main() { + if rustversion::cfg!(since(1.36)) { + println!("cargo:rustc-cfg=no_std"); + } +} +``` + +```rust +// src/lib.rs + +#![cfg_attr(no_std, no_std)] + +#[cfg(no_std)] +extern crate alloc; +``` + +<br> + +#### License + +<sup> +Licensed under either of <a href="LICENSE-APACHE">Apache License, Version +2.0</a> or <a href="LICENSE-MIT">MIT license</a> at your option. +</sup> + +<br> + +<sub> +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this crate by you, as defined in the Apache-2.0 license, shall +be dual licensed as above, without any additional terms or conditions. +</sub> diff --git a/third_party/crates/rustversion-1.0.23/src/attr.rs b/third_party/crates/rustversion-1.0.23/src/attr.rs new file mode 100644 index 0000000..6023d1e --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/attr.rs @@ -0,0 +1,35 @@ +use crate::error::{Error, Result}; +use crate::expr::{self, Expr}; +use crate::{iter, token}; +use proc_macro::{Span, TokenStream}; + +pub struct Args { + pub condition: Expr, + pub then: Then, +} + +pub enum Then { + Const(Span), + Attribute(TokenStream), +} + +pub fn parse(input: TokenStream) -> Result<Args> { + let ref mut input = iter::new(input); + let condition = expr::parse(input)?; + + token::parse_punct(input, ',')?; + if input.peek().is_none() { + return Err(Error::new(Span::call_site(), "expected one or more attrs")); + } + + let const_span = token::parse_optional_keyword(input, "const"); + let then = if let Some(const_span) = const_span { + token::parse_optional_punct(input, ','); + token::parse_end(input)?; + Then::Const(const_span) + } else { + Then::Attribute(input.collect()) + }; + + Ok(Args { condition, then }) +} diff --git a/third_party/crates/rustversion-1.0.23/src/bound.rs b/third_party/crates/rustversion-1.0.23/src/bound.rs new file mode 100644 index 0000000..8267258 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/bound.rs @@ -0,0 +1,63 @@ +use crate::date::{self, Date}; +use crate::error::{Error, Result}; +use crate::iter::Iter; +use crate::release::{self, Release}; +use crate::time; +use crate::version::{Channel::*, Version}; +use proc_macro::{Group, TokenTree}; +use std::cmp::Ordering; + +pub enum Bound { + Nightly(Date), + Stable(Release), +} + +pub fn parse(paren: Group, iter: Iter) -> Result<Bound> { + if let Some(TokenTree::Literal(literal)) = iter.peek() { + let repr = literal.to_string(); + if repr.starts_with(|ch: char| ch.is_ascii_digit()) { + if repr.contains('.') { + return release::parse(paren, iter).map(Bound::Stable); + } else { + return date::parse(paren, iter).map(Bound::Nightly); + } + } + } + let msg = format!( + "expected rustc release number like 1.85, or nightly date like {}", + time::today(), + ); + Err(Error::group(paren, msg)) +} + +impl PartialEq<Bound> for Version { + fn eq(&self, rhs: &Bound) -> bool { + match rhs { + Bound::Nightly(date) => match self.channel { + Stable | Beta | Dev => false, + Nightly(nightly) => nightly == *date, + }, + Bound::Stable(release) => { + self.minor == release.minor + && release.patch.map_or(true, |patch| self.patch == patch) + } + } + } +} + +impl PartialOrd<Bound> for Version { + fn partial_cmp(&self, rhs: &Bound) -> Option<Ordering> { + match rhs { + Bound::Nightly(date) => match self.channel { + Stable | Beta => Some(Ordering::Less), + Nightly(nightly) => Some(nightly.cmp(date)), + Dev => Some(Ordering::Greater), + }, + Bound::Stable(release) => { + let version = (self.minor, self.patch); + let bound = (release.minor, release.patch.unwrap_or(0)); + Some(version.cmp(&bound)) + } + } + } +} diff --git a/third_party/crates/rustversion-1.0.23/src/constfn.rs b/third_party/crates/rustversion-1.0.23/src/constfn.rs new file mode 100644 index 0000000..0e50c03 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/constfn.rs @@ -0,0 +1,58 @@ +use crate::error::{Error, Result}; +use proc_macro::{Ident, Span, TokenStream, TokenTree}; +use std::iter; + +#[derive(PartialOrd, PartialEq)] +enum Qualifiers { + None, + Async, + Unsafe, + Extern, + Abi, +} + +impl Qualifiers { + fn from_ident(ident: &Ident) -> Self { + match ident.to_string().as_str() { + "async" => Qualifiers::Async, + "unsafe" => Qualifiers::Unsafe, + "extern" => Qualifiers::Extern, + _ => Qualifiers::None, + } + } +} + +pub(crate) fn insert_const(input: TokenStream, const_span: Span) -> Result<TokenStream> { + let ref mut input = crate::iter::new(input); + let mut out = TokenStream::new(); + let mut qualifiers = Qualifiers::None; + let mut pending = Vec::new(); + + while let Some(token) = input.next() { + match token { + TokenTree::Ident(ref ident) if ident.to_string() == "fn" => { + let const_ident = Ident::new("const", const_span); + out.extend(iter::once(TokenTree::Ident(const_ident))); + out.extend(pending); + out.extend(iter::once(token)); + out.extend(input); + return Ok(out); + } + TokenTree::Ident(ref ident) if Qualifiers::from_ident(ident) > qualifiers => { + qualifiers = Qualifiers::from_ident(ident); + pending.push(token); + } + TokenTree::Literal(_) if qualifiers == Qualifiers::Extern => { + qualifiers = Qualifiers::Abi; + pending.push(token); + } + _ => { + qualifiers = Qualifiers::None; + out.extend(pending.drain(..)); + out.extend(iter::once(token)); + } + } + } + + Err(Error::new(const_span, "only allowed on a fn item")) +} diff --git a/third_party/crates/rustversion-1.0.23/src/date.rs b/third_party/crates/rustversion-1.0.23/src/date.rs new file mode 100644 index 0000000..aa4e710 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/date.rs @@ -0,0 +1,50 @@ +use crate::error::{Error, Result}; +use crate::iter::Iter; +use crate::{time, token}; +use proc_macro::Group; +use std::fmt::{self, Display}; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Date { + pub year: u16, + pub month: u8, + pub day: u8, +} + +impl Display for Date { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!( + formatter, + "{:04}-{:02}-{:02}", + self.year, self.month, self.day, + ) + } +} + +pub fn parse(paren: Group, iter: Iter) -> Result<Date> { + try_parse(iter).map_err(|()| { + let msg = format!("expected nightly date, like {}", time::today()); + Error::group(paren, msg) + }) +} + +fn try_parse(iter: Iter) -> Result<Date, ()> { + let year = token::parse_literal(iter).map_err(drop)?; + token::parse_punct(iter, '-').map_err(drop)?; + let month = token::parse_literal(iter).map_err(drop)?; + token::parse_punct(iter, '-').map_err(drop)?; + let day = token::parse_literal(iter).map_err(drop)?; + + let year = year.to_string().parse::<u64>().map_err(drop)?; + let month = month.to_string().parse::<u64>().map_err(drop)?; + let day = day.to_string().parse::<u64>().map_err(drop)?; + if year >= 3000 || month > 12 || day > 31 { + return Err(()); + } + + Ok(Date { + year: year as u16, + month: month as u8, + day: day as u8, + }) +} diff --git a/third_party/crates/rustversion-1.0.23/src/error.rs b/third_party/crates/rustversion-1.0.23/src/error.rs new file mode 100644 index 0000000..a99625b --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/error.rs @@ -0,0 +1,56 @@ +use proc_macro::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::fmt::Display; +use std::iter::FromIterator; + +pub type Result<T, E = Error> = std::result::Result<T, E>; + +pub struct Error { + begin: Span, + end: Span, + msg: String, +} + +impl Error { + pub fn new(span: Span, msg: impl Display) -> Self { + Self::new2(span, span, msg) + } + + pub fn new2(begin: Span, end: Span, msg: impl Display) -> Self { + Error { + begin, + end, + msg: msg.to_string(), + } + } + + pub fn group(group: Group, msg: impl Display) -> Self { + let mut iter = group.stream().into_iter(); + let delimiter = group.span(); + let begin = iter.next().map_or(delimiter, |t| t.span()); + let end = iter.last().map_or(begin, |t| t.span()); + Self::new2(begin, end, msg) + } + + pub fn into_compile_error(self) -> TokenStream { + // compile_error! { $msg } + TokenStream::from_iter(vec![ + TokenTree::Ident(Ident::new("compile_error", self.begin)), + TokenTree::Punct({ + let mut punct = Punct::new('!', Spacing::Alone); + punct.set_span(self.begin); + punct + }), + TokenTree::Group({ + let mut group = Group::new(Delimiter::Brace, { + TokenStream::from_iter(vec![TokenTree::Literal({ + let mut string = Literal::string(&self.msg); + string.set_span(self.end); + string + })]) + }); + group.set_span(self.end); + group + }), + ]) + } +} diff --git a/third_party/crates/rustversion-1.0.23/src/expand.rs b/third_party/crates/rustversion-1.0.23/src/expand.rs new file mode 100644 index 0000000..ce70f2e --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/expand.rs @@ -0,0 +1,100 @@ +use crate::attr::{self, Then}; +use crate::error::{Error, Result}; +use crate::{constfn, expr, iter, token}; +use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::iter::FromIterator; + +pub fn cfg(introducer: &str, args: TokenStream, input: TokenStream) -> TokenStream { + try_cfg(introducer, args, input).unwrap_or_else(Error::into_compile_error) +} + +fn try_cfg(introducer: &str, args: TokenStream, input: TokenStream) -> Result<TokenStream> { + let introducer = Ident::new(introducer, Span::call_site()); + + let mut full_args = TokenStream::from(TokenTree::Ident(introducer)); + if !args.is_empty() { + full_args.extend(std::iter::once(TokenTree::Group(Group::new( + Delimiter::Parenthesis, + args, + )))); + } + + let ref mut full_args = iter::new(full_args); + let expr = expr::parse(full_args)?; + token::parse_end(full_args)?; + + if expr.eval(crate::RUSTVERSION) { + Ok(allow_incompatible_msrv(input)) + } else { + Ok(TokenStream::new()) + } +} + +pub fn try_attr(args: attr::Args, input: TokenStream) -> Result<TokenStream> { + if !args.condition.eval(crate::RUSTVERSION) { + return Ok(input); + } + + let output = match args.then { + Then::Const(const_token) => constfn::insert_const(input, const_token)?, + Then::Attribute(then) => { + TokenStream::from_iter( + // #[cfg_attr(all(), #then)] + vec![ + TokenTree::Punct(Punct::new('#', Spacing::Alone)), + TokenTree::Group(Group::new( + Delimiter::Bracket, + TokenStream::from_iter(vec![ + TokenTree::Ident(Ident::new("cfg_attr", Span::call_site())), + TokenTree::Group(Group::new( + Delimiter::Parenthesis, + TokenStream::from_iter( + vec![ + TokenTree::Ident(Ident::new("all", Span::call_site())), + TokenTree::Group(Group::new( + Delimiter::Parenthesis, + TokenStream::new(), + )), + TokenTree::Punct(Punct::new(',', Spacing::Alone)), + ] + .into_iter() + .chain(then), + ), + )), + ]), + )), + ] + .into_iter() + .chain(input), + ) + } + }; + + Ok(allow_incompatible_msrv(output)) +} + +fn allow_incompatible_msrv(input: TokenStream) -> TokenStream { + TokenStream::from_iter( + // #[allow(clippy::incompatible_msrv)] + vec![ + TokenTree::Punct(Punct::new('#', Spacing::Alone)), + TokenTree::Group(Group::new( + Delimiter::Bracket, + TokenStream::from_iter(vec![ + TokenTree::Ident(Ident::new("allow", Span::call_site())), + TokenTree::Group(Group::new( + Delimiter::Parenthesis, + TokenStream::from_iter(vec![ + TokenTree::Ident(Ident::new("clippy", Span::call_site())), + TokenTree::Punct(Punct::new(':', Spacing::Joint)), + TokenTree::Punct(Punct::new(':', Spacing::Alone)), + TokenTree::Ident(Ident::new("incompatible_msrv", Span::call_site())), + ]), + )), + ]), + )), + ] + .into_iter() + .chain(input), + ) +} diff --git a/third_party/crates/rustversion-1.0.23/src/expr.rs b/third_party/crates/rustversion-1.0.23/src/expr.rs new file mode 100644 index 0000000..45ed7d2 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/expr.rs @@ -0,0 +1,163 @@ +use crate::bound::{self, Bound}; +use crate::date::{self, Date}; +use crate::error::{Error, Result}; +use crate::iter::{self, Iter}; +use crate::release::{self, Release}; +use crate::token; +use crate::version::{Channel, Version}; +use proc_macro::{Ident, Span, TokenTree}; + +pub enum Expr { + Stable, + Beta, + Nightly, + Date(Date), + Since(Bound), + Before(Bound), + Release(Release), + Not(Box<Expr>), + Any(Vec<Expr>), + All(Vec<Expr>), +} + +impl Expr { + pub fn eval(&self, rustc: Version) -> bool { + use self::Expr::*; + + match self { + Stable => rustc.channel == Channel::Stable, + Beta => rustc.channel == Channel::Beta, + Nightly => match rustc.channel { + Channel::Nightly(_) | Channel::Dev => true, + Channel::Stable | Channel::Beta => false, + }, + Date(date) => match rustc.channel { + Channel::Nightly(rustc) => rustc == *date, + Channel::Stable | Channel::Beta | Channel::Dev => false, + }, + Since(bound) => rustc >= *bound, + Before(bound) => rustc < *bound, + Release(release) => { + rustc.channel == Channel::Stable + && rustc.minor == release.minor + && release.patch.map_or(true, |patch| rustc.patch == patch) + } + Not(expr) => !expr.eval(rustc), + Any(exprs) => exprs.iter().any(|e| e.eval(rustc)), + All(exprs) => exprs.iter().all(|e| e.eval(rustc)), + } + } +} + +pub fn parse(iter: Iter) -> Result<Expr> { + match &iter.next() { + Some(TokenTree::Ident(i)) if i.to_string() == "stable" => parse_stable(iter), + Some(TokenTree::Ident(i)) if i.to_string() == "beta" => Ok(Expr::Beta), + Some(TokenTree::Ident(i)) if i.to_string() == "nightly" => parse_nightly(iter), + Some(TokenTree::Ident(i)) if i.to_string() == "since" => parse_since(i, iter), + Some(TokenTree::Ident(i)) if i.to_string() == "before" => parse_before(i, iter), + Some(TokenTree::Ident(i)) if i.to_string() == "not" => parse_not(i, iter), + Some(TokenTree::Ident(i)) if i.to_string() == "any" => parse_any(i, iter), + Some(TokenTree::Ident(i)) if i.to_string() == "all" => parse_all(i, iter), + unexpected => { + let span = unexpected + .as_ref() + .map_or_else(Span::call_site, TokenTree::span); + Err(Error::new(span, "expected one of `stable`, `beta`, `nightly`, `since`, `before`, `not`, `any`, `all`")) + } + } +} + +fn parse_nightly(iter: Iter) -> Result<Expr> { + let paren = match token::parse_optional_paren(iter) { + Some(group) => group, + None => return Ok(Expr::Nightly), + }; + + let ref mut inner = iter::new(paren.stream()); + let date = date::parse(paren, inner)?; + token::parse_optional_punct(inner, ','); + token::parse_end(inner)?; + + Ok(Expr::Date(date)) +} + +fn parse_stable(iter: Iter) -> Result<Expr> { + let paren = match token::parse_optional_paren(iter) { + Some(group) => group, + None => return Ok(Expr::Stable), + }; + + let ref mut inner = iter::new(paren.stream()); + let release = release::parse(paren, inner)?; + token::parse_optional_punct(inner, ','); + token::parse_end(inner)?; + + Ok(Expr::Release(release)) +} + +fn parse_since(introducer: &Ident, iter: Iter) -> Result<Expr> { + let paren = token::parse_paren(introducer, iter)?; + + let ref mut inner = iter::new(paren.stream()); + let bound = bound::parse(paren, inner)?; + token::parse_optional_punct(inner, ','); + token::parse_end(inner)?; + + Ok(Expr::Since(bound)) +} + +fn parse_before(introducer: &Ident, iter: Iter) -> Result<Expr> { + let paren = token::parse_paren(introducer, iter)?; + + let ref mut inner = iter::new(paren.stream()); + let bound = bound::parse(paren, inner)?; + token::parse_optional_punct(inner, ','); + token::parse_end(inner)?; + + Ok(Expr::Before(bound)) +} + +fn parse_not(introducer: &Ident, iter: Iter) -> Result<Expr> { + let paren = token::parse_paren(introducer, iter)?; + + let ref mut inner = iter::new(paren.stream()); + let expr = self::parse(inner)?; + token::parse_optional_punct(inner, ','); + token::parse_end(inner)?; + + Ok(Expr::Not(Box::new(expr))) +} + +fn parse_any(introducer: &Ident, iter: Iter) -> Result<Expr> { + let paren = token::parse_paren(introducer, iter)?; + + let ref mut inner = iter::new(paren.stream()); + let exprs = parse_comma_separated(inner)?; + + Ok(Expr::Any(exprs.into_iter().collect())) +} + +fn parse_all(introducer: &Ident, iter: Iter) -> Result<Expr> { + let paren = token::parse_paren(introducer, iter)?; + + let ref mut inner = iter::new(paren.stream()); + let exprs = parse_comma_separated(inner)?; + + Ok(Expr::All(exprs.into_iter().collect())) +} + +fn parse_comma_separated(iter: Iter) -> Result<Vec<Expr>> { + let mut exprs = Vec::new(); + + while iter.peek().is_some() { + let expr = self::parse(iter)?; + exprs.push(expr); + if iter.peek().is_none() { + break; + } + token::parse_punct(iter, ',')?; + } + + Ok(exprs) +} diff --git a/third_party/crates/rustversion-1.0.23/src/iter.rs b/third_party/crates/rustversion-1.0.23/src/iter.rs new file mode 100644 index 0000000..722013c --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/iter.rs @@ -0,0 +1,42 @@ +use proc_macro::{token_stream, Delimiter, TokenStream, TokenTree}; + +pub type Iter<'a> = &'a mut IterImpl; + +pub struct IterImpl { + stack: Vec<token_stream::IntoIter>, + peeked: Option<TokenTree>, +} + +pub fn new(tokens: TokenStream) -> IterImpl { + IterImpl { + stack: vec![tokens.into_iter()], + peeked: None, + } +} + +impl IterImpl { + pub fn peek(&mut self) -> Option<&TokenTree> { + self.peeked = self.next(); + self.peeked.as_ref() + } +} + +impl Iterator for IterImpl { + type Item = TokenTree; + + fn next(&mut self) -> Option<Self::Item> { + if let Some(tt) = self.peeked.take() { + return Some(tt); + } + loop { + let top = self.stack.last_mut()?; + match top.next() { + None => drop(self.stack.pop()), + Some(TokenTree::Group(ref group)) if group.delimiter() == Delimiter::None => { + self.stack.push(group.stream().into_iter()); + } + Some(tt) => return Some(tt), + } + } + } +} diff --git a/third_party/crates/rustversion-1.0.23/src/lib.rs b/third_party/crates/rustversion-1.0.23/src/lib.rs new file mode 100644 index 0000000..1db5ba0 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/lib.rs @@ -0,0 +1,281 @@ +//! [![github]](https://github.com/dtolnay/rustversion) [![crates-io]](https://crates.io/crates/rustversion) [![docs-rs]](https://docs.rs/rustversion) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs +//! +//! <br> +//! +//! This crate provides macros for conditional compilation according to rustc +//! compiler version, analogous to [`#[cfg(...)]`][cfg] and +//! [`#[cfg_attr(...)]`][cfg_attr]. +//! +//! [cfg]: https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg-attribute +//! [cfg_attr]: https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute +//! +//! <br> +//! +//! # Selectors +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::stable]</code></b> +//! —<br> +//! True on any stable compiler. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::stable(1.34)]</code></b> +//! —<br> +//! True on exactly the specified stable compiler. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::beta]</code></b> +//! —<br> +//! True on any beta compiler. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::nightly]</code></b> +//! —<br> +//! True on any nightly compiler or dev build. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::nightly(2025-01-01)]</code></b> +//! —<br> +//! True on exactly one nightly. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::since(1.34)]</code></b> +//! —<br> +//! True on that stable release and any later compiler, including beta and +//! nightly. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::since(2025-01-01)]</code></b> +//! —<br> +//! True on that nightly and all newer ones. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::before(</code></b><i>version or date</i><b><code style="display:inline">)]</code></b> +//! —<br> +//! Negative of <i>#[rustversion::since(...)]</i>. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::not(</code></b><i>selector</i><b><code style="display:inline">)]</code></b> +//! —<br> +//! Negative of any selector; for example <i>#[rustversion::not(nightly)]</i>. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::any(</code></b><i>selectors...</i><b><code style="display:inline">)]</code></b> +//! —<br> +//! True if any of the comma-separated selectors is true; for example +//! <i>#[rustversion::any(stable, beta)]</i>. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::all(</code></b><i>selectors...</i><b><code style="display:inline">)]</code></b> +//! —<br> +//! True if all of the comma-separated selectors are true; for example +//! <i>#[rustversion::all(since(1.31), before(1.34))]</i>. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">#[rustversion::attr(</code></b><i>selector</i><b><code style="display:inline">, </code></b><i>attribute</i><b><code style="display:inline">)]</code></b> +//! —<br> +//! For conditional inclusion of attributes; analogous to +//! <code style="display:inline">cfg_attr</code>. +//! </p> +//! +//! - <p style="margin-left:50px;text-indent:-50px"> +//! <b><code style="display:inline">rustversion::cfg!(</code></b><i>selector</i><b><code style="display:inline">)</code></b> +//! —<br> +//! An expression form of any of the above attributes; for example +//! <i>if rustversion::cfg!(any(stable, beta)) { ... }</i>. +//! </p> +//! +//! <br> +//! +//! # Use cases +//! +//! Providing additional trait impls as types are stabilized in the standard library +//! without breaking compatibility with older compilers; in this case Pin\<P\> +//! stabilized in [Rust 1.33][pin]: +//! +//! [pin]: https://blog.rust-lang.org/2019/02/28/Rust-1.33.0.html#pinning +//! +//! ``` +//! # trait MyTrait {} +//! # +//! #[rustversion::since(1.33)] +//! use std::pin::Pin; +//! +//! #[rustversion::since(1.33)] +//! impl<P: MyTrait> MyTrait for Pin<P> { +//! /* ... */ +//! } +//! ``` +//! +//! Similar but for language features; the ability to control alignment greater than +//! 1 of packed structs was stabilized in [Rust 1.33][packed]. +//! +//! [packed]: https://github.com/rust-lang/rust/blob/master/RELEASES.md#version-1330-2019-02-28 +//! +//! ``` +//! #[rustversion::attr(before(1.33), repr(packed))] +//! #[rustversion::attr(since(1.33), repr(packed(2)))] +//! struct Six(i16, i32); +//! +//! fn main() { +//! println!("{}", std::mem::align_of::<Six>()); +//! } +//! ``` +//! +//! Augmenting code with `const` as const impls are stabilized in the standard +//! library. This use of `const` as an attribute is recognized as a special case +//! by the rustversion::attr macro. +//! +//! ``` +//! use std::time::Duration; +//! +//! #[rustversion::attr(since(1.32), const)] +//! fn duration_as_days(dur: Duration) -> u64 { +//! dur.as_secs() / 60 / 60 / 24 +//! } +//! ``` +//! +//! Emitting Cargo cfg directives from a build script. Note that this requires +//! listing `rustversion` under `[build-dependencies]` in Cargo.toml, not +//! `[dependencies]`. +//! +//! ``` +//! // build.rs +//! +//! fn main() { +//! if rustversion::cfg!(since(1.36)) { +//! println!("cargo:rustc-cfg=no_std"); +//! } +//! } +//! ``` +//! +//! ``` +//! // src/lib.rs +//! +//! #![cfg_attr(no_std, no_std)] +//! +//! #[cfg(no_std)] +//! extern crate alloc; +//! ``` +//! +//! <br> + +#![doc(html_root_url = "https://docs.rs/rustversion/1.0.23")] +#![allow( + clippy::cast_lossless, + clippy::cast_possible_truncation, + clippy::derive_partial_eq_without_eq, + clippy::doc_markdown, + clippy::enum_glob_use, + // https://github.com/rust-lang/rust-clippy/issues/8539 + clippy::iter_with_drain, + clippy::module_name_repetitions, + clippy::must_use_candidate, + clippy::needless_doctest_main, + clippy::needless_pass_by_value, + clippy::redundant_else, + clippy::toplevel_ref_arg, + clippy::unreadable_literal +)] + +extern crate proc_macro; + +mod attr; +mod bound; +mod constfn; +mod date; +mod error; +mod expand; +mod expr; +mod iter; +mod release; +mod time; +mod token; +mod version; + +use crate::error::Error; +use crate::version::Version; +use proc_macro::TokenStream; + +#[cfg(not(host_os = "windows"))] +const RUSTVERSION: Version = include!(concat!(env!("OUT_DIR"), "/version.expr")); + +#[cfg(host_os = "windows")] +const RUSTVERSION: Version = include!(concat!(env!("OUT_DIR"), "\\version.expr")); + +#[proc_macro_attribute] +pub fn stable(args: TokenStream, input: TokenStream) -> TokenStream { + expand::cfg("stable", args, input) +} + +#[proc_macro_attribute] +pub fn beta(args: TokenStream, input: TokenStream) -> TokenStream { + expand::cfg("beta", args, input) +} + +#[proc_macro_attribute] +pub fn nightly(args: TokenStream, input: TokenStream) -> TokenStream { + expand::cfg("nightly", args, input) +} + +#[proc_macro_attribute] +pub fn since(args: TokenStream, input: TokenStream) -> TokenStream { + expand::cfg("since", args, input) +} + +#[proc_macro_attribute] +pub fn before(args: TokenStream, input: TokenStream) -> TokenStream { + expand::cfg("before", args, input) +} + +#[proc_macro_attribute] +pub fn not(args: TokenStream, input: TokenStream) -> TokenStream { + expand::cfg("not", args, input) +} + +#[proc_macro_attribute] +pub fn any(args: TokenStream, input: TokenStream) -> TokenStream { + expand::cfg("any", args, input) +} + +#[proc_macro_attribute] +pub fn all(args: TokenStream, input: TokenStream) -> TokenStream { + expand::cfg("all", args, input) +} + +#[proc_macro_attribute] +pub fn attr(args: TokenStream, input: TokenStream) -> TokenStream { + attr::parse(args) + .and_then(|args| expand::try_attr(args, input)) + .unwrap_or_else(Error::into_compile_error) +} + +#[cfg(not(cfg_macro_not_allowed))] +#[proc_macro] +pub fn cfg(input: TokenStream) -> TokenStream { + use proc_macro::{Ident, Span, TokenTree}; + (|| { + let ref mut args = iter::new(input); + let expr = expr::parse(args)?; + token::parse_end(args)?; + let boolean = expr.eval(RUSTVERSION); + let ident = Ident::new(&boolean.to_string(), Span::call_site()); + Ok(TokenStream::from(TokenTree::Ident(ident))) + })() + .unwrap_or_else(Error::into_compile_error) +} diff --git a/third_party/crates/rustversion-1.0.23/src/release.rs b/third_party/crates/rustversion-1.0.23/src/release.rs new file mode 100644 index 0000000..c8c0219 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/release.rs @@ -0,0 +1,34 @@ +use crate::error::{Error, Result}; +use crate::iter::Iter; +use crate::token; +use proc_macro::Group; + +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Release { + pub minor: u16, + pub patch: Option<u16>, +} + +pub fn parse(paren: Group, iter: Iter) -> Result<Release> { + try_parse(iter).map_err(|()| Error::group(paren, "expected rustc release number, like 1.31")) +} + +fn try_parse(iter: Iter) -> Result<Release, ()> { + let major_minor = token::parse_literal(iter).map_err(drop)?; + let string = major_minor.to_string(); + + if !string.starts_with("1.") { + return Err(()); + } + + let minor: u16 = string[2..].parse().map_err(drop)?; + + let patch = if token::parse_optional_punct(iter, '.').is_some() { + let int = token::parse_literal(iter).map_err(drop)?; + Some(int.to_string().parse().map_err(drop)?) + } else { + None + }; + + Ok(Release { minor, patch }) +} diff --git a/third_party/crates/rustversion-1.0.23/src/time.rs b/third_party/crates/rustversion-1.0.23/src/time.rs new file mode 100644 index 0000000..102c80b --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/time.rs @@ -0,0 +1,51 @@ +use crate::date::Date; +use std::env; +use std::time::{SystemTime, UNIX_EPOCH}; + +// Timestamp of 2016-03-01 00:00:00 in UTC. +const BASE: u64 = 1456790400; +const BASE_YEAR: u16 = 2016; +const BASE_MONTH: u8 = 3; + +// Days between leap days. +const CYCLE: u64 = 365 * 4 + 1; + +const DAYS_BY_MONTH: [u8; 12] = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + +pub fn today() -> Date { + let default = Date { + year: 2025, + month: 2, + day: 25, + }; + try_today().unwrap_or(default) +} + +fn try_today() -> Option<Date> { + if let Some(pkg_name) = env::var_os("CARGO_PKG_NAME") { + if pkg_name.to_str() == Some("rustversion-tests") { + return None; // Stable date for ui testing. + } + } + + let now = SystemTime::now(); + let since_epoch = now.duration_since(UNIX_EPOCH).ok()?; + let secs = since_epoch.as_secs(); + + let approx_days = secs.checked_sub(BASE)? / 60 / 60 / 24; + let cycle = approx_days / CYCLE; + let mut rem = approx_days % CYCLE; + + let mut year = BASE_YEAR + cycle as u16 * 4; + let mut month = BASE_MONTH; + loop { + let days_in_month = DAYS_BY_MONTH[month as usize - 1]; + if rem < days_in_month as u64 { + let day = rem as u8 + 1; + return Some(Date { year, month, day }); + } + rem -= days_in_month as u64; + year += (month == 12) as u16; + month = month % 12 + 1; + } +} diff --git a/third_party/crates/rustversion-1.0.23/src/token.rs b/third_party/crates/rustversion-1.0.23/src/token.rs new file mode 100644 index 0000000..ebb6f1c --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/token.rs @@ -0,0 +1,78 @@ +use crate::error::{Error, Result}; +use crate::iter::Iter; +use proc_macro::{Delimiter, Group, Ident, Literal, Span, TokenTree}; + +pub fn parse_punct(iter: Iter, ch: char) -> Result<()> { + match iter.next() { + Some(TokenTree::Punct(ref punct)) if punct.as_char() == ch => Ok(()), + unexpected => { + let span = unexpected + .as_ref() + .map_or_else(Span::call_site, TokenTree::span); + Err(Error::new(span, format!("expected `{}`", ch))) + } + } +} + +pub fn parse_optional_punct(iter: Iter, ch: char) -> Option<()> { + match iter.peek() { + Some(TokenTree::Punct(punct)) if punct.as_char() == ch => iter.next().map(drop), + _ => None, + } +} + +pub fn parse_optional_keyword(iter: Iter, keyword: &str) -> Option<Span> { + match iter.peek() { + Some(TokenTree::Ident(ident)) if ident.to_string() == keyword => { + Some(iter.next().unwrap().span()) + } + _ => None, + } +} + +pub fn parse_literal(iter: Iter) -> Result<Literal> { + match iter.next() { + Some(TokenTree::Literal(literal)) => Ok(literal), + unexpected => { + let span = unexpected + .as_ref() + .map_or_else(Span::call_site, TokenTree::span); + Err(Error::new(span, "expected literal")) + } + } +} + +pub fn parse_paren(introducer: &Ident, iter: Iter) -> Result<Group> { + match iter.peek() { + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::Parenthesis => { + match iter.next() { + Some(TokenTree::Group(group)) => Ok(group), + _ => unreachable!(), + } + } + Some(unexpected) => Err(Error::new(unexpected.span(), "expected `(`")), + None => Err(Error::new( + introducer.span(), + format!("expected `(` after `{}`", introducer), + )), + } +} + +pub fn parse_optional_paren(iter: Iter) -> Option<Group> { + match iter.peek() { + Some(TokenTree::Group(group)) if group.delimiter() == Delimiter::Parenthesis => { + match iter.next() { + Some(TokenTree::Group(group)) => Some(group), + _ => unreachable!(), + } + } + _ => None, + } +} + +pub fn parse_end(iter: Iter) -> Result<()> { + match iter.next() { + None => Ok(()), + Some(unexpected) => Err(Error::new(unexpected.span(), "unexpected token")), + } +} diff --git a/third_party/crates/rustversion-1.0.23/src/version.rs b/third_party/crates/rustversion-1.0.23/src/version.rs new file mode 100644 index 0000000..f4a69f5 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/src/version.rs @@ -0,0 +1,18 @@ +#![allow(dead_code)] + +use crate::date::Date; + +#[derive(Copy, Clone, Debug, PartialEq)] +pub struct Version { + pub minor: u16, + pub patch: u16, + pub channel: Channel, +} + +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum Channel { + Stable, + Beta, + Nightly(Date), + Dev, +} diff --git a/third_party/crates/rustversion-1.0.23/tests/compiletest.rs b/third_party/crates/rustversion-1.0.23/tests/compiletest.rs new file mode 100644 index 0000000..23a6a06 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/compiletest.rs @@ -0,0 +1,7 @@ +#[rustversion::attr(not(nightly), ignore = "requires nightly")] +#[cfg_attr(miri, ignore = "incompatible with miri")] +#[test] +fn ui() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/*.rs"); +} diff --git a/third_party/crates/rustversion-1.0.23/tests/test_const.rs b/third_party/crates/rustversion-1.0.23/tests/test_const.rs new file mode 100644 index 0000000..0d0c7b8 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/test_const.rs @@ -0,0 +1,42 @@ +#![allow( + clippy::semicolon_if_nothing_returned, // https://github.com/rust-lang/rust-clippy/issues/7324 + clippy::used_underscore_items, +)] + +#[rustversion::attr(all(), const)] +fn _basic() {} +const _BASIC: () = _basic(); + +#[rustversion::attr(all(), const)] +unsafe fn _unsafe() {} +const _UNSAFE: () = unsafe { _unsafe() }; + +macro_rules! item { + ($i:item) => { + #[rustversion::attr(all(), const)] + $i + }; +} + +item! {fn _item() {}} +const _ITEM: () = _item(); + +macro_rules! ident { + ($fn:ident) => { + #[rustversion::attr(all(), const)] + $fn _ident() {} + }; +} + +ident! {fn} +const _IDENT: () = _ident(); + +#[rustversion::attr(all(), const)] +/// doc +fn _doc_below() {} +const _DOC_BELOW: () = _doc_below(); + +/// doc +#[rustversion::attr(all(), const)] +fn _doc_above() {} +const _DOC_ABOVE: () = _doc_above(); diff --git a/third_party/crates/rustversion-1.0.23/tests/test_eval.rs b/third_party/crates/rustversion-1.0.23/tests/test_eval.rs new file mode 100644 index 0000000..c044e85 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/test_eval.rs @@ -0,0 +1,20 @@ +#[rustversion::any( + stable, + stable(1.34), + stable(1.34.0), + beta, + nightly, + nightly(2020-02-25), + since(1.34), + since(2020-02-25), + before(1.34), + before(2020-02-25), + not(nightly), + all(stable, beta, nightly), +)] +fn success() {} + +#[test] +fn test() { + success(); +} diff --git a/third_party/crates/rustversion-1.0.23/tests/test_parse.rs b/third_party/crates/rustversion-1.0.23/tests/test_parse.rs new file mode 100644 index 0000000..559e463 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/test_parse.rs @@ -0,0 +1,103 @@ +#![allow( + clippy::derive_partial_eq_without_eq, + clippy::enum_glob_use, + clippy::must_use_candidate +)] + +include!("../build/rustc.rs"); + +#[test] +fn test_parse() { + let cases = &[ + ( + "rustc 1.0.0 (a59de37e9 2015-05-13) (built 2015-05-14)", + Version { + minor: 0, + patch: 0, + channel: Stable, + }, + ), + ( + "rustc 1.18.0", + Version { + minor: 18, + patch: 0, + channel: Stable, + }, + ), + ( + "rustc 1.24.1 (d3ae9a9e0 2018-02-27)", + Version { + minor: 24, + patch: 1, + channel: Stable, + }, + ), + ( + "rustc 1.35.0-beta.3 (c13114dc8 2019-04-27)", + Version { + minor: 35, + patch: 0, + channel: Beta, + }, + ), + ( + "rustc 1.36.0-nightly (938d4ffe1 2019-04-27)", + Version { + minor: 36, + patch: 0, + channel: Nightly(Date { + year: 2019, + month: 4, + day: 27, + }), + }, + ), + ( + "rustc 1.36.0-dev", + Version { + minor: 36, + patch: 0, + channel: Dev, + }, + ), + ( + "rustc 1.36.0-nightly", + Version { + minor: 36, + patch: 0, + channel: Dev, + }, + ), + ( + "warning: invalid logging spec 'warning', ignoring it + rustc 1.30.0-nightly (3bc2ca7e4 2018-09-20)", + Version { + minor: 30, + patch: 0, + channel: Nightly(Date { + year: 2018, + month: 9, + day: 20, + }), + }, + ), + ( + "rustc 1.52.1-nightly (gentoo)", + Version { + minor: 52, + patch: 1, + channel: Dev, + }, + ), + ]; + + for (string, expected) in cases { + match parse(string) { + ParseResult::Success(version) => assert_eq!(version, *expected), + ParseResult::OopsClippy | ParseResult::OopsMirai | ParseResult::Unrecognized => { + panic!("unrecognized: {:?}", string); + } + } + } +} diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/bad-bound.rs b/third_party/crates/rustversion-1.0.23/tests/ui/bad-bound.rs new file mode 100644 index 0000000..add8792 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/bad-bound.rs @@ -0,0 +1,7 @@ +#[rustversion::since(stable)] +struct S; + +#[rustversion::any(since(stable))] +struct S; + +fn main() {} diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/bad-bound.stderr b/third_party/crates/rustversion-1.0.23/tests/ui/bad-bound.stderr new file mode 100644 index 0000000..2a96406 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/bad-bound.stderr @@ -0,0 +1,11 @@ +error: expected rustc release number like 1.85, or nightly date like 2025-02-25 + --> tests/ui/bad-bound.rs:1:22 + | +1 | #[rustversion::since(stable)] + | ^^^^^^ + +error: expected rustc release number like 1.85, or nightly date like 2025-02-25 + --> tests/ui/bad-bound.rs:4:26 + | +4 | #[rustversion::any(since(stable))] + | ^^^^^^ diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/bad-date.rs b/third_party/crates/rustversion-1.0.23/tests/ui/bad-date.rs new file mode 100644 index 0000000..0984625 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/bad-date.rs @@ -0,0 +1,7 @@ +#[rustversion::nightly(stable)] +struct S; + +#[rustversion::any(nightly(stable))] +struct S; + +fn main() {} diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/bad-date.stderr b/third_party/crates/rustversion-1.0.23/tests/ui/bad-date.stderr new file mode 100644 index 0000000..64c61ca --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/bad-date.stderr @@ -0,0 +1,11 @@ +error: expected nightly date, like 2025-02-25 + --> tests/ui/bad-date.rs:1:24 + | +1 | #[rustversion::nightly(stable)] + | ^^^^^^ + +error: expected nightly date, like 2025-02-25 + --> tests/ui/bad-date.rs:4:28 + | +4 | #[rustversion::any(nightly(stable))] + | ^^^^^^ diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/bad-not.rs b/third_party/crates/rustversion-1.0.23/tests/ui/bad-not.rs new file mode 100644 index 0000000..058f340 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/bad-not.rs @@ -0,0 +1,7 @@ +#[rustversion::any(not)] +struct S; + +#[rustversion::any(not, not)] +struct S; + +fn main() {} diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/bad-not.stderr b/third_party/crates/rustversion-1.0.23/tests/ui/bad-not.stderr new file mode 100644 index 0000000..2b0c699 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/bad-not.stderr @@ -0,0 +1,11 @@ +error: expected `(` after `not` + --> tests/ui/bad-not.rs:1:20 + | +1 | #[rustversion::any(not)] + | ^^^ + +error: expected `(` + --> tests/ui/bad-not.rs:4:23 + | +4 | #[rustversion::any(not, not)] + | ^ diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/bad-version.rs b/third_party/crates/rustversion-1.0.23/tests/ui/bad-version.rs new file mode 100644 index 0000000..1fe8dba --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/bad-version.rs @@ -0,0 +1,7 @@ +#[rustversion::stable(nightly)] +struct S; + +#[rustversion::any(stable(nightly))] +struct S; + +fn main() {} diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/bad-version.stderr b/third_party/crates/rustversion-1.0.23/tests/ui/bad-version.stderr new file mode 100644 index 0000000..bf3f144 --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/bad-version.stderr @@ -0,0 +1,11 @@ +error: expected rustc release number, like 1.31 + --> tests/ui/bad-version.rs:1:23 + | +1 | #[rustversion::stable(nightly)] + | ^^^^^^^ + +error: expected rustc release number, like 1.31 + --> tests/ui/bad-version.rs:4:27 + | +4 | #[rustversion::any(stable(nightly))] + | ^^^^^^^ diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/const-not-fn.rs b/third_party/crates/rustversion-1.0.23/tests/ui/const-not-fn.rs new file mode 100644 index 0000000..215df7d --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/const-not-fn.rs @@ -0,0 +1,4 @@ +#[rustversion::attr(all(), const)] +pub struct S; + +fn main() {} diff --git a/third_party/crates/rustversion-1.0.23/tests/ui/const-not-fn.stderr b/third_party/crates/rustversion-1.0.23/tests/ui/const-not-fn.stderr new file mode 100644 index 0000000..d3cb4aa --- /dev/null +++ b/third_party/crates/rustversion-1.0.23/tests/ui/const-not-fn.stderr @@ -0,0 +1,5 @@ +error: only allowed on a fn item + --> tests/ui/const-not-fn.rs:1:28 + | +1 | #[rustversion::attr(all(), const)] + | ^^^^^ diff --git a/third_party/crates/syn-2.0.119/.cargo-checksum.json b/third_party/crates/syn-2.0.119/.cargo-checksum.json new file mode 100644 index 0000000..ac8f228 --- /dev/null +++ b/third_party/crates/syn-2.0.119/.cargo-checksum.json @@ -0,0 +1 @@ +{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{".cargo_vcs_info.json":"26614adfbc9a4a7647d51fece702d79773a0d9911a6641ffed7c935efd6cc496","Cargo.lock":"dce51a1b6a7689de31019fae3dbe661a82dd8307f09e96ffcdebfa9e3e93a5cb","Cargo.toml":"da8b83ac2c66a7b404dc55d88ae42592f70714823251f51b620ae349b246a18f","Cargo.toml.orig":"33d92998d299923965137d5acc6b1ea020d1707d62014d44f3a7e70701003b33","LICENSE-APACHE":"62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"4a050aa3ee831ad3d27ab15d9b612bb32712f86517fa2e6be0b1cac37137ebe5","benches/file.rs":"6f2ae7748d1576cff48e4ef55d4e87e2f5bb7898a36aa99ea6dd6ed0f72a4e3c","benches/rust.rs":"4ad250087795831d7c79b130d47927b5c733052c9659de039c08835b74be72b1","src/attr.rs":"2108cf48c98ed0a7833ec73b841fccc254c733453a7e69127b0255541bbc9572","src/bigint.rs":"21cf6cf0c8e175421a6579321878e140d6cd2c975e5bdac4eadde25907d7e879","src/buffer.rs":"5b4daf35958f18dd389bcd49be73087b2c6f1971412eac32c1b66968ec786912","src/classify.rs":"200e81f2245dc23621026849542d95d1e7c320c775362e080857673b51bc82b7","src/custom_keyword.rs":"322114e36ae43a2f8605506fb4568efdbc2986853e2fee74bd10a4ca0fb60c69","src/custom_punctuation.rs":"67134b7685ce89eddb7bf4ee3c9a48e5eb8ecbb1703cd49a01cbb115239fb814","src/data.rs":"68b36f6feca19ad9cd2780a29113f3ca8bdd21df3276c54b71595c8ed06cb774","src/derive.rs":"4927cfd64d2c61bb9ee8462b83d5bbce77e3f85c3a03a5565ecf6f08d55367c3","src/discouraged.rs":"d5538b42691e418faf3a0b5fb2b16813efc6a80ffb59dab58a0f170c4a71cbfb","src/drops.rs":"c48e679507424c08716b48256a06d59fe55e877de336c40f0df56cfd80e23826","src/error.rs":"34a045645056452a76862f08084ae9a92aa24a38daa25586e9ab12f5fa9c3b23","src/export.rs":"d9039ba7f1455089139cf8a6871159f0ec57949cc730b42fe3c318a8397d3368","src/expr.rs":"eb2b886403cfe0b3c27122a9013e882fe7194b16ec87202c239f2b3619c36d31","src/ext.rs":"8bc0d0ffab88c73d2305c9f588e6b5285501de8869308162f91019c83bc5bdf1","src/file.rs":"0cc23703e5c0478f14d8ec285ca5c925a0c556c686f4f5d0fc55e19fc03e33f6","src/fixup.rs":"7647cde30efdce96b1488ae805788c168d4499b464b7d421abc17ea8ffde66f2","src/gen/clone.rs":"7af00b0a240453d7aac178be1b6cdf9df3b33f1281da35e02f66ba7ba55d060c","src/gen/debug.rs":"7d249085b57e0f751f4e8b9b8fe38cd056a4d56b2f474d4b4f8ba3d489068322","src/gen/eq.rs":"fa2c783b0eb58af46d9a599cce0131379c616d396c8e1124970502673e9001a3","src/gen/fold.rs":"30a91e710427a15d197957d7471282f4a1231181736ed7a8686c9b0f27248e41","src/gen/hash.rs":"f4a58047eb829f16c9ce9db08b2e73c2cc2369275cc47ef32147365778de49ba","src/gen/token.css":"3a5882d0b3aefbf56ca5d08c28c117e25ac2327eadf7242202d906b2ddb2232e","src/gen/visit.rs":"fe1443aa7953eaca10d6bf982396e627e31ce6b8aea8eb7cf949e0adeea5badb","src/gen/visit_mut.rs":"a48310aa9580af2eff96ff872b3b5093656cf48ddc0c22c02ef33884e477d633","src/generics.rs":"70852a30bf23fe368d81b343ae69d0b4ed2231881a41ab781d9b9690004cb4bc","src/group.rs":"61e067094aa930b6550dc3717a5aaeef7dabfdcf1a177f83fb636598fa7e4e72","src/ident.rs":"4e50859365382708c2c09abfb88eac4b0d35c9b3f884e58cc22392d91be2a0e1","src/item.rs":"a4f290f4c31a357c09281102e4623193adc3cf1de75acb9775c6e89fbbcca3a2","src/lib.rs":"b97929744ac2a567f1aec3a0ed90a5e95ee23c9317a3c947c55e310f6f88b0cf","src/lifetime.rs":"5bd6d716e46d489f52f4ac3075710328cae5a01a0b37818bfe26f4293f7984c7","src/lit.rs":"3be54784de36d2d87eee1baec025e0f7126a7f8c9f9754460763886465803b47","src/lookahead.rs":"ea51bbf9a23d68238f8324a51675f5815fb3e5fc08c6bab93bc2648d3342c38f","src/mac.rs":"fdce8291f71adef3f69975f229156dca2309ca232ed943061afaf96220908ab8","src/macros.rs":"9809598f9d7e533c737c2a6df994b40b7c9ca4bbdf90bb2289da72a8065345dd","src/meta.rs":"d8703f8cbc5fd6465c367214eac38efca0f8388bdef8ac5e6f863835fbd3b30f","src/op.rs":"a61757370f802e44efa3c4a1057ae2cd26e64e273f7d76c06d5ffb49602319e2","src/parse.rs":"e692d6e895ac79a14680b89a7c29611995126953238176bdb50d2a3c2fb2d9e3","src/parse_macro_input.rs":"e4e22b63d0496d06a4ca17742a22467ed93f08a739081324773828bad63175ee","src/parse_quote.rs":"32fbdf959ee601cc2d184b6a358c542dbede0da5ad69d47588f7c405258a6dec","src/pat.rs":"67b875e34a3b24bf7a929bf607897c62dc55d25b02408359f58d7138ae8f08fd","src/path.rs":"fec69a4b50fe103d406b240af5724c8e9cf3583c392ab5a811743456db8e3f64","src/precedence.rs":"301f5cc51e1bbb35aeb64f6ed1b21d6f61d83f6c048aec9896976ee0011b948e","src/print.rs":"22910bf0521ab868ebd7c62601c55912d12cfb400c65723e08e5cfa3a2d111c0","src/punctuated.rs":"89b2666be8354215c63f8af7c92d6aed3a6d309cf384965ae360d351a5007afd","src/restriction.rs":"f5099d4cf5f9d61baace5b0e16f1631c72c15bba62529c7d4598565539d29f7d","src/scan_expr.rs":"1afda43a7f4e9d426859a90d8a62a58c6654f3be41bb274d4884a56ac08f8eee","src/sealed.rs":"6ece3b3dcb30f6bb98b93d83759ca7712ee8592bef9c0511141039c38765db0e","src/span.rs":"0a48e375e5c9768f6f64174a91ba6a255f4b021e2fb3548d8494e617f142601b","src/spanned.rs":"b53ce23f10935cf202da0f5a430e9feca9f4858bbd6efbe896acf17d3cfce540","src/stmt.rs":"ee269337cfb893af2601f175f0df0226c87dd8def4b84aaff991c3b9227e458d","src/thread.rs":"b91e33c563c2ea584fcd7c5454998303173f4227bdb19504776e61d09b60675c","src/token.rs":"b9cc5ef698d1f1760c9d4bcb28b814d0767ef2a9c38e8bc494a3e10297592e79","src/tt.rs":"d5b9df076504b1506e0b05dafb9006fd0772fdb8d4b8b8dcc315dbca3277331a","src/ty.rs":"61d64932b221e9a4add4ac3dd93fd6f52d32c81a0308f932eed5ca2f314bf7cb","src/verbatim.rs":"503055ba92ebb97da71a8a7cae1e6606969aadfb965b0d6c7e56693f9ef66b0a","src/whitespace.rs":"9cdcbfe9045b259046329a795bc1105ab5a871471a6d3f7318d275ee53f7a825","tests/common/eq.rs":"95996da870e4945addedff28c0992f2a85f633ed25b738c88525eea04c4c3669","tests/common/mod.rs":"b752aa8f1faf8c6abf1286a12fb50b6c257ec1889d81bcdb3dc3257134695a89","tests/common/parse.rs":"b1685e9e0f0569f48d66ddd92d787a12933a1d2f95e7c9ae974a3648763f989b","tests/common/visit.rs":"a260ecd2ce7853cd3644e19aba08e8d358a656fd3fb0f1287cea40c59c9e62c9","tests/debug/gen.rs":"2af05f2ecc74591f1e16e170e6a8e6d538fa1dd2a76f48a4f57f3c269baea11d","tests/debug/mod.rs":"1259df940bbcaa968a837e402d6853f2efa38d2260e306d42f17f9e8ef74fae5","tests/macros/mod.rs":"d2294a79e341c623ae671dd363e99965d78dda7f340b0cc038267207adfacae2","tests/regression.rs":"e9565ea0efecb4136f099164ffcfa26e1996b0a27fb9c6659e90ad9bdd42e7b6","tests/regression/issue1108.rs":"f32db35244a674e22ff824ca9e5bbec2184e287b59f022db68c418b5878a2edc","tests/regression/issue1235.rs":"a2266b10c3f7c7af5734817ab0a3e8b309b51e7d177b63f26e67e6b744d280b0","tests/repo/mod.rs":"9a4cf4d45401adfd2d12da7d08474d505dd313ae405de2c9191a133c5beee1c4","tests/repo/progress.rs":"11219e4ae5e0175d5ddc743907a57b8c4e750e709687c3df53728d6af994a86f","tests/snapshot/mod.rs":"4a101272c5abe6ca9f3501e0cacacee9a0ccf7ca773348a239e5b046d0316a7e","tests/test_asyncness.rs":"971d560d927d5a8494eaa7fce8f0d062d6971c17c4c464fcfc31570572b7d3d7","tests/test_attribute.rs":"8a4429b7cfe2360bb73beae54a62ae3255ebbd5181467a8608d6f858c2711728","tests/test_derive_input.rs":"c8f5dbac6482dadd0fab30d0b1fe3254869256c48ea68ea484cad7f7406c8568","tests/test_expr.rs":"055cb9b33a5bb6ed5dc67491e6f1ae794a53a5a091245debd464ef57144f5edb","tests/test_generics.rs":"5c9b34813a442311fdca36b56736dd15760e7f6d6a668f82673c21711e1aacd8","tests/test_grouping.rs":"fe3de6e8824f0722ab6450c6dfc374f6e0f8fe75c87c4dd56b2cb00a2197ed58","tests/test_ident.rs":"d5850e817720e774cd397a46dbc5298c57933823c18e20805e84503fc9387e8f","tests/test_item.rs":"2e290b6be323bec34e0334cad0664fafabee15e2635cb64867086eb76278b446","tests/test_lit.rs":"4130efa425d14ed3ad9a1c2a00ef4b29782c9d1cf9e29ff9dddd3b23b2e3ddee","tests/test_meta.rs":"5b0fdee0decbd07476c9673403a662de385901b4bf60600c26ac879893f5bf9c","tests/test_parse_buffer.rs":"1c8cbbf457603bd0a83348079591f843f7ef0533a6275a0c76533e2d07dd321a","tests/test_parse_quote.rs":"85d90d2d51b82aab7c30159dd884f26c592ddb28ed31ef2baf371ee31349694c","tests/test_parse_stream.rs":"b6b533432173123d6d01d8d2cb33714bc50b30b16ffbb6116f93937221ad4594","tests/test_pat.rs":"dafa3e1f51812e8c852dc5210640a4adf6fff7cd0a0790ee17d2c4c115321846","tests/test_path.rs":"7a6763a262c41a9522068887702fe7cd4ff72b07da5253ac47761d73315b021d","tests/test_precedence.rs":"f0654b7bf85e2852f7229b14aaba7fcae87d9691f6fa91ab315bc6b0249ffccf","tests/test_punctuated.rs":"efed2c281b6965d71b065c7606631ba1989af6e7b5f5d1ca1033f8b968dc076c","tests/test_receiver.rs":"2053028236f95f3cb508ebf2eb606df43cae4f9f4dd27823661459ff6c54a39c","tests/test_round_trip.rs":"aecd7b37cf937f005ee0858f38482782341d7fa037ec8f15b3e8b26bfd6f6104","tests/test_shebang.rs":"9bc24b1ee2947b06a279d2ed40039cb45bba6caf7cd40530d93f7e2355de53c6","tests/test_size.rs":"03efaf829b80b7db1f831474c1d3ce268914fc499d0e2a7eea03cad04a482974","tests/test_stmt.rs":"b3c120059d7b56388963b85234feb2e4d379e32a0bf7f29b6683eca000dd3919","tests/test_token_trees.rs":"c30b921a96739c9334ec2bdd06552729891e0251b9d8fbdf0b8f5cc897babee5","tests/test_ty.rs":"9bb5f632941451ca6b200100310b55e62a9956190df3efe28b80d42843e75362","tests/test_unparenthesize.rs":"e5c047819afd5f70fde1bdd095467b1291d0854641f21e8183e50919986d8ce7","tests/test_visibility.rs":"7d05f05b0782976369d21477ac9f4d35a7c7f36faa42127e3a9c12ada270baf8","tests/zzz_stable.rs":"2a862e59cb446235ed99aec0e6ada8e16d3ecc30229b29d825b7c0bbc2602989"},"package":"872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"} \ No newline at end of file diff --git a/third_party/crates/syn-2.0.119/.cargo_vcs_info.json b/third_party/crates/syn-2.0.119/.cargo_vcs_info.json new file mode 100644 index 0000000..4baa697 --- /dev/null +++ b/third_party/crates/syn-2.0.119/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "3295f9e9841785ac88a5e558c884854d5fb7d67f" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/third_party/crates/syn-2.0.119/BUILD.bazel b/third_party/crates/syn-2.0.119/BUILD.bazel new file mode 100644 index 0000000..452e795 --- /dev/null +++ b/third_party/crates/syn-2.0.119/BUILD.bazel @@ -0,0 +1,125 @@ +############################################################################### +# @generated +# DO NOT MODIFY: This file is auto-generated by a crate_universe tool. To +# regenerate this file, run the following: +# +# bazel run @@//third_party:vendor_crates +############################################################################### + +load("@rules_rust//cargo:defs.bzl", "cargo_toml_env_vars") +load("@rules_rust//rust:defs.bzl", "rust_library") + +package(default_visibility = ["//visibility:public"]) + +cargo_toml_env_vars( + name = "cargo_toml_env_vars", + src = "Cargo.toml", +) + +rust_library( + name = "syn", + srcs = glob( + include = ["**/*.rs"], + allow_empty = False, + ), + compile_data = glob( + include = ["**"], + allow_empty = True, + exclude = [ + "**/* *", + ".tmp_git_root/**/*", + "BUILD", + "BUILD.bazel", + "WORKSPACE", + "WORKSPACE.bazel", + ], + ), + crate_features = [ + "clone-impls", + "default", + "derive", + "extra-traits", + "full", + "parsing", + "printing", + "proc-macro", + ], + crate_root = "src/lib.rs", + edition = "2021", + rustc_env_files = [ + ":cargo_toml_env_vars", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-bazel", + "crate-name=syn", + "manual", + "noclippy", + "norustfmt", + ], + target_compatible_with = select({ + "@rules_rust//rust/platform:aarch64-apple-darwin": [], + "@rules_rust//rust/platform:aarch64-apple-ios": [], + "@rules_rust//rust/platform:aarch64-apple-ios-macabi": [], + "@rules_rust//rust/platform:aarch64-apple-ios-sim": [], + "@rules_rust//rust/platform:aarch64-linux-android": [], + "@rules_rust//rust/platform:aarch64-pc-windows-msvc": [], + "@rules_rust//rust/platform:aarch64-unknown-fuchsia": [], + "@rules_rust//rust/platform:aarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:aarch64-unknown-none": [], + "@rules_rust//rust/platform:aarch64-unknown-nto-qnx710": [], + "@rules_rust//rust/platform:aarch64-unknown-uefi": [], + "@rules_rust//rust/platform:arm-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:arm-unknown-linux-musleabi": [], + "@rules_rust//rust/platform:armv7-linux-androideabi": [], + "@rules_rust//rust/platform:armv7-unknown-linux-gnueabi": [], + "@rules_rust//rust/platform:i686-apple-darwin": [], + "@rules_rust//rust/platform:i686-linux-android": [], + "@rules_rust//rust/platform:i686-pc-windows-msvc": [], + "@rules_rust//rust/platform:i686-unknown-freebsd": [], + "@rules_rust//rust/platform:i686-unknown-linux-gnu": [], + "@rules_rust//rust/platform:loongarch64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:mips-unknown-linux-gnu": [], + "@rules_rust//rust/platform:powerpc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv32imac-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv32imc-unknown-none-elf": [], + "@rules_rust//rust/platform:riscv64gc-unknown-linux-gnu": [], + "@rules_rust//rust/platform:riscv64gc-unknown-none-elf": [], + "@rules_rust//rust/platform:s390x-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:sparc64-unknown-netbsd": [], + "@rules_rust//rust/platform:sparc64-unknown-openbsd": [], + "@rules_rust//rust/platform:thumbv6m-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabi": [], + "@rules_rust//rust/platform:thumbv7em-none-eabihf": [], + "@rules_rust//rust/platform:thumbv7m-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabi": [], + "@rules_rust//rust/platform:thumbv8m.main-none-eabihf": [], + "@rules_rust//rust/platform:wasm32-unknown-emscripten": [], + "@rules_rust//rust/platform:wasm32-unknown-unknown": [], + "@rules_rust//rust/platform:wasm32-wasip1": [], + "@rules_rust//rust/platform:wasm32-wasip1-threads": [], + "@rules_rust//rust/platform:wasm32-wasip2": [], + "@rules_rust//rust/platform:x86_64-apple-darwin": [], + "@rules_rust//rust/platform:x86_64-apple-ios": [], + "@rules_rust//rust/platform:x86_64-apple-ios-macabi": [], + "@rules_rust//rust/platform:x86_64-linux-android": [], + "@rules_rust//rust/platform:x86_64-pc-windows-msvc": [], + "@rules_rust//rust/platform:x86_64-unknown-freebsd": [], + "@rules_rust//rust/platform:x86_64-unknown-fuchsia": [], + "@rules_rust//rust/platform:x86_64-unknown-linux-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-nixos-gnu": [], + "@rules_rust//rust/platform:x86_64-unknown-none": [], + "@rules_rust//rust/platform:x86_64-unknown-uefi": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + version = "2.0.119", + deps = [ + "//third_party/crates/proc-macro2-1.0.107:proc_macro2", + "//third_party/crates/quote-1.0.47:quote", + "//third_party/crates/unicode-ident-1.0.26:unicode_ident", + ], +) diff --git a/third_party/crates/syn-2.0.119/Cargo.lock b/third_party/crates/syn-2.0.119/Cargo.lock new file mode 100644 index 0000000..56da3b1 --- /dev/null +++ b/third_party/crates/syn-2.0.119/Cargo.lock @@ -0,0 +1,1912 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "automod" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a273e731a7fb8c2268fd2932c9e9a3469f4fd171d85d070d2687190229653bd3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "aws-lc-rs" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "console" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "insta" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" +dependencies = [ + "console", + "once_cell", + "similar", + "tempfile", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.118", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +dependencies = [ + "anyhow", + "automod", + "flate2", + "insta", + "proc-macro2", + "quote", + "rayon", + "ref-cast", + "reqwest", + "rustversion", + "syn-test-suite", + "tar", + "termcolor", + "unicode-ident", + "walkdir", +] + +[[package]] +name = "syn-test-suite" +version = "0.0.0+test" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d661992f60e67c8bdd9a7d6360d30d1301f5783abf7d59933844f656762eb5" + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.118", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] diff --git a/third_party/crates/syn-2.0.119/Cargo.toml b/third_party/crates/syn-2.0.119/Cargo.toml new file mode 100644 index 0000000..02741c0 --- /dev/null +++ b/third_party/crates/syn-2.0.119/Cargo.toml @@ -0,0 +1,272 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.71" +name = "syn" +version = "2.0.119" +authors = ["David Tolnay <dtolnay@gmail.com>"] +build = false +include = [ + "/benches/**", + "/Cargo.toml", + "/LICENSE-APACHE", + "/LICENSE-MIT", + "/README.md", + "/src/**", + "/tests/**", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Parser for Rust source code" +documentation = "https://docs.rs/syn" +readme = "README.md" +keywords = [ + "macros", + "syn", +] +categories = [ + "development-tools::procedural-macro-helpers", + "parser-implementations", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/syn" + +[package.metadata.docs.rs] +all-features = true +targets = ["x86_64-unknown-linux-gnu"] +rustdoc-args = [ + "--generate-link-to-definition", + "--generate-macro-expansion", + "--extend-css=src/gen/token.css", + "--extern-html-root-url=core=https://doc.rust-lang.org", + "--extern-html-root-url=alloc=https://doc.rust-lang.org", + "--extern-html-root-url=std=https://doc.rust-lang.org", + "--extern-html-root-url=proc_macro=https://doc.rust-lang.org", +] + +[package.metadata.playground] +features = [ + "full", + "visit", + "visit-mut", + "fold", + "extra-traits", +] + +[features] +clone-impls = [] +default = [ + "derive", + "parsing", + "printing", + "clone-impls", + "proc-macro", +] +derive = [] +extra-traits = [] +fold = [] +full = [] +parsing = [] +printing = ["dep:quote"] +proc-macro = [ + "proc-macro2/proc-macro", + "quote?/proc-macro", +] +test = ["syn-test-suite/all-features"] +visit = [] +visit-mut = [] + +[lib] +name = "syn" +path = "src/lib.rs" + +[[test]] +name = "regression" +path = "tests/regression.rs" + +[[test]] +name = "test_asyncness" +path = "tests/test_asyncness.rs" + +[[test]] +name = "test_attribute" +path = "tests/test_attribute.rs" + +[[test]] +name = "test_derive_input" +path = "tests/test_derive_input.rs" + +[[test]] +name = "test_expr" +path = "tests/test_expr.rs" + +[[test]] +name = "test_generics" +path = "tests/test_generics.rs" + +[[test]] +name = "test_grouping" +path = "tests/test_grouping.rs" + +[[test]] +name = "test_ident" +path = "tests/test_ident.rs" + +[[test]] +name = "test_item" +path = "tests/test_item.rs" + +[[test]] +name = "test_lit" +path = "tests/test_lit.rs" + +[[test]] +name = "test_meta" +path = "tests/test_meta.rs" + +[[test]] +name = "test_parse_buffer" +path = "tests/test_parse_buffer.rs" + +[[test]] +name = "test_parse_quote" +path = "tests/test_parse_quote.rs" + +[[test]] +name = "test_parse_stream" +path = "tests/test_parse_stream.rs" + +[[test]] +name = "test_pat" +path = "tests/test_pat.rs" + +[[test]] +name = "test_path" +path = "tests/test_path.rs" + +[[test]] +name = "test_precedence" +path = "tests/test_precedence.rs" + +[[test]] +name = "test_punctuated" +path = "tests/test_punctuated.rs" + +[[test]] +name = "test_receiver" +path = "tests/test_receiver.rs" + +[[test]] +name = "test_round_trip" +path = "tests/test_round_trip.rs" + +[[test]] +name = "test_shebang" +path = "tests/test_shebang.rs" + +[[test]] +name = "test_size" +path = "tests/test_size.rs" + +[[test]] +name = "test_stmt" +path = "tests/test_stmt.rs" + +[[test]] +name = "test_token_trees" +path = "tests/test_token_trees.rs" + +[[test]] +name = "test_ty" +path = "tests/test_ty.rs" + +[[test]] +name = "test_unparenthesize" +path = "tests/test_unparenthesize.rs" + +[[test]] +name = "test_visibility" +path = "tests/test_visibility.rs" + +[[test]] +name = "zzz_stable" +path = "tests/zzz_stable.rs" + +[[bench]] +name = "file" +path = "benches/file.rs" +required-features = [ + "full", + "parsing", +] + +[[bench]] +name = "rust" +path = "benches/rust.rs" +harness = false +required-features = [ + "full", + "parsing", +] + +[dependencies.proc-macro2] +version = "1.0.91" +default-features = false + +[dependencies.quote] +version = "1.0.35" +optional = true +default-features = false + +[dependencies.unicode-ident] +version = "1" + +[dev-dependencies.anyhow] +version = "1" + +[dev-dependencies.automod] +version = "1" + +[dev-dependencies.insta] +version = "1" + +[dev-dependencies.ref-cast] +version = "1" + +[dev-dependencies.rustversion] +version = "1" + +[dev-dependencies.syn-test-suite] +version = "0" + +[dev-dependencies.termcolor] +version = "1" + +[target."cfg(not(miri))".dev-dependencies.flate2] +version = "1" + +[target."cfg(not(miri))".dev-dependencies.rayon] +version = "1" + +[target."cfg(not(miri))".dev-dependencies.reqwest] +version = "0.13" +features = ["blocking"] + +[target."cfg(not(miri))".dev-dependencies.tar] +version = "0.4.16" + +[target."cfg(not(miri))".dev-dependencies.walkdir] +version = "2.3.2" diff --git a/third_party/crates/syn-2.0.119/Cargo.toml.orig b/third_party/crates/syn-2.0.119/Cargo.toml.orig new file mode 100644 index 0000000..a2a8d29 --- /dev/null +++ b/third_party/crates/syn-2.0.119/Cargo.toml.orig @@ -0,0 +1,95 @@ +[package] +name = "syn" +version = "2.0.119" +authors = ["David Tolnay <dtolnay@gmail.com>"] +categories = ["development-tools::procedural-macro-helpers", "parser-implementations"] +description = "Parser for Rust source code" +documentation = "https://docs.rs/syn" +edition = "2021" +include = [ + "/benches/**", + "/Cargo.toml", + "/LICENSE-APACHE", + "/LICENSE-MIT", + "/README.md", + "/src/**", + "/tests/**", +] +keywords = ["macros", "syn"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/syn" +rust-version = "1.71" + +[features] +default = ["derive", "parsing", "printing", "clone-impls", "proc-macro"] +derive = [] +full = [] +parsing = [] +printing = ["dep:quote"] +visit = [] +visit-mut = [] +fold = [] +clone-impls = [] +extra-traits = [] +proc-macro = ["proc-macro2/proc-macro", "quote?/proc-macro"] +test = ["syn-test-suite/all-features"] + +[dependencies] +proc-macro2 = { version = "1.0.91", default-features = false } +quote = { version = "1.0.35", optional = true, default-features = false } +unicode-ident = "1" + +[dev-dependencies] +anyhow = "1" +automod = "1" +insta = "1" +ref-cast = "1" +rustversion = "1" +syn-test-suite = { version = "0", path = "tests/features" } +termcolor = "1" + +[target.'cfg(not(miri))'.dev-dependencies] +flate2 = "1" +rayon = "1" +reqwest = { version = "0.13", features = ["blocking"] } +tar = "0.4.16" +walkdir = "2.3.2" + +[[bench]] +name = "rust" +harness = false +required-features = ["full", "parsing"] + +[[bench]] +name = "file" +required-features = ["full", "parsing"] + +[package.metadata.docs.rs] +all-features = true +targets = ["x86_64-unknown-linux-gnu"] +rustdoc-args = [ + "--generate-link-to-definition", + "--generate-macro-expansion", + "--extend-css=src/gen/token.css", + "--extern-html-root-url=core=https://doc.rust-lang.org", + "--extern-html-root-url=alloc=https://doc.rust-lang.org", + "--extern-html-root-url=std=https://doc.rust-lang.org", + "--extern-html-root-url=proc_macro=https://doc.rust-lang.org", +] + +[package.metadata.playground] +features = ["full", "visit", "visit-mut", "fold", "extra-traits"] + +[workspace] +members = [ + "dev", + "examples/dump-syntax", + "examples/heapsize/example", + "examples/heapsize/heapsize", + "examples/heapsize/heapsize_derive", + "examples/lazy-static/example", + "examples/lazy-static/lazy-static", + "examples/trace-var/example", + "examples/trace-var/trace-var", + "tests/features", +] diff --git a/third_party/crates/syn-2.0.119/LICENSE-APACHE b/third_party/crates/syn-2.0.119/LICENSE-APACHE new file mode 100644 index 0000000..1b5ec8b --- /dev/null +++ b/third_party/crates/syn-2.0.119/LICENSE-APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/third_party/crates/syn-2.0.119/LICENSE-MIT b/third_party/crates/syn-2.0.119/LICENSE-MIT new file mode 100644 index 0000000..31aa793 --- /dev/null +++ b/third_party/crates/syn-2.0.119/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third_party/crates/syn-2.0.119/README.md b/third_party/crates/syn-2.0.119/README.md new file mode 100644 index 0000000..7ac4b2d --- /dev/null +++ b/third_party/crates/syn-2.0.119/README.md @@ -0,0 +1,282 @@ +Parser for Rust source code +=========================== + +[<img alt="github" src="https://img.shields.io/badge/github-dtolnay/syn-8da0cb?style=for-the-badge&labelColor=555555&logo=github" height="20">](https://github.com/dtolnay/syn) +[<img alt="crates.io" src="https://img.shields.io/crates/v/syn.svg?style=for-the-badge&color=fc8d62&logo=rust" height="20">](https://crates.io/crates/syn) +[<img alt="docs.rs" src="https://img.shields.io/badge/docs.rs-syn-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs" height="20">](https://docs.rs/syn) +[<img alt="build status" src="https://img.shields.io/github/actions/workflow/status/dtolnay/syn/ci.yml?branch=master&style=for-the-badge" height="20">](https://github.com/dtolnay/syn/actions?query=branch%3Amaster) + +Syn is a parsing library for parsing a stream of Rust tokens into a syntax tree +of Rust source code. + +Currently this library is geared toward use in Rust procedural macros, but +contains some APIs that may be useful more generally. + +- **Data structures** — Syn provides a complete syntax tree that can represent + any valid Rust source code. The syntax tree is rooted at [`syn::File`] which + represents a full source file, but there are other entry points that may be + useful to procedural macros including [`syn::Item`], [`syn::Expr`] and + [`syn::Type`]. + +- **Derives** — Of particular interest to derive macros is [`syn::DeriveInput`] + which is any of the three legal input items to a derive macro. An example + below shows using this type in a library that can derive implementations of a + user-defined trait. + +- **Parsing** — Parsing in Syn is built around [parser functions] with the + signature `fn(ParseStream) -> Result<T>`. Every syntax tree node defined by + Syn is individually parsable and may be used as a building block for custom + syntaxes, or you may dream up your own brand new syntax without involving any + of our syntax tree types. + +- **Location information** — Every token parsed by Syn is associated with a + `Span` that tracks line and column information back to the source of that + token. These spans allow a procedural macro to display detailed error messages + pointing to all the right places in the user's code. There is an example of + this below. + +- **Feature flags** — Functionality is aggressively feature gated so your + procedural macros enable only what they need, and do not pay in compile time + for all the rest. + +[`syn::File`]: https://docs.rs/syn/2.0/syn/struct.File.html +[`syn::Item`]: https://docs.rs/syn/2.0/syn/enum.Item.html +[`syn::Expr`]: https://docs.rs/syn/2.0/syn/enum.Expr.html +[`syn::Type`]: https://docs.rs/syn/2.0/syn/enum.Type.html +[`syn::DeriveInput`]: https://docs.rs/syn/2.0/syn/struct.DeriveInput.html +[parser functions]: https://docs.rs/syn/2.0/syn/parse/index.html + +[*Release notes*](https://github.com/dtolnay/syn/releases) + +<br> + +## Resources + +The best way to learn about procedural macros is by writing some. Consider +working through [this procedural macro workshop][workshop] to get familiar with +the different types of procedural macros. The workshop contains relevant links +into the Syn documentation as you work through each project. + +[workshop]: https://github.com/dtolnay/proc-macro-workshop + +<br> + +## Example of a derive macro + +The canonical derive macro using Syn looks like this. We write an ordinary Rust +function tagged with a `proc_macro_derive` attribute and the name of the trait +we are deriving. Any time that derive appears in the user's code, the Rust +compiler passes their data structure as tokens into our macro. We get to execute +arbitrary Rust code to figure out what to do with those tokens, then hand some +tokens back to the compiler to compile into the user's crate. + +[`TokenStream`]: https://doc.rust-lang.org/proc_macro/struct.TokenStream.html + +```toml +[dependencies] +syn = "2.0" +quote = "1.0" + +[lib] +proc-macro = true +``` + +```rust +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, DeriveInput}; + +#[proc_macro_derive(MyMacro)] +pub fn my_macro(input: TokenStream) -> TokenStream { + // Parse the input tokens into a syntax tree + let input = parse_macro_input!(input as DeriveInput); + + // Build the output, possibly using quasi-quotation + let expanded = quote! { + // ... + }; + + // Hand the output tokens back to the compiler + TokenStream::from(expanded) +} +``` + +The [`heapsize`] example directory shows a complete working implementation of a +derive macro. The example derives a `HeapSize` trait which computes an estimate +of the amount of heap memory owned by a value. + +[`heapsize`]: examples/heapsize + +```rust +pub trait HeapSize { + /// Total number of bytes of heap memory owned by `self`. + fn heap_size_of_children(&self) -> usize; +} +``` + +The derive macro allows users to write `#[derive(HeapSize)]` on data structures +in their program. + +```rust +#[derive(HeapSize)] +struct Demo<'a, T: ?Sized> { + a: Box<T>, + b: u8, + c: &'a str, + d: String, +} +``` + +<br> + +## Spans and error reporting + +The token-based procedural macro API provides great control over where the +compiler's error messages are displayed in user code. Consider the error the +user sees if one of their field types does not implement `HeapSize`. + +```rust +#[derive(HeapSize)] +struct Broken { + ok: String, + bad: std::thread::Thread, +} +``` + +By tracking span information all the way through the expansion of a procedural +macro as shown in the `heapsize` example, token-based macros in Syn are able to +trigger errors that directly pinpoint the source of the problem. + +```console +error[E0277]: the trait bound `std::thread::Thread: HeapSize` is not satisfied + --> src/main.rs:7:5 + | +7 | bad: std::thread::Thread, + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `HeapSize` is not implemented for `std::thread::Thread` +``` + +<br> + +## Parsing a custom syntax + +The [`lazy-static`] example directory shows the implementation of a +`functionlike!(...)` procedural macro in which the input tokens are parsed using +Syn's parsing API. + +[`lazy-static`]: examples/lazy-static + +The example reimplements the popular `lazy_static` crate from crates.io as a +procedural macro. + +```rust +lazy_static! { + static ref USERNAME: Regex = Regex::new("^[a-z0-9_-]{3,16}$").unwrap(); +} +``` + +The implementation shows how to trigger custom warnings and error messages on +the macro input. + +```console +warning: come on, pick a more creative name + --> src/main.rs:10:16 + | +10 | static ref FOO: String = "lazy_static".to_owned(); + | ^^^ +``` + +<br> + +## Testing + +When testing macros, we often care not just that the macro can be used +successfully but also that when the macro is provided with invalid input it +produces maximally helpful error messages. Consider using the [`trybuild`] crate +to write tests for errors that are emitted by your macro or errors detected by +the Rust compiler in the expanded code following misuse of the macro. Such tests +help avoid regressions from later refactors that mistakenly make an error no +longer trigger or be less helpful than it used to be. + +[`trybuild`]: https://github.com/dtolnay/trybuild + +<br> + +## Debugging + +When developing a procedural macro it can be helpful to look at what the +generated code looks like. Use `cargo rustc -- -Zunstable-options +-Zunpretty=expanded` or the [`cargo expand`] subcommand. + +[`cargo expand`]: https://github.com/dtolnay/cargo-expand + +To show the expanded code for some crate that uses your procedural macro, run +`cargo expand` from that crate. To show the expanded code for one of your own +test cases, run `cargo expand --test the_test_case` where the last argument is +the name of the test file without the `.rs` extension. + +This write-up by Brandon W Maister discusses debugging in more detail: +[Debugging Rust's new Custom Derive system][debugging]. + +[debugging]: https://quodlibetor.github.io/posts/debugging-rusts-new-custom-derive-system/ + +<br> + +## Optional features + +Syn puts a lot of functionality behind optional features in order to optimize +compile time for the most common use cases. The following features are +available. + +- **`derive`** *(enabled by default)* — Data structures for representing the + possible input to a derive macro, including structs and enums and types. +- **`full`** — Data structures for representing the syntax tree of all valid + Rust source code, including items and expressions. +- **`parsing`** *(enabled by default)* — Ability to parse input tokens into a + syntax tree node of a chosen type. +- **`printing`** *(enabled by default)* — Ability to print a syntax tree node as + tokens of Rust source code. +- **`visit`** — Trait for traversing a syntax tree. +- **`visit-mut`** — Trait for traversing and mutating in place a syntax tree. +- **`fold`** — Trait for transforming an owned syntax tree. +- **`clone-impls`** *(enabled by default)* — Clone impls for all syntax tree + types. +- **`extra-traits`** — Debug, Eq, PartialEq, Hash impls for all syntax tree + types. +- **`proc-macro`** *(enabled by default)* — Runtime dependency on the dynamic + library libproc_macro from rustc toolchain. + +<br> + +## Proc macro shim + +Syn operates on the token representation provided by the [proc-macro2] crate +from crates.io rather than using the compiler's built in proc-macro crate +directly. This enables code using Syn to execute outside of the context of a +procedural macro, such as in unit tests or build.rs, and we avoid needing +incompatible ecosystems for proc macros vs non-macro use cases. + +In general all of your code should be written against proc-macro2 rather than +proc-macro. The one exception is in the signatures of procedural macro entry +points, which are required by the language to use `proc_macro::TokenStream`. + +The proc-macro2 crate will automatically detect and use the compiler's data +structures when a procedural macro is active. + +[proc-macro2]: https://docs.rs/proc-macro2/1.0/proc_macro2/ + +<br> + +#### License + +<sup> +Licensed under either of <a href="LICENSE-APACHE">Apache License, Version +2.0</a> or <a href="LICENSE-MIT">MIT license</a> at your option. +</sup> + +<br> + +<sub> +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this crate by you, as defined in the Apache-2.0 license, shall +be dual licensed as above, without any additional terms or conditions. +</sub> diff --git a/third_party/crates/syn-2.0.119/benches/file.rs b/third_party/crates/syn-2.0.119/benches/file.rs new file mode 100644 index 0000000..6167488 --- /dev/null +++ b/third_party/crates/syn-2.0.119/benches/file.rs @@ -0,0 +1,59 @@ +// $ cargo bench --features full,test --bench file + +#![feature(rustc_private, test)] +#![recursion_limit = "1024"] +#![allow( + clippy::elidable_lifetime_names, + clippy::items_after_statements, + clippy::manual_let_else, + clippy::match_like_matches_macro, + clippy::missing_panics_doc, + clippy::must_use_candidate, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +extern crate test; + +#[macro_use] +#[path = "../tests/macros/mod.rs"] +mod macros; + +#[allow(dead_code)] +#[path = "../tests/repo/mod.rs"] +mod repo; + +use proc_macro2::{Span, TokenStream}; +use std::fs; +use std::str::FromStr; +use syn::parse::{ParseStream, Parser}; +use test::Bencher; + +const FILE: &str = "tests/rust/library/core/src/str/mod.rs"; + +fn get_tokens() -> TokenStream { + repo::clone_rust(); + let content = fs::read_to_string(FILE).unwrap(); + TokenStream::from_str(&content).unwrap() +} + +#[bench] +fn baseline(b: &mut Bencher) { + let tokens = get_tokens(); + b.iter(|| drop(tokens.clone())); +} + +#[bench] +fn create_token_buffer(b: &mut Bencher) { + let tokens = get_tokens(); + fn immediate_fail(_input: ParseStream) -> syn::Result<()> { + Err(syn::Error::new(Span::call_site(), "")) + } + b.iter(|| immediate_fail.parse2(tokens.clone())); +} + +#[bench] +fn parse_file(b: &mut Bencher) { + let tokens = get_tokens(); + b.iter(|| syn::parse2::<syn::File>(tokens.clone())); +} diff --git a/third_party/crates/syn-2.0.119/benches/rust.rs b/third_party/crates/syn-2.0.119/benches/rust.rs new file mode 100644 index 0000000..c0e82c6 --- /dev/null +++ b/third_party/crates/syn-2.0.119/benches/rust.rs @@ -0,0 +1,189 @@ +// $ cargo bench --features full,test --bench rust +// +// Syn only, useful for profiling: +// $ RUSTFLAGS='--cfg syn_only' cargo build --release --features full,test --bench rust + +#![cfg_attr(not(syn_only), feature(rustc_private))] +#![recursion_limit = "1024"] +#![allow( + clippy::arc_with_non_send_sync, + clippy::cast_lossless, + clippy::elidable_lifetime_names, + clippy::let_underscore_untyped, + clippy::manual_let_else, + clippy::match_like_matches_macro, + clippy::needless_lifetimes, + clippy::uninlined_format_args, + clippy::unnecessary_wraps +)] + +#[macro_use] +#[path = "../tests/macros/mod.rs"] +mod macros; + +#[allow(dead_code)] +#[path = "../tests/repo/mod.rs"] +mod repo; + +use std::fs; +use std::path::Path; +use std::time::{Duration, Instant}; + +#[cfg(not(syn_only))] +mod tokenstream_parse { + use proc_macro2::TokenStream; + use std::path::Path; + use std::str::FromStr; + + pub fn bench(_path: &Path, content: &str) -> Result<(), ()> { + TokenStream::from_str(content).map(drop).map_err(drop) + } +} + +mod syn_parse { + use std::path::Path; + + pub fn bench(_path: &Path, content: &str) -> Result<(), ()> { + syn::parse_file(content).map(drop).map_err(drop) + } +} + +#[cfg(not(syn_only))] +mod librustc_parse { + extern crate rustc_data_structures; + extern crate rustc_driver; + extern crate rustc_error_messages; + extern crate rustc_errors; + extern crate rustc_parse; + extern crate rustc_session; + extern crate rustc_span; + + use crate::repo; + use rustc_errors::emitter::Emitter; + use rustc_errors::{DiagCtxt, DiagInner}; + use rustc_parse::lexer::StripTokens; + use rustc_session::parse::ParseSess; + use rustc_span::source_map::{FilePathMapping, SourceMap}; + use rustc_span::FileName; + use std::path::Path; + use std::sync::Arc; + + pub fn bench(path: &Path, content: &str) -> Result<(), ()> { + struct SilentEmitter; + + impl Emitter for SilentEmitter { + fn emit_diagnostic(&mut self, _diag: DiagInner) {} + fn source_map(&self) -> Option<&SourceMap> { + None + } + } + + let edition = repo::edition(path).parse().unwrap(); + rustc_span::create_session_if_not_set_then(edition, |_| { + let source_map = Arc::new(SourceMap::new(FilePathMapping::empty())); + let emitter = Box::new(SilentEmitter); + let handler = DiagCtxt::new(emitter); + let sess = ParseSess::with_dcx(handler, source_map); + let name = FileName::Custom("bench".to_owned()); + let mut parser = rustc_parse::new_parser_from_source_str( + &sess, + name, + content.to_owned(), + StripTokens::ShebangAndFrontmatter, + ) + .unwrap(); + if let Err(diagnostic) = parser.parse_crate_mod() { + diagnostic.cancel(); + return Err(()); + } + Ok(()) + }) + } +} + +#[cfg(not(syn_only))] +mod read_from_disk { + use std::path::Path; + + pub fn bench(_path: &Path, content: &str) -> Result<(), ()> { + let _ = content; + Ok(()) + } +} + +fn exec(mut codepath: impl FnMut(&Path, &str) -> Result<(), ()>) -> Duration { + let begin = Instant::now(); + let mut success = 0; + let mut total = 0; + + ["tests/rust/compiler", "tests/rust/library"] + .iter() + .flat_map(|dir| { + walkdir::WalkDir::new(dir) + .into_iter() + .filter_entry(repo::base_dir_filter) + }) + .for_each(|entry| { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_dir() { + return; + } + let content = fs::read_to_string(path).unwrap(); + let ok = codepath(path, &content).is_ok(); + success += ok as usize; + total += 1; + if !ok { + eprintln!("FAIL {}", path.display()); + } + }); + + assert_eq!(success, total); + begin.elapsed() +} + +fn main() { + repo::clone_rust(); + + macro_rules! testcases { + ($($(#[$cfg:meta])* $name:ident,)*) => { + [ + $( + $(#[$cfg])* + (stringify!($name), $name::bench as fn(&Path, &str) -> Result<(), ()>), + )* + ] + }; + } + + #[cfg(not(syn_only))] + { + let mut lines = 0; + let mut files = 0; + exec(|_path, content| { + lines += content.lines().count(); + files += 1; + Ok(()) + }); + eprintln!("\n{} lines in {} files", lines, files); + } + + for (name, f) in testcases!( + #[cfg(not(syn_only))] + read_from_disk, + #[cfg(not(syn_only))] + tokenstream_parse, + syn_parse, + #[cfg(not(syn_only))] + librustc_parse, + ) { + eprint!("{:20}", format!("{}:", name)); + let elapsed = exec(f); + eprintln!( + "elapsed={}.{:03}s", + elapsed.as_secs(), + elapsed.subsec_millis(), + ); + } + eprintln!(); +} diff --git a/third_party/crates/syn-2.0.119/src/attr.rs b/third_party/crates/syn-2.0.119/src/attr.rs new file mode 100644 index 0000000..5e2d7a7 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/attr.rs @@ -0,0 +1,841 @@ +#[cfg(feature = "parsing")] +use crate::error::Error; +#[cfg(feature = "parsing")] +use crate::error::Result; +use crate::expr::Expr; +use crate::mac::MacroDelimiter; +#[cfg(feature = "parsing")] +use crate::meta::{self, ParseNestedMeta}; +#[cfg(feature = "parsing")] +use crate::parse::{Parse, ParseStream, Parser}; +use crate::path::Path; +use crate::token; +#[cfg(feature = "parsing")] +use alloc::format; +#[cfg(feature = "parsing")] +use alloc::vec::Vec; +#[cfg(feature = "printing")] +use core::iter; +#[cfg(feature = "printing")] +use core::slice; +use proc_macro2::TokenStream; + +ast_struct! { + /// An attribute, like `#[repr(transparent)]`. + /// + /// <br> + /// + /// # Syntax + /// + /// Rust has six types of attributes. + /// + /// - Outer attributes like `#[repr(transparent)]`. These appear outside or + /// in front of the item they describe. + /// + /// - Inner attributes like `#![feature(proc_macro)]`. These appear inside + /// of the item they describe, usually a module. + /// + /// - Outer one-line doc comments like `/// Example`. + /// + /// - Inner one-line doc comments like `//! Please file an issue`. + /// + /// - Outer documentation blocks `/** Example */`. + /// + /// - Inner documentation blocks `/*! Please file an issue */`. + /// + /// The `style` field of type `AttrStyle` distinguishes whether an attribute + /// is outer or inner. + /// + /// Every attribute has a `path` that indicates the intended interpretation + /// of the rest of the attribute's contents. The path and the optional + /// additional contents are represented together in the `meta` field of the + /// attribute in three possible varieties: + /// + /// - Meta::Path — attributes whose information content conveys just a + /// path, for example the `#[test]` attribute. + /// + /// - Meta::List — attributes that carry arbitrary tokens after the + /// path, surrounded by a delimiter (parenthesis, bracket, or brace). For + /// example `#[derive(Copy)]` or `#[precondition(x < 5)]`. + /// + /// - Meta::NameValue — attributes with an `=` sign after the path, + /// followed by a Rust expression. For example `#[path = + /// "sys/windows.rs"]`. + /// + /// All doc comments are represented in the NameValue style with a path of + /// "doc", as this is how they are processed by the compiler and by + /// `macro_rules!` macros. + /// + /// ```text + /// #[derive(Copy, Clone)] + /// ~~~~~~Path + /// ^^^^^^^^^^^^^^^^^^^Meta::List + /// + /// #[path = "sys/windows.rs"] + /// ~~~~Path + /// ^^^^^^^^^^^^^^^^^^^^^^^Meta::NameValue + /// + /// #[test] + /// ^^^^Meta::Path + /// ``` + /// + /// <br> + /// + /// # Parsing from tokens to Attribute + /// + /// This type does not implement the [`Parse`] trait and thus cannot be + /// parsed directly by [`ParseStream::parse`]. Instead use + /// [`ParseStream::call`] with one of the two parser functions + /// [`Attribute::parse_outer`] or [`Attribute::parse_inner`] depending on + /// which you intend to parse. + /// + /// [`Parse`]: crate::parse::Parse + /// [`ParseStream::parse`]: crate::parse::ParseBuffer::parse + /// [`ParseStream::call`]: crate::parse::ParseBuffer::call + /// + /// ``` + /// use syn::{Attribute, Ident, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parses a unit struct with attributes. + /// // + /// // #[path = "s.tmpl"] + /// // struct S; + /// struct UnitStruct { + /// attrs: Vec<Attribute>, + /// struct_token: Token![struct], + /// name: Ident, + /// semi_token: Token![;], + /// } + /// + /// impl Parse for UnitStruct { + /// fn parse(input: ParseStream) -> Result<Self> { + /// Ok(UnitStruct { + /// attrs: input.call(Attribute::parse_outer)?, + /// struct_token: input.parse()?, + /// name: input.parse()?, + /// semi_token: input.parse()?, + /// }) + /// } + /// } + /// ``` + /// + /// <p><br></p> + /// + /// # Parsing from Attribute to structured arguments + /// + /// The grammar of attributes in Rust is very flexible, which makes the + /// syntax tree not that useful on its own. In particular, arguments of the + /// `Meta::List` variety of attribute are held in an arbitrary `tokens: + /// TokenStream`. Macros are expected to check the `path` of the attribute, + /// decide whether they recognize it, and then parse the remaining tokens + /// according to whatever grammar they wish to require for that kind of + /// attribute. Use [`parse_args()`] to parse those tokens into the expected + /// data structure. + /// + /// [`parse_args()`]: Attribute::parse_args + /// + /// <p><br></p> + /// + /// # Doc comments + /// + /// The compiler transforms doc comments, such as `/// comment` and `/*! + /// comment */`, into attributes before macros are expanded. Each comment is + /// expanded into an attribute of the form `#[doc = r"comment"]`. + /// + /// As an example, the following `mod` items are expanded identically: + /// + /// ``` + /// # use syn::{ItemMod, parse_quote}; + /// let doc: ItemMod = parse_quote! { + /// /// Single line doc comments + /// /// We write so many! + /// /** + /// * Multi-line comments... + /// * May span many lines + /// */ + /// mod example { + /// //! Of course, they can be inner too + /// /*! And fit in a single line */ + /// } + /// }; + /// let attr: ItemMod = parse_quote! { + /// #[doc = r" Single line doc comments"] + /// #[doc = r" We write so many!"] + /// #[doc = r" + /// * Multi-line comments... + /// * May span many lines + /// "] + /// mod example { + /// #![doc = r" Of course, they can be inner too"] + /// #![doc = r" And fit in a single line "] + /// } + /// }; + /// assert_eq!(doc, attr); + /// ``` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Attribute { + pub pound_token: Token![#], + pub style: AttrStyle, + pub bracket_token: token::Bracket, + pub meta: Meta, + } +} + +impl Attribute { + /// Returns the path that identifies the interpretation of this attribute. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + self.meta.path() + } + + /// Parse the arguments to the attribute as a syntax tree. + /// + /// This is similar to pulling out the `TokenStream` from `Meta::List` and + /// doing `syn::parse2::<T>(meta_list.tokens)`, except that using + /// `parse_args` the error message has a more useful span when `tokens` is + /// empty. + /// + /// The surrounding delimiters are *not* included in the input to the + /// parser. + /// + /// ```text + /// #[my_attr(value < 5)] + /// ^^^^^^^^^ what gets parsed + /// ``` + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute, Expr}; + /// + /// let attr: Attribute = parse_quote! { + /// #[precondition(value < 5)] + /// }; + /// + /// if attr.path().is_ident("precondition") { + /// let precondition: Expr = attr.parse_args()?; + /// // ... + /// } + /// # anyhow::Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args<T: Parse>(&self) -> Result<T> { + self.parse_args_with(T::parse) + } + + /// Parse the arguments to the attribute using the given parser. + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute}; + /// + /// let attr: Attribute = parse_quote! { + /// #[inception { #[brrrrrrraaaaawwwwrwrrrmrmrmmrmrmmmmm] }] + /// }; + /// + /// let bwom = attr.parse_args_with(Attribute::parse_outer)?; + /// + /// // Attribute does not have a Parse impl, so we couldn't directly do: + /// // let bwom: Attribute = attr.parse_args()?; + /// # anyhow::Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args_with<F: Parser>(&self, parser: F) -> Result<F::Output> { + match &self.meta { + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new( + meta.eq_token.span, + format_args!( + "expected parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(&meta.path), + ), + )), + Meta::List(meta) => meta.parse_args_with(parser), + } + } + + /// Parse the arguments to the attribute, expecting it to follow the + /// conventional structure used by most of Rust's built-in attributes. + /// + /// The [*Meta Item Attribute Syntax*][syntax] section in the Rust reference + /// explains the convention in more detail. Not all attributes follow this + /// convention, so [`parse_args()`][Self::parse_args] is available if you + /// need to parse arbitrarily goofy attribute syntax. + /// + /// [syntax]: https://doc.rust-lang.org/reference/attributes.html#meta-item-attribute-syntax + /// + /// # Example + /// + /// We'll parse a struct, and then parse some of Rust's `#[repr]` attribute + /// syntax. + /// + /// ``` + /// use syn::{parenthesized, parse_quote, token, ItemStruct, LitInt}; + /// + /// let input: ItemStruct = parse_quote! { + /// #[repr(C, align(4))] + /// pub struct MyStruct(u16, u32); + /// }; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::<usize>; + /// let mut repr_packed = None::<usize>; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// attr.parse_nested_meta(|meta| { + /// // #[repr(C)] + /// if meta.path.is_ident("C") { + /// repr_c = true; + /// return Ok(()); + /// } + /// + /// // #[repr(transparent)] + /// if meta.path.is_ident("transparent") { + /// repr_transparent = true; + /// return Ok(()); + /// } + /// + /// // #[repr(align(N))] + /// if meta.path.is_ident("align") { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// return Ok(()); + /// } + /// + /// // #[repr(packed)] or #[repr(packed(N))], omitted N means 1 + /// if meta.path.is_ident("packed") { + /// if meta.input.peek(token::Paren) { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_packed = Some(n); + /// } else { + /// repr_packed = Some(1); + /// } + /// return Ok(()); + /// } + /// + /// Err(meta.error("unrecognized repr")) + /// })?; + /// } + /// } + /// # anyhow::Ok(()) + /// ``` + /// + /// # Alternatives + /// + /// In some cases, for attributes which have nested layers of structured + /// content, the following less flexible approach might be more convenient: + /// + /// ``` + /// # use syn::{parse_quote, ItemStruct}; + /// # + /// # let input: ItemStruct = parse_quote! { + /// # #[repr(C, align(4))] + /// # pub struct MyStruct(u16, u32); + /// # }; + /// # + /// use syn::punctuated::Punctuated; + /// use syn::{parenthesized, token, Error, LitInt, Meta, Token}; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::<usize>; + /// let mut repr_packed = None::<usize>; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// let nested = attr.parse_args_with(Punctuated::<Meta, Token![,]>::parse_terminated)?; + /// for meta in nested { + /// match meta { + /// // #[repr(C)] + /// Meta::Path(path) if path.is_ident("C") => { + /// repr_c = true; + /// } + /// + /// // #[repr(align(N))] + /// Meta::List(meta) if meta.path.is_ident("align") => { + /// let lit: LitInt = meta.parse_args()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// } + /// + /// /* ... */ + /// + /// _ => { + /// return Err(Error::new_spanned(meta, "unrecognized repr")); + /// } + /// } + /// } + /// } + /// } + /// # Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } + + /// Parses zero or more outer attributes from the stream. + /// + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_outer(input: ParseStream) -> Result<Vec<Self>> { + let mut attrs = Vec::new(); + while input.peek(Token![#]) { + attrs.push(input.call(parsing::single_parse_outer)?); + } + Ok(attrs) + } + + /// Parses zero or more inner attributes from the stream. + /// + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_inner(input: ParseStream) -> Result<Vec<Self>> { + let mut attrs = Vec::new(); + parsing::parse_inner(input, &mut attrs)?; + Ok(attrs) + } +} + +ast_enum! { + /// Distinguishes between attributes that decorate an item and attributes + /// that are contained within an item. + /// + /// # Outer attributes + /// + /// - `#[repr(transparent)]` + /// - `/// # Example` + /// - `/** Please file an issue */` + /// + /// # Inner attributes + /// + /// - `#![feature(proc_macro)]` + /// - `//! # Example` + /// - `/*! Please file an issue */` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum AttrStyle { + Outer, + Inner(Token![!]), + } +} + +ast_enum! { + /// Content of a compile-time structured attribute. + /// + /// ## Path + /// + /// A meta path is like the `test` in `#[test]`. + /// + /// ## List + /// + /// A meta list is like the `derive(Copy)` in `#[derive(Copy)]`. + /// + /// ## NameValue + /// + /// A name-value meta is like the `path = "..."` in `#[path = + /// "sys/windows.rs"]`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Meta { + Path(Path), + + /// A structured list within an attribute, like `derive(Copy, Clone)`. + List(MetaList), + + /// A name-value pair within an attribute, like `feature = "nightly"`. + NameValue(MetaNameValue), + } +} + +ast_struct! { + /// A structured list within an attribute, like `derive(Copy, Clone)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct MetaList { + pub path: Path, + pub delimiter: MacroDelimiter, + pub tokens: TokenStream, + } +} + +ast_struct! { + /// A name-value pair within an attribute, like `feature = "nightly"`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct MetaNameValue { + pub path: Path, + pub eq_token: Token![=], + pub value: Expr, + } +} + +impl Meta { + /// Returns the path that begins this structured meta item. + /// + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + match self { + Meta::Path(path) => path, + Meta::List(meta) => &meta.path, + Meta::NameValue(meta) => &meta.path, + } + } + + /// Error if this is a `Meta::List` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_path_only(&self) -> Result<&Path> { + let error_span = match self { + Meta::Path(path) => return Ok(path), + Meta::List(meta) => meta.delimiter.span().open(), + Meta::NameValue(meta) => meta.eq_token.span, + }; + Err(Error::new(error_span, "unexpected token in attribute")) + } + + /// Error if this is a `Meta::Path` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_list(&self) -> Result<&MetaList> { + match self { + Meta::List(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: `{}(...)`", + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new(meta.eq_token.span, "expected `(`")), + } + } + + /// Error if this is a `Meta::Path` or `Meta::List`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_name_value(&self) -> Result<&MetaNameValue> { + match self { + Meta::NameValue(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected a value for this attribute: `{} = ...`", + parsing::DisplayPath(path), + ), + )), + Meta::List(meta) => Err(Error::new(meta.delimiter.span().open(), "expected `=`")), + } + } +} + +impl MetaList { + /// See [`Attribute::parse_args`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args<T: Parse>(&self) -> Result<T> { + self.parse_args_with(T::parse) + } + + /// See [`Attribute::parse_args_with`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_args_with<F: Parser>(&self, parser: F) -> Result<F::Output> { + let scope = self.delimiter.span().close(); + crate::parse::parse_scoped(parser, scope, self.tokens.clone()) + } + + /// See [`Attribute::parse_nested_meta`]. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } +} + +#[cfg(feature = "printing")] +pub(crate) trait FilterAttrs<'a> { + type Ret: Iterator<Item = &'a Attribute>; + + fn outer(self) -> Self::Ret; + #[cfg(feature = "full")] + fn inner(self) -> Self::Ret; +} + +#[cfg(feature = "printing")] +impl<'a> FilterAttrs<'a> for &'a [Attribute] { + type Ret = iter::Filter<slice::Iter<'a, Attribute>, fn(&&Attribute) -> bool>; + + fn outer(self) -> Self::Ret { + fn is_outer(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Outer => true, + AttrStyle::Inner(_) => false, + } + } + self.iter().filter(is_outer) + } + + #[cfg(feature = "full")] + fn inner(self) -> Self::Ret { + fn is_inner(attr: &&Attribute) -> bool { + match attr.style { + AttrStyle::Inner(_) => true, + AttrStyle::Outer => false, + } + } + self.iter().filter(is_inner) + } +} + +impl From<Path> for Meta { + fn from(meta: Path) -> Meta { + Meta::Path(meta) + } +} + +impl From<MetaList> for Meta { + fn from(meta: MetaList) -> Meta { + Meta::List(meta) + } +} + +impl From<MetaNameValue> for Meta { + fn from(meta: MetaNameValue) -> Meta { + Meta::NameValue(meta) + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + use crate::error::Result; + use crate::expr::{Expr, ExprLit}; + use crate::lit::Lit; + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::path::Path; + use crate::{mac, token}; + use alloc::vec::Vec; + use core::fmt::{self, Display}; + use proc_macro2::Ident; + + pub(crate) fn parse_inner(input: ParseStream, attrs: &mut Vec<Attribute>) -> Result<()> { + while input.peek(Token![#]) && input.peek2(Token![!]) { + attrs.push(input.call(single_parse_inner)?); + } + Ok(()) + } + + pub(crate) fn single_parse_inner(input: ParseStream) -> Result<Attribute> { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Inner(input.parse()?), + bracket_token: bracketed!(content in input), + meta: content.parse()?, + }) + } + + pub(crate) fn single_parse_outer(input: ParseStream) -> Result<Attribute> { + let content; + Ok(Attribute { + pound_token: input.parse()?, + style: AttrStyle::Outer, + bracket_token: bracketed!(content in input), + meta: content.parse()?, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Meta { + fn parse(input: ParseStream) -> Result<Self> { + let path = parse_outermost_meta_path(input)?; + parse_meta_after_path(path, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for MetaList { + fn parse(input: ParseStream) -> Result<Self> { + let path = parse_outermost_meta_path(input)?; + parse_meta_list_after_path(path, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for MetaNameValue { + fn parse(input: ParseStream) -> Result<Self> { + let path = parse_outermost_meta_path(input)?; + parse_meta_name_value_after_path(path, input) + } + } + + // Unlike meta::parse_meta_path which accepts arbitrary keywords in the path, + // only the `unsafe` keyword is accepted as an attribute's outermost path. + fn parse_outermost_meta_path(input: ParseStream) -> Result<Path> { + if input.peek(Token![unsafe]) { + let unsafe_token: Token![unsafe] = input.parse()?; + Ok(Path::from(Ident::new("unsafe", unsafe_token.span))) + } else { + Path::parse_mod_style(input) + } + } + + pub(crate) fn parse_meta_after_path(path: Path, input: ParseStream) -> Result<Meta> { + if input.peek(token::Paren) || input.peek(token::Bracket) || input.peek(token::Brace) { + parse_meta_list_after_path(path, input).map(Meta::List) + } else if input.peek(Token![=]) && !input.peek(Token![==]) && !input.peek(Token![=>]) { + parse_meta_name_value_after_path(path, input).map(Meta::NameValue) + } else { + Ok(Meta::Path(path)) + } + } + + fn parse_meta_list_after_path(path: Path, input: ParseStream) -> Result<MetaList> { + let (delimiter, tokens) = mac::parse_delimiter(input)?; + Ok(MetaList { + path, + delimiter, + tokens, + }) + } + + fn parse_meta_name_value_after_path(path: Path, input: ParseStream) -> Result<MetaNameValue> { + let eq_token: Token![=] = input.parse()?; + let ahead = input.fork(); + let lit: Option<Lit> = ahead.parse()?; + let value = if let (Some(lit), true) = (lit, ahead.is_empty()) { + input.advance_to(&ahead); + Expr::Lit(ExprLit { + attrs: Vec::new(), + lit, + }) + } else if input.peek(Token![#]) && input.peek2(token::Bracket) { + return Err(input.error("unexpected attribute inside of attribute")); + } else { + input.parse()? + }; + Ok(MetaNameValue { + path, + eq_token, + value, + }) + } + + pub(super) struct DisplayAttrStyle<'a>(pub &'a AttrStyle); + + impl<'a> Display for DisplayAttrStyle<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(match self.0 { + AttrStyle::Outer => "#", + AttrStyle::Inner(_) => "#!", + }) + } + } + + pub(super) struct DisplayPath<'a>(pub &'a Path); + + impl<'a> Display for DisplayPath<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + for (i, segment) in self.0.segments.iter().enumerate() { + if i > 0 || self.0.leading_colon.is_some() { + formatter.write_str("::")?; + } + write!(formatter, "{}", segment.ident)?; + } + Ok(()) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + use crate::path; + use crate::path::printing::PathStyle; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Attribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pound_token.to_tokens(tokens); + if let AttrStyle::Inner(b) = &self.style { + b.to_tokens(tokens); + } + self.bracket_token.surround(tokens, |tokens| { + self.meta.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Meta { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Meta::Path(path) => path::printing::print_path(tokens, path, PathStyle::Mod), + Meta::List(meta_list) => meta_list.to_tokens(tokens), + Meta::NameValue(meta_name_value) => meta_name_value.to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for MetaList { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.delimiter.surround(tokens, self.tokens.clone()); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for MetaNameValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.eq_token.to_tokens(tokens); + self.value.to_tokens(tokens); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/bigint.rs b/third_party/crates/syn-2.0.119/src/bigint.rs new file mode 100644 index 0000000..d32298b --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/bigint.rs @@ -0,0 +1,68 @@ +use alloc::string::String; +use alloc::vec::Vec; +use core::ops::{AddAssign, MulAssign}; + +// For implementing base10_digits() accessor on LitInt. +pub(crate) struct BigInt { + digits: Vec<u8>, +} + +impl BigInt { + pub(crate) fn new() -> Self { + BigInt { digits: Vec::new() } + } + + pub(crate) fn to_string(&self) -> String { + let mut repr = String::with_capacity(self.digits.len()); + + let mut has_nonzero = false; + for digit in self.digits.iter().rev() { + has_nonzero |= *digit != 0; + if has_nonzero { + repr.push((*digit + b'0') as char); + } + } + + if repr.is_empty() { + repr.push('0'); + } + + repr + } + + fn reserve_two_digits(&mut self) { + let len = self.digits.len(); + let desired = + len + !self.digits.ends_with(&[0, 0]) as usize + !self.digits.ends_with(&[0]) as usize; + self.digits.resize(desired, 0); + } +} + +impl AddAssign<u8> for BigInt { + // Assumes increment <16. + fn add_assign(&mut self, mut increment: u8) { + self.reserve_two_digits(); + + let mut i = 0; + while increment > 0 { + let sum = self.digits[i] + increment; + self.digits[i] = sum % 10; + increment = sum / 10; + i += 1; + } + } +} + +impl MulAssign<u8> for BigInt { + // Assumes base <=16. + fn mul_assign(&mut self, base: u8) { + self.reserve_two_digits(); + + let mut carry = 0; + for digit in &mut self.digits { + let prod = *digit * base + carry; + *digit = prod % 10; + carry = prod / 10; + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/buffer.rs b/third_party/crates/syn-2.0.119/src/buffer.rs new file mode 100644 index 0000000..277093d --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/buffer.rs @@ -0,0 +1,437 @@ +//! A stably addressed token buffer supporting efficient traversal based on a +//! cheaply copyable cursor. + +// This module is heavily commented as it contains most of the unsafe code in +// Syn, and caution should be used when editing it. The public-facing interface +// is 100% safe but the implementation is fragile internally. + +use crate::ext::TokenStreamExt as _; +use crate::Lifetime; +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::cmp::Ordering; +use core::marker::PhantomData; +use core::ptr; +use proc_macro2::extra::DelimSpan; +use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; + +/// Internal type which is used instead of `TokenTree` to represent a token tree +/// within a `TokenBuffer`. +enum Entry { + // Mimicking types from proc-macro. + // Group entries contain the offset to the matching End entry. + Group(Group, usize), + Ident(Ident), + Punct(Punct), + Literal(Literal), + // End entries contain the offset (negative) to the start of the buffer, and + // offset (negative) to the matching Group entry. + End(isize, isize), +} + +/// A buffer that can be efficiently traversed multiple times, unlike +/// `TokenStream` which requires a deep copy in order to traverse more than +/// once. +pub struct TokenBuffer { + // NOTE: Do not implement clone on this - while the current design could be + // cloned, other designs which could be desirable may not be cloneable. + entries: Box<[Entry]>, +} + +impl TokenBuffer { + fn recursive_new(entries: &mut Vec<Entry>, stream: TokenStream) { + for tt in stream { + match tt { + TokenTree::Ident(ident) => entries.push(Entry::Ident(ident)), + TokenTree::Punct(punct) => entries.push(Entry::Punct(punct)), + TokenTree::Literal(literal) => entries.push(Entry::Literal(literal)), + TokenTree::Group(group) => { + let group_start_index = entries.len(); + entries.push(Entry::End(0, 0)); // we replace this below + Self::recursive_new(entries, group.stream()); + let group_end_index = entries.len(); + let group_offset = group_end_index - group_start_index; + entries.push(Entry::End( + -(group_end_index as isize), + -(group_offset as isize), + )); + entries[group_start_index] = Entry::Group(group, group_offset); + } + } + } + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `proc_macro::TokenStream`. + #[cfg(feature = "proc-macro")] + #[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] + pub fn new(stream: proc_macro::TokenStream) -> Self { + Self::new2(stream.into()) + } + + /// Creates a `TokenBuffer` containing all the tokens from the input + /// `proc_macro2::TokenStream`. + pub fn new2(stream: TokenStream) -> Self { + let mut entries = Vec::new(); + Self::recursive_new(&mut entries, stream); + entries.push(Entry::End(-(entries.len() as isize), 0)); + Self { + entries: entries.into_boxed_slice(), + } + } + + /// Creates a cursor referencing the first token in the buffer and able to + /// traverse until the end of the buffer. + pub fn begin(&self) -> Cursor { + let ptr = self.entries.as_ptr(); + unsafe { Cursor::create(ptr, ptr.add(self.entries.len() - 1)) } + } +} + +/// A cheaply copyable cursor into a `TokenBuffer`. +/// +/// This cursor holds a shared reference into the immutable data which is used +/// internally to represent a `TokenStream`, and can be efficiently manipulated +/// and copied around. +/// +/// An empty `Cursor` can be created directly, or one may create a `TokenBuffer` +/// object and get a cursor to its first token with `begin()`. +pub struct Cursor<'a> { + // The current entry which the `Cursor` is pointing at. + ptr: *const Entry, + // This is the only `Entry::End` object which this cursor is allowed to + // point at. All other `End` objects are skipped over in `Cursor::create`. + scope: *const Entry, + // Cursor is covariant in 'a. This field ensures that our pointers are still + // valid. + marker: PhantomData<&'a Entry>, +} + +impl<'a> Cursor<'a> { + /// Creates a cursor referencing a static empty TokenStream. + pub fn empty() -> Self { + // It's safe in this situation for us to put an `Entry` object in global + // storage, despite it not actually being safe to send across threads + // (`Ident` is a reference into a thread-local table). This is because + // this entry never includes a `Ident` object. + // + // This wrapper struct allows us to break the rules and put a `Sync` + // object in global storage. + struct UnsafeSyncEntry(Entry); + unsafe impl Sync for UnsafeSyncEntry {} + static EMPTY_ENTRY: UnsafeSyncEntry = UnsafeSyncEntry(Entry::End(0, 0)); + + Cursor { + ptr: &EMPTY_ENTRY.0, + scope: &EMPTY_ENTRY.0, + marker: PhantomData, + } + } + + /// This create method intelligently exits non-explicitly-entered + /// `None`-delimited scopes when the cursor reaches the end of them, + /// allowing for them to be treated transparently. + unsafe fn create(mut ptr: *const Entry, scope: *const Entry) -> Self { + // NOTE: If we're looking at a `End`, we want to advance the cursor + // past it, unless `ptr == scope`, which means that we're at the edge of + // our cursor's scope. We should only have `ptr != scope` at the exit + // from None-delimited groups entered with `ignore_none`. + while let Entry::End(..) = unsafe { &*ptr } { + if ptr::eq(ptr, scope) { + break; + } + ptr = unsafe { ptr.add(1) }; + } + + Cursor { + ptr, + scope, + marker: PhantomData, + } + } + + /// Get the current entry. + fn entry(self) -> &'a Entry { + unsafe { &*self.ptr } + } + + /// Bump the cursor to point at the next token after the current one. This + /// is undefined behavior if the cursor is currently looking at an + /// `Entry::End`. + /// + /// If the cursor is looking at an `Entry::Group`, the bumped cursor will + /// point at the first token in the group (with the same scope end). + unsafe fn bump_ignore_group(self) -> Cursor<'a> { + unsafe { Cursor::create(self.ptr.add(1), self.scope) } + } + + /// While the cursor is looking at a `None`-delimited group, move it to look + /// at the first token inside instead. If the group is empty, this will move + /// the cursor past the `None`-delimited group. + /// + /// WARNING: This mutates its argument. + fn ignore_none(&mut self) { + while let Entry::Group(group, _) = self.entry() { + if group.delimiter() == Delimiter::None { + unsafe { *self = self.bump_ignore_group() }; + } else { + break; + } + } + } + + /// Checks whether the cursor is currently pointing at the end of its valid + /// scope. + pub fn eof(self) -> bool { + // We're at eof if we're at the end of our scope. + ptr::eq(self.ptr, self.scope) + } + + /// If the cursor is pointing at a `Ident`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Ident(ident) => Some((ident.clone(), unsafe { self.bump_ignore_group() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Punct`, returns it along with a cursor + /// pointing at the next `TokenTree`. + pub fn punct(mut self) -> Option<(Punct, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(punct) if punct.as_char() != '\'' => { + Some((punct.clone(), unsafe { self.bump_ignore_group() })) + } + _ => None, + } + } + + /// If the cursor is pointing at a `Literal`, return it along with a cursor + /// pointing at the next `TokenTree`. + pub fn literal(mut self) -> Option<(Literal, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Literal(literal) => Some((literal.clone(), unsafe { self.bump_ignore_group() })), + _ => None, + } + } + + /// If the cursor is pointing at a `Lifetime`, returns it along with a + /// cursor pointing at the next `TokenTree`. + pub fn lifetime(mut self) -> Option<(Lifetime, Cursor<'a>)> { + self.ignore_none(); + match self.entry() { + Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { + let next = unsafe { self.bump_ignore_group() }; + let (ident, rest) = next.ident()?; + let lifetime = Lifetime { + apostrophe: punct.span(), + ident, + }; + Some((lifetime, rest)) + } + _ => None, + } + } + + /// If the cursor is pointing at a `Group` with the given delimiter, returns + /// a cursor into that group and one pointing to the next `TokenTree`. + pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, DelimSpan, Cursor<'a>)> { + // If we're not trying to enter a none-delimited group, we want to + // ignore them. We have to make sure to _not_ ignore them when we want + // to enter them, of course. For obvious reasons. + if delim != Delimiter::None { + self.ignore_none(); + } + + if let Entry::Group(group, end_offset) = self.entry() { + if group.delimiter() == delim { + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, span, after_group)); + } + } + + None + } + + /// If the cursor is pointing at a `Group`, returns a cursor into the group + /// and one pointing to the next `TokenTree`. + pub fn any_group(self) -> Option<(Cursor<'a>, Delimiter, DelimSpan, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let delimiter = group.delimiter(); + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, delimiter, span, after_group)); + } + + None + } + + pub(crate) fn any_group_token(self) -> Option<(Group, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((group.clone(), after_group)); + } + + None + } + + /// Copies all remaining tokens visible from this cursor into a + /// `TokenStream`. + pub fn token_stream(self) -> TokenStream { + let mut tokens = TokenStream::new(); + let mut cursor = self; + while let Some((tt, rest)) = cursor.token_tree() { + tokens.append(tt); + cursor = rest; + } + tokens + } + + /// If the cursor is pointing at a `TokenTree`, returns it along with a + /// cursor pointing at the next `TokenTree`. + /// + /// Returns `None` if the cursor has reached the end of its stream. + /// + /// This method does not treat `None`-delimited groups as transparent, and + /// will return a `Group(None, ..)` if the cursor is looking at one. + pub fn token_tree(self) -> Option<(TokenTree, Cursor<'a>)> { + let (tree, len) = match self.entry() { + Entry::Group(group, end_offset) => (group.clone().into(), *end_offset), + Entry::Literal(literal) => (literal.clone().into(), 1), + Entry::Ident(ident) => (ident.clone().into(), 1), + Entry::Punct(punct) => (punct.clone().into(), 1), + Entry::End(..) => return None, + }; + + let rest = unsafe { Cursor::create(self.ptr.add(len), self.scope) }; + Some((tree, rest)) + } + + /// Returns the `Span` of the current token, or `Span::call_site()` if this + /// cursor points to eof. + pub fn span(mut self) -> Span { + match self.entry() { + Entry::Group(group, _) => group.span(), + Entry::Literal(literal) => literal.span(), + Entry::Ident(ident) => ident.span(), + Entry::Punct(punct) => punct.span(), + Entry::End(_, offset) => { + self.ptr = unsafe { self.ptr.offset(*offset) }; + if let Entry::Group(group, _) = self.entry() { + group.span_close() + } else { + Span::call_site() + } + } + } + } + + /// Returns the `Span` of the token immediately prior to the position of + /// this cursor, or of the current token if there is no previous one. + #[cfg(any(feature = "full", feature = "derive"))] + pub(crate) fn prev_span(mut self) -> Span { + if start_of_buffer(self) < self.ptr { + self.ptr = unsafe { self.ptr.sub(1) }; + } + self.span() + } + + /// Skip over the next token that is not a None-delimited group, without + /// cloning it. Returns `None` if this cursor points to eof. + /// + /// This method treats `'lifetimes` as a single token. + pub(crate) fn skip(mut self) -> Option<Cursor<'a>> { + self.ignore_none(); + + let len = match self.entry() { + Entry::End(..) => return None, + + // Treat lifetimes as a single tt for the purposes of 'skip'. + Entry::Punct(punct) if punct.as_char() == '\'' && punct.spacing() == Spacing::Joint => { + match unsafe { &*self.ptr.add(1) } { + Entry::Ident(_) => 2, + _ => 1, + } + } + + Entry::Group(_, end_offset) => *end_offset, + _ => 1, + }; + + Some(unsafe { Cursor::create(self.ptr.add(len), self.scope) }) + } + + pub(crate) fn scope_delimiter(self) -> Delimiter { + match unsafe { &*self.scope } { + Entry::End(_, offset) => match unsafe { &*self.scope.offset(*offset) } { + Entry::Group(group, _) => group.delimiter(), + _ => Delimiter::None, + }, + _ => unreachable!(), + } + } +} + +impl<'a> Copy for Cursor<'a> {} + +impl<'a> Clone for Cursor<'a> { + fn clone(&self) -> Self { + *self + } +} + +impl<'a> Eq for Cursor<'a> {} + +impl<'a> PartialEq for Cursor<'a> { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self.ptr, other.ptr) + } +} + +impl<'a> PartialOrd for Cursor<'a> { + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + if same_buffer(*self, *other) { + Some(cmp_assuming_same_buffer(*self, *other)) + } else { + None + } + } +} + +pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool { + ptr::eq(a.scope, b.scope) +} + +pub(crate) fn same_buffer(a: Cursor, b: Cursor) -> bool { + ptr::eq(start_of_buffer(a), start_of_buffer(b)) +} + +fn start_of_buffer(cursor: Cursor) -> *const Entry { + unsafe { + match &*cursor.scope { + Entry::End(offset, _) => cursor.scope.offset(*offset), + _ => unreachable!(), + } + } +} + +pub(crate) fn cmp_assuming_same_buffer(a: Cursor, b: Cursor) -> Ordering { + a.ptr.cmp(&b.ptr) +} + +pub(crate) fn open_span_of_group(cursor: Cursor) -> Span { + match cursor.entry() { + Entry::Group(group, _) => group.span_open(), + _ => cursor.span(), + } +} diff --git a/third_party/crates/syn-2.0.119/src/classify.rs b/third_party/crates/syn-2.0.119/src/classify.rs new file mode 100644 index 0000000..6d5edbd --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/classify.rs @@ -0,0 +1,311 @@ +#[cfg(feature = "full")] +use crate::expr::Expr; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::generics::TypeParamBound; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::path::{Path, PathArguments}; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::punctuated::Punctuated; +#[cfg(any(feature = "printing", feature = "full"))] +use crate::ty::{ReturnType, Type}; +#[cfg(any(feature = "printing", feature = "full"))] +use core::ops::ControlFlow; +#[cfg(feature = "full")] +use proc_macro2::{Delimiter, TokenStream, TokenTree}; + +#[cfg(feature = "full")] +pub(crate) fn requires_semi_to_be_stmt(expr: &Expr) -> bool { + match expr { + Expr::Macro(expr) => !expr.mac.delimiter.is_brace(), + _ => requires_comma_to_be_match_arm(expr), + } +} + +#[cfg(feature = "full")] +pub(crate) fn requires_comma_to_be_match_arm(expr: &Expr) -> bool { + match expr { + Expr::If(_) + | Expr::Match(_) + | Expr::Block(_) | Expr::Unsafe(_) // both under ExprKind::Block in rustc + | Expr::While(_) + | Expr::Loop(_) + | Expr::ForLoop(_) + | Expr::TryBlock(_) + | Expr::Const(_) => false, + + Expr::Array(_) + | Expr::Assign(_) + | Expr::Async(_) + | Expr::Await(_) + | Expr::Binary(_) + | Expr::Break(_) + | Expr::Call(_) + | Expr::Cast(_) + | Expr::Closure(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::Group(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Macro(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Range(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::Try(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Yield(_) + | Expr::Verbatim(_) => true, + } +} + +#[cfg(feature = "printing")] +pub(crate) fn trailing_unparameterized_path(mut ty: &Type) -> bool { + loop { + match ty { + Type::BareFn(t) => match &t.output { + ReturnType::Default => return false, + ReturnType::Type(_, ret) => ty = ret, + }, + Type::ImplTrait(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + Type::Path(t) => match last_type_in_path(&t.path) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + Type::Ptr(t) => ty = &t.elem, + Type::Reference(t) => ty = &t.elem, + Type::TraitObject(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_path) => return trailing_path, + ControlFlow::Continue(t) => ty = t, + }, + + Type::Array(_) + | Type::Group(_) + | Type::Infer(_) + | Type::Macro(_) + | Type::Never(_) + | Type::Paren(_) + | Type::Slice(_) + | Type::Tuple(_) + | Type::Verbatim(_) => return false, + } + } + + fn last_type_in_path(path: &Path) -> ControlFlow<bool, &Type> { + match &path.segments.last().unwrap().arguments { + PathArguments::None => ControlFlow::Break(true), + PathArguments::AngleBracketed(_) => ControlFlow::Break(false), + PathArguments::Parenthesized(arg) => match &arg.output { + ReturnType::Default => ControlFlow::Break(false), + ReturnType::Type(_, ret) => ControlFlow::Continue(ret), + }, + } + } + + fn last_type_in_bounds( + bounds: &Punctuated<TypeParamBound, Token![+]>, + ) -> ControlFlow<bool, &Type> { + match bounds.last().unwrap() { + TypeParamBound::Trait(t) => last_type_in_path(&t.path), + TypeParamBound::Lifetime(_) + | TypeParamBound::PreciseCapture(_) + | TypeParamBound::Verbatim(_) => ControlFlow::Break(false), + } + } +} + +/// Whether the expression's first token is the label of a loop/block. +#[cfg(all(feature = "printing", feature = "full"))] +pub(crate) fn expr_leading_label(mut expr: &Expr) -> bool { + loop { + match expr { + Expr::Block(e) => return e.label.is_some(), + Expr::ForLoop(e) => return e.label.is_some(), + Expr::Loop(e) => return e.label.is_some(), + Expr::While(e) => return e.label.is_some(), + + Expr::Assign(e) => expr = &e.left, + Expr::Await(e) => expr = &e.base, + Expr::Binary(e) => expr = &e.left, + Expr::Call(e) => expr = &e.func, + Expr::Cast(e) => expr = &e.expr, + Expr::Field(e) => expr = &e.base, + Expr::Index(e) => expr = &e.expr, + Expr::MethodCall(e) => expr = &e.receiver, + Expr::Range(e) => match &e.start { + Some(start) => expr = start, + None => return false, + }, + Expr::Try(e) => expr = &e.expr, + + Expr::Array(_) + | Expr::Async(_) + | Expr::Break(_) + | Expr::Closure(_) + | Expr::Const(_) + | Expr::Continue(_) + | Expr::Group(_) + | Expr::If(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Macro(_) + | Expr::Match(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::TryBlock(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Unsafe(_) + | Expr::Verbatim(_) + | Expr::Yield(_) => return false, + } + } +} + +/// Whether the expression's last token is `}`. +#[cfg(feature = "full")] +pub(crate) fn expr_trailing_brace(mut expr: &Expr) -> bool { + loop { + match expr { + Expr::Async(_) + | Expr::Block(_) + | Expr::Const(_) + | Expr::ForLoop(_) + | Expr::If(_) + | Expr::Loop(_) + | Expr::Match(_) + | Expr::Struct(_) + | Expr::TryBlock(_) + | Expr::Unsafe(_) + | Expr::While(_) => return true, + + Expr::Assign(e) => expr = &e.right, + Expr::Binary(e) => expr = &e.right, + Expr::Break(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + Expr::Cast(e) => return type_trailing_brace(&e.ty), + Expr::Closure(e) => expr = &e.body, + Expr::Let(e) => expr = &e.expr, + Expr::Macro(e) => return e.mac.delimiter.is_brace(), + Expr::Range(e) => match &e.end { + Some(end) => expr = end, + None => return false, + }, + Expr::RawAddr(e) => expr = &e.expr, + Expr::Reference(e) => expr = &e.expr, + Expr::Return(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + Expr::Unary(e) => expr = &e.expr, + Expr::Verbatim(e) => return tokens_trailing_brace(e), + Expr::Yield(e) => match &e.expr { + Some(e) => expr = e, + None => return false, + }, + + Expr::Array(_) + | Expr::Await(_) + | Expr::Call(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::Group(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Lit(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Repeat(_) + | Expr::Try(_) + | Expr::Tuple(_) => return false, + } + } + + fn type_trailing_brace(mut ty: &Type) -> bool { + loop { + match ty { + Type::BareFn(t) => match &t.output { + ReturnType::Default => return false, + ReturnType::Type(_, ret) => ty = ret, + }, + Type::ImplTrait(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_brace) => return trailing_brace, + ControlFlow::Continue(t) => ty = t, + }, + Type::Macro(t) => return t.mac.delimiter.is_brace(), + Type::Path(t) => match last_type_in_path(&t.path) { + Some(t) => ty = t, + None => return false, + }, + Type::Ptr(t) => ty = &t.elem, + Type::Reference(t) => ty = &t.elem, + Type::TraitObject(t) => match last_type_in_bounds(&t.bounds) { + ControlFlow::Break(trailing_brace) => return trailing_brace, + ControlFlow::Continue(t) => ty = t, + }, + Type::Verbatim(t) => return tokens_trailing_brace(t), + + Type::Array(_) + | Type::Group(_) + | Type::Infer(_) + | Type::Never(_) + | Type::Paren(_) + | Type::Slice(_) + | Type::Tuple(_) => return false, + } + } + } + + fn last_type_in_path(path: &Path) -> Option<&Type> { + match &path.segments.last().unwrap().arguments { + PathArguments::None | PathArguments::AngleBracketed(_) => None, + PathArguments::Parenthesized(arg) => match &arg.output { + ReturnType::Default => None, + ReturnType::Type(_, ret) => Some(ret), + }, + } + } + + fn last_type_in_bounds( + bounds: &Punctuated<TypeParamBound, Token![+]>, + ) -> ControlFlow<bool, &Type> { + match bounds.last().unwrap() { + TypeParamBound::Trait(t) => match last_type_in_path(&t.path) { + Some(t) => ControlFlow::Continue(t), + None => ControlFlow::Break(false), + }, + TypeParamBound::Lifetime(_) | TypeParamBound::PreciseCapture(_) => { + ControlFlow::Break(false) + } + TypeParamBound::Verbatim(t) => ControlFlow::Break(tokens_trailing_brace(t)), + } + } + + fn tokens_trailing_brace(tokens: &TokenStream) -> bool { + if let Some(TokenTree::Group(last)) = tokens.clone().into_iter().last() { + last.delimiter() == Delimiter::Brace + } else { + false + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/custom_keyword.rs b/third_party/crates/syn-2.0.119/src/custom_keyword.rs new file mode 100644 index 0000000..cc4f632 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/custom_keyword.rs @@ -0,0 +1,260 @@ +/// Define a type that supports parsing and printing a given identifier as if it +/// were a keyword. +/// +/// # Usage +/// +/// As a convention, it is recommended that this macro be invoked within a +/// module called `kw` or `keyword` and that the resulting parser be invoked +/// with a `kw::` or `keyword::` prefix. +/// +/// ``` +/// mod kw { +/// syn::custom_keyword!(whatever); +/// } +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in keyword token. +/// +/// - [Peeking] — `input.peek(kw::whatever)` +/// +/// - [Parsing] — `input.parse::<kw::whatever>()?` +/// +/// - [Printing] — `quote!( ... #whatever_token ... )` +/// +/// - Construction from a [`Span`] — `let whatever_token = kw::whatever(sp)` +/// +/// - Field access to its span — `let sp = whatever_token.span` +/// +/// [Peeking]: crate::parse::ParseBuffer::peek +/// [Parsing]: crate::parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// This example parses input that looks like `bool = true` or `str = "value"`. +/// The key must be either the identifier `bool` or the identifier `str`. If +/// `bool`, the value may be either `true` or `false`. If `str`, the value may +/// be any string literal. +/// +/// The symbols `bool` and `str` are not reserved keywords in Rust so these are +/// not considered keywords in the `syn::token` module. Like any other +/// identifier that is not a keyword, these can be declared as custom keywords +/// by crates that need to use them as such. +/// +/// ``` +/// use syn::{LitBool, LitStr, Result, Token}; +/// use syn::parse::{Parse, ParseStream}; +/// +/// mod kw { +/// syn::custom_keyword!(bool); +/// syn::custom_keyword!(str); +/// } +/// +/// enum Argument { +/// Bool { +/// bool_token: kw::bool, +/// eq_token: Token![=], +/// value: LitBool, +/// }, +/// Str { +/// str_token: kw::str, +/// eq_token: Token![=], +/// value: LitStr, +/// }, +/// } +/// +/// impl Parse for Argument { +/// fn parse(input: ParseStream) -> Result<Self> { +/// let lookahead = input.lookahead1(); +/// if lookahead.peek(kw::bool) { +/// Ok(Argument::Bool { +/// bool_token: input.parse::<kw::bool>()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else if lookahead.peek(kw::str) { +/// Ok(Argument::Str { +/// str_token: input.parse::<kw::str>()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else { +/// Err(lookahead.error()) +/// } +/// } +/// } +/// ``` +#[macro_export] +macro_rules! custom_keyword { + ($ident:ident) => { + #[allow(non_camel_case_types)] + pub struct $ident { + #[allow(dead_code)] + pub span: $crate::__private::Span, + } + + #[doc(hidden)] + #[allow(dead_code, non_snake_case)] + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::__private::Span>>( + span: __S, + ) -> $ident { + $ident { + span: $crate::__private::IntoSpans::into_spans(span), + } + } + + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident { + span: $crate::__private::Span::call_site(), + } + } + } + + $crate::impl_parse_for_custom_keyword!($ident); + $crate::impl_to_tokens_for_custom_keyword!($ident); + $crate::impl_clone_for_custom_keyword!($ident); + $crate::impl_extra_traits_for_custom_keyword!($ident); + }; + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => { + // For peek. + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + if let $crate::__private::Some((ident, _rest)) = cursor.ident() { + ident == $crate::__private::stringify!($ident) + } else { + false + } + } + + fn display() -> &'static $crate::__private::str { + $crate::__private::concat!("`", $crate::__private::stringify!($ident), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + input.step(|cursor| { + if let $crate::__private::Some((ident, rest)) = cursor.ident() { + if ident == $crate::__private::stringify!($ident) { + return $crate::__private::Ok(($ident { span: ident.span() }, rest)); + } + } + $crate::__private::Err(cursor.error($crate::__private::concat!( + "expected `", + $crate::__private::stringify!($ident), + "`", + ))) + }) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { + let ident = $crate::Ident::new($crate::__private::stringify!($ident), self.span); + $crate::__private::TokenStreamExt::append(tokens, ident); + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::Copy for $ident {} + + #[allow(clippy::expl_impl_clone_on_copy)] + impl $crate::__private::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => { + impl $crate::__private::Debug for $ident { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str( + f, + $crate::__private::concat!( + "Keyword [", + $crate::__private::stringify!($ident), + "]", + ), + ) + } + } + + impl $crate::__private::Eq for $ident {} + + impl $crate::__private::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::__private::bool { + true + } + } + + impl $crate::__private::Hash for $ident { + fn hash<__H: $crate::__private::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => {}; +} diff --git a/third_party/crates/syn-2.0.119/src/custom_punctuation.rs b/third_party/crates/syn-2.0.119/src/custom_punctuation.rs new file mode 100644 index 0000000..58955c7 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/custom_punctuation.rs @@ -0,0 +1,305 @@ +/// Define a type that supports parsing and printing a multi-character symbol +/// as if it were a punctuation token. +/// +/// # Usage +/// +/// ``` +/// syn::custom_punctuation!(LeftRightArrow, <=>); +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in punctuation token. +/// +/// - [Peeking] — `input.peek(LeftRightArrow)` +/// +/// - [Parsing] — `input.parse::<LeftRightArrow>()?` +/// +/// - [Printing] — `quote!( ... #lrarrow ... )` +/// +/// - Construction from a [`Span`] — `let lrarrow = LeftRightArrow(sp)` +/// +/// - Construction from multiple [`Span`] — `let lrarrow = LeftRightArrow([sp, sp, sp])` +/// +/// - Field access to its spans — `let spans = lrarrow.spans` +/// +/// [Peeking]: crate::parse::ParseBuffer::peek +/// [Parsing]: crate::parse::ParseBuffer::parse +/// [Printing]: quote::ToTokens +/// [`Span`]: proc_macro2::Span +/// +/// # Example +/// +/// ``` +/// use core::iter; +/// use proc_macro2::{TokenStream, TokenTree}; +/// use syn::parse::{Parse, ParseStream, Peek, Result}; +/// use syn::punctuated::Punctuated; +/// use syn::Expr; +/// +/// syn::custom_punctuation!(PathSeparator, </>); +/// +/// // expr </> expr </> expr ... +/// struct PathSegments { +/// segments: Punctuated<Expr, PathSeparator>, +/// } +/// +/// impl Parse for PathSegments { +/// fn parse(input: ParseStream) -> Result<Self> { +/// let mut segments = Punctuated::new(); +/// +/// let first = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(first)?); +/// +/// while input.peek(PathSeparator) { +/// segments.push_punct(input.parse()?); +/// +/// let next = parse_until(input, PathSeparator)?; +/// segments.push_value(syn::parse2(next)?); +/// } +/// +/// Ok(PathSegments { segments }) +/// } +/// } +/// +/// fn parse_until<E: Peek>(input: ParseStream, end: E) -> Result<TokenStream> { +/// let mut tokens = TokenStream::new(); +/// while !input.is_empty() && !input.peek(end) { +/// let next: TokenTree = input.parse()?; +/// tokens.extend(iter::once(next)); +/// } +/// Ok(tokens) +/// } +/// +/// fn main() { +/// let input = r#" a::b </> c::d::e "#; +/// let _: PathSegments = syn::parse_str(input).unwrap(); +/// } +/// ``` +#[macro_export] +macro_rules! custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + pub struct $ident { + #[allow(dead_code)] + pub spans: $crate::custom_punctuation_repr!($($tt)+), + } + + #[doc(hidden)] + #[allow(dead_code, non_snake_case)] + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::custom_punctuation_repr!($($tt)+)>>( + spans: __S, + ) -> $ident { + let _validate_len = 0 $(+ $crate::custom_punctuation_len!(strict, $tt))*; + $ident { + spans: $crate::__private::IntoSpans::into_spans(spans) + } + } + + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident($crate::__private::Span::call_site()) + } + } + + $crate::impl_parse_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_clone_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); + }; + }; +} + +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + $crate::__private::peek_punct(cursor, $crate::stringify_punct!($($tt)+)) + } + + fn display() -> &'static $crate::__private::str { + $crate::__private::concat!("`", $crate::stringify_punct!($($tt)+), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + let spans: $crate::custom_punctuation_repr!($($tt)+) = + $crate::__private::parse_punct(input, $crate::stringify_punct!($($tt)+))?; + Ok($ident(spans)) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { + $crate::__private::print_punct($crate::stringify_punct!($($tt)+), &self.spans, tokens) + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::Copy for $ident {} + + #[allow(clippy::expl_impl_clone_on_copy)] + impl $crate::__private::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => { + impl $crate::__private::Debug for $ident { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str(f, $crate::__private::stringify!($ident)) + } + } + + impl $crate::__private::Eq for $ident {} + + impl $crate::__private::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::__private::bool { + true + } + } + + impl $crate::__private::Hash for $ident { + fn hash<__H: $crate::__private::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_punctuation { + ($ident:ident, $($tt:tt)+) => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_repr { + ($($tt:tt)+) => { + [$crate::__private::Span; 0 $(+ $crate::custom_punctuation_len!(lenient, $tt))+] + }; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +#[rustfmt::skip] +macro_rules! custom_punctuation_len { + ($mode:ident, &) => { 1 }; + ($mode:ident, &&) => { 2 }; + ($mode:ident, &=) => { 2 }; + ($mode:ident, @) => { 1 }; + ($mode:ident, ^) => { 1 }; + ($mode:ident, ^=) => { 2 }; + ($mode:ident, :) => { 1 }; + ($mode:ident, ,) => { 1 }; + ($mode:ident, $) => { 1 }; + ($mode:ident, .) => { 1 }; + ($mode:ident, ..) => { 2 }; + ($mode:ident, ...) => { 3 }; + ($mode:ident, ..=) => { 3 }; + ($mode:ident, =) => { 1 }; + ($mode:ident, ==) => { 2 }; + ($mode:ident, =>) => { 2 }; + ($mode:ident, >=) => { 2 }; + ($mode:ident, >) => { 1 }; + ($mode:ident, <-) => { 2 }; + ($mode:ident, <=) => { 2 }; + ($mode:ident, <) => { 1 }; + ($mode:ident, -) => { 1 }; + ($mode:ident, -=) => { 2 }; + ($mode:ident, !=) => { 2 }; + ($mode:ident, !) => { 1 }; + ($mode:ident, |) => { 1 }; + ($mode:ident, |=) => { 2 }; + ($mode:ident, ||) => { 2 }; + ($mode:ident, ::) => { 2 }; + ($mode:ident, %) => { 1 }; + ($mode:ident, %=) => { 2 }; + ($mode:ident, +) => { 1 }; + ($mode:ident, +=) => { 2 }; + ($mode:ident, #) => { 1 }; + ($mode:ident, ?) => { 1 }; + ($mode:ident, ->) => { 2 }; + ($mode:ident, ;) => { 1 }; + ($mode:ident, <<) => { 2 }; + ($mode:ident, <<=) => { 3 }; + ($mode:ident, >>) => { 2 }; + ($mode:ident, >>=) => { 3 }; + ($mode:ident, /) => { 1 }; + ($mode:ident, /=) => { 2 }; + ($mode:ident, *) => { 1 }; + ($mode:ident, *=) => { 2 }; + ($mode:ident, ~) => { 1 }; + (lenient, $tt:tt) => { 0 }; + (strict, $tt:tt) => {{ $crate::custom_punctuation_unexpected!($tt); 0 }}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! custom_punctuation_unexpected { + () => {}; +} + +// Not public API. +#[doc(hidden)] +#[macro_export] +macro_rules! stringify_punct { + ($($tt:tt)+) => { + $crate::__private::concat!($($crate::__private::stringify!($tt)),+) + }; +} diff --git a/third_party/crates/syn-2.0.119/src/data.rs b/third_party/crates/syn-2.0.119/src/data.rs new file mode 100644 index 0000000..ce02902 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/data.rs @@ -0,0 +1,425 @@ +use crate::attr::Attribute; +use crate::expr::{Expr, Index, Member}; +use crate::ident::Ident; +use crate::punctuated::{self, Punctuated}; +use crate::restriction::{FieldMutability, Visibility}; +use crate::token; +use crate::ty::Type; +use alloc::vec::Vec; + +ast_struct! { + /// An enum variant. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Variant { + pub attrs: Vec<Attribute>, + + /// Name of the variant. + pub ident: Ident, + + /// Content stored in the variant. + pub fields: Fields, + + /// Explicit discriminant: `Variant = 1` + pub discriminant: Option<(Token![=], Expr)>, + } +} + +ast_enum_of_structs! { + /// Data stored within an enum variant or struct. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Fields { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + Named(FieldsNamed), + + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + Unnamed(FieldsUnnamed), + + /// Unit struct or unit variant such as `None`. + Unit, + } +} + +ast_struct! { + /// Named fields of a struct or struct variant such as `Point { x: f64, + /// y: f64 }`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldsNamed { + pub brace_token: token::Brace, + pub named: Punctuated<Field, Token![,]>, + } +} + +ast_struct! { + /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldsUnnamed { + pub paren_token: token::Paren, + pub unnamed: Punctuated<Field, Token![,]>, + } +} + +impl Fields { + /// Get an iterator over the borrowed [`Field`] items in this object. This + /// iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + pub fn iter(&self) -> punctuated::Iter<Field> { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter(), + Fields::Named(f) => f.named.iter(), + Fields::Unnamed(f) => f.unnamed.iter(), + } + } + + /// Get an iterator over the mutably borrowed [`Field`] items in this + /// object. This iterator can be used to iterate over a named or unnamed + /// struct or variant's fields uniformly. + pub fn iter_mut(&mut self) -> punctuated::IterMut<Field> { + match self { + Fields::Unit => crate::punctuated::empty_punctuated_iter_mut(), + Fields::Named(f) => f.named.iter_mut(), + Fields::Unnamed(f) => f.unnamed.iter_mut(), + } + } + + /// Returns the number of fields. + pub fn len(&self) -> usize { + match self { + Fields::Unit => 0, + Fields::Named(f) => f.named.len(), + Fields::Unnamed(f) => f.unnamed.len(), + } + } + + /// Returns `true` if there are zero fields. + pub fn is_empty(&self) -> bool { + match self { + Fields::Unit => true, + Fields::Named(f) => f.named.is_empty(), + Fields::Unnamed(f) => f.unnamed.is_empty(), + } + } + + return_impl_trait! { + /// Get an iterator over the fields of a struct or variant as [`Member`]s. + /// This iterator can be used to iterate over a named or unnamed struct or + /// variant's fields uniformly. + /// + /// # Example + /// + /// The following is a simplistic [`Clone`] derive for structs. (A more + /// complete implementation would additionally want to infer trait bounds on + /// the generic type parameters.) + /// + /// ``` + /// # use quote::quote; + /// # + /// fn derive_clone(input: &syn::ItemStruct) -> proc_macro2::TokenStream { + /// let ident = &input.ident; + /// let members = input.fields.members(); + /// let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + /// quote! { + /// impl #impl_generics Clone for #ident #ty_generics #where_clause { + /// fn clone(&self) -> Self { + /// Self { + /// #(#members: self.#members.clone()),* + /// } + /// } + /// } + /// } + /// } + /// ``` + /// + /// For structs with named fields, it produces an expression like `Self { a: + /// self.a.clone() }`. For structs with unnamed fields, `Self { 0: + /// self.0.clone() }`. And for unit structs, `Self {}`. + pub fn members(&self) -> impl Iterator<Item = Member> + Clone + '_ [Members] { + Members { + fields: self.iter(), + index: 0, + } + } + } +} + +impl IntoIterator for Fields { + type Item = Field; + type IntoIter = punctuated::IntoIter<Field>; + + fn into_iter(self) -> Self::IntoIter { + match self { + Fields::Unit => Punctuated::<Field, ()>::new().into_iter(), + Fields::Named(f) => f.named.into_iter(), + Fields::Unnamed(f) => f.unnamed.into_iter(), + } + } +} + +impl<'a> IntoIterator for &'a Fields { + type Item = &'a Field; + type IntoIter = punctuated::Iter<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl<'a> IntoIterator for &'a mut Fields { + type Item = &'a mut Field; + type IntoIter = punctuated::IterMut<'a, Field>; + + fn into_iter(self) -> Self::IntoIter { + self.iter_mut() + } +} + +ast_struct! { + /// A field of a struct or enum variant. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Field { + pub attrs: Vec<Attribute>, + + pub vis: Visibility, + + pub mutability: FieldMutability, + + /// Name of the field, if any. + /// + /// Fields of tuple structs have no names. + pub ident: Option<Ident>, + + pub colon_token: Option<Token![:]>, + + pub ty: Type, + } +} + +pub struct Members<'a> { + fields: punctuated::Iter<'a, Field>, + index: u32, +} + +impl<'a> Iterator for Members<'a> { + type Item = Member; + + fn next(&mut self) -> Option<Self::Item> { + let field = self.fields.next()?; + let member = match &field.ident { + Some(ident) => Member::Named(ident.clone()), + None => { + #[cfg(all(feature = "parsing", feature = "printing"))] + let span = crate::spanned::Spanned::span(&field.ty); + #[cfg(not(all(feature = "parsing", feature = "printing")))] + let span = proc_macro2::Span::call_site(); + Member::Unnamed(Index { + index: self.index, + span, + }) + } + }; + self.index += 1; + Some(member) + } +} + +impl<'a> Clone for Members<'a> { + fn clone(&self) -> Self { + Members { + fields: self.fields.clone(), + index: self.index, + } + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::data::{Field, Fields, FieldsNamed, FieldsUnnamed, Variant}; + use crate::error::Result; + use crate::expr::Expr; + use crate::ext::IdentExt as _; + use crate::ident::Ident; + #[cfg(not(feature = "full"))] + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::restriction::{FieldMutability, Visibility}; + #[cfg(not(feature = "full"))] + use crate::scan_expr::scan_expr; + use crate::token; + use crate::ty::Type; + use crate::verbatim; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Variant { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let _visibility: Visibility = input.parse()?; + let ident: Ident = input.parse()?; + let fields = if input.peek(token::Brace) { + Fields::Named(input.parse()?) + } else if input.peek(token::Paren) { + Fields::Unnamed(input.parse()?) + } else { + Fields::Unit + }; + let discriminant = if input.peek(Token![=]) { + let eq_token: Token![=] = input.parse()?; + #[cfg(feature = "full")] + let discriminant: Expr = input.parse()?; + #[cfg(not(feature = "full"))] + let discriminant = { + let begin = input.cursor(); + let ahead = input.fork(); + let mut discriminant: Result<Expr> = ahead.parse(); + if discriminant.is_ok() { + input.advance_to(&ahead); + } else if scan_expr(input).is_ok() { + discriminant = Ok(Expr::Verbatim(verbatim::between(begin, input.cursor()))); + } + discriminant? + }; + Some((eq_token, discriminant)) + } else { + None + }; + Ok(Variant { + attrs, + ident, + fields, + discriminant, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldsNamed { + fn parse(input: ParseStream) -> Result<Self> { + let content; + Ok(FieldsNamed { + brace_token: braced!(content in input), + named: content.parse_terminated(Field::parse_named, Token![,])?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldsUnnamed { + fn parse(input: ParseStream) -> Result<Self> { + let content; + Ok(FieldsUnnamed { + paren_token: parenthesized!(content in input), + unnamed: content.parse_terminated(Field::parse_unnamed, Token![,])?, + }) + } + } + + impl Field { + /// Parses a named (braced struct) field. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_named(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + + let unnamed_field = cfg!(feature = "full") && input.peek(Token![_]); + let ident = if unnamed_field { + input.call(Ident::parse_any) + } else { + input.parse() + }?; + + let colon_token: Token![:] = input.parse()?; + + let ty: Type = if unnamed_field + && (input.peek(Token![struct]) + || input.peek(Token![union]) && input.peek2(token::Brace)) + { + let begin = input.cursor(); + input.call(Ident::parse_any)?; + input.parse::<FieldsNamed>()?; + Type::Verbatim(verbatim::between(begin, input.cursor())) + } else { + input.parse()? + }; + + Ok(Field { + attrs, + vis, + mutability: FieldMutability::None, + ident: Some(ident), + colon_token: Some(colon_token), + ty, + }) + } + + /// Parses an unnamed (tuple struct) field. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_unnamed(input: ParseStream) -> Result<Self> { + Ok(Field { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + mutability: FieldMutability::None, + ident: None, + colon_token: None, + ty: input.parse()?, + }) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::data::{Field, FieldsNamed, FieldsUnnamed, Variant}; + use crate::print::TokensOrDefault; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Variant { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.ident.to_tokens(tokens); + self.fields.to_tokens(tokens); + if let Some((eq_token, disc)) = &self.discriminant { + eq_token.to_tokens(tokens); + disc.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldsNamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.brace_token.surround(tokens, |tokens| { + self.named.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldsUnnamed { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.paren_token.surround(tokens, |tokens| { + self.unnamed.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Field { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.vis.to_tokens(tokens); + if let Some(ident) = &self.ident { + ident.to_tokens(tokens); + TokensOrDefault(&self.colon_token).to_tokens(tokens); + } + self.ty.to_tokens(tokens); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/derive.rs b/third_party/crates/syn-2.0.119/src/derive.rs new file mode 100644 index 0000000..573b88f --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/derive.rs @@ -0,0 +1,260 @@ +use crate::attr::Attribute; +use crate::data::{Fields, FieldsNamed, Variant}; +use crate::generics::Generics; +use crate::ident::Ident; +use crate::punctuated::Punctuated; +use crate::restriction::Visibility; +use crate::token; +use alloc::vec::Vec; + +ast_struct! { + /// Data structure sent to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DeriveInput { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub ident: Ident, + pub generics: Generics, + pub data: Data, + } +} + +ast_enum! { + /// The storage of a struct, enum or union data structure. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub enum Data { + Struct(DataStruct), + Enum(DataEnum), + Union(DataUnion), + } +} + +ast_struct! { + /// A struct input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataStruct { + pub struct_token: Token![struct], + pub fields: Fields, + pub semi_token: Option<Token![;]>, + } +} + +ast_struct! { + /// An enum input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataEnum { + pub enum_token: Token![enum], + pub brace_token: token::Brace, + pub variants: Punctuated<Variant, Token![,]>, + } +} + +ast_struct! { + /// An untagged union input to a `proc_macro_derive` macro. + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + pub struct DataUnion { + pub union_token: Token![union], + pub fields: FieldsNamed, + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::data::{Fields, FieldsNamed, Variant}; + use crate::derive::{Data, DataEnum, DataStruct, DataUnion, DeriveInput}; + use crate::error::Result; + use crate::generics::{Generics, WhereClause}; + use crate::ident::Ident; + use crate::parse::{Parse, ParseStream}; + use crate::punctuated::Punctuated; + use crate::restriction::Visibility; + use crate::token; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for DeriveInput { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse::<Visibility>()?; + + let lookahead = input.lookahead1(); + if lookahead.peek(Token![struct]) { + let struct_token = input.parse::<Token![struct]>()?; + let ident = input.parse::<Ident>()?; + let generics = input.parse::<Generics>()?; + let (where_clause, fields, semi) = data_struct(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Struct(DataStruct { + struct_token, + fields, + semi_token: semi, + }), + }) + } else if lookahead.peek(Token![enum]) { + let enum_token = input.parse::<Token![enum]>()?; + let ident = input.parse::<Ident>()?; + let generics = input.parse::<Generics>()?; + let (where_clause, brace, variants) = data_enum(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Enum(DataEnum { + enum_token, + brace_token: brace, + variants, + }), + }) + } else if lookahead.peek(Token![union]) { + let union_token = input.parse::<Token![union]>()?; + let ident = input.parse::<Ident>()?; + let generics = input.parse::<Generics>()?; + let (where_clause, fields) = data_union(input)?; + Ok(DeriveInput { + attrs, + vis, + ident, + generics: Generics { + where_clause, + ..generics + }, + data: Data::Union(DataUnion { + union_token, + fields, + }), + }) + } else { + Err(lookahead.error()) + } + } + } + + pub(crate) fn data_struct( + input: ParseStream, + ) -> Result<(Option<WhereClause>, Fields, Option<Token![;]>)> { + let mut lookahead = input.lookahead1(); + let mut where_clause = None; + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if where_clause.is_none() && lookahead.peek(token::Paren) { + let fields = input.parse()?; + + lookahead = input.lookahead1(); + if lookahead.peek(Token![where]) { + where_clause = Some(input.parse()?); + lookahead = input.lookahead1(); + } + + if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unnamed(fields), Some(semi))) + } else { + Err(lookahead.error()) + } + } else if lookahead.peek(token::Brace) { + let fields = input.parse()?; + Ok((where_clause, Fields::Named(fields), None)) + } else if lookahead.peek(Token![;]) { + let semi = input.parse()?; + Ok((where_clause, Fields::Unit, Some(semi))) + } else { + Err(lookahead.error()) + } + } + + pub(crate) fn data_enum( + input: ParseStream, + ) -> Result<( + Option<WhereClause>, + token::Brace, + Punctuated<Variant, Token![,]>, + )> { + let where_clause = input.parse()?; + + let content; + let brace = braced!(content in input); + let variants = content.parse_terminated(Variant::parse, Token![,])?; + + Ok((where_clause, brace, variants)) + } + + pub(crate) fn data_union(input: ParseStream) -> Result<(Option<WhereClause>, FieldsNamed)> { + let where_clause = input.parse()?; + let fields = input.parse()?; + Ok((where_clause, fields)) + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::FilterAttrs; + use crate::data::Fields; + use crate::derive::{Data, DeriveInput}; + use crate::print::TokensOrDefault; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for DeriveInput { + fn to_tokens(&self, tokens: &mut TokenStream) { + for attr in self.attrs.outer() { + attr.to_tokens(tokens); + } + self.vis.to_tokens(tokens); + match &self.data { + Data::Struct(d) => d.struct_token.to_tokens(tokens), + Data::Enum(d) => d.enum_token.to_tokens(tokens), + Data::Union(d) => d.union_token.to_tokens(tokens), + } + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + match &self.data { + Data::Struct(data) => match &data.fields { + Fields::Named(fields) => { + self.generics.where_clause.to_tokens(tokens); + fields.to_tokens(tokens); + } + Fields::Unnamed(fields) => { + fields.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + Fields::Unit => { + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&data.semi_token).to_tokens(tokens); + } + }, + Data::Enum(data) => { + self.generics.where_clause.to_tokens(tokens); + data.brace_token.surround(tokens, |tokens| { + data.variants.to_tokens(tokens); + }); + } + Data::Union(data) => { + self.generics.where_clause.to_tokens(tokens); + data.fields.to_tokens(tokens); + } + } + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/discouraged.rs b/third_party/crates/syn-2.0.119/src/discouraged.rs new file mode 100644 index 0000000..9e26bd9 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/discouraged.rs @@ -0,0 +1,225 @@ +//! Extensions to the parsing API with niche applicability. + +use crate::buffer::Cursor; +use crate::error::Result; +use crate::parse::{inner_unexpected, ParseBuffer, Unexpected}; +use alloc::rc::Rc; +use core::cell::Cell; +use core::mem; +use proc_macro2::extra::DelimSpan; +use proc_macro2::Delimiter; + +/// Extensions to the `ParseStream` API to support speculative parsing. +pub trait Speculative { + /// Advance this parse stream to the position of a forked parse stream. + /// + /// This is the opposite operation to [`ParseStream::fork`]. You can fork a + /// parse stream, perform some speculative parsing, then join the original + /// stream to the fork to "commit" the parsing from the fork to the main + /// stream. + /// + /// If you can avoid doing this, you should, as it limits the ability to + /// generate useful errors. That said, it is often the only way to parse + /// syntax of the form `A* B*` for arbitrary syntax `A` and `B`. The problem + /// is that when the fork fails to parse an `A`, it's impossible to tell + /// whether that was because of a syntax error and the user meant to provide + /// an `A`, or that the `A`s are finished and it's time to start parsing + /// `B`s. Use with care. + /// + /// Also note that if `A` is a subset of `B`, `A* B*` can be parsed by + /// parsing `B*` and removing the leading members of `A` from the + /// repetition, bypassing the need to involve the downsides associated with + /// speculative parsing. + /// + /// [`ParseStream::fork`]: ParseBuffer::fork + /// + /// # Example + /// + /// There has been chatter about the possibility of making the colons in the + /// turbofish syntax like `path::to::<T>` no longer required by accepting + /// `path::to<T>` in expression position. Specifically, according to [RFC + /// 2544], [`PathSegment`] parsing should always try to consume a following + /// `<` token as the start of generic arguments, and reset to the `<` if + /// that fails (e.g. the token is acting as a less-than operator). + /// + /// This is the exact kind of parsing behavior which requires the "fork, + /// try, commit" behavior that [`ParseStream::fork`] discourages. With + /// `advance_to`, we can avoid having to parse the speculatively parsed + /// content a second time. + /// + /// This change in behavior can be implemented in syn by replacing just the + /// `Parse` implementation for `PathSegment`: + /// + /// ``` + /// # use syn::ext::IdentExt; + /// use syn::parse::discouraged::Speculative; + /// # use syn::parse::{Parse, ParseStream}; + /// # use syn::{Ident, PathArguments, Result, Token}; + /// + /// pub struct PathSegment { + /// pub ident: Ident, + /// pub arguments: PathArguments, + /// } + /// # + /// # impl<T> From<T> for PathSegment + /// # where + /// # T: Into<Ident>, + /// # { + /// # fn from(ident: T) -> Self { + /// # PathSegment { + /// # ident: ident.into(), + /// # arguments: PathArguments::None, + /// # } + /// # } + /// # } + /// + /// impl Parse for PathSegment { + /// fn parse(input: ParseStream) -> Result<Self> { + /// if input.peek(Token![super]) + /// || input.peek(Token![self]) + /// || input.peek(Token![Self]) + /// || input.peek(Token![crate]) + /// { + /// let ident = input.call(Ident::parse_any)?; + /// return Ok(PathSegment::from(ident)); + /// } + /// + /// let ident = input.parse()?; + /// if input.peek(Token![::]) && input.peek3(Token![<]) { + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(input.parse()?), + /// }); + /// } + /// if input.peek(Token![<]) && !input.peek(Token![<=]) { + /// let fork = input.fork(); + /// if let Ok(arguments) = fork.parse() { + /// input.advance_to(&fork); + /// return Ok(PathSegment { + /// ident, + /// arguments: PathArguments::AngleBracketed(arguments), + /// }); + /// } + /// } + /// Ok(PathSegment::from(ident)) + /// } + /// } + /// + /// # syn::parse_str::<PathSegment>("a<b,c>").unwrap(); + /// ``` + /// + /// # Drawbacks + /// + /// The main drawback of this style of speculative parsing is in error + /// presentation. Even if the lookahead is the "correct" parse, the error + /// that is shown is that of the "fallback" parse. To use the same example + /// as the turbofish above, take the following unfinished "turbofish": + /// + /// ```text + /// let _ = f<&'a fn(), for<'a> serde::>(); + /// ``` + /// + /// If this is parsed as generic arguments, we can provide the error message + /// + /// ```text + /// error: expected identifier + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^ + /// ``` + /// + /// but if parsed using the above speculative parsing, it falls back to + /// assuming that the `<` is a less-than when it fails to parse the generic + /// arguments, and tries to interpret the `&'a` as the start of a labelled + /// loop, resulting in the much less helpful error + /// + /// ```text + /// error: expected `:` + /// --> src.rs:L:C + /// | + /// L | let _ = f<&'a fn(), for<'a> serde::>(); + /// | ^^ + /// ``` + /// + /// This can be mitigated with various heuristics (two examples: show both + /// forks' parse errors, or show the one that consumed more tokens), but + /// when you can control the grammar, sticking to something that can be + /// parsed LL(3) and without the LL(*) speculative parsing this makes + /// possible, displaying reasonable errors becomes much more simple. + /// + /// [RFC 2544]: https://github.com/rust-lang/rfcs/pull/2544 + /// [`PathSegment`]: crate::PathSegment + /// + /// # Performance + /// + /// This method performs a cheap fixed amount of work that does not depend + /// on how far apart the two streams are positioned. + /// + /// # Panics + /// + /// The forked stream in the argument of `advance_to` must have been + /// obtained by forking `self`. Attempting to advance to any other stream + /// will cause a panic. + fn advance_to(&self, fork: &Self); +} + +impl<'a> Speculative for ParseBuffer<'a> { + fn advance_to(&self, fork: &Self) { + if !crate::buffer::same_scope(self.cursor(), fork.cursor()) { + panic!("fork was not derived from the advancing parse stream"); + } + + let (self_unexp, self_sp) = inner_unexpected(self); + let (fork_unexp, fork_sp) = inner_unexpected(fork); + if !Rc::ptr_eq(&self_unexp, &fork_unexp) { + match (fork_sp, self_sp) { + // Unexpected set on the fork, but not on `self`, copy it over. + (Some((span, delimiter)), None) => { + self_unexp.set(Unexpected::Some(span, delimiter)); + } + // Unexpected unset. Use chain to propagate errors from fork. + (None, None) => { + fork_unexp.set(Unexpected::Chain(self_unexp)); + + // Ensure toplevel 'unexpected' tokens from the fork don't + // propagate up the chain by replacing the root `unexpected` + // pointer, only 'unexpected' tokens from existing group + // parsers should propagate. + fork.unexpected + .set(Some(Rc::new(Cell::new(Unexpected::None)))); + } + // Unexpected has been set on `self`. No changes needed. + (_, Some(_)) => {} + } + } + + // See comment on `cell` in the struct definition. + self.cell + .set(unsafe { mem::transmute::<Cursor, Cursor<'static>>(fork.cursor()) }); + } +} + +/// Extensions to the `ParseStream` API to support manipulating invisible +/// delimiters the same as if they were visible. +pub trait AnyDelimiter { + /// Returns the delimiter, the span of the delimiter token, and the nested + /// contents for further parsing. + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)>; +} + +impl<'a> AnyDelimiter for ParseBuffer<'a> { + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)> { + self.step(|cursor| { + if let Some((content, delimiter, span, rest)) = cursor.any_group() { + let scope = span.close(); + let nested = crate::parse::advance_step_cursor(cursor, content); + let unexpected = crate::parse::get_unexpected(self); + let content = crate::parse::new_parse_buffer(scope, nested, unexpected); + Ok(((delimiter, span, content), rest)) + } else { + Err(cursor.error("expected any delimiter")) + } + }) + } +} diff --git a/third_party/crates/syn-2.0.119/src/drops.rs b/third_party/crates/syn-2.0.119/src/drops.rs new file mode 100644 index 0000000..20ab75e --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/drops.rs @@ -0,0 +1,58 @@ +use core::iter; +use core::mem::ManuallyDrop; +use core::ops::{Deref, DerefMut}; +use core::option; +use core::slice; + +#[repr(transparent)] +pub(crate) struct NoDrop<T: ?Sized>(ManuallyDrop<T>); + +impl<T> NoDrop<T> { + pub(crate) fn new(value: T) -> Self + where + T: TrivialDrop, + { + NoDrop(ManuallyDrop::new(value)) + } +} + +impl<T: ?Sized> Deref for NoDrop<T> { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<T: ?Sized> DerefMut for NoDrop<T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +pub(crate) trait TrivialDrop {} + +impl<T> TrivialDrop for iter::Empty<T> {} +impl<T> TrivialDrop for slice::Iter<'_, T> {} +impl<T> TrivialDrop for slice::IterMut<'_, T> {} +impl<T> TrivialDrop for option::IntoIter<&T> {} +impl<T> TrivialDrop for option::IntoIter<&mut T> {} + +#[test] +fn test_needs_drop() { + use core::mem::needs_drop; + + struct NeedsDrop; + + impl Drop for NeedsDrop { + fn drop(&mut self) {} + } + + assert!(needs_drop::<NeedsDrop>()); + + // Test each of the types with a handwritten TrivialDrop impl above. + assert!(!needs_drop::<iter::Empty<NeedsDrop>>()); + assert!(!needs_drop::<slice::Iter<NeedsDrop>>()); + assert!(!needs_drop::<slice::IterMut<NeedsDrop>>()); + assert!(!needs_drop::<option::IntoIter<&NeedsDrop>>()); + assert!(!needs_drop::<option::IntoIter<&mut NeedsDrop>>()); +} diff --git a/third_party/crates/syn-2.0.119/src/error.rs b/third_party/crates/syn-2.0.119/src/error.rs new file mode 100644 index 0000000..2b753ea --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/error.rs @@ -0,0 +1,473 @@ +#[cfg(feature = "parsing")] +use crate::buffer::Cursor; +use crate::ext::{PunctExt as _, TokenStreamExt as _}; +use crate::thread::ThreadBound; +#[cfg(feature = "parsing")] +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec; +use alloc::vec::Vec; +use core::fmt::{self, Debug, Display}; +use core::slice; +use proc_macro2::{ + Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree, +}; +#[cfg(feature = "printing")] +use quote::ToTokens; + +/// The result of a Syn parser. +pub type Result<T> = core::result::Result<T, Error>; + +/// Error returned when a Syn parser cannot parse the input tokens. +/// +/// # Error reporting in proc macros +/// +/// The correct way to report errors back to the compiler from a procedural +/// macro is by emitting an appropriately spanned invocation of +/// [`compile_error!`] in the generated code. This produces a better diagnostic +/// message than simply panicking the macro. +/// +/// [`compile_error!`]: core::compile_error! +/// +/// When parsing macro input, the [`parse_macro_input!`] macro handles the +/// conversion to `compile_error!` automatically. +/// +/// [`parse_macro_input!`]: crate::parse_macro_input! +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::parse::{Parse, ParseStream, Result}; +/// use syn::{parse_macro_input, ItemFn}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { +/// let args = parse_macro_input!(args as MyAttrArgs); +/// let input = parse_macro_input!(input as ItemFn); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// +/// struct MyAttrArgs { +/// # _k: [(); { stringify! { +/// ... +/// # }; 0 }] +/// } +/// +/// impl Parse for MyAttrArgs { +/// fn parse(input: ParseStream) -> Result<Self> { +/// # stringify! { +/// ... +/// # }; +/// # unimplemented!() +/// } +/// } +/// ``` +/// +/// For errors that arise later than the initial parsing stage, the +/// [`.to_compile_error()`] or [`.into_compile_error()`] methods can be used to +/// perform an explicit conversion to `compile_error!`. +/// +/// [`.to_compile_error()`]: Error::to_compile_error +/// [`.into_compile_error()`]: Error::into_compile_error +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// # use proc_macro::TokenStream; +/// # use syn::{parse_macro_input, DeriveInput}; +/// # +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_derive(MyDerive)] +/// # }; +/// pub fn my_derive(input: TokenStream) -> TokenStream { +/// let input = parse_macro_input!(input as DeriveInput); +/// +/// // fn(DeriveInput) -> syn::Result<proc_macro2::TokenStream> +/// expand::my_derive(input) +/// .unwrap_or_else(syn::Error::into_compile_error) +/// .into() +/// } +/// # +/// # mod expand { +/// # use proc_macro2::TokenStream; +/// # use syn::{DeriveInput, Result}; +/// # +/// # pub fn my_derive(input: DeriveInput) -> Result<TokenStream> { +/// # unimplemented!() +/// # } +/// # } +/// ``` +pub struct Error { + messages: Vec<ErrorMessage>, +} + +struct ErrorMessage { + // Span is implemented as an index into a thread-local interner to keep the + // size small. It is not safe to access from a different thread. We want + // errors to be Send and Sync to play nicely with ecosystem crates for error + // handling, so pin the span we're given to its original thread and assume + // it is Span::call_site if accessed from any other thread. + span: ThreadBound<SpanRange>, + message: String, +} + +// Cannot use core::ops::Range<Span> because that does not implement Copy, +// whereas ThreadBound<T> requires a Copy impl as a way to ensure no Drop impls +// are involved. +struct SpanRange { + start: Span, + end: Span, +} + +#[cfg(test)] +struct _Test +where + Error: Send + Sync; + +impl Error { + /// Usually the [`ParseStream::error`] method will be used instead, which + /// automatically uses the correct span from the current position of the + /// parse stream. + /// + /// Use `Error::new` when the error needs to be triggered on some span other + /// than where the parse stream is currently positioned. + /// + /// [`ParseStream::error`]: crate::parse::ParseBuffer::error + /// + /// # Example + /// + /// ``` + /// use syn::{Error, Ident, LitStr, Result, Token}; + /// use syn::parse::ParseStream; + /// + /// // Parses input that looks like `name = "string"` where the key must be + /// // the identifier `name` and the value may be any string literal. + /// // Returns the string literal. + /// fn parse_name(input: ParseStream) -> Result<LitStr> { + /// let name_token: Ident = input.parse()?; + /// if name_token != "name" { + /// // Trigger an error not on the current position of the stream, + /// // but on the position of the unexpected identifier. + /// return Err(Error::new(name_token.span(), "expected `name`")); + /// } + /// input.parse::<Token![=]>()?; + /// let s: LitStr = input.parse()?; + /// Ok(s) + /// } + /// ``` + pub fn new<T: Display>(span: Span, message: T) -> Self { + return new(span, message.to_string()); + + fn new(span: Span, message: String) -> Error { + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { + start: span, + end: span, + }), + message, + }], + } + } + } + + /// Creates an error with the specified message spanning the given syntax + /// tree node. + /// + /// Unlike the `Error::new` constructor, this constructor takes an argument + /// `tokens` which is a syntax tree node. This allows the resulting `Error` + /// to attempt to span all tokens inside of `tokens`. While you would + /// typically be able to use the `Spanned` trait with the above `Error::new` + /// constructor, implementation limitations today mean that + /// `Error::new_spanned` may provide a higher-quality error message on + /// stable Rust. + /// + /// When in doubt it's recommended to stick to `Error::new` (or + /// `ParseStream::error`)! + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + pub fn new_spanned<T: ToTokens, U: Display>(tokens: T, message: U) -> Self { + return new_spanned(tokens.into_token_stream(), message.to_string()); + + fn new_spanned(tokens: TokenStream, message: String) -> Error { + let mut iter = tokens.into_iter(); + let start = iter.next().map_or_else(Span::call_site, |t| t.span()); + let end = iter.last().map_or(start, |t| t.span()); + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { start, end }), + message, + }], + } + } + } + + /// The source location of the error. + /// + /// Spans are not thread-safe so this function returns `Span::call_site()` + /// if called from a different thread than the one on which the `Error` was + /// originally created. + pub fn span(&self) -> Span { + let SpanRange { start, end } = match self.messages[0].span.get() { + Some(span) => *span, + None => return Span::call_site(), + }; + start.join(end).unwrap_or(start) + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// The [`parse_macro_input!`] macro provides a convenient way to invoke + /// this method correctly in a procedural macro. + /// + /// [`compile_error!`]: core::compile_error! + /// [`parse_macro_input!`]: crate::parse_macro_input! + pub fn to_compile_error(&self) -> TokenStream { + let mut tokens = TokenStream::new(); + for msg in &self.messages { + ErrorMessage::to_compile_error(msg, &mut tokens); + } + tokens + } + + /// Render the error as an invocation of [`compile_error!`]. + /// + /// [`compile_error!`]: core::compile_error! + /// + /// # Example + /// + /// ``` + /// # extern crate proc_macro; + /// # + /// use proc_macro::TokenStream; + /// use syn::{parse_macro_input, DeriveInput, Error}; + /// + /// # const _: &str = stringify! { + /// #[proc_macro_derive(MyTrait)] + /// # }; + /// pub fn derive_my_trait(input: TokenStream) -> TokenStream { + /// let input = parse_macro_input!(input as DeriveInput); + /// my_trait::expand(input) + /// .unwrap_or_else(Error::into_compile_error) + /// .into() + /// } + /// + /// mod my_trait { + /// use proc_macro2::TokenStream; + /// use syn::{DeriveInput, Result}; + /// + /// pub(crate) fn expand(input: DeriveInput) -> Result<TokenStream> { + /// /* ... */ + /// # unimplemented!() + /// } + /// } + /// ``` + pub fn into_compile_error(self) -> TokenStream { + self.to_compile_error() + } + + /// Add another error message to self such that when `to_compile_error()` is + /// called, both errors will be emitted together. + pub fn combine(&mut self, another: Error) { + self.messages.extend(another.messages); + } +} + +impl ErrorMessage { + fn to_compile_error(&self, tokens: &mut TokenStream) { + let (start, end) = match self.span.get() { + Some(range) => (range.start, range.end), + None => (Span::call_site(), Span::call_site()), + }; + + // ::core::compile_error!($message) + tokens.append(TokenTree::Punct(Punct::new_spanned( + ':', + Spacing::Joint, + start, + ))); + tokens.append(TokenTree::Punct(Punct::new_spanned( + ':', + Spacing::Alone, + start, + ))); + tokens.append(TokenTree::Ident(Ident::new("core", start))); + tokens.append(TokenTree::Punct(Punct::new_spanned( + ':', + Spacing::Joint, + start, + ))); + tokens.append(TokenTree::Punct(Punct::new_spanned( + ':', + Spacing::Alone, + start, + ))); + tokens.append(TokenTree::Ident(Ident::new("compile_error", start))); + tokens.append(TokenTree::Punct(Punct::new_spanned( + '!', + Spacing::Alone, + start, + ))); + tokens.append(TokenTree::Group({ + let mut group = Group::new( + Delimiter::Brace, + TokenStream::from({ + let mut string = Literal::string(&self.message); + string.set_span(end); + TokenTree::Literal(string) + }), + ); + group.set_span(end); + group + })); + } +} + +#[cfg(feature = "parsing")] +pub(crate) fn new_at<T: Display>(scope: Span, cursor: Cursor, message: T) -> Error { + if cursor.eof() { + Error::new(scope, format!("unexpected end of input, {}", message)) + } else { + let span = crate::buffer::open_span_of_group(cursor); + Error::new(span, message) + } +} + +#[cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))] +pub(crate) fn new2<T: Display>(start: Span, end: Span, message: T) -> Error { + return new2(start, end, message.to_string()); + + fn new2(start: Span, end: Span, message: String) -> Error { + Error { + messages: vec![ErrorMessage { + span: ThreadBound::new(SpanRange { start, end }), + message, + }], + } + } +} + +impl Debug for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + if self.messages.len() == 1 { + formatter + .debug_tuple("Error") + .field(&self.messages[0]) + .finish() + } else { + formatter + .debug_tuple("Error") + .field(&self.messages) + .finish() + } + } +} + +impl Debug for ErrorMessage { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.message, formatter) + } +} + +impl Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(&self.messages[0].message) + } +} + +impl Clone for Error { + fn clone(&self) -> Self { + Error { + messages: self.messages.clone(), + } + } +} + +impl Clone for ErrorMessage { + fn clone(&self) -> Self { + ErrorMessage { + span: self.span, + message: self.message.clone(), + } + } +} + +impl Clone for SpanRange { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for SpanRange {} + +// TODO: impl core::error::Error (requires Rust 1.81+) +impl std::error::Error for Error {} + +impl From<LexError> for Error { + fn from(err: LexError) -> Self { + Error::new(err.span(), err) + } +} + +impl IntoIterator for Error { + type Item = Error; + type IntoIter = IntoIter; + + fn into_iter(self) -> Self::IntoIter { + IntoIter { + messages: self.messages.into_iter(), + } + } +} + +pub struct IntoIter { + messages: vec::IntoIter<ErrorMessage>, +} + +impl Iterator for IntoIter { + type Item = Error; + + fn next(&mut self) -> Option<Self::Item> { + Some(Error { + messages: vec![self.messages.next()?], + }) + } +} + +impl<'a> IntoIterator for &'a Error { + type Item = Error; + type IntoIter = Iter<'a>; + + fn into_iter(self) -> Self::IntoIter { + Iter { + messages: self.messages.iter(), + } + } +} + +pub struct Iter<'a> { + messages: slice::Iter<'a, ErrorMessage>, +} + +impl<'a> Iterator for Iter<'a> { + type Item = Error; + + fn next(&mut self) -> Option<Self::Item> { + Some(Error { + messages: vec![self.messages.next()?.clone()], + }) + } +} + +impl Extend<Error> for Error { + fn extend<T: IntoIterator<Item = Error>>(&mut self, iter: T) { + for err in iter { + self.combine(err); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/export.rs b/third_party/crates/syn-2.0.119/src/export.rs new file mode 100644 index 0000000..eca67f6 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/export.rs @@ -0,0 +1,73 @@ +#[doc(hidden)] +pub use core::clone::Clone; +#[doc(hidden)] +pub use core::cmp::{Eq, PartialEq}; +#[doc(hidden)] +pub use core::concat; +#[doc(hidden)] +pub use core::default::Default; +#[doc(hidden)] +pub use core::fmt::Debug; +#[doc(hidden)] +pub use core::hash::{Hash, Hasher}; +#[doc(hidden)] +pub use core::marker::Copy; +#[doc(hidden)] +pub use core::option::Option::{None, Some}; +#[doc(hidden)] +pub use core::result::Result::{Err, Ok}; +#[doc(hidden)] +pub use core::stringify; + +#[doc(hidden)] +pub type Formatter<'a> = core::fmt::Formatter<'a>; +#[doc(hidden)] +pub type FmtResult = core::fmt::Result; + +#[doc(hidden)] +pub type bool = core::primitive::bool; +#[doc(hidden)] +pub type str = core::primitive::str; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use quote; + +#[doc(hidden)] +pub type Span = proc_macro2::Span; +#[doc(hidden)] +pub type TokenStream2 = proc_macro2::TokenStream; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::group::{parse_braces, parse_brackets, parse_parens}; + +#[doc(hidden)] +pub use crate::span::IntoSpans; + +#[cfg(all(feature = "parsing", feature = "printing"))] +#[doc(hidden)] +pub use crate::parse_quote::parse as parse_quote; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::parsing::{peek_punct, punct as parse_punct}; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use crate::token::printing::punct as print_punct; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::private::CustomToken; + +#[cfg(feature = "proc-macro")] +#[doc(hidden)] +pub type TokenStream = proc_macro::TokenStream; + +#[cfg(feature = "printing")] +#[doc(hidden)] +pub use quote::{ToTokens, TokenStreamExt}; + +#[doc(hidden)] +pub struct private(pub(crate) ()); diff --git a/third_party/crates/syn-2.0.119/src/expr.rs b/third_party/crates/syn-2.0.119/src/expr.rs new file mode 100644 index 0000000..0388e70 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/expr.rs @@ -0,0 +1,4179 @@ +use crate::attr::Attribute; +#[cfg(all(feature = "parsing", feature = "full"))] +use crate::error::Result; +#[cfg(feature = "parsing")] +use crate::ext::IdentExt as _; +#[cfg(feature = "full")] +use crate::generics::BoundLifetimes; +use crate::ident::Ident; +#[cfg(any(feature = "parsing", feature = "full"))] +use crate::lifetime::Lifetime; +use crate::lit::Lit; +use crate::mac::Macro; +use crate::op::{BinOp, UnOp}; +#[cfg(feature = "parsing")] +use crate::parse::ParseStream; +#[cfg(feature = "full")] +use crate::pat::Pat; +use crate::path::{AngleBracketedGenericArguments, Path, QSelf}; +use crate::punctuated::Punctuated; +#[cfg(feature = "full")] +use crate::stmt::Block; +use crate::token; +#[cfg(feature = "full")] +use crate::ty::ReturnType; +use crate::ty::Type; +use alloc::boxed::Box; +use alloc::vec::Vec; +#[cfg(feature = "printing")] +use core::fmt::{self, Display}; +use core::hash::{Hash, Hasher}; +#[cfg(all(feature = "parsing", feature = "full"))] +use core::mem; +use proc_macro2::{Span, TokenStream}; +#[cfg(feature = "printing")] +use quote::IdentFragment; + +ast_enum_of_structs! { + /// A Rust expression. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature, but most of the variants are not available unless "full" is enabled.* + /// + /// # Syntax tree enums + /// + /// This type is a syntax tree enum. In Syn this and other syntax tree enums + /// are designed to be traversed using the following rebinding idiom. + /// + /// ``` + /// # use syn::Expr; + /// # + /// # fn example(expr: Expr) { + /// # const IGNORE: &str = stringify! { + /// let expr: Expr = /* ... */; + /// # }; + /// match expr { + /// Expr::MethodCall(expr) => { + /// /* ... */ + /// } + /// Expr::Cast(expr) => { + /// /* ... */ + /// } + /// Expr::If(expr) => { + /// /* ... */ + /// } + /// + /// /* ... */ + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// We begin with a variable `expr` of type `Expr` that has no fields + /// (because it is an enum), and by matching on it and rebinding a variable + /// with the same name `expr` we effectively imbue our variable with all of + /// the data fields provided by the variant that it turned out to be. So for + /// example above if we ended up in the `MethodCall` case then we get to use + /// `expr.receiver`, `expr.args` etc; if we ended up in the `If` case we get + /// to use `expr.cond`, `expr.then_branch`, `expr.else_branch`. + /// + /// This approach avoids repeating the variant names twice on every line. + /// + /// ``` + /// # use syn::{Expr, ExprMethodCall}; + /// # + /// # fn example(expr: Expr) { + /// // Repetitive; recommend not doing this. + /// match expr { + /// Expr::MethodCall(ExprMethodCall { method, args, .. }) => { + /// # } + /// # _ => {} + /// # } + /// # } + /// ``` + /// + /// In general, the name to which a syntax tree enum variant is bound should + /// be a suitable name for the complete syntax tree enum type. + /// + /// ``` + /// # use syn::{Expr, ExprField}; + /// # + /// # fn example(discriminant: ExprField) { + /// // Binding is called `base` which is the name I would use if I were + /// // assigning `*discriminant.base` without an `if let`. + /// if let Expr::Tuple(base) = *discriminant.base { + /// # } + /// # } + /// ``` + /// + /// A sign that you may not be choosing the right variable names is if you + /// see names getting repeated in your code, like accessing + /// `receiver.receiver` or `pat.pat` or `cond.cond`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum Expr { + /// A slice literal expression: `[a, b, c, d]`. + Array(ExprArray), + + /// An assignment expression: `a = compute()`. + Assign(ExprAssign), + + /// An async block: `async { ... }`. + Async(ExprAsync), + + /// An await expression: `fut.await`. + Await(ExprAwait), + + /// A binary operation: `a + b`, `a += b`. + Binary(ExprBinary), + + /// A blocked scope: `{ ... }`. + Block(ExprBlock), + + /// A `break`, with an optional label to break and an optional + /// expression. + Break(ExprBreak), + + /// A function call expression: `invoke(a, b)`. + Call(ExprCall), + + /// A cast expression: `foo as f64`. + Cast(ExprCast), + + /// A closure expression: `|a, b| a + b`. + Closure(ExprClosure), + + /// A const block: `const { ... }`. + Const(ExprConst), + + /// A `continue`, with an optional label. + Continue(ExprContinue), + + /// Access of a named struct field (`obj.k`) or unnamed tuple struct + /// field (`obj.0`). + Field(ExprField), + + /// A for loop: `for pat in expr { ... }`. + ForLoop(ExprForLoop), + + /// An expression contained within invisible delimiters. + /// + /// This variant is important for faithfully representing the precedence + /// of expressions and is related to `None`-delimited spans in a + /// `TokenStream`. + Group(ExprGroup), + + /// An `if` expression with an optional `else` block: `if expr { ... } + /// else { ... }`. + /// + /// The `else` branch expression may only be an `If` or `Block` + /// expression, not any of the other types of expression. + If(ExprIf), + + /// A square bracketed indexing expression: `vector[2]`. + Index(ExprIndex), + + /// The inferred value of a const generic argument, denoted `_`. + Infer(ExprInfer), + + /// A `let` guard: `let Some(x) = opt`. + Let(ExprLet), + + /// A literal in place of an expression: `1`, `"foo"`. + Lit(ExprLit), + + /// Conditionless loop: `loop { ... }`. + Loop(ExprLoop), + + /// A macro invocation expression: `format!("{}", q)`. + Macro(ExprMacro), + + /// A `match` expression: `match n { Some(n) => {}, None => {} }`. + Match(ExprMatch), + + /// A method call expression: `x.foo::<T>(a, b)`. + MethodCall(ExprMethodCall), + + /// A parenthesized expression: `(a + b)`. + Paren(ExprParen), + + /// A path like `core::mem::replace` possibly containing generic + /// parameters and a qualified self-type. + /// + /// A plain identifier like `x` is a path of length 1. + Path(ExprPath), + + /// A range expression: `1..2`, `1..`, `..2`, `1..=2`, `..=2`. + Range(ExprRange), + + /// Address-of operation: `&raw const place` or `&raw mut place`. + RawAddr(ExprRawAddr), + + /// A referencing operation: `&a` or `&mut a`. + Reference(ExprReference), + + /// An array literal constructed from one repeated element: `[0u8; N]`. + Repeat(ExprRepeat), + + /// A `return`, with an optional value to be returned. + Return(ExprReturn), + + /// A struct literal expression: `Point { x: 1, y: 1 }`. + /// + /// The `rest` provides the value of the remaining fields as in `S { a: + /// 1, b: 1, ..rest }`. + Struct(ExprStruct), + + /// A try-expression: `expr?`. + Try(ExprTry), + + /// A try block: `try { ... }`. + TryBlock(ExprTryBlock), + + /// A tuple expression: `(a, b, c, d)`. + Tuple(ExprTuple), + + /// A unary operation: `!x`, `*x`, `-x`. + Unary(ExprUnary), + + /// An unsafe block: `unsafe { ... }`. + Unsafe(ExprUnsafe), + + /// Tokens in expression position not interpreted by Syn. + Verbatim(TokenStream), + + /// A while loop: `while expr { ... }`. + While(ExprWhile), + + /// A yield expression: `yield expr`. + Yield(ExprYield), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match expr { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // Expr::Array(expr) => {...} + // Expr::Assign(expr) => {...} + // ... + // Expr::Yield(expr) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// A slice literal expression: `[a, b, c, d]`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprArray #full { + pub attrs: Vec<Attribute>, + pub bracket_token: token::Bracket, + pub elems: Punctuated<Expr, Token![,]>, + } +} + +ast_struct! { + /// An assignment expression: `a = compute()`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAssign #full { + pub attrs: Vec<Attribute>, + pub left: Box<Expr>, + pub eq_token: Token![=], + pub right: Box<Expr>, + } +} + +ast_struct! { + /// An async block: `async { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAsync #full { + pub attrs: Vec<Attribute>, + pub async_token: Token![async], + pub capture: Option<Token![move]>, + pub block: Block, + } +} + +ast_struct! { + /// An await expression: `fut.await`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprAwait #full { + pub attrs: Vec<Attribute>, + pub base: Box<Expr>, + pub dot_token: Token![.], + pub await_token: Token![await], + } +} + +ast_struct! { + /// A binary operation: `a + b`, `a += b`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprBinary { + pub attrs: Vec<Attribute>, + pub left: Box<Expr>, + pub op: BinOp, + pub right: Box<Expr>, + } +} + +ast_struct! { + /// A blocked scope: `{ ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprBlock #full { + pub attrs: Vec<Attribute>, + pub label: Option<Label>, + pub block: Block, + } +} + +ast_struct! { + /// A `break`, with an optional label to break and an optional + /// expression. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprBreak #full { + pub attrs: Vec<Attribute>, + pub break_token: Token![break], + pub label: Option<Lifetime>, + pub expr: Option<Box<Expr>>, + } +} + +ast_struct! { + /// A function call expression: `invoke(a, b)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprCall { + pub attrs: Vec<Attribute>, + pub func: Box<Expr>, + pub paren_token: token::Paren, + pub args: Punctuated<Expr, Token![,]>, + } +} + +ast_struct! { + /// A cast expression: `foo as f64`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprCast { + pub attrs: Vec<Attribute>, + pub expr: Box<Expr>, + pub as_token: Token![as], + pub ty: Box<Type>, + } +} + +ast_struct! { + /// A closure expression: `|a, b| a + b`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprClosure #full { + pub attrs: Vec<Attribute>, + pub lifetimes: Option<BoundLifetimes>, + pub constness: Option<Token![const]>, + pub movability: Option<Token![static]>, + pub asyncness: Option<Token![async]>, + pub capture: Option<Token![move]>, + pub or1_token: Token![|], + pub inputs: Punctuated<Pat, Token![,]>, + pub or2_token: Token![|], + pub output: ReturnType, + pub body: Box<Expr>, + } +} + +ast_struct! { + /// A const block: `const { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprConst #full { + pub attrs: Vec<Attribute>, + pub const_token: Token![const], + pub block: Block, + } +} + +ast_struct! { + /// A `continue`, with an optional label. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprContinue #full { + pub attrs: Vec<Attribute>, + pub continue_token: Token![continue], + pub label: Option<Lifetime>, + } +} + +ast_struct! { + /// Access of a named struct field (`obj.k`) or unnamed tuple struct + /// field (`obj.0`). + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprField { + pub attrs: Vec<Attribute>, + pub base: Box<Expr>, + pub dot_token: Token![.], + pub member: Member, + } +} + +ast_struct! { + /// A for loop: `for pat in expr { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprForLoop #full { + pub attrs: Vec<Attribute>, + pub label: Option<Label>, + pub for_token: Token![for], + pub pat: Box<Pat>, + pub in_token: Token![in], + pub expr: Box<Expr>, + pub body: Block, + } +} + +ast_struct! { + /// An expression contained within invisible delimiters. + /// + /// This variant is important for faithfully representing the precedence + /// of expressions and is related to `None`-delimited spans in a + /// `TokenStream`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprGroup { + pub attrs: Vec<Attribute>, + pub group_token: token::Group, + pub expr: Box<Expr>, + } +} + +ast_struct! { + /// An `if` expression with an optional `else` block: `if expr { ... } + /// else { ... }`. + /// + /// The `else` branch expression may only be an `If` or `Block` + /// expression, not any of the other types of expression. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprIf #full { + pub attrs: Vec<Attribute>, + pub if_token: Token![if], + pub cond: Box<Expr>, + pub then_branch: Block, + pub else_branch: Option<(Token![else], Box<Expr>)>, + } +} + +ast_struct! { + /// A square bracketed indexing expression: `vector[2]`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprIndex { + pub attrs: Vec<Attribute>, + pub expr: Box<Expr>, + pub bracket_token: token::Bracket, + pub index: Box<Expr>, + } +} + +ast_struct! { + /// The inferred value of a const generic argument, denoted `_`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprInfer #full { + pub attrs: Vec<Attribute>, + pub underscore_token: Token![_], + } +} + +ast_struct! { + /// A `let` guard: `let Some(x) = opt`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprLet #full { + pub attrs: Vec<Attribute>, + pub let_token: Token![let], + pub pat: Box<Pat>, + pub eq_token: Token![=], + pub expr: Box<Expr>, + } +} + +ast_struct! { + /// A literal in place of an expression: `1`, `"foo"`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprLit { + pub attrs: Vec<Attribute>, + pub lit: Lit, + } +} + +ast_struct! { + /// Conditionless loop: `loop { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprLoop #full { + pub attrs: Vec<Attribute>, + pub label: Option<Label>, + pub loop_token: Token![loop], + pub body: Block, + } +} + +ast_struct! { + /// A macro invocation expression: `format!("{}", q)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprMacro { + pub attrs: Vec<Attribute>, + pub mac: Macro, + } +} + +ast_struct! { + /// A `match` expression: `match n { Some(n) => {}, None => {} }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprMatch #full { + pub attrs: Vec<Attribute>, + pub match_token: Token![match], + pub expr: Box<Expr>, + pub brace_token: token::Brace, + pub arms: Vec<Arm>, + } +} + +ast_struct! { + /// A method call expression: `x.foo::<T>(a, b)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprMethodCall { + pub attrs: Vec<Attribute>, + pub receiver: Box<Expr>, + pub dot_token: Token![.], + pub method: Ident, + pub turbofish: Option<AngleBracketedGenericArguments>, + pub paren_token: token::Paren, + pub args: Punctuated<Expr, Token![,]>, + } +} + +ast_struct! { + /// A parenthesized expression: `(a + b)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprParen { + pub attrs: Vec<Attribute>, + pub paren_token: token::Paren, + pub expr: Box<Expr>, + } +} + +ast_struct! { + /// A path like `core::mem::replace` possibly containing generic + /// parameters and a qualified self-type. + /// + /// A plain identifier like `x` is a path of length 1. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprPath { + pub attrs: Vec<Attribute>, + pub qself: Option<QSelf>, + pub path: Path, + } +} + +ast_struct! { + /// A range expression: `1..2`, `1..`, `..2`, `1..=2`, `..=2`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprRange #full { + pub attrs: Vec<Attribute>, + pub start: Option<Box<Expr>>, + pub limits: RangeLimits, + pub end: Option<Box<Expr>>, + } +} + +ast_struct! { + /// Address-of operation: `&raw const place` or `&raw mut place`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprRawAddr #full { + pub attrs: Vec<Attribute>, + pub and_token: Token![&], + pub raw: Token![raw], + pub mutability: PointerMutability, + pub expr: Box<Expr>, + } +} + +ast_struct! { + /// A referencing operation: `&a` or `&mut a`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprReference { + pub attrs: Vec<Attribute>, + pub and_token: Token![&], + pub mutability: Option<Token![mut]>, + pub expr: Box<Expr>, + } +} + +ast_struct! { + /// An array literal constructed from one repeated element: `[0u8; N]`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprRepeat #full { + pub attrs: Vec<Attribute>, + pub bracket_token: token::Bracket, + pub expr: Box<Expr>, + pub semi_token: Token![;], + pub len: Box<Expr>, + } +} + +ast_struct! { + /// A `return`, with an optional value to be returned. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprReturn #full { + pub attrs: Vec<Attribute>, + pub return_token: Token![return], + pub expr: Option<Box<Expr>>, + } +} + +ast_struct! { + /// A struct literal expression: `Point { x: 1, y: 1 }`. + /// + /// The `rest` provides the value of the remaining fields as in `S { a: + /// 1, b: 1, ..rest }`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprStruct { + pub attrs: Vec<Attribute>, + pub qself: Option<QSelf>, + pub path: Path, + pub brace_token: token::Brace, + pub fields: Punctuated<FieldValue, Token![,]>, + pub dot2_token: Option<Token![..]>, + pub rest: Option<Box<Expr>>, + } +} + +ast_struct! { + /// A try-expression: `expr?`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprTry #full { + pub attrs: Vec<Attribute>, + pub expr: Box<Expr>, + pub question_token: Token![?], + } +} + +ast_struct! { + /// A try block: `try { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprTryBlock #full { + pub attrs: Vec<Attribute>, + pub try_token: Token![try], + pub block: Block, + } +} + +ast_struct! { + /// A tuple expression: `(a, b, c, d)`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprTuple { + pub attrs: Vec<Attribute>, + pub paren_token: token::Paren, + pub elems: Punctuated<Expr, Token![,]>, + } +} + +ast_struct! { + /// A unary operation: `!x`, `*x`, `-x`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ExprUnary { + pub attrs: Vec<Attribute>, + pub op: UnOp, + pub expr: Box<Expr>, + } +} + +ast_struct! { + /// An unsafe block: `unsafe { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprUnsafe #full { + pub attrs: Vec<Attribute>, + pub unsafe_token: Token![unsafe], + pub block: Block, + } +} + +ast_struct! { + /// A while loop: `while expr { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprWhile #full { + pub attrs: Vec<Attribute>, + pub label: Option<Label>, + pub while_token: Token![while], + pub cond: Box<Expr>, + pub body: Block, + } +} + +ast_struct! { + /// A yield expression: `yield expr`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ExprYield #full { + pub attrs: Vec<Attribute>, + pub yield_token: Token![yield], + pub expr: Option<Box<Expr>>, + } +} + +impl Expr { + /// An unspecified invalid expression. + /// + /// ``` + /// use core::mem; + /// use quote::ToTokens; + /// use syn::{parse_quote, Expr}; + /// + /// fn unparenthesize(e: &mut Expr) { + /// while let Expr::Paren(paren) = e { + /// *e = mem::replace(&mut *paren.expr, Expr::PLACEHOLDER); + /// } + /// } + /// + /// fn main() { + /// let mut e: Expr = parse_quote! { ((1 + 1)) }; + /// unparenthesize(&mut e); + /// assert_eq!("1 + 1", e.to_token_stream().to_string()); + /// } + /// ``` + pub const PLACEHOLDER: Self = Expr::Path(ExprPath { + attrs: Vec::new(), + qself: None, + path: Path { + leading_colon: None, + segments: Punctuated::new(), + }, + }); + + /// An alternative to the primary `Expr::parse` parser (from the [`Parse`] + /// trait) for ambiguous syntactic positions in which a trailing brace + /// should not be taken as part of the expression. + /// + /// [`Parse`]: crate::parse::Parse + /// + /// Rust grammar has an ambiguity where braces sometimes turn a path + /// expression into a struct initialization and sometimes do not. In the + /// following code, the expression `S {}` is one expression. Presumably + /// there is an empty struct `struct S {}` defined somewhere which it is + /// instantiating. + /// + /// ``` + /// # struct S; + /// # impl core::ops::Deref for S { + /// # type Target = bool; + /// # fn deref(&self) -> &Self::Target { + /// # &true + /// # } + /// # } + /// let _ = *S {}; + /// + /// // parsed by rustc as: `*(S {})` + /// ``` + /// + /// We would want to parse the above using `Expr::parse` after the `=` + /// token. + /// + /// But in the following, `S {}` is *not* a struct init expression. + /// + /// ``` + /// # const S: &bool = &true; + /// if *S {} {} + /// + /// // parsed by rustc as: + /// // + /// // if (*S) { + /// // /* empty block */ + /// // } + /// // { + /// // /* another empty block */ + /// // } + /// ``` + /// + /// For that reason we would want to parse if-conditions using + /// `Expr::parse_without_eager_brace` after the `if` token. Same for similar + /// syntactic positions such as the condition expr after a `while` token or + /// the expr at the top of a `match`. + /// + /// The Rust grammar's choices around which way this ambiguity is resolved + /// at various syntactic positions is fairly arbitrary. Really either parse + /// behavior could work in most positions, and language designers just + /// decide each case based on which is more likely to be what the programmer + /// had in mind most of the time. + /// + /// ``` + /// # struct S; + /// # fn doc() -> S { + /// if return S {} {} + /// # unreachable!() + /// # } + /// + /// // parsed by rustc as: + /// // + /// // if (return (S {})) { + /// // } + /// // + /// // but could equally well have been this other arbitrary choice: + /// // + /// // if (return S) { + /// // } + /// // {} + /// ``` + /// + /// Note the grammar ambiguity on trailing braces is distinct from + /// precedence and is not captured by assigning a precedence level to the + /// braced struct init expr in relation to other operators. This can be + /// illustrated by `return 0..S {}` vs `match 0..S {}`. The former parses as + /// `return (0..(S {}))` implying tighter precedence for struct init than + /// `..`, while the latter parses as `match (0..S) {}` implying tighter + /// precedence for `..` than struct init, a contradiction. + #[cfg(all(feature = "full", feature = "parsing"))] + #[cfg_attr(docsrs, doc(cfg(all(feature = "full", feature = "parsing"))))] + pub fn parse_without_eager_brace(input: ParseStream) -> Result<Expr> { + parsing::ambiguous_expr(input, parsing::AllowStruct(false)) + } + + /// An alternative to the primary `Expr::parse` parser (from the [`Parse`] + /// trait) for syntactic positions in which expression boundaries are placed + /// more eagerly than done by the typical expression grammar. This includes + /// expressions at the head of a statement or in the right-hand side of a + /// `match` arm. + /// + /// [`Parse`]: crate::parse::Parse + /// + /// Compare the following cases: + /// + /// 1. + /// ``` + /// # let result = (); + /// # let guard = false; + /// # let cond = true; + /// # let f = true; + /// # let g = f; + /// # + /// let _ = match result { + /// () if guard => if cond { f } else { g } + /// () => false, + /// }; + /// ``` + /// + /// 2. + /// ``` + /// # let cond = true; + /// # let f = (); + /// # let g = f; + /// # + /// let _ = || { + /// if cond { f } else { g } + /// () + /// }; + /// ``` + /// + /// 3. + /// ``` + /// # let cond = true; + /// # let f = || (); + /// # let g = f; + /// # + /// let _ = [if cond { f } else { g } ()]; + /// ``` + /// + /// The same sequence of tokens `if cond { f } else { g } ()` appears in + /// expression position 3 times. The first two syntactic positions use eager + /// placement of expression boundaries, and parse as `Expr::If`, with the + /// adjacent `()` becoming `Pat::Tuple` or `Expr::Tuple`. In contrast, the + /// third case uses standard expression boundaries and parses as + /// `Expr::Call`. + /// + /// As with [`parse_without_eager_brace`], this ambiguity in the Rust + /// grammar is independent of precedence. + /// + /// [`parse_without_eager_brace`]: Self::parse_without_eager_brace + #[cfg(all(feature = "full", feature = "parsing"))] + #[cfg_attr(docsrs, doc(cfg(all(feature = "full", feature = "parsing"))))] + pub fn parse_with_earlier_boundary_rule(input: ParseStream) -> Result<Expr> { + parsing::parse_with_earlier_boundary_rule(input) + } + + /// Returns whether the next token in the parse stream is one that might + /// possibly form the beginning of an expr. + /// + /// This classification is a load-bearing part of the grammar of some Rust + /// expressions, notably `return` and `break`. For example `return < …` will + /// never parse `<` as a binary operator regardless of what comes after, + /// because `<` is a legal starting token for an expression and so it's + /// required to be continued as a return value, such as `return <Struct as + /// Trait>::CONST`. Meanwhile `return > …` treats the `>` as a binary + /// operator because it cannot be a starting token for any Rust expression. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn peek(input: ParseStream) -> bool { + input.peek(Ident::peek_any) && !input.peek(Token![as]) // value name or keyword + || input.peek(token::Paren) // tuple + || input.peek(token::Bracket) // array + || input.peek(token::Brace) // block + || input.peek(Lit) // literal + || input.peek(Token![!]) && !input.peek(Token![!=]) // operator not + || input.peek(Token![-]) && !input.peek(Token![-=]) && !input.peek(Token![->]) // unary minus + || input.peek(Token![*]) && !input.peek(Token![*=]) // dereference + || input.peek(Token![|]) && !input.peek(Token![|=]) // closure + || input.peek(Token![&]) && !input.peek(Token![&=]) // reference + || input.peek(Token![..]) // range + || input.peek(Token![<]) && !input.peek(Token![<=]) && !input.peek(Token![<<=]) // associated path + || input.peek(Token![::]) // absolute path + || input.peek(Lifetime) // labeled loop + || input.peek(Token![#]) // expression attributes + } + + #[cfg(all(feature = "parsing", feature = "full"))] + pub(crate) fn replace_attrs(&mut self, new: Vec<Attribute>) -> Vec<Attribute> { + match self { + Expr::Array(ExprArray { attrs, .. }) + | Expr::Assign(ExprAssign { attrs, .. }) + | Expr::Async(ExprAsync { attrs, .. }) + | Expr::Await(ExprAwait { attrs, .. }) + | Expr::Binary(ExprBinary { attrs, .. }) + | Expr::Block(ExprBlock { attrs, .. }) + | Expr::Break(ExprBreak { attrs, .. }) + | Expr::Call(ExprCall { attrs, .. }) + | Expr::Cast(ExprCast { attrs, .. }) + | Expr::Closure(ExprClosure { attrs, .. }) + | Expr::Const(ExprConst { attrs, .. }) + | Expr::Continue(ExprContinue { attrs, .. }) + | Expr::Field(ExprField { attrs, .. }) + | Expr::ForLoop(ExprForLoop { attrs, .. }) + | Expr::Group(ExprGroup { attrs, .. }) + | Expr::If(ExprIf { attrs, .. }) + | Expr::Index(ExprIndex { attrs, .. }) + | Expr::Infer(ExprInfer { attrs, .. }) + | Expr::Let(ExprLet { attrs, .. }) + | Expr::Lit(ExprLit { attrs, .. }) + | Expr::Loop(ExprLoop { attrs, .. }) + | Expr::Macro(ExprMacro { attrs, .. }) + | Expr::Match(ExprMatch { attrs, .. }) + | Expr::MethodCall(ExprMethodCall { attrs, .. }) + | Expr::Paren(ExprParen { attrs, .. }) + | Expr::Path(ExprPath { attrs, .. }) + | Expr::Range(ExprRange { attrs, .. }) + | Expr::RawAddr(ExprRawAddr { attrs, .. }) + | Expr::Reference(ExprReference { attrs, .. }) + | Expr::Repeat(ExprRepeat { attrs, .. }) + | Expr::Return(ExprReturn { attrs, .. }) + | Expr::Struct(ExprStruct { attrs, .. }) + | Expr::Try(ExprTry { attrs, .. }) + | Expr::TryBlock(ExprTryBlock { attrs, .. }) + | Expr::Tuple(ExprTuple { attrs, .. }) + | Expr::Unary(ExprUnary { attrs, .. }) + | Expr::Unsafe(ExprUnsafe { attrs, .. }) + | Expr::While(ExprWhile { attrs, .. }) + | Expr::Yield(ExprYield { attrs, .. }) => mem::replace(attrs, new), + Expr::Verbatim(_) => Vec::new(), + } + } +} + +ast_enum! { + /// A struct or tuple struct field accessed in a struct literal or field + /// expression. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Member { + /// A named field like `self.x`. + Named(Ident), + /// An unnamed field like `self.0`. + Unnamed(Index), + } +} + +impl From<Ident> for Member { + fn from(ident: Ident) -> Member { + Member::Named(ident) + } +} + +impl From<Index> for Member { + fn from(index: Index) -> Member { + Member::Unnamed(index) + } +} + +impl From<usize> for Member { + fn from(index: usize) -> Member { + Member::Unnamed(Index::from(index)) + } +} + +impl Eq for Member {} + +impl PartialEq for Member { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Member::Named(this), Member::Named(other)) => this == other, + (Member::Unnamed(this), Member::Unnamed(other)) => this == other, + _ => false, + } + } +} + +impl Hash for Member { + fn hash<H: Hasher>(&self, state: &mut H) { + match self { + Member::Named(m) => m.hash(state), + Member::Unnamed(m) => m.hash(state), + } + } +} + +#[cfg(feature = "printing")] +impl IdentFragment for Member { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self { + Member::Named(m) => Display::fmt(m, formatter), + Member::Unnamed(m) => Display::fmt(&m.index, formatter), + } + } + + fn span(&self) -> Option<Span> { + match self { + Member::Named(m) => Some(m.span()), + Member::Unnamed(m) => Some(m.span), + } + } +} + +#[cfg(any(feature = "parsing", feature = "printing"))] +impl Member { + pub(crate) fn is_named(&self) -> bool { + match self { + Member::Named(_) => true, + Member::Unnamed(_) => false, + } + } +} + +ast_struct! { + /// The index of an unnamed tuple struct field. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Index { + pub index: u32, + pub span: Span, + } +} + +impl From<usize> for Index { + fn from(index: usize) -> Index { + assert!(index < u32::MAX as usize); + Index { + index: index as u32, + span: Span::call_site(), + } + } +} + +impl Eq for Index {} + +impl PartialEq for Index { + fn eq(&self, other: &Self) -> bool { + self.index == other.index + } +} + +impl Hash for Index { + fn hash<H: Hasher>(&self, state: &mut H) { + self.index.hash(state); + } +} + +#[cfg(feature = "printing")] +impl IdentFragment for Index { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.index, formatter) + } + + fn span(&self) -> Option<Span> { + Some(self.span) + } +} + +ast_struct! { + /// A field-value pair in a struct literal. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct FieldValue { + pub attrs: Vec<Attribute>, + pub member: Member, + + /// The colon in `Struct { x: x }`. If written in shorthand like + /// `Struct { x }`, there is no colon. + pub colon_token: Option<Token![:]>, + + pub expr: Expr, + } +} + +#[cfg(feature = "full")] +ast_struct! { + /// A lifetime labeling a `for`, `while`, or `loop`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct Label { + pub name: Lifetime, + pub colon_token: Token![:], + } +} + +#[cfg(feature = "full")] +ast_struct! { + /// One arm of a `match` expression: `0..=10 => { return true; }`. + /// + /// As in: + /// + /// ``` + /// # fn f() -> bool { + /// # let n = 0; + /// match n { + /// 0..=10 => { + /// return true; + /// } + /// // ... + /// # _ => {} + /// } + /// # false + /// # } + /// ``` + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct Arm { + pub attrs: Vec<Attribute>, + pub pat: Pat, + pub guard: Option<(Token![if], Box<Expr>)>, + pub fat_arrow_token: Token![=>], + pub body: Box<Expr>, + pub comma: Option<Token![,]>, + } +} + +#[cfg(feature = "full")] +ast_enum! { + /// Limit types of a range, inclusive or exclusive. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub enum RangeLimits { + /// Inclusive at the beginning, exclusive at the end. + HalfOpen(Token![..]), + /// Inclusive at the beginning and end. + Closed(Token![..=]), + } +} + +#[cfg(feature = "full")] +ast_enum! { + /// Mutability of a raw pointer (`*const T`, `*mut T`), in which non-mutable + /// isn't the implicit default. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub enum PointerMutability { + Const(Token![const]), + Mut(Token![mut]), + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + #[cfg(feature = "full")] + use crate::attr; + use crate::attr::Attribute; + #[cfg(feature = "full")] + use crate::buffer::Cursor; + #[cfg(feature = "full")] + use crate::classify; + use crate::error::{Error, Result}; + #[cfg(feature = "full")] + use crate::expr::{ + Arm, ExprArray, ExprAssign, ExprAsync, ExprAwait, ExprBlock, ExprBreak, ExprClosure, + ExprConst, ExprContinue, ExprForLoop, ExprIf, ExprInfer, ExprLet, ExprLoop, ExprMatch, + ExprRange, ExprRawAddr, ExprRepeat, ExprReturn, ExprTry, ExprTryBlock, ExprUnsafe, + ExprWhile, ExprYield, Label, PointerMutability, RangeLimits, + }; + use crate::expr::{ + Expr, ExprBinary, ExprCall, ExprCast, ExprField, ExprGroup, ExprIndex, ExprLit, ExprMacro, + ExprMethodCall, ExprParen, ExprPath, ExprReference, ExprStruct, ExprTuple, ExprUnary, + FieldValue, Index, Member, + }; + #[cfg(feature = "full")] + use crate::generics::{self, BoundLifetimes}; + use crate::ident::Ident; + #[cfg(feature = "full")] + use crate::lifetime::Lifetime; + use crate::lit::{Lit, LitFloat, LitInt}; + use crate::mac::{self, Macro}; + use crate::op::BinOp; + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + #[cfg(feature = "full")] + use crate::pat::{Pat, PatType}; + use crate::path::{self, AngleBracketedGenericArguments, Path, QSelf}; + use crate::precedence::Precedence; + use crate::punctuated::Punctuated; + #[cfg(feature = "full")] + use crate::stmt::Block; + use crate::token; + use crate::ty; + #[cfg(feature = "full")] + use crate::ty::{ReturnType, Type}; + use crate::verbatim; + use alloc::boxed::Box; + use alloc::format; + use alloc::string::ToString; + use alloc::vec::Vec; + use core::mem; + #[cfg(feature = "full")] + use proc_macro2::{Span, TokenStream}; + + // When we're parsing expressions which occur before blocks, like in an if + // statement's condition, we cannot parse a struct literal. + // + // Struct literals are ambiguous in certain positions + // https://github.com/rust-lang/rfcs/pull/92 + #[cfg(feature = "full")] + pub(super) struct AllowStruct(pub bool); + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Expr { + fn parse(input: ParseStream) -> Result<Self> { + ambiguous_expr( + input, + #[cfg(feature = "full")] + AllowStruct(true), + ) + } + } + + #[cfg(feature = "full")] + pub(super) fn parse_with_earlier_boundary_rule(input: ParseStream) -> Result<Expr> { + let mut attrs = input.call(expr_attrs)?; + let mut expr = if input.peek(token::Group) { + let allow_struct = AllowStruct(true); + let atom = expr_group(input, allow_struct)?; + if continue_parsing_early(&atom) { + trailer_helper(input, atom)? + } else { + atom + } + } else if input.peek(Token![if]) { + Expr::If(input.parse()?) + } else if input.peek(Token![while]) { + Expr::While(input.parse()?) + } else if input.peek(Token![for]) + && !generics::parsing::choose_generics_over_qpath_after_keyword(input) + { + Expr::ForLoop(input.parse()?) + } else if input.peek(Token![loop]) { + Expr::Loop(input.parse()?) + } else if input.peek(Token![match]) { + Expr::Match(input.parse()?) + } else if input.peek(Token![try]) && input.peek2(token::Brace) { + Expr::TryBlock(input.parse()?) + } else if input.peek(Token![unsafe]) { + Expr::Unsafe(input.parse()?) + } else if input.peek(Token![const]) && input.peek2(token::Brace) { + Expr::Const(input.parse()?) + } else if input.peek(token::Brace) { + Expr::Block(input.parse()?) + } else if input.peek(Lifetime) { + atom_labeled(input)? + } else { + let allow_struct = AllowStruct(true); + unary_expr(input, allow_struct)? + }; + + if continue_parsing_early(&expr) { + attrs.extend(expr.replace_attrs(Vec::new())); + expr.replace_attrs(attrs); + + let allow_struct = AllowStruct(true); + return parse_expr(input, expr, allow_struct, Precedence::MIN); + } + + if input.peek(Token![.]) && !input.peek(Token![..]) || input.peek(Token![?]) { + expr = trailer_helper(input, expr)?; + + attrs.extend(expr.replace_attrs(Vec::new())); + expr.replace_attrs(attrs); + + let allow_struct = AllowStruct(true); + return parse_expr(input, expr, allow_struct, Precedence::MIN); + } + + attrs.extend(expr.replace_attrs(Vec::new())); + expr.replace_attrs(attrs); + Ok(expr) + } + + #[cfg(feature = "full")] + impl Copy for AllowStruct {} + + #[cfg(feature = "full")] + impl Clone for AllowStruct { + fn clone(&self) -> Self { + *self + } + } + + #[cfg(feature = "full")] + fn parse_expr( + input: ParseStream, + mut lhs: Expr, + allow_struct: AllowStruct, + base: Precedence, + ) -> Result<Expr> { + loop { + let ahead = input.fork(); + if let Expr::Range(_) = lhs { + // A range cannot be the left-hand side of another binary operator. + break; + } else if let Ok(op) = ahead.parse::<BinOp>() { + let precedence = Precedence::of_binop(&op); + if precedence < base { + break; + } + if precedence == Precedence::Assign { + if let Expr::Range(_) = lhs { + break; + } + } + if precedence == Precedence::Compare { + if let Expr::Binary(lhs) = &lhs { + if Precedence::of_binop(&lhs.op) == Precedence::Compare { + return Err(input.error("comparison operators cannot be chained")); + } + } + } + input.advance_to(&ahead); + let right = parse_binop_rhs(input, allow_struct, precedence)?; + lhs = Expr::Binary(ExprBinary { + attrs: Vec::new(), + left: Box::new(lhs), + op, + right, + }); + } else if Precedence::Assign >= base + && input.peek(Token![=]) + && !input.peek(Token![=>]) + && match lhs { + Expr::Range(_) => false, + _ => true, + } + { + let eq_token: Token![=] = input.parse()?; + let right = parse_binop_rhs(input, allow_struct, Precedence::Assign)?; + lhs = Expr::Assign(ExprAssign { + attrs: Vec::new(), + left: Box::new(lhs), + eq_token, + right, + }); + } else if Precedence::Range >= base && input.peek(Token![..]) { + let limits: RangeLimits = input.parse()?; + let end = parse_range_end(input, &limits, allow_struct)?; + lhs = Expr::Range(ExprRange { + attrs: Vec::new(), + start: Some(Box::new(lhs)), + limits, + end, + }); + } else if Precedence::Cast >= base && input.peek(Token![as]) { + let as_token: Token![as] = input.parse()?; + let allow_plus = false; + let allow_group_generic = false; + let ty = ty::parsing::ambig_ty(input, allow_plus, allow_group_generic)?; + check_cast(input)?; + lhs = Expr::Cast(ExprCast { + attrs: Vec::new(), + expr: Box::new(lhs), + as_token, + ty: Box::new(ty), + }); + } else { + break; + } + } + Ok(lhs) + } + + #[cfg(not(feature = "full"))] + fn parse_expr(input: ParseStream, mut lhs: Expr, base: Precedence) -> Result<Expr> { + loop { + let ahead = input.fork(); + if let Ok(op) = ahead.parse::<BinOp>() { + let precedence = Precedence::of_binop(&op); + if precedence < base { + break; + } + if precedence == Precedence::Compare { + if let Expr::Binary(lhs) = &lhs { + if Precedence::of_binop(&lhs.op) == Precedence::Compare { + return Err(input.error("comparison operators cannot be chained")); + } + } + } + input.advance_to(&ahead); + let right = parse_binop_rhs(input, precedence)?; + lhs = Expr::Binary(ExprBinary { + attrs: Vec::new(), + left: Box::new(lhs), + op, + right, + }); + } else if Precedence::Cast >= base && input.peek(Token![as]) { + let as_token: Token![as] = input.parse()?; + let allow_plus = false; + let allow_group_generic = false; + let ty = ty::parsing::ambig_ty(input, allow_plus, allow_group_generic)?; + check_cast(input)?; + lhs = Expr::Cast(ExprCast { + attrs: Vec::new(), + expr: Box::new(lhs), + as_token, + ty: Box::new(ty), + }); + } else { + break; + } + } + Ok(lhs) + } + + fn parse_binop_rhs( + input: ParseStream, + #[cfg(feature = "full")] allow_struct: AllowStruct, + precedence: Precedence, + ) -> Result<Box<Expr>> { + let mut rhs = unary_expr( + input, + #[cfg(feature = "full")] + allow_struct, + )?; + loop { + let next = peek_precedence(input); + if next > precedence || next == precedence && precedence == Precedence::Assign { + let cursor = input.cursor(); + rhs = parse_expr( + input, + rhs, + #[cfg(feature = "full")] + allow_struct, + next, + )?; + if cursor == input.cursor() { + // Bespoke grammar restrictions separate from precedence can + // cause parsing to not advance, such as `..a` being + // disallowed in the left-hand side of binary operators, + // even ones that have lower precedence than `..`. + break; + } + } else { + break; + } + } + Ok(Box::new(rhs)) + } + + fn peek_precedence(input: ParseStream) -> Precedence { + if let Ok(op) = input.fork().parse() { + Precedence::of_binop(&op) + } else if input.peek(Token![=]) && !input.peek(Token![=>]) { + Precedence::Assign + } else if input.peek(Token![..]) { + Precedence::Range + } else if input.peek(Token![as]) { + Precedence::Cast + } else { + Precedence::MIN + } + } + + // Parse an arbitrary expression. + pub(super) fn ambiguous_expr( + input: ParseStream, + #[cfg(feature = "full")] allow_struct: AllowStruct, + ) -> Result<Expr> { + let lhs = unary_expr( + input, + #[cfg(feature = "full")] + allow_struct, + )?; + parse_expr( + input, + lhs, + #[cfg(feature = "full")] + allow_struct, + Precedence::MIN, + ) + } + + #[cfg(feature = "full")] + fn expr_attrs(input: ParseStream) -> Result<Vec<Attribute>> { + let mut attrs = Vec::new(); + while !input.peek(token::Group) && input.peek(Token![#]) { + attrs.push(input.call(attr::parsing::single_parse_outer)?); + } + Ok(attrs) + } + + // <UnOp> <trailer> + // & <trailer> + // &mut <trailer> + // box <trailer> + #[cfg(feature = "full")] + fn unary_expr(input: ParseStream, allow_struct: AllowStruct) -> Result<Expr> { + let begin = input.cursor(); + let attrs = input.call(expr_attrs)?; + if input.peek(token::Group) { + return trailer_expr(begin, attrs, input, allow_struct); + } + + if input.peek(Token![&]) { + let and_token: Token![&] = input.parse()?; + let raw: Option<Token![raw]> = if input.peek(Token![raw]) + && (input.peek2(Token![mut]) || input.peek2(Token![const])) + { + Some(input.parse()?) + } else { + None + }; + let mutability: Option<Token![mut]> = input.parse()?; + let const_token: Option<Token![const]> = if raw.is_some() && mutability.is_none() { + Some(input.parse()?) + } else { + None + }; + let expr = Box::new(unary_expr(input, allow_struct)?); + if let Some(raw) = raw { + Ok(Expr::RawAddr(ExprRawAddr { + attrs, + and_token, + raw, + mutability: match mutability { + Some(mut_token) => PointerMutability::Mut(mut_token), + None => PointerMutability::Const(const_token.unwrap()), + }, + expr, + })) + } else { + Ok(Expr::Reference(ExprReference { + attrs, + and_token, + mutability, + expr, + })) + } + } else if input.peek(Token![*]) || input.peek(Token![!]) || input.peek(Token![-]) { + expr_unary(input, attrs, allow_struct).map(Expr::Unary) + } else { + trailer_expr(begin, attrs, input, allow_struct) + } + } + + #[cfg(not(feature = "full"))] + fn unary_expr(input: ParseStream) -> Result<Expr> { + if input.peek(Token![&]) { + Ok(Expr::Reference(ExprReference { + attrs: Vec::new(), + and_token: input.parse()?, + mutability: input.parse()?, + expr: Box::new(unary_expr(input)?), + })) + } else if input.peek(Token![*]) || input.peek(Token![!]) || input.peek(Token![-]) { + Ok(Expr::Unary(ExprUnary { + attrs: Vec::new(), + op: input.parse()?, + expr: Box::new(unary_expr(input)?), + })) + } else { + trailer_expr(input) + } + } + + // <atom> (..<args>) ... + // <atom> . <ident> (..<args>) ... + // <atom> . <ident> ... + // <atom> . <lit> ... + // <atom> [ <expr> ] ... + // <atom> ? ... + #[cfg(feature = "full")] + fn trailer_expr( + begin: Cursor, + mut attrs: Vec<Attribute>, + input: ParseStream, + allow_struct: AllowStruct, + ) -> Result<Expr> { + let atom = atom_expr(input, allow_struct)?; + let mut e = trailer_helper(input, atom)?; + + if let Expr::Verbatim(tokens) = &mut e { + *tokens = verbatim::between(begin, input.cursor()); + } else if !attrs.is_empty() { + if let Expr::Range(range) = e { + let spans: &[Span] = match &range.limits { + RangeLimits::HalfOpen(limits) => &limits.spans, + RangeLimits::Closed(limits) => &limits.spans, + }; + return Err(crate::error::new2( + spans[0], + *spans.last().unwrap(), + "attributes are not allowed on range expressions starting with `..`", + )); + } + let inner_attrs = e.replace_attrs(Vec::new()); + attrs.extend(inner_attrs); + e.replace_attrs(attrs); + } + + Ok(e) + } + + #[cfg(feature = "full")] + fn trailer_helper(input: ParseStream, mut e: Expr) -> Result<Expr> { + loop { + if input.peek(token::Paren) { + let content; + e = Expr::Call(ExprCall { + attrs: Vec::new(), + func: Box::new(e), + paren_token: parenthesized!(content in input), + args: content.parse_terminated(Expr::parse, Token![,])?, + }); + } else if input.peek(Token![.]) + && !input.peek(Token![..]) + && match e { + Expr::Range(_) => false, + _ => true, + } + { + let mut dot_token: Token![.] = input.parse()?; + + let float_token: Option<LitFloat> = input.parse()?; + if let Some(float_token) = float_token { + if multi_index(&mut e, &mut dot_token, float_token)? { + continue; + } + } + + let await_token: Option<Token![await]> = input.parse()?; + if let Some(await_token) = await_token { + e = Expr::Await(ExprAwait { + attrs: Vec::new(), + base: Box::new(e), + dot_token, + await_token, + }); + continue; + } + + let member: Member = input.parse()?; + let turbofish = if member.is_named() && input.peek(Token![::]) { + Some(AngleBracketedGenericArguments::parse_turbofish(input)?) + } else { + None + }; + + if turbofish.is_some() || input.peek(token::Paren) { + if let Member::Named(method) = member { + let content; + e = Expr::MethodCall(ExprMethodCall { + attrs: Vec::new(), + receiver: Box::new(e), + dot_token, + method, + turbofish, + paren_token: parenthesized!(content in input), + args: content.parse_terminated(Expr::parse, Token![,])?, + }); + continue; + } + } + + e = Expr::Field(ExprField { + attrs: Vec::new(), + base: Box::new(e), + dot_token, + member, + }); + } else if input.peek(token::Bracket) { + let content; + e = Expr::Index(ExprIndex { + attrs: Vec::new(), + expr: Box::new(e), + bracket_token: bracketed!(content in input), + index: content.parse()?, + }); + } else if input.peek(Token![?]) + && match e { + Expr::Range(_) => false, + _ => true, + } + { + e = Expr::Try(ExprTry { + attrs: Vec::new(), + expr: Box::new(e), + question_token: input.parse()?, + }); + } else { + break; + } + } + Ok(e) + } + + #[cfg(not(feature = "full"))] + fn trailer_expr(input: ParseStream) -> Result<Expr> { + let mut e = atom_expr(input)?; + + loop { + if input.peek(token::Paren) { + let content; + e = Expr::Call(ExprCall { + attrs: Vec::new(), + func: Box::new(e), + paren_token: parenthesized!(content in input), + args: content.parse_terminated(Expr::parse, Token![,])?, + }); + } else if input.peek(Token![.]) + && !input.peek(Token![..]) + && !input.peek2(Token![await]) + { + let mut dot_token: Token![.] = input.parse()?; + + let float_token: Option<LitFloat> = input.parse()?; + if let Some(float_token) = float_token { + if multi_index(&mut e, &mut dot_token, float_token)? { + continue; + } + } + + let member: Member = input.parse()?; + let turbofish = if member.is_named() && input.peek(Token![::]) { + let colon2_token: Token![::] = input.parse()?; + let turbofish = + AngleBracketedGenericArguments::do_parse(Some(colon2_token), input)?; + Some(turbofish) + } else { + None + }; + + if turbofish.is_some() || input.peek(token::Paren) { + if let Member::Named(method) = member { + let content; + e = Expr::MethodCall(ExprMethodCall { + attrs: Vec::new(), + receiver: Box::new(e), + dot_token, + method, + turbofish, + paren_token: parenthesized!(content in input), + args: content.parse_terminated(Expr::parse, Token![,])?, + }); + continue; + } + } + + e = Expr::Field(ExprField { + attrs: Vec::new(), + base: Box::new(e), + dot_token, + member, + }); + } else if input.peek(token::Bracket) { + let content; + e = Expr::Index(ExprIndex { + attrs: Vec::new(), + expr: Box::new(e), + bracket_token: bracketed!(content in input), + index: content.parse()?, + }); + } else { + break; + } + } + + Ok(e) + } + + // Parse all atomic expressions which don't have to worry about precedence + // interactions, as they are fully contained. + #[cfg(feature = "full")] + fn atom_expr(input: ParseStream, allow_struct: AllowStruct) -> Result<Expr> { + if input.peek(token::Group) { + expr_group(input, allow_struct) + } else if input.peek(Lit) { + input.parse().map(Expr::Lit) + } else if input.peek(Token![async]) + && (input.peek2(token::Brace) || input.peek2(Token![move]) && input.peek3(token::Brace)) + { + input.parse().map(Expr::Async) + } else if input.peek(Token![try]) && input.peek2(token::Brace) { + input.parse().map(Expr::TryBlock) + } else if input.peek(Token![|]) + || input.peek(Token![move]) + || input.peek(Token![for]) + && generics::parsing::choose_generics_over_qpath_after_keyword(input) + || input.peek(Token![const]) && !input.peek2(token::Brace) + || input.peek(Token![static]) + || input.peek(Token![async]) && (input.peek2(Token![|]) || input.peek2(Token![move])) + { + expr_closure(input, allow_struct).map(Expr::Closure) + } else if token::parsing::peek_keyword(input.cursor(), "builtin") && input.peek2(Token![#]) + { + expr_builtin(input) + } else if input.peek(Ident) + || input.peek(Token![::]) + || input.peek(Token![<]) + || input.peek(Token![self]) + || input.peek(Token![Self]) + || input.peek(Token![super]) + || input.peek(Token![crate]) + || input.peek(Token![try]) && (input.peek2(Token![!]) || input.peek2(Token![::])) + { + path_or_macro_or_struct(input, allow_struct) + } else if input.peek(token::Paren) { + paren_or_tuple(input) + } else if input.peek(Token![break]) { + expr_break(input, allow_struct).map(Expr::Break) + } else if input.peek(Token![continue]) { + input.parse().map(Expr::Continue) + } else if input.peek(Token![return]) { + input.parse().map(Expr::Return) + } else if input.peek(Token![become]) { + expr_become(input) + } else if input.peek(token::Bracket) { + array_or_repeat(input) + } else if input.peek(Token![let]) { + expr_let(input, allow_struct).map(Expr::Let) + } else if input.peek(Token![if]) { + input.parse().map(Expr::If) + } else if input.peek(Token![while]) { + input.parse().map(Expr::While) + } else if input.peek(Token![for]) { + input.parse().map(Expr::ForLoop) + } else if input.peek(Token![loop]) { + input.parse().map(Expr::Loop) + } else if input.peek(Token![match]) { + input.parse().map(Expr::Match) + } else if input.peek(Token![yield]) { + input.parse().map(Expr::Yield) + } else if input.peek(Token![unsafe]) { + input.parse().map(Expr::Unsafe) + } else if input.peek(Token![const]) { + input.parse().map(Expr::Const) + } else if input.peek(token::Brace) { + input.parse().map(Expr::Block) + } else if input.peek(Token![..]) { + expr_range(input, allow_struct).map(Expr::Range) + } else if input.peek(Token![_]) { + input.parse().map(Expr::Infer) + } else if input.peek(Lifetime) { + atom_labeled(input) + } else { + Err(input.error("expected an expression")) + } + } + + #[cfg(feature = "full")] + fn atom_labeled(input: ParseStream) -> Result<Expr> { + let the_label: Label = input.parse()?; + let mut expr = if input.peek(Token![while]) { + Expr::While(input.parse()?) + } else if input.peek(Token![for]) { + Expr::ForLoop(input.parse()?) + } else if input.peek(Token![loop]) { + Expr::Loop(input.parse()?) + } else if input.peek(token::Brace) { + Expr::Block(input.parse()?) + } else { + return Err(input.error("expected loop or block expression")); + }; + match &mut expr { + Expr::While(ExprWhile { label, .. }) + | Expr::ForLoop(ExprForLoop { label, .. }) + | Expr::Loop(ExprLoop { label, .. }) + | Expr::Block(ExprBlock { label, .. }) => *label = Some(the_label), + _ => unreachable!(), + } + Ok(expr) + } + + #[cfg(not(feature = "full"))] + fn atom_expr(input: ParseStream) -> Result<Expr> { + if input.peek(token::Group) { + expr_group(input) + } else if input.peek(Lit) { + input.parse().map(Expr::Lit) + } else if input.peek(token::Paren) { + paren_or_tuple(input) + } else if input.peek(Ident) + || input.peek(Token![::]) + || input.peek(Token![<]) + || input.peek(Token![self]) + || input.peek(Token![Self]) + || input.peek(Token![super]) + || input.peek(Token![crate]) + { + path_or_macro_or_struct(input) + } else if input.is_empty() { + Err(input.error("expected an expression")) + } else { + if input.peek(token::Brace) { + let scan = input.fork(); + let content; + braced!(content in scan); + if content.parse::<Expr>().is_ok() && content.is_empty() { + let expr_block = verbatim::between(input.cursor(), scan.cursor()); + input.advance_to(&scan); + return Ok(Expr::Verbatim(expr_block)); + } + } + Err(input.error("unsupported expression; enable syn's features=[\"full\"]")) + } + } + + #[cfg(feature = "full")] + fn expr_builtin(input: ParseStream) -> Result<Expr> { + let begin = input.cursor(); + + token::parsing::keyword(input, "builtin")?; + input.parse::<Token![#]>()?; + input.parse::<Ident>()?; + + let args; + parenthesized!(args in input); + args.parse::<TokenStream>()?; + + Ok(Expr::Verbatim(verbatim::between(begin, input.cursor()))) + } + + fn path_or_macro_or_struct( + input: ParseStream, + #[cfg(feature = "full")] allow_struct: AllowStruct, + ) -> Result<Expr> { + let expr_style = true; + let (qself, path) = path::parsing::qpath(input, expr_style)?; + rest_of_path_or_macro_or_struct( + qself, + path, + input, + #[cfg(feature = "full")] + allow_struct, + ) + } + + fn rest_of_path_or_macro_or_struct( + qself: Option<QSelf>, + path: Path, + input: ParseStream, + #[cfg(feature = "full")] allow_struct: AllowStruct, + ) -> Result<Expr> { + if qself.is_none() + && input.peek(Token![!]) + && !input.peek(Token![!=]) + && path.is_mod_style() + { + let bang_token: Token![!] = input.parse()?; + let (delimiter, tokens) = mac::parse_delimiter(input)?; + return Ok(Expr::Macro(ExprMacro { + attrs: Vec::new(), + mac: Macro { + path, + bang_token, + delimiter, + tokens, + }, + })); + } + + #[cfg(not(feature = "full"))] + let allow_struct = (true,); + if allow_struct.0 && input.peek(token::Brace) { + return expr_struct_helper(input, qself, path).map(Expr::Struct); + } + + Ok(Expr::Path(ExprPath { + attrs: Vec::new(), + qself, + path, + })) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprMacro { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ExprMacro { + attrs: Vec::new(), + mac: input.parse()?, + }) + } + } + + fn paren_or_tuple(input: ParseStream) -> Result<Expr> { + let content; + let paren_token = parenthesized!(content in input); + if content.is_empty() { + return Ok(Expr::Tuple(ExprTuple { + attrs: Vec::new(), + paren_token, + elems: Punctuated::new(), + })); + } + + let first: Expr = content.parse()?; + if content.is_empty() { + return Ok(Expr::Paren(ExprParen { + attrs: Vec::new(), + paren_token, + expr: Box::new(first), + })); + } + + let mut elems = Punctuated::new(); + elems.push_value(first); + while !content.is_empty() { + let punct = content.parse()?; + elems.push_punct(punct); + if content.is_empty() { + break; + } + let value = content.parse()?; + elems.push_value(value); + } + Ok(Expr::Tuple(ExprTuple { + attrs: Vec::new(), + paren_token, + elems, + })) + } + + #[cfg(feature = "full")] + fn array_or_repeat(input: ParseStream) -> Result<Expr> { + let content; + let bracket_token = bracketed!(content in input); + if content.is_empty() { + return Ok(Expr::Array(ExprArray { + attrs: Vec::new(), + bracket_token, + elems: Punctuated::new(), + })); + } + + let first: Expr = content.parse()?; + if content.is_empty() || content.peek(Token![,]) { + let mut elems = Punctuated::new(); + elems.push_value(first); + while !content.is_empty() { + let punct = content.parse()?; + elems.push_punct(punct); + if content.is_empty() { + break; + } + let value = content.parse()?; + elems.push_value(value); + } + Ok(Expr::Array(ExprArray { + attrs: Vec::new(), + bracket_token, + elems, + })) + } else if content.peek(Token![;]) { + let semi_token: Token![;] = content.parse()?; + let len: Expr = content.parse()?; + Ok(Expr::Repeat(ExprRepeat { + attrs: Vec::new(), + bracket_token, + expr: Box::new(first), + semi_token, + len: Box::new(len), + })) + } else { + Err(content.error("expected `,` or `;`")) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprArray { + fn parse(input: ParseStream) -> Result<Self> { + let content; + let bracket_token = bracketed!(content in input); + let mut elems = Punctuated::new(); + + while !content.is_empty() { + let first: Expr = content.parse()?; + elems.push_value(first); + if content.is_empty() { + break; + } + let punct = content.parse()?; + elems.push_punct(punct); + } + + Ok(ExprArray { + attrs: Vec::new(), + bracket_token, + elems, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprRepeat { + fn parse(input: ParseStream) -> Result<Self> { + let content; + Ok(ExprRepeat { + bracket_token: bracketed!(content in input), + attrs: Vec::new(), + expr: content.parse()?, + semi_token: content.parse()?, + len: content.parse()?, + }) + } + } + + #[cfg(feature = "full")] + fn continue_parsing_early(mut expr: &Expr) -> bool { + while let Expr::Group(group) = expr { + expr = &group.expr; + } + match expr { + Expr::If(_) + | Expr::While(_) + | Expr::ForLoop(_) + | Expr::Loop(_) + | Expr::Match(_) + | Expr::TryBlock(_) + | Expr::Unsafe(_) + | Expr::Const(_) + | Expr::Block(_) => false, + _ => true, + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprLit { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ExprLit { + attrs: Vec::new(), + lit: input.parse()?, + }) + } + } + + fn expr_group( + input: ParseStream, + #[cfg(feature = "full")] allow_struct: AllowStruct, + ) -> Result<Expr> { + let group = crate::group::parse_group(input)?; + let mut inner: Expr = group.content.parse()?; + + match inner { + Expr::Path(mut expr) if expr.attrs.is_empty() => { + let grouped_len = expr.path.segments.len(); + Path::parse_rest(input, &mut expr.path, true)?; + match rest_of_path_or_macro_or_struct( + expr.qself, + expr.path, + input, + #[cfg(feature = "full")] + allow_struct, + )? { + Expr::Path(expr) if expr.path.segments.len() == grouped_len => { + inner = Expr::Path(expr); + } + extended => return Ok(extended), + } + } + _ => {} + } + + Ok(Expr::Group(ExprGroup { + attrs: Vec::new(), + group_token: group.token, + expr: Box::new(inner), + })) + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprParen { + fn parse(input: ParseStream) -> Result<Self> { + let content; + Ok(ExprParen { + attrs: Vec::new(), + paren_token: parenthesized!(content in input), + expr: content.parse()?, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprLet { + fn parse(input: ParseStream) -> Result<Self> { + let allow_struct = AllowStruct(true); + expr_let(input, allow_struct) + } + } + + #[cfg(feature = "full")] + fn expr_let(input: ParseStream, allow_struct: AllowStruct) -> Result<ExprLet> { + Ok(ExprLet { + attrs: Vec::new(), + let_token: input.parse()?, + pat: Box::new(Pat::parse_multi_with_leading_vert(input)?), + eq_token: input.parse()?, + expr: Box::new({ + let lhs = unary_expr(input, allow_struct)?; + parse_expr(input, lhs, allow_struct, Precedence::Compare)? + }), + }) + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprIf { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + + let mut clauses = Vec::new(); + let mut expr; + loop { + let if_token: Token![if] = input.parse()?; + let cond = input.call(Expr::parse_without_eager_brace)?; + let then_branch: Block = input.parse()?; + + expr = ExprIf { + attrs: Vec::new(), + if_token, + cond: Box::new(cond), + then_branch, + else_branch: None, + }; + + if !input.peek(Token![else]) { + break; + } + + let else_token: Token![else] = input.parse()?; + let lookahead = input.lookahead1(); + if lookahead.peek(Token![if]) { + expr.else_branch = Some((else_token, Box::new(Expr::PLACEHOLDER))); + clauses.push(expr); + } else if lookahead.peek(token::Brace) { + expr.else_branch = Some(( + else_token, + Box::new(Expr::Block(ExprBlock { + attrs: Vec::new(), + label: None, + block: input.parse()?, + })), + )); + break; + } else { + return Err(lookahead.error()); + } + } + + while let Some(mut prev) = clauses.pop() { + *prev.else_branch.as_mut().unwrap().1 = Expr::If(expr); + expr = prev; + } + expr.attrs = attrs; + Ok(expr) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprInfer { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ExprInfer { + attrs: input.call(Attribute::parse_outer)?, + underscore_token: input.parse()?, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprForLoop { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = input.call(Attribute::parse_outer)?; + let label: Option<Label> = input.parse()?; + let for_token: Token![for] = input.parse()?; + + let pat = Pat::parse_multi_with_leading_vert(input)?; + + let in_token: Token![in] = input.parse()?; + let expr: Expr = input.call(Expr::parse_without_eager_brace)?; + + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + let stmts = content.call(Block::parse_within)?; + + Ok(ExprForLoop { + attrs, + label, + for_token, + pat: Box::new(pat), + in_token, + expr: Box::new(expr), + body: Block { brace_token, stmts }, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprLoop { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = input.call(Attribute::parse_outer)?; + let label: Option<Label> = input.parse()?; + let loop_token: Token![loop] = input.parse()?; + + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + let stmts = content.call(Block::parse_within)?; + + Ok(ExprLoop { + attrs, + label, + loop_token, + body: Block { brace_token, stmts }, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprMatch { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = input.call(Attribute::parse_outer)?; + let match_token: Token![match] = input.parse()?; + let expr = Expr::parse_without_eager_brace(input)?; + + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + + let arms = Arm::parse_multiple(&content)?; + + Ok(ExprMatch { + attrs, + match_token, + expr: Box::new(expr), + brace_token, + arms, + }) + } + } + + macro_rules! impl_by_parsing_expr { + ( + $( + $expr_type:ty, $variant:ident, $msg:expr, + )* + ) => { + $( + #[cfg(all(feature = "full", feature = "printing"))] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for $expr_type { + fn parse(input: ParseStream) -> Result<Self> { + let mut expr: Expr = input.parse()?; + loop { + match expr { + Expr::$variant(inner) => return Ok(inner), + Expr::Group(next) => expr = *next.expr, + _ => return Err(Error::new_spanned(expr, $msg)), + } + } + } + } + )* + }; + } + + impl_by_parsing_expr! { + ExprAssign, Assign, "expected assignment expression", + ExprAwait, Await, "expected await expression", + ExprBinary, Binary, "expected binary operation", + ExprCall, Call, "expected function call expression", + ExprCast, Cast, "expected cast expression", + ExprField, Field, "expected struct field access", + ExprIndex, Index, "expected indexing expression", + ExprMethodCall, MethodCall, "expected method call expression", + ExprRange, Range, "expected range expression", + ExprTry, Try, "expected try expression", + ExprTuple, Tuple, "expected tuple expression", + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprUnary { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = Vec::new(); + let allow_struct = AllowStruct(true); + expr_unary(input, attrs, allow_struct) + } + } + + #[cfg(feature = "full")] + fn expr_unary( + input: ParseStream, + attrs: Vec<Attribute>, + allow_struct: AllowStruct, + ) -> Result<ExprUnary> { + Ok(ExprUnary { + attrs, + op: input.parse()?, + expr: Box::new(unary_expr(input, allow_struct)?), + }) + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprClosure { + fn parse(input: ParseStream) -> Result<Self> { + let allow_struct = AllowStruct(true); + expr_closure(input, allow_struct) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprRawAddr { + fn parse(input: ParseStream) -> Result<Self> { + let allow_struct = AllowStruct(true); + Ok(ExprRawAddr { + attrs: Vec::new(), + and_token: input.parse()?, + raw: input.parse()?, + mutability: input.parse()?, + expr: Box::new(unary_expr(input, allow_struct)?), + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprReference { + fn parse(input: ParseStream) -> Result<Self> { + let allow_struct = AllowStruct(true); + Ok(ExprReference { + attrs: Vec::new(), + and_token: input.parse()?, + mutability: input.parse()?, + expr: Box::new(unary_expr(input, allow_struct)?), + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprBreak { + fn parse(input: ParseStream) -> Result<Self> { + let allow_struct = AllowStruct(true); + expr_break(input, allow_struct) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprReturn { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ExprReturn { + attrs: Vec::new(), + return_token: input.parse()?, + expr: { + if Expr::peek(input) { + Some(input.parse()?) + } else { + None + } + }, + }) + } + } + + #[cfg(feature = "full")] + fn expr_become(input: ParseStream) -> Result<Expr> { + let begin = input.cursor(); + input.parse::<Token![become]>()?; + input.parse::<Expr>()?; + Ok(Expr::Verbatim(verbatim::between(begin, input.cursor()))) + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprTryBlock { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ExprTryBlock { + attrs: Vec::new(), + try_token: input.parse()?, + block: input.parse()?, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprYield { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ExprYield { + attrs: Vec::new(), + yield_token: input.parse()?, + expr: { + if Expr::peek(input) { + Some(input.parse()?) + } else { + None + } + }, + }) + } + } + + #[cfg(feature = "full")] + fn expr_closure(input: ParseStream, allow_struct: AllowStruct) -> Result<ExprClosure> { + let lifetimes: Option<BoundLifetimes> = input.parse()?; + let constness: Option<Token![const]> = input.parse()?; + let movability: Option<Token![static]> = input.parse()?; + let asyncness: Option<Token![async]> = input.parse()?; + let capture: Option<Token![move]> = input.parse()?; + let or1_token: Token![|] = input.parse()?; + + let mut inputs = Punctuated::new(); + loop { + if input.peek(Token![|]) { + break; + } + let value = closure_arg(input)?; + inputs.push_value(value); + if input.peek(Token![|]) { + break; + } + let punct: Token![,] = input.parse()?; + inputs.push_punct(punct); + } + + let or2_token: Token![|] = input.parse()?; + + let (output, body) = if input.peek(Token![->]) { + let arrow_token: Token![->] = input.parse()?; + let ty: Type = input.parse()?; + let body: Block = input.parse()?; + let output = ReturnType::Type(arrow_token, Box::new(ty)); + let block = Expr::Block(ExprBlock { + attrs: Vec::new(), + label: None, + block: body, + }); + (output, block) + } else { + let body = ambiguous_expr(input, allow_struct)?; + (ReturnType::Default, body) + }; + + Ok(ExprClosure { + attrs: Vec::new(), + lifetimes, + constness, + movability, + asyncness, + capture, + or1_token, + inputs, + or2_token, + output, + body: Box::new(body), + }) + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprAsync { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ExprAsync { + attrs: Vec::new(), + async_token: input.parse()?, + capture: input.parse()?, + block: input.parse()?, + }) + } + } + + #[cfg(feature = "full")] + fn closure_arg(input: ParseStream) -> Result<Pat> { + let attrs = input.call(Attribute::parse_outer)?; + let mut pat = Pat::parse_single(input)?; + + if input.peek(Token![:]) { + Ok(Pat::Type(PatType { + attrs, + pat: Box::new(pat), + colon_token: input.parse()?, + ty: input.parse()?, + })) + } else { + match &mut pat { + Pat::Const(pat) => pat.attrs = attrs, + Pat::Ident(pat) => pat.attrs = attrs, + Pat::Lit(pat) => pat.attrs = attrs, + Pat::Macro(pat) => pat.attrs = attrs, + Pat::Or(pat) => pat.attrs = attrs, + Pat::Paren(pat) => pat.attrs = attrs, + Pat::Path(pat) => pat.attrs = attrs, + Pat::Range(pat) => pat.attrs = attrs, + Pat::Reference(pat) => pat.attrs = attrs, + Pat::Rest(pat) => pat.attrs = attrs, + Pat::Slice(pat) => pat.attrs = attrs, + Pat::Struct(pat) => pat.attrs = attrs, + Pat::Tuple(pat) => pat.attrs = attrs, + Pat::TupleStruct(pat) => pat.attrs = attrs, + Pat::Type(_) => unreachable!(), + Pat::Verbatim(_) => {} + Pat::Wild(pat) => pat.attrs = attrs, + } + Ok(pat) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprWhile { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = input.call(Attribute::parse_outer)?; + let label: Option<Label> = input.parse()?; + let while_token: Token![while] = input.parse()?; + let cond = Expr::parse_without_eager_brace(input)?; + + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + let stmts = content.call(Block::parse_within)?; + + Ok(ExprWhile { + attrs, + label, + while_token, + cond: Box::new(cond), + body: Block { brace_token, stmts }, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprConst { + fn parse(input: ParseStream) -> Result<Self> { + let const_token: Token![const] = input.parse()?; + + let content; + let brace_token = braced!(content in input); + let inner_attrs = content.call(Attribute::parse_inner)?; + let stmts = content.call(Block::parse_within)?; + + Ok(ExprConst { + attrs: inner_attrs, + const_token, + block: Block { brace_token, stmts }, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Label { + fn parse(input: ParseStream) -> Result<Self> { + Ok(Label { + name: input.parse()?, + colon_token: input.parse()?, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Option<Label> { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Lifetime) { + input.parse().map(Some) + } else { + Ok(None) + } + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprContinue { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ExprContinue { + attrs: Vec::new(), + continue_token: input.parse()?, + label: input.parse()?, + }) + } + } + + #[cfg(feature = "full")] + fn expr_break(input: ParseStream, allow_struct: AllowStruct) -> Result<ExprBreak> { + let break_token: Token![break] = input.parse()?; + + let ahead = input.fork(); + let label: Option<Lifetime> = ahead.parse()?; + if label.is_some() && ahead.peek(Token![:]) { + // Not allowed: `break 'label: loop {...}` + // Parentheses are required. `break ('label: loop {...})` + let _: Expr = input.parse()?; + let start_span = label.unwrap().apostrophe; + let end_span = input.cursor().prev_span(); + return Err(crate::error::new2( + start_span, + end_span, + "parentheses required", + )); + } + + input.advance_to(&ahead); + let expr = if Expr::peek(input) && (allow_struct.0 || !input.peek(token::Brace)) { + Some(input.parse()?) + } else { + None + }; + + Ok(ExprBreak { + attrs: Vec::new(), + break_token, + label, + expr, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FieldValue { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let member: Member = input.parse()?; + let (colon_token, value) = if input.peek(Token![:]) || !member.is_named() { + let colon_token: Token![:] = input.parse()?; + let value: Expr = input.parse()?; + (Some(colon_token), value) + } else if let Member::Named(ident) = &member { + let value = Expr::Path(ExprPath { + attrs: Vec::new(), + qself: None, + path: Path::from(ident.clone()), + }); + (None, value) + } else { + unreachable!() + }; + + Ok(FieldValue { + attrs, + member, + colon_token, + expr: value, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprStruct { + fn parse(input: ParseStream) -> Result<Self> { + let expr_style = true; + let (qself, path) = path::parsing::qpath(input, expr_style)?; + expr_struct_helper(input, qself, path) + } + } + + fn expr_struct_helper( + input: ParseStream, + qself: Option<QSelf>, + path: Path, + ) -> Result<ExprStruct> { + let content; + let brace_token = braced!(content in input); + + let mut fields = Punctuated::new(); + while !content.is_empty() { + if content.peek(Token![..]) { + return Ok(ExprStruct { + attrs: Vec::new(), + qself, + path, + brace_token, + fields, + dot2_token: Some(content.parse()?), + rest: if content.is_empty() { + None + } else { + Some(Box::new(content.parse()?)) + }, + }); + } + + fields.push(content.parse()?); + if content.is_empty() { + break; + } + let punct: Token![,] = content.parse()?; + fields.push_punct(punct); + } + + Ok(ExprStruct { + attrs: Vec::new(), + qself, + path, + brace_token, + fields, + dot2_token: None, + rest: None, + }) + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprUnsafe { + fn parse(input: ParseStream) -> Result<Self> { + let unsafe_token: Token![unsafe] = input.parse()?; + + let content; + let brace_token = braced!(content in input); + let inner_attrs = content.call(Attribute::parse_inner)?; + let stmts = content.call(Block::parse_within)?; + + Ok(ExprUnsafe { + attrs: inner_attrs, + unsafe_token, + block: Block { brace_token, stmts }, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprBlock { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = input.call(Attribute::parse_outer)?; + let label: Option<Label> = input.parse()?; + + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + let stmts = content.call(Block::parse_within)?; + + Ok(ExprBlock { + attrs, + label, + block: Block { brace_token, stmts }, + }) + } + } + + #[cfg(feature = "full")] + fn expr_range(input: ParseStream, allow_struct: AllowStruct) -> Result<ExprRange> { + let limits: RangeLimits = input.parse()?; + let end = parse_range_end(input, &limits, allow_struct)?; + Ok(ExprRange { + attrs: Vec::new(), + start: None, + limits, + end, + }) + } + + #[cfg(feature = "full")] + fn parse_range_end( + input: ParseStream, + limits: &RangeLimits, + allow_struct: AllowStruct, + ) -> Result<Option<Box<Expr>>> { + if matches!(limits, RangeLimits::HalfOpen(_)) + && (input.is_empty() + || input.peek(Token![,]) + || input.peek(Token![;]) + || input.peek(Token![.]) && !input.peek(Token![..]) + || input.peek(Token![?]) + || input.peek(Token![=>]) + || !allow_struct.0 && input.peek(token::Brace) + || input.peek(Token![=]) + || input.peek(Token![+]) + || input.peek(Token![/]) + || input.peek(Token![%]) + || input.peek(Token![^]) + || input.peek(Token![>]) + || input.peek(Token![<=]) + || input.peek(Token![!=]) + || input.peek(Token![-=]) + || input.peek(Token![*=]) + || input.peek(Token![&=]) + || input.peek(Token![|=]) + || input.peek(Token![<<=]) + || input.peek(Token![as])) + { + Ok(None) + } else { + let end = parse_binop_rhs(input, allow_struct, Precedence::Range)?; + Ok(Some(end)) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for RangeLimits { + fn parse(input: ParseStream) -> Result<Self> { + let lookahead = input.lookahead1(); + let dot_dot = lookahead.peek(Token![..]); + let dot_dot_eq = dot_dot && lookahead.peek(Token![..=]); + let dot_dot_dot = dot_dot && input.peek(Token![...]); + if dot_dot_eq { + input.parse().map(RangeLimits::Closed) + } else if dot_dot && !dot_dot_dot { + input.parse().map(RangeLimits::HalfOpen) + } else { + Err(lookahead.error()) + } + } + } + + #[cfg(feature = "full")] + impl RangeLimits { + pub(crate) fn parse_obsolete(input: ParseStream) -> Result<Self> { + let lookahead = input.lookahead1(); + let dot_dot = lookahead.peek(Token![..]); + let dot_dot_eq = dot_dot && lookahead.peek(Token![..=]); + let dot_dot_dot = dot_dot && input.peek(Token![...]); + if dot_dot_eq { + input.parse().map(RangeLimits::Closed) + } else if dot_dot_dot { + let dot3: Token![...] = input.parse()?; + Ok(RangeLimits::Closed(Token![..=](dot3.spans))) + } else if dot_dot { + input.parse().map(RangeLimits::HalfOpen) + } else { + Err(lookahead.error()) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ExprPath { + fn parse(input: ParseStream) -> Result<Self> { + #[cfg(not(feature = "full"))] + let attrs = Vec::new(); + #[cfg(feature = "full")] + let attrs = input.call(Attribute::parse_outer)?; + + let expr_style = true; + let (qself, path) = path::parsing::qpath(input, expr_style)?; + + Ok(ExprPath { attrs, qself, path }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Member { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Ident) { + input.parse().map(Member::Named) + } else if input.peek(LitInt) { + input.parse().map(Member::Unnamed) + } else { + Err(input.error("expected identifier or integer")) + } + } + } + + #[cfg(feature = "full")] + impl Arm { + pub(crate) fn parse_multiple(input: ParseStream) -> Result<Vec<Self>> { + let mut arms = Vec::new(); + while !input.is_empty() { + arms.push(input.call(Arm::parse)?); + } + Ok(arms) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Arm { + fn parse(input: ParseStream) -> Result<Arm> { + let requires_comma; + Ok(Arm { + attrs: input.call(Attribute::parse_outer)?, + pat: Pat::parse_multi_with_leading_vert(input)?, + guard: { + if input.peek(Token![if]) { + let if_token: Token![if] = input.parse()?; + let guard: Expr = input.parse()?; + Some((if_token, Box::new(guard))) + } else { + None + } + }, + fat_arrow_token: input.parse()?, + body: { + let body = Expr::parse_with_earlier_boundary_rule(input)?; + requires_comma = classify::requires_comma_to_be_match_arm(&body); + Box::new(body) + }, + comma: { + if requires_comma && !input.is_empty() { + Some(input.parse()?) + } else { + input.parse()? + } + }, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Index { + fn parse(input: ParseStream) -> Result<Self> { + let lit: LitInt = input.parse()?; + if lit.suffix().is_empty() { + Ok(Index { + index: lit + .base10_digits() + .parse() + .map_err(|err| Error::new(lit.span(), err))?, + span: lit.span(), + }) + } else { + Err(Error::new(lit.span(), "expected unsuffixed integer")) + } + } + } + + fn multi_index(e: &mut Expr, dot_token: &mut Token![.], float: LitFloat) -> Result<bool> { + let float_token = float.token(); + let float_span = float_token.span(); + let mut float_repr = float_token.to_string(); + let trailing_dot = float_repr.ends_with('.'); + if trailing_dot { + float_repr.truncate(float_repr.len() - 1); + } + + let mut offset = 0; + for part in float_repr.split('.') { + let mut index: Index = + crate::parse_str(part).map_err(|err| Error::new(float_span, err))?; + let part_end = offset + part.len(); + index.span = float_token.subspan(offset..part_end).unwrap_or(float_span); + + let base = mem::replace(e, Expr::PLACEHOLDER); + *e = Expr::Field(ExprField { + attrs: Vec::new(), + base: Box::new(base), + dot_token: Token![.](dot_token.span), + member: Member::Unnamed(index), + }); + + let dot_span = float_token + .subspan(part_end..part_end + 1) + .unwrap_or(float_span); + *dot_token = Token![.](dot_span); + offset = part_end + 1; + } + + Ok(!trailing_dot) + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for PointerMutability { + fn parse(input: ParseStream) -> Result<Self> { + let lookahead = input.lookahead1(); + if lookahead.peek(Token![const]) { + Ok(PointerMutability::Const(input.parse()?)) + } else if lookahead.peek(Token![mut]) { + Ok(PointerMutability::Mut(input.parse()?)) + } else { + Err(lookahead.error()) + } + } + } + + fn check_cast(input: ParseStream) -> Result<()> { + let kind = if input.peek(Token![.]) && !input.peek(Token![..]) { + if input.peek2(Token![await]) { + "`.await`" + } else if input.peek2(Ident) && (input.peek3(token::Paren) || input.peek3(Token![::])) { + "a method call" + } else { + "a field access" + } + } else if input.peek(Token![?]) { + "`?`" + } else if input.peek(token::Bracket) { + "indexing" + } else if input.peek(token::Paren) { + "a function call" + } else { + return Ok(()); + }; + let msg = format!("casts cannot be followed by {}", kind); + Err(input.error(msg)) + } +} + +#[cfg(feature = "printing")] +pub(crate) mod printing { + use crate::attr::Attribute; + #[cfg(feature = "full")] + use crate::attr::FilterAttrs; + #[cfg(feature = "full")] + use crate::classify; + #[cfg(feature = "full")] + use crate::expr::{ + Arm, ExprArray, ExprAssign, ExprAsync, ExprAwait, ExprBlock, ExprBreak, ExprClosure, + ExprConst, ExprContinue, ExprForLoop, ExprIf, ExprInfer, ExprLet, ExprLoop, ExprMatch, + ExprRange, ExprRawAddr, ExprRepeat, ExprReturn, ExprTry, ExprTryBlock, ExprUnsafe, + ExprWhile, ExprYield, Label, PointerMutability, RangeLimits, + }; + use crate::expr::{ + Expr, ExprBinary, ExprCall, ExprCast, ExprField, ExprGroup, ExprIndex, ExprLit, ExprMacro, + ExprMethodCall, ExprParen, ExprPath, ExprReference, ExprStruct, ExprTuple, ExprUnary, + FieldValue, Index, Member, + }; + use crate::fixup::FixupContext; + use crate::op::BinOp; + use crate::path; + use crate::path::printing::PathStyle; + use crate::precedence::Precedence; + use crate::token; + #[cfg(feature = "full")] + use crate::ty::ReturnType; + use proc_macro2::{Literal, Span, TokenStream}; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg(feature = "full")] + pub(crate) fn outer_attrs_to_tokens(attrs: &[Attribute], tokens: &mut TokenStream) { + tokens.append_all(attrs.outer()); + } + + #[cfg(feature = "full")] + fn inner_attrs_to_tokens(attrs: &[Attribute], tokens: &mut TokenStream) { + tokens.append_all(attrs.inner()); + } + + #[cfg(not(feature = "full"))] + pub(crate) fn outer_attrs_to_tokens(_attrs: &[Attribute], _tokens: &mut TokenStream) {} + + pub(crate) fn print_subexpression( + expr: &Expr, + needs_group: bool, + tokens: &mut TokenStream, + mut fixup: FixupContext, + ) { + if needs_group { + // If we are surrounding the whole cond in parentheses, such as: + // + // if (return Struct {}) {} + // + // then there is no need for parenthesizing the individual struct + // expressions within. On the other hand if the whole cond is not + // parenthesized, then print_expr must parenthesize exterior struct + // literals. + // + // if x == (Struct {}) {} + // + fixup = FixupContext::NONE; + } + + let do_print_expr = |tokens: &mut TokenStream| print_expr(expr, tokens, fixup); + + if needs_group { + token::Paren::default().surround(tokens, do_print_expr); + } else { + do_print_expr(tokens); + } + } + + pub(crate) fn print_expr(expr: &Expr, tokens: &mut TokenStream, mut fixup: FixupContext) { + #[cfg(feature = "full")] + let needs_group = fixup.parenthesize(expr); + #[cfg(not(feature = "full"))] + let needs_group = false; + + if needs_group { + fixup = FixupContext::NONE; + } + + let do_print_expr = |tokens: &mut TokenStream| match expr { + #[cfg(feature = "full")] + Expr::Array(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Assign(e) => print_expr_assign(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::Async(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Await(e) => print_expr_await(e, tokens, fixup), + Expr::Binary(e) => print_expr_binary(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::Block(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Break(e) => print_expr_break(e, tokens, fixup), + Expr::Call(e) => print_expr_call(e, tokens, fixup), + Expr::Cast(e) => print_expr_cast(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::Closure(e) => print_expr_closure(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::Const(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Continue(e) => e.to_tokens(tokens), + Expr::Field(e) => print_expr_field(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::ForLoop(e) => e.to_tokens(tokens), + Expr::Group(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::If(e) => e.to_tokens(tokens), + Expr::Index(e) => print_expr_index(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::Infer(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Let(e) => print_expr_let(e, tokens, fixup), + Expr::Lit(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Loop(e) => e.to_tokens(tokens), + Expr::Macro(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Match(e) => e.to_tokens(tokens), + Expr::MethodCall(e) => print_expr_method_call(e, tokens, fixup), + Expr::Paren(e) => e.to_tokens(tokens), + Expr::Path(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Range(e) => print_expr_range(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::RawAddr(e) => print_expr_raw_addr(e, tokens, fixup), + Expr::Reference(e) => print_expr_reference(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::Repeat(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Return(e) => print_expr_return(e, tokens, fixup), + Expr::Struct(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Try(e) => print_expr_try(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::TryBlock(e) => e.to_tokens(tokens), + Expr::Tuple(e) => e.to_tokens(tokens), + Expr::Unary(e) => print_expr_unary(e, tokens, fixup), + #[cfg(feature = "full")] + Expr::Unsafe(e) => e.to_tokens(tokens), + Expr::Verbatim(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::While(e) => e.to_tokens(tokens), + #[cfg(feature = "full")] + Expr::Yield(e) => print_expr_yield(e, tokens, fixup), + + #[cfg(not(feature = "full"))] + _ => unreachable!(), + }; + + if needs_group { + token::Paren::default().surround(tokens, do_print_expr); + } else { + do_print_expr(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprArray { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.bracket_token.surround(tokens, |tokens| { + self.elems.to_tokens(tokens); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprAssign { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_assign(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_assign(e: &ExprAssign, tokens: &mut TokenStream, mut fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + + let needs_group = !e.attrs.is_empty(); + if needs_group { + fixup = FixupContext::NONE; + } + + let do_print_expr = |tokens: &mut TokenStream| { + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_operator( + &e.left, + false, + false, + Precedence::Assign, + ); + print_subexpression(&e.left, left_prec <= Precedence::Range, tokens, left_fixup); + e.eq_token.to_tokens(tokens); + print_expr( + &e.right, + tokens, + fixup.rightmost_subexpression_fixup(false, false, Precedence::Assign), + ); + }; + + if needs_group { + token::Paren::default().surround(tokens, do_print_expr); + } else { + do_print_expr(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprAsync { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.async_token.to_tokens(tokens); + self.capture.to_tokens(tokens); + self.block.to_tokens(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprAwait { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_await(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_await(e: &ExprAwait, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_dot(&e.base); + print_subexpression( + &e.base, + left_prec < Precedence::Unambiguous, + tokens, + left_fixup, + ); + e.dot_token.to_tokens(tokens); + e.await_token.to_tokens(tokens); + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprBinary { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_binary(self, tokens, FixupContext::NONE); + } + } + + fn print_expr_binary(e: &ExprBinary, tokens: &mut TokenStream, mut fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + + let needs_group = !e.attrs.is_empty(); + if needs_group { + fixup = FixupContext::NONE; + } + + let do_print_expr = |tokens: &mut TokenStream| { + let binop_prec = Precedence::of_binop(&e.op); + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_operator( + &e.left, + #[cfg(feature = "full")] + match &e.op { + BinOp::Sub(_) + | BinOp::Mul(_) + | BinOp::And(_) + | BinOp::Or(_) + | BinOp::BitAnd(_) + | BinOp::BitOr(_) + | BinOp::Shl(_) + | BinOp::Lt(_) => true, + _ => false, + }, + match &e.op { + BinOp::Shl(_) | BinOp::Lt(_) => true, + _ => false, + }, + #[cfg(feature = "full")] + binop_prec, + ); + let left_needs_group = match binop_prec { + Precedence::Assign => left_prec <= Precedence::Range, + Precedence::Compare => left_prec <= binop_prec, + _ => left_prec < binop_prec, + }; + + let right_fixup = fixup.rightmost_subexpression_fixup( + #[cfg(feature = "full")] + false, + #[cfg(feature = "full")] + false, + #[cfg(feature = "full")] + binop_prec, + ); + let right_needs_group = binop_prec != Precedence::Assign + && right_fixup.rightmost_subexpression_precedence(&e.right) <= binop_prec; + + print_subexpression(&e.left, left_needs_group, tokens, left_fixup); + e.op.to_tokens(tokens); + print_subexpression(&e.right, right_needs_group, tokens, right_fixup); + }; + + if needs_group { + token::Paren::default().surround(tokens, do_print_expr); + } else { + do_print_expr(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprBlock { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.label.to_tokens(tokens); + self.block.brace_token.surround(tokens, |tokens| { + inner_attrs_to_tokens(&self.attrs, tokens); + tokens.append_all(&self.block.stmts); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprBreak { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_break(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_break(e: &ExprBreak, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + e.break_token.to_tokens(tokens); + e.label.to_tokens(tokens); + if let Some(value) = &e.expr { + print_subexpression( + value, + // Parenthesize `break 'inner: loop { break 'inner 1 } + 1` + // ^---------------------------------^ + e.label.is_none() && classify::expr_leading_label(value), + tokens, + fixup.rightmost_subexpression_fixup(true, true, Precedence::Jump), + ); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprCall { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_call(self, tokens, FixupContext::NONE); + } + } + + fn print_expr_call(e: &ExprCall, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_operator( + &e.func, + #[cfg(feature = "full")] + true, + false, + #[cfg(feature = "full")] + Precedence::Unambiguous, + ); + let needs_group = if let Expr::Field(func) = &*e.func { + func.member.is_named() + } else { + left_prec < Precedence::Unambiguous + }; + print_subexpression(&e.func, needs_group, tokens, left_fixup); + + e.paren_token.surround(tokens, |tokens| { + e.args.to_tokens(tokens); + }); + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprCast { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_cast(self, tokens, FixupContext::NONE); + } + } + + fn print_expr_cast(e: &ExprCast, tokens: &mut TokenStream, mut fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + + let needs_group = !e.attrs.is_empty(); + if needs_group { + fixup = FixupContext::NONE; + } + + let do_print_expr = |tokens: &mut TokenStream| { + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_operator( + &e.expr, + #[cfg(feature = "full")] + false, + false, + #[cfg(feature = "full")] + Precedence::Cast, + ); + print_subexpression(&e.expr, left_prec < Precedence::Cast, tokens, left_fixup); + e.as_token.to_tokens(tokens); + e.ty.to_tokens(tokens); + }; + + if needs_group { + token::Paren::default().surround(tokens, do_print_expr); + } else { + do_print_expr(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprClosure { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_closure(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_closure(e: &ExprClosure, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + e.lifetimes.to_tokens(tokens); + e.constness.to_tokens(tokens); + e.movability.to_tokens(tokens); + e.asyncness.to_tokens(tokens); + e.capture.to_tokens(tokens); + e.or1_token.to_tokens(tokens); + e.inputs.to_tokens(tokens); + e.or2_token.to_tokens(tokens); + e.output.to_tokens(tokens); + if matches!(e.output, ReturnType::Default) + || matches!(&*e.body, Expr::Block(body) if body.attrs.is_empty() && body.label.is_none()) + { + print_expr( + &e.body, + tokens, + fixup.rightmost_subexpression_fixup(false, false, Precedence::Jump), + ); + } else { + token::Brace::default().surround(tokens, |tokens| { + print_expr(&e.body, tokens, FixupContext::new_stmt()); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprConst { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.const_token.to_tokens(tokens); + self.block.brace_token.surround(tokens, |tokens| { + inner_attrs_to_tokens(&self.attrs, tokens); + tokens.append_all(&self.block.stmts); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprContinue { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.continue_token.to_tokens(tokens); + self.label.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprField { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_field(self, tokens, FixupContext::NONE); + } + } + + fn print_expr_field(e: &ExprField, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_dot(&e.base); + print_subexpression( + &e.base, + left_prec < Precedence::Unambiguous, + tokens, + left_fixup, + ); + e.dot_token.to_tokens(tokens); + e.member.to_tokens(tokens); + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprForLoop { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.label.to_tokens(tokens); + self.for_token.to_tokens(tokens); + self.pat.to_tokens(tokens); + self.in_token.to_tokens(tokens); + print_expr(&self.expr, tokens, FixupContext::new_condition()); + self.body.brace_token.surround(tokens, |tokens| { + inner_attrs_to_tokens(&self.attrs, tokens); + tokens.append_all(&self.body.stmts); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprGroup { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.group_token.surround(tokens, |tokens| { + self.expr.to_tokens(tokens); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprIf { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + + let mut expr = self; + loop { + expr.if_token.to_tokens(tokens); + print_expr(&expr.cond, tokens, FixupContext::new_condition()); + expr.then_branch.to_tokens(tokens); + + let (else_token, else_) = match &expr.else_branch { + Some(else_branch) => else_branch, + None => break, + }; + + else_token.to_tokens(tokens); + match &**else_ { + Expr::If(next) => { + expr = next; + } + Expr::Block(last) => { + last.to_tokens(tokens); + break; + } + // If this is not one of the valid expressions to exist in + // an else clause, wrap it in a block. + other => { + token::Brace::default().surround(tokens, |tokens| { + print_expr(other, tokens, FixupContext::new_stmt()); + }); + break; + } + } + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprIndex { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_index(self, tokens, FixupContext::NONE); + } + } + + fn print_expr_index(e: &ExprIndex, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_operator( + &e.expr, + #[cfg(feature = "full")] + true, + false, + #[cfg(feature = "full")] + Precedence::Unambiguous, + ); + print_subexpression( + &e.expr, + left_prec < Precedence::Unambiguous, + tokens, + left_fixup, + ); + e.bracket_token.surround(tokens, |tokens| { + e.index.to_tokens(tokens); + }); + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprInfer { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.underscore_token.to_tokens(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprLet { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_let(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_let(e: &ExprLet, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + e.let_token.to_tokens(tokens); + e.pat.to_tokens(tokens); + e.eq_token.to_tokens(tokens); + let (right_prec, right_fixup) = fixup.rightmost_subexpression(&e.expr, Precedence::Let); + print_subexpression(&e.expr, right_prec < Precedence::Let, tokens, right_fixup); + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprLit { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.lit.to_tokens(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprLoop { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.label.to_tokens(tokens); + self.loop_token.to_tokens(tokens); + self.body.brace_token.surround(tokens, |tokens| { + inner_attrs_to_tokens(&self.attrs, tokens); + tokens.append_all(&self.body.stmts); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprMacro { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.mac.to_tokens(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprMatch { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.match_token.to_tokens(tokens); + print_expr(&self.expr, tokens, FixupContext::new_condition()); + self.brace_token.surround(tokens, |tokens| { + inner_attrs_to_tokens(&self.attrs, tokens); + for (i, arm) in self.arms.iter().enumerate() { + arm.to_tokens(tokens); + // Ensure that we have a comma after a non-block arm, except + // for the last one. + let is_last = i == self.arms.len() - 1; + if !is_last + && classify::requires_comma_to_be_match_arm(&arm.body) + && arm.comma.is_none() + { + <Token![,]>::default().to_tokens(tokens); + } + } + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprMethodCall { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_method_call(self, tokens, FixupContext::NONE); + } + } + + fn print_expr_method_call(e: &ExprMethodCall, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_dot(&e.receiver); + print_subexpression( + &e.receiver, + left_prec < Precedence::Unambiguous, + tokens, + left_fixup, + ); + e.dot_token.to_tokens(tokens); + e.method.to_tokens(tokens); + if let Some(turbofish) = &e.turbofish { + path::printing::print_angle_bracketed_generic_arguments( + tokens, + turbofish, + PathStyle::Expr, + ); + } + e.paren_token.surround(tokens, |tokens| { + e.args.to_tokens(tokens); + }); + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprParen { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.paren_token.surround(tokens, |tokens| { + self.expr.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprPath { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + path::printing::print_qpath(tokens, &self.qself, &self.path, PathStyle::Expr); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprRange { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_range(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_range(e: &ExprRange, tokens: &mut TokenStream, mut fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + + let needs_group = !e.attrs.is_empty(); + if needs_group { + fixup = FixupContext::NONE; + } + + let do_print_expr = |tokens: &mut TokenStream| { + if let Some(start) = &e.start { + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_operator( + start, + true, + false, + Precedence::Range, + ); + print_subexpression(start, left_prec <= Precedence::Range, tokens, left_fixup); + } + e.limits.to_tokens(tokens); + if let Some(end) = &e.end { + let right_fixup = + fixup.rightmost_subexpression_fixup(false, true, Precedence::Range); + let right_prec = right_fixup.rightmost_subexpression_precedence(end); + print_subexpression(end, right_prec <= Precedence::Range, tokens, right_fixup); + } + }; + + if needs_group { + token::Paren::default().surround(tokens, do_print_expr); + } else { + do_print_expr(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprRawAddr { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_raw_addr(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_raw_addr(e: &ExprRawAddr, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + e.and_token.to_tokens(tokens); + e.raw.to_tokens(tokens); + e.mutability.to_tokens(tokens); + let (right_prec, right_fixup) = fixup.rightmost_subexpression(&e.expr, Precedence::Prefix); + print_subexpression( + &e.expr, + right_prec < Precedence::Prefix, + tokens, + right_fixup, + ); + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprReference { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_reference(self, tokens, FixupContext::NONE); + } + } + + fn print_expr_reference(e: &ExprReference, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + e.and_token.to_tokens(tokens); + e.mutability.to_tokens(tokens); + let (right_prec, right_fixup) = fixup.rightmost_subexpression( + &e.expr, + #[cfg(feature = "full")] + Precedence::Prefix, + ); + print_subexpression( + &e.expr, + right_prec < Precedence::Prefix, + tokens, + right_fixup, + ); + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprRepeat { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.bracket_token.surround(tokens, |tokens| { + self.expr.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + self.len.to_tokens(tokens); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprReturn { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_return(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_return(e: &ExprReturn, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + e.return_token.to_tokens(tokens); + if let Some(expr) = &e.expr { + print_expr( + expr, + tokens, + fixup.rightmost_subexpression_fixup(true, false, Precedence::Jump), + ); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprStruct { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + path::printing::print_qpath(tokens, &self.qself, &self.path, PathStyle::Expr); + self.brace_token.surround(tokens, |tokens| { + self.fields.to_tokens(tokens); + if let Some(dot2_token) = &self.dot2_token { + dot2_token.to_tokens(tokens); + } else if self.rest.is_some() { + Token![..](Span::call_site()).to_tokens(tokens); + } + self.rest.to_tokens(tokens); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprTry { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_try(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_try(e: &ExprTry, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + let (left_prec, left_fixup) = fixup.leftmost_subexpression_with_dot(&e.expr); + print_subexpression( + &e.expr, + left_prec < Precedence::Unambiguous, + tokens, + left_fixup, + ); + e.question_token.to_tokens(tokens); + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprTryBlock { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.try_token.to_tokens(tokens); + self.block.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprTuple { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.paren_token.surround(tokens, |tokens| { + self.elems.to_tokens(tokens); + // If we only have one argument, we need a trailing comma to + // distinguish ExprTuple from ExprParen. + if self.elems.len() == 1 && !self.elems.trailing_punct() { + <Token![,]>::default().to_tokens(tokens); + } + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprUnary { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_unary(self, tokens, FixupContext::NONE); + } + } + + fn print_expr_unary(e: &ExprUnary, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + e.op.to_tokens(tokens); + let (right_prec, right_fixup) = fixup.rightmost_subexpression( + &e.expr, + #[cfg(feature = "full")] + Precedence::Prefix, + ); + print_subexpression( + &e.expr, + right_prec < Precedence::Prefix, + tokens, + right_fixup, + ); + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprUnsafe { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.unsafe_token.to_tokens(tokens); + self.block.brace_token.surround(tokens, |tokens| { + inner_attrs_to_tokens(&self.attrs, tokens); + tokens.append_all(&self.block.stmts); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprWhile { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.label.to_tokens(tokens); + self.while_token.to_tokens(tokens); + print_expr(&self.cond, tokens, FixupContext::new_condition()); + self.body.brace_token.surround(tokens, |tokens| { + inner_attrs_to_tokens(&self.attrs, tokens); + tokens.append_all(&self.body.stmts); + }); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ExprYield { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_expr_yield(self, tokens, FixupContext::NONE); + } + } + + #[cfg(feature = "full")] + fn print_expr_yield(e: &ExprYield, tokens: &mut TokenStream, fixup: FixupContext) { + outer_attrs_to_tokens(&e.attrs, tokens); + e.yield_token.to_tokens(tokens); + if let Some(expr) = &e.expr { + print_expr( + expr, + tokens, + fixup.rightmost_subexpression_fixup(true, false, Precedence::Jump), + ); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Arm { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(&self.attrs); + self.pat.to_tokens(tokens); + if let Some((if_token, guard)) = &self.guard { + if_token.to_tokens(tokens); + guard.to_tokens(tokens); + } + self.fat_arrow_token.to_tokens(tokens); + print_expr(&self.body, tokens, FixupContext::new_match_arm()); + self.comma.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + outer_attrs_to_tokens(&self.attrs, tokens); + self.member.to_tokens(tokens); + if let Some(colon_token) = &self.colon_token { + colon_token.to_tokens(tokens); + self.expr.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Index { + fn to_tokens(&self, tokens: &mut TokenStream) { + let mut lit = Literal::i64_unsuffixed(i64::from(self.index)); + lit.set_span(self.span); + tokens.append(lit); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Label { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.name.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Member { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Member::Named(ident) => ident.to_tokens(tokens), + Member::Unnamed(index) => index.to_tokens(tokens), + } + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for RangeLimits { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + RangeLimits::HalfOpen(t) => t.to_tokens(tokens), + RangeLimits::Closed(t) => t.to_tokens(tokens), + } + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PointerMutability { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + PointerMutability::Const(const_token) => const_token.to_tokens(tokens), + PointerMutability::Mut(mut_token) => mut_token.to_tokens(tokens), + } + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/ext.rs b/third_party/crates/syn-2.0.119/src/ext.rs new file mode 100644 index 0000000..212e223 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/ext.rs @@ -0,0 +1,180 @@ +//! Extension traits to provide parsing methods on foreign types. + +#[cfg(feature = "parsing")] +use crate::buffer::Cursor; +#[cfg(feature = "parsing")] +use crate::error::Result; +#[cfg(feature = "parsing")] +use crate::parse::ParseStream; +#[cfg(feature = "parsing")] +use crate::parse::Peek; +#[cfg(feature = "parsing")] +use crate::sealed::lookahead; +#[cfg(feature = "parsing")] +use crate::token::CustomToken; +use alloc::string::ToString; +use core::iter; +use proc_macro2::{Ident, Punct, Spacing, Span, TokenStream, TokenTree}; + +/// Additional methods for `Ident` not provided by proc-macro2 or libproc_macro. +/// +/// This trait is sealed and cannot be implemented for types outside of Syn. It +/// is implemented only for `proc_macro2::Ident`. +pub trait IdentExt: Sized + private::Sealed { + /// Parses any identifier including keywords. + /// + /// This is useful when parsing macro input which allows Rust keywords as + /// identifiers. + /// + /// # Example + /// + /// ``` + /// use syn::{Error, Ident, Result, Token}; + /// use syn::ext::IdentExt; + /// use syn::parse::ParseStream; + /// + /// mod kw { + /// syn::custom_keyword!(name); + /// } + /// + /// // Parses input that looks like `name = NAME` where `NAME` can be + /// // any identifier. + /// // + /// // Examples: + /// // + /// // name = anything + /// // name = impl + /// fn parse_dsl(input: ParseStream) -> Result<Ident> { + /// input.parse::<kw::name>()?; + /// input.parse::<Token![=]>()?; + /// let name = input.call(Ident::parse_any)?; + /// Ok(name) + /// } + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + fn parse_any(input: ParseStream) -> Result<Self>; + + /// Peeks any identifier including keywords. Usage: + /// `input.peek(Ident::peek_any)` + /// + /// This is different from `input.peek(Ident)` which only returns true in + /// the case of an ident which is not a Rust keyword. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + #[allow(non_upper_case_globals)] + const peek_any: private::PeekFn = private::PeekFn; + + /// Strips the raw marker `r#`, if any, from the beginning of an ident. + /// + /// - unraw(`x`) = `x` + /// - unraw(`move`) = `move` + /// - unraw(`r#move`) = `move` + /// + /// # Example + /// + /// In the case of interop with other languages like Python that have a + /// different set of keywords than Rust, we might come across macro input + /// that involves raw identifiers to refer to ordinary variables in the + /// other language with a name that happens to be a Rust keyword. + /// + /// The function below appends an identifier from the caller's input onto a + /// fixed prefix. Without using `unraw()`, this would tend to produce + /// invalid identifiers like `__pyo3_get_r#move`. + /// + /// ``` + /// use proc_macro2::Span; + /// use syn::Ident; + /// use syn::ext::IdentExt; + /// + /// fn ident_for_getter(variable: &Ident) -> Ident { + /// let getter = format!("__pyo3_get_{}", variable.unraw()); + /// Ident::new(&getter, Span::call_site()) + /// } + /// ``` + fn unraw(&self) -> Ident; +} + +impl IdentExt for Ident { + #[cfg(feature = "parsing")] + fn parse_any(input: ParseStream) -> Result<Self> { + input.step(|cursor| match cursor.ident() { + Some((ident, rest)) => Ok((ident, rest)), + None => Err(cursor.error("expected ident")), + }) + } + + fn unraw(&self) -> Ident { + let string = self.to_string(); + if let Some(string) = string.strip_prefix("r#") { + Ident::new(string, self.span()) + } else { + self.clone() + } + } +} + +#[cfg(feature = "parsing")] +impl Peek for private::PeekFn { + type Token = private::IdentAny; +} + +#[cfg(feature = "parsing")] +impl CustomToken for private::IdentAny { + fn peek(cursor: Cursor) -> bool { + cursor.ident().is_some() + } + + fn display() -> &'static str { + "identifier" + } +} + +#[cfg(feature = "parsing")] +impl lookahead::Sealed for private::PeekFn {} + +pub(crate) trait TokenStreamExt { + fn append(&mut self, token: TokenTree); +} + +impl TokenStreamExt for TokenStream { + fn append(&mut self, token: TokenTree) { + self.extend(iter::once(token)); + } +} + +pub(crate) trait PunctExt { + fn new_spanned(ch: char, spacing: Spacing, span: Span) -> Self; +} + +impl PunctExt for Punct { + fn new_spanned(ch: char, spacing: Spacing, span: Span) -> Self { + let mut punct = Punct::new(ch, spacing); + punct.set_span(span); + punct + } +} + +mod private { + use proc_macro2::Ident; + + pub trait Sealed {} + + impl Sealed for Ident {} + + #[cfg(feature = "parsing")] + pub struct PeekFn; + + #[cfg(feature = "parsing")] + pub struct IdentAny; + + #[cfg(feature = "parsing")] + impl Copy for PeekFn {} + + #[cfg(feature = "parsing")] + impl Clone for PeekFn { + fn clone(&self) -> Self { + *self + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/file.rs b/third_party/crates/syn-2.0.119/src/file.rs new file mode 100644 index 0000000..dbb1bcb --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/file.rs @@ -0,0 +1,128 @@ +use crate::attr::Attribute; +use crate::item::Item; +use alloc::string::String; +use alloc::vec::Vec; + +ast_struct! { + /// A complete file of Rust source code. + /// + /// Typically `File` objects are created with [`parse_file`]. + /// + /// [`parse_file`]: crate::parse_file + /// + /// # Example + /// + /// Parse a Rust source file into a `syn::File` and print out a debug + /// representation of the syntax tree. + /// + /// ``` + /// use std::env; + /// use std::fs; + /// use std::process; + /// + /// fn main() { + /// # } + /// # + /// # fn fake_main() { + /// let mut args = env::args(); + /// let _ = args.next(); // executable name + /// + /// let filename = match (args.next(), args.next()) { + /// (Some(filename), None) => filename, + /// _ => { + /// eprintln!("Usage: dump-syntax path/to/filename.rs"); + /// process::exit(1); + /// } + /// }; + /// + /// let src = fs::read_to_string(&filename).expect("unable to read file"); + /// let syntax = syn::parse_file(&src).expect("unable to parse file"); + /// + /// // Debug impl is available if Syn is built with "extra-traits" feature. + /// println!("{:#?}", syntax); + /// } + /// ``` + /// + /// Running with its own source code as input, this program prints output + /// that begins with: + /// + /// ```text + /// File { + /// shebang: None, + /// attrs: [], + /// items: [ + /// Use( + /// ItemUse { + /// attrs: [], + /// vis: Inherited, + /// use_token: Use, + /// leading_colon: None, + /// tree: Path( + /// UsePath { + /// ident: Ident( + /// std, + /// ), + /// colon2_token: Colon2, + /// tree: Name( + /// UseName { + /// ident: Ident( + /// env, + /// ), + /// }, + /// ), + /// }, + /// ), + /// semi_token: Semi, + /// }, + /// ), + /// ... + /// ``` + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct File { + pub shebang: Option<String>, + pub attrs: Vec<Attribute>, + pub items: Vec<Item>, + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::error::Result; + use crate::file::File; + use crate::parse::{Parse, ParseStream}; + use alloc::vec::Vec; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for File { + fn parse(input: ParseStream) -> Result<Self> { + Ok(File { + shebang: None, + attrs: input.call(Attribute::parse_inner)?, + items: { + let mut items = Vec::new(); + while !input.is_empty() { + items.push(input.parse()?); + } + items + }, + }) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::FilterAttrs; + use crate::file::File; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for File { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.inner()); + tokens.append_all(&self.items); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/fixup.rs b/third_party/crates/syn-2.0.119/src/fixup.rs new file mode 100644 index 0000000..6d2c309 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/fixup.rs @@ -0,0 +1,773 @@ +use crate::classify; +use crate::expr::Expr; +#[cfg(feature = "full")] +use crate::expr::{ + ExprBreak, ExprRange, ExprRawAddr, ExprReference, ExprReturn, ExprUnary, ExprYield, +}; +use crate::precedence::Precedence; +#[cfg(feature = "full")] +use crate::ty::ReturnType; + +pub(crate) struct FixupContext { + #[cfg(feature = "full")] + previous_operator: Precedence, + #[cfg(feature = "full")] + next_operator: Precedence, + + // Print expression such that it can be parsed back as a statement + // consisting of the original expression. + // + // The effect of this is for binary operators in statement position to set + // `leftmost_subexpression_in_stmt` when printing their left-hand operand. + // + // (match x {}) - 1; // match needs parens when LHS of binary operator + // + // match x {}; // not when its own statement + // + #[cfg(feature = "full")] + stmt: bool, + + // This is the difference between: + // + // (match x {}) - 1; // subexpression needs parens + // + // let _ = match x {} - 1; // no parens + // + // There are 3 distinguishable contexts in which `print_expr` might be + // called with the expression `$match` as its argument, where `$match` + // represents an expression of kind `ExprKind::Match`: + // + // - stmt=false leftmost_subexpression_in_stmt=false + // + // Example: `let _ = $match - 1;` + // + // No parentheses required. + // + // - stmt=false leftmost_subexpression_in_stmt=true + // + // Example: `$match - 1;` + // + // Must parenthesize `($match)`, otherwise parsing back the output as a + // statement would terminate the statement after the closing brace of + // the match, parsing `-1;` as a separate statement. + // + // - stmt=true leftmost_subexpression_in_stmt=false + // + // Example: `$match;` + // + // No parentheses required. + #[cfg(feature = "full")] + leftmost_subexpression_in_stmt: bool, + + // Print expression such that it can be parsed as a match arm. + // + // This is almost equivalent to `stmt`, but the grammar diverges a tiny bit + // between statements and match arms when it comes to braced macro calls. + // Macro calls with brace delimiter terminate a statement without a + // semicolon, but do not terminate a match-arm without comma. + // + // m! {} - 1; // two statements: a macro call followed by -1 literal + // + // match () { + // _ => m! {} - 1, // binary subtraction operator + // } + // + #[cfg(feature = "full")] + match_arm: bool, + + // This is almost equivalent to `leftmost_subexpression_in_stmt`, other than + // for braced macro calls. + // + // If we have `m! {} - 1` as an expression, the leftmost subexpression + // `m! {}` will need to be parenthesized in the statement case but not the + // match-arm case. + // + // (m! {}) - 1; // subexpression needs parens + // + // match () { + // _ => m! {} - 1, // no parens + // } + // + #[cfg(feature = "full")] + leftmost_subexpression_in_match_arm: bool, + + // This is the difference between: + // + // if let _ = (Struct {}) {} // needs parens + // + // match () { + // () if let _ = Struct {} => {} // no parens + // } + // + #[cfg(feature = "full")] + condition: bool, + + // This is the difference between: + // + // if break Struct {} == (break) {} // needs parens + // + // if break break == Struct {} {} // no parens + // + #[cfg(feature = "full")] + rightmost_subexpression_in_condition: bool, + + // This is the difference between: + // + // if break ({ x }).field + 1 {} needs parens + // + // if break 1 + { x }.field {} // no parens + // + #[cfg(feature = "full")] + leftmost_subexpression_in_optional_operand: bool, + + // This is the difference between: + // + // let _ = (return) - 1; // without paren, this would return -1 + // + // let _ = return + 1; // no paren because '+' cannot begin expr + // + #[cfg(feature = "full")] + next_operator_can_begin_expr: bool, + + // This is the difference between: + // + // let _ = 1 + return 1; // no parens if rightmost subexpression + // + // let _ = 1 + (return 1) + 1; // needs parens + // + #[cfg(feature = "full")] + next_operator_can_continue_expr: bool, + + // This is the difference between: + // + // let _ = x as u8 + T; + // + // let _ = (x as u8) < T; + // + // Without parens, the latter would want to parse `u8<T...` as a type. + next_operator_can_begin_generics: bool, +} + +impl FixupContext { + /// The default amount of fixing is minimal fixing. Fixups should be turned + /// on in a targeted fashion where needed. + pub const NONE: Self = FixupContext { + #[cfg(feature = "full")] + previous_operator: Precedence::MIN, + #[cfg(feature = "full")] + next_operator: Precedence::MIN, + #[cfg(feature = "full")] + stmt: false, + #[cfg(feature = "full")] + leftmost_subexpression_in_stmt: false, + #[cfg(feature = "full")] + match_arm: false, + #[cfg(feature = "full")] + leftmost_subexpression_in_match_arm: false, + #[cfg(feature = "full")] + condition: false, + #[cfg(feature = "full")] + rightmost_subexpression_in_condition: false, + #[cfg(feature = "full")] + leftmost_subexpression_in_optional_operand: false, + #[cfg(feature = "full")] + next_operator_can_begin_expr: false, + #[cfg(feature = "full")] + next_operator_can_continue_expr: false, + next_operator_can_begin_generics: false, + }; + + /// Create the initial fixup for printing an expression in statement + /// position. + #[cfg(feature = "full")] + pub fn new_stmt() -> Self { + FixupContext { + stmt: true, + ..FixupContext::NONE + } + } + + /// Create the initial fixup for printing an expression as the right-hand + /// side of a match arm. + #[cfg(feature = "full")] + pub fn new_match_arm() -> Self { + FixupContext { + match_arm: true, + ..FixupContext::NONE + } + } + + /// Create the initial fixup for printing an expression as the "condition" + /// of an `if` or `while`. There are a few other positions which are + /// grammatically equivalent and also use this, such as the iterator + /// expression in `for` and the scrutinee in `match`. + #[cfg(feature = "full")] + pub fn new_condition() -> Self { + FixupContext { + condition: true, + rightmost_subexpression_in_condition: true, + ..FixupContext::NONE + } + } + + /// Transform this fixup into the one that should apply when printing the + /// leftmost subexpression of the current expression. + /// + /// The leftmost subexpression is any subexpression that has the same first + /// token as the current expression, but has a different last token. + /// + /// For example in `$a + $b` and `$a.method()`, the subexpression `$a` is a + /// leftmost subexpression. + /// + /// Not every expression has a leftmost subexpression. For example neither + /// `-$a` nor `[$a]` have one. + pub fn leftmost_subexpression_with_operator( + self, + expr: &Expr, + #[cfg(feature = "full")] next_operator_can_begin_expr: bool, + next_operator_can_begin_generics: bool, + #[cfg(feature = "full")] precedence: Precedence, + ) -> (Precedence, Self) { + let fixup = FixupContext { + #[cfg(feature = "full")] + next_operator: precedence, + #[cfg(feature = "full")] + stmt: false, + #[cfg(feature = "full")] + leftmost_subexpression_in_stmt: self.stmt || self.leftmost_subexpression_in_stmt, + #[cfg(feature = "full")] + match_arm: false, + #[cfg(feature = "full")] + leftmost_subexpression_in_match_arm: self.match_arm + || self.leftmost_subexpression_in_match_arm, + #[cfg(feature = "full")] + rightmost_subexpression_in_condition: false, + #[cfg(feature = "full")] + next_operator_can_begin_expr, + #[cfg(feature = "full")] + next_operator_can_continue_expr: true, + next_operator_can_begin_generics, + ..self + }; + + (fixup.leftmost_subexpression_precedence(expr), fixup) + } + + /// Transform this fixup into the one that should apply when printing a + /// leftmost subexpression followed by a `.` or `?` token, which confer + /// different statement boundary rules compared to other leftmost + /// subexpressions. + pub fn leftmost_subexpression_with_dot(self, expr: &Expr) -> (Precedence, Self) { + let fixup = FixupContext { + #[cfg(feature = "full")] + next_operator: Precedence::Unambiguous, + #[cfg(feature = "full")] + stmt: self.stmt || self.leftmost_subexpression_in_stmt, + #[cfg(feature = "full")] + leftmost_subexpression_in_stmt: false, + #[cfg(feature = "full")] + match_arm: self.match_arm || self.leftmost_subexpression_in_match_arm, + #[cfg(feature = "full")] + leftmost_subexpression_in_match_arm: false, + #[cfg(feature = "full")] + rightmost_subexpression_in_condition: false, + #[cfg(feature = "full")] + next_operator_can_begin_expr: false, + #[cfg(feature = "full")] + next_operator_can_continue_expr: true, + next_operator_can_begin_generics: false, + ..self + }; + + (fixup.leftmost_subexpression_precedence(expr), fixup) + } + + fn leftmost_subexpression_precedence(self, expr: &Expr) -> Precedence { + #[cfg(feature = "full")] + if !self.next_operator_can_begin_expr || self.next_operator == Precedence::Range { + if let Scan::Bailout = scan_right(expr, self, Precedence::MIN, 0, 0) { + if scan_left(expr, self) { + return Precedence::Unambiguous; + } + } + } + + self.precedence(expr) + } + + /// Transform this fixup into the one that should apply when printing the + /// rightmost subexpression of the current expression. + /// + /// The rightmost subexpression is any subexpression that has a different + /// first token than the current expression, but has the same last token. + /// + /// For example in `$a + $b` and `-$b`, the subexpression `$b` is a + /// rightmost subexpression. + /// + /// Not every expression has a rightmost subexpression. For example neither + /// `[$b]` nor `$a.f($b)` have one. + pub fn rightmost_subexpression( + self, + expr: &Expr, + #[cfg(feature = "full")] precedence: Precedence, + ) -> (Precedence, Self) { + let fixup = self.rightmost_subexpression_fixup( + #[cfg(feature = "full")] + false, + #[cfg(feature = "full")] + false, + #[cfg(feature = "full")] + precedence, + ); + (fixup.rightmost_subexpression_precedence(expr), fixup) + } + + pub fn rightmost_subexpression_fixup( + self, + #[cfg(feature = "full")] reset_allow_struct: bool, + #[cfg(feature = "full")] optional_operand: bool, + #[cfg(feature = "full")] precedence: Precedence, + ) -> Self { + FixupContext { + #[cfg(feature = "full")] + previous_operator: precedence, + #[cfg(feature = "full")] + stmt: false, + #[cfg(feature = "full")] + leftmost_subexpression_in_stmt: false, + #[cfg(feature = "full")] + match_arm: false, + #[cfg(feature = "full")] + leftmost_subexpression_in_match_arm: false, + #[cfg(feature = "full")] + condition: self.condition && !reset_allow_struct, + #[cfg(feature = "full")] + leftmost_subexpression_in_optional_operand: self.condition && optional_operand, + ..self + } + } + + pub fn rightmost_subexpression_precedence(self, expr: &Expr) -> Precedence { + let default_prec = self.precedence(expr); + + #[cfg(feature = "full")] + if match self.previous_operator { + Precedence::Assign | Precedence::Let | Precedence::Prefix => { + default_prec < self.previous_operator + } + _ => default_prec <= self.previous_operator, + } && match self.next_operator { + Precedence::Range | Precedence::Or | Precedence::And => true, + _ => !self.next_operator_can_begin_expr, + } { + if let Scan::Bailout | Scan::Fail = scan_right(expr, self, self.previous_operator, 1, 0) + { + if scan_left(expr, self) { + return Precedence::Prefix; + } + } + } + + default_prec + } + + /// Determine whether parentheses are needed around the given expression to + /// head off the early termination of a statement or condition. + #[cfg(feature = "full")] + pub fn parenthesize(self, expr: &Expr) -> bool { + (self.leftmost_subexpression_in_stmt && !classify::requires_semi_to_be_stmt(expr)) + || ((self.stmt || self.leftmost_subexpression_in_stmt) && matches!(expr, Expr::Let(_))) + || (self.leftmost_subexpression_in_match_arm + && !classify::requires_comma_to_be_match_arm(expr)) + || (self.condition && matches!(expr, Expr::Struct(_))) + || (self.rightmost_subexpression_in_condition + && matches!( + expr, + Expr::Return(ExprReturn { expr: None, .. }) + | Expr::Yield(ExprYield { expr: None, .. }) + )) + || (self.rightmost_subexpression_in_condition + && !self.condition + && matches!( + expr, + Expr::Break(ExprBreak { expr: None, .. }) + | Expr::Path(_) + | Expr::Range(ExprRange { end: None, .. }) + )) + || (self.leftmost_subexpression_in_optional_operand + && matches!(expr, Expr::Block(expr) if expr.attrs.is_empty() && expr.label.is_none())) + } + + /// Determines the effective precedence of a subexpression. Some expressions + /// have higher or lower precedence when adjacent to particular operators. + fn precedence(self, expr: &Expr) -> Precedence { + #[cfg(feature = "full")] + if self.next_operator_can_begin_expr { + // Decrease precedence of value-less jumps when followed by an + // operator that would otherwise get interpreted as beginning a + // value for the jump. + if let Expr::Break(ExprBreak { expr: None, .. }) + | Expr::Return(ExprReturn { expr: None, .. }) + | Expr::Yield(ExprYield { expr: None, .. }) = expr + { + return Precedence::Jump; + } + } + + #[cfg(feature = "full")] + if !self.next_operator_can_continue_expr { + match expr { + // Increase precedence of expressions that extend to the end of + // current statement or group. + Expr::Break(_) + | Expr::Closure(_) + | Expr::Let(_) + | Expr::Return(_) + | Expr::Yield(_) => { + return Precedence::Prefix; + } + Expr::Range(e) if e.start.is_none() => return Precedence::Prefix, + _ => {} + } + } + + if self.next_operator_can_begin_generics { + if let Expr::Cast(cast) = expr { + if classify::trailing_unparameterized_path(&cast.ty) { + return Precedence::MIN; + } + } + } + + Precedence::of(expr) + } +} + +impl Copy for FixupContext {} + +impl Clone for FixupContext { + fn clone(&self) -> Self { + *self + } +} + +#[cfg(feature = "full")] +enum Scan { + Fail, + Bailout, + Consume, +} + +#[cfg(feature = "full")] +impl Copy for Scan {} + +#[cfg(feature = "full")] +impl Clone for Scan { + fn clone(&self) -> Self { + *self + } +} + +#[cfg(feature = "full")] +impl PartialEq for Scan { + fn eq(&self, other: &Self) -> bool { + *self as u8 == *other as u8 + } +} + +#[cfg(feature = "full")] +fn scan_left(expr: &Expr, fixup: FixupContext) -> bool { + match expr { + Expr::Assign(_) => fixup.previous_operator <= Precedence::Assign, + Expr::Binary(e) => match Precedence::of_binop(&e.op) { + Precedence::Assign => fixup.previous_operator <= Precedence::Assign, + binop_prec => fixup.previous_operator < binop_prec, + }, + Expr::Cast(_) => fixup.previous_operator < Precedence::Cast, + Expr::Range(e) => e.start.is_none() || fixup.previous_operator < Precedence::Assign, + _ => true, + } +} + +#[cfg(feature = "full")] +fn scan_right( + expr: &Expr, + fixup: FixupContext, + precedence: Precedence, + fail_offset: u8, + bailout_offset: u8, +) -> Scan { + let consume_by_precedence = if match precedence { + Precedence::Assign | Precedence::Compare => precedence <= fixup.next_operator, + _ => precedence < fixup.next_operator, + } || fixup.next_operator == Precedence::MIN + { + Scan::Consume + } else { + Scan::Bailout + }; + if fixup.parenthesize(expr) { + return consume_by_precedence; + } + match expr { + Expr::Assign(e) if e.attrs.is_empty() => { + if match fixup.next_operator { + Precedence::Unambiguous => fail_offset >= 2, + _ => bailout_offset >= 1, + } { + return Scan::Consume; + } + let right_fixup = fixup.rightmost_subexpression_fixup(false, false, Precedence::Assign); + let scan = scan_right( + &e.right, + right_fixup, + Precedence::Assign, + match fixup.next_operator { + Precedence::Unambiguous => fail_offset, + _ => 1, + }, + 1, + ); + if let Scan::Bailout | Scan::Consume = scan { + Scan::Consume + } else if let Precedence::Unambiguous = fixup.next_operator { + Scan::Fail + } else { + Scan::Bailout + } + } + Expr::Binary(e) if e.attrs.is_empty() => { + if match fixup.next_operator { + Precedence::Unambiguous => { + fail_offset >= 2 + && (consume_by_precedence == Scan::Consume || bailout_offset >= 1) + } + _ => bailout_offset >= 1, + } { + return Scan::Consume; + } + let binop_prec = Precedence::of_binop(&e.op); + if binop_prec == Precedence::Compare && fixup.next_operator == Precedence::Compare { + return Scan::Consume; + } + let right_fixup = fixup.rightmost_subexpression_fixup(false, false, binop_prec); + let scan = scan_right( + &e.right, + right_fixup, + binop_prec, + match fixup.next_operator { + Precedence::Unambiguous => fail_offset, + _ => 1, + }, + consume_by_precedence as u8 - Scan::Bailout as u8, + ); + match scan { + Scan::Fail => {} + Scan::Bailout => return consume_by_precedence, + Scan::Consume => return Scan::Consume, + } + let right_needs_group = binop_prec != Precedence::Assign + && right_fixup.rightmost_subexpression_precedence(&e.right) <= binop_prec; + if right_needs_group { + consume_by_precedence + } else if let (Scan::Fail, Precedence::Unambiguous) = (scan, fixup.next_operator) { + Scan::Fail + } else { + Scan::Bailout + } + } + Expr::RawAddr(ExprRawAddr { expr, .. }) + | Expr::Reference(ExprReference { expr, .. }) + | Expr::Unary(ExprUnary { expr, .. }) => { + if match fixup.next_operator { + Precedence::Unambiguous => { + fail_offset >= 2 + && (consume_by_precedence == Scan::Consume || bailout_offset >= 1) + } + _ => bailout_offset >= 1, + } { + return Scan::Consume; + } + let right_fixup = fixup.rightmost_subexpression_fixup(false, false, Precedence::Prefix); + let scan = scan_right( + expr, + right_fixup, + precedence, + match fixup.next_operator { + Precedence::Unambiguous => fail_offset, + _ => 1, + }, + consume_by_precedence as u8 - Scan::Bailout as u8, + ); + match scan { + Scan::Fail => {} + Scan::Bailout => return consume_by_precedence, + Scan::Consume => return Scan::Consume, + } + if right_fixup.rightmost_subexpression_precedence(expr) < Precedence::Prefix { + consume_by_precedence + } else if let (Scan::Fail, Precedence::Unambiguous) = (scan, fixup.next_operator) { + Scan::Fail + } else { + Scan::Bailout + } + } + Expr::Range(e) if e.attrs.is_empty() => match &e.end { + Some(end) => { + if fail_offset >= 2 { + return Scan::Consume; + } + let right_fixup = + fixup.rightmost_subexpression_fixup(false, true, Precedence::Range); + let scan = scan_right( + end, + right_fixup, + Precedence::Range, + fail_offset, + match fixup.next_operator { + Precedence::Assign | Precedence::Range => 0, + _ => 1, + }, + ); + if match (scan, fixup.next_operator) { + (Scan::Fail, _) => false, + (Scan::Bailout, Precedence::Assign | Precedence::Range) => false, + (Scan::Bailout | Scan::Consume, _) => true, + } { + return Scan::Consume; + } + if right_fixup.rightmost_subexpression_precedence(end) <= Precedence::Range { + Scan::Consume + } else { + Scan::Fail + } + } + None => { + if fixup.next_operator_can_begin_expr { + Scan::Consume + } else { + Scan::Fail + } + } + }, + Expr::Break(e) => match &e.expr { + Some(value) => { + if bailout_offset >= 1 || e.label.is_none() && classify::expr_leading_label(value) { + return Scan::Consume; + } + let right_fixup = fixup.rightmost_subexpression_fixup(true, true, Precedence::Jump); + match scan_right(value, right_fixup, Precedence::Jump, 1, 1) { + Scan::Fail => Scan::Bailout, + Scan::Bailout | Scan::Consume => Scan::Consume, + } + } + None => match fixup.next_operator { + Precedence::Assign if precedence > Precedence::Assign => Scan::Fail, + _ => Scan::Consume, + }, + }, + Expr::Return(ExprReturn { expr, .. }) | Expr::Yield(ExprYield { expr, .. }) => match expr { + Some(e) => { + if bailout_offset >= 1 { + return Scan::Consume; + } + let right_fixup = + fixup.rightmost_subexpression_fixup(true, false, Precedence::Jump); + match scan_right(e, right_fixup, Precedence::Jump, 1, 1) { + Scan::Fail => Scan::Bailout, + Scan::Bailout | Scan::Consume => Scan::Consume, + } + } + None => match fixup.next_operator { + Precedence::Assign if precedence > Precedence::Assign => Scan::Fail, + _ => Scan::Consume, + }, + }, + Expr::Closure(e) => { + if matches!(e.output, ReturnType::Default) + || matches!(&*e.body, Expr::Block(body) if body.attrs.is_empty() && body.label.is_none()) + { + if bailout_offset >= 1 { + return Scan::Consume; + } + let right_fixup = + fixup.rightmost_subexpression_fixup(false, false, Precedence::Jump); + match scan_right(&e.body, right_fixup, Precedence::Jump, 1, 1) { + Scan::Fail => Scan::Bailout, + Scan::Bailout | Scan::Consume => Scan::Consume, + } + } else { + Scan::Consume + } + } + Expr::Let(e) => { + if bailout_offset >= 1 { + return Scan::Consume; + } + let right_fixup = fixup.rightmost_subexpression_fixup(false, false, Precedence::Let); + let scan = scan_right( + &e.expr, + right_fixup, + Precedence::Let, + 1, + if fixup.next_operator < Precedence::Let { + 0 + } else { + 1 + }, + ); + match scan { + Scan::Fail | Scan::Bailout if fixup.next_operator < Precedence::Let => { + return Scan::Bailout; + } + Scan::Consume => return Scan::Consume, + _ => {} + } + if right_fixup.rightmost_subexpression_precedence(&e.expr) < Precedence::Let { + Scan::Consume + } else if let Scan::Fail = scan { + Scan::Bailout + } else { + Scan::Consume + } + } + Expr::Array(_) + | Expr::Assign(_) + | Expr::Async(_) + | Expr::Await(_) + | Expr::Binary(_) + | Expr::Block(_) + | Expr::Call(_) + | Expr::Cast(_) + | Expr::Const(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::ForLoop(_) + | Expr::Group(_) + | Expr::If(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Lit(_) + | Expr::Loop(_) + | Expr::Macro(_) + | Expr::Match(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Range(_) + | Expr::Repeat(_) + | Expr::Struct(_) + | Expr::Try(_) + | Expr::TryBlock(_) + | Expr::Tuple(_) + | Expr::Unsafe(_) + | Expr::Verbatim(_) + | Expr::While(_) => match fixup.next_operator { + Precedence::Assign | Precedence::Range if precedence == Precedence::Range => Scan::Fail, + _ if precedence == Precedence::Let && fixup.next_operator < Precedence::Let => { + Scan::Fail + } + _ => consume_by_precedence, + }, + } +} diff --git a/third_party/crates/syn-2.0.119/src/gen/clone.rs b/third_party/crates/syn-2.0.119/src/gen/clone.rs new file mode 100644 index 0000000..be2b698 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/gen/clone.rs @@ -0,0 +1,2267 @@ +// This file is @generated by syn-internal-codegen. +// It is not intended for manual editing. + +#![allow(clippy::clone_on_copy, clippy::expl_impl_clone_on_copy)] +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Abi { + fn clone(&self) -> Self { + crate::Abi { + extern_token: self.extern_token.clone(), + name: self.name.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::AngleBracketedGenericArguments { + fn clone(&self) -> Self { + crate::AngleBracketedGenericArguments { + colon2_token: self.colon2_token.clone(), + lt_token: self.lt_token.clone(), + args: self.args.clone(), + gt_token: self.gt_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Arm { + fn clone(&self) -> Self { + crate::Arm { + attrs: self.attrs.clone(), + pat: self.pat.clone(), + guard: self.guard.clone(), + fat_arrow_token: self.fat_arrow_token.clone(), + body: self.body.clone(), + comma: self.comma.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::AssocConst { + fn clone(&self) -> Self { + crate::AssocConst { + ident: self.ident.clone(), + generics: self.generics.clone(), + eq_token: self.eq_token.clone(), + value: self.value.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::AssocType { + fn clone(&self) -> Self { + crate::AssocType { + ident: self.ident.clone(), + generics: self.generics.clone(), + eq_token: self.eq_token.clone(), + ty: self.ty.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Copy for crate::AttrStyle {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::AttrStyle { + fn clone(&self) -> Self { + *self + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Attribute { + fn clone(&self) -> Self { + crate::Attribute { + pound_token: self.pound_token.clone(), + style: self.style.clone(), + bracket_token: self.bracket_token.clone(), + meta: self.meta.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::BareFnArg { + fn clone(&self) -> Self { + crate::BareFnArg { + attrs: self.attrs.clone(), + name: self.name.clone(), + ty: self.ty.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::BareVariadic { + fn clone(&self) -> Self { + crate::BareVariadic { + attrs: self.attrs.clone(), + name: self.name.clone(), + dots: self.dots.clone(), + comma: self.comma.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Copy for crate::BinOp {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::BinOp { + fn clone(&self) -> Self { + *self + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Block { + fn clone(&self) -> Self { + crate::Block { + brace_token: self.brace_token.clone(), + stmts: self.stmts.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::BoundLifetimes { + fn clone(&self) -> Self { + crate::BoundLifetimes { + for_token: self.for_token.clone(), + lt_token: self.lt_token.clone(), + lifetimes: self.lifetimes.clone(), + gt_token: self.gt_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::CapturedParam { + fn clone(&self) -> Self { + match self { + crate::CapturedParam::Lifetime(v0) => { + crate::CapturedParam::Lifetime(v0.clone()) + } + crate::CapturedParam::Ident(v0) => crate::CapturedParam::Ident(v0.clone()), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ConstParam { + fn clone(&self) -> Self { + crate::ConstParam { + attrs: self.attrs.clone(), + const_token: self.const_token.clone(), + ident: self.ident.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + eq_token: self.eq_token.clone(), + default: self.default.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Constraint { + fn clone(&self) -> Self { + crate::Constraint { + ident: self.ident.clone(), + generics: self.generics.clone(), + colon_token: self.colon_token.clone(), + bounds: self.bounds.clone(), + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Data { + fn clone(&self) -> Self { + match self { + crate::Data::Struct(v0) => crate::Data::Struct(v0.clone()), + crate::Data::Enum(v0) => crate::Data::Enum(v0.clone()), + crate::Data::Union(v0) => crate::Data::Union(v0.clone()), + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::DataEnum { + fn clone(&self) -> Self { + crate::DataEnum { + enum_token: self.enum_token.clone(), + brace_token: self.brace_token.clone(), + variants: self.variants.clone(), + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::DataStruct { + fn clone(&self) -> Self { + crate::DataStruct { + struct_token: self.struct_token.clone(), + fields: self.fields.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::DataUnion { + fn clone(&self) -> Self { + crate::DataUnion { + union_token: self.union_token.clone(), + fields: self.fields.clone(), + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::DeriveInput { + fn clone(&self) -> Self { + crate::DeriveInput { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + data: self.data.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Expr { + fn clone(&self) -> Self { + match self { + #[cfg(feature = "full")] + crate::Expr::Array(v0) => crate::Expr::Array(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Assign(v0) => crate::Expr::Assign(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Async(v0) => crate::Expr::Async(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Await(v0) => crate::Expr::Await(v0.clone()), + crate::Expr::Binary(v0) => crate::Expr::Binary(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Block(v0) => crate::Expr::Block(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Break(v0) => crate::Expr::Break(v0.clone()), + crate::Expr::Call(v0) => crate::Expr::Call(v0.clone()), + crate::Expr::Cast(v0) => crate::Expr::Cast(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Closure(v0) => crate::Expr::Closure(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Const(v0) => crate::Expr::Const(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Continue(v0) => crate::Expr::Continue(v0.clone()), + crate::Expr::Field(v0) => crate::Expr::Field(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::ForLoop(v0) => crate::Expr::ForLoop(v0.clone()), + crate::Expr::Group(v0) => crate::Expr::Group(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::If(v0) => crate::Expr::If(v0.clone()), + crate::Expr::Index(v0) => crate::Expr::Index(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Infer(v0) => crate::Expr::Infer(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Let(v0) => crate::Expr::Let(v0.clone()), + crate::Expr::Lit(v0) => crate::Expr::Lit(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Loop(v0) => crate::Expr::Loop(v0.clone()), + crate::Expr::Macro(v0) => crate::Expr::Macro(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Match(v0) => crate::Expr::Match(v0.clone()), + crate::Expr::MethodCall(v0) => crate::Expr::MethodCall(v0.clone()), + crate::Expr::Paren(v0) => crate::Expr::Paren(v0.clone()), + crate::Expr::Path(v0) => crate::Expr::Path(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Range(v0) => crate::Expr::Range(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::RawAddr(v0) => crate::Expr::RawAddr(v0.clone()), + crate::Expr::Reference(v0) => crate::Expr::Reference(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Repeat(v0) => crate::Expr::Repeat(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Return(v0) => crate::Expr::Return(v0.clone()), + crate::Expr::Struct(v0) => crate::Expr::Struct(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Try(v0) => crate::Expr::Try(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::TryBlock(v0) => crate::Expr::TryBlock(v0.clone()), + crate::Expr::Tuple(v0) => crate::Expr::Tuple(v0.clone()), + crate::Expr::Unary(v0) => crate::Expr::Unary(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Unsafe(v0) => crate::Expr::Unsafe(v0.clone()), + crate::Expr::Verbatim(v0) => crate::Expr::Verbatim(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::While(v0) => crate::Expr::While(v0.clone()), + #[cfg(feature = "full")] + crate::Expr::Yield(v0) => crate::Expr::Yield(v0.clone()), + #[cfg(not(feature = "full"))] + _ => unreachable!(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprArray { + fn clone(&self) -> Self { + crate::ExprArray { + attrs: self.attrs.clone(), + bracket_token: self.bracket_token.clone(), + elems: self.elems.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprAssign { + fn clone(&self) -> Self { + crate::ExprAssign { + attrs: self.attrs.clone(), + left: self.left.clone(), + eq_token: self.eq_token.clone(), + right: self.right.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprAsync { + fn clone(&self) -> Self { + crate::ExprAsync { + attrs: self.attrs.clone(), + async_token: self.async_token.clone(), + capture: self.capture.clone(), + block: self.block.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprAwait { + fn clone(&self) -> Self { + crate::ExprAwait { + attrs: self.attrs.clone(), + base: self.base.clone(), + dot_token: self.dot_token.clone(), + await_token: self.await_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprBinary { + fn clone(&self) -> Self { + crate::ExprBinary { + attrs: self.attrs.clone(), + left: self.left.clone(), + op: self.op.clone(), + right: self.right.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprBlock { + fn clone(&self) -> Self { + crate::ExprBlock { + attrs: self.attrs.clone(), + label: self.label.clone(), + block: self.block.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprBreak { + fn clone(&self) -> Self { + crate::ExprBreak { + attrs: self.attrs.clone(), + break_token: self.break_token.clone(), + label: self.label.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprCall { + fn clone(&self) -> Self { + crate::ExprCall { + attrs: self.attrs.clone(), + func: self.func.clone(), + paren_token: self.paren_token.clone(), + args: self.args.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprCast { + fn clone(&self) -> Self { + crate::ExprCast { + attrs: self.attrs.clone(), + expr: self.expr.clone(), + as_token: self.as_token.clone(), + ty: self.ty.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprClosure { + fn clone(&self) -> Self { + crate::ExprClosure { + attrs: self.attrs.clone(), + lifetimes: self.lifetimes.clone(), + constness: self.constness.clone(), + movability: self.movability.clone(), + asyncness: self.asyncness.clone(), + capture: self.capture.clone(), + or1_token: self.or1_token.clone(), + inputs: self.inputs.clone(), + or2_token: self.or2_token.clone(), + output: self.output.clone(), + body: self.body.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprConst { + fn clone(&self) -> Self { + crate::ExprConst { + attrs: self.attrs.clone(), + const_token: self.const_token.clone(), + block: self.block.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprContinue { + fn clone(&self) -> Self { + crate::ExprContinue { + attrs: self.attrs.clone(), + continue_token: self.continue_token.clone(), + label: self.label.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprField { + fn clone(&self) -> Self { + crate::ExprField { + attrs: self.attrs.clone(), + base: self.base.clone(), + dot_token: self.dot_token.clone(), + member: self.member.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprForLoop { + fn clone(&self) -> Self { + crate::ExprForLoop { + attrs: self.attrs.clone(), + label: self.label.clone(), + for_token: self.for_token.clone(), + pat: self.pat.clone(), + in_token: self.in_token.clone(), + expr: self.expr.clone(), + body: self.body.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprGroup { + fn clone(&self) -> Self { + crate::ExprGroup { + attrs: self.attrs.clone(), + group_token: self.group_token.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprIf { + fn clone(&self) -> Self { + crate::ExprIf { + attrs: self.attrs.clone(), + if_token: self.if_token.clone(), + cond: self.cond.clone(), + then_branch: self.then_branch.clone(), + else_branch: self.else_branch.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprIndex { + fn clone(&self) -> Self { + crate::ExprIndex { + attrs: self.attrs.clone(), + expr: self.expr.clone(), + bracket_token: self.bracket_token.clone(), + index: self.index.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprInfer { + fn clone(&self) -> Self { + crate::ExprInfer { + attrs: self.attrs.clone(), + underscore_token: self.underscore_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprLet { + fn clone(&self) -> Self { + crate::ExprLet { + attrs: self.attrs.clone(), + let_token: self.let_token.clone(), + pat: self.pat.clone(), + eq_token: self.eq_token.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprLit { + fn clone(&self) -> Self { + crate::ExprLit { + attrs: self.attrs.clone(), + lit: self.lit.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprLoop { + fn clone(&self) -> Self { + crate::ExprLoop { + attrs: self.attrs.clone(), + label: self.label.clone(), + loop_token: self.loop_token.clone(), + body: self.body.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprMacro { + fn clone(&self) -> Self { + crate::ExprMacro { + attrs: self.attrs.clone(), + mac: self.mac.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprMatch { + fn clone(&self) -> Self { + crate::ExprMatch { + attrs: self.attrs.clone(), + match_token: self.match_token.clone(), + expr: self.expr.clone(), + brace_token: self.brace_token.clone(), + arms: self.arms.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprMethodCall { + fn clone(&self) -> Self { + crate::ExprMethodCall { + attrs: self.attrs.clone(), + receiver: self.receiver.clone(), + dot_token: self.dot_token.clone(), + method: self.method.clone(), + turbofish: self.turbofish.clone(), + paren_token: self.paren_token.clone(), + args: self.args.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprParen { + fn clone(&self) -> Self { + crate::ExprParen { + attrs: self.attrs.clone(), + paren_token: self.paren_token.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprPath { + fn clone(&self) -> Self { + crate::ExprPath { + attrs: self.attrs.clone(), + qself: self.qself.clone(), + path: self.path.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprRange { + fn clone(&self) -> Self { + crate::ExprRange { + attrs: self.attrs.clone(), + start: self.start.clone(), + limits: self.limits.clone(), + end: self.end.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprRawAddr { + fn clone(&self) -> Self { + crate::ExprRawAddr { + attrs: self.attrs.clone(), + and_token: self.and_token.clone(), + raw: self.raw.clone(), + mutability: self.mutability.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprReference { + fn clone(&self) -> Self { + crate::ExprReference { + attrs: self.attrs.clone(), + and_token: self.and_token.clone(), + mutability: self.mutability.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprRepeat { + fn clone(&self) -> Self { + crate::ExprRepeat { + attrs: self.attrs.clone(), + bracket_token: self.bracket_token.clone(), + expr: self.expr.clone(), + semi_token: self.semi_token.clone(), + len: self.len.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprReturn { + fn clone(&self) -> Self { + crate::ExprReturn { + attrs: self.attrs.clone(), + return_token: self.return_token.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprStruct { + fn clone(&self) -> Self { + crate::ExprStruct { + attrs: self.attrs.clone(), + qself: self.qself.clone(), + path: self.path.clone(), + brace_token: self.brace_token.clone(), + fields: self.fields.clone(), + dot2_token: self.dot2_token.clone(), + rest: self.rest.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprTry { + fn clone(&self) -> Self { + crate::ExprTry { + attrs: self.attrs.clone(), + expr: self.expr.clone(), + question_token: self.question_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprTryBlock { + fn clone(&self) -> Self { + crate::ExprTryBlock { + attrs: self.attrs.clone(), + try_token: self.try_token.clone(), + block: self.block.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprTuple { + fn clone(&self) -> Self { + crate::ExprTuple { + attrs: self.attrs.clone(), + paren_token: self.paren_token.clone(), + elems: self.elems.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprUnary { + fn clone(&self) -> Self { + crate::ExprUnary { + attrs: self.attrs.clone(), + op: self.op.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprUnsafe { + fn clone(&self) -> Self { + crate::ExprUnsafe { + attrs: self.attrs.clone(), + unsafe_token: self.unsafe_token.clone(), + block: self.block.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprWhile { + fn clone(&self) -> Self { + crate::ExprWhile { + attrs: self.attrs.clone(), + label: self.label.clone(), + while_token: self.while_token.clone(), + cond: self.cond.clone(), + body: self.body.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ExprYield { + fn clone(&self) -> Self { + crate::ExprYield { + attrs: self.attrs.clone(), + yield_token: self.yield_token.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Field { + fn clone(&self) -> Self { + crate::Field { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + mutability: self.mutability.clone(), + ident: self.ident.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::FieldMutability { + fn clone(&self) -> Self { + match self { + crate::FieldMutability::None => crate::FieldMutability::None, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::FieldPat { + fn clone(&self) -> Self { + crate::FieldPat { + attrs: self.attrs.clone(), + member: self.member.clone(), + colon_token: self.colon_token.clone(), + pat: self.pat.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::FieldValue { + fn clone(&self) -> Self { + crate::FieldValue { + attrs: self.attrs.clone(), + member: self.member.clone(), + colon_token: self.colon_token.clone(), + expr: self.expr.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Fields { + fn clone(&self) -> Self { + match self { + crate::Fields::Named(v0) => crate::Fields::Named(v0.clone()), + crate::Fields::Unnamed(v0) => crate::Fields::Unnamed(v0.clone()), + crate::Fields::Unit => crate::Fields::Unit, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::FieldsNamed { + fn clone(&self) -> Self { + crate::FieldsNamed { + brace_token: self.brace_token.clone(), + named: self.named.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::FieldsUnnamed { + fn clone(&self) -> Self { + crate::FieldsUnnamed { + paren_token: self.paren_token.clone(), + unnamed: self.unnamed.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::File { + fn clone(&self) -> Self { + crate::File { + shebang: self.shebang.clone(), + attrs: self.attrs.clone(), + items: self.items.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::FnArg { + fn clone(&self) -> Self { + match self { + crate::FnArg::Receiver(v0) => crate::FnArg::Receiver(v0.clone()), + crate::FnArg::Typed(v0) => crate::FnArg::Typed(v0.clone()), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ForeignItem { + fn clone(&self) -> Self { + match self { + crate::ForeignItem::Fn(v0) => crate::ForeignItem::Fn(v0.clone()), + crate::ForeignItem::Static(v0) => crate::ForeignItem::Static(v0.clone()), + crate::ForeignItem::Type(v0) => crate::ForeignItem::Type(v0.clone()), + crate::ForeignItem::Macro(v0) => crate::ForeignItem::Macro(v0.clone()), + crate::ForeignItem::Verbatim(v0) => crate::ForeignItem::Verbatim(v0.clone()), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ForeignItemFn { + fn clone(&self) -> Self { + crate::ForeignItemFn { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + sig: self.sig.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ForeignItemMacro { + fn clone(&self) -> Self { + crate::ForeignItemMacro { + attrs: self.attrs.clone(), + mac: self.mac.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ForeignItemStatic { + fn clone(&self) -> Self { + crate::ForeignItemStatic { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + static_token: self.static_token.clone(), + mutability: self.mutability.clone(), + ident: self.ident.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ForeignItemType { + fn clone(&self) -> Self { + crate::ForeignItemType { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + type_token: self.type_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::GenericArgument { + fn clone(&self) -> Self { + match self { + crate::GenericArgument::Lifetime(v0) => { + crate::GenericArgument::Lifetime(v0.clone()) + } + crate::GenericArgument::Type(v0) => crate::GenericArgument::Type(v0.clone()), + crate::GenericArgument::Const(v0) => { + crate::GenericArgument::Const(v0.clone()) + } + crate::GenericArgument::AssocType(v0) => { + crate::GenericArgument::AssocType(v0.clone()) + } + crate::GenericArgument::AssocConst(v0) => { + crate::GenericArgument::AssocConst(v0.clone()) + } + crate::GenericArgument::Constraint(v0) => { + crate::GenericArgument::Constraint(v0.clone()) + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::GenericParam { + fn clone(&self) -> Self { + match self { + crate::GenericParam::Lifetime(v0) => { + crate::GenericParam::Lifetime(v0.clone()) + } + crate::GenericParam::Type(v0) => crate::GenericParam::Type(v0.clone()), + crate::GenericParam::Const(v0) => crate::GenericParam::Const(v0.clone()), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Generics { + fn clone(&self) -> Self { + crate::Generics { + lt_token: self.lt_token.clone(), + params: self.params.clone(), + gt_token: self.gt_token.clone(), + where_clause: self.where_clause.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ImplItem { + fn clone(&self) -> Self { + match self { + crate::ImplItem::Const(v0) => crate::ImplItem::Const(v0.clone()), + crate::ImplItem::Fn(v0) => crate::ImplItem::Fn(v0.clone()), + crate::ImplItem::Type(v0) => crate::ImplItem::Type(v0.clone()), + crate::ImplItem::Macro(v0) => crate::ImplItem::Macro(v0.clone()), + crate::ImplItem::Verbatim(v0) => crate::ImplItem::Verbatim(v0.clone()), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ImplItemConst { + fn clone(&self) -> Self { + crate::ImplItemConst { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + defaultness: self.defaultness.clone(), + const_token: self.const_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + eq_token: self.eq_token.clone(), + expr: self.expr.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ImplItemFn { + fn clone(&self) -> Self { + crate::ImplItemFn { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + defaultness: self.defaultness.clone(), + sig: self.sig.clone(), + block: self.block.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ImplItemMacro { + fn clone(&self) -> Self { + crate::ImplItemMacro { + attrs: self.attrs.clone(), + mac: self.mac.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ImplItemType { + fn clone(&self) -> Self { + crate::ImplItemType { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + defaultness: self.defaultness.clone(), + type_token: self.type_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + eq_token: self.eq_token.clone(), + ty: self.ty.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ImplRestriction { + fn clone(&self) -> Self { + match *self {} + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Index { + fn clone(&self) -> Self { + crate::Index { + index: self.index.clone(), + span: self.span.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Item { + fn clone(&self) -> Self { + match self { + crate::Item::Const(v0) => crate::Item::Const(v0.clone()), + crate::Item::Enum(v0) => crate::Item::Enum(v0.clone()), + crate::Item::ExternCrate(v0) => crate::Item::ExternCrate(v0.clone()), + crate::Item::Fn(v0) => crate::Item::Fn(v0.clone()), + crate::Item::ForeignMod(v0) => crate::Item::ForeignMod(v0.clone()), + crate::Item::Impl(v0) => crate::Item::Impl(v0.clone()), + crate::Item::Macro(v0) => crate::Item::Macro(v0.clone()), + crate::Item::Mod(v0) => crate::Item::Mod(v0.clone()), + crate::Item::Static(v0) => crate::Item::Static(v0.clone()), + crate::Item::Struct(v0) => crate::Item::Struct(v0.clone()), + crate::Item::Trait(v0) => crate::Item::Trait(v0.clone()), + crate::Item::TraitAlias(v0) => crate::Item::TraitAlias(v0.clone()), + crate::Item::Type(v0) => crate::Item::Type(v0.clone()), + crate::Item::Union(v0) => crate::Item::Union(v0.clone()), + crate::Item::Use(v0) => crate::Item::Use(v0.clone()), + crate::Item::Verbatim(v0) => crate::Item::Verbatim(v0.clone()), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemConst { + fn clone(&self) -> Self { + crate::ItemConst { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + const_token: self.const_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + eq_token: self.eq_token.clone(), + expr: self.expr.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemEnum { + fn clone(&self) -> Self { + crate::ItemEnum { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + enum_token: self.enum_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + brace_token: self.brace_token.clone(), + variants: self.variants.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemExternCrate { + fn clone(&self) -> Self { + crate::ItemExternCrate { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + extern_token: self.extern_token.clone(), + crate_token: self.crate_token.clone(), + ident: self.ident.clone(), + rename: self.rename.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemFn { + fn clone(&self) -> Self { + crate::ItemFn { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + sig: self.sig.clone(), + block: self.block.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemForeignMod { + fn clone(&self) -> Self { + crate::ItemForeignMod { + attrs: self.attrs.clone(), + unsafety: self.unsafety.clone(), + abi: self.abi.clone(), + brace_token: self.brace_token.clone(), + items: self.items.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemImpl { + fn clone(&self) -> Self { + crate::ItemImpl { + attrs: self.attrs.clone(), + defaultness: self.defaultness.clone(), + unsafety: self.unsafety.clone(), + impl_token: self.impl_token.clone(), + generics: self.generics.clone(), + trait_: self.trait_.clone(), + self_ty: self.self_ty.clone(), + brace_token: self.brace_token.clone(), + items: self.items.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemMacro { + fn clone(&self) -> Self { + crate::ItemMacro { + attrs: self.attrs.clone(), + ident: self.ident.clone(), + mac: self.mac.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemMod { + fn clone(&self) -> Self { + crate::ItemMod { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + unsafety: self.unsafety.clone(), + mod_token: self.mod_token.clone(), + ident: self.ident.clone(), + content: self.content.clone(), + semi: self.semi.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemStatic { + fn clone(&self) -> Self { + crate::ItemStatic { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + static_token: self.static_token.clone(), + mutability: self.mutability.clone(), + ident: self.ident.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + eq_token: self.eq_token.clone(), + expr: self.expr.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemStruct { + fn clone(&self) -> Self { + crate::ItemStruct { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + struct_token: self.struct_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + fields: self.fields.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemTrait { + fn clone(&self) -> Self { + crate::ItemTrait { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + unsafety: self.unsafety.clone(), + auto_token: self.auto_token.clone(), + restriction: self.restriction.clone(), + trait_token: self.trait_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + colon_token: self.colon_token.clone(), + supertraits: self.supertraits.clone(), + brace_token: self.brace_token.clone(), + items: self.items.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemTraitAlias { + fn clone(&self) -> Self { + crate::ItemTraitAlias { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + trait_token: self.trait_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + eq_token: self.eq_token.clone(), + bounds: self.bounds.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemType { + fn clone(&self) -> Self { + crate::ItemType { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + type_token: self.type_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + eq_token: self.eq_token.clone(), + ty: self.ty.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemUnion { + fn clone(&self) -> Self { + crate::ItemUnion { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + union_token: self.union_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + fields: self.fields.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ItemUse { + fn clone(&self) -> Self { + crate::ItemUse { + attrs: self.attrs.clone(), + vis: self.vis.clone(), + use_token: self.use_token.clone(), + leading_colon: self.leading_colon.clone(), + tree: self.tree.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Label { + fn clone(&self) -> Self { + crate::Label { + name: self.name.clone(), + colon_token: self.colon_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::LifetimeParam { + fn clone(&self) -> Self { + crate::LifetimeParam { + attrs: self.attrs.clone(), + lifetime: self.lifetime.clone(), + colon_token: self.colon_token.clone(), + bounds: self.bounds.clone(), + } + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Lit { + fn clone(&self) -> Self { + match self { + crate::Lit::Str(v0) => crate::Lit::Str(v0.clone()), + crate::Lit::ByteStr(v0) => crate::Lit::ByteStr(v0.clone()), + crate::Lit::CStr(v0) => crate::Lit::CStr(v0.clone()), + crate::Lit::Byte(v0) => crate::Lit::Byte(v0.clone()), + crate::Lit::Char(v0) => crate::Lit::Char(v0.clone()), + crate::Lit::Int(v0) => crate::Lit::Int(v0.clone()), + crate::Lit::Float(v0) => crate::Lit::Float(v0.clone()), + crate::Lit::Bool(v0) => crate::Lit::Bool(v0.clone()), + crate::Lit::Verbatim(v0) => crate::Lit::Verbatim(v0.clone()), + } + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::LitBool { + fn clone(&self) -> Self { + crate::LitBool { + value: self.value.clone(), + span: self.span.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Local { + fn clone(&self) -> Self { + crate::Local { + attrs: self.attrs.clone(), + let_token: self.let_token.clone(), + pat: self.pat.clone(), + init: self.init.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::LocalInit { + fn clone(&self) -> Self { + crate::LocalInit { + eq_token: self.eq_token.clone(), + expr: self.expr.clone(), + diverge: self.diverge.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Macro { + fn clone(&self) -> Self { + crate::Macro { + path: self.path.clone(), + bang_token: self.bang_token.clone(), + delimiter: self.delimiter.clone(), + tokens: self.tokens.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::MacroDelimiter { + fn clone(&self) -> Self { + match self { + crate::MacroDelimiter::Paren(v0) => crate::MacroDelimiter::Paren(v0.clone()), + crate::MacroDelimiter::Brace(v0) => crate::MacroDelimiter::Brace(v0.clone()), + crate::MacroDelimiter::Bracket(v0) => { + crate::MacroDelimiter::Bracket(v0.clone()) + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Member { + fn clone(&self) -> Self { + match self { + crate::Member::Named(v0) => crate::Member::Named(v0.clone()), + crate::Member::Unnamed(v0) => crate::Member::Unnamed(v0.clone()), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Meta { + fn clone(&self) -> Self { + match self { + crate::Meta::Path(v0) => crate::Meta::Path(v0.clone()), + crate::Meta::List(v0) => crate::Meta::List(v0.clone()), + crate::Meta::NameValue(v0) => crate::Meta::NameValue(v0.clone()), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::MetaList { + fn clone(&self) -> Self { + crate::MetaList { + path: self.path.clone(), + delimiter: self.delimiter.clone(), + tokens: self.tokens.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::MetaNameValue { + fn clone(&self) -> Self { + crate::MetaNameValue { + path: self.path.clone(), + eq_token: self.eq_token.clone(), + value: self.value.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ParenthesizedGenericArguments { + fn clone(&self) -> Self { + crate::ParenthesizedGenericArguments { + paren_token: self.paren_token.clone(), + inputs: self.inputs.clone(), + output: self.output.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Pat { + fn clone(&self) -> Self { + match self { + crate::Pat::Const(v0) => crate::Pat::Const(v0.clone()), + crate::Pat::Ident(v0) => crate::Pat::Ident(v0.clone()), + crate::Pat::Lit(v0) => crate::Pat::Lit(v0.clone()), + crate::Pat::Macro(v0) => crate::Pat::Macro(v0.clone()), + crate::Pat::Or(v0) => crate::Pat::Or(v0.clone()), + crate::Pat::Paren(v0) => crate::Pat::Paren(v0.clone()), + crate::Pat::Path(v0) => crate::Pat::Path(v0.clone()), + crate::Pat::Range(v0) => crate::Pat::Range(v0.clone()), + crate::Pat::Reference(v0) => crate::Pat::Reference(v0.clone()), + crate::Pat::Rest(v0) => crate::Pat::Rest(v0.clone()), + crate::Pat::Slice(v0) => crate::Pat::Slice(v0.clone()), + crate::Pat::Struct(v0) => crate::Pat::Struct(v0.clone()), + crate::Pat::Tuple(v0) => crate::Pat::Tuple(v0.clone()), + crate::Pat::TupleStruct(v0) => crate::Pat::TupleStruct(v0.clone()), + crate::Pat::Type(v0) => crate::Pat::Type(v0.clone()), + crate::Pat::Verbatim(v0) => crate::Pat::Verbatim(v0.clone()), + crate::Pat::Wild(v0) => crate::Pat::Wild(v0.clone()), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatIdent { + fn clone(&self) -> Self { + crate::PatIdent { + attrs: self.attrs.clone(), + by_ref: self.by_ref.clone(), + mutability: self.mutability.clone(), + ident: self.ident.clone(), + subpat: self.subpat.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatOr { + fn clone(&self) -> Self { + crate::PatOr { + attrs: self.attrs.clone(), + leading_vert: self.leading_vert.clone(), + cases: self.cases.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatParen { + fn clone(&self) -> Self { + crate::PatParen { + attrs: self.attrs.clone(), + paren_token: self.paren_token.clone(), + pat: self.pat.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatReference { + fn clone(&self) -> Self { + crate::PatReference { + attrs: self.attrs.clone(), + and_token: self.and_token.clone(), + mutability: self.mutability.clone(), + pat: self.pat.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatRest { + fn clone(&self) -> Self { + crate::PatRest { + attrs: self.attrs.clone(), + dot2_token: self.dot2_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatSlice { + fn clone(&self) -> Self { + crate::PatSlice { + attrs: self.attrs.clone(), + bracket_token: self.bracket_token.clone(), + elems: self.elems.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatStruct { + fn clone(&self) -> Self { + crate::PatStruct { + attrs: self.attrs.clone(), + qself: self.qself.clone(), + path: self.path.clone(), + brace_token: self.brace_token.clone(), + fields: self.fields.clone(), + rest: self.rest.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatTuple { + fn clone(&self) -> Self { + crate::PatTuple { + attrs: self.attrs.clone(), + paren_token: self.paren_token.clone(), + elems: self.elems.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatTupleStruct { + fn clone(&self) -> Self { + crate::PatTupleStruct { + attrs: self.attrs.clone(), + qself: self.qself.clone(), + path: self.path.clone(), + paren_token: self.paren_token.clone(), + elems: self.elems.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatType { + fn clone(&self) -> Self { + crate::PatType { + attrs: self.attrs.clone(), + pat: self.pat.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PatWild { + fn clone(&self) -> Self { + crate::PatWild { + attrs: self.attrs.clone(), + underscore_token: self.underscore_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Path { + fn clone(&self) -> Self { + crate::Path { + leading_colon: self.leading_colon.clone(), + segments: self.segments.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PathArguments { + fn clone(&self) -> Self { + match self { + crate::PathArguments::None => crate::PathArguments::None, + crate::PathArguments::AngleBracketed(v0) => { + crate::PathArguments::AngleBracketed(v0.clone()) + } + crate::PathArguments::Parenthesized(v0) => { + crate::PathArguments::Parenthesized(v0.clone()) + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PathSegment { + fn clone(&self) -> Self { + crate::PathSegment { + ident: self.ident.clone(), + arguments: self.arguments.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PointerMutability { + fn clone(&self) -> Self { + match self { + crate::PointerMutability::Const(v0) => { + crate::PointerMutability::Const(v0.clone()) + } + crate::PointerMutability::Mut(v0) => { + crate::PointerMutability::Mut(v0.clone()) + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PreciseCapture { + fn clone(&self) -> Self { + crate::PreciseCapture { + use_token: self.use_token.clone(), + lt_token: self.lt_token.clone(), + params: self.params.clone(), + gt_token: self.gt_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PredicateLifetime { + fn clone(&self) -> Self { + crate::PredicateLifetime { + lifetime: self.lifetime.clone(), + colon_token: self.colon_token.clone(), + bounds: self.bounds.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::PredicateType { + fn clone(&self) -> Self { + crate::PredicateType { + lifetimes: self.lifetimes.clone(), + bounded_ty: self.bounded_ty.clone(), + colon_token: self.colon_token.clone(), + bounds: self.bounds.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::QSelf { + fn clone(&self) -> Self { + crate::QSelf { + lt_token: self.lt_token.clone(), + ty: self.ty.clone(), + position: self.position.clone(), + as_token: self.as_token.clone(), + gt_token: self.gt_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Copy for crate::RangeLimits {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::RangeLimits { + fn clone(&self) -> Self { + *self + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Receiver { + fn clone(&self) -> Self { + crate::Receiver { + attrs: self.attrs.clone(), + reference: self.reference.clone(), + mutability: self.mutability.clone(), + self_token: self.self_token.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::ReturnType { + fn clone(&self) -> Self { + match self { + crate::ReturnType::Default => crate::ReturnType::Default, + crate::ReturnType::Type(v0, v1) => { + crate::ReturnType::Type(v0.clone(), v1.clone()) + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Signature { + fn clone(&self) -> Self { + crate::Signature { + constness: self.constness.clone(), + asyncness: self.asyncness.clone(), + unsafety: self.unsafety.clone(), + abi: self.abi.clone(), + fn_token: self.fn_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + paren_token: self.paren_token.clone(), + inputs: self.inputs.clone(), + variadic: self.variadic.clone(), + output: self.output.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::StaticMutability { + fn clone(&self) -> Self { + match self { + crate::StaticMutability::Mut(v0) => crate::StaticMutability::Mut(v0.clone()), + crate::StaticMutability::None => crate::StaticMutability::None, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Stmt { + fn clone(&self) -> Self { + match self { + crate::Stmt::Local(v0) => crate::Stmt::Local(v0.clone()), + crate::Stmt::Item(v0) => crate::Stmt::Item(v0.clone()), + crate::Stmt::Expr(v0, v1) => crate::Stmt::Expr(v0.clone(), v1.clone()), + crate::Stmt::Macro(v0) => crate::Stmt::Macro(v0.clone()), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::StmtMacro { + fn clone(&self) -> Self { + crate::StmtMacro { + attrs: self.attrs.clone(), + mac: self.mac.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TraitBound { + fn clone(&self) -> Self { + crate::TraitBound { + paren_token: self.paren_token.clone(), + modifier: self.modifier.clone(), + lifetimes: self.lifetimes.clone(), + path: self.path.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Copy for crate::TraitBoundModifier {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TraitBoundModifier { + fn clone(&self) -> Self { + *self + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TraitItem { + fn clone(&self) -> Self { + match self { + crate::TraitItem::Const(v0) => crate::TraitItem::Const(v0.clone()), + crate::TraitItem::Fn(v0) => crate::TraitItem::Fn(v0.clone()), + crate::TraitItem::Type(v0) => crate::TraitItem::Type(v0.clone()), + crate::TraitItem::Macro(v0) => crate::TraitItem::Macro(v0.clone()), + crate::TraitItem::Verbatim(v0) => crate::TraitItem::Verbatim(v0.clone()), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TraitItemConst { + fn clone(&self) -> Self { + crate::TraitItemConst { + attrs: self.attrs.clone(), + const_token: self.const_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + colon_token: self.colon_token.clone(), + ty: self.ty.clone(), + default: self.default.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TraitItemFn { + fn clone(&self) -> Self { + crate::TraitItemFn { + attrs: self.attrs.clone(), + sig: self.sig.clone(), + default: self.default.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TraitItemMacro { + fn clone(&self) -> Self { + crate::TraitItemMacro { + attrs: self.attrs.clone(), + mac: self.mac.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TraitItemType { + fn clone(&self) -> Self { + crate::TraitItemType { + attrs: self.attrs.clone(), + type_token: self.type_token.clone(), + ident: self.ident.clone(), + generics: self.generics.clone(), + colon_token: self.colon_token.clone(), + bounds: self.bounds.clone(), + default: self.default.clone(), + semi_token: self.semi_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Type { + fn clone(&self) -> Self { + match self { + crate::Type::Array(v0) => crate::Type::Array(v0.clone()), + crate::Type::BareFn(v0) => crate::Type::BareFn(v0.clone()), + crate::Type::Group(v0) => crate::Type::Group(v0.clone()), + crate::Type::ImplTrait(v0) => crate::Type::ImplTrait(v0.clone()), + crate::Type::Infer(v0) => crate::Type::Infer(v0.clone()), + crate::Type::Macro(v0) => crate::Type::Macro(v0.clone()), + crate::Type::Never(v0) => crate::Type::Never(v0.clone()), + crate::Type::Paren(v0) => crate::Type::Paren(v0.clone()), + crate::Type::Path(v0) => crate::Type::Path(v0.clone()), + crate::Type::Ptr(v0) => crate::Type::Ptr(v0.clone()), + crate::Type::Reference(v0) => crate::Type::Reference(v0.clone()), + crate::Type::Slice(v0) => crate::Type::Slice(v0.clone()), + crate::Type::TraitObject(v0) => crate::Type::TraitObject(v0.clone()), + crate::Type::Tuple(v0) => crate::Type::Tuple(v0.clone()), + crate::Type::Verbatim(v0) => crate::Type::Verbatim(v0.clone()), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeArray { + fn clone(&self) -> Self { + crate::TypeArray { + bracket_token: self.bracket_token.clone(), + elem: self.elem.clone(), + semi_token: self.semi_token.clone(), + len: self.len.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeBareFn { + fn clone(&self) -> Self { + crate::TypeBareFn { + lifetimes: self.lifetimes.clone(), + unsafety: self.unsafety.clone(), + abi: self.abi.clone(), + fn_token: self.fn_token.clone(), + paren_token: self.paren_token.clone(), + inputs: self.inputs.clone(), + variadic: self.variadic.clone(), + output: self.output.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeGroup { + fn clone(&self) -> Self { + crate::TypeGroup { + group_token: self.group_token.clone(), + elem: self.elem.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeImplTrait { + fn clone(&self) -> Self { + crate::TypeImplTrait { + impl_token: self.impl_token.clone(), + bounds: self.bounds.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeInfer { + fn clone(&self) -> Self { + crate::TypeInfer { + underscore_token: self.underscore_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeMacro { + fn clone(&self) -> Self { + crate::TypeMacro { + mac: self.mac.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeNever { + fn clone(&self) -> Self { + crate::TypeNever { + bang_token: self.bang_token.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeParam { + fn clone(&self) -> Self { + crate::TypeParam { + attrs: self.attrs.clone(), + ident: self.ident.clone(), + colon_token: self.colon_token.clone(), + bounds: self.bounds.clone(), + eq_token: self.eq_token.clone(), + default: self.default.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeParamBound { + fn clone(&self) -> Self { + match self { + crate::TypeParamBound::Trait(v0) => crate::TypeParamBound::Trait(v0.clone()), + crate::TypeParamBound::Lifetime(v0) => { + crate::TypeParamBound::Lifetime(v0.clone()) + } + #[cfg(feature = "full")] + crate::TypeParamBound::PreciseCapture(v0) => { + crate::TypeParamBound::PreciseCapture(v0.clone()) + } + crate::TypeParamBound::Verbatim(v0) => { + crate::TypeParamBound::Verbatim(v0.clone()) + } + #[cfg(not(feature = "full"))] + _ => unreachable!(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeParen { + fn clone(&self) -> Self { + crate::TypeParen { + paren_token: self.paren_token.clone(), + elem: self.elem.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypePath { + fn clone(&self) -> Self { + crate::TypePath { + qself: self.qself.clone(), + path: self.path.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypePtr { + fn clone(&self) -> Self { + crate::TypePtr { + star_token: self.star_token.clone(), + const_token: self.const_token.clone(), + mutability: self.mutability.clone(), + elem: self.elem.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeReference { + fn clone(&self) -> Self { + crate::TypeReference { + and_token: self.and_token.clone(), + lifetime: self.lifetime.clone(), + mutability: self.mutability.clone(), + elem: self.elem.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeSlice { + fn clone(&self) -> Self { + crate::TypeSlice { + bracket_token: self.bracket_token.clone(), + elem: self.elem.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeTraitObject { + fn clone(&self) -> Self { + crate::TypeTraitObject { + dyn_token: self.dyn_token.clone(), + bounds: self.bounds.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::TypeTuple { + fn clone(&self) -> Self { + crate::TypeTuple { + paren_token: self.paren_token.clone(), + elems: self.elems.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Copy for crate::UnOp {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::UnOp { + fn clone(&self) -> Self { + *self + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::UseGlob { + fn clone(&self) -> Self { + crate::UseGlob { + star_token: self.star_token.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::UseGroup { + fn clone(&self) -> Self { + crate::UseGroup { + brace_token: self.brace_token.clone(), + items: self.items.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::UseName { + fn clone(&self) -> Self { + crate::UseName { + ident: self.ident.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::UsePath { + fn clone(&self) -> Self { + crate::UsePath { + ident: self.ident.clone(), + colon2_token: self.colon2_token.clone(), + tree: self.tree.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::UseRename { + fn clone(&self) -> Self { + crate::UseRename { + ident: self.ident.clone(), + as_token: self.as_token.clone(), + rename: self.rename.clone(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::UseTree { + fn clone(&self) -> Self { + match self { + crate::UseTree::Path(v0) => crate::UseTree::Path(v0.clone()), + crate::UseTree::Name(v0) => crate::UseTree::Name(v0.clone()), + crate::UseTree::Rename(v0) => crate::UseTree::Rename(v0.clone()), + crate::UseTree::Glob(v0) => crate::UseTree::Glob(v0.clone()), + crate::UseTree::Group(v0) => crate::UseTree::Group(v0.clone()), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Variadic { + fn clone(&self) -> Self { + crate::Variadic { + attrs: self.attrs.clone(), + pat: self.pat.clone(), + dots: self.dots.clone(), + comma: self.comma.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Variant { + fn clone(&self) -> Self { + crate::Variant { + attrs: self.attrs.clone(), + ident: self.ident.clone(), + fields: self.fields.clone(), + discriminant: self.discriminant.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::VisRestricted { + fn clone(&self) -> Self { + crate::VisRestricted { + pub_token: self.pub_token.clone(), + paren_token: self.paren_token.clone(), + in_token: self.in_token.clone(), + path: self.path.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::Visibility { + fn clone(&self) -> Self { + match self { + crate::Visibility::Public(v0) => crate::Visibility::Public(v0.clone()), + crate::Visibility::Restricted(v0) => { + crate::Visibility::Restricted(v0.clone()) + } + crate::Visibility::Inherited => crate::Visibility::Inherited, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::WhereClause { + fn clone(&self) -> Self { + crate::WhereClause { + where_token: self.where_token.clone(), + predicates: self.predicates.clone(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for crate::WherePredicate { + fn clone(&self) -> Self { + match self { + crate::WherePredicate::Lifetime(v0) => { + crate::WherePredicate::Lifetime(v0.clone()) + } + crate::WherePredicate::Type(v0) => crate::WherePredicate::Type(v0.clone()), + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/gen/debug.rs b/third_party/crates/syn-2.0.119/src/gen/debug.rs new file mode 100644 index 0000000..bf0784d --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/gen/debug.rs @@ -0,0 +1,3238 @@ +// This file is @generated by syn-internal-codegen. +// It is not intended for manual editing. + +#![allow(unknown_lints, non_local_definitions)] +use core::fmt::{self, Debug}; +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Abi { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Abi"); + formatter.field("extern_token", &self.extern_token); + formatter.field("name", &self.name); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::AngleBracketedGenericArguments { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "AngleBracketedGenericArguments") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::AngleBracketedGenericArguments { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("colon2_token", &self.colon2_token); + formatter.field("lt_token", &self.lt_token); + formatter.field("args", &self.args); + formatter.field("gt_token", &self.gt_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Arm { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Arm"); + formatter.field("attrs", &self.attrs); + formatter.field("pat", &self.pat); + formatter.field("guard", &self.guard); + formatter.field("fat_arrow_token", &self.fat_arrow_token); + formatter.field("body", &self.body); + formatter.field("comma", &self.comma); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::AssocConst { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("AssocConst"); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("eq_token", &self.eq_token); + formatter.field("value", &self.value); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::AssocType { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("AssocType"); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("eq_token", &self.eq_token); + formatter.field("ty", &self.ty); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::AttrStyle { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("AttrStyle::")?; + match self { + crate::AttrStyle::Outer => formatter.write_str("Outer"), + crate::AttrStyle::Inner(v0) => { + let mut formatter = formatter.debug_tuple("Inner"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Attribute { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Attribute"); + formatter.field("pound_token", &self.pound_token); + formatter.field("style", &self.style); + formatter.field("bracket_token", &self.bracket_token); + formatter.field("meta", &self.meta); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::BareFnArg { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("BareFnArg"); + formatter.field("attrs", &self.attrs); + formatter.field("name", &self.name); + formatter.field("ty", &self.ty); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::BareVariadic { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("BareVariadic"); + formatter.field("attrs", &self.attrs); + formatter.field("name", &self.name); + formatter.field("dots", &self.dots); + formatter.field("comma", &self.comma); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::BinOp { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("BinOp::")?; + match self { + crate::BinOp::Add(v0) => { + let mut formatter = formatter.debug_tuple("Add"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Sub(v0) => { + let mut formatter = formatter.debug_tuple("Sub"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Mul(v0) => { + let mut formatter = formatter.debug_tuple("Mul"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Div(v0) => { + let mut formatter = formatter.debug_tuple("Div"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Rem(v0) => { + let mut formatter = formatter.debug_tuple("Rem"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::And(v0) => { + let mut formatter = formatter.debug_tuple("And"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Or(v0) => { + let mut formatter = formatter.debug_tuple("Or"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::BitXor(v0) => { + let mut formatter = formatter.debug_tuple("BitXor"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::BitAnd(v0) => { + let mut formatter = formatter.debug_tuple("BitAnd"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::BitOr(v0) => { + let mut formatter = formatter.debug_tuple("BitOr"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Shl(v0) => { + let mut formatter = formatter.debug_tuple("Shl"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Shr(v0) => { + let mut formatter = formatter.debug_tuple("Shr"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Eq(v0) => { + let mut formatter = formatter.debug_tuple("Eq"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Lt(v0) => { + let mut formatter = formatter.debug_tuple("Lt"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Le(v0) => { + let mut formatter = formatter.debug_tuple("Le"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Ne(v0) => { + let mut formatter = formatter.debug_tuple("Ne"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Ge(v0) => { + let mut formatter = formatter.debug_tuple("Ge"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::Gt(v0) => { + let mut formatter = formatter.debug_tuple("Gt"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::AddAssign(v0) => { + let mut formatter = formatter.debug_tuple("AddAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::SubAssign(v0) => { + let mut formatter = formatter.debug_tuple("SubAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::MulAssign(v0) => { + let mut formatter = formatter.debug_tuple("MulAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::DivAssign(v0) => { + let mut formatter = formatter.debug_tuple("DivAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::RemAssign(v0) => { + let mut formatter = formatter.debug_tuple("RemAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::BitXorAssign(v0) => { + let mut formatter = formatter.debug_tuple("BitXorAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::BitAndAssign(v0) => { + let mut formatter = formatter.debug_tuple("BitAndAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::BitOrAssign(v0) => { + let mut formatter = formatter.debug_tuple("BitOrAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::ShlAssign(v0) => { + let mut formatter = formatter.debug_tuple("ShlAssign"); + formatter.field(v0); + formatter.finish() + } + crate::BinOp::ShrAssign(v0) => { + let mut formatter = formatter.debug_tuple("ShrAssign"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Block { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Block"); + formatter.field("brace_token", &self.brace_token); + formatter.field("stmts", &self.stmts); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::BoundLifetimes { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("BoundLifetimes"); + formatter.field("for_token", &self.for_token); + formatter.field("lt_token", &self.lt_token); + formatter.field("lifetimes", &self.lifetimes); + formatter.field("gt_token", &self.gt_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::CapturedParam { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("CapturedParam::")?; + match self { + crate::CapturedParam::Lifetime(v0) => { + let mut formatter = formatter.debug_tuple("Lifetime"); + formatter.field(v0); + formatter.finish() + } + crate::CapturedParam::Ident(v0) => { + let mut formatter = formatter.debug_tuple("Ident"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ConstParam { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ConstParam"); + formatter.field("attrs", &self.attrs); + formatter.field("const_token", &self.const_token); + formatter.field("ident", &self.ident); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.field("eq_token", &self.eq_token); + formatter.field("default", &self.default); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Constraint { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Constraint"); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("colon_token", &self.colon_token); + formatter.field("bounds", &self.bounds); + formatter.finish() + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Data { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Data::")?; + match self { + crate::Data::Struct(v0) => v0.debug(formatter, "Struct"), + crate::Data::Enum(v0) => v0.debug(formatter, "Enum"), + crate::Data::Union(v0) => v0.debug(formatter, "Union"), + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::DataEnum { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "DataEnum") + } +} +#[cfg(feature = "derive")] +impl crate::DataEnum { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("enum_token", &self.enum_token); + formatter.field("brace_token", &self.brace_token); + formatter.field("variants", &self.variants); + formatter.finish() + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::DataStruct { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "DataStruct") + } +} +#[cfg(feature = "derive")] +impl crate::DataStruct { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("struct_token", &self.struct_token); + formatter.field("fields", &self.fields); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::DataUnion { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "DataUnion") + } +} +#[cfg(feature = "derive")] +impl crate::DataUnion { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("union_token", &self.union_token); + formatter.field("fields", &self.fields); + formatter.finish() + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::DeriveInput { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("DeriveInput"); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("data", &self.data); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Expr { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Expr::")?; + match self { + #[cfg(feature = "full")] + crate::Expr::Array(v0) => v0.debug(formatter, "Array"), + #[cfg(feature = "full")] + crate::Expr::Assign(v0) => v0.debug(formatter, "Assign"), + #[cfg(feature = "full")] + crate::Expr::Async(v0) => v0.debug(formatter, "Async"), + #[cfg(feature = "full")] + crate::Expr::Await(v0) => v0.debug(formatter, "Await"), + crate::Expr::Binary(v0) => v0.debug(formatter, "Binary"), + #[cfg(feature = "full")] + crate::Expr::Block(v0) => v0.debug(formatter, "Block"), + #[cfg(feature = "full")] + crate::Expr::Break(v0) => v0.debug(formatter, "Break"), + crate::Expr::Call(v0) => v0.debug(formatter, "Call"), + crate::Expr::Cast(v0) => v0.debug(formatter, "Cast"), + #[cfg(feature = "full")] + crate::Expr::Closure(v0) => v0.debug(formatter, "Closure"), + #[cfg(feature = "full")] + crate::Expr::Const(v0) => v0.debug(formatter, "Const"), + #[cfg(feature = "full")] + crate::Expr::Continue(v0) => v0.debug(formatter, "Continue"), + crate::Expr::Field(v0) => v0.debug(formatter, "Field"), + #[cfg(feature = "full")] + crate::Expr::ForLoop(v0) => v0.debug(formatter, "ForLoop"), + crate::Expr::Group(v0) => v0.debug(formatter, "Group"), + #[cfg(feature = "full")] + crate::Expr::If(v0) => v0.debug(formatter, "If"), + crate::Expr::Index(v0) => v0.debug(formatter, "Index"), + #[cfg(feature = "full")] + crate::Expr::Infer(v0) => v0.debug(formatter, "Infer"), + #[cfg(feature = "full")] + crate::Expr::Let(v0) => v0.debug(formatter, "Let"), + crate::Expr::Lit(v0) => v0.debug(formatter, "Lit"), + #[cfg(feature = "full")] + crate::Expr::Loop(v0) => v0.debug(formatter, "Loop"), + crate::Expr::Macro(v0) => v0.debug(formatter, "Macro"), + #[cfg(feature = "full")] + crate::Expr::Match(v0) => v0.debug(formatter, "Match"), + crate::Expr::MethodCall(v0) => v0.debug(formatter, "MethodCall"), + crate::Expr::Paren(v0) => v0.debug(formatter, "Paren"), + crate::Expr::Path(v0) => v0.debug(formatter, "Path"), + #[cfg(feature = "full")] + crate::Expr::Range(v0) => v0.debug(formatter, "Range"), + #[cfg(feature = "full")] + crate::Expr::RawAddr(v0) => v0.debug(formatter, "RawAddr"), + crate::Expr::Reference(v0) => v0.debug(formatter, "Reference"), + #[cfg(feature = "full")] + crate::Expr::Repeat(v0) => v0.debug(formatter, "Repeat"), + #[cfg(feature = "full")] + crate::Expr::Return(v0) => v0.debug(formatter, "Return"), + crate::Expr::Struct(v0) => v0.debug(formatter, "Struct"), + #[cfg(feature = "full")] + crate::Expr::Try(v0) => v0.debug(formatter, "Try"), + #[cfg(feature = "full")] + crate::Expr::TryBlock(v0) => v0.debug(formatter, "TryBlock"), + crate::Expr::Tuple(v0) => v0.debug(formatter, "Tuple"), + crate::Expr::Unary(v0) => v0.debug(formatter, "Unary"), + #[cfg(feature = "full")] + crate::Expr::Unsafe(v0) => v0.debug(formatter, "Unsafe"), + crate::Expr::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + #[cfg(feature = "full")] + crate::Expr::While(v0) => v0.debug(formatter, "While"), + #[cfg(feature = "full")] + crate::Expr::Yield(v0) => v0.debug(formatter, "Yield"), + #[cfg(not(feature = "full"))] + _ => unreachable!(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprArray { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprArray") + } +} +#[cfg(feature = "full")] +impl crate::ExprArray { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("bracket_token", &self.bracket_token); + formatter.field("elems", &self.elems); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprAssign { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprAssign") + } +} +#[cfg(feature = "full")] +impl crate::ExprAssign { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("left", &self.left); + formatter.field("eq_token", &self.eq_token); + formatter.field("right", &self.right); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprAsync { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprAsync") + } +} +#[cfg(feature = "full")] +impl crate::ExprAsync { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("async_token", &self.async_token); + formatter.field("capture", &self.capture); + formatter.field("block", &self.block); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprAwait { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprAwait") + } +} +#[cfg(feature = "full")] +impl crate::ExprAwait { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("base", &self.base); + formatter.field("dot_token", &self.dot_token); + formatter.field("await_token", &self.await_token); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprBinary { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprBinary") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprBinary { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("left", &self.left); + formatter.field("op", &self.op); + formatter.field("right", &self.right); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprBlock { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprBlock") + } +} +#[cfg(feature = "full")] +impl crate::ExprBlock { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("label", &self.label); + formatter.field("block", &self.block); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprBreak { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprBreak") + } +} +#[cfg(feature = "full")] +impl crate::ExprBreak { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("break_token", &self.break_token); + formatter.field("label", &self.label); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprCall { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprCall") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprCall { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("func", &self.func); + formatter.field("paren_token", &self.paren_token); + formatter.field("args", &self.args); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprCast { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprCast") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprCast { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("expr", &self.expr); + formatter.field("as_token", &self.as_token); + formatter.field("ty", &self.ty); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprClosure { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprClosure") + } +} +#[cfg(feature = "full")] +impl crate::ExprClosure { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("lifetimes", &self.lifetimes); + formatter.field("constness", &self.constness); + formatter.field("movability", &self.movability); + formatter.field("asyncness", &self.asyncness); + formatter.field("capture", &self.capture); + formatter.field("or1_token", &self.or1_token); + formatter.field("inputs", &self.inputs); + formatter.field("or2_token", &self.or2_token); + formatter.field("output", &self.output); + formatter.field("body", &self.body); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprConst { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprConst") + } +} +#[cfg(feature = "full")] +impl crate::ExprConst { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("const_token", &self.const_token); + formatter.field("block", &self.block); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprContinue { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprContinue") + } +} +#[cfg(feature = "full")] +impl crate::ExprContinue { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("continue_token", &self.continue_token); + formatter.field("label", &self.label); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprField { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprField") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprField { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("base", &self.base); + formatter.field("dot_token", &self.dot_token); + formatter.field("member", &self.member); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprForLoop { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprForLoop") + } +} +#[cfg(feature = "full")] +impl crate::ExprForLoop { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("label", &self.label); + formatter.field("for_token", &self.for_token); + formatter.field("pat", &self.pat); + formatter.field("in_token", &self.in_token); + formatter.field("expr", &self.expr); + formatter.field("body", &self.body); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprGroup { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprGroup") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprGroup { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("group_token", &self.group_token); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprIf { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprIf") + } +} +#[cfg(feature = "full")] +impl crate::ExprIf { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("if_token", &self.if_token); + formatter.field("cond", &self.cond); + formatter.field("then_branch", &self.then_branch); + formatter.field("else_branch", &self.else_branch); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprIndex { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprIndex") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprIndex { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("expr", &self.expr); + formatter.field("bracket_token", &self.bracket_token); + formatter.field("index", &self.index); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprInfer { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprInfer") + } +} +#[cfg(feature = "full")] +impl crate::ExprInfer { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("underscore_token", &self.underscore_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprLet { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprLet") + } +} +#[cfg(feature = "full")] +impl crate::ExprLet { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("let_token", &self.let_token); + formatter.field("pat", &self.pat); + formatter.field("eq_token", &self.eq_token); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprLit { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprLit") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprLit { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("lit", &self.lit); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprLoop { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprLoop") + } +} +#[cfg(feature = "full")] +impl crate::ExprLoop { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("label", &self.label); + formatter.field("loop_token", &self.loop_token); + formatter.field("body", &self.body); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprMacro { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprMacro") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprMacro { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("mac", &self.mac); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprMatch { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprMatch") + } +} +#[cfg(feature = "full")] +impl crate::ExprMatch { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("match_token", &self.match_token); + formatter.field("expr", &self.expr); + formatter.field("brace_token", &self.brace_token); + formatter.field("arms", &self.arms); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprMethodCall { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprMethodCall") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprMethodCall { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("receiver", &self.receiver); + formatter.field("dot_token", &self.dot_token); + formatter.field("method", &self.method); + formatter.field("turbofish", &self.turbofish); + formatter.field("paren_token", &self.paren_token); + formatter.field("args", &self.args); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprParen { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprParen") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprParen { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("paren_token", &self.paren_token); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprPath { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprPath") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprPath { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("qself", &self.qself); + formatter.field("path", &self.path); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprRange { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprRange") + } +} +#[cfg(feature = "full")] +impl crate::ExprRange { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("start", &self.start); + formatter.field("limits", &self.limits); + formatter.field("end", &self.end); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprRawAddr { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprRawAddr") + } +} +#[cfg(feature = "full")] +impl crate::ExprRawAddr { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("and_token", &self.and_token); + formatter.field("raw", &self.raw); + formatter.field("mutability", &self.mutability); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprReference { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprReference") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprReference { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("and_token", &self.and_token); + formatter.field("mutability", &self.mutability); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprRepeat { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprRepeat") + } +} +#[cfg(feature = "full")] +impl crate::ExprRepeat { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("bracket_token", &self.bracket_token); + formatter.field("expr", &self.expr); + formatter.field("semi_token", &self.semi_token); + formatter.field("len", &self.len); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprReturn { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprReturn") + } +} +#[cfg(feature = "full")] +impl crate::ExprReturn { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("return_token", &self.return_token); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprStruct { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprStruct") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprStruct { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("qself", &self.qself); + formatter.field("path", &self.path); + formatter.field("brace_token", &self.brace_token); + formatter.field("fields", &self.fields); + formatter.field("dot2_token", &self.dot2_token); + formatter.field("rest", &self.rest); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprTry { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprTry") + } +} +#[cfg(feature = "full")] +impl crate::ExprTry { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("expr", &self.expr); + formatter.field("question_token", &self.question_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprTryBlock { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprTryBlock") + } +} +#[cfg(feature = "full")] +impl crate::ExprTryBlock { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("try_token", &self.try_token); + formatter.field("block", &self.block); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprTuple { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprTuple") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprTuple { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("paren_token", &self.paren_token); + formatter.field("elems", &self.elems); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprUnary { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprUnary") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ExprUnary { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("op", &self.op); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprUnsafe { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprUnsafe") + } +} +#[cfg(feature = "full")] +impl crate::ExprUnsafe { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("unsafe_token", &self.unsafe_token); + formatter.field("block", &self.block); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprWhile { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprWhile") + } +} +#[cfg(feature = "full")] +impl crate::ExprWhile { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("label", &self.label); + formatter.field("while_token", &self.while_token); + formatter.field("cond", &self.cond); + formatter.field("body", &self.body); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ExprYield { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ExprYield") + } +} +#[cfg(feature = "full")] +impl crate::ExprYield { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("yield_token", &self.yield_token); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Field { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Field"); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("mutability", &self.mutability); + formatter.field("ident", &self.ident); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::FieldMutability { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("FieldMutability::")?; + match self { + crate::FieldMutability::None => formatter.write_str("None"), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::FieldPat { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("FieldPat"); + formatter.field("attrs", &self.attrs); + formatter.field("member", &self.member); + formatter.field("colon_token", &self.colon_token); + formatter.field("pat", &self.pat); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::FieldValue { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("FieldValue"); + formatter.field("attrs", &self.attrs); + formatter.field("member", &self.member); + formatter.field("colon_token", &self.colon_token); + formatter.field("expr", &self.expr); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Fields { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Fields::")?; + match self { + crate::Fields::Named(v0) => v0.debug(formatter, "Named"), + crate::Fields::Unnamed(v0) => v0.debug(formatter, "Unnamed"), + crate::Fields::Unit => formatter.write_str("Unit"), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::FieldsNamed { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "FieldsNamed") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::FieldsNamed { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("brace_token", &self.brace_token); + formatter.field("named", &self.named); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::FieldsUnnamed { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "FieldsUnnamed") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::FieldsUnnamed { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("paren_token", &self.paren_token); + formatter.field("unnamed", &self.unnamed); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::File { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("File"); + formatter.field("shebang", &self.shebang); + formatter.field("attrs", &self.attrs); + formatter.field("items", &self.items); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::FnArg { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("FnArg::")?; + match self { + crate::FnArg::Receiver(v0) => { + let mut formatter = formatter.debug_tuple("Receiver"); + formatter.field(v0); + formatter.finish() + } + crate::FnArg::Typed(v0) => { + let mut formatter = formatter.debug_tuple("Typed"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ForeignItem { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("ForeignItem::")?; + match self { + crate::ForeignItem::Fn(v0) => v0.debug(formatter, "Fn"), + crate::ForeignItem::Static(v0) => v0.debug(formatter, "Static"), + crate::ForeignItem::Type(v0) => v0.debug(formatter, "Type"), + crate::ForeignItem::Macro(v0) => v0.debug(formatter, "Macro"), + crate::ForeignItem::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ForeignItemFn { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ForeignItemFn") + } +} +#[cfg(feature = "full")] +impl crate::ForeignItemFn { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("sig", &self.sig); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ForeignItemMacro { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ForeignItemMacro") + } +} +#[cfg(feature = "full")] +impl crate::ForeignItemMacro { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("mac", &self.mac); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ForeignItemStatic { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ForeignItemStatic") + } +} +#[cfg(feature = "full")] +impl crate::ForeignItemStatic { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("static_token", &self.static_token); + formatter.field("mutability", &self.mutability); + formatter.field("ident", &self.ident); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ForeignItemType { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ForeignItemType") + } +} +#[cfg(feature = "full")] +impl crate::ForeignItemType { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("type_token", &self.type_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::GenericArgument { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("GenericArgument::")?; + match self { + crate::GenericArgument::Lifetime(v0) => { + let mut formatter = formatter.debug_tuple("Lifetime"); + formatter.field(v0); + formatter.finish() + } + crate::GenericArgument::Type(v0) => { + let mut formatter = formatter.debug_tuple("Type"); + formatter.field(v0); + formatter.finish() + } + crate::GenericArgument::Const(v0) => { + let mut formatter = formatter.debug_tuple("Const"); + formatter.field(v0); + formatter.finish() + } + crate::GenericArgument::AssocType(v0) => { + let mut formatter = formatter.debug_tuple("AssocType"); + formatter.field(v0); + formatter.finish() + } + crate::GenericArgument::AssocConst(v0) => { + let mut formatter = formatter.debug_tuple("AssocConst"); + formatter.field(v0); + formatter.finish() + } + crate::GenericArgument::Constraint(v0) => { + let mut formatter = formatter.debug_tuple("Constraint"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::GenericParam { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("GenericParam::")?; + match self { + crate::GenericParam::Lifetime(v0) => { + let mut formatter = formatter.debug_tuple("Lifetime"); + formatter.field(v0); + formatter.finish() + } + crate::GenericParam::Type(v0) => { + let mut formatter = formatter.debug_tuple("Type"); + formatter.field(v0); + formatter.finish() + } + crate::GenericParam::Const(v0) => { + let mut formatter = formatter.debug_tuple("Const"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Generics { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Generics"); + formatter.field("lt_token", &self.lt_token); + formatter.field("params", &self.params); + formatter.field("gt_token", &self.gt_token); + formatter.field("where_clause", &self.where_clause); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ImplItem { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("ImplItem::")?; + match self { + crate::ImplItem::Const(v0) => v0.debug(formatter, "Const"), + crate::ImplItem::Fn(v0) => v0.debug(formatter, "Fn"), + crate::ImplItem::Type(v0) => v0.debug(formatter, "Type"), + crate::ImplItem::Macro(v0) => v0.debug(formatter, "Macro"), + crate::ImplItem::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ImplItemConst { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ImplItemConst") + } +} +#[cfg(feature = "full")] +impl crate::ImplItemConst { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("defaultness", &self.defaultness); + formatter.field("const_token", &self.const_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.field("eq_token", &self.eq_token); + formatter.field("expr", &self.expr); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ImplItemFn { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ImplItemFn") + } +} +#[cfg(feature = "full")] +impl crate::ImplItemFn { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("defaultness", &self.defaultness); + formatter.field("sig", &self.sig); + formatter.field("block", &self.block); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ImplItemMacro { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ImplItemMacro") + } +} +#[cfg(feature = "full")] +impl crate::ImplItemMacro { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("mac", &self.mac); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ImplItemType { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ImplItemType") + } +} +#[cfg(feature = "full")] +impl crate::ImplItemType { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("defaultness", &self.defaultness); + formatter.field("type_token", &self.type_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("eq_token", &self.eq_token); + formatter.field("ty", &self.ty); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ImplRestriction { + fn fmt(&self, _formatter: &mut fmt::Formatter) -> fmt::Result { + match *self {} + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Index { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Index"); + formatter.field("index", &self.index); + formatter.field("span", &self.span); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Item { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Item::")?; + match self { + crate::Item::Const(v0) => v0.debug(formatter, "Const"), + crate::Item::Enum(v0) => v0.debug(formatter, "Enum"), + crate::Item::ExternCrate(v0) => v0.debug(formatter, "ExternCrate"), + crate::Item::Fn(v0) => v0.debug(formatter, "Fn"), + crate::Item::ForeignMod(v0) => v0.debug(formatter, "ForeignMod"), + crate::Item::Impl(v0) => v0.debug(formatter, "Impl"), + crate::Item::Macro(v0) => v0.debug(formatter, "Macro"), + crate::Item::Mod(v0) => v0.debug(formatter, "Mod"), + crate::Item::Static(v0) => v0.debug(formatter, "Static"), + crate::Item::Struct(v0) => v0.debug(formatter, "Struct"), + crate::Item::Trait(v0) => v0.debug(formatter, "Trait"), + crate::Item::TraitAlias(v0) => v0.debug(formatter, "TraitAlias"), + crate::Item::Type(v0) => v0.debug(formatter, "Type"), + crate::Item::Union(v0) => v0.debug(formatter, "Union"), + crate::Item::Use(v0) => v0.debug(formatter, "Use"), + crate::Item::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemConst { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemConst") + } +} +#[cfg(feature = "full")] +impl crate::ItemConst { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("const_token", &self.const_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.field("eq_token", &self.eq_token); + formatter.field("expr", &self.expr); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemEnum { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemEnum") + } +} +#[cfg(feature = "full")] +impl crate::ItemEnum { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("enum_token", &self.enum_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("brace_token", &self.brace_token); + formatter.field("variants", &self.variants); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemExternCrate { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemExternCrate") + } +} +#[cfg(feature = "full")] +impl crate::ItemExternCrate { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("extern_token", &self.extern_token); + formatter.field("crate_token", &self.crate_token); + formatter.field("ident", &self.ident); + formatter.field("rename", &self.rename); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemFn { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemFn") + } +} +#[cfg(feature = "full")] +impl crate::ItemFn { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("sig", &self.sig); + formatter.field("block", &self.block); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemForeignMod { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemForeignMod") + } +} +#[cfg(feature = "full")] +impl crate::ItemForeignMod { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("unsafety", &self.unsafety); + formatter.field("abi", &self.abi); + formatter.field("brace_token", &self.brace_token); + formatter.field("items", &self.items); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemImpl { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemImpl") + } +} +#[cfg(feature = "full")] +impl crate::ItemImpl { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("defaultness", &self.defaultness); + formatter.field("unsafety", &self.unsafety); + formatter.field("impl_token", &self.impl_token); + formatter.field("generics", &self.generics); + formatter.field("trait_", &self.trait_); + formatter.field("self_ty", &self.self_ty); + formatter.field("brace_token", &self.brace_token); + formatter.field("items", &self.items); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemMacro { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemMacro") + } +} +#[cfg(feature = "full")] +impl crate::ItemMacro { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("ident", &self.ident); + formatter.field("mac", &self.mac); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemMod { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemMod") + } +} +#[cfg(feature = "full")] +impl crate::ItemMod { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("unsafety", &self.unsafety); + formatter.field("mod_token", &self.mod_token); + formatter.field("ident", &self.ident); + formatter.field("content", &self.content); + formatter.field("semi", &self.semi); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemStatic { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemStatic") + } +} +#[cfg(feature = "full")] +impl crate::ItemStatic { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("static_token", &self.static_token); + formatter.field("mutability", &self.mutability); + formatter.field("ident", &self.ident); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.field("eq_token", &self.eq_token); + formatter.field("expr", &self.expr); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemStruct { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemStruct") + } +} +#[cfg(feature = "full")] +impl crate::ItemStruct { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("struct_token", &self.struct_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("fields", &self.fields); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemTrait { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemTrait") + } +} +#[cfg(feature = "full")] +impl crate::ItemTrait { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("unsafety", &self.unsafety); + formatter.field("auto_token", &self.auto_token); + formatter.field("restriction", &self.restriction); + formatter.field("trait_token", &self.trait_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("colon_token", &self.colon_token); + formatter.field("supertraits", &self.supertraits); + formatter.field("brace_token", &self.brace_token); + formatter.field("items", &self.items); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemTraitAlias { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemTraitAlias") + } +} +#[cfg(feature = "full")] +impl crate::ItemTraitAlias { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("trait_token", &self.trait_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("eq_token", &self.eq_token); + formatter.field("bounds", &self.bounds); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemType { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemType") + } +} +#[cfg(feature = "full")] +impl crate::ItemType { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("type_token", &self.type_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("eq_token", &self.eq_token); + formatter.field("ty", &self.ty); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemUnion { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemUnion") + } +} +#[cfg(feature = "full")] +impl crate::ItemUnion { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("union_token", &self.union_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("fields", &self.fields); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ItemUse { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ItemUse") + } +} +#[cfg(feature = "full")] +impl crate::ItemUse { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("vis", &self.vis); + formatter.field("use_token", &self.use_token); + formatter.field("leading_colon", &self.leading_colon); + formatter.field("tree", &self.tree); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Label { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Label"); + formatter.field("name", &self.name); + formatter.field("colon_token", &self.colon_token); + formatter.finish() + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Lifetime { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "Lifetime") + } +} +impl crate::Lifetime { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("apostrophe", &self.apostrophe); + formatter.field("ident", &self.ident); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::LifetimeParam { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("LifetimeParam"); + formatter.field("attrs", &self.attrs); + formatter.field("lifetime", &self.lifetime); + formatter.field("colon_token", &self.colon_token); + formatter.field("bounds", &self.bounds); + formatter.finish() + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Lit { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Lit::")?; + match self { + crate::Lit::Str(v0) => v0.debug(formatter, "Str"), + crate::Lit::ByteStr(v0) => v0.debug(formatter, "ByteStr"), + crate::Lit::CStr(v0) => v0.debug(formatter, "CStr"), + crate::Lit::Byte(v0) => v0.debug(formatter, "Byte"), + crate::Lit::Char(v0) => v0.debug(formatter, "Char"), + crate::Lit::Int(v0) => v0.debug(formatter, "Int"), + crate::Lit::Float(v0) => v0.debug(formatter, "Float"), + crate::Lit::Bool(v0) => v0.debug(formatter, "Bool"), + crate::Lit::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Local { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "Local") + } +} +#[cfg(feature = "full")] +impl crate::Local { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("let_token", &self.let_token); + formatter.field("pat", &self.pat); + formatter.field("init", &self.init); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::LocalInit { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("LocalInit"); + formatter.field("eq_token", &self.eq_token); + formatter.field("expr", &self.expr); + formatter.field("diverge", &self.diverge); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Macro { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Macro"); + formatter.field("path", &self.path); + formatter.field("bang_token", &self.bang_token); + formatter.field("delimiter", &self.delimiter); + formatter.field("tokens", &self.tokens); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::MacroDelimiter { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("MacroDelimiter::")?; + match self { + crate::MacroDelimiter::Paren(v0) => { + let mut formatter = formatter.debug_tuple("Paren"); + formatter.field(v0); + formatter.finish() + } + crate::MacroDelimiter::Brace(v0) => { + let mut formatter = formatter.debug_tuple("Brace"); + formatter.field(v0); + formatter.finish() + } + crate::MacroDelimiter::Bracket(v0) => { + let mut formatter = formatter.debug_tuple("Bracket"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Member { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Member::")?; + match self { + crate::Member::Named(v0) => { + let mut formatter = formatter.debug_tuple("Named"); + formatter.field(v0); + formatter.finish() + } + crate::Member::Unnamed(v0) => { + let mut formatter = formatter.debug_tuple("Unnamed"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Meta { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Meta::")?; + match self { + crate::Meta::Path(v0) => v0.debug(formatter, "Path"), + crate::Meta::List(v0) => v0.debug(formatter, "List"), + crate::Meta::NameValue(v0) => v0.debug(formatter, "NameValue"), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::MetaList { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "MetaList") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::MetaList { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("path", &self.path); + formatter.field("delimiter", &self.delimiter); + formatter.field("tokens", &self.tokens); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::MetaNameValue { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "MetaNameValue") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::MetaNameValue { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("path", &self.path); + formatter.field("eq_token", &self.eq_token); + formatter.field("value", &self.value); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ParenthesizedGenericArguments { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "ParenthesizedGenericArguments") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::ParenthesizedGenericArguments { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("paren_token", &self.paren_token); + formatter.field("inputs", &self.inputs); + formatter.field("output", &self.output); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Pat { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Pat::")?; + match self { + crate::Pat::Const(v0) => v0.debug(formatter, "Const"), + crate::Pat::Ident(v0) => v0.debug(formatter, "Ident"), + crate::Pat::Lit(v0) => v0.debug(formatter, "Lit"), + crate::Pat::Macro(v0) => v0.debug(formatter, "Macro"), + crate::Pat::Or(v0) => v0.debug(formatter, "Or"), + crate::Pat::Paren(v0) => v0.debug(formatter, "Paren"), + crate::Pat::Path(v0) => v0.debug(formatter, "Path"), + crate::Pat::Range(v0) => v0.debug(formatter, "Range"), + crate::Pat::Reference(v0) => v0.debug(formatter, "Reference"), + crate::Pat::Rest(v0) => v0.debug(formatter, "Rest"), + crate::Pat::Slice(v0) => v0.debug(formatter, "Slice"), + crate::Pat::Struct(v0) => v0.debug(formatter, "Struct"), + crate::Pat::Tuple(v0) => v0.debug(formatter, "Tuple"), + crate::Pat::TupleStruct(v0) => v0.debug(formatter, "TupleStruct"), + crate::Pat::Type(v0) => v0.debug(formatter, "Type"), + crate::Pat::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + crate::Pat::Wild(v0) => v0.debug(formatter, "Wild"), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatIdent { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatIdent") + } +} +#[cfg(feature = "full")] +impl crate::PatIdent { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("by_ref", &self.by_ref); + formatter.field("mutability", &self.mutability); + formatter.field("ident", &self.ident); + formatter.field("subpat", &self.subpat); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatOr { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatOr") + } +} +#[cfg(feature = "full")] +impl crate::PatOr { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("leading_vert", &self.leading_vert); + formatter.field("cases", &self.cases); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatParen { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatParen") + } +} +#[cfg(feature = "full")] +impl crate::PatParen { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("paren_token", &self.paren_token); + formatter.field("pat", &self.pat); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatReference { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatReference") + } +} +#[cfg(feature = "full")] +impl crate::PatReference { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("and_token", &self.and_token); + formatter.field("mutability", &self.mutability); + formatter.field("pat", &self.pat); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatRest { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatRest") + } +} +#[cfg(feature = "full")] +impl crate::PatRest { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("dot2_token", &self.dot2_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatSlice { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatSlice") + } +} +#[cfg(feature = "full")] +impl crate::PatSlice { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("bracket_token", &self.bracket_token); + formatter.field("elems", &self.elems); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatStruct { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatStruct") + } +} +#[cfg(feature = "full")] +impl crate::PatStruct { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("qself", &self.qself); + formatter.field("path", &self.path); + formatter.field("brace_token", &self.brace_token); + formatter.field("fields", &self.fields); + formatter.field("rest", &self.rest); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatTuple { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatTuple") + } +} +#[cfg(feature = "full")] +impl crate::PatTuple { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("paren_token", &self.paren_token); + formatter.field("elems", &self.elems); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatTupleStruct { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatTupleStruct") + } +} +#[cfg(feature = "full")] +impl crate::PatTupleStruct { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("qself", &self.qself); + formatter.field("path", &self.path); + formatter.field("paren_token", &self.paren_token); + formatter.field("elems", &self.elems); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatType { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatType") + } +} +#[cfg(feature = "full")] +impl crate::PatType { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("pat", &self.pat); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PatWild { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "PatWild") + } +} +#[cfg(feature = "full")] +impl crate::PatWild { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("underscore_token", &self.underscore_token); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Path { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "Path") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::Path { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("leading_colon", &self.leading_colon); + formatter.field("segments", &self.segments); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PathArguments { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("PathArguments::")?; + match self { + crate::PathArguments::None => formatter.write_str("None"), + crate::PathArguments::AngleBracketed(v0) => { + v0.debug(formatter, "AngleBracketed") + } + crate::PathArguments::Parenthesized(v0) => { + v0.debug(formatter, "Parenthesized") + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PathSegment { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PathSegment"); + formatter.field("ident", &self.ident); + formatter.field("arguments", &self.arguments); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PointerMutability { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("PointerMutability::")?; + match self { + crate::PointerMutability::Const(v0) => { + let mut formatter = formatter.debug_tuple("Const"); + formatter.field(v0); + formatter.finish() + } + crate::PointerMutability::Mut(v0) => { + let mut formatter = formatter.debug_tuple("Mut"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PreciseCapture { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PreciseCapture"); + formatter.field("use_token", &self.use_token); + formatter.field("lt_token", &self.lt_token); + formatter.field("params", &self.params); + formatter.field("gt_token", &self.gt_token); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PredicateLifetime { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PredicateLifetime"); + formatter.field("lifetime", &self.lifetime); + formatter.field("colon_token", &self.colon_token); + formatter.field("bounds", &self.bounds); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::PredicateType { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PredicateType"); + formatter.field("lifetimes", &self.lifetimes); + formatter.field("bounded_ty", &self.bounded_ty); + formatter.field("colon_token", &self.colon_token); + formatter.field("bounds", &self.bounds); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::QSelf { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("QSelf"); + formatter.field("lt_token", &self.lt_token); + formatter.field("ty", &self.ty); + formatter.field("position", &self.position); + formatter.field("as_token", &self.as_token); + formatter.field("gt_token", &self.gt_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::RangeLimits { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("RangeLimits::")?; + match self { + crate::RangeLimits::HalfOpen(v0) => { + let mut formatter = formatter.debug_tuple("HalfOpen"); + formatter.field(v0); + formatter.finish() + } + crate::RangeLimits::Closed(v0) => { + let mut formatter = formatter.debug_tuple("Closed"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Receiver { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Receiver"); + formatter.field("attrs", &self.attrs); + formatter.field("reference", &self.reference); + formatter.field("mutability", &self.mutability); + formatter.field("self_token", &self.self_token); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::ReturnType { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("ReturnType::")?; + match self { + crate::ReturnType::Default => formatter.write_str("Default"), + crate::ReturnType::Type(v0, v1) => { + let mut formatter = formatter.debug_tuple("Type"); + formatter.field(v0); + formatter.field(v1); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Signature { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Signature"); + formatter.field("constness", &self.constness); + formatter.field("asyncness", &self.asyncness); + formatter.field("unsafety", &self.unsafety); + formatter.field("abi", &self.abi); + formatter.field("fn_token", &self.fn_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("paren_token", &self.paren_token); + formatter.field("inputs", &self.inputs); + formatter.field("variadic", &self.variadic); + formatter.field("output", &self.output); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::StaticMutability { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("StaticMutability::")?; + match self { + crate::StaticMutability::Mut(v0) => { + let mut formatter = formatter.debug_tuple("Mut"); + formatter.field(v0); + formatter.finish() + } + crate::StaticMutability::None => formatter.write_str("None"), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Stmt { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Stmt::")?; + match self { + crate::Stmt::Local(v0) => v0.debug(formatter, "Local"), + crate::Stmt::Item(v0) => { + let mut formatter = formatter.debug_tuple("Item"); + formatter.field(v0); + formatter.finish() + } + crate::Stmt::Expr(v0, v1) => { + let mut formatter = formatter.debug_tuple("Expr"); + formatter.field(v0); + formatter.field(v1); + formatter.finish() + } + crate::Stmt::Macro(v0) => v0.debug(formatter, "Macro"), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::StmtMacro { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "StmtMacro") + } +} +#[cfg(feature = "full")] +impl crate::StmtMacro { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("mac", &self.mac); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TraitBound { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TraitBound"); + formatter.field("paren_token", &self.paren_token); + formatter.field("modifier", &self.modifier); + formatter.field("lifetimes", &self.lifetimes); + formatter.field("path", &self.path); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TraitBoundModifier { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("TraitBoundModifier::")?; + match self { + crate::TraitBoundModifier::None => formatter.write_str("None"), + crate::TraitBoundModifier::Maybe(v0) => { + let mut formatter = formatter.debug_tuple("Maybe"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TraitItem { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("TraitItem::")?; + match self { + crate::TraitItem::Const(v0) => v0.debug(formatter, "Const"), + crate::TraitItem::Fn(v0) => v0.debug(formatter, "Fn"), + crate::TraitItem::Type(v0) => v0.debug(formatter, "Type"), + crate::TraitItem::Macro(v0) => v0.debug(formatter, "Macro"), + crate::TraitItem::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TraitItemConst { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TraitItemConst") + } +} +#[cfg(feature = "full")] +impl crate::TraitItemConst { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("const_token", &self.const_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("colon_token", &self.colon_token); + formatter.field("ty", &self.ty); + formatter.field("default", &self.default); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TraitItemFn { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TraitItemFn") + } +} +#[cfg(feature = "full")] +impl crate::TraitItemFn { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("sig", &self.sig); + formatter.field("default", &self.default); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TraitItemMacro { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TraitItemMacro") + } +} +#[cfg(feature = "full")] +impl crate::TraitItemMacro { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("mac", &self.mac); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TraitItemType { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TraitItemType") + } +} +#[cfg(feature = "full")] +impl crate::TraitItemType { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("attrs", &self.attrs); + formatter.field("type_token", &self.type_token); + formatter.field("ident", &self.ident); + formatter.field("generics", &self.generics); + formatter.field("colon_token", &self.colon_token); + formatter.field("bounds", &self.bounds); + formatter.field("default", &self.default); + formatter.field("semi_token", &self.semi_token); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Type { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Type::")?; + match self { + crate::Type::Array(v0) => v0.debug(formatter, "Array"), + crate::Type::BareFn(v0) => v0.debug(formatter, "BareFn"), + crate::Type::Group(v0) => v0.debug(formatter, "Group"), + crate::Type::ImplTrait(v0) => v0.debug(formatter, "ImplTrait"), + crate::Type::Infer(v0) => v0.debug(formatter, "Infer"), + crate::Type::Macro(v0) => v0.debug(formatter, "Macro"), + crate::Type::Never(v0) => v0.debug(formatter, "Never"), + crate::Type::Paren(v0) => v0.debug(formatter, "Paren"), + crate::Type::Path(v0) => v0.debug(formatter, "Path"), + crate::Type::Ptr(v0) => v0.debug(formatter, "Ptr"), + crate::Type::Reference(v0) => v0.debug(formatter, "Reference"), + crate::Type::Slice(v0) => v0.debug(formatter, "Slice"), + crate::Type::TraitObject(v0) => v0.debug(formatter, "TraitObject"), + crate::Type::Tuple(v0) => v0.debug(formatter, "Tuple"), + crate::Type::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeArray { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeArray") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeArray { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("bracket_token", &self.bracket_token); + formatter.field("elem", &self.elem); + formatter.field("semi_token", &self.semi_token); + formatter.field("len", &self.len); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeBareFn { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeBareFn") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeBareFn { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("lifetimes", &self.lifetimes); + formatter.field("unsafety", &self.unsafety); + formatter.field("abi", &self.abi); + formatter.field("fn_token", &self.fn_token); + formatter.field("paren_token", &self.paren_token); + formatter.field("inputs", &self.inputs); + formatter.field("variadic", &self.variadic); + formatter.field("output", &self.output); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeGroup { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeGroup") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeGroup { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("group_token", &self.group_token); + formatter.field("elem", &self.elem); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeImplTrait { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeImplTrait") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeImplTrait { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("impl_token", &self.impl_token); + formatter.field("bounds", &self.bounds); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeInfer { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeInfer") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeInfer { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("underscore_token", &self.underscore_token); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeMacro { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeMacro") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeMacro { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("mac", &self.mac); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeNever { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeNever") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeNever { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("bang_token", &self.bang_token); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeParam { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeParam"); + formatter.field("attrs", &self.attrs); + formatter.field("ident", &self.ident); + formatter.field("colon_token", &self.colon_token); + formatter.field("bounds", &self.bounds); + formatter.field("eq_token", &self.eq_token); + formatter.field("default", &self.default); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeParamBound { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("TypeParamBound::")?; + match self { + crate::TypeParamBound::Trait(v0) => { + let mut formatter = formatter.debug_tuple("Trait"); + formatter.field(v0); + formatter.finish() + } + crate::TypeParamBound::Lifetime(v0) => v0.debug(formatter, "Lifetime"), + #[cfg(feature = "full")] + crate::TypeParamBound::PreciseCapture(v0) => { + let mut formatter = formatter.debug_tuple("PreciseCapture"); + formatter.field(v0); + formatter.finish() + } + crate::TypeParamBound::Verbatim(v0) => { + let mut formatter = formatter.debug_tuple("Verbatim"); + formatter.field(v0); + formatter.finish() + } + #[cfg(not(feature = "full"))] + _ => unreachable!(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeParen { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeParen") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeParen { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("paren_token", &self.paren_token); + formatter.field("elem", &self.elem); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypePath { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypePath") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypePath { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("qself", &self.qself); + formatter.field("path", &self.path); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypePtr { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypePtr") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypePtr { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("star_token", &self.star_token); + formatter.field("const_token", &self.const_token); + formatter.field("mutability", &self.mutability); + formatter.field("elem", &self.elem); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeReference { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeReference") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeReference { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("and_token", &self.and_token); + formatter.field("lifetime", &self.lifetime); + formatter.field("mutability", &self.mutability); + formatter.field("elem", &self.elem); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeSlice { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeSlice") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeSlice { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("bracket_token", &self.bracket_token); + formatter.field("elem", &self.elem); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeTraitObject { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeTraitObject") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeTraitObject { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("dyn_token", &self.dyn_token); + formatter.field("bounds", &self.bounds); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::TypeTuple { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "TypeTuple") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::TypeTuple { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("paren_token", &self.paren_token); + formatter.field("elems", &self.elems); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::UnOp { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("UnOp::")?; + match self { + crate::UnOp::Deref(v0) => { + let mut formatter = formatter.debug_tuple("Deref"); + formatter.field(v0); + formatter.finish() + } + crate::UnOp::Not(v0) => { + let mut formatter = formatter.debug_tuple("Not"); + formatter.field(v0); + formatter.finish() + } + crate::UnOp::Neg(v0) => { + let mut formatter = formatter.debug_tuple("Neg"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::UseGlob { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UseGlob"); + formatter.field("star_token", &self.star_token); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::UseGroup { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UseGroup"); + formatter.field("brace_token", &self.brace_token); + formatter.field("items", &self.items); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::UseName { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UseName"); + formatter.field("ident", &self.ident); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::UsePath { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UsePath"); + formatter.field("ident", &self.ident); + formatter.field("colon2_token", &self.colon2_token); + formatter.field("tree", &self.tree); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::UseRename { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UseRename"); + formatter.field("ident", &self.ident); + formatter.field("as_token", &self.as_token); + formatter.field("rename", &self.rename); + formatter.finish() + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::UseTree { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("UseTree::")?; + match self { + crate::UseTree::Path(v0) => { + let mut formatter = formatter.debug_tuple("Path"); + formatter.field(v0); + formatter.finish() + } + crate::UseTree::Name(v0) => { + let mut formatter = formatter.debug_tuple("Name"); + formatter.field(v0); + formatter.finish() + } + crate::UseTree::Rename(v0) => { + let mut formatter = formatter.debug_tuple("Rename"); + formatter.field(v0); + formatter.finish() + } + crate::UseTree::Glob(v0) => { + let mut formatter = formatter.debug_tuple("Glob"); + formatter.field(v0); + formatter.finish() + } + crate::UseTree::Group(v0) => { + let mut formatter = formatter.debug_tuple("Group"); + formatter.field(v0); + formatter.finish() + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Variadic { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Variadic"); + formatter.field("attrs", &self.attrs); + formatter.field("pat", &self.pat); + formatter.field("dots", &self.dots); + formatter.field("comma", &self.comma); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Variant { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Variant"); + formatter.field("attrs", &self.attrs); + formatter.field("ident", &self.ident); + formatter.field("fields", &self.fields); + formatter.field("discriminant", &self.discriminant); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::VisRestricted { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "VisRestricted") + } +} +#[cfg(any(feature = "derive", feature = "full"))] +impl crate::VisRestricted { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + let mut formatter = formatter.debug_struct(name); + formatter.field("pub_token", &self.pub_token); + formatter.field("paren_token", &self.paren_token); + formatter.field("in_token", &self.in_token); + formatter.field("path", &self.path); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::Visibility { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Visibility::")?; + match self { + crate::Visibility::Public(v0) => { + let mut formatter = formatter.debug_tuple("Public"); + formatter.field(v0); + formatter.finish() + } + crate::Visibility::Restricted(v0) => v0.debug(formatter, "Restricted"), + crate::Visibility::Inherited => formatter.write_str("Inherited"), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::WhereClause { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("WhereClause"); + formatter.field("where_token", &self.where_token); + formatter.field("predicates", &self.predicates); + formatter.finish() + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for crate::WherePredicate { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("WherePredicate::")?; + match self { + crate::WherePredicate::Lifetime(v0) => { + let mut formatter = formatter.debug_tuple("Lifetime"); + formatter.field(v0); + formatter.finish() + } + crate::WherePredicate::Type(v0) => { + let mut formatter = formatter.debug_tuple("Type"); + formatter.field(v0); + formatter.finish() + } + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/gen/eq.rs b/third_party/crates/syn-2.0.119/src/gen/eq.rs new file mode 100644 index 0000000..5ba1572 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/gen/eq.rs @@ -0,0 +1,2308 @@ +// This file is @generated by syn-internal-codegen. +// It is not intended for manual editing. + +#[cfg(any(feature = "derive", feature = "full"))] +use crate::tt::TokenStreamHelper; +#[cfg(feature = "extra-traits")] +use alloc::string::ToString; +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Abi {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Abi { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::AngleBracketedGenericArguments {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::AngleBracketedGenericArguments { + fn eq(&self, other: &Self) -> bool { + self.colon2_token == other.colon2_token && self.args == other.args + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Arm {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Arm { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.pat == other.pat && self.guard == other.guard + && self.body == other.body && self.comma == other.comma + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::AssocConst {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::AssocConst { + fn eq(&self, other: &Self) -> bool { + self.ident == other.ident && self.generics == other.generics + && self.value == other.value + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::AssocType {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::AssocType { + fn eq(&self, other: &Self) -> bool { + self.ident == other.ident && self.generics == other.generics + && self.ty == other.ty + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::AttrStyle {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::AttrStyle { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::AttrStyle::Outer, crate::AttrStyle::Outer) => true, + (crate::AttrStyle::Inner(_), crate::AttrStyle::Inner(_)) => true, + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Attribute {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Attribute { + fn eq(&self, other: &Self) -> bool { + self.style == other.style && self.meta == other.meta + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::BareFnArg {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::BareFnArg { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.name == other.name && self.ty == other.ty + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::BareVariadic {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::BareVariadic { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.name == other.name && self.comma == other.comma + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::BinOp {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::BinOp { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::BinOp::Add(_), crate::BinOp::Add(_)) => true, + (crate::BinOp::Sub(_), crate::BinOp::Sub(_)) => true, + (crate::BinOp::Mul(_), crate::BinOp::Mul(_)) => true, + (crate::BinOp::Div(_), crate::BinOp::Div(_)) => true, + (crate::BinOp::Rem(_), crate::BinOp::Rem(_)) => true, + (crate::BinOp::And(_), crate::BinOp::And(_)) => true, + (crate::BinOp::Or(_), crate::BinOp::Or(_)) => true, + (crate::BinOp::BitXor(_), crate::BinOp::BitXor(_)) => true, + (crate::BinOp::BitAnd(_), crate::BinOp::BitAnd(_)) => true, + (crate::BinOp::BitOr(_), crate::BinOp::BitOr(_)) => true, + (crate::BinOp::Shl(_), crate::BinOp::Shl(_)) => true, + (crate::BinOp::Shr(_), crate::BinOp::Shr(_)) => true, + (crate::BinOp::Eq(_), crate::BinOp::Eq(_)) => true, + (crate::BinOp::Lt(_), crate::BinOp::Lt(_)) => true, + (crate::BinOp::Le(_), crate::BinOp::Le(_)) => true, + (crate::BinOp::Ne(_), crate::BinOp::Ne(_)) => true, + (crate::BinOp::Ge(_), crate::BinOp::Ge(_)) => true, + (crate::BinOp::Gt(_), crate::BinOp::Gt(_)) => true, + (crate::BinOp::AddAssign(_), crate::BinOp::AddAssign(_)) => true, + (crate::BinOp::SubAssign(_), crate::BinOp::SubAssign(_)) => true, + (crate::BinOp::MulAssign(_), crate::BinOp::MulAssign(_)) => true, + (crate::BinOp::DivAssign(_), crate::BinOp::DivAssign(_)) => true, + (crate::BinOp::RemAssign(_), crate::BinOp::RemAssign(_)) => true, + (crate::BinOp::BitXorAssign(_), crate::BinOp::BitXorAssign(_)) => true, + (crate::BinOp::BitAndAssign(_), crate::BinOp::BitAndAssign(_)) => true, + (crate::BinOp::BitOrAssign(_), crate::BinOp::BitOrAssign(_)) => true, + (crate::BinOp::ShlAssign(_), crate::BinOp::ShlAssign(_)) => true, + (crate::BinOp::ShrAssign(_), crate::BinOp::ShrAssign(_)) => true, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Block {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Block { + fn eq(&self, other: &Self) -> bool { + self.stmts == other.stmts + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::BoundLifetimes {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::BoundLifetimes { + fn eq(&self, other: &Self) -> bool { + self.lifetimes == other.lifetimes + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::CapturedParam {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::CapturedParam { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + crate::CapturedParam::Lifetime(self0), + crate::CapturedParam::Lifetime(other0), + ) => self0 == other0, + (crate::CapturedParam::Ident(self0), crate::CapturedParam::Ident(other0)) => { + self0 == other0 + } + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ConstParam {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ConstParam { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.ident == other.ident && self.ty == other.ty + && self.eq_token == other.eq_token && self.default == other.default + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Constraint {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Constraint { + fn eq(&self, other: &Self) -> bool { + self.ident == other.ident && self.generics == other.generics + && self.bounds == other.bounds + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Data {} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Data { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Data::Struct(self0), crate::Data::Struct(other0)) => self0 == other0, + (crate::Data::Enum(self0), crate::Data::Enum(other0)) => self0 == other0, + (crate::Data::Union(self0), crate::Data::Union(other0)) => self0 == other0, + _ => false, + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::DataEnum {} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::DataEnum { + fn eq(&self, other: &Self) -> bool { + self.variants == other.variants + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::DataStruct {} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::DataStruct { + fn eq(&self, other: &Self) -> bool { + self.fields == other.fields && self.semi_token == other.semi_token + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::DataUnion {} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::DataUnion { + fn eq(&self, other: &Self) -> bool { + self.fields == other.fields + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::DeriveInput {} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::DeriveInput { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.generics == other.generics && self.data == other.data + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Expr {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Expr { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + #[cfg(feature = "full")] + (crate::Expr::Array(self0), crate::Expr::Array(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Assign(self0), crate::Expr::Assign(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Async(self0), crate::Expr::Async(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Await(self0), crate::Expr::Await(other0)) => self0 == other0, + (crate::Expr::Binary(self0), crate::Expr::Binary(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Block(self0), crate::Expr::Block(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Break(self0), crate::Expr::Break(other0)) => self0 == other0, + (crate::Expr::Call(self0), crate::Expr::Call(other0)) => self0 == other0, + (crate::Expr::Cast(self0), crate::Expr::Cast(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Closure(self0), crate::Expr::Closure(other0)) => { + self0 == other0 + } + #[cfg(feature = "full")] + (crate::Expr::Const(self0), crate::Expr::Const(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Continue(self0), crate::Expr::Continue(other0)) => { + self0 == other0 + } + (crate::Expr::Field(self0), crate::Expr::Field(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::ForLoop(self0), crate::Expr::ForLoop(other0)) => { + self0 == other0 + } + (crate::Expr::Group(self0), crate::Expr::Group(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::If(self0), crate::Expr::If(other0)) => self0 == other0, + (crate::Expr::Index(self0), crate::Expr::Index(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Infer(self0), crate::Expr::Infer(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Let(self0), crate::Expr::Let(other0)) => self0 == other0, + (crate::Expr::Lit(self0), crate::Expr::Lit(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Loop(self0), crate::Expr::Loop(other0)) => self0 == other0, + (crate::Expr::Macro(self0), crate::Expr::Macro(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Match(self0), crate::Expr::Match(other0)) => self0 == other0, + (crate::Expr::MethodCall(self0), crate::Expr::MethodCall(other0)) => { + self0 == other0 + } + (crate::Expr::Paren(self0), crate::Expr::Paren(other0)) => self0 == other0, + (crate::Expr::Path(self0), crate::Expr::Path(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Range(self0), crate::Expr::Range(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::RawAddr(self0), crate::Expr::RawAddr(other0)) => { + self0 == other0 + } + (crate::Expr::Reference(self0), crate::Expr::Reference(other0)) => { + self0 == other0 + } + #[cfg(feature = "full")] + (crate::Expr::Repeat(self0), crate::Expr::Repeat(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Return(self0), crate::Expr::Return(other0)) => self0 == other0, + (crate::Expr::Struct(self0), crate::Expr::Struct(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Try(self0), crate::Expr::Try(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::TryBlock(self0), crate::Expr::TryBlock(other0)) => { + self0 == other0 + } + (crate::Expr::Tuple(self0), crate::Expr::Tuple(other0)) => self0 == other0, + (crate::Expr::Unary(self0), crate::Expr::Unary(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Unsafe(self0), crate::Expr::Unsafe(other0)) => self0 == other0, + (crate::Expr::Verbatim(self0), crate::Expr::Verbatim(other0)) => { + TokenStreamHelper(self0) == TokenStreamHelper(other0) + } + #[cfg(feature = "full")] + (crate::Expr::While(self0), crate::Expr::While(other0)) => self0 == other0, + #[cfg(feature = "full")] + (crate::Expr::Yield(self0), crate::Expr::Yield(other0)) => self0 == other0, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprArray {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprArray { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.elems == other.elems + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprAssign {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprAssign { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.left == other.left && self.right == other.right + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprAsync {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprAsync { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.capture == other.capture + && self.block == other.block + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprAwait {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprAwait { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.base == other.base + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprBinary {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprBinary { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.left == other.left && self.op == other.op + && self.right == other.right + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprBlock {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprBlock { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.label == other.label + && self.block == other.block + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprBreak {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprBreak { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.label == other.label && self.expr == other.expr + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprCall {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprCall { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.func == other.func && self.args == other.args + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprCast {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprCast { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr && self.ty == other.ty + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprClosure {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprClosure { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.lifetimes == other.lifetimes + && self.constness == other.constness && self.movability == other.movability + && self.asyncness == other.asyncness && self.capture == other.capture + && self.inputs == other.inputs && self.output == other.output + && self.body == other.body + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprConst {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprConst { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.block == other.block + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprContinue {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprContinue { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.label == other.label + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprField {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprField { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.base == other.base + && self.member == other.member + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprForLoop {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprForLoop { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.label == other.label && self.pat == other.pat + && self.expr == other.expr && self.body == other.body + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprGroup {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprGroup { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprIf {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprIf { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.cond == other.cond + && self.then_branch == other.then_branch + && self.else_branch == other.else_branch + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprIndex {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprIndex { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr && self.index == other.index + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprInfer {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprInfer { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprLet {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprLet { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.pat == other.pat && self.expr == other.expr + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprLit {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprLit { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.lit == other.lit + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprLoop {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprLoop { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.label == other.label && self.body == other.body + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprMacro {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprMacro { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.mac == other.mac + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprMatch {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprMatch { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr && self.arms == other.arms + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprMethodCall {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprMethodCall { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.receiver == other.receiver + && self.method == other.method && self.turbofish == other.turbofish + && self.args == other.args + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprParen {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprParen { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprPath {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprPath { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.qself == other.qself && self.path == other.path + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprRange {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprRange { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.start == other.start + && self.limits == other.limits && self.end == other.end + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprRawAddr {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprRawAddr { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.mutability == other.mutability + && self.expr == other.expr + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprReference {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprReference { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.mutability == other.mutability + && self.expr == other.expr + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprRepeat {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprRepeat { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr && self.len == other.len + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprReturn {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprReturn { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprStruct {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprStruct { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.qself == other.qself && self.path == other.path + && self.fields == other.fields && self.dot2_token == other.dot2_token + && self.rest == other.rest + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprTry {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprTry { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprTryBlock {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprTryBlock { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.block == other.block + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprTuple {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprTuple { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.elems == other.elems + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprUnary {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprUnary { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.op == other.op && self.expr == other.expr + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprUnsafe {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprUnsafe { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.block == other.block + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprWhile {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprWhile { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.label == other.label && self.cond == other.cond + && self.body == other.body + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ExprYield {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ExprYield { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.expr == other.expr + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Field {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Field { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.mutability == other.mutability && self.ident == other.ident + && self.colon_token == other.colon_token && self.ty == other.ty + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::FieldMutability {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::FieldMutability { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::FieldMutability::None, crate::FieldMutability::None) => true, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::FieldPat {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::FieldPat { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.member == other.member + && self.colon_token == other.colon_token && self.pat == other.pat + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::FieldValue {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::FieldValue { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.member == other.member + && self.colon_token == other.colon_token && self.expr == other.expr + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Fields {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Fields { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Fields::Named(self0), crate::Fields::Named(other0)) => { + self0 == other0 + } + (crate::Fields::Unnamed(self0), crate::Fields::Unnamed(other0)) => { + self0 == other0 + } + (crate::Fields::Unit, crate::Fields::Unit) => true, + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::FieldsNamed {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::FieldsNamed { + fn eq(&self, other: &Self) -> bool { + self.named == other.named + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::FieldsUnnamed {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::FieldsUnnamed { + fn eq(&self, other: &Self) -> bool { + self.unnamed == other.unnamed + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::File {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::File { + fn eq(&self, other: &Self) -> bool { + self.shebang == other.shebang && self.attrs == other.attrs + && self.items == other.items + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::FnArg {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::FnArg { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::FnArg::Receiver(self0), crate::FnArg::Receiver(other0)) => { + self0 == other0 + } + (crate::FnArg::Typed(self0), crate::FnArg::Typed(other0)) => self0 == other0, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ForeignItem {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ForeignItem { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::ForeignItem::Fn(self0), crate::ForeignItem::Fn(other0)) => { + self0 == other0 + } + (crate::ForeignItem::Static(self0), crate::ForeignItem::Static(other0)) => { + self0 == other0 + } + (crate::ForeignItem::Type(self0), crate::ForeignItem::Type(other0)) => { + self0 == other0 + } + (crate::ForeignItem::Macro(self0), crate::ForeignItem::Macro(other0)) => { + self0 == other0 + } + ( + crate::ForeignItem::Verbatim(self0), + crate::ForeignItem::Verbatim(other0), + ) => TokenStreamHelper(self0) == TokenStreamHelper(other0), + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ForeignItemFn {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ForeignItemFn { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.sig == other.sig + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ForeignItemMacro {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ForeignItemMacro { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.mac == other.mac + && self.semi_token == other.semi_token + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ForeignItemStatic {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ForeignItemStatic { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.mutability == other.mutability && self.ident == other.ident + && self.ty == other.ty + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ForeignItemType {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ForeignItemType { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.generics == other.generics + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::GenericArgument {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::GenericArgument { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + crate::GenericArgument::Lifetime(self0), + crate::GenericArgument::Lifetime(other0), + ) => self0 == other0, + ( + crate::GenericArgument::Type(self0), + crate::GenericArgument::Type(other0), + ) => self0 == other0, + ( + crate::GenericArgument::Const(self0), + crate::GenericArgument::Const(other0), + ) => self0 == other0, + ( + crate::GenericArgument::AssocType(self0), + crate::GenericArgument::AssocType(other0), + ) => self0 == other0, + ( + crate::GenericArgument::AssocConst(self0), + crate::GenericArgument::AssocConst(other0), + ) => self0 == other0, + ( + crate::GenericArgument::Constraint(self0), + crate::GenericArgument::Constraint(other0), + ) => self0 == other0, + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::GenericParam {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::GenericParam { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + crate::GenericParam::Lifetime(self0), + crate::GenericParam::Lifetime(other0), + ) => self0 == other0, + (crate::GenericParam::Type(self0), crate::GenericParam::Type(other0)) => { + self0 == other0 + } + (crate::GenericParam::Const(self0), crate::GenericParam::Const(other0)) => { + self0 == other0 + } + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Generics {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Generics { + fn eq(&self, other: &Self) -> bool { + self.lt_token == other.lt_token && self.params == other.params + && self.gt_token == other.gt_token && self.where_clause == other.where_clause + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ImplItem {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ImplItem { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::ImplItem::Const(self0), crate::ImplItem::Const(other0)) => { + self0 == other0 + } + (crate::ImplItem::Fn(self0), crate::ImplItem::Fn(other0)) => self0 == other0, + (crate::ImplItem::Type(self0), crate::ImplItem::Type(other0)) => { + self0 == other0 + } + (crate::ImplItem::Macro(self0), crate::ImplItem::Macro(other0)) => { + self0 == other0 + } + (crate::ImplItem::Verbatim(self0), crate::ImplItem::Verbatim(other0)) => { + TokenStreamHelper(self0) == TokenStreamHelper(other0) + } + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ImplItemConst {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ImplItemConst { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.defaultness == other.defaultness && self.ident == other.ident + && self.generics == other.generics && self.ty == other.ty + && self.expr == other.expr + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ImplItemFn {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ImplItemFn { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.defaultness == other.defaultness && self.sig == other.sig + && self.block == other.block + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ImplItemMacro {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ImplItemMacro { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.mac == other.mac + && self.semi_token == other.semi_token + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ImplItemType {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ImplItemType { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.defaultness == other.defaultness && self.ident == other.ident + && self.generics == other.generics && self.ty == other.ty + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ImplRestriction {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ImplRestriction { + fn eq(&self, _other: &Self) -> bool { + match *self {} + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Item {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Item { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Item::Const(self0), crate::Item::Const(other0)) => self0 == other0, + (crate::Item::Enum(self0), crate::Item::Enum(other0)) => self0 == other0, + (crate::Item::ExternCrate(self0), crate::Item::ExternCrate(other0)) => { + self0 == other0 + } + (crate::Item::Fn(self0), crate::Item::Fn(other0)) => self0 == other0, + (crate::Item::ForeignMod(self0), crate::Item::ForeignMod(other0)) => { + self0 == other0 + } + (crate::Item::Impl(self0), crate::Item::Impl(other0)) => self0 == other0, + (crate::Item::Macro(self0), crate::Item::Macro(other0)) => self0 == other0, + (crate::Item::Mod(self0), crate::Item::Mod(other0)) => self0 == other0, + (crate::Item::Static(self0), crate::Item::Static(other0)) => self0 == other0, + (crate::Item::Struct(self0), crate::Item::Struct(other0)) => self0 == other0, + (crate::Item::Trait(self0), crate::Item::Trait(other0)) => self0 == other0, + (crate::Item::TraitAlias(self0), crate::Item::TraitAlias(other0)) => { + self0 == other0 + } + (crate::Item::Type(self0), crate::Item::Type(other0)) => self0 == other0, + (crate::Item::Union(self0), crate::Item::Union(other0)) => self0 == other0, + (crate::Item::Use(self0), crate::Item::Use(other0)) => self0 == other0, + (crate::Item::Verbatim(self0), crate::Item::Verbatim(other0)) => { + TokenStreamHelper(self0) == TokenStreamHelper(other0) + } + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemConst {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemConst { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.generics == other.generics && self.ty == other.ty + && self.expr == other.expr + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemEnum {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemEnum { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.generics == other.generics && self.variants == other.variants + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemExternCrate {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemExternCrate { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.rename == other.rename + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemFn {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemFn { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.sig == other.sig + && self.block == other.block + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemForeignMod {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemForeignMod { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.unsafety == other.unsafety + && self.abi == other.abi && self.items == other.items + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemImpl {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemImpl { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.defaultness == other.defaultness + && self.unsafety == other.unsafety && self.generics == other.generics + && self.trait_ == other.trait_ && self.self_ty == other.self_ty + && self.items == other.items + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemMacro {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemMacro { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.ident == other.ident && self.mac == other.mac + && self.semi_token == other.semi_token + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemMod {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemMod { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.unsafety == other.unsafety && self.ident == other.ident + && self.content == other.content && self.semi == other.semi + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemStatic {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemStatic { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.mutability == other.mutability && self.ident == other.ident + && self.ty == other.ty && self.expr == other.expr + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemStruct {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemStruct { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.generics == other.generics && self.fields == other.fields + && self.semi_token == other.semi_token + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemTrait {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemTrait { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.unsafety == other.unsafety && self.auto_token == other.auto_token + && self.restriction == other.restriction && self.ident == other.ident + && self.generics == other.generics && self.colon_token == other.colon_token + && self.supertraits == other.supertraits && self.items == other.items + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemTraitAlias {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemTraitAlias { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.generics == other.generics && self.bounds == other.bounds + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemType {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemType { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.generics == other.generics && self.ty == other.ty + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemUnion {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemUnion { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis && self.ident == other.ident + && self.generics == other.generics && self.fields == other.fields + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ItemUse {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ItemUse { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.vis == other.vis + && self.leading_colon == other.leading_colon && self.tree == other.tree + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Label {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Label { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LifetimeParam {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::LifetimeParam { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.lifetime == other.lifetime + && self.colon_token == other.colon_token && self.bounds == other.bounds + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Lit {} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Lit { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Lit::Str(self0), crate::Lit::Str(other0)) => self0 == other0, + (crate::Lit::ByteStr(self0), crate::Lit::ByteStr(other0)) => self0 == other0, + (crate::Lit::CStr(self0), crate::Lit::CStr(other0)) => self0 == other0, + (crate::Lit::Byte(self0), crate::Lit::Byte(other0)) => self0 == other0, + (crate::Lit::Char(self0), crate::Lit::Char(other0)) => self0 == other0, + (crate::Lit::Int(self0), crate::Lit::Int(other0)) => self0 == other0, + (crate::Lit::Float(self0), crate::Lit::Float(other0)) => self0 == other0, + (crate::Lit::Bool(self0), crate::Lit::Bool(other0)) => self0 == other0, + (crate::Lit::Verbatim(self0), crate::Lit::Verbatim(other0)) => { + self0.to_string() == other0.to_string() + } + _ => false, + } + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LitBool {} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::LitBool { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LitByte {} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LitByteStr {} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LitCStr {} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LitChar {} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LitFloat {} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LitInt {} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LitStr {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Local {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Local { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.pat == other.pat && self.init == other.init + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::LocalInit {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::LocalInit { + fn eq(&self, other: &Self) -> bool { + self.expr == other.expr && self.diverge == other.diverge + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Macro {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Macro { + fn eq(&self, other: &Self) -> bool { + self.path == other.path && self.delimiter == other.delimiter + && TokenStreamHelper(&self.tokens) == TokenStreamHelper(&other.tokens) + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::MacroDelimiter {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::MacroDelimiter { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::MacroDelimiter::Paren(_), crate::MacroDelimiter::Paren(_)) => true, + (crate::MacroDelimiter::Brace(_), crate::MacroDelimiter::Brace(_)) => true, + (crate::MacroDelimiter::Bracket(_), crate::MacroDelimiter::Bracket(_)) => { + true + } + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Meta {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Meta { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Meta::Path(self0), crate::Meta::Path(other0)) => self0 == other0, + (crate::Meta::List(self0), crate::Meta::List(other0)) => self0 == other0, + (crate::Meta::NameValue(self0), crate::Meta::NameValue(other0)) => { + self0 == other0 + } + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::MetaList {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::MetaList { + fn eq(&self, other: &Self) -> bool { + self.path == other.path && self.delimiter == other.delimiter + && TokenStreamHelper(&self.tokens) == TokenStreamHelper(&other.tokens) + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::MetaNameValue {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::MetaNameValue { + fn eq(&self, other: &Self) -> bool { + self.path == other.path && self.value == other.value + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ParenthesizedGenericArguments {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ParenthesizedGenericArguments { + fn eq(&self, other: &Self) -> bool { + self.inputs == other.inputs && self.output == other.output + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Pat {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Pat { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Pat::Const(self0), crate::Pat::Const(other0)) => self0 == other0, + (crate::Pat::Ident(self0), crate::Pat::Ident(other0)) => self0 == other0, + (crate::Pat::Lit(self0), crate::Pat::Lit(other0)) => self0 == other0, + (crate::Pat::Macro(self0), crate::Pat::Macro(other0)) => self0 == other0, + (crate::Pat::Or(self0), crate::Pat::Or(other0)) => self0 == other0, + (crate::Pat::Paren(self0), crate::Pat::Paren(other0)) => self0 == other0, + (crate::Pat::Path(self0), crate::Pat::Path(other0)) => self0 == other0, + (crate::Pat::Range(self0), crate::Pat::Range(other0)) => self0 == other0, + (crate::Pat::Reference(self0), crate::Pat::Reference(other0)) => { + self0 == other0 + } + (crate::Pat::Rest(self0), crate::Pat::Rest(other0)) => self0 == other0, + (crate::Pat::Slice(self0), crate::Pat::Slice(other0)) => self0 == other0, + (crate::Pat::Struct(self0), crate::Pat::Struct(other0)) => self0 == other0, + (crate::Pat::Tuple(self0), crate::Pat::Tuple(other0)) => self0 == other0, + (crate::Pat::TupleStruct(self0), crate::Pat::TupleStruct(other0)) => { + self0 == other0 + } + (crate::Pat::Type(self0), crate::Pat::Type(other0)) => self0 == other0, + (crate::Pat::Verbatim(self0), crate::Pat::Verbatim(other0)) => { + TokenStreamHelper(self0) == TokenStreamHelper(other0) + } + (crate::Pat::Wild(self0), crate::Pat::Wild(other0)) => self0 == other0, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatIdent {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatIdent { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.by_ref == other.by_ref + && self.mutability == other.mutability && self.ident == other.ident + && self.subpat == other.subpat + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatOr {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatOr { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.leading_vert == other.leading_vert + && self.cases == other.cases + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatParen {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatParen { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.pat == other.pat + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatReference {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatReference { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.mutability == other.mutability + && self.pat == other.pat + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatRest {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatRest { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatSlice {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatSlice { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.elems == other.elems + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatStruct {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatStruct { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.qself == other.qself && self.path == other.path + && self.fields == other.fields && self.rest == other.rest + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatTuple {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatTuple { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.elems == other.elems + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatTupleStruct {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatTupleStruct { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.qself == other.qself && self.path == other.path + && self.elems == other.elems + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatType {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatType { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.pat == other.pat && self.ty == other.ty + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PatWild {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PatWild { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Path {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Path { + fn eq(&self, other: &Self) -> bool { + self.leading_colon == other.leading_colon && self.segments == other.segments + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PathArguments {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PathArguments { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::PathArguments::None, crate::PathArguments::None) => true, + ( + crate::PathArguments::AngleBracketed(self0), + crate::PathArguments::AngleBracketed(other0), + ) => self0 == other0, + ( + crate::PathArguments::Parenthesized(self0), + crate::PathArguments::Parenthesized(other0), + ) => self0 == other0, + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PathSegment {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PathSegment { + fn eq(&self, other: &Self) -> bool { + self.ident == other.ident && self.arguments == other.arguments + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PointerMutability {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PointerMutability { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::PointerMutability::Const(_), crate::PointerMutability::Const(_)) => { + true + } + (crate::PointerMutability::Mut(_), crate::PointerMutability::Mut(_)) => true, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PreciseCapture {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PreciseCapture { + fn eq(&self, other: &Self) -> bool { + self.params == other.params + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PredicateLifetime {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PredicateLifetime { + fn eq(&self, other: &Self) -> bool { + self.lifetime == other.lifetime && self.bounds == other.bounds + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::PredicateType {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::PredicateType { + fn eq(&self, other: &Self) -> bool { + self.lifetimes == other.lifetimes && self.bounded_ty == other.bounded_ty + && self.bounds == other.bounds + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::QSelf {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::QSelf { + fn eq(&self, other: &Self) -> bool { + self.ty == other.ty && self.position == other.position + && self.as_token == other.as_token + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::RangeLimits {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::RangeLimits { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::RangeLimits::HalfOpen(_), crate::RangeLimits::HalfOpen(_)) => true, + (crate::RangeLimits::Closed(_), crate::RangeLimits::Closed(_)) => true, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Receiver {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Receiver { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.reference == other.reference + && self.mutability == other.mutability + && self.colon_token == other.colon_token && self.ty == other.ty + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::ReturnType {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::ReturnType { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::ReturnType::Default, crate::ReturnType::Default) => true, + (crate::ReturnType::Type(_, self1), crate::ReturnType::Type(_, other1)) => { + self1 == other1 + } + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Signature {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Signature { + fn eq(&self, other: &Self) -> bool { + self.constness == other.constness && self.asyncness == other.asyncness + && self.unsafety == other.unsafety && self.abi == other.abi + && self.ident == other.ident && self.generics == other.generics + && self.inputs == other.inputs && self.variadic == other.variadic + && self.output == other.output + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::StaticMutability {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::StaticMutability { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::StaticMutability::Mut(_), crate::StaticMutability::Mut(_)) => true, + (crate::StaticMutability::None, crate::StaticMutability::None) => true, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Stmt {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Stmt { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Stmt::Local(self0), crate::Stmt::Local(other0)) => self0 == other0, + (crate::Stmt::Item(self0), crate::Stmt::Item(other0)) => self0 == other0, + (crate::Stmt::Expr(self0, self1), crate::Stmt::Expr(other0, other1)) => { + self0 == other0 && self1 == other1 + } + (crate::Stmt::Macro(self0), crate::Stmt::Macro(other0)) => self0 == other0, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::StmtMacro {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::StmtMacro { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.mac == other.mac + && self.semi_token == other.semi_token + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TraitBound {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TraitBound { + fn eq(&self, other: &Self) -> bool { + self.paren_token == other.paren_token && self.modifier == other.modifier + && self.lifetimes == other.lifetimes && self.path == other.path + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TraitBoundModifier {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TraitBoundModifier { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::TraitBoundModifier::None, crate::TraitBoundModifier::None) => true, + ( + crate::TraitBoundModifier::Maybe(_), + crate::TraitBoundModifier::Maybe(_), + ) => true, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TraitItem {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TraitItem { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::TraitItem::Const(self0), crate::TraitItem::Const(other0)) => { + self0 == other0 + } + (crate::TraitItem::Fn(self0), crate::TraitItem::Fn(other0)) => { + self0 == other0 + } + (crate::TraitItem::Type(self0), crate::TraitItem::Type(other0)) => { + self0 == other0 + } + (crate::TraitItem::Macro(self0), crate::TraitItem::Macro(other0)) => { + self0 == other0 + } + (crate::TraitItem::Verbatim(self0), crate::TraitItem::Verbatim(other0)) => { + TokenStreamHelper(self0) == TokenStreamHelper(other0) + } + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TraitItemConst {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TraitItemConst { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.ident == other.ident + && self.generics == other.generics && self.ty == other.ty + && self.default == other.default + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TraitItemFn {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TraitItemFn { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.sig == other.sig + && self.default == other.default && self.semi_token == other.semi_token + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TraitItemMacro {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TraitItemMacro { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.mac == other.mac + && self.semi_token == other.semi_token + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TraitItemType {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TraitItemType { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.ident == other.ident + && self.generics == other.generics && self.colon_token == other.colon_token + && self.bounds == other.bounds && self.default == other.default + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Type {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Type { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Type::Array(self0), crate::Type::Array(other0)) => self0 == other0, + (crate::Type::BareFn(self0), crate::Type::BareFn(other0)) => self0 == other0, + (crate::Type::Group(self0), crate::Type::Group(other0)) => self0 == other0, + (crate::Type::ImplTrait(self0), crate::Type::ImplTrait(other0)) => { + self0 == other0 + } + (crate::Type::Infer(self0), crate::Type::Infer(other0)) => self0 == other0, + (crate::Type::Macro(self0), crate::Type::Macro(other0)) => self0 == other0, + (crate::Type::Never(self0), crate::Type::Never(other0)) => self0 == other0, + (crate::Type::Paren(self0), crate::Type::Paren(other0)) => self0 == other0, + (crate::Type::Path(self0), crate::Type::Path(other0)) => self0 == other0, + (crate::Type::Ptr(self0), crate::Type::Ptr(other0)) => self0 == other0, + (crate::Type::Reference(self0), crate::Type::Reference(other0)) => { + self0 == other0 + } + (crate::Type::Slice(self0), crate::Type::Slice(other0)) => self0 == other0, + (crate::Type::TraitObject(self0), crate::Type::TraitObject(other0)) => { + self0 == other0 + } + (crate::Type::Tuple(self0), crate::Type::Tuple(other0)) => self0 == other0, + (crate::Type::Verbatim(self0), crate::Type::Verbatim(other0)) => { + TokenStreamHelper(self0) == TokenStreamHelper(other0) + } + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeArray {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeArray { + fn eq(&self, other: &Self) -> bool { + self.elem == other.elem && self.len == other.len + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeBareFn {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeBareFn { + fn eq(&self, other: &Self) -> bool { + self.lifetimes == other.lifetimes && self.unsafety == other.unsafety + && self.abi == other.abi && self.inputs == other.inputs + && self.variadic == other.variadic && self.output == other.output + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeGroup {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeGroup { + fn eq(&self, other: &Self) -> bool { + self.elem == other.elem + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeImplTrait {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeImplTrait { + fn eq(&self, other: &Self) -> bool { + self.bounds == other.bounds + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeInfer {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeInfer { + fn eq(&self, _other: &Self) -> bool { + true + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeMacro {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeMacro { + fn eq(&self, other: &Self) -> bool { + self.mac == other.mac + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeNever {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeNever { + fn eq(&self, _other: &Self) -> bool { + true + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeParam {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeParam { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.ident == other.ident + && self.colon_token == other.colon_token && self.bounds == other.bounds + && self.eq_token == other.eq_token && self.default == other.default + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeParamBound {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeParamBound { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + crate::TypeParamBound::Trait(self0), + crate::TypeParamBound::Trait(other0), + ) => self0 == other0, + ( + crate::TypeParamBound::Lifetime(self0), + crate::TypeParamBound::Lifetime(other0), + ) => self0 == other0, + #[cfg(feature = "full")] + ( + crate::TypeParamBound::PreciseCapture(self0), + crate::TypeParamBound::PreciseCapture(other0), + ) => self0 == other0, + ( + crate::TypeParamBound::Verbatim(self0), + crate::TypeParamBound::Verbatim(other0), + ) => TokenStreamHelper(self0) == TokenStreamHelper(other0), + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeParen {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeParen { + fn eq(&self, other: &Self) -> bool { + self.elem == other.elem + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypePath {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypePath { + fn eq(&self, other: &Self) -> bool { + self.qself == other.qself && self.path == other.path + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypePtr {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypePtr { + fn eq(&self, other: &Self) -> bool { + self.const_token == other.const_token && self.mutability == other.mutability + && self.elem == other.elem + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeReference {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeReference { + fn eq(&self, other: &Self) -> bool { + self.lifetime == other.lifetime && self.mutability == other.mutability + && self.elem == other.elem + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeSlice {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeSlice { + fn eq(&self, other: &Self) -> bool { + self.elem == other.elem + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeTraitObject {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeTraitObject { + fn eq(&self, other: &Self) -> bool { + self.dyn_token == other.dyn_token && self.bounds == other.bounds + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::TypeTuple {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::TypeTuple { + fn eq(&self, other: &Self) -> bool { + self.elems == other.elems + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::UnOp {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::UnOp { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::UnOp::Deref(_), crate::UnOp::Deref(_)) => true, + (crate::UnOp::Not(_), crate::UnOp::Not(_)) => true, + (crate::UnOp::Neg(_), crate::UnOp::Neg(_)) => true, + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::UseGlob {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::UseGlob { + fn eq(&self, _other: &Self) -> bool { + true + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::UseGroup {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::UseGroup { + fn eq(&self, other: &Self) -> bool { + self.items == other.items + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::UseName {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::UseName { + fn eq(&self, other: &Self) -> bool { + self.ident == other.ident + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::UsePath {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::UsePath { + fn eq(&self, other: &Self) -> bool { + self.ident == other.ident && self.tree == other.tree + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::UseRename {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::UseRename { + fn eq(&self, other: &Self) -> bool { + self.ident == other.ident && self.rename == other.rename + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::UseTree {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::UseTree { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::UseTree::Path(self0), crate::UseTree::Path(other0)) => { + self0 == other0 + } + (crate::UseTree::Name(self0), crate::UseTree::Name(other0)) => { + self0 == other0 + } + (crate::UseTree::Rename(self0), crate::UseTree::Rename(other0)) => { + self0 == other0 + } + (crate::UseTree::Glob(self0), crate::UseTree::Glob(other0)) => { + self0 == other0 + } + (crate::UseTree::Group(self0), crate::UseTree::Group(other0)) => { + self0 == other0 + } + _ => false, + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Variadic {} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Variadic { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.pat == other.pat && self.comma == other.comma + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Variant {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Variant { + fn eq(&self, other: &Self) -> bool { + self.attrs == other.attrs && self.ident == other.ident + && self.fields == other.fields && self.discriminant == other.discriminant + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::VisRestricted {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::VisRestricted { + fn eq(&self, other: &Self) -> bool { + self.in_token == other.in_token && self.path == other.path + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::Visibility {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::Visibility { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (crate::Visibility::Public(_), crate::Visibility::Public(_)) => true, + ( + crate::Visibility::Restricted(self0), + crate::Visibility::Restricted(other0), + ) => self0 == other0, + (crate::Visibility::Inherited, crate::Visibility::Inherited) => true, + _ => false, + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::WhereClause {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::WhereClause { + fn eq(&self, other: &Self) -> bool { + self.predicates == other.predicates + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for crate::WherePredicate {} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for crate::WherePredicate { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + ( + crate::WherePredicate::Lifetime(self0), + crate::WherePredicate::Lifetime(other0), + ) => self0 == other0, + (crate::WherePredicate::Type(self0), crate::WherePredicate::Type(other0)) => { + self0 == other0 + } + _ => false, + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/gen/fold.rs b/third_party/crates/syn-2.0.119/src/gen/fold.rs new file mode 100644 index 0000000..b71421f --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/gen/fold.rs @@ -0,0 +1,3906 @@ +// This file is @generated by syn-internal-codegen. +// It is not intended for manual editing. + +#![allow(unreachable_code, unused_variables)] +#![allow( + clippy::match_wildcard_for_single_variants, + clippy::needless_match, + clippy::needless_pass_by_ref_mut, +)] +#[cfg(any(feature = "derive", feature = "full"))] +use alloc::boxed::Box; +#[cfg(any(feature = "derive", feature = "full"))] +use alloc::vec::Vec; +#[cfg(feature = "full")] +macro_rules! full { + ($e:expr) => { + $e + }; +} +#[cfg(all(feature = "derive", not(feature = "full")))] +macro_rules! full { + ($e:expr) => { + unreachable!() + }; +} +/// Syntax tree traversal to transform the nodes of an owned syntax tree. +/// +/// See the [module documentation] for details. +/// +/// [module documentation]: self +pub trait Fold { + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_abi(&mut self, i: crate::Abi) -> crate::Abi { + fold_abi(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_angle_bracketed_generic_arguments( + &mut self, + i: crate::AngleBracketedGenericArguments, + ) -> crate::AngleBracketedGenericArguments { + fold_angle_bracketed_generic_arguments(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_arm(&mut self, i: crate::Arm) -> crate::Arm { + fold_arm(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_assoc_const(&mut self, i: crate::AssocConst) -> crate::AssocConst { + fold_assoc_const(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_assoc_type(&mut self, i: crate::AssocType) -> crate::AssocType { + fold_assoc_type(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_attr_style(&mut self, i: crate::AttrStyle) -> crate::AttrStyle { + fold_attr_style(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_attribute(&mut self, i: crate::Attribute) -> crate::Attribute { + fold_attribute(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_attributes(&mut self, i: Vec<crate::Attribute>) -> Vec<crate::Attribute> { + fold_vec(i, self, Self::fold_attribute) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_bare_fn_arg(&mut self, i: crate::BareFnArg) -> crate::BareFnArg { + fold_bare_fn_arg(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_bare_variadic(&mut self, i: crate::BareVariadic) -> crate::BareVariadic { + fold_bare_variadic(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_bin_op(&mut self, i: crate::BinOp) -> crate::BinOp { + fold_bin_op(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_block(&mut self, i: crate::Block) -> crate::Block { + fold_block(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_bound_lifetimes( + &mut self, + i: crate::BoundLifetimes, + ) -> crate::BoundLifetimes { + fold_bound_lifetimes(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_captured_param(&mut self, i: crate::CapturedParam) -> crate::CapturedParam { + fold_captured_param(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_const_param(&mut self, i: crate::ConstParam) -> crate::ConstParam { + fold_const_param(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_constraint(&mut self, i: crate::Constraint) -> crate::Constraint { + fold_constraint(self, i) + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn fold_data(&mut self, i: crate::Data) -> crate::Data { + fold_data(self, i) + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn fold_data_enum(&mut self, i: crate::DataEnum) -> crate::DataEnum { + fold_data_enum(self, i) + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn fold_data_struct(&mut self, i: crate::DataStruct) -> crate::DataStruct { + fold_data_struct(self, i) + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn fold_data_union(&mut self, i: crate::DataUnion) -> crate::DataUnion { + fold_data_union(self, i) + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn fold_derive_input(&mut self, i: crate::DeriveInput) -> crate::DeriveInput { + fold_derive_input(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr(&mut self, i: crate::Expr) -> crate::Expr { + fold_expr(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_array(&mut self, i: crate::ExprArray) -> crate::ExprArray { + fold_expr_array(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_assign(&mut self, i: crate::ExprAssign) -> crate::ExprAssign { + fold_expr_assign(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_async(&mut self, i: crate::ExprAsync) -> crate::ExprAsync { + fold_expr_async(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_await(&mut self, i: crate::ExprAwait) -> crate::ExprAwait { + fold_expr_await(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_binary(&mut self, i: crate::ExprBinary) -> crate::ExprBinary { + fold_expr_binary(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_block(&mut self, i: crate::ExprBlock) -> crate::ExprBlock { + fold_expr_block(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_break(&mut self, i: crate::ExprBreak) -> crate::ExprBreak { + fold_expr_break(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_call(&mut self, i: crate::ExprCall) -> crate::ExprCall { + fold_expr_call(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_cast(&mut self, i: crate::ExprCast) -> crate::ExprCast { + fold_expr_cast(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_closure(&mut self, i: crate::ExprClosure) -> crate::ExprClosure { + fold_expr_closure(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_const(&mut self, i: crate::ExprConst) -> crate::ExprConst { + fold_expr_const(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_continue(&mut self, i: crate::ExprContinue) -> crate::ExprContinue { + fold_expr_continue(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_field(&mut self, i: crate::ExprField) -> crate::ExprField { + fold_expr_field(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_for_loop(&mut self, i: crate::ExprForLoop) -> crate::ExprForLoop { + fold_expr_for_loop(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_group(&mut self, i: crate::ExprGroup) -> crate::ExprGroup { + fold_expr_group(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_if(&mut self, i: crate::ExprIf) -> crate::ExprIf { + fold_expr_if(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_index(&mut self, i: crate::ExprIndex) -> crate::ExprIndex { + fold_expr_index(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_infer(&mut self, i: crate::ExprInfer) -> crate::ExprInfer { + fold_expr_infer(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_let(&mut self, i: crate::ExprLet) -> crate::ExprLet { + fold_expr_let(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_lit(&mut self, i: crate::ExprLit) -> crate::ExprLit { + fold_expr_lit(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_loop(&mut self, i: crate::ExprLoop) -> crate::ExprLoop { + fold_expr_loop(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_macro(&mut self, i: crate::ExprMacro) -> crate::ExprMacro { + fold_expr_macro(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_match(&mut self, i: crate::ExprMatch) -> crate::ExprMatch { + fold_expr_match(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_method_call( + &mut self, + i: crate::ExprMethodCall, + ) -> crate::ExprMethodCall { + fold_expr_method_call(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_paren(&mut self, i: crate::ExprParen) -> crate::ExprParen { + fold_expr_paren(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_path(&mut self, i: crate::ExprPath) -> crate::ExprPath { + fold_expr_path(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_range(&mut self, i: crate::ExprRange) -> crate::ExprRange { + fold_expr_range(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_raw_addr(&mut self, i: crate::ExprRawAddr) -> crate::ExprRawAddr { + fold_expr_raw_addr(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_reference(&mut self, i: crate::ExprReference) -> crate::ExprReference { + fold_expr_reference(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_repeat(&mut self, i: crate::ExprRepeat) -> crate::ExprRepeat { + fold_expr_repeat(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_return(&mut self, i: crate::ExprReturn) -> crate::ExprReturn { + fold_expr_return(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_struct(&mut self, i: crate::ExprStruct) -> crate::ExprStruct { + fold_expr_struct(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_try(&mut self, i: crate::ExprTry) -> crate::ExprTry { + fold_expr_try(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_try_block(&mut self, i: crate::ExprTryBlock) -> crate::ExprTryBlock { + fold_expr_try_block(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_tuple(&mut self, i: crate::ExprTuple) -> crate::ExprTuple { + fold_expr_tuple(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_expr_unary(&mut self, i: crate::ExprUnary) -> crate::ExprUnary { + fold_expr_unary(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_unsafe(&mut self, i: crate::ExprUnsafe) -> crate::ExprUnsafe { + fold_expr_unsafe(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_while(&mut self, i: crate::ExprWhile) -> crate::ExprWhile { + fold_expr_while(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_expr_yield(&mut self, i: crate::ExprYield) -> crate::ExprYield { + fold_expr_yield(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_field(&mut self, i: crate::Field) -> crate::Field { + fold_field(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_field_mutability( + &mut self, + i: crate::FieldMutability, + ) -> crate::FieldMutability { + fold_field_mutability(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_field_pat(&mut self, i: crate::FieldPat) -> crate::FieldPat { + fold_field_pat(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_field_value(&mut self, i: crate::FieldValue) -> crate::FieldValue { + fold_field_value(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_fields(&mut self, i: crate::Fields) -> crate::Fields { + fold_fields(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_fields_named(&mut self, i: crate::FieldsNamed) -> crate::FieldsNamed { + fold_fields_named(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_fields_unnamed(&mut self, i: crate::FieldsUnnamed) -> crate::FieldsUnnamed { + fold_fields_unnamed(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_file(&mut self, i: crate::File) -> crate::File { + fold_file(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_fn_arg(&mut self, i: crate::FnArg) -> crate::FnArg { + fold_fn_arg(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_foreign_item(&mut self, i: crate::ForeignItem) -> crate::ForeignItem { + fold_foreign_item(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_foreign_item_fn(&mut self, i: crate::ForeignItemFn) -> crate::ForeignItemFn { + fold_foreign_item_fn(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_foreign_item_macro( + &mut self, + i: crate::ForeignItemMacro, + ) -> crate::ForeignItemMacro { + fold_foreign_item_macro(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_foreign_item_static( + &mut self, + i: crate::ForeignItemStatic, + ) -> crate::ForeignItemStatic { + fold_foreign_item_static(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_foreign_item_type( + &mut self, + i: crate::ForeignItemType, + ) -> crate::ForeignItemType { + fold_foreign_item_type(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_generic_argument( + &mut self, + i: crate::GenericArgument, + ) -> crate::GenericArgument { + fold_generic_argument(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_generic_param(&mut self, i: crate::GenericParam) -> crate::GenericParam { + fold_generic_param(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_generics(&mut self, i: crate::Generics) -> crate::Generics { + fold_generics(self, i) + } + fn fold_ident(&mut self, i: proc_macro2::Ident) -> proc_macro2::Ident { + fold_ident(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_impl_item(&mut self, i: crate::ImplItem) -> crate::ImplItem { + fold_impl_item(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_impl_item_const(&mut self, i: crate::ImplItemConst) -> crate::ImplItemConst { + fold_impl_item_const(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_impl_item_fn(&mut self, i: crate::ImplItemFn) -> crate::ImplItemFn { + fold_impl_item_fn(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_impl_item_macro(&mut self, i: crate::ImplItemMacro) -> crate::ImplItemMacro { + fold_impl_item_macro(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_impl_item_type(&mut self, i: crate::ImplItemType) -> crate::ImplItemType { + fold_impl_item_type(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_impl_restriction( + &mut self, + i: crate::ImplRestriction, + ) -> crate::ImplRestriction { + fold_impl_restriction(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_index(&mut self, i: crate::Index) -> crate::Index { + fold_index(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item(&mut self, i: crate::Item) -> crate::Item { + fold_item(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_const(&mut self, i: crate::ItemConst) -> crate::ItemConst { + fold_item_const(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_enum(&mut self, i: crate::ItemEnum) -> crate::ItemEnum { + fold_item_enum(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_extern_crate( + &mut self, + i: crate::ItemExternCrate, + ) -> crate::ItemExternCrate { + fold_item_extern_crate(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_fn(&mut self, i: crate::ItemFn) -> crate::ItemFn { + fold_item_fn(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_foreign_mod( + &mut self, + i: crate::ItemForeignMod, + ) -> crate::ItemForeignMod { + fold_item_foreign_mod(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_impl(&mut self, i: crate::ItemImpl) -> crate::ItemImpl { + fold_item_impl(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_macro(&mut self, i: crate::ItemMacro) -> crate::ItemMacro { + fold_item_macro(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_mod(&mut self, i: crate::ItemMod) -> crate::ItemMod { + fold_item_mod(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_static(&mut self, i: crate::ItemStatic) -> crate::ItemStatic { + fold_item_static(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_struct(&mut self, i: crate::ItemStruct) -> crate::ItemStruct { + fold_item_struct(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_trait(&mut self, i: crate::ItemTrait) -> crate::ItemTrait { + fold_item_trait(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_trait_alias( + &mut self, + i: crate::ItemTraitAlias, + ) -> crate::ItemTraitAlias { + fold_item_trait_alias(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_type(&mut self, i: crate::ItemType) -> crate::ItemType { + fold_item_type(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_union(&mut self, i: crate::ItemUnion) -> crate::ItemUnion { + fold_item_union(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_item_use(&mut self, i: crate::ItemUse) -> crate::ItemUse { + fold_item_use(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_label(&mut self, i: crate::Label) -> crate::Label { + fold_label(self, i) + } + fn fold_lifetime(&mut self, i: crate::Lifetime) -> crate::Lifetime { + fold_lifetime(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_lifetime_param(&mut self, i: crate::LifetimeParam) -> crate::LifetimeParam { + fold_lifetime_param(self, i) + } + fn fold_lit(&mut self, i: crate::Lit) -> crate::Lit { + fold_lit(self, i) + } + fn fold_lit_bool(&mut self, i: crate::LitBool) -> crate::LitBool { + fold_lit_bool(self, i) + } + fn fold_lit_byte(&mut self, i: crate::LitByte) -> crate::LitByte { + fold_lit_byte(self, i) + } + fn fold_lit_byte_str(&mut self, i: crate::LitByteStr) -> crate::LitByteStr { + fold_lit_byte_str(self, i) + } + fn fold_lit_cstr(&mut self, i: crate::LitCStr) -> crate::LitCStr { + fold_lit_cstr(self, i) + } + fn fold_lit_char(&mut self, i: crate::LitChar) -> crate::LitChar { + fold_lit_char(self, i) + } + fn fold_lit_float(&mut self, i: crate::LitFloat) -> crate::LitFloat { + fold_lit_float(self, i) + } + fn fold_lit_int(&mut self, i: crate::LitInt) -> crate::LitInt { + fold_lit_int(self, i) + } + fn fold_lit_str(&mut self, i: crate::LitStr) -> crate::LitStr { + fold_lit_str(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_local(&mut self, i: crate::Local) -> crate::Local { + fold_local(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_local_init(&mut self, i: crate::LocalInit) -> crate::LocalInit { + fold_local_init(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_macro(&mut self, i: crate::Macro) -> crate::Macro { + fold_macro(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_macro_delimiter( + &mut self, + i: crate::MacroDelimiter, + ) -> crate::MacroDelimiter { + fold_macro_delimiter(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_member(&mut self, i: crate::Member) -> crate::Member { + fold_member(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_meta(&mut self, i: crate::Meta) -> crate::Meta { + fold_meta(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_meta_list(&mut self, i: crate::MetaList) -> crate::MetaList { + fold_meta_list(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_meta_name_value(&mut self, i: crate::MetaNameValue) -> crate::MetaNameValue { + fold_meta_name_value(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_parenthesized_generic_arguments( + &mut self, + i: crate::ParenthesizedGenericArguments, + ) -> crate::ParenthesizedGenericArguments { + fold_parenthesized_generic_arguments(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat(&mut self, i: crate::Pat) -> crate::Pat { + fold_pat(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_ident(&mut self, i: crate::PatIdent) -> crate::PatIdent { + fold_pat_ident(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_or(&mut self, i: crate::PatOr) -> crate::PatOr { + fold_pat_or(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_paren(&mut self, i: crate::PatParen) -> crate::PatParen { + fold_pat_paren(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_reference(&mut self, i: crate::PatReference) -> crate::PatReference { + fold_pat_reference(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_rest(&mut self, i: crate::PatRest) -> crate::PatRest { + fold_pat_rest(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_slice(&mut self, i: crate::PatSlice) -> crate::PatSlice { + fold_pat_slice(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_struct(&mut self, i: crate::PatStruct) -> crate::PatStruct { + fold_pat_struct(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_tuple(&mut self, i: crate::PatTuple) -> crate::PatTuple { + fold_pat_tuple(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_tuple_struct( + &mut self, + i: crate::PatTupleStruct, + ) -> crate::PatTupleStruct { + fold_pat_tuple_struct(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_type(&mut self, i: crate::PatType) -> crate::PatType { + fold_pat_type(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pat_wild(&mut self, i: crate::PatWild) -> crate::PatWild { + fold_pat_wild(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_path(&mut self, i: crate::Path) -> crate::Path { + fold_path(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_path_arguments(&mut self, i: crate::PathArguments) -> crate::PathArguments { + fold_path_arguments(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_path_segment(&mut self, i: crate::PathSegment) -> crate::PathSegment { + fold_path_segment(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_pointer_mutability( + &mut self, + i: crate::PointerMutability, + ) -> crate::PointerMutability { + fold_pointer_mutability(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_precise_capture( + &mut self, + i: crate::PreciseCapture, + ) -> crate::PreciseCapture { + fold_precise_capture(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_predicate_lifetime( + &mut self, + i: crate::PredicateLifetime, + ) -> crate::PredicateLifetime { + fold_predicate_lifetime(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_predicate_type(&mut self, i: crate::PredicateType) -> crate::PredicateType { + fold_predicate_type(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_qself(&mut self, i: crate::QSelf) -> crate::QSelf { + fold_qself(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_range_limits(&mut self, i: crate::RangeLimits) -> crate::RangeLimits { + fold_range_limits(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_receiver(&mut self, i: crate::Receiver) -> crate::Receiver { + fold_receiver(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_return_type(&mut self, i: crate::ReturnType) -> crate::ReturnType { + fold_return_type(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_signature(&mut self, i: crate::Signature) -> crate::Signature { + fold_signature(self, i) + } + fn fold_span(&mut self, i: proc_macro2::Span) -> proc_macro2::Span { + i + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_static_mutability( + &mut self, + i: crate::StaticMutability, + ) -> crate::StaticMutability { + fold_static_mutability(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_stmt(&mut self, i: crate::Stmt) -> crate::Stmt { + fold_stmt(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_stmt_macro(&mut self, i: crate::StmtMacro) -> crate::StmtMacro { + fold_stmt_macro(self, i) + } + fn fold_token_stream( + &mut self, + i: proc_macro2::TokenStream, + ) -> proc_macro2::TokenStream { + i + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_trait_bound(&mut self, i: crate::TraitBound) -> crate::TraitBound { + fold_trait_bound(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_trait_bound_modifier( + &mut self, + i: crate::TraitBoundModifier, + ) -> crate::TraitBoundModifier { + fold_trait_bound_modifier(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_trait_item(&mut self, i: crate::TraitItem) -> crate::TraitItem { + fold_trait_item(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_trait_item_const( + &mut self, + i: crate::TraitItemConst, + ) -> crate::TraitItemConst { + fold_trait_item_const(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_trait_item_fn(&mut self, i: crate::TraitItemFn) -> crate::TraitItemFn { + fold_trait_item_fn(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_trait_item_macro( + &mut self, + i: crate::TraitItemMacro, + ) -> crate::TraitItemMacro { + fold_trait_item_macro(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_trait_item_type(&mut self, i: crate::TraitItemType) -> crate::TraitItemType { + fold_trait_item_type(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type(&mut self, i: crate::Type) -> crate::Type { + fold_type(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_array(&mut self, i: crate::TypeArray) -> crate::TypeArray { + fold_type_array(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_bare_fn(&mut self, i: crate::TypeBareFn) -> crate::TypeBareFn { + fold_type_bare_fn(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_group(&mut self, i: crate::TypeGroup) -> crate::TypeGroup { + fold_type_group(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_impl_trait(&mut self, i: crate::TypeImplTrait) -> crate::TypeImplTrait { + fold_type_impl_trait(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_infer(&mut self, i: crate::TypeInfer) -> crate::TypeInfer { + fold_type_infer(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_macro(&mut self, i: crate::TypeMacro) -> crate::TypeMacro { + fold_type_macro(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_never(&mut self, i: crate::TypeNever) -> crate::TypeNever { + fold_type_never(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_param(&mut self, i: crate::TypeParam) -> crate::TypeParam { + fold_type_param(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_param_bound( + &mut self, + i: crate::TypeParamBound, + ) -> crate::TypeParamBound { + fold_type_param_bound(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_paren(&mut self, i: crate::TypeParen) -> crate::TypeParen { + fold_type_paren(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_path(&mut self, i: crate::TypePath) -> crate::TypePath { + fold_type_path(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_ptr(&mut self, i: crate::TypePtr) -> crate::TypePtr { + fold_type_ptr(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_reference(&mut self, i: crate::TypeReference) -> crate::TypeReference { + fold_type_reference(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_slice(&mut self, i: crate::TypeSlice) -> crate::TypeSlice { + fold_type_slice(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_trait_object( + &mut self, + i: crate::TypeTraitObject, + ) -> crate::TypeTraitObject { + fold_type_trait_object(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_type_tuple(&mut self, i: crate::TypeTuple) -> crate::TypeTuple { + fold_type_tuple(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_un_op(&mut self, i: crate::UnOp) -> crate::UnOp { + fold_un_op(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_use_glob(&mut self, i: crate::UseGlob) -> crate::UseGlob { + fold_use_glob(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_use_group(&mut self, i: crate::UseGroup) -> crate::UseGroup { + fold_use_group(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_use_name(&mut self, i: crate::UseName) -> crate::UseName { + fold_use_name(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_use_path(&mut self, i: crate::UsePath) -> crate::UsePath { + fold_use_path(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_use_rename(&mut self, i: crate::UseRename) -> crate::UseRename { + fold_use_rename(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_use_tree(&mut self, i: crate::UseTree) -> crate::UseTree { + fold_use_tree(self, i) + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn fold_variadic(&mut self, i: crate::Variadic) -> crate::Variadic { + fold_variadic(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_variant(&mut self, i: crate::Variant) -> crate::Variant { + fold_variant(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_vis_restricted(&mut self, i: crate::VisRestricted) -> crate::VisRestricted { + fold_vis_restricted(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_visibility(&mut self, i: crate::Visibility) -> crate::Visibility { + fold_visibility(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_where_clause(&mut self, i: crate::WhereClause) -> crate::WhereClause { + fold_where_clause(self, i) + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn fold_where_predicate( + &mut self, + i: crate::WherePredicate, + ) -> crate::WherePredicate { + fold_where_predicate(self, i) + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_abi<F>(f: &mut F, node: crate::Abi) -> crate::Abi +where + F: Fold + ?Sized, +{ + crate::Abi { + extern_token: node.extern_token, + name: (node.name).map(|it| f.fold_lit_str(it)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_angle_bracketed_generic_arguments<F>( + f: &mut F, + node: crate::AngleBracketedGenericArguments, +) -> crate::AngleBracketedGenericArguments +where + F: Fold + ?Sized, +{ + crate::AngleBracketedGenericArguments { + colon2_token: node.colon2_token, + lt_token: node.lt_token, + args: crate::punctuated::fold(node.args, f, F::fold_generic_argument), + gt_token: node.gt_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_arm<F>(f: &mut F, node: crate::Arm) -> crate::Arm +where + F: Fold + ?Sized, +{ + crate::Arm { + attrs: f.fold_attributes(node.attrs), + pat: f.fold_pat(node.pat), + guard: (node.guard).map(|it| ((it).0, Box::new(f.fold_expr(*(it).1)))), + fat_arrow_token: node.fat_arrow_token, + body: Box::new(f.fold_expr(*node.body)), + comma: node.comma, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_assoc_const<F>(f: &mut F, node: crate::AssocConst) -> crate::AssocConst +where + F: Fold + ?Sized, +{ + crate::AssocConst { + ident: f.fold_ident(node.ident), + generics: (node.generics).map(|it| f.fold_angle_bracketed_generic_arguments(it)), + eq_token: node.eq_token, + value: f.fold_expr(node.value), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_assoc_type<F>(f: &mut F, node: crate::AssocType) -> crate::AssocType +where + F: Fold + ?Sized, +{ + crate::AssocType { + ident: f.fold_ident(node.ident), + generics: (node.generics).map(|it| f.fold_angle_bracketed_generic_arguments(it)), + eq_token: node.eq_token, + ty: f.fold_type(node.ty), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_attr_style<F>(f: &mut F, node: crate::AttrStyle) -> crate::AttrStyle +where + F: Fold + ?Sized, +{ + match node { + crate::AttrStyle::Outer => crate::AttrStyle::Outer, + crate::AttrStyle::Inner(_binding_0) => crate::AttrStyle::Inner(_binding_0), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_attribute<F>(f: &mut F, node: crate::Attribute) -> crate::Attribute +where + F: Fold + ?Sized, +{ + crate::Attribute { + pound_token: node.pound_token, + style: f.fold_attr_style(node.style), + bracket_token: node.bracket_token, + meta: f.fold_meta(node.meta), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_bare_fn_arg<F>(f: &mut F, node: crate::BareFnArg) -> crate::BareFnArg +where + F: Fold + ?Sized, +{ + crate::BareFnArg { + attrs: f.fold_attributes(node.attrs), + name: (node.name).map(|it| (f.fold_ident((it).0), (it).1)), + ty: f.fold_type(node.ty), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_bare_variadic<F>(f: &mut F, node: crate::BareVariadic) -> crate::BareVariadic +where + F: Fold + ?Sized, +{ + crate::BareVariadic { + attrs: f.fold_attributes(node.attrs), + name: (node.name).map(|it| (f.fold_ident((it).0), (it).1)), + dots: node.dots, + comma: node.comma, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_bin_op<F>(f: &mut F, node: crate::BinOp) -> crate::BinOp +where + F: Fold + ?Sized, +{ + match node { + crate::BinOp::Add(_binding_0) => crate::BinOp::Add(_binding_0), + crate::BinOp::Sub(_binding_0) => crate::BinOp::Sub(_binding_0), + crate::BinOp::Mul(_binding_0) => crate::BinOp::Mul(_binding_0), + crate::BinOp::Div(_binding_0) => crate::BinOp::Div(_binding_0), + crate::BinOp::Rem(_binding_0) => crate::BinOp::Rem(_binding_0), + crate::BinOp::And(_binding_0) => crate::BinOp::And(_binding_0), + crate::BinOp::Or(_binding_0) => crate::BinOp::Or(_binding_0), + crate::BinOp::BitXor(_binding_0) => crate::BinOp::BitXor(_binding_0), + crate::BinOp::BitAnd(_binding_0) => crate::BinOp::BitAnd(_binding_0), + crate::BinOp::BitOr(_binding_0) => crate::BinOp::BitOr(_binding_0), + crate::BinOp::Shl(_binding_0) => crate::BinOp::Shl(_binding_0), + crate::BinOp::Shr(_binding_0) => crate::BinOp::Shr(_binding_0), + crate::BinOp::Eq(_binding_0) => crate::BinOp::Eq(_binding_0), + crate::BinOp::Lt(_binding_0) => crate::BinOp::Lt(_binding_0), + crate::BinOp::Le(_binding_0) => crate::BinOp::Le(_binding_0), + crate::BinOp::Ne(_binding_0) => crate::BinOp::Ne(_binding_0), + crate::BinOp::Ge(_binding_0) => crate::BinOp::Ge(_binding_0), + crate::BinOp::Gt(_binding_0) => crate::BinOp::Gt(_binding_0), + crate::BinOp::AddAssign(_binding_0) => crate::BinOp::AddAssign(_binding_0), + crate::BinOp::SubAssign(_binding_0) => crate::BinOp::SubAssign(_binding_0), + crate::BinOp::MulAssign(_binding_0) => crate::BinOp::MulAssign(_binding_0), + crate::BinOp::DivAssign(_binding_0) => crate::BinOp::DivAssign(_binding_0), + crate::BinOp::RemAssign(_binding_0) => crate::BinOp::RemAssign(_binding_0), + crate::BinOp::BitXorAssign(_binding_0) => crate::BinOp::BitXorAssign(_binding_0), + crate::BinOp::BitAndAssign(_binding_0) => crate::BinOp::BitAndAssign(_binding_0), + crate::BinOp::BitOrAssign(_binding_0) => crate::BinOp::BitOrAssign(_binding_0), + crate::BinOp::ShlAssign(_binding_0) => crate::BinOp::ShlAssign(_binding_0), + crate::BinOp::ShrAssign(_binding_0) => crate::BinOp::ShrAssign(_binding_0), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_block<F>(f: &mut F, node: crate::Block) -> crate::Block +where + F: Fold + ?Sized, +{ + crate::Block { + brace_token: node.brace_token, + stmts: fold_vec(node.stmts, f, F::fold_stmt), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_bound_lifetimes<F>( + f: &mut F, + node: crate::BoundLifetimes, +) -> crate::BoundLifetimes +where + F: Fold + ?Sized, +{ + crate::BoundLifetimes { + for_token: node.for_token, + lt_token: node.lt_token, + lifetimes: crate::punctuated::fold(node.lifetimes, f, F::fold_generic_param), + gt_token: node.gt_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_captured_param<F>( + f: &mut F, + node: crate::CapturedParam, +) -> crate::CapturedParam +where + F: Fold + ?Sized, +{ + match node { + crate::CapturedParam::Lifetime(_binding_0) => { + crate::CapturedParam::Lifetime(f.fold_lifetime(_binding_0)) + } + crate::CapturedParam::Ident(_binding_0) => { + crate::CapturedParam::Ident(f.fold_ident(_binding_0)) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_const_param<F>(f: &mut F, node: crate::ConstParam) -> crate::ConstParam +where + F: Fold + ?Sized, +{ + crate::ConstParam { + attrs: f.fold_attributes(node.attrs), + const_token: node.const_token, + ident: f.fold_ident(node.ident), + colon_token: node.colon_token, + ty: f.fold_type(node.ty), + eq_token: node.eq_token, + default: (node.default).map(|it| f.fold_expr(it)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_constraint<F>(f: &mut F, node: crate::Constraint) -> crate::Constraint +where + F: Fold + ?Sized, +{ + crate::Constraint { + ident: f.fold_ident(node.ident), + generics: (node.generics).map(|it| f.fold_angle_bracketed_generic_arguments(it)), + colon_token: node.colon_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_type_param_bound), + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn fold_data<F>(f: &mut F, node: crate::Data) -> crate::Data +where + F: Fold + ?Sized, +{ + match node { + crate::Data::Struct(_binding_0) => { + crate::Data::Struct(f.fold_data_struct(_binding_0)) + } + crate::Data::Enum(_binding_0) => crate::Data::Enum(f.fold_data_enum(_binding_0)), + crate::Data::Union(_binding_0) => { + crate::Data::Union(f.fold_data_union(_binding_0)) + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn fold_data_enum<F>(f: &mut F, node: crate::DataEnum) -> crate::DataEnum +where + F: Fold + ?Sized, +{ + crate::DataEnum { + enum_token: node.enum_token, + brace_token: node.brace_token, + variants: crate::punctuated::fold(node.variants, f, F::fold_variant), + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn fold_data_struct<F>(f: &mut F, node: crate::DataStruct) -> crate::DataStruct +where + F: Fold + ?Sized, +{ + crate::DataStruct { + struct_token: node.struct_token, + fields: f.fold_fields(node.fields), + semi_token: node.semi_token, + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn fold_data_union<F>(f: &mut F, node: crate::DataUnion) -> crate::DataUnion +where + F: Fold + ?Sized, +{ + crate::DataUnion { + union_token: node.union_token, + fields: f.fold_fields_named(node.fields), + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn fold_derive_input<F>(f: &mut F, node: crate::DeriveInput) -> crate::DeriveInput +where + F: Fold + ?Sized, +{ + crate::DeriveInput { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + data: f.fold_data(node.data), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr<F>(f: &mut F, node: crate::Expr) -> crate::Expr +where + F: Fold + ?Sized, +{ + match node { + crate::Expr::Array(_binding_0) => { + crate::Expr::Array(full!(f.fold_expr_array(_binding_0))) + } + crate::Expr::Assign(_binding_0) => { + crate::Expr::Assign(full!(f.fold_expr_assign(_binding_0))) + } + crate::Expr::Async(_binding_0) => { + crate::Expr::Async(full!(f.fold_expr_async(_binding_0))) + } + crate::Expr::Await(_binding_0) => { + crate::Expr::Await(full!(f.fold_expr_await(_binding_0))) + } + crate::Expr::Binary(_binding_0) => { + crate::Expr::Binary(f.fold_expr_binary(_binding_0)) + } + crate::Expr::Block(_binding_0) => { + crate::Expr::Block(full!(f.fold_expr_block(_binding_0))) + } + crate::Expr::Break(_binding_0) => { + crate::Expr::Break(full!(f.fold_expr_break(_binding_0))) + } + crate::Expr::Call(_binding_0) => crate::Expr::Call(f.fold_expr_call(_binding_0)), + crate::Expr::Cast(_binding_0) => crate::Expr::Cast(f.fold_expr_cast(_binding_0)), + crate::Expr::Closure(_binding_0) => { + crate::Expr::Closure(full!(f.fold_expr_closure(_binding_0))) + } + crate::Expr::Const(_binding_0) => { + crate::Expr::Const(full!(f.fold_expr_const(_binding_0))) + } + crate::Expr::Continue(_binding_0) => { + crate::Expr::Continue(full!(f.fold_expr_continue(_binding_0))) + } + crate::Expr::Field(_binding_0) => { + crate::Expr::Field(f.fold_expr_field(_binding_0)) + } + crate::Expr::ForLoop(_binding_0) => { + crate::Expr::ForLoop(full!(f.fold_expr_for_loop(_binding_0))) + } + crate::Expr::Group(_binding_0) => { + crate::Expr::Group(f.fold_expr_group(_binding_0)) + } + crate::Expr::If(_binding_0) => crate::Expr::If(full!(f.fold_expr_if(_binding_0))), + crate::Expr::Index(_binding_0) => { + crate::Expr::Index(f.fold_expr_index(_binding_0)) + } + crate::Expr::Infer(_binding_0) => { + crate::Expr::Infer(full!(f.fold_expr_infer(_binding_0))) + } + crate::Expr::Let(_binding_0) => { + crate::Expr::Let(full!(f.fold_expr_let(_binding_0))) + } + crate::Expr::Lit(_binding_0) => crate::Expr::Lit(f.fold_expr_lit(_binding_0)), + crate::Expr::Loop(_binding_0) => { + crate::Expr::Loop(full!(f.fold_expr_loop(_binding_0))) + } + crate::Expr::Macro(_binding_0) => { + crate::Expr::Macro(f.fold_expr_macro(_binding_0)) + } + crate::Expr::Match(_binding_0) => { + crate::Expr::Match(full!(f.fold_expr_match(_binding_0))) + } + crate::Expr::MethodCall(_binding_0) => { + crate::Expr::MethodCall(f.fold_expr_method_call(_binding_0)) + } + crate::Expr::Paren(_binding_0) => { + crate::Expr::Paren(f.fold_expr_paren(_binding_0)) + } + crate::Expr::Path(_binding_0) => crate::Expr::Path(f.fold_expr_path(_binding_0)), + crate::Expr::Range(_binding_0) => { + crate::Expr::Range(full!(f.fold_expr_range(_binding_0))) + } + crate::Expr::RawAddr(_binding_0) => { + crate::Expr::RawAddr(full!(f.fold_expr_raw_addr(_binding_0))) + } + crate::Expr::Reference(_binding_0) => { + crate::Expr::Reference(f.fold_expr_reference(_binding_0)) + } + crate::Expr::Repeat(_binding_0) => { + crate::Expr::Repeat(full!(f.fold_expr_repeat(_binding_0))) + } + crate::Expr::Return(_binding_0) => { + crate::Expr::Return(full!(f.fold_expr_return(_binding_0))) + } + crate::Expr::Struct(_binding_0) => { + crate::Expr::Struct(f.fold_expr_struct(_binding_0)) + } + crate::Expr::Try(_binding_0) => { + crate::Expr::Try(full!(f.fold_expr_try(_binding_0))) + } + crate::Expr::TryBlock(_binding_0) => { + crate::Expr::TryBlock(full!(f.fold_expr_try_block(_binding_0))) + } + crate::Expr::Tuple(_binding_0) => { + crate::Expr::Tuple(f.fold_expr_tuple(_binding_0)) + } + crate::Expr::Unary(_binding_0) => { + crate::Expr::Unary(f.fold_expr_unary(_binding_0)) + } + crate::Expr::Unsafe(_binding_0) => { + crate::Expr::Unsafe(full!(f.fold_expr_unsafe(_binding_0))) + } + crate::Expr::Verbatim(_binding_0) => { + crate::Expr::Verbatim(f.fold_token_stream(_binding_0)) + } + crate::Expr::While(_binding_0) => { + crate::Expr::While(full!(f.fold_expr_while(_binding_0))) + } + crate::Expr::Yield(_binding_0) => { + crate::Expr::Yield(full!(f.fold_expr_yield(_binding_0))) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_array<F>(f: &mut F, node: crate::ExprArray) -> crate::ExprArray +where + F: Fold + ?Sized, +{ + crate::ExprArray { + attrs: f.fold_attributes(node.attrs), + bracket_token: node.bracket_token, + elems: crate::punctuated::fold(node.elems, f, F::fold_expr), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_assign<F>(f: &mut F, node: crate::ExprAssign) -> crate::ExprAssign +where + F: Fold + ?Sized, +{ + crate::ExprAssign { + attrs: f.fold_attributes(node.attrs), + left: Box::new(f.fold_expr(*node.left)), + eq_token: node.eq_token, + right: Box::new(f.fold_expr(*node.right)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_async<F>(f: &mut F, node: crate::ExprAsync) -> crate::ExprAsync +where + F: Fold + ?Sized, +{ + crate::ExprAsync { + attrs: f.fold_attributes(node.attrs), + async_token: node.async_token, + capture: node.capture, + block: f.fold_block(node.block), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_await<F>(f: &mut F, node: crate::ExprAwait) -> crate::ExprAwait +where + F: Fold + ?Sized, +{ + crate::ExprAwait { + attrs: f.fold_attributes(node.attrs), + base: Box::new(f.fold_expr(*node.base)), + dot_token: node.dot_token, + await_token: node.await_token, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_binary<F>(f: &mut F, node: crate::ExprBinary) -> crate::ExprBinary +where + F: Fold + ?Sized, +{ + crate::ExprBinary { + attrs: f.fold_attributes(node.attrs), + left: Box::new(f.fold_expr(*node.left)), + op: f.fold_bin_op(node.op), + right: Box::new(f.fold_expr(*node.right)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_block<F>(f: &mut F, node: crate::ExprBlock) -> crate::ExprBlock +where + F: Fold + ?Sized, +{ + crate::ExprBlock { + attrs: f.fold_attributes(node.attrs), + label: (node.label).map(|it| f.fold_label(it)), + block: f.fold_block(node.block), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_break<F>(f: &mut F, node: crate::ExprBreak) -> crate::ExprBreak +where + F: Fold + ?Sized, +{ + crate::ExprBreak { + attrs: f.fold_attributes(node.attrs), + break_token: node.break_token, + label: (node.label).map(|it| f.fold_lifetime(it)), + expr: (node.expr).map(|it| Box::new(f.fold_expr(*it))), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_call<F>(f: &mut F, node: crate::ExprCall) -> crate::ExprCall +where + F: Fold + ?Sized, +{ + crate::ExprCall { + attrs: f.fold_attributes(node.attrs), + func: Box::new(f.fold_expr(*node.func)), + paren_token: node.paren_token, + args: crate::punctuated::fold(node.args, f, F::fold_expr), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_cast<F>(f: &mut F, node: crate::ExprCast) -> crate::ExprCast +where + F: Fold + ?Sized, +{ + crate::ExprCast { + attrs: f.fold_attributes(node.attrs), + expr: Box::new(f.fold_expr(*node.expr)), + as_token: node.as_token, + ty: Box::new(f.fold_type(*node.ty)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_closure<F>(f: &mut F, node: crate::ExprClosure) -> crate::ExprClosure +where + F: Fold + ?Sized, +{ + crate::ExprClosure { + attrs: f.fold_attributes(node.attrs), + lifetimes: (node.lifetimes).map(|it| f.fold_bound_lifetimes(it)), + constness: node.constness, + movability: node.movability, + asyncness: node.asyncness, + capture: node.capture, + or1_token: node.or1_token, + inputs: crate::punctuated::fold(node.inputs, f, F::fold_pat), + or2_token: node.or2_token, + output: f.fold_return_type(node.output), + body: Box::new(f.fold_expr(*node.body)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_const<F>(f: &mut F, node: crate::ExprConst) -> crate::ExprConst +where + F: Fold + ?Sized, +{ + crate::ExprConst { + attrs: f.fold_attributes(node.attrs), + const_token: node.const_token, + block: f.fold_block(node.block), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_continue<F>(f: &mut F, node: crate::ExprContinue) -> crate::ExprContinue +where + F: Fold + ?Sized, +{ + crate::ExprContinue { + attrs: f.fold_attributes(node.attrs), + continue_token: node.continue_token, + label: (node.label).map(|it| f.fold_lifetime(it)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_field<F>(f: &mut F, node: crate::ExprField) -> crate::ExprField +where + F: Fold + ?Sized, +{ + crate::ExprField { + attrs: f.fold_attributes(node.attrs), + base: Box::new(f.fold_expr(*node.base)), + dot_token: node.dot_token, + member: f.fold_member(node.member), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_for_loop<F>(f: &mut F, node: crate::ExprForLoop) -> crate::ExprForLoop +where + F: Fold + ?Sized, +{ + crate::ExprForLoop { + attrs: f.fold_attributes(node.attrs), + label: (node.label).map(|it| f.fold_label(it)), + for_token: node.for_token, + pat: Box::new(f.fold_pat(*node.pat)), + in_token: node.in_token, + expr: Box::new(f.fold_expr(*node.expr)), + body: f.fold_block(node.body), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_group<F>(f: &mut F, node: crate::ExprGroup) -> crate::ExprGroup +where + F: Fold + ?Sized, +{ + crate::ExprGroup { + attrs: f.fold_attributes(node.attrs), + group_token: node.group_token, + expr: Box::new(f.fold_expr(*node.expr)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_if<F>(f: &mut F, node: crate::ExprIf) -> crate::ExprIf +where + F: Fold + ?Sized, +{ + crate::ExprIf { + attrs: f.fold_attributes(node.attrs), + if_token: node.if_token, + cond: Box::new(f.fold_expr(*node.cond)), + then_branch: f.fold_block(node.then_branch), + else_branch: (node.else_branch) + .map(|it| ((it).0, Box::new(f.fold_expr(*(it).1)))), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_index<F>(f: &mut F, node: crate::ExprIndex) -> crate::ExprIndex +where + F: Fold + ?Sized, +{ + crate::ExprIndex { + attrs: f.fold_attributes(node.attrs), + expr: Box::new(f.fold_expr(*node.expr)), + bracket_token: node.bracket_token, + index: Box::new(f.fold_expr(*node.index)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_infer<F>(f: &mut F, node: crate::ExprInfer) -> crate::ExprInfer +where + F: Fold + ?Sized, +{ + crate::ExprInfer { + attrs: f.fold_attributes(node.attrs), + underscore_token: node.underscore_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_let<F>(f: &mut F, node: crate::ExprLet) -> crate::ExprLet +where + F: Fold + ?Sized, +{ + crate::ExprLet { + attrs: f.fold_attributes(node.attrs), + let_token: node.let_token, + pat: Box::new(f.fold_pat(*node.pat)), + eq_token: node.eq_token, + expr: Box::new(f.fold_expr(*node.expr)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_lit<F>(f: &mut F, node: crate::ExprLit) -> crate::ExprLit +where + F: Fold + ?Sized, +{ + crate::ExprLit { + attrs: f.fold_attributes(node.attrs), + lit: f.fold_lit(node.lit), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_loop<F>(f: &mut F, node: crate::ExprLoop) -> crate::ExprLoop +where + F: Fold + ?Sized, +{ + crate::ExprLoop { + attrs: f.fold_attributes(node.attrs), + label: (node.label).map(|it| f.fold_label(it)), + loop_token: node.loop_token, + body: f.fold_block(node.body), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_macro<F>(f: &mut F, node: crate::ExprMacro) -> crate::ExprMacro +where + F: Fold + ?Sized, +{ + crate::ExprMacro { + attrs: f.fold_attributes(node.attrs), + mac: f.fold_macro(node.mac), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_match<F>(f: &mut F, node: crate::ExprMatch) -> crate::ExprMatch +where + F: Fold + ?Sized, +{ + crate::ExprMatch { + attrs: f.fold_attributes(node.attrs), + match_token: node.match_token, + expr: Box::new(f.fold_expr(*node.expr)), + brace_token: node.brace_token, + arms: fold_vec(node.arms, f, F::fold_arm), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_method_call<F>( + f: &mut F, + node: crate::ExprMethodCall, +) -> crate::ExprMethodCall +where + F: Fold + ?Sized, +{ + crate::ExprMethodCall { + attrs: f.fold_attributes(node.attrs), + receiver: Box::new(f.fold_expr(*node.receiver)), + dot_token: node.dot_token, + method: f.fold_ident(node.method), + turbofish: (node.turbofish) + .map(|it| f.fold_angle_bracketed_generic_arguments(it)), + paren_token: node.paren_token, + args: crate::punctuated::fold(node.args, f, F::fold_expr), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_paren<F>(f: &mut F, node: crate::ExprParen) -> crate::ExprParen +where + F: Fold + ?Sized, +{ + crate::ExprParen { + attrs: f.fold_attributes(node.attrs), + paren_token: node.paren_token, + expr: Box::new(f.fold_expr(*node.expr)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_path<F>(f: &mut F, node: crate::ExprPath) -> crate::ExprPath +where + F: Fold + ?Sized, +{ + crate::ExprPath { + attrs: f.fold_attributes(node.attrs), + qself: (node.qself).map(|it| f.fold_qself(it)), + path: f.fold_path(node.path), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_range<F>(f: &mut F, node: crate::ExprRange) -> crate::ExprRange +where + F: Fold + ?Sized, +{ + crate::ExprRange { + attrs: f.fold_attributes(node.attrs), + start: (node.start).map(|it| Box::new(f.fold_expr(*it))), + limits: f.fold_range_limits(node.limits), + end: (node.end).map(|it| Box::new(f.fold_expr(*it))), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_raw_addr<F>(f: &mut F, node: crate::ExprRawAddr) -> crate::ExprRawAddr +where + F: Fold + ?Sized, +{ + crate::ExprRawAddr { + attrs: f.fold_attributes(node.attrs), + and_token: node.and_token, + raw: node.raw, + mutability: f.fold_pointer_mutability(node.mutability), + expr: Box::new(f.fold_expr(*node.expr)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_reference<F>( + f: &mut F, + node: crate::ExprReference, +) -> crate::ExprReference +where + F: Fold + ?Sized, +{ + crate::ExprReference { + attrs: f.fold_attributes(node.attrs), + and_token: node.and_token, + mutability: node.mutability, + expr: Box::new(f.fold_expr(*node.expr)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_repeat<F>(f: &mut F, node: crate::ExprRepeat) -> crate::ExprRepeat +where + F: Fold + ?Sized, +{ + crate::ExprRepeat { + attrs: f.fold_attributes(node.attrs), + bracket_token: node.bracket_token, + expr: Box::new(f.fold_expr(*node.expr)), + semi_token: node.semi_token, + len: Box::new(f.fold_expr(*node.len)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_return<F>(f: &mut F, node: crate::ExprReturn) -> crate::ExprReturn +where + F: Fold + ?Sized, +{ + crate::ExprReturn { + attrs: f.fold_attributes(node.attrs), + return_token: node.return_token, + expr: (node.expr).map(|it| Box::new(f.fold_expr(*it))), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_struct<F>(f: &mut F, node: crate::ExprStruct) -> crate::ExprStruct +where + F: Fold + ?Sized, +{ + crate::ExprStruct { + attrs: f.fold_attributes(node.attrs), + qself: (node.qself).map(|it| f.fold_qself(it)), + path: f.fold_path(node.path), + brace_token: node.brace_token, + fields: crate::punctuated::fold(node.fields, f, F::fold_field_value), + dot2_token: node.dot2_token, + rest: (node.rest).map(|it| Box::new(f.fold_expr(*it))), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_try<F>(f: &mut F, node: crate::ExprTry) -> crate::ExprTry +where + F: Fold + ?Sized, +{ + crate::ExprTry { + attrs: f.fold_attributes(node.attrs), + expr: Box::new(f.fold_expr(*node.expr)), + question_token: node.question_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_try_block<F>( + f: &mut F, + node: crate::ExprTryBlock, +) -> crate::ExprTryBlock +where + F: Fold + ?Sized, +{ + crate::ExprTryBlock { + attrs: f.fold_attributes(node.attrs), + try_token: node.try_token, + block: f.fold_block(node.block), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_tuple<F>(f: &mut F, node: crate::ExprTuple) -> crate::ExprTuple +where + F: Fold + ?Sized, +{ + crate::ExprTuple { + attrs: f.fold_attributes(node.attrs), + paren_token: node.paren_token, + elems: crate::punctuated::fold(node.elems, f, F::fold_expr), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_expr_unary<F>(f: &mut F, node: crate::ExprUnary) -> crate::ExprUnary +where + F: Fold + ?Sized, +{ + crate::ExprUnary { + attrs: f.fold_attributes(node.attrs), + op: f.fold_un_op(node.op), + expr: Box::new(f.fold_expr(*node.expr)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_unsafe<F>(f: &mut F, node: crate::ExprUnsafe) -> crate::ExprUnsafe +where + F: Fold + ?Sized, +{ + crate::ExprUnsafe { + attrs: f.fold_attributes(node.attrs), + unsafe_token: node.unsafe_token, + block: f.fold_block(node.block), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_while<F>(f: &mut F, node: crate::ExprWhile) -> crate::ExprWhile +where + F: Fold + ?Sized, +{ + crate::ExprWhile { + attrs: f.fold_attributes(node.attrs), + label: (node.label).map(|it| f.fold_label(it)), + while_token: node.while_token, + cond: Box::new(f.fold_expr(*node.cond)), + body: f.fold_block(node.body), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_expr_yield<F>(f: &mut F, node: crate::ExprYield) -> crate::ExprYield +where + F: Fold + ?Sized, +{ + crate::ExprYield { + attrs: f.fold_attributes(node.attrs), + yield_token: node.yield_token, + expr: (node.expr).map(|it| Box::new(f.fold_expr(*it))), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_field<F>(f: &mut F, node: crate::Field) -> crate::Field +where + F: Fold + ?Sized, +{ + crate::Field { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + mutability: f.fold_field_mutability(node.mutability), + ident: (node.ident).map(|it| f.fold_ident(it)), + colon_token: node.colon_token, + ty: f.fold_type(node.ty), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_field_mutability<F>( + f: &mut F, + node: crate::FieldMutability, +) -> crate::FieldMutability +where + F: Fold + ?Sized, +{ + match node { + crate::FieldMutability::None => crate::FieldMutability::None, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_field_pat<F>(f: &mut F, node: crate::FieldPat) -> crate::FieldPat +where + F: Fold + ?Sized, +{ + crate::FieldPat { + attrs: f.fold_attributes(node.attrs), + member: f.fold_member(node.member), + colon_token: node.colon_token, + pat: Box::new(f.fold_pat(*node.pat)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_field_value<F>(f: &mut F, node: crate::FieldValue) -> crate::FieldValue +where + F: Fold + ?Sized, +{ + crate::FieldValue { + attrs: f.fold_attributes(node.attrs), + member: f.fold_member(node.member), + colon_token: node.colon_token, + expr: f.fold_expr(node.expr), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_fields<F>(f: &mut F, node: crate::Fields) -> crate::Fields +where + F: Fold + ?Sized, +{ + match node { + crate::Fields::Named(_binding_0) => { + crate::Fields::Named(f.fold_fields_named(_binding_0)) + } + crate::Fields::Unnamed(_binding_0) => { + crate::Fields::Unnamed(f.fold_fields_unnamed(_binding_0)) + } + crate::Fields::Unit => crate::Fields::Unit, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_fields_named<F>(f: &mut F, node: crate::FieldsNamed) -> crate::FieldsNamed +where + F: Fold + ?Sized, +{ + crate::FieldsNamed { + brace_token: node.brace_token, + named: crate::punctuated::fold(node.named, f, F::fold_field), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_fields_unnamed<F>( + f: &mut F, + node: crate::FieldsUnnamed, +) -> crate::FieldsUnnamed +where + F: Fold + ?Sized, +{ + crate::FieldsUnnamed { + paren_token: node.paren_token, + unnamed: crate::punctuated::fold(node.unnamed, f, F::fold_field), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_file<F>(f: &mut F, node: crate::File) -> crate::File +where + F: Fold + ?Sized, +{ + crate::File { + shebang: node.shebang, + attrs: f.fold_attributes(node.attrs), + items: fold_vec(node.items, f, F::fold_item), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_fn_arg<F>(f: &mut F, node: crate::FnArg) -> crate::FnArg +where + F: Fold + ?Sized, +{ + match node { + crate::FnArg::Receiver(_binding_0) => { + crate::FnArg::Receiver(f.fold_receiver(_binding_0)) + } + crate::FnArg::Typed(_binding_0) => { + crate::FnArg::Typed(f.fold_pat_type(_binding_0)) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_foreign_item<F>(f: &mut F, node: crate::ForeignItem) -> crate::ForeignItem +where + F: Fold + ?Sized, +{ + match node { + crate::ForeignItem::Fn(_binding_0) => { + crate::ForeignItem::Fn(f.fold_foreign_item_fn(_binding_0)) + } + crate::ForeignItem::Static(_binding_0) => { + crate::ForeignItem::Static(f.fold_foreign_item_static(_binding_0)) + } + crate::ForeignItem::Type(_binding_0) => { + crate::ForeignItem::Type(f.fold_foreign_item_type(_binding_0)) + } + crate::ForeignItem::Macro(_binding_0) => { + crate::ForeignItem::Macro(f.fold_foreign_item_macro(_binding_0)) + } + crate::ForeignItem::Verbatim(_binding_0) => { + crate::ForeignItem::Verbatim(f.fold_token_stream(_binding_0)) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_foreign_item_fn<F>( + f: &mut F, + node: crate::ForeignItemFn, +) -> crate::ForeignItemFn +where + F: Fold + ?Sized, +{ + crate::ForeignItemFn { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + sig: f.fold_signature(node.sig), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_foreign_item_macro<F>( + f: &mut F, + node: crate::ForeignItemMacro, +) -> crate::ForeignItemMacro +where + F: Fold + ?Sized, +{ + crate::ForeignItemMacro { + attrs: f.fold_attributes(node.attrs), + mac: f.fold_macro(node.mac), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_foreign_item_static<F>( + f: &mut F, + node: crate::ForeignItemStatic, +) -> crate::ForeignItemStatic +where + F: Fold + ?Sized, +{ + crate::ForeignItemStatic { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + static_token: node.static_token, + mutability: f.fold_static_mutability(node.mutability), + ident: f.fold_ident(node.ident), + colon_token: node.colon_token, + ty: Box::new(f.fold_type(*node.ty)), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_foreign_item_type<F>( + f: &mut F, + node: crate::ForeignItemType, +) -> crate::ForeignItemType +where + F: Fold + ?Sized, +{ + crate::ForeignItemType { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + type_token: node.type_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + semi_token: node.semi_token, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_generic_argument<F>( + f: &mut F, + node: crate::GenericArgument, +) -> crate::GenericArgument +where + F: Fold + ?Sized, +{ + match node { + crate::GenericArgument::Lifetime(_binding_0) => { + crate::GenericArgument::Lifetime(f.fold_lifetime(_binding_0)) + } + crate::GenericArgument::Type(_binding_0) => { + crate::GenericArgument::Type(f.fold_type(_binding_0)) + } + crate::GenericArgument::Const(_binding_0) => { + crate::GenericArgument::Const(f.fold_expr(_binding_0)) + } + crate::GenericArgument::AssocType(_binding_0) => { + crate::GenericArgument::AssocType(f.fold_assoc_type(_binding_0)) + } + crate::GenericArgument::AssocConst(_binding_0) => { + crate::GenericArgument::AssocConst(f.fold_assoc_const(_binding_0)) + } + crate::GenericArgument::Constraint(_binding_0) => { + crate::GenericArgument::Constraint(f.fold_constraint(_binding_0)) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_generic_param<F>(f: &mut F, node: crate::GenericParam) -> crate::GenericParam +where + F: Fold + ?Sized, +{ + match node { + crate::GenericParam::Lifetime(_binding_0) => { + crate::GenericParam::Lifetime(f.fold_lifetime_param(_binding_0)) + } + crate::GenericParam::Type(_binding_0) => { + crate::GenericParam::Type(f.fold_type_param(_binding_0)) + } + crate::GenericParam::Const(_binding_0) => { + crate::GenericParam::Const(f.fold_const_param(_binding_0)) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_generics<F>(f: &mut F, node: crate::Generics) -> crate::Generics +where + F: Fold + ?Sized, +{ + crate::Generics { + lt_token: node.lt_token, + params: crate::punctuated::fold(node.params, f, F::fold_generic_param), + gt_token: node.gt_token, + where_clause: (node.where_clause).map(|it| f.fold_where_clause(it)), + } +} +pub fn fold_ident<F>(f: &mut F, node: proc_macro2::Ident) -> proc_macro2::Ident +where + F: Fold + ?Sized, +{ + let mut node = node; + let span = f.fold_span(node.span()); + node.set_span(span); + node +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_impl_item<F>(f: &mut F, node: crate::ImplItem) -> crate::ImplItem +where + F: Fold + ?Sized, +{ + match node { + crate::ImplItem::Const(_binding_0) => { + crate::ImplItem::Const(f.fold_impl_item_const(_binding_0)) + } + crate::ImplItem::Fn(_binding_0) => { + crate::ImplItem::Fn(f.fold_impl_item_fn(_binding_0)) + } + crate::ImplItem::Type(_binding_0) => { + crate::ImplItem::Type(f.fold_impl_item_type(_binding_0)) + } + crate::ImplItem::Macro(_binding_0) => { + crate::ImplItem::Macro(f.fold_impl_item_macro(_binding_0)) + } + crate::ImplItem::Verbatim(_binding_0) => { + crate::ImplItem::Verbatim(f.fold_token_stream(_binding_0)) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_impl_item_const<F>( + f: &mut F, + node: crate::ImplItemConst, +) -> crate::ImplItemConst +where + F: Fold + ?Sized, +{ + crate::ImplItemConst { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + defaultness: node.defaultness, + const_token: node.const_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + colon_token: node.colon_token, + ty: f.fold_type(node.ty), + eq_token: node.eq_token, + expr: f.fold_expr(node.expr), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_impl_item_fn<F>(f: &mut F, node: crate::ImplItemFn) -> crate::ImplItemFn +where + F: Fold + ?Sized, +{ + crate::ImplItemFn { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + defaultness: node.defaultness, + sig: f.fold_signature(node.sig), + block: f.fold_block(node.block), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_impl_item_macro<F>( + f: &mut F, + node: crate::ImplItemMacro, +) -> crate::ImplItemMacro +where + F: Fold + ?Sized, +{ + crate::ImplItemMacro { + attrs: f.fold_attributes(node.attrs), + mac: f.fold_macro(node.mac), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_impl_item_type<F>( + f: &mut F, + node: crate::ImplItemType, +) -> crate::ImplItemType +where + F: Fold + ?Sized, +{ + crate::ImplItemType { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + defaultness: node.defaultness, + type_token: node.type_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + eq_token: node.eq_token, + ty: f.fold_type(node.ty), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_impl_restriction<F>( + f: &mut F, + node: crate::ImplRestriction, +) -> crate::ImplRestriction +where + F: Fold + ?Sized, +{ + match node {} +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_index<F>(f: &mut F, node: crate::Index) -> crate::Index +where + F: Fold + ?Sized, +{ + crate::Index { + index: node.index, + span: f.fold_span(node.span), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item<F>(f: &mut F, node: crate::Item) -> crate::Item +where + F: Fold + ?Sized, +{ + match node { + crate::Item::Const(_binding_0) => { + crate::Item::Const(f.fold_item_const(_binding_0)) + } + crate::Item::Enum(_binding_0) => crate::Item::Enum(f.fold_item_enum(_binding_0)), + crate::Item::ExternCrate(_binding_0) => { + crate::Item::ExternCrate(f.fold_item_extern_crate(_binding_0)) + } + crate::Item::Fn(_binding_0) => crate::Item::Fn(f.fold_item_fn(_binding_0)), + crate::Item::ForeignMod(_binding_0) => { + crate::Item::ForeignMod(f.fold_item_foreign_mod(_binding_0)) + } + crate::Item::Impl(_binding_0) => crate::Item::Impl(f.fold_item_impl(_binding_0)), + crate::Item::Macro(_binding_0) => { + crate::Item::Macro(f.fold_item_macro(_binding_0)) + } + crate::Item::Mod(_binding_0) => crate::Item::Mod(f.fold_item_mod(_binding_0)), + crate::Item::Static(_binding_0) => { + crate::Item::Static(f.fold_item_static(_binding_0)) + } + crate::Item::Struct(_binding_0) => { + crate::Item::Struct(f.fold_item_struct(_binding_0)) + } + crate::Item::Trait(_binding_0) => { + crate::Item::Trait(f.fold_item_trait(_binding_0)) + } + crate::Item::TraitAlias(_binding_0) => { + crate::Item::TraitAlias(f.fold_item_trait_alias(_binding_0)) + } + crate::Item::Type(_binding_0) => crate::Item::Type(f.fold_item_type(_binding_0)), + crate::Item::Union(_binding_0) => { + crate::Item::Union(f.fold_item_union(_binding_0)) + } + crate::Item::Use(_binding_0) => crate::Item::Use(f.fold_item_use(_binding_0)), + crate::Item::Verbatim(_binding_0) => { + crate::Item::Verbatim(f.fold_token_stream(_binding_0)) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_const<F>(f: &mut F, node: crate::ItemConst) -> crate::ItemConst +where + F: Fold + ?Sized, +{ + crate::ItemConst { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + const_token: node.const_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + colon_token: node.colon_token, + ty: Box::new(f.fold_type(*node.ty)), + eq_token: node.eq_token, + expr: Box::new(f.fold_expr(*node.expr)), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_enum<F>(f: &mut F, node: crate::ItemEnum) -> crate::ItemEnum +where + F: Fold + ?Sized, +{ + crate::ItemEnum { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + enum_token: node.enum_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + brace_token: node.brace_token, + variants: crate::punctuated::fold(node.variants, f, F::fold_variant), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_extern_crate<F>( + f: &mut F, + node: crate::ItemExternCrate, +) -> crate::ItemExternCrate +where + F: Fold + ?Sized, +{ + crate::ItemExternCrate { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + extern_token: node.extern_token, + crate_token: node.crate_token, + ident: f.fold_ident(node.ident), + rename: (node.rename).map(|it| ((it).0, f.fold_ident((it).1))), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_fn<F>(f: &mut F, node: crate::ItemFn) -> crate::ItemFn +where + F: Fold + ?Sized, +{ + crate::ItemFn { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + sig: f.fold_signature(node.sig), + block: Box::new(f.fold_block(*node.block)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_foreign_mod<F>( + f: &mut F, + node: crate::ItemForeignMod, +) -> crate::ItemForeignMod +where + F: Fold + ?Sized, +{ + crate::ItemForeignMod { + attrs: f.fold_attributes(node.attrs), + unsafety: node.unsafety, + abi: f.fold_abi(node.abi), + brace_token: node.brace_token, + items: fold_vec(node.items, f, F::fold_foreign_item), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_impl<F>(f: &mut F, node: crate::ItemImpl) -> crate::ItemImpl +where + F: Fold + ?Sized, +{ + crate::ItemImpl { + attrs: f.fold_attributes(node.attrs), + defaultness: node.defaultness, + unsafety: node.unsafety, + impl_token: node.impl_token, + generics: f.fold_generics(node.generics), + trait_: (node.trait_).map(|it| ((it).0, f.fold_path((it).1), (it).2)), + self_ty: Box::new(f.fold_type(*node.self_ty)), + brace_token: node.brace_token, + items: fold_vec(node.items, f, F::fold_impl_item), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_macro<F>(f: &mut F, node: crate::ItemMacro) -> crate::ItemMacro +where + F: Fold + ?Sized, +{ + crate::ItemMacro { + attrs: f.fold_attributes(node.attrs), + ident: (node.ident).map(|it| f.fold_ident(it)), + mac: f.fold_macro(node.mac), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_mod<F>(f: &mut F, node: crate::ItemMod) -> crate::ItemMod +where + F: Fold + ?Sized, +{ + crate::ItemMod { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + unsafety: node.unsafety, + mod_token: node.mod_token, + ident: f.fold_ident(node.ident), + content: (node.content).map(|it| ((it).0, fold_vec((it).1, f, F::fold_item))), + semi: node.semi, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_static<F>(f: &mut F, node: crate::ItemStatic) -> crate::ItemStatic +where + F: Fold + ?Sized, +{ + crate::ItemStatic { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + static_token: node.static_token, + mutability: f.fold_static_mutability(node.mutability), + ident: f.fold_ident(node.ident), + colon_token: node.colon_token, + ty: Box::new(f.fold_type(*node.ty)), + eq_token: node.eq_token, + expr: Box::new(f.fold_expr(*node.expr)), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_struct<F>(f: &mut F, node: crate::ItemStruct) -> crate::ItemStruct +where + F: Fold + ?Sized, +{ + crate::ItemStruct { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + struct_token: node.struct_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + fields: f.fold_fields(node.fields), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_trait<F>(f: &mut F, node: crate::ItemTrait) -> crate::ItemTrait +where + F: Fold + ?Sized, +{ + crate::ItemTrait { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + unsafety: node.unsafety, + auto_token: node.auto_token, + restriction: (node.restriction).map(|it| f.fold_impl_restriction(it)), + trait_token: node.trait_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + colon_token: node.colon_token, + supertraits: crate::punctuated::fold( + node.supertraits, + f, + F::fold_type_param_bound, + ), + brace_token: node.brace_token, + items: fold_vec(node.items, f, F::fold_trait_item), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_trait_alias<F>( + f: &mut F, + node: crate::ItemTraitAlias, +) -> crate::ItemTraitAlias +where + F: Fold + ?Sized, +{ + crate::ItemTraitAlias { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + trait_token: node.trait_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + eq_token: node.eq_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_type_param_bound), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_type<F>(f: &mut F, node: crate::ItemType) -> crate::ItemType +where + F: Fold + ?Sized, +{ + crate::ItemType { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + type_token: node.type_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + eq_token: node.eq_token, + ty: Box::new(f.fold_type(*node.ty)), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_union<F>(f: &mut F, node: crate::ItemUnion) -> crate::ItemUnion +where + F: Fold + ?Sized, +{ + crate::ItemUnion { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + union_token: node.union_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + fields: f.fold_fields_named(node.fields), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_item_use<F>(f: &mut F, node: crate::ItemUse) -> crate::ItemUse +where + F: Fold + ?Sized, +{ + crate::ItemUse { + attrs: f.fold_attributes(node.attrs), + vis: f.fold_visibility(node.vis), + use_token: node.use_token, + leading_colon: node.leading_colon, + tree: f.fold_use_tree(node.tree), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_label<F>(f: &mut F, node: crate::Label) -> crate::Label +where + F: Fold + ?Sized, +{ + crate::Label { + name: f.fold_lifetime(node.name), + colon_token: node.colon_token, + } +} +pub fn fold_lifetime<F>(f: &mut F, node: crate::Lifetime) -> crate::Lifetime +where + F: Fold + ?Sized, +{ + crate::Lifetime { + apostrophe: f.fold_span(node.apostrophe), + ident: f.fold_ident(node.ident), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_lifetime_param<F>( + f: &mut F, + node: crate::LifetimeParam, +) -> crate::LifetimeParam +where + F: Fold + ?Sized, +{ + crate::LifetimeParam { + attrs: f.fold_attributes(node.attrs), + lifetime: f.fold_lifetime(node.lifetime), + colon_token: node.colon_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_lifetime), + } +} +pub fn fold_lit<F>(f: &mut F, node: crate::Lit) -> crate::Lit +where + F: Fold + ?Sized, +{ + match node { + crate::Lit::Str(_binding_0) => crate::Lit::Str(f.fold_lit_str(_binding_0)), + crate::Lit::ByteStr(_binding_0) => { + crate::Lit::ByteStr(f.fold_lit_byte_str(_binding_0)) + } + crate::Lit::CStr(_binding_0) => crate::Lit::CStr(f.fold_lit_cstr(_binding_0)), + crate::Lit::Byte(_binding_0) => crate::Lit::Byte(f.fold_lit_byte(_binding_0)), + crate::Lit::Char(_binding_0) => crate::Lit::Char(f.fold_lit_char(_binding_0)), + crate::Lit::Int(_binding_0) => crate::Lit::Int(f.fold_lit_int(_binding_0)), + crate::Lit::Float(_binding_0) => crate::Lit::Float(f.fold_lit_float(_binding_0)), + crate::Lit::Bool(_binding_0) => crate::Lit::Bool(f.fold_lit_bool(_binding_0)), + crate::Lit::Verbatim(_binding_0) => crate::Lit::Verbatim(_binding_0), + } +} +pub fn fold_lit_bool<F>(f: &mut F, node: crate::LitBool) -> crate::LitBool +where + F: Fold + ?Sized, +{ + crate::LitBool { + value: node.value, + span: f.fold_span(node.span), + } +} +pub fn fold_lit_byte<F>(f: &mut F, node: crate::LitByte) -> crate::LitByte +where + F: Fold + ?Sized, +{ + let span = f.fold_span(node.span()); + let mut node = node; + node.set_span(span); + node +} +pub fn fold_lit_byte_str<F>(f: &mut F, node: crate::LitByteStr) -> crate::LitByteStr +where + F: Fold + ?Sized, +{ + let span = f.fold_span(node.span()); + let mut node = node; + node.set_span(span); + node +} +pub fn fold_lit_cstr<F>(f: &mut F, node: crate::LitCStr) -> crate::LitCStr +where + F: Fold + ?Sized, +{ + let span = f.fold_span(node.span()); + let mut node = node; + node.set_span(span); + node +} +pub fn fold_lit_char<F>(f: &mut F, node: crate::LitChar) -> crate::LitChar +where + F: Fold + ?Sized, +{ + let span = f.fold_span(node.span()); + let mut node = node; + node.set_span(span); + node +} +pub fn fold_lit_float<F>(f: &mut F, node: crate::LitFloat) -> crate::LitFloat +where + F: Fold + ?Sized, +{ + let span = f.fold_span(node.span()); + let mut node = node; + node.set_span(span); + node +} +pub fn fold_lit_int<F>(f: &mut F, node: crate::LitInt) -> crate::LitInt +where + F: Fold + ?Sized, +{ + let span = f.fold_span(node.span()); + let mut node = node; + node.set_span(span); + node +} +pub fn fold_lit_str<F>(f: &mut F, node: crate::LitStr) -> crate::LitStr +where + F: Fold + ?Sized, +{ + let span = f.fold_span(node.span()); + let mut node = node; + node.set_span(span); + node +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_local<F>(f: &mut F, node: crate::Local) -> crate::Local +where + F: Fold + ?Sized, +{ + crate::Local { + attrs: f.fold_attributes(node.attrs), + let_token: node.let_token, + pat: f.fold_pat(node.pat), + init: (node.init).map(|it| f.fold_local_init(it)), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_local_init<F>(f: &mut F, node: crate::LocalInit) -> crate::LocalInit +where + F: Fold + ?Sized, +{ + crate::LocalInit { + eq_token: node.eq_token, + expr: Box::new(f.fold_expr(*node.expr)), + diverge: (node.diverge).map(|it| ((it).0, Box::new(f.fold_expr(*(it).1)))), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_macro<F>(f: &mut F, node: crate::Macro) -> crate::Macro +where + F: Fold + ?Sized, +{ + crate::Macro { + path: f.fold_path(node.path), + bang_token: node.bang_token, + delimiter: f.fold_macro_delimiter(node.delimiter), + tokens: f.fold_token_stream(node.tokens), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_macro_delimiter<F>( + f: &mut F, + node: crate::MacroDelimiter, +) -> crate::MacroDelimiter +where + F: Fold + ?Sized, +{ + match node { + crate::MacroDelimiter::Paren(_binding_0) => { + crate::MacroDelimiter::Paren(_binding_0) + } + crate::MacroDelimiter::Brace(_binding_0) => { + crate::MacroDelimiter::Brace(_binding_0) + } + crate::MacroDelimiter::Bracket(_binding_0) => { + crate::MacroDelimiter::Bracket(_binding_0) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_member<F>(f: &mut F, node: crate::Member) -> crate::Member +where + F: Fold + ?Sized, +{ + match node { + crate::Member::Named(_binding_0) => { + crate::Member::Named(f.fold_ident(_binding_0)) + } + crate::Member::Unnamed(_binding_0) => { + crate::Member::Unnamed(f.fold_index(_binding_0)) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_meta<F>(f: &mut F, node: crate::Meta) -> crate::Meta +where + F: Fold + ?Sized, +{ + match node { + crate::Meta::Path(_binding_0) => crate::Meta::Path(f.fold_path(_binding_0)), + crate::Meta::List(_binding_0) => crate::Meta::List(f.fold_meta_list(_binding_0)), + crate::Meta::NameValue(_binding_0) => { + crate::Meta::NameValue(f.fold_meta_name_value(_binding_0)) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_meta_list<F>(f: &mut F, node: crate::MetaList) -> crate::MetaList +where + F: Fold + ?Sized, +{ + crate::MetaList { + path: f.fold_path(node.path), + delimiter: f.fold_macro_delimiter(node.delimiter), + tokens: f.fold_token_stream(node.tokens), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_meta_name_value<F>( + f: &mut F, + node: crate::MetaNameValue, +) -> crate::MetaNameValue +where + F: Fold + ?Sized, +{ + crate::MetaNameValue { + path: f.fold_path(node.path), + eq_token: node.eq_token, + value: f.fold_expr(node.value), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_parenthesized_generic_arguments<F>( + f: &mut F, + node: crate::ParenthesizedGenericArguments, +) -> crate::ParenthesizedGenericArguments +where + F: Fold + ?Sized, +{ + crate::ParenthesizedGenericArguments { + paren_token: node.paren_token, + inputs: crate::punctuated::fold(node.inputs, f, F::fold_type), + output: f.fold_return_type(node.output), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat<F>(f: &mut F, node: crate::Pat) -> crate::Pat +where + F: Fold + ?Sized, +{ + match node { + crate::Pat::Const(_binding_0) => crate::Pat::Const(f.fold_expr_const(_binding_0)), + crate::Pat::Ident(_binding_0) => crate::Pat::Ident(f.fold_pat_ident(_binding_0)), + crate::Pat::Lit(_binding_0) => crate::Pat::Lit(f.fold_expr_lit(_binding_0)), + crate::Pat::Macro(_binding_0) => crate::Pat::Macro(f.fold_expr_macro(_binding_0)), + crate::Pat::Or(_binding_0) => crate::Pat::Or(f.fold_pat_or(_binding_0)), + crate::Pat::Paren(_binding_0) => crate::Pat::Paren(f.fold_pat_paren(_binding_0)), + crate::Pat::Path(_binding_0) => crate::Pat::Path(f.fold_expr_path(_binding_0)), + crate::Pat::Range(_binding_0) => crate::Pat::Range(f.fold_expr_range(_binding_0)), + crate::Pat::Reference(_binding_0) => { + crate::Pat::Reference(f.fold_pat_reference(_binding_0)) + } + crate::Pat::Rest(_binding_0) => crate::Pat::Rest(f.fold_pat_rest(_binding_0)), + crate::Pat::Slice(_binding_0) => crate::Pat::Slice(f.fold_pat_slice(_binding_0)), + crate::Pat::Struct(_binding_0) => { + crate::Pat::Struct(f.fold_pat_struct(_binding_0)) + } + crate::Pat::Tuple(_binding_0) => crate::Pat::Tuple(f.fold_pat_tuple(_binding_0)), + crate::Pat::TupleStruct(_binding_0) => { + crate::Pat::TupleStruct(f.fold_pat_tuple_struct(_binding_0)) + } + crate::Pat::Type(_binding_0) => crate::Pat::Type(f.fold_pat_type(_binding_0)), + crate::Pat::Verbatim(_binding_0) => { + crate::Pat::Verbatim(f.fold_token_stream(_binding_0)) + } + crate::Pat::Wild(_binding_0) => crate::Pat::Wild(f.fold_pat_wild(_binding_0)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_ident<F>(f: &mut F, node: crate::PatIdent) -> crate::PatIdent +where + F: Fold + ?Sized, +{ + crate::PatIdent { + attrs: f.fold_attributes(node.attrs), + by_ref: node.by_ref, + mutability: node.mutability, + ident: f.fold_ident(node.ident), + subpat: (node.subpat).map(|it| ((it).0, Box::new(f.fold_pat(*(it).1)))), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_or<F>(f: &mut F, node: crate::PatOr) -> crate::PatOr +where + F: Fold + ?Sized, +{ + crate::PatOr { + attrs: f.fold_attributes(node.attrs), + leading_vert: node.leading_vert, + cases: crate::punctuated::fold(node.cases, f, F::fold_pat), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_paren<F>(f: &mut F, node: crate::PatParen) -> crate::PatParen +where + F: Fold + ?Sized, +{ + crate::PatParen { + attrs: f.fold_attributes(node.attrs), + paren_token: node.paren_token, + pat: Box::new(f.fold_pat(*node.pat)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_reference<F>(f: &mut F, node: crate::PatReference) -> crate::PatReference +where + F: Fold + ?Sized, +{ + crate::PatReference { + attrs: f.fold_attributes(node.attrs), + and_token: node.and_token, + mutability: node.mutability, + pat: Box::new(f.fold_pat(*node.pat)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_rest<F>(f: &mut F, node: crate::PatRest) -> crate::PatRest +where + F: Fold + ?Sized, +{ + crate::PatRest { + attrs: f.fold_attributes(node.attrs), + dot2_token: node.dot2_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_slice<F>(f: &mut F, node: crate::PatSlice) -> crate::PatSlice +where + F: Fold + ?Sized, +{ + crate::PatSlice { + attrs: f.fold_attributes(node.attrs), + bracket_token: node.bracket_token, + elems: crate::punctuated::fold(node.elems, f, F::fold_pat), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_struct<F>(f: &mut F, node: crate::PatStruct) -> crate::PatStruct +where + F: Fold + ?Sized, +{ + crate::PatStruct { + attrs: f.fold_attributes(node.attrs), + qself: (node.qself).map(|it| f.fold_qself(it)), + path: f.fold_path(node.path), + brace_token: node.brace_token, + fields: crate::punctuated::fold(node.fields, f, F::fold_field_pat), + rest: (node.rest).map(|it| f.fold_pat_rest(it)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_tuple<F>(f: &mut F, node: crate::PatTuple) -> crate::PatTuple +where + F: Fold + ?Sized, +{ + crate::PatTuple { + attrs: f.fold_attributes(node.attrs), + paren_token: node.paren_token, + elems: crate::punctuated::fold(node.elems, f, F::fold_pat), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_tuple_struct<F>( + f: &mut F, + node: crate::PatTupleStruct, +) -> crate::PatTupleStruct +where + F: Fold + ?Sized, +{ + crate::PatTupleStruct { + attrs: f.fold_attributes(node.attrs), + qself: (node.qself).map(|it| f.fold_qself(it)), + path: f.fold_path(node.path), + paren_token: node.paren_token, + elems: crate::punctuated::fold(node.elems, f, F::fold_pat), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_type<F>(f: &mut F, node: crate::PatType) -> crate::PatType +where + F: Fold + ?Sized, +{ + crate::PatType { + attrs: f.fold_attributes(node.attrs), + pat: Box::new(f.fold_pat(*node.pat)), + colon_token: node.colon_token, + ty: Box::new(f.fold_type(*node.ty)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pat_wild<F>(f: &mut F, node: crate::PatWild) -> crate::PatWild +where + F: Fold + ?Sized, +{ + crate::PatWild { + attrs: f.fold_attributes(node.attrs), + underscore_token: node.underscore_token, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_path<F>(f: &mut F, node: crate::Path) -> crate::Path +where + F: Fold + ?Sized, +{ + crate::Path { + leading_colon: node.leading_colon, + segments: crate::punctuated::fold(node.segments, f, F::fold_path_segment), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_path_arguments<F>( + f: &mut F, + node: crate::PathArguments, +) -> crate::PathArguments +where + F: Fold + ?Sized, +{ + match node { + crate::PathArguments::None => crate::PathArguments::None, + crate::PathArguments::AngleBracketed(_binding_0) => { + crate::PathArguments::AngleBracketed( + f.fold_angle_bracketed_generic_arguments(_binding_0), + ) + } + crate::PathArguments::Parenthesized(_binding_0) => { + crate::PathArguments::Parenthesized( + f.fold_parenthesized_generic_arguments(_binding_0), + ) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_path_segment<F>(f: &mut F, node: crate::PathSegment) -> crate::PathSegment +where + F: Fold + ?Sized, +{ + crate::PathSegment { + ident: f.fold_ident(node.ident), + arguments: f.fold_path_arguments(node.arguments), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_pointer_mutability<F>( + f: &mut F, + node: crate::PointerMutability, +) -> crate::PointerMutability +where + F: Fold + ?Sized, +{ + match node { + crate::PointerMutability::Const(_binding_0) => { + crate::PointerMutability::Const(_binding_0) + } + crate::PointerMutability::Mut(_binding_0) => { + crate::PointerMutability::Mut(_binding_0) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_precise_capture<F>( + f: &mut F, + node: crate::PreciseCapture, +) -> crate::PreciseCapture +where + F: Fold + ?Sized, +{ + crate::PreciseCapture { + use_token: node.use_token, + lt_token: node.lt_token, + params: crate::punctuated::fold(node.params, f, F::fold_captured_param), + gt_token: node.gt_token, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_predicate_lifetime<F>( + f: &mut F, + node: crate::PredicateLifetime, +) -> crate::PredicateLifetime +where + F: Fold + ?Sized, +{ + crate::PredicateLifetime { + lifetime: f.fold_lifetime(node.lifetime), + colon_token: node.colon_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_lifetime), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_predicate_type<F>( + f: &mut F, + node: crate::PredicateType, +) -> crate::PredicateType +where + F: Fold + ?Sized, +{ + crate::PredicateType { + lifetimes: (node.lifetimes).map(|it| f.fold_bound_lifetimes(it)), + bounded_ty: f.fold_type(node.bounded_ty), + colon_token: node.colon_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_type_param_bound), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_qself<F>(f: &mut F, node: crate::QSelf) -> crate::QSelf +where + F: Fold + ?Sized, +{ + crate::QSelf { + lt_token: node.lt_token, + ty: Box::new(f.fold_type(*node.ty)), + position: node.position, + as_token: node.as_token, + gt_token: node.gt_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_range_limits<F>(f: &mut F, node: crate::RangeLimits) -> crate::RangeLimits +where + F: Fold + ?Sized, +{ + match node { + crate::RangeLimits::HalfOpen(_binding_0) => { + crate::RangeLimits::HalfOpen(_binding_0) + } + crate::RangeLimits::Closed(_binding_0) => crate::RangeLimits::Closed(_binding_0), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_receiver<F>(f: &mut F, node: crate::Receiver) -> crate::Receiver +where + F: Fold + ?Sized, +{ + crate::Receiver { + attrs: f.fold_attributes(node.attrs), + reference: (node.reference) + .map(|it| ((it).0, ((it).1).map(|it| f.fold_lifetime(it)))), + mutability: node.mutability, + self_token: node.self_token, + colon_token: node.colon_token, + ty: Box::new(f.fold_type(*node.ty)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_return_type<F>(f: &mut F, node: crate::ReturnType) -> crate::ReturnType +where + F: Fold + ?Sized, +{ + match node { + crate::ReturnType::Default => crate::ReturnType::Default, + crate::ReturnType::Type(_binding_0, _binding_1) => { + crate::ReturnType::Type(_binding_0, Box::new(f.fold_type(*_binding_1))) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_signature<F>(f: &mut F, node: crate::Signature) -> crate::Signature +where + F: Fold + ?Sized, +{ + crate::Signature { + constness: node.constness, + asyncness: node.asyncness, + unsafety: node.unsafety, + abi: (node.abi).map(|it| f.fold_abi(it)), + fn_token: node.fn_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + paren_token: node.paren_token, + inputs: crate::punctuated::fold(node.inputs, f, F::fold_fn_arg), + variadic: (node.variadic).map(|it| f.fold_variadic(it)), + output: f.fold_return_type(node.output), + } +} +pub fn fold_span<F>(f: &mut F, node: proc_macro2::Span) -> proc_macro2::Span +where + F: Fold + ?Sized, +{ + node +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_static_mutability<F>( + f: &mut F, + node: crate::StaticMutability, +) -> crate::StaticMutability +where + F: Fold + ?Sized, +{ + match node { + crate::StaticMutability::Mut(_binding_0) => { + crate::StaticMutability::Mut(_binding_0) + } + crate::StaticMutability::None => crate::StaticMutability::None, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_stmt<F>(f: &mut F, node: crate::Stmt) -> crate::Stmt +where + F: Fold + ?Sized, +{ + match node { + crate::Stmt::Local(_binding_0) => crate::Stmt::Local(f.fold_local(_binding_0)), + crate::Stmt::Item(_binding_0) => crate::Stmt::Item(f.fold_item(_binding_0)), + crate::Stmt::Expr(_binding_0, _binding_1) => { + crate::Stmt::Expr(f.fold_expr(_binding_0), _binding_1) + } + crate::Stmt::Macro(_binding_0) => { + crate::Stmt::Macro(f.fold_stmt_macro(_binding_0)) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_stmt_macro<F>(f: &mut F, node: crate::StmtMacro) -> crate::StmtMacro +where + F: Fold + ?Sized, +{ + crate::StmtMacro { + attrs: f.fold_attributes(node.attrs), + mac: f.fold_macro(node.mac), + semi_token: node.semi_token, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_trait_bound<F>(f: &mut F, node: crate::TraitBound) -> crate::TraitBound +where + F: Fold + ?Sized, +{ + crate::TraitBound { + paren_token: node.paren_token, + modifier: f.fold_trait_bound_modifier(node.modifier), + lifetimes: (node.lifetimes).map(|it| f.fold_bound_lifetimes(it)), + path: f.fold_path(node.path), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_trait_bound_modifier<F>( + f: &mut F, + node: crate::TraitBoundModifier, +) -> crate::TraitBoundModifier +where + F: Fold + ?Sized, +{ + match node { + crate::TraitBoundModifier::None => crate::TraitBoundModifier::None, + crate::TraitBoundModifier::Maybe(_binding_0) => { + crate::TraitBoundModifier::Maybe(_binding_0) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_trait_item<F>(f: &mut F, node: crate::TraitItem) -> crate::TraitItem +where + F: Fold + ?Sized, +{ + match node { + crate::TraitItem::Const(_binding_0) => { + crate::TraitItem::Const(f.fold_trait_item_const(_binding_0)) + } + crate::TraitItem::Fn(_binding_0) => { + crate::TraitItem::Fn(f.fold_trait_item_fn(_binding_0)) + } + crate::TraitItem::Type(_binding_0) => { + crate::TraitItem::Type(f.fold_trait_item_type(_binding_0)) + } + crate::TraitItem::Macro(_binding_0) => { + crate::TraitItem::Macro(f.fold_trait_item_macro(_binding_0)) + } + crate::TraitItem::Verbatim(_binding_0) => { + crate::TraitItem::Verbatim(f.fold_token_stream(_binding_0)) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_trait_item_const<F>( + f: &mut F, + node: crate::TraitItemConst, +) -> crate::TraitItemConst +where + F: Fold + ?Sized, +{ + crate::TraitItemConst { + attrs: f.fold_attributes(node.attrs), + const_token: node.const_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + colon_token: node.colon_token, + ty: f.fold_type(node.ty), + default: (node.default).map(|it| ((it).0, f.fold_expr((it).1))), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_trait_item_fn<F>(f: &mut F, node: crate::TraitItemFn) -> crate::TraitItemFn +where + F: Fold + ?Sized, +{ + crate::TraitItemFn { + attrs: f.fold_attributes(node.attrs), + sig: f.fold_signature(node.sig), + default: (node.default).map(|it| f.fold_block(it)), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_trait_item_macro<F>( + f: &mut F, + node: crate::TraitItemMacro, +) -> crate::TraitItemMacro +where + F: Fold + ?Sized, +{ + crate::TraitItemMacro { + attrs: f.fold_attributes(node.attrs), + mac: f.fold_macro(node.mac), + semi_token: node.semi_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_trait_item_type<F>( + f: &mut F, + node: crate::TraitItemType, +) -> crate::TraitItemType +where + F: Fold + ?Sized, +{ + crate::TraitItemType { + attrs: f.fold_attributes(node.attrs), + type_token: node.type_token, + ident: f.fold_ident(node.ident), + generics: f.fold_generics(node.generics), + colon_token: node.colon_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_type_param_bound), + default: (node.default).map(|it| ((it).0, f.fold_type((it).1))), + semi_token: node.semi_token, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type<F>(f: &mut F, node: crate::Type) -> crate::Type +where + F: Fold + ?Sized, +{ + match node { + crate::Type::Array(_binding_0) => { + crate::Type::Array(f.fold_type_array(_binding_0)) + } + crate::Type::BareFn(_binding_0) => { + crate::Type::BareFn(f.fold_type_bare_fn(_binding_0)) + } + crate::Type::Group(_binding_0) => { + crate::Type::Group(f.fold_type_group(_binding_0)) + } + crate::Type::ImplTrait(_binding_0) => { + crate::Type::ImplTrait(f.fold_type_impl_trait(_binding_0)) + } + crate::Type::Infer(_binding_0) => { + crate::Type::Infer(f.fold_type_infer(_binding_0)) + } + crate::Type::Macro(_binding_0) => { + crate::Type::Macro(f.fold_type_macro(_binding_0)) + } + crate::Type::Never(_binding_0) => { + crate::Type::Never(f.fold_type_never(_binding_0)) + } + crate::Type::Paren(_binding_0) => { + crate::Type::Paren(f.fold_type_paren(_binding_0)) + } + crate::Type::Path(_binding_0) => crate::Type::Path(f.fold_type_path(_binding_0)), + crate::Type::Ptr(_binding_0) => crate::Type::Ptr(f.fold_type_ptr(_binding_0)), + crate::Type::Reference(_binding_0) => { + crate::Type::Reference(f.fold_type_reference(_binding_0)) + } + crate::Type::Slice(_binding_0) => { + crate::Type::Slice(f.fold_type_slice(_binding_0)) + } + crate::Type::TraitObject(_binding_0) => { + crate::Type::TraitObject(f.fold_type_trait_object(_binding_0)) + } + crate::Type::Tuple(_binding_0) => { + crate::Type::Tuple(f.fold_type_tuple(_binding_0)) + } + crate::Type::Verbatim(_binding_0) => { + crate::Type::Verbatim(f.fold_token_stream(_binding_0)) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_array<F>(f: &mut F, node: crate::TypeArray) -> crate::TypeArray +where + F: Fold + ?Sized, +{ + crate::TypeArray { + bracket_token: node.bracket_token, + elem: Box::new(f.fold_type(*node.elem)), + semi_token: node.semi_token, + len: f.fold_expr(node.len), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_bare_fn<F>(f: &mut F, node: crate::TypeBareFn) -> crate::TypeBareFn +where + F: Fold + ?Sized, +{ + crate::TypeBareFn { + lifetimes: (node.lifetimes).map(|it| f.fold_bound_lifetimes(it)), + unsafety: node.unsafety, + abi: (node.abi).map(|it| f.fold_abi(it)), + fn_token: node.fn_token, + paren_token: node.paren_token, + inputs: crate::punctuated::fold(node.inputs, f, F::fold_bare_fn_arg), + variadic: (node.variadic).map(|it| f.fold_bare_variadic(it)), + output: f.fold_return_type(node.output), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_group<F>(f: &mut F, node: crate::TypeGroup) -> crate::TypeGroup +where + F: Fold + ?Sized, +{ + crate::TypeGroup { + group_token: node.group_token, + elem: Box::new(f.fold_type(*node.elem)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_impl_trait<F>( + f: &mut F, + node: crate::TypeImplTrait, +) -> crate::TypeImplTrait +where + F: Fold + ?Sized, +{ + crate::TypeImplTrait { + impl_token: node.impl_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_type_param_bound), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_infer<F>(f: &mut F, node: crate::TypeInfer) -> crate::TypeInfer +where + F: Fold + ?Sized, +{ + crate::TypeInfer { + underscore_token: node.underscore_token, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_macro<F>(f: &mut F, node: crate::TypeMacro) -> crate::TypeMacro +where + F: Fold + ?Sized, +{ + crate::TypeMacro { + mac: f.fold_macro(node.mac), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_never<F>(f: &mut F, node: crate::TypeNever) -> crate::TypeNever +where + F: Fold + ?Sized, +{ + crate::TypeNever { + bang_token: node.bang_token, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_param<F>(f: &mut F, node: crate::TypeParam) -> crate::TypeParam +where + F: Fold + ?Sized, +{ + crate::TypeParam { + attrs: f.fold_attributes(node.attrs), + ident: f.fold_ident(node.ident), + colon_token: node.colon_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_type_param_bound), + eq_token: node.eq_token, + default: (node.default).map(|it| f.fold_type(it)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_param_bound<F>( + f: &mut F, + node: crate::TypeParamBound, +) -> crate::TypeParamBound +where + F: Fold + ?Sized, +{ + match node { + crate::TypeParamBound::Trait(_binding_0) => { + crate::TypeParamBound::Trait(f.fold_trait_bound(_binding_0)) + } + crate::TypeParamBound::Lifetime(_binding_0) => { + crate::TypeParamBound::Lifetime(f.fold_lifetime(_binding_0)) + } + crate::TypeParamBound::PreciseCapture(_binding_0) => { + crate::TypeParamBound::PreciseCapture( + full!(f.fold_precise_capture(_binding_0)), + ) + } + crate::TypeParamBound::Verbatim(_binding_0) => { + crate::TypeParamBound::Verbatim(f.fold_token_stream(_binding_0)) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_paren<F>(f: &mut F, node: crate::TypeParen) -> crate::TypeParen +where + F: Fold + ?Sized, +{ + crate::TypeParen { + paren_token: node.paren_token, + elem: Box::new(f.fold_type(*node.elem)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_path<F>(f: &mut F, node: crate::TypePath) -> crate::TypePath +where + F: Fold + ?Sized, +{ + crate::TypePath { + qself: (node.qself).map(|it| f.fold_qself(it)), + path: f.fold_path(node.path), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_ptr<F>(f: &mut F, node: crate::TypePtr) -> crate::TypePtr +where + F: Fold + ?Sized, +{ + crate::TypePtr { + star_token: node.star_token, + const_token: node.const_token, + mutability: node.mutability, + elem: Box::new(f.fold_type(*node.elem)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_reference<F>( + f: &mut F, + node: crate::TypeReference, +) -> crate::TypeReference +where + F: Fold + ?Sized, +{ + crate::TypeReference { + and_token: node.and_token, + lifetime: (node.lifetime).map(|it| f.fold_lifetime(it)), + mutability: node.mutability, + elem: Box::new(f.fold_type(*node.elem)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_slice<F>(f: &mut F, node: crate::TypeSlice) -> crate::TypeSlice +where + F: Fold + ?Sized, +{ + crate::TypeSlice { + bracket_token: node.bracket_token, + elem: Box::new(f.fold_type(*node.elem)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_trait_object<F>( + f: &mut F, + node: crate::TypeTraitObject, +) -> crate::TypeTraitObject +where + F: Fold + ?Sized, +{ + crate::TypeTraitObject { + dyn_token: node.dyn_token, + bounds: crate::punctuated::fold(node.bounds, f, F::fold_type_param_bound), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_type_tuple<F>(f: &mut F, node: crate::TypeTuple) -> crate::TypeTuple +where + F: Fold + ?Sized, +{ + crate::TypeTuple { + paren_token: node.paren_token, + elems: crate::punctuated::fold(node.elems, f, F::fold_type), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_un_op<F>(f: &mut F, node: crate::UnOp) -> crate::UnOp +where + F: Fold + ?Sized, +{ + match node { + crate::UnOp::Deref(_binding_0) => crate::UnOp::Deref(_binding_0), + crate::UnOp::Not(_binding_0) => crate::UnOp::Not(_binding_0), + crate::UnOp::Neg(_binding_0) => crate::UnOp::Neg(_binding_0), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_use_glob<F>(f: &mut F, node: crate::UseGlob) -> crate::UseGlob +where + F: Fold + ?Sized, +{ + crate::UseGlob { + star_token: node.star_token, + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_use_group<F>(f: &mut F, node: crate::UseGroup) -> crate::UseGroup +where + F: Fold + ?Sized, +{ + crate::UseGroup { + brace_token: node.brace_token, + items: crate::punctuated::fold(node.items, f, F::fold_use_tree), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_use_name<F>(f: &mut F, node: crate::UseName) -> crate::UseName +where + F: Fold + ?Sized, +{ + crate::UseName { + ident: f.fold_ident(node.ident), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_use_path<F>(f: &mut F, node: crate::UsePath) -> crate::UsePath +where + F: Fold + ?Sized, +{ + crate::UsePath { + ident: f.fold_ident(node.ident), + colon2_token: node.colon2_token, + tree: Box::new(f.fold_use_tree(*node.tree)), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_use_rename<F>(f: &mut F, node: crate::UseRename) -> crate::UseRename +where + F: Fold + ?Sized, +{ + crate::UseRename { + ident: f.fold_ident(node.ident), + as_token: node.as_token, + rename: f.fold_ident(node.rename), + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_use_tree<F>(f: &mut F, node: crate::UseTree) -> crate::UseTree +where + F: Fold + ?Sized, +{ + match node { + crate::UseTree::Path(_binding_0) => { + crate::UseTree::Path(f.fold_use_path(_binding_0)) + } + crate::UseTree::Name(_binding_0) => { + crate::UseTree::Name(f.fold_use_name(_binding_0)) + } + crate::UseTree::Rename(_binding_0) => { + crate::UseTree::Rename(f.fold_use_rename(_binding_0)) + } + crate::UseTree::Glob(_binding_0) => { + crate::UseTree::Glob(f.fold_use_glob(_binding_0)) + } + crate::UseTree::Group(_binding_0) => { + crate::UseTree::Group(f.fold_use_group(_binding_0)) + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn fold_variadic<F>(f: &mut F, node: crate::Variadic) -> crate::Variadic +where + F: Fold + ?Sized, +{ + crate::Variadic { + attrs: f.fold_attributes(node.attrs), + pat: (node.pat).map(|it| (Box::new(f.fold_pat(*(it).0)), (it).1)), + dots: node.dots, + comma: node.comma, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_variant<F>(f: &mut F, node: crate::Variant) -> crate::Variant +where + F: Fold + ?Sized, +{ + crate::Variant { + attrs: f.fold_attributes(node.attrs), + ident: f.fold_ident(node.ident), + fields: f.fold_fields(node.fields), + discriminant: (node.discriminant).map(|it| ((it).0, f.fold_expr((it).1))), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_vis_restricted<F>( + f: &mut F, + node: crate::VisRestricted, +) -> crate::VisRestricted +where + F: Fold + ?Sized, +{ + crate::VisRestricted { + pub_token: node.pub_token, + paren_token: node.paren_token, + in_token: node.in_token, + path: Box::new(f.fold_path(*node.path)), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_visibility<F>(f: &mut F, node: crate::Visibility) -> crate::Visibility +where + F: Fold + ?Sized, +{ + match node { + crate::Visibility::Public(_binding_0) => crate::Visibility::Public(_binding_0), + crate::Visibility::Restricted(_binding_0) => { + crate::Visibility::Restricted(f.fold_vis_restricted(_binding_0)) + } + crate::Visibility::Inherited => crate::Visibility::Inherited, + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_where_clause<F>(f: &mut F, node: crate::WhereClause) -> crate::WhereClause +where + F: Fold + ?Sized, +{ + crate::WhereClause { + where_token: node.where_token, + predicates: crate::punctuated::fold(node.predicates, f, F::fold_where_predicate), + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn fold_where_predicate<F>( + f: &mut F, + node: crate::WherePredicate, +) -> crate::WherePredicate +where + F: Fold + ?Sized, +{ + match node { + crate::WherePredicate::Lifetime(_binding_0) => { + crate::WherePredicate::Lifetime(f.fold_predicate_lifetime(_binding_0)) + } + crate::WherePredicate::Type(_binding_0) => { + crate::WherePredicate::Type(f.fold_predicate_type(_binding_0)) + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +fn fold_vec<T, V, F>(vec: Vec<T>, fold: &mut V, mut f: F) -> Vec<T> +where + V: ?Sized, + F: FnMut(&mut V, T) -> T, +{ + vec.into_iter().map(|it| f(fold, it)).collect() +} diff --git a/third_party/crates/syn-2.0.119/src/gen/hash.rs b/third_party/crates/syn-2.0.119/src/gen/hash.rs new file mode 100644 index 0000000..508af52 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/gen/hash.rs @@ -0,0 +1,2878 @@ +// This file is @generated by syn-internal-codegen. +// It is not intended for manual editing. + +#[cfg(any(feature = "derive", feature = "full"))] +use crate::tt::TokenStreamHelper; +#[cfg(feature = "extra-traits")] +use alloc::string::ToString; +use core::hash::{Hash, Hasher}; +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Abi { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.name.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::AngleBracketedGenericArguments { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.colon2_token.hash(state); + self.args.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Arm { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.pat.hash(state); + self.guard.hash(state); + self.body.hash(state); + self.comma.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::AssocConst { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.ident.hash(state); + self.generics.hash(state); + self.value.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::AssocType { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.ident.hash(state); + self.generics.hash(state); + self.ty.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::AttrStyle { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::AttrStyle::Outer => { + state.write_u8(0u8); + } + crate::AttrStyle::Inner(_) => { + state.write_u8(1u8); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Attribute { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.style.hash(state); + self.meta.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::BareFnArg { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.name.hash(state); + self.ty.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::BareVariadic { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.name.hash(state); + self.comma.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::BinOp { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::BinOp::Add(_) => { + state.write_u8(0u8); + } + crate::BinOp::Sub(_) => { + state.write_u8(1u8); + } + crate::BinOp::Mul(_) => { + state.write_u8(2u8); + } + crate::BinOp::Div(_) => { + state.write_u8(3u8); + } + crate::BinOp::Rem(_) => { + state.write_u8(4u8); + } + crate::BinOp::And(_) => { + state.write_u8(5u8); + } + crate::BinOp::Or(_) => { + state.write_u8(6u8); + } + crate::BinOp::BitXor(_) => { + state.write_u8(7u8); + } + crate::BinOp::BitAnd(_) => { + state.write_u8(8u8); + } + crate::BinOp::BitOr(_) => { + state.write_u8(9u8); + } + crate::BinOp::Shl(_) => { + state.write_u8(10u8); + } + crate::BinOp::Shr(_) => { + state.write_u8(11u8); + } + crate::BinOp::Eq(_) => { + state.write_u8(12u8); + } + crate::BinOp::Lt(_) => { + state.write_u8(13u8); + } + crate::BinOp::Le(_) => { + state.write_u8(14u8); + } + crate::BinOp::Ne(_) => { + state.write_u8(15u8); + } + crate::BinOp::Ge(_) => { + state.write_u8(16u8); + } + crate::BinOp::Gt(_) => { + state.write_u8(17u8); + } + crate::BinOp::AddAssign(_) => { + state.write_u8(18u8); + } + crate::BinOp::SubAssign(_) => { + state.write_u8(19u8); + } + crate::BinOp::MulAssign(_) => { + state.write_u8(20u8); + } + crate::BinOp::DivAssign(_) => { + state.write_u8(21u8); + } + crate::BinOp::RemAssign(_) => { + state.write_u8(22u8); + } + crate::BinOp::BitXorAssign(_) => { + state.write_u8(23u8); + } + crate::BinOp::BitAndAssign(_) => { + state.write_u8(24u8); + } + crate::BinOp::BitOrAssign(_) => { + state.write_u8(25u8); + } + crate::BinOp::ShlAssign(_) => { + state.write_u8(26u8); + } + crate::BinOp::ShrAssign(_) => { + state.write_u8(27u8); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Block { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.stmts.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::BoundLifetimes { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.lifetimes.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::CapturedParam { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::CapturedParam::Lifetime(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::CapturedParam::Ident(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ConstParam { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.ident.hash(state); + self.ty.hash(state); + self.eq_token.hash(state); + self.default.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Constraint { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.ident.hash(state); + self.generics.hash(state); + self.bounds.hash(state); + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Data { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Data::Struct(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::Data::Enum(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Data::Union(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::DataEnum { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.variants.hash(state); + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::DataStruct { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.fields.hash(state); + self.semi_token.hash(state); + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::DataUnion { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.fields.hash(state); + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::DeriveInput { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.data.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Expr { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + #[cfg(feature = "full")] + crate::Expr::Array(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Assign(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Async(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Await(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::Expr::Binary(v0) => { + state.write_u8(4u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Block(v0) => { + state.write_u8(5u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Break(v0) => { + state.write_u8(6u8); + v0.hash(state); + } + crate::Expr::Call(v0) => { + state.write_u8(7u8); + v0.hash(state); + } + crate::Expr::Cast(v0) => { + state.write_u8(8u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Closure(v0) => { + state.write_u8(9u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Const(v0) => { + state.write_u8(10u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Continue(v0) => { + state.write_u8(11u8); + v0.hash(state); + } + crate::Expr::Field(v0) => { + state.write_u8(12u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::ForLoop(v0) => { + state.write_u8(13u8); + v0.hash(state); + } + crate::Expr::Group(v0) => { + state.write_u8(14u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::If(v0) => { + state.write_u8(15u8); + v0.hash(state); + } + crate::Expr::Index(v0) => { + state.write_u8(16u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Infer(v0) => { + state.write_u8(17u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Let(v0) => { + state.write_u8(18u8); + v0.hash(state); + } + crate::Expr::Lit(v0) => { + state.write_u8(19u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Loop(v0) => { + state.write_u8(20u8); + v0.hash(state); + } + crate::Expr::Macro(v0) => { + state.write_u8(21u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Match(v0) => { + state.write_u8(22u8); + v0.hash(state); + } + crate::Expr::MethodCall(v0) => { + state.write_u8(23u8); + v0.hash(state); + } + crate::Expr::Paren(v0) => { + state.write_u8(24u8); + v0.hash(state); + } + crate::Expr::Path(v0) => { + state.write_u8(25u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Range(v0) => { + state.write_u8(26u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::RawAddr(v0) => { + state.write_u8(27u8); + v0.hash(state); + } + crate::Expr::Reference(v0) => { + state.write_u8(28u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Repeat(v0) => { + state.write_u8(29u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Return(v0) => { + state.write_u8(30u8); + v0.hash(state); + } + crate::Expr::Struct(v0) => { + state.write_u8(31u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Try(v0) => { + state.write_u8(32u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::TryBlock(v0) => { + state.write_u8(33u8); + v0.hash(state); + } + crate::Expr::Tuple(v0) => { + state.write_u8(34u8); + v0.hash(state); + } + crate::Expr::Unary(v0) => { + state.write_u8(35u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Unsafe(v0) => { + state.write_u8(36u8); + v0.hash(state); + } + crate::Expr::Verbatim(v0) => { + state.write_u8(37u8); + TokenStreamHelper(v0).hash(state); + } + #[cfg(feature = "full")] + crate::Expr::While(v0) => { + state.write_u8(38u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::Expr::Yield(v0) => { + state.write_u8(39u8); + v0.hash(state); + } + #[cfg(not(feature = "full"))] + _ => unreachable!(), + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprArray { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.elems.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprAssign { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.left.hash(state); + self.right.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprAsync { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.capture.hash(state); + self.block.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprAwait { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.base.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprBinary { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.left.hash(state); + self.op.hash(state); + self.right.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprBlock { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.label.hash(state); + self.block.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprBreak { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.label.hash(state); + self.expr.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprCall { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.func.hash(state); + self.args.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprCast { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + self.ty.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprClosure { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.lifetimes.hash(state); + self.constness.hash(state); + self.movability.hash(state); + self.asyncness.hash(state); + self.capture.hash(state); + self.inputs.hash(state); + self.output.hash(state); + self.body.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprConst { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.block.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprContinue { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.label.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprField { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.base.hash(state); + self.member.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprForLoop { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.label.hash(state); + self.pat.hash(state); + self.expr.hash(state); + self.body.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprGroup { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprIf { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.cond.hash(state); + self.then_branch.hash(state); + self.else_branch.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprIndex { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + self.index.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprInfer { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprLet { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.pat.hash(state); + self.expr.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprLit { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.lit.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprLoop { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.label.hash(state); + self.body.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprMacro { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.mac.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprMatch { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + self.arms.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprMethodCall { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.receiver.hash(state); + self.method.hash(state); + self.turbofish.hash(state); + self.args.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprParen { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprPath { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.qself.hash(state); + self.path.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprRange { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.start.hash(state); + self.limits.hash(state); + self.end.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprRawAddr { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.mutability.hash(state); + self.expr.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprReference { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.mutability.hash(state); + self.expr.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprRepeat { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + self.len.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprReturn { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprStruct { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.qself.hash(state); + self.path.hash(state); + self.fields.hash(state); + self.dot2_token.hash(state); + self.rest.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprTry { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprTryBlock { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.block.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprTuple { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.elems.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprUnary { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.op.hash(state); + self.expr.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprUnsafe { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.block.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprWhile { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.label.hash(state); + self.cond.hash(state); + self.body.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ExprYield { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.expr.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Field { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.mutability.hash(state); + self.ident.hash(state); + self.colon_token.hash(state); + self.ty.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::FieldMutability { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::FieldMutability::None => { + state.write_u8(0u8); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::FieldPat { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.member.hash(state); + self.colon_token.hash(state); + self.pat.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::FieldValue { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.member.hash(state); + self.colon_token.hash(state); + self.expr.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Fields { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Fields::Named(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::Fields::Unnamed(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Fields::Unit => { + state.write_u8(2u8); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::FieldsNamed { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.named.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::FieldsUnnamed { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.unnamed.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::File { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.shebang.hash(state); + self.attrs.hash(state); + self.items.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::FnArg { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::FnArg::Receiver(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::FnArg::Typed(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ForeignItem { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::ForeignItem::Fn(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::ForeignItem::Static(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::ForeignItem::Type(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::ForeignItem::Macro(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::ForeignItem::Verbatim(v0) => { + state.write_u8(4u8); + TokenStreamHelper(v0).hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ForeignItemFn { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.sig.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ForeignItemMacro { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.mac.hash(state); + self.semi_token.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ForeignItemStatic { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.mutability.hash(state); + self.ident.hash(state); + self.ty.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ForeignItemType { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.generics.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::GenericArgument { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::GenericArgument::Lifetime(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::GenericArgument::Type(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::GenericArgument::Const(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::GenericArgument::AssocType(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::GenericArgument::AssocConst(v0) => { + state.write_u8(4u8); + v0.hash(state); + } + crate::GenericArgument::Constraint(v0) => { + state.write_u8(5u8); + v0.hash(state); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::GenericParam { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::GenericParam::Lifetime(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::GenericParam::Type(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::GenericParam::Const(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Generics { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.lt_token.hash(state); + self.params.hash(state); + self.gt_token.hash(state); + self.where_clause.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ImplItem { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::ImplItem::Const(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::ImplItem::Fn(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::ImplItem::Type(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::ImplItem::Macro(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::ImplItem::Verbatim(v0) => { + state.write_u8(4u8); + TokenStreamHelper(v0).hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ImplItemConst { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.defaultness.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.ty.hash(state); + self.expr.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ImplItemFn { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.defaultness.hash(state); + self.sig.hash(state); + self.block.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ImplItemMacro { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.mac.hash(state); + self.semi_token.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ImplItemType { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.defaultness.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.ty.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ImplRestriction { + fn hash<H>(&self, _state: &mut H) + where + H: Hasher, + { + match *self {} + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Item { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Item::Const(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::Item::Enum(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Item::ExternCrate(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::Item::Fn(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::Item::ForeignMod(v0) => { + state.write_u8(4u8); + v0.hash(state); + } + crate::Item::Impl(v0) => { + state.write_u8(5u8); + v0.hash(state); + } + crate::Item::Macro(v0) => { + state.write_u8(6u8); + v0.hash(state); + } + crate::Item::Mod(v0) => { + state.write_u8(7u8); + v0.hash(state); + } + crate::Item::Static(v0) => { + state.write_u8(8u8); + v0.hash(state); + } + crate::Item::Struct(v0) => { + state.write_u8(9u8); + v0.hash(state); + } + crate::Item::Trait(v0) => { + state.write_u8(10u8); + v0.hash(state); + } + crate::Item::TraitAlias(v0) => { + state.write_u8(11u8); + v0.hash(state); + } + crate::Item::Type(v0) => { + state.write_u8(12u8); + v0.hash(state); + } + crate::Item::Union(v0) => { + state.write_u8(13u8); + v0.hash(state); + } + crate::Item::Use(v0) => { + state.write_u8(14u8); + v0.hash(state); + } + crate::Item::Verbatim(v0) => { + state.write_u8(15u8); + TokenStreamHelper(v0).hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemConst { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.ty.hash(state); + self.expr.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemEnum { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.variants.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemExternCrate { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.rename.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemFn { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.sig.hash(state); + self.block.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemForeignMod { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.unsafety.hash(state); + self.abi.hash(state); + self.items.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemImpl { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.defaultness.hash(state); + self.unsafety.hash(state); + self.generics.hash(state); + self.trait_.hash(state); + self.self_ty.hash(state); + self.items.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemMacro { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.ident.hash(state); + self.mac.hash(state); + self.semi_token.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemMod { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.unsafety.hash(state); + self.ident.hash(state); + self.content.hash(state); + self.semi.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemStatic { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.mutability.hash(state); + self.ident.hash(state); + self.ty.hash(state); + self.expr.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemStruct { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.fields.hash(state); + self.semi_token.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemTrait { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.unsafety.hash(state); + self.auto_token.hash(state); + self.restriction.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.colon_token.hash(state); + self.supertraits.hash(state); + self.items.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemTraitAlias { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.bounds.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemType { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.ty.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemUnion { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.fields.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ItemUse { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.vis.hash(state); + self.leading_colon.hash(state); + self.tree.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Label { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.name.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::LifetimeParam { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.lifetime.hash(state); + self.colon_token.hash(state); + self.bounds.hash(state); + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Lit { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Lit::Str(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::Lit::ByteStr(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Lit::CStr(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::Lit::Byte(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::Lit::Char(v0) => { + state.write_u8(4u8); + v0.hash(state); + } + crate::Lit::Int(v0) => { + state.write_u8(5u8); + v0.hash(state); + } + crate::Lit::Float(v0) => { + state.write_u8(6u8); + v0.hash(state); + } + crate::Lit::Bool(v0) => { + state.write_u8(7u8); + v0.hash(state); + } + crate::Lit::Verbatim(v0) => { + state.write_u8(8u8); + v0.to_string().hash(state); + } + } + } +} +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::LitBool { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.value.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Local { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.pat.hash(state); + self.init.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::LocalInit { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.expr.hash(state); + self.diverge.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Macro { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.path.hash(state); + self.delimiter.hash(state); + TokenStreamHelper(&self.tokens).hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::MacroDelimiter { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::MacroDelimiter::Paren(_) => { + state.write_u8(0u8); + } + crate::MacroDelimiter::Brace(_) => { + state.write_u8(1u8); + } + crate::MacroDelimiter::Bracket(_) => { + state.write_u8(2u8); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Meta { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Meta::Path(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::Meta::List(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Meta::NameValue(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::MetaList { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.path.hash(state); + self.delimiter.hash(state); + TokenStreamHelper(&self.tokens).hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::MetaNameValue { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.path.hash(state); + self.value.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ParenthesizedGenericArguments { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.inputs.hash(state); + self.output.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Pat { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Pat::Const(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::Pat::Ident(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Pat::Lit(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::Pat::Macro(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::Pat::Or(v0) => { + state.write_u8(4u8); + v0.hash(state); + } + crate::Pat::Paren(v0) => { + state.write_u8(5u8); + v0.hash(state); + } + crate::Pat::Path(v0) => { + state.write_u8(6u8); + v0.hash(state); + } + crate::Pat::Range(v0) => { + state.write_u8(7u8); + v0.hash(state); + } + crate::Pat::Reference(v0) => { + state.write_u8(8u8); + v0.hash(state); + } + crate::Pat::Rest(v0) => { + state.write_u8(9u8); + v0.hash(state); + } + crate::Pat::Slice(v0) => { + state.write_u8(10u8); + v0.hash(state); + } + crate::Pat::Struct(v0) => { + state.write_u8(11u8); + v0.hash(state); + } + crate::Pat::Tuple(v0) => { + state.write_u8(12u8); + v0.hash(state); + } + crate::Pat::TupleStruct(v0) => { + state.write_u8(13u8); + v0.hash(state); + } + crate::Pat::Type(v0) => { + state.write_u8(14u8); + v0.hash(state); + } + crate::Pat::Verbatim(v0) => { + state.write_u8(15u8); + TokenStreamHelper(v0).hash(state); + } + crate::Pat::Wild(v0) => { + state.write_u8(16u8); + v0.hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatIdent { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.by_ref.hash(state); + self.mutability.hash(state); + self.ident.hash(state); + self.subpat.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatOr { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.leading_vert.hash(state); + self.cases.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatParen { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.pat.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatReference { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.mutability.hash(state); + self.pat.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatRest { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatSlice { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.elems.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatStruct { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.qself.hash(state); + self.path.hash(state); + self.fields.hash(state); + self.rest.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatTuple { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.elems.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatTupleStruct { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.qself.hash(state); + self.path.hash(state); + self.elems.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatType { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.pat.hash(state); + self.ty.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PatWild { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Path { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.leading_colon.hash(state); + self.segments.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PathArguments { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::PathArguments::None => { + state.write_u8(0u8); + } + crate::PathArguments::AngleBracketed(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::PathArguments::Parenthesized(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PathSegment { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.ident.hash(state); + self.arguments.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PointerMutability { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::PointerMutability::Const(_) => { + state.write_u8(0u8); + } + crate::PointerMutability::Mut(_) => { + state.write_u8(1u8); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PreciseCapture { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.params.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PredicateLifetime { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.lifetime.hash(state); + self.bounds.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::PredicateType { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.lifetimes.hash(state); + self.bounded_ty.hash(state); + self.bounds.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::QSelf { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.ty.hash(state); + self.position.hash(state); + self.as_token.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::RangeLimits { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::RangeLimits::HalfOpen(_) => { + state.write_u8(0u8); + } + crate::RangeLimits::Closed(_) => { + state.write_u8(1u8); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Receiver { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.reference.hash(state); + self.mutability.hash(state); + self.colon_token.hash(state); + self.ty.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::ReturnType { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::ReturnType::Default => { + state.write_u8(0u8); + } + crate::ReturnType::Type(_, v1) => { + state.write_u8(1u8); + v1.hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Signature { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.constness.hash(state); + self.asyncness.hash(state); + self.unsafety.hash(state); + self.abi.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.inputs.hash(state); + self.variadic.hash(state); + self.output.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::StaticMutability { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::StaticMutability::Mut(_) => { + state.write_u8(0u8); + } + crate::StaticMutability::None => { + state.write_u8(1u8); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Stmt { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Stmt::Local(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::Stmt::Item(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Stmt::Expr(v0, v1) => { + state.write_u8(2u8); + v0.hash(state); + v1.hash(state); + } + crate::Stmt::Macro(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::StmtMacro { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.mac.hash(state); + self.semi_token.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TraitBound { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.paren_token.hash(state); + self.modifier.hash(state); + self.lifetimes.hash(state); + self.path.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TraitBoundModifier { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::TraitBoundModifier::None => { + state.write_u8(0u8); + } + crate::TraitBoundModifier::Maybe(_) => { + state.write_u8(1u8); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TraitItem { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::TraitItem::Const(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::TraitItem::Fn(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::TraitItem::Type(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::TraitItem::Macro(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::TraitItem::Verbatim(v0) => { + state.write_u8(4u8); + TokenStreamHelper(v0).hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TraitItemConst { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.ty.hash(state); + self.default.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TraitItemFn { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.sig.hash(state); + self.default.hash(state); + self.semi_token.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TraitItemMacro { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.mac.hash(state); + self.semi_token.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TraitItemType { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.ident.hash(state); + self.generics.hash(state); + self.colon_token.hash(state); + self.bounds.hash(state); + self.default.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Type { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Type::Array(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::Type::BareFn(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Type::Group(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::Type::ImplTrait(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::Type::Infer(v0) => { + state.write_u8(4u8); + v0.hash(state); + } + crate::Type::Macro(v0) => { + state.write_u8(5u8); + v0.hash(state); + } + crate::Type::Never(v0) => { + state.write_u8(6u8); + v0.hash(state); + } + crate::Type::Paren(v0) => { + state.write_u8(7u8); + v0.hash(state); + } + crate::Type::Path(v0) => { + state.write_u8(8u8); + v0.hash(state); + } + crate::Type::Ptr(v0) => { + state.write_u8(9u8); + v0.hash(state); + } + crate::Type::Reference(v0) => { + state.write_u8(10u8); + v0.hash(state); + } + crate::Type::Slice(v0) => { + state.write_u8(11u8); + v0.hash(state); + } + crate::Type::TraitObject(v0) => { + state.write_u8(12u8); + v0.hash(state); + } + crate::Type::Tuple(v0) => { + state.write_u8(13u8); + v0.hash(state); + } + crate::Type::Verbatim(v0) => { + state.write_u8(14u8); + TokenStreamHelper(v0).hash(state); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeArray { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.elem.hash(state); + self.len.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeBareFn { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.lifetimes.hash(state); + self.unsafety.hash(state); + self.abi.hash(state); + self.inputs.hash(state); + self.variadic.hash(state); + self.output.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeGroup { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.elem.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeImplTrait { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.bounds.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeInfer { + fn hash<H>(&self, _state: &mut H) + where + H: Hasher, + {} +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeMacro { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.mac.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeNever { + fn hash<H>(&self, _state: &mut H) + where + H: Hasher, + {} +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeParam { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.ident.hash(state); + self.colon_token.hash(state); + self.bounds.hash(state); + self.eq_token.hash(state); + self.default.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeParamBound { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::TypeParamBound::Trait(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::TypeParamBound::Lifetime(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + #[cfg(feature = "full")] + crate::TypeParamBound::PreciseCapture(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::TypeParamBound::Verbatim(v0) => { + state.write_u8(3u8); + TokenStreamHelper(v0).hash(state); + } + #[cfg(not(feature = "full"))] + _ => unreachable!(), + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeParen { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.elem.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypePath { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.qself.hash(state); + self.path.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypePtr { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.const_token.hash(state); + self.mutability.hash(state); + self.elem.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeReference { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.lifetime.hash(state); + self.mutability.hash(state); + self.elem.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeSlice { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.elem.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeTraitObject { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.dyn_token.hash(state); + self.bounds.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::TypeTuple { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.elems.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::UnOp { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::UnOp::Deref(_) => { + state.write_u8(0u8); + } + crate::UnOp::Not(_) => { + state.write_u8(1u8); + } + crate::UnOp::Neg(_) => { + state.write_u8(2u8); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::UseGlob { + fn hash<H>(&self, _state: &mut H) + where + H: Hasher, + {} +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::UseGroup { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.items.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::UseName { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.ident.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::UsePath { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.ident.hash(state); + self.tree.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::UseRename { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.ident.hash(state); + self.rename.hash(state); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::UseTree { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::UseTree::Path(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::UseTree::Name(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::UseTree::Rename(v0) => { + state.write_u8(2u8); + v0.hash(state); + } + crate::UseTree::Glob(v0) => { + state.write_u8(3u8); + v0.hash(state); + } + crate::UseTree::Group(v0) => { + state.write_u8(4u8); + v0.hash(state); + } + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Variadic { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.pat.hash(state); + self.comma.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Variant { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.attrs.hash(state); + self.ident.hash(state); + self.fields.hash(state); + self.discriminant.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::VisRestricted { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.in_token.hash(state); + self.path.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::Visibility { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::Visibility::Public(_) => { + state.write_u8(0u8); + } + crate::Visibility::Restricted(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + crate::Visibility::Inherited => { + state.write_u8(2u8); + } + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::WhereClause { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.predicates.hash(state); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for crate::WherePredicate { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + match self { + crate::WherePredicate::Lifetime(v0) => { + state.write_u8(0u8); + v0.hash(state); + } + crate::WherePredicate::Type(v0) => { + state.write_u8(1u8); + v0.hash(state); + } + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/gen/token.css b/third_party/crates/syn-2.0.119/src/gen/token.css new file mode 100644 index 0000000..ed82ae1 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/gen/token.css @@ -0,0 +1,737 @@ +a.struct[title="struct syn::token::Abstract"], +a.struct[title="struct syn::token::And"], +a.struct[title="struct syn::token::AndAnd"], +a.struct[title="struct syn::token::AndEq"], +a.struct[title="struct syn::token::As"], +a.struct[title="struct syn::token::Async"], +a.struct[title="struct syn::token::At"], +a.struct[title="struct syn::token::Auto"], +a.struct[title="struct syn::token::Await"], +a.struct[title="struct syn::token::Become"], +a.struct[title="struct syn::token::Box"], +a.struct[title="struct syn::token::Break"], +a.struct[title="struct syn::token::Caret"], +a.struct[title="struct syn::token::CaretEq"], +a.struct[title="struct syn::token::Colon"], +a.struct[title="struct syn::token::Comma"], +a.struct[title="struct syn::token::Const"], +a.struct[title="struct syn::token::Continue"], +a.struct[title="struct syn::token::Crate"], +a.struct[title="struct syn::token::Default"], +a.struct[title="struct syn::token::Do"], +a.struct[title="struct syn::token::Dollar"], +a.struct[title="struct syn::token::Dot"], +a.struct[title="struct syn::token::DotDot"], +a.struct[title="struct syn::token::DotDotDot"], +a.struct[title="struct syn::token::DotDotEq"], +a.struct[title="struct syn::token::Dyn"], +a.struct[title="struct syn::token::Else"], +a.struct[title="struct syn::token::Enum"], +a.struct[title="struct syn::token::Eq"], +a.struct[title="struct syn::token::EqEq"], +a.struct[title="struct syn::token::Extern"], +a.struct[title="struct syn::token::FatArrow"], +a.struct[title="struct syn::token::Final"], +a.struct[title="struct syn::token::Fn"], +a.struct[title="struct syn::token::For"], +a.struct[title="struct syn::token::Ge"], +a.struct[title="struct syn::token::Gt"], +a.struct[title="struct syn::token::If"], +a.struct[title="struct syn::token::Impl"], +a.struct[title="struct syn::token::In"], +a.struct[title="struct syn::token::LArrow"], +a.struct[title="struct syn::token::Le"], +a.struct[title="struct syn::token::Let"], +a.struct[title="struct syn::token::Loop"], +a.struct[title="struct syn::token::Lt"], +a.struct[title="struct syn::token::Macro"], +a.struct[title="struct syn::token::Match"], +a.struct[title="struct syn::token::Minus"], +a.struct[title="struct syn::token::MinusEq"], +a.struct[title="struct syn::token::Mod"], +a.struct[title="struct syn::token::Move"], +a.struct[title="struct syn::token::Mut"], +a.struct[title="struct syn::token::Ne"], +a.struct[title="struct syn::token::Not"], +a.struct[title="struct syn::token::Or"], +a.struct[title="struct syn::token::OrEq"], +a.struct[title="struct syn::token::OrOr"], +a.struct[title="struct syn::token::Override"], +a.struct[title="struct syn::token::PathSep"], +a.struct[title="struct syn::token::Percent"], +a.struct[title="struct syn::token::PercentEq"], +a.struct[title="struct syn::token::Plus"], +a.struct[title="struct syn::token::PlusEq"], +a.struct[title="struct syn::token::Pound"], +a.struct[title="struct syn::token::Priv"], +a.struct[title="struct syn::token::Pub"], +a.struct[title="struct syn::token::Question"], +a.struct[title="struct syn::token::RArrow"], +a.struct[title="struct syn::token::Raw"], +a.struct[title="struct syn::token::Ref"], +a.struct[title="struct syn::token::Return"], +a.struct[title="struct syn::token::SelfType"], +a.struct[title="struct syn::token::SelfValue"], +a.struct[title="struct syn::token::Semi"], +a.struct[title="struct syn::token::Shl"], +a.struct[title="struct syn::token::ShlEq"], +a.struct[title="struct syn::token::Shr"], +a.struct[title="struct syn::token::ShrEq"], +a.struct[title="struct syn::token::Slash"], +a.struct[title="struct syn::token::SlashEq"], +a.struct[title="struct syn::token::Star"], +a.struct[title="struct syn::token::StarEq"], +a.struct[title="struct syn::token::Static"], +a.struct[title="struct syn::token::Struct"], +a.struct[title="struct syn::token::Super"], +a.struct[title="struct syn::token::Tilde"], +a.struct[title="struct syn::token::Trait"], +a.struct[title="struct syn::token::Try"], +a.struct[title="struct syn::token::Type"], +a.struct[title="struct syn::token::Typeof"], +a.struct[title="struct syn::token::Underscore"], +a.struct[title="struct syn::token::Union"], +a.struct[title="struct syn::token::Unsafe"], +a.struct[title="struct syn::token::Unsized"], +a.struct[title="struct syn::token::Use"], +a.struct[title="struct syn::token::Virtual"], +a.struct[title="struct syn::token::Where"], +a.struct[title="struct syn::token::While"], +a.struct[title="struct syn::token::Yield"] { + display: inline-block; + color: transparent; + white-space: nowrap; +} + +a.struct[title="struct syn::token::Abstract"]::before, +a.struct[title="struct syn::token::And"]::before, +a.struct[title="struct syn::token::AndAnd"]::before, +a.struct[title="struct syn::token::AndEq"]::before, +a.struct[title="struct syn::token::As"]::before, +a.struct[title="struct syn::token::Async"]::before, +a.struct[title="struct syn::token::At"]::before, +a.struct[title="struct syn::token::Auto"]::before, +a.struct[title="struct syn::token::Await"]::before, +a.struct[title="struct syn::token::Become"]::before, +a.struct[title="struct syn::token::Box"]::before, +a.struct[title="struct syn::token::Break"]::before, +a.struct[title="struct syn::token::Caret"]::before, +a.struct[title="struct syn::token::CaretEq"]::before, +a.struct[title="struct syn::token::Colon"]::before, +a.struct[title="struct syn::token::Comma"]::before, +a.struct[title="struct syn::token::Const"]::before, +a.struct[title="struct syn::token::Continue"]::before, +a.struct[title="struct syn::token::Crate"]::before, +a.struct[title="struct syn::token::Default"]::before, +a.struct[title="struct syn::token::Do"]::before, +a.struct[title="struct syn::token::Dollar"]::before, +a.struct[title="struct syn::token::Dot"]::before, +a.struct[title="struct syn::token::DotDot"]::before, +a.struct[title="struct syn::token::DotDotDot"]::before, +a.struct[title="struct syn::token::DotDotEq"]::before, +a.struct[title="struct syn::token::Dyn"]::before, +a.struct[title="struct syn::token::Else"]::before, +a.struct[title="struct syn::token::Enum"]::before, +a.struct[title="struct syn::token::Eq"]::before, +a.struct[title="struct syn::token::EqEq"]::before, +a.struct[title="struct syn::token::Extern"]::before, +a.struct[title="struct syn::token::FatArrow"]::before, +a.struct[title="struct syn::token::Final"]::before, +a.struct[title="struct syn::token::Fn"]::before, +a.struct[title="struct syn::token::For"]::before, +a.struct[title="struct syn::token::Ge"]::before, +a.struct[title="struct syn::token::Gt"]::before, +a.struct[title="struct syn::token::If"]::before, +a.struct[title="struct syn::token::Impl"]::before, +a.struct[title="struct syn::token::In"]::before, +a.struct[title="struct syn::token::LArrow"]::before, +a.struct[title="struct syn::token::Le"]::before, +a.struct[title="struct syn::token::Let"]::before, +a.struct[title="struct syn::token::Loop"]::before, +a.struct[title="struct syn::token::Lt"]::before, +a.struct[title="struct syn::token::Macro"]::before, +a.struct[title="struct syn::token::Match"]::before, +a.struct[title="struct syn::token::Minus"]::before, +a.struct[title="struct syn::token::MinusEq"]::before, +a.struct[title="struct syn::token::Mod"]::before, +a.struct[title="struct syn::token::Move"]::before, +a.struct[title="struct syn::token::Mut"]::before, +a.struct[title="struct syn::token::Ne"]::before, +a.struct[title="struct syn::token::Not"]::before, +a.struct[title="struct syn::token::Or"]::before, +a.struct[title="struct syn::token::OrEq"]::before, +a.struct[title="struct syn::token::OrOr"]::before, +a.struct[title="struct syn::token::Override"]::before, +a.struct[title="struct syn::token::PathSep"]::before, +a.struct[title="struct syn::token::Percent"]::before, +a.struct[title="struct syn::token::PercentEq"]::before, +a.struct[title="struct syn::token::Plus"]::before, +a.struct[title="struct syn::token::PlusEq"]::before, +a.struct[title="struct syn::token::Pound"]::before, +a.struct[title="struct syn::token::Priv"]::before, +a.struct[title="struct syn::token::Pub"]::before, +a.struct[title="struct syn::token::Question"]::before, +a.struct[title="struct syn::token::RArrow"]::before, +a.struct[title="struct syn::token::Raw"]::before, +a.struct[title="struct syn::token::Ref"]::before, +a.struct[title="struct syn::token::Return"]::before, +a.struct[title="struct syn::token::SelfType"]::before, +a.struct[title="struct syn::token::SelfValue"]::before, +a.struct[title="struct syn::token::Semi"]::before, +a.struct[title="struct syn::token::Shl"]::before, +a.struct[title="struct syn::token::ShlEq"]::before, +a.struct[title="struct syn::token::Shr"]::before, +a.struct[title="struct syn::token::ShrEq"]::before, +a.struct[title="struct syn::token::Slash"]::before, +a.struct[title="struct syn::token::SlashEq"]::before, +a.struct[title="struct syn::token::Star"]::before, +a.struct[title="struct syn::token::StarEq"]::before, +a.struct[title="struct syn::token::Static"]::before, +a.struct[title="struct syn::token::Struct"]::before, +a.struct[title="struct syn::token::Super"]::before, +a.struct[title="struct syn::token::Tilde"]::before, +a.struct[title="struct syn::token::Trait"]::before, +a.struct[title="struct syn::token::Try"]::before, +a.struct[title="struct syn::token::Type"]::before, +a.struct[title="struct syn::token::Typeof"]::before, +a.struct[title="struct syn::token::Underscore"]::before, +a.struct[title="struct syn::token::Union"]::before, +a.struct[title="struct syn::token::Unsafe"]::before, +a.struct[title="struct syn::token::Unsized"]::before, +a.struct[title="struct syn::token::Use"]::before, +a.struct[title="struct syn::token::Virtual"]::before, +a.struct[title="struct syn::token::Where"]::before, +a.struct[title="struct syn::token::While"]::before, +a.struct[title="struct syn::token::Yield"]::before { + display: inline-block; + color: var(--type-link-color); + width: 0; +} + +a.struct[title="struct syn::token::Abstract"]::before { + content: "Token![abstract]"; +} + +a.struct[title="struct syn::token::And"]::before { + content: "Token![&]"; +} + +a.struct[title="struct syn::token::AndAnd"]::before { + content: "Token![&&]"; +} + +a.struct[title="struct syn::token::AndEq"]::before { + content: "Token![&=]"; +} + +a.struct[title="struct syn::token::As"]::before { + content: "Token![as]"; +} + +a.struct[title="struct syn::token::Async"]::before { + content: "Token![async]"; +} + +a.struct[title="struct syn::token::At"]::before { + content: "Token![@]"; +} + +a.struct[title="struct syn::token::Auto"]::before { + content: "Token![auto]"; +} + +a.struct[title="struct syn::token::Await"]::before { + content: "Token![await]"; +} + +a.struct[title="struct syn::token::Become"]::before { + content: "Token![become]"; +} + +a.struct[title="struct syn::token::Box"]::before { + content: "Token![box]"; +} + +a.struct[title="struct syn::token::Break"]::before { + content: "Token![break]"; +} + +a.struct[title="struct syn::token::Caret"]::before { + content: "Token![^]"; +} + +a.struct[title="struct syn::token::CaretEq"]::before { + content: "Token![^=]"; +} + +a.struct[title="struct syn::token::Colon"]::before { + content: "Token![:]"; +} + +a.struct[title="struct syn::token::Comma"]::before { + content: "Token![,]"; +} + +a.struct[title="struct syn::token::Const"]::before { + content: "Token![const]"; +} + +a.struct[title="struct syn::token::Continue"]::before { + content: "Token![continue]"; +} + +a.struct[title="struct syn::token::Crate"]::before { + content: "Token![crate]"; +} + +a.struct[title="struct syn::token::Default"]::before { + content: "Token![default]"; +} + +a.struct[title="struct syn::token::Do"]::before { + content: "Token![do]"; +} + +a.struct[title="struct syn::token::Dollar"]::before { + content: "Token![$]"; +} + +a.struct[title="struct syn::token::Dot"]::before { + content: "Token![.]"; +} + +a.struct[title="struct syn::token::DotDot"]::before { + content: "Token![..]"; +} + +a.struct[title="struct syn::token::DotDotDot"]::before { + content: "Token![...]"; +} + +a.struct[title="struct syn::token::DotDotEq"]::before { + content: "Token![..=]"; +} + +a.struct[title="struct syn::token::Dyn"]::before { + content: "Token![dyn]"; +} + +a.struct[title="struct syn::token::Else"]::before { + content: "Token![else]"; +} + +a.struct[title="struct syn::token::Enum"]::before { + content: "Token![enum]"; +} + +a.struct[title="struct syn::token::Eq"]::before { + content: "Token![=]"; +} + +a.struct[title="struct syn::token::EqEq"]::before { + content: "Token![==]"; +} + +a.struct[title="struct syn::token::Extern"]::before { + content: "Token![extern]"; +} + +a.struct[title="struct syn::token::FatArrow"]::before { + content: "Token![=>]"; +} + +a.struct[title="struct syn::token::Final"]::before { + content: "Token![final]"; +} + +a.struct[title="struct syn::token::Fn"]::before { + content: "Token![fn]"; +} + +a.struct[title="struct syn::token::For"]::before { + content: "Token![for]"; +} + +a.struct[title="struct syn::token::Ge"]::before { + content: "Token![>=]"; +} + +a.struct[title="struct syn::token::Gt"]::before { + content: "Token![>]"; +} + +a.struct[title="struct syn::token::If"]::before { + content: "Token![if]"; +} + +a.struct[title="struct syn::token::Impl"]::before { + content: "Token![impl]"; +} + +a.struct[title="struct syn::token::In"]::before { + content: "Token![in]"; +} + +a.struct[title="struct syn::token::LArrow"]::before { + content: "Token![<-]"; +} + +a.struct[title="struct syn::token::Le"]::before { + content: "Token![<=]"; +} + +a.struct[title="struct syn::token::Let"]::before { + content: "Token![let]"; +} + +a.struct[title="struct syn::token::Loop"]::before { + content: "Token![loop]"; +} + +a.struct[title="struct syn::token::Lt"]::before { + content: "Token![<]"; +} + +a.struct[title="struct syn::token::Macro"]::before { + content: "Token![macro]"; +} + +a.struct[title="struct syn::token::Match"]::before { + content: "Token![match]"; +} + +a.struct[title="struct syn::token::Minus"]::before { + content: "Token![-]"; +} + +a.struct[title="struct syn::token::MinusEq"]::before { + content: "Token![-=]"; +} + +a.struct[title="struct syn::token::Mod"]::before { + content: "Token![mod]"; +} + +a.struct[title="struct syn::token::Move"]::before { + content: "Token![move]"; +} + +a.struct[title="struct syn::token::Mut"]::before { + content: "Token![mut]"; +} + +a.struct[title="struct syn::token::Ne"]::before { + content: "Token![!=]"; +} + +a.struct[title="struct syn::token::Not"]::before { + content: "Token![!]"; +} + +a.struct[title="struct syn::token::Or"]::before { + content: "Token![|]"; +} + +a.struct[title="struct syn::token::OrEq"]::before { + content: "Token![|=]"; +} + +a.struct[title="struct syn::token::OrOr"]::before { + content: "Token![||]"; +} + +a.struct[title="struct syn::token::Override"]::before { + content: "Token![override]"; +} + +a.struct[title="struct syn::token::PathSep"]::before { + content: "Token![::]"; +} + +a.struct[title="struct syn::token::Percent"]::before { + content: "Token![%]"; +} + +a.struct[title="struct syn::token::PercentEq"]::before { + content: "Token![%=]"; +} + +a.struct[title="struct syn::token::Plus"]::before { + content: "Token![+]"; +} + +a.struct[title="struct syn::token::PlusEq"]::before { + content: "Token![+=]"; +} + +a.struct[title="struct syn::token::Pound"]::before { + content: "Token![#]"; +} + +a.struct[title="struct syn::token::Priv"]::before { + content: "Token![priv]"; +} + +a.struct[title="struct syn::token::Pub"]::before { + content: "Token![pub]"; +} + +a.struct[title="struct syn::token::Question"]::before { + content: "Token![?]"; +} + +a.struct[title="struct syn::token::RArrow"]::before { + content: "Token![->]"; +} + +a.struct[title="struct syn::token::Raw"]::before { + content: "Token![raw]"; +} + +a.struct[title="struct syn::token::Ref"]::before { + content: "Token![ref]"; +} + +a.struct[title="struct syn::token::Return"]::before { + content: "Token![return]"; +} + +a.struct[title="struct syn::token::SelfType"]::before { + content: "Token![Self]"; +} + +a.struct[title="struct syn::token::SelfValue"]::before { + content: "Token![self]"; +} + +a.struct[title="struct syn::token::Semi"]::before { + content: "Token![;]"; +} + +a.struct[title="struct syn::token::Shl"]::before { + content: "Token![<<]"; +} + +a.struct[title="struct syn::token::ShlEq"]::before { + content: "Token![<<=]"; +} + +a.struct[title="struct syn::token::Shr"]::before { + content: "Token![>>]"; +} + +a.struct[title="struct syn::token::ShrEq"]::before { + content: "Token![>>=]"; +} + +a.struct[title="struct syn::token::Slash"]::before { + content: "Token![/]"; +} + +a.struct[title="struct syn::token::SlashEq"]::before { + content: "Token![/=]"; +} + +a.struct[title="struct syn::token::Star"]::before { + content: "Token![*]"; +} + +a.struct[title="struct syn::token::StarEq"]::before { + content: "Token![*=]"; +} + +a.struct[title="struct syn::token::Static"]::before { + content: "Token![static]"; +} + +a.struct[title="struct syn::token::Struct"]::before { + content: "Token![struct]"; +} + +a.struct[title="struct syn::token::Super"]::before { + content: "Token![super]"; +} + +a.struct[title="struct syn::token::Tilde"]::before { + content: "Token![~]"; +} + +a.struct[title="struct syn::token::Trait"]::before { + content: "Token![trait]"; +} + +a.struct[title="struct syn::token::Try"]::before { + content: "Token![try]"; +} + +a.struct[title="struct syn::token::Type"]::before { + content: "Token![type]"; +} + +a.struct[title="struct syn::token::Typeof"]::before { + content: "Token![typeof]"; +} + +a.struct[title="struct syn::token::Underscore"]::before { + content: "Token![_]"; + font-size: calc(100% * 10 / 9); +} + +a.struct[title="struct syn::token::Union"]::before { + content: "Token![union]"; +} + +a.struct[title="struct syn::token::Unsafe"]::before { + content: "Token![unsafe]"; +} + +a.struct[title="struct syn::token::Unsized"]::before { + content: "Token![unsized]"; +} + +a.struct[title="struct syn::token::Use"]::before { + content: "Token![use]"; +} + +a.struct[title="struct syn::token::Virtual"]::before { + content: "Token![virtual]"; +} + +a.struct[title="struct syn::token::Where"]::before { + content: "Token![where]"; +} + +a.struct[title="struct syn::token::While"]::before { + content: "Token![while]"; +} + +a.struct[title="struct syn::token::Yield"]::before { + content: "Token![yield]"; +} + +a.struct[title="struct syn::token::Underscore"] { + font-size: calc(100% * 9 / 10); +} + +a.struct[title="struct syn::token::PercentEq"]::after, +a.struct[title="struct syn::token::Question"]::after { + content: "."; +} + +a.struct[title="struct syn::token::DotDotDot"]::after, +a.struct[title="struct syn::token::FatArrow"]::after, +a.struct[title="struct syn::token::Percent"]::after { + content: ".."; +} + +a.struct[title="struct syn::token::CaretEq"]::after, +a.struct[title="struct syn::token::Dollar"]::after, +a.struct[title="struct syn::token::DotDotEq"]::after, +a.struct[title="struct syn::token::MinusEq"]::after, +a.struct[title="struct syn::token::PathSep"]::after, +a.struct[title="struct syn::token::SelfValue"]::after, +a.struct[title="struct syn::token::SlashEq"]::after { + content: "..."; +} + +a.struct[title="struct syn::token::AndAnd"]::after, +a.struct[title="struct syn::token::Caret"]::after, +a.struct[title="struct syn::token::Colon"]::after, +a.struct[title="struct syn::token::Comma"]::after, +a.struct[title="struct syn::token::DotDot"]::after, +a.struct[title="struct syn::token::LArrow"]::after, +a.struct[title="struct syn::token::Minus"]::after, +a.struct[title="struct syn::token::PlusEq"]::after, +a.struct[title="struct syn::token::Pound"]::after, +a.struct[title="struct syn::token::RArrow"]::after, +a.struct[title="struct syn::token::SelfType"]::after, +a.struct[title="struct syn::token::Slash"]::after, +a.struct[title="struct syn::token::StarEq"]::after, +a.struct[title="struct syn::token::Tilde"]::after { + content: "...."; +} + +a.struct[title="struct syn::token::AndEq"]::after, +a.struct[title="struct syn::token::Plus"]::after, +a.struct[title="struct syn::token::Semi"]::after, +a.struct[title="struct syn::token::Star"]::after { + content: "....."; +} + +a.struct[title="struct syn::token::And"]::after, +a.struct[title="struct syn::token::Dot"]::after, +a.struct[title="struct syn::token::EqEq"]::after, +a.struct[title="struct syn::token::Not"]::after, +a.struct[title="struct syn::token::OrEq"]::after, +a.struct[title="struct syn::token::OrOr"]::after, +a.struct[title="struct syn::token::ShlEq"]::after, +a.struct[title="struct syn::token::ShrEq"]::after { + content: "......"; +} + +a.struct[title="struct syn::token::At"]::after, +a.struct[title="struct syn::token::Eq"]::after, +a.struct[title="struct syn::token::Gt"]::after, +a.struct[title="struct syn::token::Lt"]::after, +a.struct[title="struct syn::token::Or"]::after, +a.struct[title="struct syn::token::Shl"]::after, +a.struct[title="struct syn::token::Shr"]::after { + content: "......."; +} + +a.struct[title="struct syn::token::Abstract"]::after, +a.struct[title="struct syn::token::As"]::after, +a.struct[title="struct syn::token::Async"]::after, +a.struct[title="struct syn::token::Auto"]::after, +a.struct[title="struct syn::token::Await"]::after, +a.struct[title="struct syn::token::Become"]::after, +a.struct[title="struct syn::token::Box"]::after, +a.struct[title="struct syn::token::Break"]::after, +a.struct[title="struct syn::token::Const"]::after, +a.struct[title="struct syn::token::Continue"]::after, +a.struct[title="struct syn::token::Crate"]::after, +a.struct[title="struct syn::token::Default"]::after, +a.struct[title="struct syn::token::Do"]::after, +a.struct[title="struct syn::token::Dyn"]::after, +a.struct[title="struct syn::token::Else"]::after, +a.struct[title="struct syn::token::Enum"]::after, +a.struct[title="struct syn::token::Extern"]::after, +a.struct[title="struct syn::token::Final"]::after, +a.struct[title="struct syn::token::Fn"]::after, +a.struct[title="struct syn::token::For"]::after, +a.struct[title="struct syn::token::Ge"]::after, +a.struct[title="struct syn::token::If"]::after, +a.struct[title="struct syn::token::Impl"]::after, +a.struct[title="struct syn::token::In"]::after, +a.struct[title="struct syn::token::Le"]::after, +a.struct[title="struct syn::token::Let"]::after, +a.struct[title="struct syn::token::Loop"]::after, +a.struct[title="struct syn::token::Macro"]::after, +a.struct[title="struct syn::token::Match"]::after, +a.struct[title="struct syn::token::Mod"]::after, +a.struct[title="struct syn::token::Move"]::after, +a.struct[title="struct syn::token::Mut"]::after, +a.struct[title="struct syn::token::Ne"]::after, +a.struct[title="struct syn::token::Override"]::after, +a.struct[title="struct syn::token::Priv"]::after, +a.struct[title="struct syn::token::Pub"]::after, +a.struct[title="struct syn::token::Raw"]::after, +a.struct[title="struct syn::token::Ref"]::after, +a.struct[title="struct syn::token::Return"]::after, +a.struct[title="struct syn::token::Static"]::after, +a.struct[title="struct syn::token::Struct"]::after, +a.struct[title="struct syn::token::Super"]::after, +a.struct[title="struct syn::token::Trait"]::after, +a.struct[title="struct syn::token::Try"]::after, +a.struct[title="struct syn::token::Type"]::after, +a.struct[title="struct syn::token::Typeof"]::after, +a.struct[title="struct syn::token::Union"]::after, +a.struct[title="struct syn::token::Unsafe"]::after, +a.struct[title="struct syn::token::Unsized"]::after, +a.struct[title="struct syn::token::Use"]::after, +a.struct[title="struct syn::token::Virtual"]::after, +a.struct[title="struct syn::token::Where"]::after, +a.struct[title="struct syn::token::While"]::after, +a.struct[title="struct syn::token::Yield"]::after { + content: "........"; +} diff --git a/third_party/crates/syn-2.0.119/src/gen/visit.rs b/third_party/crates/syn-2.0.119/src/gen/visit.rs new file mode 100644 index 0000000..cd258fc --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/gen/visit.rs @@ -0,0 +1,3941 @@ +// This file is @generated by syn-internal-codegen. +// It is not intended for manual editing. + +#![allow(unused_variables)] +#![allow(clippy::needless_pass_by_ref_mut)] +#[cfg(any(feature = "full", feature = "derive"))] +use crate::punctuated::Punctuated; +#[cfg(feature = "full")] +macro_rules! full { + ($e:expr) => { + $e + }; +} +#[cfg(all(feature = "derive", not(feature = "full")))] +macro_rules! full { + ($e:expr) => { + unreachable!() + }; +} +macro_rules! skip { + ($($tt:tt)*) => {}; +} +/// Syntax tree traversal to walk a shared borrow of a syntax tree. +/// +/// See the [module documentation] for details. +/// +/// [module documentation]: self +pub trait Visit<'ast> { + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_abi(&mut self, i: &'ast crate::Abi) { + visit_abi(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_angle_bracketed_generic_arguments( + &mut self, + i: &'ast crate::AngleBracketedGenericArguments, + ) { + visit_angle_bracketed_generic_arguments(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_arm(&mut self, i: &'ast crate::Arm) { + visit_arm(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_assoc_const(&mut self, i: &'ast crate::AssocConst) { + visit_assoc_const(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_assoc_type(&mut self, i: &'ast crate::AssocType) { + visit_assoc_type(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_attr_style(&mut self, i: &'ast crate::AttrStyle) { + visit_attr_style(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_attribute(&mut self, i: &'ast crate::Attribute) { + visit_attribute(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_bare_fn_arg(&mut self, i: &'ast crate::BareFnArg) { + visit_bare_fn_arg(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_bare_variadic(&mut self, i: &'ast crate::BareVariadic) { + visit_bare_variadic(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_bin_op(&mut self, i: &'ast crate::BinOp) { + visit_bin_op(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_block(&mut self, i: &'ast crate::Block) { + visit_block(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_bound_lifetimes(&mut self, i: &'ast crate::BoundLifetimes) { + visit_bound_lifetimes(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_captured_param(&mut self, i: &'ast crate::CapturedParam) { + visit_captured_param(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_const_param(&mut self, i: &'ast crate::ConstParam) { + visit_const_param(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_constraint(&mut self, i: &'ast crate::Constraint) { + visit_constraint(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_data(&mut self, i: &'ast crate::Data) { + visit_data(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_data_enum(&mut self, i: &'ast crate::DataEnum) { + visit_data_enum(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_data_struct(&mut self, i: &'ast crate::DataStruct) { + visit_data_struct(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_data_union(&mut self, i: &'ast crate::DataUnion) { + visit_data_union(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_derive_input(&mut self, i: &'ast crate::DeriveInput) { + visit_derive_input(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr(&mut self, i: &'ast crate::Expr) { + visit_expr(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_array(&mut self, i: &'ast crate::ExprArray) { + visit_expr_array(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_assign(&mut self, i: &'ast crate::ExprAssign) { + visit_expr_assign(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_async(&mut self, i: &'ast crate::ExprAsync) { + visit_expr_async(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_await(&mut self, i: &'ast crate::ExprAwait) { + visit_expr_await(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_binary(&mut self, i: &'ast crate::ExprBinary) { + visit_expr_binary(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_block(&mut self, i: &'ast crate::ExprBlock) { + visit_expr_block(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_break(&mut self, i: &'ast crate::ExprBreak) { + visit_expr_break(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_call(&mut self, i: &'ast crate::ExprCall) { + visit_expr_call(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_cast(&mut self, i: &'ast crate::ExprCast) { + visit_expr_cast(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_closure(&mut self, i: &'ast crate::ExprClosure) { + visit_expr_closure(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_const(&mut self, i: &'ast crate::ExprConst) { + visit_expr_const(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_continue(&mut self, i: &'ast crate::ExprContinue) { + visit_expr_continue(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_field(&mut self, i: &'ast crate::ExprField) { + visit_expr_field(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_for_loop(&mut self, i: &'ast crate::ExprForLoop) { + visit_expr_for_loop(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_group(&mut self, i: &'ast crate::ExprGroup) { + visit_expr_group(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_if(&mut self, i: &'ast crate::ExprIf) { + visit_expr_if(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_index(&mut self, i: &'ast crate::ExprIndex) { + visit_expr_index(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_infer(&mut self, i: &'ast crate::ExprInfer) { + visit_expr_infer(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_let(&mut self, i: &'ast crate::ExprLet) { + visit_expr_let(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_lit(&mut self, i: &'ast crate::ExprLit) { + visit_expr_lit(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_loop(&mut self, i: &'ast crate::ExprLoop) { + visit_expr_loop(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_macro(&mut self, i: &'ast crate::ExprMacro) { + visit_expr_macro(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_match(&mut self, i: &'ast crate::ExprMatch) { + visit_expr_match(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_method_call(&mut self, i: &'ast crate::ExprMethodCall) { + visit_expr_method_call(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_paren(&mut self, i: &'ast crate::ExprParen) { + visit_expr_paren(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_path(&mut self, i: &'ast crate::ExprPath) { + visit_expr_path(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_range(&mut self, i: &'ast crate::ExprRange) { + visit_expr_range(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_raw_addr(&mut self, i: &'ast crate::ExprRawAddr) { + visit_expr_raw_addr(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_reference(&mut self, i: &'ast crate::ExprReference) { + visit_expr_reference(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_repeat(&mut self, i: &'ast crate::ExprRepeat) { + visit_expr_repeat(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_return(&mut self, i: &'ast crate::ExprReturn) { + visit_expr_return(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_struct(&mut self, i: &'ast crate::ExprStruct) { + visit_expr_struct(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_try(&mut self, i: &'ast crate::ExprTry) { + visit_expr_try(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_try_block(&mut self, i: &'ast crate::ExprTryBlock) { + visit_expr_try_block(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_tuple(&mut self, i: &'ast crate::ExprTuple) { + visit_expr_tuple(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_unary(&mut self, i: &'ast crate::ExprUnary) { + visit_expr_unary(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_unsafe(&mut self, i: &'ast crate::ExprUnsafe) { + visit_expr_unsafe(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_while(&mut self, i: &'ast crate::ExprWhile) { + visit_expr_while(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_yield(&mut self, i: &'ast crate::ExprYield) { + visit_expr_yield(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_field(&mut self, i: &'ast crate::Field) { + visit_field(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_field_mutability(&mut self, i: &'ast crate::FieldMutability) { + visit_field_mutability(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_field_pat(&mut self, i: &'ast crate::FieldPat) { + visit_field_pat(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_field_value(&mut self, i: &'ast crate::FieldValue) { + visit_field_value(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_fields(&mut self, i: &'ast crate::Fields) { + visit_fields(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_fields_named(&mut self, i: &'ast crate::FieldsNamed) { + visit_fields_named(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_fields_unnamed(&mut self, i: &'ast crate::FieldsUnnamed) { + visit_fields_unnamed(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_file(&mut self, i: &'ast crate::File) { + visit_file(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_fn_arg(&mut self, i: &'ast crate::FnArg) { + visit_fn_arg(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item(&mut self, i: &'ast crate::ForeignItem) { + visit_foreign_item(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_fn(&mut self, i: &'ast crate::ForeignItemFn) { + visit_foreign_item_fn(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_macro(&mut self, i: &'ast crate::ForeignItemMacro) { + visit_foreign_item_macro(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_static(&mut self, i: &'ast crate::ForeignItemStatic) { + visit_foreign_item_static(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_type(&mut self, i: &'ast crate::ForeignItemType) { + visit_foreign_item_type(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_generic_argument(&mut self, i: &'ast crate::GenericArgument) { + visit_generic_argument(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_generic_param(&mut self, i: &'ast crate::GenericParam) { + visit_generic_param(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_generics(&mut self, i: &'ast crate::Generics) { + visit_generics(self, i); + } + fn visit_ident(&mut self, i: &'ast proc_macro2::Ident) { + visit_ident(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item(&mut self, i: &'ast crate::ImplItem) { + visit_impl_item(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_const(&mut self, i: &'ast crate::ImplItemConst) { + visit_impl_item_const(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_fn(&mut self, i: &'ast crate::ImplItemFn) { + visit_impl_item_fn(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_macro(&mut self, i: &'ast crate::ImplItemMacro) { + visit_impl_item_macro(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_type(&mut self, i: &'ast crate::ImplItemType) { + visit_impl_item_type(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_restriction(&mut self, i: &'ast crate::ImplRestriction) { + visit_impl_restriction(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_index(&mut self, i: &'ast crate::Index) { + visit_index(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item(&mut self, i: &'ast crate::Item) { + visit_item(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_const(&mut self, i: &'ast crate::ItemConst) { + visit_item_const(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_enum(&mut self, i: &'ast crate::ItemEnum) { + visit_item_enum(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_extern_crate(&mut self, i: &'ast crate::ItemExternCrate) { + visit_item_extern_crate(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_fn(&mut self, i: &'ast crate::ItemFn) { + visit_item_fn(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_foreign_mod(&mut self, i: &'ast crate::ItemForeignMod) { + visit_item_foreign_mod(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_impl(&mut self, i: &'ast crate::ItemImpl) { + visit_item_impl(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_macro(&mut self, i: &'ast crate::ItemMacro) { + visit_item_macro(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_mod(&mut self, i: &'ast crate::ItemMod) { + visit_item_mod(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_static(&mut self, i: &'ast crate::ItemStatic) { + visit_item_static(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_struct(&mut self, i: &'ast crate::ItemStruct) { + visit_item_struct(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_trait(&mut self, i: &'ast crate::ItemTrait) { + visit_item_trait(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_trait_alias(&mut self, i: &'ast crate::ItemTraitAlias) { + visit_item_trait_alias(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_type(&mut self, i: &'ast crate::ItemType) { + visit_item_type(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_union(&mut self, i: &'ast crate::ItemUnion) { + visit_item_union(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_use(&mut self, i: &'ast crate::ItemUse) { + visit_item_use(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_label(&mut self, i: &'ast crate::Label) { + visit_label(self, i); + } + fn visit_lifetime(&mut self, i: &'ast crate::Lifetime) { + visit_lifetime(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_lifetime_param(&mut self, i: &'ast crate::LifetimeParam) { + visit_lifetime_param(self, i); + } + fn visit_lit(&mut self, i: &'ast crate::Lit) { + visit_lit(self, i); + } + fn visit_lit_bool(&mut self, i: &'ast crate::LitBool) { + visit_lit_bool(self, i); + } + fn visit_lit_byte(&mut self, i: &'ast crate::LitByte) { + visit_lit_byte(self, i); + } + fn visit_lit_byte_str(&mut self, i: &'ast crate::LitByteStr) { + visit_lit_byte_str(self, i); + } + fn visit_lit_cstr(&mut self, i: &'ast crate::LitCStr) { + visit_lit_cstr(self, i); + } + fn visit_lit_char(&mut self, i: &'ast crate::LitChar) { + visit_lit_char(self, i); + } + fn visit_lit_float(&mut self, i: &'ast crate::LitFloat) { + visit_lit_float(self, i); + } + fn visit_lit_int(&mut self, i: &'ast crate::LitInt) { + visit_lit_int(self, i); + } + fn visit_lit_str(&mut self, i: &'ast crate::LitStr) { + visit_lit_str(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_local(&mut self, i: &'ast crate::Local) { + visit_local(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_local_init(&mut self, i: &'ast crate::LocalInit) { + visit_local_init(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_macro(&mut self, i: &'ast crate::Macro) { + visit_macro(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_macro_delimiter(&mut self, i: &'ast crate::MacroDelimiter) { + visit_macro_delimiter(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_member(&mut self, i: &'ast crate::Member) { + visit_member(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_meta(&mut self, i: &'ast crate::Meta) { + visit_meta(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_meta_list(&mut self, i: &'ast crate::MetaList) { + visit_meta_list(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_meta_name_value(&mut self, i: &'ast crate::MetaNameValue) { + visit_meta_name_value(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_parenthesized_generic_arguments( + &mut self, + i: &'ast crate::ParenthesizedGenericArguments, + ) { + visit_parenthesized_generic_arguments(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat(&mut self, i: &'ast crate::Pat) { + visit_pat(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_ident(&mut self, i: &'ast crate::PatIdent) { + visit_pat_ident(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_or(&mut self, i: &'ast crate::PatOr) { + visit_pat_or(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_paren(&mut self, i: &'ast crate::PatParen) { + visit_pat_paren(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_reference(&mut self, i: &'ast crate::PatReference) { + visit_pat_reference(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_rest(&mut self, i: &'ast crate::PatRest) { + visit_pat_rest(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_slice(&mut self, i: &'ast crate::PatSlice) { + visit_pat_slice(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_struct(&mut self, i: &'ast crate::PatStruct) { + visit_pat_struct(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_tuple(&mut self, i: &'ast crate::PatTuple) { + visit_pat_tuple(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_tuple_struct(&mut self, i: &'ast crate::PatTupleStruct) { + visit_pat_tuple_struct(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_type(&mut self, i: &'ast crate::PatType) { + visit_pat_type(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_wild(&mut self, i: &'ast crate::PatWild) { + visit_pat_wild(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_path(&mut self, i: &'ast crate::Path) { + visit_path(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_path_arguments(&mut self, i: &'ast crate::PathArguments) { + visit_path_arguments(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_path_segment(&mut self, i: &'ast crate::PathSegment) { + visit_path_segment(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pointer_mutability(&mut self, i: &'ast crate::PointerMutability) { + visit_pointer_mutability(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_precise_capture(&mut self, i: &'ast crate::PreciseCapture) { + visit_precise_capture(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_predicate_lifetime(&mut self, i: &'ast crate::PredicateLifetime) { + visit_predicate_lifetime(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_predicate_type(&mut self, i: &'ast crate::PredicateType) { + visit_predicate_type(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_qself(&mut self, i: &'ast crate::QSelf) { + visit_qself(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_range_limits(&mut self, i: &'ast crate::RangeLimits) { + visit_range_limits(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_receiver(&mut self, i: &'ast crate::Receiver) { + visit_receiver(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_return_type(&mut self, i: &'ast crate::ReturnType) { + visit_return_type(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_signature(&mut self, i: &'ast crate::Signature) { + visit_signature(self, i); + } + fn visit_span(&mut self, i: &proc_macro2::Span) {} + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_static_mutability(&mut self, i: &'ast crate::StaticMutability) { + visit_static_mutability(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_stmt(&mut self, i: &'ast crate::Stmt) { + visit_stmt(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_stmt_macro(&mut self, i: &'ast crate::StmtMacro) { + visit_stmt_macro(self, i); + } + fn visit_token_stream(&mut self, i: &'ast proc_macro2::TokenStream) {} + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_trait_bound(&mut self, i: &'ast crate::TraitBound) { + visit_trait_bound(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_trait_bound_modifier(&mut self, i: &'ast crate::TraitBoundModifier) { + visit_trait_bound_modifier(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item(&mut self, i: &'ast crate::TraitItem) { + visit_trait_item(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_const(&mut self, i: &'ast crate::TraitItemConst) { + visit_trait_item_const(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_fn(&mut self, i: &'ast crate::TraitItemFn) { + visit_trait_item_fn(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_macro(&mut self, i: &'ast crate::TraitItemMacro) { + visit_trait_item_macro(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_type(&mut self, i: &'ast crate::TraitItemType) { + visit_trait_item_type(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type(&mut self, i: &'ast crate::Type) { + visit_type(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_array(&mut self, i: &'ast crate::TypeArray) { + visit_type_array(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_bare_fn(&mut self, i: &'ast crate::TypeBareFn) { + visit_type_bare_fn(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_group(&mut self, i: &'ast crate::TypeGroup) { + visit_type_group(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_impl_trait(&mut self, i: &'ast crate::TypeImplTrait) { + visit_type_impl_trait(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_infer(&mut self, i: &'ast crate::TypeInfer) { + visit_type_infer(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_macro(&mut self, i: &'ast crate::TypeMacro) { + visit_type_macro(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_never(&mut self, i: &'ast crate::TypeNever) { + visit_type_never(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_param(&mut self, i: &'ast crate::TypeParam) { + visit_type_param(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_param_bound(&mut self, i: &'ast crate::TypeParamBound) { + visit_type_param_bound(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_paren(&mut self, i: &'ast crate::TypeParen) { + visit_type_paren(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_path(&mut self, i: &'ast crate::TypePath) { + visit_type_path(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_ptr(&mut self, i: &'ast crate::TypePtr) { + visit_type_ptr(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_reference(&mut self, i: &'ast crate::TypeReference) { + visit_type_reference(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_slice(&mut self, i: &'ast crate::TypeSlice) { + visit_type_slice(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_trait_object(&mut self, i: &'ast crate::TypeTraitObject) { + visit_type_trait_object(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_tuple(&mut self, i: &'ast crate::TypeTuple) { + visit_type_tuple(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_un_op(&mut self, i: &'ast crate::UnOp) { + visit_un_op(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_glob(&mut self, i: &'ast crate::UseGlob) { + visit_use_glob(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_group(&mut self, i: &'ast crate::UseGroup) { + visit_use_group(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_name(&mut self, i: &'ast crate::UseName) { + visit_use_name(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_path(&mut self, i: &'ast crate::UsePath) { + visit_use_path(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_rename(&mut self, i: &'ast crate::UseRename) { + visit_use_rename(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_tree(&mut self, i: &'ast crate::UseTree) { + visit_use_tree(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_variadic(&mut self, i: &'ast crate::Variadic) { + visit_variadic(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_variant(&mut self, i: &'ast crate::Variant) { + visit_variant(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_vis_restricted(&mut self, i: &'ast crate::VisRestricted) { + visit_vis_restricted(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_visibility(&mut self, i: &'ast crate::Visibility) { + visit_visibility(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_where_clause(&mut self, i: &'ast crate::WhereClause) { + visit_where_clause(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_where_predicate(&mut self, i: &'ast crate::WherePredicate) { + visit_where_predicate(self, i); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_abi<'ast, V>(v: &mut V, node: &'ast crate::Abi) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.extern_token); + if let Some(it) = &node.name { + v.visit_lit_str(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_angle_bracketed_generic_arguments<'ast, V>( + v: &mut V, + node: &'ast crate::AngleBracketedGenericArguments, +) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.colon2_token); + skip!(node.lt_token); + for el in Punctuated::pairs(&node.args) { + let it = el.value(); + v.visit_generic_argument(it); + } + skip!(node.gt_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_arm<'ast, V>(v: &mut V, node: &'ast crate::Arm) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_pat(&node.pat); + if let Some(it) = &node.guard { + skip!((it).0); + v.visit_expr(&*(it).1); + } + skip!(node.fat_arrow_token); + v.visit_expr(&*node.body); + skip!(node.comma); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_assoc_const<'ast, V>(v: &mut V, node: &'ast crate::AssocConst) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_ident(&node.ident); + if let Some(it) = &node.generics { + v.visit_angle_bracketed_generic_arguments(it); + } + skip!(node.eq_token); + v.visit_expr(&node.value); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_assoc_type<'ast, V>(v: &mut V, node: &'ast crate::AssocType) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_ident(&node.ident); + if let Some(it) = &node.generics { + v.visit_angle_bracketed_generic_arguments(it); + } + skip!(node.eq_token); + v.visit_type(&node.ty); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_attr_style<'ast, V>(v: &mut V, node: &'ast crate::AttrStyle) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::AttrStyle::Outer => {} + crate::AttrStyle::Inner(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_attribute<'ast, V>(v: &mut V, node: &'ast crate::Attribute) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.pound_token); + v.visit_attr_style(&node.style); + skip!(node.bracket_token); + v.visit_meta(&node.meta); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_bare_fn_arg<'ast, V>(v: &mut V, node: &'ast crate::BareFnArg) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.name { + v.visit_ident(&(it).0); + skip!((it).1); + } + v.visit_type(&node.ty); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_bare_variadic<'ast, V>(v: &mut V, node: &'ast crate::BareVariadic) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.name { + v.visit_ident(&(it).0); + skip!((it).1); + } + skip!(node.dots); + skip!(node.comma); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_bin_op<'ast, V>(v: &mut V, node: &'ast crate::BinOp) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::BinOp::Add(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Sub(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Mul(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Div(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Rem(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::And(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Or(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitXor(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitAnd(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitOr(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Shl(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Shr(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Eq(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Lt(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Le(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Ne(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Ge(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Gt(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::AddAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::SubAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::MulAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::DivAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::RemAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitXorAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitAndAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitOrAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::ShlAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::ShrAssign(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_block<'ast, V>(v: &mut V, node: &'ast crate::Block) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.brace_token); + for it in &node.stmts { + v.visit_stmt(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_bound_lifetimes<'ast, V>(v: &mut V, node: &'ast crate::BoundLifetimes) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.for_token); + skip!(node.lt_token); + for el in Punctuated::pairs(&node.lifetimes) { + let it = el.value(); + v.visit_generic_param(it); + } + skip!(node.gt_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_captured_param<'ast, V>(v: &mut V, node: &'ast crate::CapturedParam) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::CapturedParam::Lifetime(_binding_0) => { + v.visit_lifetime(_binding_0); + } + crate::CapturedParam::Ident(_binding_0) => { + v.visit_ident(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_const_param<'ast, V>(v: &mut V, node: &'ast crate::ConstParam) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.const_token); + v.visit_ident(&node.ident); + skip!(node.colon_token); + v.visit_type(&node.ty); + skip!(node.eq_token); + if let Some(it) = &node.default { + v.visit_expr(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_constraint<'ast, V>(v: &mut V, node: &'ast crate::Constraint) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_ident(&node.ident); + if let Some(it) = &node.generics { + v.visit_angle_bracketed_generic_arguments(it); + } + skip!(node.colon_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_type_param_bound(it); + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_data<'ast, V>(v: &mut V, node: &'ast crate::Data) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Data::Struct(_binding_0) => { + v.visit_data_struct(_binding_0); + } + crate::Data::Enum(_binding_0) => { + v.visit_data_enum(_binding_0); + } + crate::Data::Union(_binding_0) => { + v.visit_data_union(_binding_0); + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_data_enum<'ast, V>(v: &mut V, node: &'ast crate::DataEnum) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.enum_token); + skip!(node.brace_token); + for el in Punctuated::pairs(&node.variants) { + let it = el.value(); + v.visit_variant(it); + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_data_struct<'ast, V>(v: &mut V, node: &'ast crate::DataStruct) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.struct_token); + v.visit_fields(&node.fields); + skip!(node.semi_token); +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_data_union<'ast, V>(v: &mut V, node: &'ast crate::DataUnion) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.union_token); + v.visit_fields_named(&node.fields); +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_derive_input<'ast, V>(v: &mut V, node: &'ast crate::DeriveInput) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + v.visit_data(&node.data); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr<'ast, V>(v: &mut V, node: &'ast crate::Expr) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Expr::Array(_binding_0) => { + full!(v.visit_expr_array(_binding_0)); + } + crate::Expr::Assign(_binding_0) => { + full!(v.visit_expr_assign(_binding_0)); + } + crate::Expr::Async(_binding_0) => { + full!(v.visit_expr_async(_binding_0)); + } + crate::Expr::Await(_binding_0) => { + full!(v.visit_expr_await(_binding_0)); + } + crate::Expr::Binary(_binding_0) => { + v.visit_expr_binary(_binding_0); + } + crate::Expr::Block(_binding_0) => { + full!(v.visit_expr_block(_binding_0)); + } + crate::Expr::Break(_binding_0) => { + full!(v.visit_expr_break(_binding_0)); + } + crate::Expr::Call(_binding_0) => { + v.visit_expr_call(_binding_0); + } + crate::Expr::Cast(_binding_0) => { + v.visit_expr_cast(_binding_0); + } + crate::Expr::Closure(_binding_0) => { + full!(v.visit_expr_closure(_binding_0)); + } + crate::Expr::Const(_binding_0) => { + full!(v.visit_expr_const(_binding_0)); + } + crate::Expr::Continue(_binding_0) => { + full!(v.visit_expr_continue(_binding_0)); + } + crate::Expr::Field(_binding_0) => { + v.visit_expr_field(_binding_0); + } + crate::Expr::ForLoop(_binding_0) => { + full!(v.visit_expr_for_loop(_binding_0)); + } + crate::Expr::Group(_binding_0) => { + v.visit_expr_group(_binding_0); + } + crate::Expr::If(_binding_0) => { + full!(v.visit_expr_if(_binding_0)); + } + crate::Expr::Index(_binding_0) => { + v.visit_expr_index(_binding_0); + } + crate::Expr::Infer(_binding_0) => { + full!(v.visit_expr_infer(_binding_0)); + } + crate::Expr::Let(_binding_0) => { + full!(v.visit_expr_let(_binding_0)); + } + crate::Expr::Lit(_binding_0) => { + v.visit_expr_lit(_binding_0); + } + crate::Expr::Loop(_binding_0) => { + full!(v.visit_expr_loop(_binding_0)); + } + crate::Expr::Macro(_binding_0) => { + v.visit_expr_macro(_binding_0); + } + crate::Expr::Match(_binding_0) => { + full!(v.visit_expr_match(_binding_0)); + } + crate::Expr::MethodCall(_binding_0) => { + v.visit_expr_method_call(_binding_0); + } + crate::Expr::Paren(_binding_0) => { + v.visit_expr_paren(_binding_0); + } + crate::Expr::Path(_binding_0) => { + v.visit_expr_path(_binding_0); + } + crate::Expr::Range(_binding_0) => { + full!(v.visit_expr_range(_binding_0)); + } + crate::Expr::RawAddr(_binding_0) => { + full!(v.visit_expr_raw_addr(_binding_0)); + } + crate::Expr::Reference(_binding_0) => { + v.visit_expr_reference(_binding_0); + } + crate::Expr::Repeat(_binding_0) => { + full!(v.visit_expr_repeat(_binding_0)); + } + crate::Expr::Return(_binding_0) => { + full!(v.visit_expr_return(_binding_0)); + } + crate::Expr::Struct(_binding_0) => { + v.visit_expr_struct(_binding_0); + } + crate::Expr::Try(_binding_0) => { + full!(v.visit_expr_try(_binding_0)); + } + crate::Expr::TryBlock(_binding_0) => { + full!(v.visit_expr_try_block(_binding_0)); + } + crate::Expr::Tuple(_binding_0) => { + v.visit_expr_tuple(_binding_0); + } + crate::Expr::Unary(_binding_0) => { + v.visit_expr_unary(_binding_0); + } + crate::Expr::Unsafe(_binding_0) => { + full!(v.visit_expr_unsafe(_binding_0)); + } + crate::Expr::Verbatim(_binding_0) => { + v.visit_token_stream(_binding_0); + } + crate::Expr::While(_binding_0) => { + full!(v.visit_expr_while(_binding_0)); + } + crate::Expr::Yield(_binding_0) => { + full!(v.visit_expr_yield(_binding_0)); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_array<'ast, V>(v: &mut V, node: &'ast crate::ExprArray) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.bracket_token); + for el in Punctuated::pairs(&node.elems) { + let it = el.value(); + v.visit_expr(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_assign<'ast, V>(v: &mut V, node: &'ast crate::ExprAssign) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.left); + skip!(node.eq_token); + v.visit_expr(&*node.right); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_async<'ast, V>(v: &mut V, node: &'ast crate::ExprAsync) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.async_token); + skip!(node.capture); + v.visit_block(&node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_await<'ast, V>(v: &mut V, node: &'ast crate::ExprAwait) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.base); + skip!(node.dot_token); + skip!(node.await_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_binary<'ast, V>(v: &mut V, node: &'ast crate::ExprBinary) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.left); + v.visit_bin_op(&node.op); + v.visit_expr(&*node.right); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_block<'ast, V>(v: &mut V, node: &'ast crate::ExprBlock) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.label { + v.visit_label(it); + } + v.visit_block(&node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_break<'ast, V>(v: &mut V, node: &'ast crate::ExprBreak) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.break_token); + if let Some(it) = &node.label { + v.visit_lifetime(it); + } + if let Some(it) = &node.expr { + v.visit_expr(&**it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_call<'ast, V>(v: &mut V, node: &'ast crate::ExprCall) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.func); + skip!(node.paren_token); + for el in Punctuated::pairs(&node.args) { + let it = el.value(); + v.visit_expr(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_cast<'ast, V>(v: &mut V, node: &'ast crate::ExprCast) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.expr); + skip!(node.as_token); + v.visit_type(&*node.ty); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_closure<'ast, V>(v: &mut V, node: &'ast crate::ExprClosure) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.lifetimes { + v.visit_bound_lifetimes(it); + } + skip!(node.constness); + skip!(node.movability); + skip!(node.asyncness); + skip!(node.capture); + skip!(node.or1_token); + for el in Punctuated::pairs(&node.inputs) { + let it = el.value(); + v.visit_pat(it); + } + skip!(node.or2_token); + v.visit_return_type(&node.output); + v.visit_expr(&*node.body); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_const<'ast, V>(v: &mut V, node: &'ast crate::ExprConst) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.const_token); + v.visit_block(&node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_continue<'ast, V>(v: &mut V, node: &'ast crate::ExprContinue) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.continue_token); + if let Some(it) = &node.label { + v.visit_lifetime(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_field<'ast, V>(v: &mut V, node: &'ast crate::ExprField) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.base); + skip!(node.dot_token); + v.visit_member(&node.member); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_for_loop<'ast, V>(v: &mut V, node: &'ast crate::ExprForLoop) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.label { + v.visit_label(it); + } + skip!(node.for_token); + v.visit_pat(&*node.pat); + skip!(node.in_token); + v.visit_expr(&*node.expr); + v.visit_block(&node.body); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_group<'ast, V>(v: &mut V, node: &'ast crate::ExprGroup) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.group_token); + v.visit_expr(&*node.expr); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_if<'ast, V>(v: &mut V, node: &'ast crate::ExprIf) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.if_token); + v.visit_expr(&*node.cond); + v.visit_block(&node.then_branch); + if let Some(it) = &node.else_branch { + skip!((it).0); + v.visit_expr(&*(it).1); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_index<'ast, V>(v: &mut V, node: &'ast crate::ExprIndex) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.expr); + skip!(node.bracket_token); + v.visit_expr(&*node.index); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_infer<'ast, V>(v: &mut V, node: &'ast crate::ExprInfer) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.underscore_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_let<'ast, V>(v: &mut V, node: &'ast crate::ExprLet) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.let_token); + v.visit_pat(&*node.pat); + skip!(node.eq_token); + v.visit_expr(&*node.expr); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_lit<'ast, V>(v: &mut V, node: &'ast crate::ExprLit) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_lit(&node.lit); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_loop<'ast, V>(v: &mut V, node: &'ast crate::ExprLoop) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.label { + v.visit_label(it); + } + skip!(node.loop_token); + v.visit_block(&node.body); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_macro<'ast, V>(v: &mut V, node: &'ast crate::ExprMacro) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_macro(&node.mac); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_match<'ast, V>(v: &mut V, node: &'ast crate::ExprMatch) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.match_token); + v.visit_expr(&*node.expr); + skip!(node.brace_token); + for it in &node.arms { + v.visit_arm(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_method_call<'ast, V>(v: &mut V, node: &'ast crate::ExprMethodCall) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.receiver); + skip!(node.dot_token); + v.visit_ident(&node.method); + if let Some(it) = &node.turbofish { + v.visit_angle_bracketed_generic_arguments(it); + } + skip!(node.paren_token); + for el in Punctuated::pairs(&node.args) { + let it = el.value(); + v.visit_expr(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_paren<'ast, V>(v: &mut V, node: &'ast crate::ExprParen) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.paren_token); + v.visit_expr(&*node.expr); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_path<'ast, V>(v: &mut V, node: &'ast crate::ExprPath) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.qself { + v.visit_qself(it); + } + v.visit_path(&node.path); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_range<'ast, V>(v: &mut V, node: &'ast crate::ExprRange) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.start { + v.visit_expr(&**it); + } + v.visit_range_limits(&node.limits); + if let Some(it) = &node.end { + v.visit_expr(&**it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_raw_addr<'ast, V>(v: &mut V, node: &'ast crate::ExprRawAddr) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.and_token); + skip!(node.raw); + v.visit_pointer_mutability(&node.mutability); + v.visit_expr(&*node.expr); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_reference<'ast, V>(v: &mut V, node: &'ast crate::ExprReference) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.and_token); + skip!(node.mutability); + v.visit_expr(&*node.expr); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_repeat<'ast, V>(v: &mut V, node: &'ast crate::ExprRepeat) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.bracket_token); + v.visit_expr(&*node.expr); + skip!(node.semi_token); + v.visit_expr(&*node.len); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_return<'ast, V>(v: &mut V, node: &'ast crate::ExprReturn) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.return_token); + if let Some(it) = &node.expr { + v.visit_expr(&**it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_struct<'ast, V>(v: &mut V, node: &'ast crate::ExprStruct) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.qself { + v.visit_qself(it); + } + v.visit_path(&node.path); + skip!(node.brace_token); + for el in Punctuated::pairs(&node.fields) { + let it = el.value(); + v.visit_field_value(it); + } + skip!(node.dot2_token); + if let Some(it) = &node.rest { + v.visit_expr(&**it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_try<'ast, V>(v: &mut V, node: &'ast crate::ExprTry) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_expr(&*node.expr); + skip!(node.question_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_try_block<'ast, V>(v: &mut V, node: &'ast crate::ExprTryBlock) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.try_token); + v.visit_block(&node.block); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_tuple<'ast, V>(v: &mut V, node: &'ast crate::ExprTuple) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.paren_token); + for el in Punctuated::pairs(&node.elems) { + let it = el.value(); + v.visit_expr(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_unary<'ast, V>(v: &mut V, node: &'ast crate::ExprUnary) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_un_op(&node.op); + v.visit_expr(&*node.expr); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_unsafe<'ast, V>(v: &mut V, node: &'ast crate::ExprUnsafe) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.unsafe_token); + v.visit_block(&node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_while<'ast, V>(v: &mut V, node: &'ast crate::ExprWhile) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.label { + v.visit_label(it); + } + skip!(node.while_token); + v.visit_expr(&*node.cond); + v.visit_block(&node.body); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_yield<'ast, V>(v: &mut V, node: &'ast crate::ExprYield) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.yield_token); + if let Some(it) = &node.expr { + v.visit_expr(&**it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_field<'ast, V>(v: &mut V, node: &'ast crate::Field) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + v.visit_field_mutability(&node.mutability); + if let Some(it) = &node.ident { + v.visit_ident(it); + } + skip!(node.colon_token); + v.visit_type(&node.ty); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_field_mutability<'ast, V>(v: &mut V, node: &'ast crate::FieldMutability) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::FieldMutability::None => {} + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_field_pat<'ast, V>(v: &mut V, node: &'ast crate::FieldPat) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_member(&node.member); + skip!(node.colon_token); + v.visit_pat(&*node.pat); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_field_value<'ast, V>(v: &mut V, node: &'ast crate::FieldValue) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_member(&node.member); + skip!(node.colon_token); + v.visit_expr(&node.expr); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_fields<'ast, V>(v: &mut V, node: &'ast crate::Fields) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Fields::Named(_binding_0) => { + v.visit_fields_named(_binding_0); + } + crate::Fields::Unnamed(_binding_0) => { + v.visit_fields_unnamed(_binding_0); + } + crate::Fields::Unit => {} + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_fields_named<'ast, V>(v: &mut V, node: &'ast crate::FieldsNamed) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.brace_token); + for el in Punctuated::pairs(&node.named) { + let it = el.value(); + v.visit_field(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_fields_unnamed<'ast, V>(v: &mut V, node: &'ast crate::FieldsUnnamed) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.paren_token); + for el in Punctuated::pairs(&node.unnamed) { + let it = el.value(); + v.visit_field(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_file<'ast, V>(v: &mut V, node: &'ast crate::File) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.shebang); + for it in &node.attrs { + v.visit_attribute(it); + } + for it in &node.items { + v.visit_item(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_fn_arg<'ast, V>(v: &mut V, node: &'ast crate::FnArg) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::FnArg::Receiver(_binding_0) => { + v.visit_receiver(_binding_0); + } + crate::FnArg::Typed(_binding_0) => { + v.visit_pat_type(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item<'ast, V>(v: &mut V, node: &'ast crate::ForeignItem) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::ForeignItem::Fn(_binding_0) => { + v.visit_foreign_item_fn(_binding_0); + } + crate::ForeignItem::Static(_binding_0) => { + v.visit_foreign_item_static(_binding_0); + } + crate::ForeignItem::Type(_binding_0) => { + v.visit_foreign_item_type(_binding_0); + } + crate::ForeignItem::Macro(_binding_0) => { + v.visit_foreign_item_macro(_binding_0); + } + crate::ForeignItem::Verbatim(_binding_0) => { + v.visit_token_stream(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_fn<'ast, V>(v: &mut V, node: &'ast crate::ForeignItemFn) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + v.visit_signature(&node.sig); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_macro<'ast, V>(v: &mut V, node: &'ast crate::ForeignItemMacro) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_macro(&node.mac); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_static<'ast, V>( + v: &mut V, + node: &'ast crate::ForeignItemStatic, +) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.static_token); + v.visit_static_mutability(&node.mutability); + v.visit_ident(&node.ident); + skip!(node.colon_token); + v.visit_type(&*node.ty); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_type<'ast, V>(v: &mut V, node: &'ast crate::ForeignItemType) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.type_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.semi_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_generic_argument<'ast, V>(v: &mut V, node: &'ast crate::GenericArgument) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::GenericArgument::Lifetime(_binding_0) => { + v.visit_lifetime(_binding_0); + } + crate::GenericArgument::Type(_binding_0) => { + v.visit_type(_binding_0); + } + crate::GenericArgument::Const(_binding_0) => { + v.visit_expr(_binding_0); + } + crate::GenericArgument::AssocType(_binding_0) => { + v.visit_assoc_type(_binding_0); + } + crate::GenericArgument::AssocConst(_binding_0) => { + v.visit_assoc_const(_binding_0); + } + crate::GenericArgument::Constraint(_binding_0) => { + v.visit_constraint(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_generic_param<'ast, V>(v: &mut V, node: &'ast crate::GenericParam) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::GenericParam::Lifetime(_binding_0) => { + v.visit_lifetime_param(_binding_0); + } + crate::GenericParam::Type(_binding_0) => { + v.visit_type_param(_binding_0); + } + crate::GenericParam::Const(_binding_0) => { + v.visit_const_param(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_generics<'ast, V>(v: &mut V, node: &'ast crate::Generics) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.lt_token); + for el in Punctuated::pairs(&node.params) { + let it = el.value(); + v.visit_generic_param(it); + } + skip!(node.gt_token); + if let Some(it) = &node.where_clause { + v.visit_where_clause(it); + } +} +pub fn visit_ident<'ast, V>(v: &mut V, node: &'ast proc_macro2::Ident) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_span(&node.span()); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item<'ast, V>(v: &mut V, node: &'ast crate::ImplItem) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::ImplItem::Const(_binding_0) => { + v.visit_impl_item_const(_binding_0); + } + crate::ImplItem::Fn(_binding_0) => { + v.visit_impl_item_fn(_binding_0); + } + crate::ImplItem::Type(_binding_0) => { + v.visit_impl_item_type(_binding_0); + } + crate::ImplItem::Macro(_binding_0) => { + v.visit_impl_item_macro(_binding_0); + } + crate::ImplItem::Verbatim(_binding_0) => { + v.visit_token_stream(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_const<'ast, V>(v: &mut V, node: &'ast crate::ImplItemConst) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.defaultness); + skip!(node.const_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.colon_token); + v.visit_type(&node.ty); + skip!(node.eq_token); + v.visit_expr(&node.expr); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_fn<'ast, V>(v: &mut V, node: &'ast crate::ImplItemFn) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.defaultness); + v.visit_signature(&node.sig); + v.visit_block(&node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_macro<'ast, V>(v: &mut V, node: &'ast crate::ImplItemMacro) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_macro(&node.mac); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_type<'ast, V>(v: &mut V, node: &'ast crate::ImplItemType) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.defaultness); + skip!(node.type_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.eq_token); + v.visit_type(&node.ty); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_restriction<'ast, V>(v: &mut V, node: &'ast crate::ImplRestriction) +where + V: Visit<'ast> + ?Sized, +{ + match *node {} +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_index<'ast, V>(v: &mut V, node: &'ast crate::Index) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.index); + v.visit_span(&node.span); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item<'ast, V>(v: &mut V, node: &'ast crate::Item) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Item::Const(_binding_0) => { + v.visit_item_const(_binding_0); + } + crate::Item::Enum(_binding_0) => { + v.visit_item_enum(_binding_0); + } + crate::Item::ExternCrate(_binding_0) => { + v.visit_item_extern_crate(_binding_0); + } + crate::Item::Fn(_binding_0) => { + v.visit_item_fn(_binding_0); + } + crate::Item::ForeignMod(_binding_0) => { + v.visit_item_foreign_mod(_binding_0); + } + crate::Item::Impl(_binding_0) => { + v.visit_item_impl(_binding_0); + } + crate::Item::Macro(_binding_0) => { + v.visit_item_macro(_binding_0); + } + crate::Item::Mod(_binding_0) => { + v.visit_item_mod(_binding_0); + } + crate::Item::Static(_binding_0) => { + v.visit_item_static(_binding_0); + } + crate::Item::Struct(_binding_0) => { + v.visit_item_struct(_binding_0); + } + crate::Item::Trait(_binding_0) => { + v.visit_item_trait(_binding_0); + } + crate::Item::TraitAlias(_binding_0) => { + v.visit_item_trait_alias(_binding_0); + } + crate::Item::Type(_binding_0) => { + v.visit_item_type(_binding_0); + } + crate::Item::Union(_binding_0) => { + v.visit_item_union(_binding_0); + } + crate::Item::Use(_binding_0) => { + v.visit_item_use(_binding_0); + } + crate::Item::Verbatim(_binding_0) => { + v.visit_token_stream(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_const<'ast, V>(v: &mut V, node: &'ast crate::ItemConst) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.const_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.colon_token); + v.visit_type(&*node.ty); + skip!(node.eq_token); + v.visit_expr(&*node.expr); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_enum<'ast, V>(v: &mut V, node: &'ast crate::ItemEnum) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.enum_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.brace_token); + for el in Punctuated::pairs(&node.variants) { + let it = el.value(); + v.visit_variant(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_extern_crate<'ast, V>(v: &mut V, node: &'ast crate::ItemExternCrate) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.extern_token); + skip!(node.crate_token); + v.visit_ident(&node.ident); + if let Some(it) = &node.rename { + skip!((it).0); + v.visit_ident(&(it).1); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_fn<'ast, V>(v: &mut V, node: &'ast crate::ItemFn) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + v.visit_signature(&node.sig); + v.visit_block(&*node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_foreign_mod<'ast, V>(v: &mut V, node: &'ast crate::ItemForeignMod) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.unsafety); + v.visit_abi(&node.abi); + skip!(node.brace_token); + for it in &node.items { + v.visit_foreign_item(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_impl<'ast, V>(v: &mut V, node: &'ast crate::ItemImpl) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.defaultness); + skip!(node.unsafety); + skip!(node.impl_token); + v.visit_generics(&node.generics); + if let Some(it) = &node.trait_ { + skip!((it).0); + v.visit_path(&(it).1); + skip!((it).2); + } + v.visit_type(&*node.self_ty); + skip!(node.brace_token); + for it in &node.items { + v.visit_impl_item(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_macro<'ast, V>(v: &mut V, node: &'ast crate::ItemMacro) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.ident { + v.visit_ident(it); + } + v.visit_macro(&node.mac); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_mod<'ast, V>(v: &mut V, node: &'ast crate::ItemMod) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.unsafety); + skip!(node.mod_token); + v.visit_ident(&node.ident); + if let Some(it) = &node.content { + skip!((it).0); + for it in &(it).1 { + v.visit_item(it); + } + } + skip!(node.semi); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_static<'ast, V>(v: &mut V, node: &'ast crate::ItemStatic) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.static_token); + v.visit_static_mutability(&node.mutability); + v.visit_ident(&node.ident); + skip!(node.colon_token); + v.visit_type(&*node.ty); + skip!(node.eq_token); + v.visit_expr(&*node.expr); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_struct<'ast, V>(v: &mut V, node: &'ast crate::ItemStruct) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.struct_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + v.visit_fields(&node.fields); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_trait<'ast, V>(v: &mut V, node: &'ast crate::ItemTrait) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.unsafety); + skip!(node.auto_token); + if let Some(it) = &node.restriction { + v.visit_impl_restriction(it); + } + skip!(node.trait_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.colon_token); + for el in Punctuated::pairs(&node.supertraits) { + let it = el.value(); + v.visit_type_param_bound(it); + } + skip!(node.brace_token); + for it in &node.items { + v.visit_trait_item(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_trait_alias<'ast, V>(v: &mut V, node: &'ast crate::ItemTraitAlias) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.trait_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.eq_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_type_param_bound(it); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_type<'ast, V>(v: &mut V, node: &'ast crate::ItemType) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.type_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.eq_token); + v.visit_type(&*node.ty); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_union<'ast, V>(v: &mut V, node: &'ast crate::ItemUnion) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.union_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + v.visit_fields_named(&node.fields); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_use<'ast, V>(v: &mut V, node: &'ast crate::ItemUse) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_visibility(&node.vis); + skip!(node.use_token); + skip!(node.leading_colon); + v.visit_use_tree(&node.tree); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_label<'ast, V>(v: &mut V, node: &'ast crate::Label) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_lifetime(&node.name); + skip!(node.colon_token); +} +pub fn visit_lifetime<'ast, V>(v: &mut V, node: &'ast crate::Lifetime) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_span(&node.apostrophe); + v.visit_ident(&node.ident); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_lifetime_param<'ast, V>(v: &mut V, node: &'ast crate::LifetimeParam) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_lifetime(&node.lifetime); + skip!(node.colon_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_lifetime(it); + } +} +pub fn visit_lit<'ast, V>(v: &mut V, node: &'ast crate::Lit) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Lit::Str(_binding_0) => { + v.visit_lit_str(_binding_0); + } + crate::Lit::ByteStr(_binding_0) => { + v.visit_lit_byte_str(_binding_0); + } + crate::Lit::CStr(_binding_0) => { + v.visit_lit_cstr(_binding_0); + } + crate::Lit::Byte(_binding_0) => { + v.visit_lit_byte(_binding_0); + } + crate::Lit::Char(_binding_0) => { + v.visit_lit_char(_binding_0); + } + crate::Lit::Int(_binding_0) => { + v.visit_lit_int(_binding_0); + } + crate::Lit::Float(_binding_0) => { + v.visit_lit_float(_binding_0); + } + crate::Lit::Bool(_binding_0) => { + v.visit_lit_bool(_binding_0); + } + crate::Lit::Verbatim(_binding_0) => { + skip!(_binding_0); + } + } +} +pub fn visit_lit_bool<'ast, V>(v: &mut V, node: &'ast crate::LitBool) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.value); + v.visit_span(&node.span); +} +pub fn visit_lit_byte<'ast, V>(v: &mut V, node: &'ast crate::LitByte) +where + V: Visit<'ast> + ?Sized, +{} +pub fn visit_lit_byte_str<'ast, V>(v: &mut V, node: &'ast crate::LitByteStr) +where + V: Visit<'ast> + ?Sized, +{} +pub fn visit_lit_cstr<'ast, V>(v: &mut V, node: &'ast crate::LitCStr) +where + V: Visit<'ast> + ?Sized, +{} +pub fn visit_lit_char<'ast, V>(v: &mut V, node: &'ast crate::LitChar) +where + V: Visit<'ast> + ?Sized, +{} +pub fn visit_lit_float<'ast, V>(v: &mut V, node: &'ast crate::LitFloat) +where + V: Visit<'ast> + ?Sized, +{} +pub fn visit_lit_int<'ast, V>(v: &mut V, node: &'ast crate::LitInt) +where + V: Visit<'ast> + ?Sized, +{} +pub fn visit_lit_str<'ast, V>(v: &mut V, node: &'ast crate::LitStr) +where + V: Visit<'ast> + ?Sized, +{} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_local<'ast, V>(v: &mut V, node: &'ast crate::Local) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.let_token); + v.visit_pat(&node.pat); + if let Some(it) = &node.init { + v.visit_local_init(it); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_local_init<'ast, V>(v: &mut V, node: &'ast crate::LocalInit) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.eq_token); + v.visit_expr(&*node.expr); + if let Some(it) = &node.diverge { + skip!((it).0); + v.visit_expr(&*(it).1); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_macro<'ast, V>(v: &mut V, node: &'ast crate::Macro) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_path(&node.path); + skip!(node.bang_token); + v.visit_macro_delimiter(&node.delimiter); + v.visit_token_stream(&node.tokens); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_macro_delimiter<'ast, V>(v: &mut V, node: &'ast crate::MacroDelimiter) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::MacroDelimiter::Paren(_binding_0) => { + skip!(_binding_0); + } + crate::MacroDelimiter::Brace(_binding_0) => { + skip!(_binding_0); + } + crate::MacroDelimiter::Bracket(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_member<'ast, V>(v: &mut V, node: &'ast crate::Member) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Member::Named(_binding_0) => { + v.visit_ident(_binding_0); + } + crate::Member::Unnamed(_binding_0) => { + v.visit_index(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_meta<'ast, V>(v: &mut V, node: &'ast crate::Meta) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Meta::Path(_binding_0) => { + v.visit_path(_binding_0); + } + crate::Meta::List(_binding_0) => { + v.visit_meta_list(_binding_0); + } + crate::Meta::NameValue(_binding_0) => { + v.visit_meta_name_value(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_meta_list<'ast, V>(v: &mut V, node: &'ast crate::MetaList) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_path(&node.path); + v.visit_macro_delimiter(&node.delimiter); + v.visit_token_stream(&node.tokens); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_meta_name_value<'ast, V>(v: &mut V, node: &'ast crate::MetaNameValue) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_path(&node.path); + skip!(node.eq_token); + v.visit_expr(&node.value); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_parenthesized_generic_arguments<'ast, V>( + v: &mut V, + node: &'ast crate::ParenthesizedGenericArguments, +) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.paren_token); + for el in Punctuated::pairs(&node.inputs) { + let it = el.value(); + v.visit_type(it); + } + v.visit_return_type(&node.output); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat<'ast, V>(v: &mut V, node: &'ast crate::Pat) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Pat::Const(_binding_0) => { + v.visit_expr_const(_binding_0); + } + crate::Pat::Ident(_binding_0) => { + v.visit_pat_ident(_binding_0); + } + crate::Pat::Lit(_binding_0) => { + v.visit_expr_lit(_binding_0); + } + crate::Pat::Macro(_binding_0) => { + v.visit_expr_macro(_binding_0); + } + crate::Pat::Or(_binding_0) => { + v.visit_pat_or(_binding_0); + } + crate::Pat::Paren(_binding_0) => { + v.visit_pat_paren(_binding_0); + } + crate::Pat::Path(_binding_0) => { + v.visit_expr_path(_binding_0); + } + crate::Pat::Range(_binding_0) => { + v.visit_expr_range(_binding_0); + } + crate::Pat::Reference(_binding_0) => { + v.visit_pat_reference(_binding_0); + } + crate::Pat::Rest(_binding_0) => { + v.visit_pat_rest(_binding_0); + } + crate::Pat::Slice(_binding_0) => { + v.visit_pat_slice(_binding_0); + } + crate::Pat::Struct(_binding_0) => { + v.visit_pat_struct(_binding_0); + } + crate::Pat::Tuple(_binding_0) => { + v.visit_pat_tuple(_binding_0); + } + crate::Pat::TupleStruct(_binding_0) => { + v.visit_pat_tuple_struct(_binding_0); + } + crate::Pat::Type(_binding_0) => { + v.visit_pat_type(_binding_0); + } + crate::Pat::Verbatim(_binding_0) => { + v.visit_token_stream(_binding_0); + } + crate::Pat::Wild(_binding_0) => { + v.visit_pat_wild(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_ident<'ast, V>(v: &mut V, node: &'ast crate::PatIdent) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.by_ref); + skip!(node.mutability); + v.visit_ident(&node.ident); + if let Some(it) = &node.subpat { + skip!((it).0); + v.visit_pat(&*(it).1); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_or<'ast, V>(v: &mut V, node: &'ast crate::PatOr) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.leading_vert); + for el in Punctuated::pairs(&node.cases) { + let it = el.value(); + v.visit_pat(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_paren<'ast, V>(v: &mut V, node: &'ast crate::PatParen) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.paren_token); + v.visit_pat(&*node.pat); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_reference<'ast, V>(v: &mut V, node: &'ast crate::PatReference) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.and_token); + skip!(node.mutability); + v.visit_pat(&*node.pat); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_rest<'ast, V>(v: &mut V, node: &'ast crate::PatRest) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.dot2_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_slice<'ast, V>(v: &mut V, node: &'ast crate::PatSlice) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.bracket_token); + for el in Punctuated::pairs(&node.elems) { + let it = el.value(); + v.visit_pat(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_struct<'ast, V>(v: &mut V, node: &'ast crate::PatStruct) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.qself { + v.visit_qself(it); + } + v.visit_path(&node.path); + skip!(node.brace_token); + for el in Punctuated::pairs(&node.fields) { + let it = el.value(); + v.visit_field_pat(it); + } + if let Some(it) = &node.rest { + v.visit_pat_rest(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_tuple<'ast, V>(v: &mut V, node: &'ast crate::PatTuple) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.paren_token); + for el in Punctuated::pairs(&node.elems) { + let it = el.value(); + v.visit_pat(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_tuple_struct<'ast, V>(v: &mut V, node: &'ast crate::PatTupleStruct) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.qself { + v.visit_qself(it); + } + v.visit_path(&node.path); + skip!(node.paren_token); + for el in Punctuated::pairs(&node.elems) { + let it = el.value(); + v.visit_pat(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_type<'ast, V>(v: &mut V, node: &'ast crate::PatType) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_pat(&*node.pat); + skip!(node.colon_token); + v.visit_type(&*node.ty); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_wild<'ast, V>(v: &mut V, node: &'ast crate::PatWild) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.underscore_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_path<'ast, V>(v: &mut V, node: &'ast crate::Path) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.leading_colon); + for el in Punctuated::pairs(&node.segments) { + let it = el.value(); + v.visit_path_segment(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_path_arguments<'ast, V>(v: &mut V, node: &'ast crate::PathArguments) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::PathArguments::None => {} + crate::PathArguments::AngleBracketed(_binding_0) => { + v.visit_angle_bracketed_generic_arguments(_binding_0); + } + crate::PathArguments::Parenthesized(_binding_0) => { + v.visit_parenthesized_generic_arguments(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_path_segment<'ast, V>(v: &mut V, node: &'ast crate::PathSegment) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_ident(&node.ident); + v.visit_path_arguments(&node.arguments); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pointer_mutability<'ast, V>(v: &mut V, node: &'ast crate::PointerMutability) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::PointerMutability::Const(_binding_0) => { + skip!(_binding_0); + } + crate::PointerMutability::Mut(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_precise_capture<'ast, V>(v: &mut V, node: &'ast crate::PreciseCapture) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.use_token); + skip!(node.lt_token); + for el in Punctuated::pairs(&node.params) { + let it = el.value(); + v.visit_captured_param(it); + } + skip!(node.gt_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_predicate_lifetime<'ast, V>(v: &mut V, node: &'ast crate::PredicateLifetime) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_lifetime(&node.lifetime); + skip!(node.colon_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_lifetime(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_predicate_type<'ast, V>(v: &mut V, node: &'ast crate::PredicateType) +where + V: Visit<'ast> + ?Sized, +{ + if let Some(it) = &node.lifetimes { + v.visit_bound_lifetimes(it); + } + v.visit_type(&node.bounded_ty); + skip!(node.colon_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_type_param_bound(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_qself<'ast, V>(v: &mut V, node: &'ast crate::QSelf) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.lt_token); + v.visit_type(&*node.ty); + skip!(node.position); + skip!(node.as_token); + skip!(node.gt_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_range_limits<'ast, V>(v: &mut V, node: &'ast crate::RangeLimits) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::RangeLimits::HalfOpen(_binding_0) => { + skip!(_binding_0); + } + crate::RangeLimits::Closed(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_receiver<'ast, V>(v: &mut V, node: &'ast crate::Receiver) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.reference { + skip!((it).0); + if let Some(it) = &(it).1 { + v.visit_lifetime(it); + } + } + skip!(node.mutability); + skip!(node.self_token); + skip!(node.colon_token); + v.visit_type(&*node.ty); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_return_type<'ast, V>(v: &mut V, node: &'ast crate::ReturnType) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::ReturnType::Default => {} + crate::ReturnType::Type(_binding_0, _binding_1) => { + skip!(_binding_0); + v.visit_type(&**_binding_1); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_signature<'ast, V>(v: &mut V, node: &'ast crate::Signature) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.constness); + skip!(node.asyncness); + skip!(node.unsafety); + if let Some(it) = &node.abi { + v.visit_abi(it); + } + skip!(node.fn_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.paren_token); + for el in Punctuated::pairs(&node.inputs) { + let it = el.value(); + v.visit_fn_arg(it); + } + if let Some(it) = &node.variadic { + v.visit_variadic(it); + } + v.visit_return_type(&node.output); +} +pub fn visit_span<'ast, V>(v: &mut V, node: &proc_macro2::Span) +where + V: Visit<'ast> + ?Sized, +{} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_static_mutability<'ast, V>(v: &mut V, node: &'ast crate::StaticMutability) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::StaticMutability::Mut(_binding_0) => { + skip!(_binding_0); + } + crate::StaticMutability::None => {} + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_stmt<'ast, V>(v: &mut V, node: &'ast crate::Stmt) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Stmt::Local(_binding_0) => { + v.visit_local(_binding_0); + } + crate::Stmt::Item(_binding_0) => { + v.visit_item(_binding_0); + } + crate::Stmt::Expr(_binding_0, _binding_1) => { + v.visit_expr(_binding_0); + skip!(_binding_1); + } + crate::Stmt::Macro(_binding_0) => { + v.visit_stmt_macro(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_stmt_macro<'ast, V>(v: &mut V, node: &'ast crate::StmtMacro) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_macro(&node.mac); + skip!(node.semi_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_trait_bound<'ast, V>(v: &mut V, node: &'ast crate::TraitBound) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.paren_token); + v.visit_trait_bound_modifier(&node.modifier); + if let Some(it) = &node.lifetimes { + v.visit_bound_lifetimes(it); + } + v.visit_path(&node.path); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_trait_bound_modifier<'ast, V>( + v: &mut V, + node: &'ast crate::TraitBoundModifier, +) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::TraitBoundModifier::None => {} + crate::TraitBoundModifier::Maybe(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item<'ast, V>(v: &mut V, node: &'ast crate::TraitItem) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::TraitItem::Const(_binding_0) => { + v.visit_trait_item_const(_binding_0); + } + crate::TraitItem::Fn(_binding_0) => { + v.visit_trait_item_fn(_binding_0); + } + crate::TraitItem::Type(_binding_0) => { + v.visit_trait_item_type(_binding_0); + } + crate::TraitItem::Macro(_binding_0) => { + v.visit_trait_item_macro(_binding_0); + } + crate::TraitItem::Verbatim(_binding_0) => { + v.visit_token_stream(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_const<'ast, V>(v: &mut V, node: &'ast crate::TraitItemConst) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.const_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.colon_token); + v.visit_type(&node.ty); + if let Some(it) = &node.default { + skip!((it).0); + v.visit_expr(&(it).1); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_fn<'ast, V>(v: &mut V, node: &'ast crate::TraitItemFn) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_signature(&node.sig); + if let Some(it) = &node.default { + v.visit_block(it); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_macro<'ast, V>(v: &mut V, node: &'ast crate::TraitItemMacro) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_macro(&node.mac); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_type<'ast, V>(v: &mut V, node: &'ast crate::TraitItemType) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + skip!(node.type_token); + v.visit_ident(&node.ident); + v.visit_generics(&node.generics); + skip!(node.colon_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_type_param_bound(it); + } + if let Some(it) = &node.default { + skip!((it).0); + v.visit_type(&(it).1); + } + skip!(node.semi_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type<'ast, V>(v: &mut V, node: &'ast crate::Type) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Type::Array(_binding_0) => { + v.visit_type_array(_binding_0); + } + crate::Type::BareFn(_binding_0) => { + v.visit_type_bare_fn(_binding_0); + } + crate::Type::Group(_binding_0) => { + v.visit_type_group(_binding_0); + } + crate::Type::ImplTrait(_binding_0) => { + v.visit_type_impl_trait(_binding_0); + } + crate::Type::Infer(_binding_0) => { + v.visit_type_infer(_binding_0); + } + crate::Type::Macro(_binding_0) => { + v.visit_type_macro(_binding_0); + } + crate::Type::Never(_binding_0) => { + v.visit_type_never(_binding_0); + } + crate::Type::Paren(_binding_0) => { + v.visit_type_paren(_binding_0); + } + crate::Type::Path(_binding_0) => { + v.visit_type_path(_binding_0); + } + crate::Type::Ptr(_binding_0) => { + v.visit_type_ptr(_binding_0); + } + crate::Type::Reference(_binding_0) => { + v.visit_type_reference(_binding_0); + } + crate::Type::Slice(_binding_0) => { + v.visit_type_slice(_binding_0); + } + crate::Type::TraitObject(_binding_0) => { + v.visit_type_trait_object(_binding_0); + } + crate::Type::Tuple(_binding_0) => { + v.visit_type_tuple(_binding_0); + } + crate::Type::Verbatim(_binding_0) => { + v.visit_token_stream(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_array<'ast, V>(v: &mut V, node: &'ast crate::TypeArray) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.bracket_token); + v.visit_type(&*node.elem); + skip!(node.semi_token); + v.visit_expr(&node.len); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_bare_fn<'ast, V>(v: &mut V, node: &'ast crate::TypeBareFn) +where + V: Visit<'ast> + ?Sized, +{ + if let Some(it) = &node.lifetimes { + v.visit_bound_lifetimes(it); + } + skip!(node.unsafety); + if let Some(it) = &node.abi { + v.visit_abi(it); + } + skip!(node.fn_token); + skip!(node.paren_token); + for el in Punctuated::pairs(&node.inputs) { + let it = el.value(); + v.visit_bare_fn_arg(it); + } + if let Some(it) = &node.variadic { + v.visit_bare_variadic(it); + } + v.visit_return_type(&node.output); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_group<'ast, V>(v: &mut V, node: &'ast crate::TypeGroup) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.group_token); + v.visit_type(&*node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_impl_trait<'ast, V>(v: &mut V, node: &'ast crate::TypeImplTrait) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.impl_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_type_param_bound(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_infer<'ast, V>(v: &mut V, node: &'ast crate::TypeInfer) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.underscore_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_macro<'ast, V>(v: &mut V, node: &'ast crate::TypeMacro) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_macro(&node.mac); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_never<'ast, V>(v: &mut V, node: &'ast crate::TypeNever) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.bang_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_param<'ast, V>(v: &mut V, node: &'ast crate::TypeParam) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_ident(&node.ident); + skip!(node.colon_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_type_param_bound(it); + } + skip!(node.eq_token); + if let Some(it) = &node.default { + v.visit_type(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_param_bound<'ast, V>(v: &mut V, node: &'ast crate::TypeParamBound) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::TypeParamBound::Trait(_binding_0) => { + v.visit_trait_bound(_binding_0); + } + crate::TypeParamBound::Lifetime(_binding_0) => { + v.visit_lifetime(_binding_0); + } + crate::TypeParamBound::PreciseCapture(_binding_0) => { + full!(v.visit_precise_capture(_binding_0)); + } + crate::TypeParamBound::Verbatim(_binding_0) => { + v.visit_token_stream(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_paren<'ast, V>(v: &mut V, node: &'ast crate::TypeParen) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.paren_token); + v.visit_type(&*node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_path<'ast, V>(v: &mut V, node: &'ast crate::TypePath) +where + V: Visit<'ast> + ?Sized, +{ + if let Some(it) = &node.qself { + v.visit_qself(it); + } + v.visit_path(&node.path); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_ptr<'ast, V>(v: &mut V, node: &'ast crate::TypePtr) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.star_token); + skip!(node.const_token); + skip!(node.mutability); + v.visit_type(&*node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_reference<'ast, V>(v: &mut V, node: &'ast crate::TypeReference) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.and_token); + if let Some(it) = &node.lifetime { + v.visit_lifetime(it); + } + skip!(node.mutability); + v.visit_type(&*node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_slice<'ast, V>(v: &mut V, node: &'ast crate::TypeSlice) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.bracket_token); + v.visit_type(&*node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_trait_object<'ast, V>(v: &mut V, node: &'ast crate::TypeTraitObject) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.dyn_token); + for el in Punctuated::pairs(&node.bounds) { + let it = el.value(); + v.visit_type_param_bound(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_tuple<'ast, V>(v: &mut V, node: &'ast crate::TypeTuple) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.paren_token); + for el in Punctuated::pairs(&node.elems) { + let it = el.value(); + v.visit_type(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_un_op<'ast, V>(v: &mut V, node: &'ast crate::UnOp) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::UnOp::Deref(_binding_0) => { + skip!(_binding_0); + } + crate::UnOp::Not(_binding_0) => { + skip!(_binding_0); + } + crate::UnOp::Neg(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_glob<'ast, V>(v: &mut V, node: &'ast crate::UseGlob) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.star_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_group<'ast, V>(v: &mut V, node: &'ast crate::UseGroup) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.brace_token); + for el in Punctuated::pairs(&node.items) { + let it = el.value(); + v.visit_use_tree(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_name<'ast, V>(v: &mut V, node: &'ast crate::UseName) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_ident(&node.ident); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_path<'ast, V>(v: &mut V, node: &'ast crate::UsePath) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_ident(&node.ident); + skip!(node.colon2_token); + v.visit_use_tree(&*node.tree); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_rename<'ast, V>(v: &mut V, node: &'ast crate::UseRename) +where + V: Visit<'ast> + ?Sized, +{ + v.visit_ident(&node.ident); + skip!(node.as_token); + v.visit_ident(&node.rename); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_tree<'ast, V>(v: &mut V, node: &'ast crate::UseTree) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::UseTree::Path(_binding_0) => { + v.visit_use_path(_binding_0); + } + crate::UseTree::Name(_binding_0) => { + v.visit_use_name(_binding_0); + } + crate::UseTree::Rename(_binding_0) => { + v.visit_use_rename(_binding_0); + } + crate::UseTree::Glob(_binding_0) => { + v.visit_use_glob(_binding_0); + } + crate::UseTree::Group(_binding_0) => { + v.visit_use_group(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_variadic<'ast, V>(v: &mut V, node: &'ast crate::Variadic) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + if let Some(it) = &node.pat { + v.visit_pat(&*(it).0); + skip!((it).1); + } + skip!(node.dots); + skip!(node.comma); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_variant<'ast, V>(v: &mut V, node: &'ast crate::Variant) +where + V: Visit<'ast> + ?Sized, +{ + for it in &node.attrs { + v.visit_attribute(it); + } + v.visit_ident(&node.ident); + v.visit_fields(&node.fields); + if let Some(it) = &node.discriminant { + skip!((it).0); + v.visit_expr(&(it).1); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_vis_restricted<'ast, V>(v: &mut V, node: &'ast crate::VisRestricted) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.pub_token); + skip!(node.paren_token); + skip!(node.in_token); + v.visit_path(&*node.path); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_visibility<'ast, V>(v: &mut V, node: &'ast crate::Visibility) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::Visibility::Public(_binding_0) => { + skip!(_binding_0); + } + crate::Visibility::Restricted(_binding_0) => { + v.visit_vis_restricted(_binding_0); + } + crate::Visibility::Inherited => {} + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_where_clause<'ast, V>(v: &mut V, node: &'ast crate::WhereClause) +where + V: Visit<'ast> + ?Sized, +{ + skip!(node.where_token); + for el in Punctuated::pairs(&node.predicates) { + let it = el.value(); + v.visit_where_predicate(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_where_predicate<'ast, V>(v: &mut V, node: &'ast crate::WherePredicate) +where + V: Visit<'ast> + ?Sized, +{ + match node { + crate::WherePredicate::Lifetime(_binding_0) => { + v.visit_predicate_lifetime(_binding_0); + } + crate::WherePredicate::Type(_binding_0) => { + v.visit_predicate_type(_binding_0); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/gen/visit_mut.rs b/third_party/crates/syn-2.0.119/src/gen/visit_mut.rs new file mode 100644 index 0000000..6588182 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/gen/visit_mut.rs @@ -0,0 +1,3761 @@ +// This file is @generated by syn-internal-codegen. +// It is not intended for manual editing. + +#![allow(unused_variables)] +#![allow(clippy::needless_pass_by_ref_mut)] +#[cfg(any(feature = "full", feature = "derive"))] +use crate::punctuated::Punctuated; +#[cfg(any(feature = "derive", feature = "full"))] +use alloc::vec::Vec; +#[cfg(feature = "full")] +macro_rules! full { + ($e:expr) => { + $e + }; +} +#[cfg(all(feature = "derive", not(feature = "full")))] +macro_rules! full { + ($e:expr) => { + unreachable!() + }; +} +macro_rules! skip { + ($($tt:tt)*) => {}; +} +/// Syntax tree traversal to mutate an exclusive borrow of a syntax tree in +/// place. +/// +/// See the [module documentation] for details. +/// +/// [module documentation]: self +pub trait VisitMut { + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_abi_mut(&mut self, i: &mut crate::Abi) { + visit_abi_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_angle_bracketed_generic_arguments_mut( + &mut self, + i: &mut crate::AngleBracketedGenericArguments, + ) { + visit_angle_bracketed_generic_arguments_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_arm_mut(&mut self, i: &mut crate::Arm) { + visit_arm_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_assoc_const_mut(&mut self, i: &mut crate::AssocConst) { + visit_assoc_const_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_assoc_type_mut(&mut self, i: &mut crate::AssocType) { + visit_assoc_type_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_attr_style_mut(&mut self, i: &mut crate::AttrStyle) { + visit_attr_style_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_attribute_mut(&mut self, i: &mut crate::Attribute) { + visit_attribute_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_attributes_mut(&mut self, i: &mut Vec<crate::Attribute>) { + for attr in i { + self.visit_attribute_mut(attr); + } + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_bare_fn_arg_mut(&mut self, i: &mut crate::BareFnArg) { + visit_bare_fn_arg_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_bare_variadic_mut(&mut self, i: &mut crate::BareVariadic) { + visit_bare_variadic_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_bin_op_mut(&mut self, i: &mut crate::BinOp) { + visit_bin_op_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_block_mut(&mut self, i: &mut crate::Block) { + visit_block_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_bound_lifetimes_mut(&mut self, i: &mut crate::BoundLifetimes) { + visit_bound_lifetimes_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_captured_param_mut(&mut self, i: &mut crate::CapturedParam) { + visit_captured_param_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_const_param_mut(&mut self, i: &mut crate::ConstParam) { + visit_const_param_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_constraint_mut(&mut self, i: &mut crate::Constraint) { + visit_constraint_mut(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_data_mut(&mut self, i: &mut crate::Data) { + visit_data_mut(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_data_enum_mut(&mut self, i: &mut crate::DataEnum) { + visit_data_enum_mut(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_data_struct_mut(&mut self, i: &mut crate::DataStruct) { + visit_data_struct_mut(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_data_union_mut(&mut self, i: &mut crate::DataUnion) { + visit_data_union_mut(self, i); + } + #[cfg(feature = "derive")] + #[cfg_attr(docsrs, doc(cfg(feature = "derive")))] + fn visit_derive_input_mut(&mut self, i: &mut crate::DeriveInput) { + visit_derive_input_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_mut(&mut self, i: &mut crate::Expr) { + visit_expr_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_array_mut(&mut self, i: &mut crate::ExprArray) { + visit_expr_array_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_assign_mut(&mut self, i: &mut crate::ExprAssign) { + visit_expr_assign_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_async_mut(&mut self, i: &mut crate::ExprAsync) { + visit_expr_async_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_await_mut(&mut self, i: &mut crate::ExprAwait) { + visit_expr_await_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_binary_mut(&mut self, i: &mut crate::ExprBinary) { + visit_expr_binary_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_block_mut(&mut self, i: &mut crate::ExprBlock) { + visit_expr_block_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_break_mut(&mut self, i: &mut crate::ExprBreak) { + visit_expr_break_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_call_mut(&mut self, i: &mut crate::ExprCall) { + visit_expr_call_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_cast_mut(&mut self, i: &mut crate::ExprCast) { + visit_expr_cast_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_closure_mut(&mut self, i: &mut crate::ExprClosure) { + visit_expr_closure_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_const_mut(&mut self, i: &mut crate::ExprConst) { + visit_expr_const_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_continue_mut(&mut self, i: &mut crate::ExprContinue) { + visit_expr_continue_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_field_mut(&mut self, i: &mut crate::ExprField) { + visit_expr_field_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_for_loop_mut(&mut self, i: &mut crate::ExprForLoop) { + visit_expr_for_loop_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_group_mut(&mut self, i: &mut crate::ExprGroup) { + visit_expr_group_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_if_mut(&mut self, i: &mut crate::ExprIf) { + visit_expr_if_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_index_mut(&mut self, i: &mut crate::ExprIndex) { + visit_expr_index_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_infer_mut(&mut self, i: &mut crate::ExprInfer) { + visit_expr_infer_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_let_mut(&mut self, i: &mut crate::ExprLet) { + visit_expr_let_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_lit_mut(&mut self, i: &mut crate::ExprLit) { + visit_expr_lit_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_loop_mut(&mut self, i: &mut crate::ExprLoop) { + visit_expr_loop_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_macro_mut(&mut self, i: &mut crate::ExprMacro) { + visit_expr_macro_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_match_mut(&mut self, i: &mut crate::ExprMatch) { + visit_expr_match_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_method_call_mut(&mut self, i: &mut crate::ExprMethodCall) { + visit_expr_method_call_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_paren_mut(&mut self, i: &mut crate::ExprParen) { + visit_expr_paren_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_path_mut(&mut self, i: &mut crate::ExprPath) { + visit_expr_path_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_range_mut(&mut self, i: &mut crate::ExprRange) { + visit_expr_range_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_raw_addr_mut(&mut self, i: &mut crate::ExprRawAddr) { + visit_expr_raw_addr_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_reference_mut(&mut self, i: &mut crate::ExprReference) { + visit_expr_reference_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_repeat_mut(&mut self, i: &mut crate::ExprRepeat) { + visit_expr_repeat_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_return_mut(&mut self, i: &mut crate::ExprReturn) { + visit_expr_return_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_struct_mut(&mut self, i: &mut crate::ExprStruct) { + visit_expr_struct_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_try_mut(&mut self, i: &mut crate::ExprTry) { + visit_expr_try_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_try_block_mut(&mut self, i: &mut crate::ExprTryBlock) { + visit_expr_try_block_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_tuple_mut(&mut self, i: &mut crate::ExprTuple) { + visit_expr_tuple_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_expr_unary_mut(&mut self, i: &mut crate::ExprUnary) { + visit_expr_unary_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_unsafe_mut(&mut self, i: &mut crate::ExprUnsafe) { + visit_expr_unsafe_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_while_mut(&mut self, i: &mut crate::ExprWhile) { + visit_expr_while_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_expr_yield_mut(&mut self, i: &mut crate::ExprYield) { + visit_expr_yield_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_field_mut(&mut self, i: &mut crate::Field) { + visit_field_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_field_mutability_mut(&mut self, i: &mut crate::FieldMutability) { + visit_field_mutability_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_field_pat_mut(&mut self, i: &mut crate::FieldPat) { + visit_field_pat_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_field_value_mut(&mut self, i: &mut crate::FieldValue) { + visit_field_value_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_fields_mut(&mut self, i: &mut crate::Fields) { + visit_fields_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_fields_named_mut(&mut self, i: &mut crate::FieldsNamed) { + visit_fields_named_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_fields_unnamed_mut(&mut self, i: &mut crate::FieldsUnnamed) { + visit_fields_unnamed_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_file_mut(&mut self, i: &mut crate::File) { + visit_file_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_fn_arg_mut(&mut self, i: &mut crate::FnArg) { + visit_fn_arg_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_mut(&mut self, i: &mut crate::ForeignItem) { + visit_foreign_item_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_fn_mut(&mut self, i: &mut crate::ForeignItemFn) { + visit_foreign_item_fn_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_macro_mut(&mut self, i: &mut crate::ForeignItemMacro) { + visit_foreign_item_macro_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_static_mut(&mut self, i: &mut crate::ForeignItemStatic) { + visit_foreign_item_static_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_foreign_item_type_mut(&mut self, i: &mut crate::ForeignItemType) { + visit_foreign_item_type_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_generic_argument_mut(&mut self, i: &mut crate::GenericArgument) { + visit_generic_argument_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_generic_param_mut(&mut self, i: &mut crate::GenericParam) { + visit_generic_param_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_generics_mut(&mut self, i: &mut crate::Generics) { + visit_generics_mut(self, i); + } + fn visit_ident_mut(&mut self, i: &mut proc_macro2::Ident) { + visit_ident_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_mut(&mut self, i: &mut crate::ImplItem) { + visit_impl_item_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_const_mut(&mut self, i: &mut crate::ImplItemConst) { + visit_impl_item_const_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_fn_mut(&mut self, i: &mut crate::ImplItemFn) { + visit_impl_item_fn_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_macro_mut(&mut self, i: &mut crate::ImplItemMacro) { + visit_impl_item_macro_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_item_type_mut(&mut self, i: &mut crate::ImplItemType) { + visit_impl_item_type_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_impl_restriction_mut(&mut self, i: &mut crate::ImplRestriction) { + visit_impl_restriction_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_index_mut(&mut self, i: &mut crate::Index) { + visit_index_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_mut(&mut self, i: &mut crate::Item) { + visit_item_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_const_mut(&mut self, i: &mut crate::ItemConst) { + visit_item_const_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_enum_mut(&mut self, i: &mut crate::ItemEnum) { + visit_item_enum_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_extern_crate_mut(&mut self, i: &mut crate::ItemExternCrate) { + visit_item_extern_crate_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_fn_mut(&mut self, i: &mut crate::ItemFn) { + visit_item_fn_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_foreign_mod_mut(&mut self, i: &mut crate::ItemForeignMod) { + visit_item_foreign_mod_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_impl_mut(&mut self, i: &mut crate::ItemImpl) { + visit_item_impl_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_macro_mut(&mut self, i: &mut crate::ItemMacro) { + visit_item_macro_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_mod_mut(&mut self, i: &mut crate::ItemMod) { + visit_item_mod_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_static_mut(&mut self, i: &mut crate::ItemStatic) { + visit_item_static_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_struct_mut(&mut self, i: &mut crate::ItemStruct) { + visit_item_struct_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_trait_mut(&mut self, i: &mut crate::ItemTrait) { + visit_item_trait_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_trait_alias_mut(&mut self, i: &mut crate::ItemTraitAlias) { + visit_item_trait_alias_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_type_mut(&mut self, i: &mut crate::ItemType) { + visit_item_type_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_union_mut(&mut self, i: &mut crate::ItemUnion) { + visit_item_union_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_item_use_mut(&mut self, i: &mut crate::ItemUse) { + visit_item_use_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_label_mut(&mut self, i: &mut crate::Label) { + visit_label_mut(self, i); + } + fn visit_lifetime_mut(&mut self, i: &mut crate::Lifetime) { + visit_lifetime_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_lifetime_param_mut(&mut self, i: &mut crate::LifetimeParam) { + visit_lifetime_param_mut(self, i); + } + fn visit_lit_mut(&mut self, i: &mut crate::Lit) { + visit_lit_mut(self, i); + } + fn visit_lit_bool_mut(&mut self, i: &mut crate::LitBool) { + visit_lit_bool_mut(self, i); + } + fn visit_lit_byte_mut(&mut self, i: &mut crate::LitByte) { + visit_lit_byte_mut(self, i); + } + fn visit_lit_byte_str_mut(&mut self, i: &mut crate::LitByteStr) { + visit_lit_byte_str_mut(self, i); + } + fn visit_lit_cstr_mut(&mut self, i: &mut crate::LitCStr) { + visit_lit_cstr_mut(self, i); + } + fn visit_lit_char_mut(&mut self, i: &mut crate::LitChar) { + visit_lit_char_mut(self, i); + } + fn visit_lit_float_mut(&mut self, i: &mut crate::LitFloat) { + visit_lit_float_mut(self, i); + } + fn visit_lit_int_mut(&mut self, i: &mut crate::LitInt) { + visit_lit_int_mut(self, i); + } + fn visit_lit_str_mut(&mut self, i: &mut crate::LitStr) { + visit_lit_str_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_local_mut(&mut self, i: &mut crate::Local) { + visit_local_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_local_init_mut(&mut self, i: &mut crate::LocalInit) { + visit_local_init_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_macro_mut(&mut self, i: &mut crate::Macro) { + visit_macro_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_macro_delimiter_mut(&mut self, i: &mut crate::MacroDelimiter) { + visit_macro_delimiter_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_member_mut(&mut self, i: &mut crate::Member) { + visit_member_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_meta_mut(&mut self, i: &mut crate::Meta) { + visit_meta_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_meta_list_mut(&mut self, i: &mut crate::MetaList) { + visit_meta_list_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_meta_name_value_mut(&mut self, i: &mut crate::MetaNameValue) { + visit_meta_name_value_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_parenthesized_generic_arguments_mut( + &mut self, + i: &mut crate::ParenthesizedGenericArguments, + ) { + visit_parenthesized_generic_arguments_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_mut(&mut self, i: &mut crate::Pat) { + visit_pat_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_ident_mut(&mut self, i: &mut crate::PatIdent) { + visit_pat_ident_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_or_mut(&mut self, i: &mut crate::PatOr) { + visit_pat_or_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_paren_mut(&mut self, i: &mut crate::PatParen) { + visit_pat_paren_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_reference_mut(&mut self, i: &mut crate::PatReference) { + visit_pat_reference_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_rest_mut(&mut self, i: &mut crate::PatRest) { + visit_pat_rest_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_slice_mut(&mut self, i: &mut crate::PatSlice) { + visit_pat_slice_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_struct_mut(&mut self, i: &mut crate::PatStruct) { + visit_pat_struct_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_tuple_mut(&mut self, i: &mut crate::PatTuple) { + visit_pat_tuple_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_tuple_struct_mut(&mut self, i: &mut crate::PatTupleStruct) { + visit_pat_tuple_struct_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_type_mut(&mut self, i: &mut crate::PatType) { + visit_pat_type_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pat_wild_mut(&mut self, i: &mut crate::PatWild) { + visit_pat_wild_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_path_mut(&mut self, i: &mut crate::Path) { + visit_path_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_path_arguments_mut(&mut self, i: &mut crate::PathArguments) { + visit_path_arguments_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_path_segment_mut(&mut self, i: &mut crate::PathSegment) { + visit_path_segment_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_pointer_mutability_mut(&mut self, i: &mut crate::PointerMutability) { + visit_pointer_mutability_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_precise_capture_mut(&mut self, i: &mut crate::PreciseCapture) { + visit_precise_capture_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_predicate_lifetime_mut(&mut self, i: &mut crate::PredicateLifetime) { + visit_predicate_lifetime_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_predicate_type_mut(&mut self, i: &mut crate::PredicateType) { + visit_predicate_type_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_qself_mut(&mut self, i: &mut crate::QSelf) { + visit_qself_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_range_limits_mut(&mut self, i: &mut crate::RangeLimits) { + visit_range_limits_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_receiver_mut(&mut self, i: &mut crate::Receiver) { + visit_receiver_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_return_type_mut(&mut self, i: &mut crate::ReturnType) { + visit_return_type_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_signature_mut(&mut self, i: &mut crate::Signature) { + visit_signature_mut(self, i); + } + fn visit_span_mut(&mut self, i: &mut proc_macro2::Span) {} + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_static_mutability_mut(&mut self, i: &mut crate::StaticMutability) { + visit_static_mutability_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_stmt_mut(&mut self, i: &mut crate::Stmt) { + visit_stmt_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_stmt_macro_mut(&mut self, i: &mut crate::StmtMacro) { + visit_stmt_macro_mut(self, i); + } + fn visit_token_stream_mut(&mut self, i: &mut proc_macro2::TokenStream) {} + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_trait_bound_mut(&mut self, i: &mut crate::TraitBound) { + visit_trait_bound_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_trait_bound_modifier_mut(&mut self, i: &mut crate::TraitBoundModifier) { + visit_trait_bound_modifier_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_mut(&mut self, i: &mut crate::TraitItem) { + visit_trait_item_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_const_mut(&mut self, i: &mut crate::TraitItemConst) { + visit_trait_item_const_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_fn_mut(&mut self, i: &mut crate::TraitItemFn) { + visit_trait_item_fn_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_macro_mut(&mut self, i: &mut crate::TraitItemMacro) { + visit_trait_item_macro_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_trait_item_type_mut(&mut self, i: &mut crate::TraitItemType) { + visit_trait_item_type_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_mut(&mut self, i: &mut crate::Type) { + visit_type_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_array_mut(&mut self, i: &mut crate::TypeArray) { + visit_type_array_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_bare_fn_mut(&mut self, i: &mut crate::TypeBareFn) { + visit_type_bare_fn_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_group_mut(&mut self, i: &mut crate::TypeGroup) { + visit_type_group_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_impl_trait_mut(&mut self, i: &mut crate::TypeImplTrait) { + visit_type_impl_trait_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_infer_mut(&mut self, i: &mut crate::TypeInfer) { + visit_type_infer_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_macro_mut(&mut self, i: &mut crate::TypeMacro) { + visit_type_macro_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_never_mut(&mut self, i: &mut crate::TypeNever) { + visit_type_never_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_param_mut(&mut self, i: &mut crate::TypeParam) { + visit_type_param_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_param_bound_mut(&mut self, i: &mut crate::TypeParamBound) { + visit_type_param_bound_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_paren_mut(&mut self, i: &mut crate::TypeParen) { + visit_type_paren_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_path_mut(&mut self, i: &mut crate::TypePath) { + visit_type_path_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_ptr_mut(&mut self, i: &mut crate::TypePtr) { + visit_type_ptr_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_reference_mut(&mut self, i: &mut crate::TypeReference) { + visit_type_reference_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_slice_mut(&mut self, i: &mut crate::TypeSlice) { + visit_type_slice_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_trait_object_mut(&mut self, i: &mut crate::TypeTraitObject) { + visit_type_trait_object_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_type_tuple_mut(&mut self, i: &mut crate::TypeTuple) { + visit_type_tuple_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_un_op_mut(&mut self, i: &mut crate::UnOp) { + visit_un_op_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_glob_mut(&mut self, i: &mut crate::UseGlob) { + visit_use_glob_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_group_mut(&mut self, i: &mut crate::UseGroup) { + visit_use_group_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_name_mut(&mut self, i: &mut crate::UseName) { + visit_use_name_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_path_mut(&mut self, i: &mut crate::UsePath) { + visit_use_path_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_rename_mut(&mut self, i: &mut crate::UseRename) { + visit_use_rename_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_use_tree_mut(&mut self, i: &mut crate::UseTree) { + visit_use_tree_mut(self, i); + } + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + fn visit_variadic_mut(&mut self, i: &mut crate::Variadic) { + visit_variadic_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_variant_mut(&mut self, i: &mut crate::Variant) { + visit_variant_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_vis_restricted_mut(&mut self, i: &mut crate::VisRestricted) { + visit_vis_restricted_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_visibility_mut(&mut self, i: &mut crate::Visibility) { + visit_visibility_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_where_clause_mut(&mut self, i: &mut crate::WhereClause) { + visit_where_clause_mut(self, i); + } + #[cfg(any(feature = "derive", feature = "full"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] + fn visit_where_predicate_mut(&mut self, i: &mut crate::WherePredicate) { + visit_where_predicate_mut(self, i); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_abi_mut<V>(v: &mut V, node: &mut crate::Abi) +where + V: VisitMut + ?Sized, +{ + skip!(node.extern_token); + if let Some(it) = &mut node.name { + v.visit_lit_str_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_angle_bracketed_generic_arguments_mut<V>( + v: &mut V, + node: &mut crate::AngleBracketedGenericArguments, +) +where + V: VisitMut + ?Sized, +{ + skip!(node.colon2_token); + skip!(node.lt_token); + for mut el in Punctuated::pairs_mut(&mut node.args) { + let it = el.value_mut(); + v.visit_generic_argument_mut(it); + } + skip!(node.gt_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_arm_mut<V>(v: &mut V, node: &mut crate::Arm) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_pat_mut(&mut node.pat); + if let Some(it) = &mut node.guard { + skip!((it).0); + v.visit_expr_mut(&mut *(it).1); + } + skip!(node.fat_arrow_token); + v.visit_expr_mut(&mut *node.body); + skip!(node.comma); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_assoc_const_mut<V>(v: &mut V, node: &mut crate::AssocConst) +where + V: VisitMut + ?Sized, +{ + v.visit_ident_mut(&mut node.ident); + if let Some(it) = &mut node.generics { + v.visit_angle_bracketed_generic_arguments_mut(it); + } + skip!(node.eq_token); + v.visit_expr_mut(&mut node.value); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_assoc_type_mut<V>(v: &mut V, node: &mut crate::AssocType) +where + V: VisitMut + ?Sized, +{ + v.visit_ident_mut(&mut node.ident); + if let Some(it) = &mut node.generics { + v.visit_angle_bracketed_generic_arguments_mut(it); + } + skip!(node.eq_token); + v.visit_type_mut(&mut node.ty); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_attr_style_mut<V>(v: &mut V, node: &mut crate::AttrStyle) +where + V: VisitMut + ?Sized, +{ + match node { + crate::AttrStyle::Outer => {} + crate::AttrStyle::Inner(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_attribute_mut<V>(v: &mut V, node: &mut crate::Attribute) +where + V: VisitMut + ?Sized, +{ + skip!(node.pound_token); + v.visit_attr_style_mut(&mut node.style); + skip!(node.bracket_token); + v.visit_meta_mut(&mut node.meta); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_bare_fn_arg_mut<V>(v: &mut V, node: &mut crate::BareFnArg) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.name { + v.visit_ident_mut(&mut (it).0); + skip!((it).1); + } + v.visit_type_mut(&mut node.ty); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_bare_variadic_mut<V>(v: &mut V, node: &mut crate::BareVariadic) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.name { + v.visit_ident_mut(&mut (it).0); + skip!((it).1); + } + skip!(node.dots); + skip!(node.comma); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_bin_op_mut<V>(v: &mut V, node: &mut crate::BinOp) +where + V: VisitMut + ?Sized, +{ + match node { + crate::BinOp::Add(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Sub(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Mul(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Div(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Rem(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::And(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Or(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitXor(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitAnd(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitOr(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Shl(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Shr(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Eq(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Lt(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Le(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Ne(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Ge(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::Gt(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::AddAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::SubAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::MulAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::DivAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::RemAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitXorAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitAndAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::BitOrAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::ShlAssign(_binding_0) => { + skip!(_binding_0); + } + crate::BinOp::ShrAssign(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_block_mut<V>(v: &mut V, node: &mut crate::Block) +where + V: VisitMut + ?Sized, +{ + skip!(node.brace_token); + for it in &mut node.stmts { + v.visit_stmt_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_bound_lifetimes_mut<V>(v: &mut V, node: &mut crate::BoundLifetimes) +where + V: VisitMut + ?Sized, +{ + skip!(node.for_token); + skip!(node.lt_token); + for mut el in Punctuated::pairs_mut(&mut node.lifetimes) { + let it = el.value_mut(); + v.visit_generic_param_mut(it); + } + skip!(node.gt_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_captured_param_mut<V>(v: &mut V, node: &mut crate::CapturedParam) +where + V: VisitMut + ?Sized, +{ + match node { + crate::CapturedParam::Lifetime(_binding_0) => { + v.visit_lifetime_mut(_binding_0); + } + crate::CapturedParam::Ident(_binding_0) => { + v.visit_ident_mut(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_const_param_mut<V>(v: &mut V, node: &mut crate::ConstParam) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.const_token); + v.visit_ident_mut(&mut node.ident); + skip!(node.colon_token); + v.visit_type_mut(&mut node.ty); + skip!(node.eq_token); + if let Some(it) = &mut node.default { + v.visit_expr_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_constraint_mut<V>(v: &mut V, node: &mut crate::Constraint) +where + V: VisitMut + ?Sized, +{ + v.visit_ident_mut(&mut node.ident); + if let Some(it) = &mut node.generics { + v.visit_angle_bracketed_generic_arguments_mut(it); + } + skip!(node.colon_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_type_param_bound_mut(it); + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_data_mut<V>(v: &mut V, node: &mut crate::Data) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Data::Struct(_binding_0) => { + v.visit_data_struct_mut(_binding_0); + } + crate::Data::Enum(_binding_0) => { + v.visit_data_enum_mut(_binding_0); + } + crate::Data::Union(_binding_0) => { + v.visit_data_union_mut(_binding_0); + } + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_data_enum_mut<V>(v: &mut V, node: &mut crate::DataEnum) +where + V: VisitMut + ?Sized, +{ + skip!(node.enum_token); + skip!(node.brace_token); + for mut el in Punctuated::pairs_mut(&mut node.variants) { + let it = el.value_mut(); + v.visit_variant_mut(it); + } +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_data_struct_mut<V>(v: &mut V, node: &mut crate::DataStruct) +where + V: VisitMut + ?Sized, +{ + skip!(node.struct_token); + v.visit_fields_mut(&mut node.fields); + skip!(node.semi_token); +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_data_union_mut<V>(v: &mut V, node: &mut crate::DataUnion) +where + V: VisitMut + ?Sized, +{ + skip!(node.union_token); + v.visit_fields_named_mut(&mut node.fields); +} +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub fn visit_derive_input_mut<V>(v: &mut V, node: &mut crate::DeriveInput) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + v.visit_data_mut(&mut node.data); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_mut<V>(v: &mut V, node: &mut crate::Expr) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Expr::Array(_binding_0) => { + full!(v.visit_expr_array_mut(_binding_0)); + } + crate::Expr::Assign(_binding_0) => { + full!(v.visit_expr_assign_mut(_binding_0)); + } + crate::Expr::Async(_binding_0) => { + full!(v.visit_expr_async_mut(_binding_0)); + } + crate::Expr::Await(_binding_0) => { + full!(v.visit_expr_await_mut(_binding_0)); + } + crate::Expr::Binary(_binding_0) => { + v.visit_expr_binary_mut(_binding_0); + } + crate::Expr::Block(_binding_0) => { + full!(v.visit_expr_block_mut(_binding_0)); + } + crate::Expr::Break(_binding_0) => { + full!(v.visit_expr_break_mut(_binding_0)); + } + crate::Expr::Call(_binding_0) => { + v.visit_expr_call_mut(_binding_0); + } + crate::Expr::Cast(_binding_0) => { + v.visit_expr_cast_mut(_binding_0); + } + crate::Expr::Closure(_binding_0) => { + full!(v.visit_expr_closure_mut(_binding_0)); + } + crate::Expr::Const(_binding_0) => { + full!(v.visit_expr_const_mut(_binding_0)); + } + crate::Expr::Continue(_binding_0) => { + full!(v.visit_expr_continue_mut(_binding_0)); + } + crate::Expr::Field(_binding_0) => { + v.visit_expr_field_mut(_binding_0); + } + crate::Expr::ForLoop(_binding_0) => { + full!(v.visit_expr_for_loop_mut(_binding_0)); + } + crate::Expr::Group(_binding_0) => { + v.visit_expr_group_mut(_binding_0); + } + crate::Expr::If(_binding_0) => { + full!(v.visit_expr_if_mut(_binding_0)); + } + crate::Expr::Index(_binding_0) => { + v.visit_expr_index_mut(_binding_0); + } + crate::Expr::Infer(_binding_0) => { + full!(v.visit_expr_infer_mut(_binding_0)); + } + crate::Expr::Let(_binding_0) => { + full!(v.visit_expr_let_mut(_binding_0)); + } + crate::Expr::Lit(_binding_0) => { + v.visit_expr_lit_mut(_binding_0); + } + crate::Expr::Loop(_binding_0) => { + full!(v.visit_expr_loop_mut(_binding_0)); + } + crate::Expr::Macro(_binding_0) => { + v.visit_expr_macro_mut(_binding_0); + } + crate::Expr::Match(_binding_0) => { + full!(v.visit_expr_match_mut(_binding_0)); + } + crate::Expr::MethodCall(_binding_0) => { + v.visit_expr_method_call_mut(_binding_0); + } + crate::Expr::Paren(_binding_0) => { + v.visit_expr_paren_mut(_binding_0); + } + crate::Expr::Path(_binding_0) => { + v.visit_expr_path_mut(_binding_0); + } + crate::Expr::Range(_binding_0) => { + full!(v.visit_expr_range_mut(_binding_0)); + } + crate::Expr::RawAddr(_binding_0) => { + full!(v.visit_expr_raw_addr_mut(_binding_0)); + } + crate::Expr::Reference(_binding_0) => { + v.visit_expr_reference_mut(_binding_0); + } + crate::Expr::Repeat(_binding_0) => { + full!(v.visit_expr_repeat_mut(_binding_0)); + } + crate::Expr::Return(_binding_0) => { + full!(v.visit_expr_return_mut(_binding_0)); + } + crate::Expr::Struct(_binding_0) => { + v.visit_expr_struct_mut(_binding_0); + } + crate::Expr::Try(_binding_0) => { + full!(v.visit_expr_try_mut(_binding_0)); + } + crate::Expr::TryBlock(_binding_0) => { + full!(v.visit_expr_try_block_mut(_binding_0)); + } + crate::Expr::Tuple(_binding_0) => { + v.visit_expr_tuple_mut(_binding_0); + } + crate::Expr::Unary(_binding_0) => { + v.visit_expr_unary_mut(_binding_0); + } + crate::Expr::Unsafe(_binding_0) => { + full!(v.visit_expr_unsafe_mut(_binding_0)); + } + crate::Expr::Verbatim(_binding_0) => { + v.visit_token_stream_mut(_binding_0); + } + crate::Expr::While(_binding_0) => { + full!(v.visit_expr_while_mut(_binding_0)); + } + crate::Expr::Yield(_binding_0) => { + full!(v.visit_expr_yield_mut(_binding_0)); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_array_mut<V>(v: &mut V, node: &mut crate::ExprArray) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.bracket_token); + for mut el in Punctuated::pairs_mut(&mut node.elems) { + let it = el.value_mut(); + v.visit_expr_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_assign_mut<V>(v: &mut V, node: &mut crate::ExprAssign) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.left); + skip!(node.eq_token); + v.visit_expr_mut(&mut *node.right); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_async_mut<V>(v: &mut V, node: &mut crate::ExprAsync) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.async_token); + skip!(node.capture); + v.visit_block_mut(&mut node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_await_mut<V>(v: &mut V, node: &mut crate::ExprAwait) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.base); + skip!(node.dot_token); + skip!(node.await_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_binary_mut<V>(v: &mut V, node: &mut crate::ExprBinary) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.left); + v.visit_bin_op_mut(&mut node.op); + v.visit_expr_mut(&mut *node.right); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_block_mut<V>(v: &mut V, node: &mut crate::ExprBlock) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.label { + v.visit_label_mut(it); + } + v.visit_block_mut(&mut node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_break_mut<V>(v: &mut V, node: &mut crate::ExprBreak) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.break_token); + if let Some(it) = &mut node.label { + v.visit_lifetime_mut(it); + } + if let Some(it) = &mut node.expr { + v.visit_expr_mut(&mut **it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_call_mut<V>(v: &mut V, node: &mut crate::ExprCall) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.func); + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.args) { + let it = el.value_mut(); + v.visit_expr_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_cast_mut<V>(v: &mut V, node: &mut crate::ExprCast) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.expr); + skip!(node.as_token); + v.visit_type_mut(&mut *node.ty); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_closure_mut<V>(v: &mut V, node: &mut crate::ExprClosure) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.lifetimes { + v.visit_bound_lifetimes_mut(it); + } + skip!(node.constness); + skip!(node.movability); + skip!(node.asyncness); + skip!(node.capture); + skip!(node.or1_token); + for mut el in Punctuated::pairs_mut(&mut node.inputs) { + let it = el.value_mut(); + v.visit_pat_mut(it); + } + skip!(node.or2_token); + v.visit_return_type_mut(&mut node.output); + v.visit_expr_mut(&mut *node.body); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_const_mut<V>(v: &mut V, node: &mut crate::ExprConst) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.const_token); + v.visit_block_mut(&mut node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_continue_mut<V>(v: &mut V, node: &mut crate::ExprContinue) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.continue_token); + if let Some(it) = &mut node.label { + v.visit_lifetime_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_field_mut<V>(v: &mut V, node: &mut crate::ExprField) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.base); + skip!(node.dot_token); + v.visit_member_mut(&mut node.member); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_for_loop_mut<V>(v: &mut V, node: &mut crate::ExprForLoop) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.label { + v.visit_label_mut(it); + } + skip!(node.for_token); + v.visit_pat_mut(&mut *node.pat); + skip!(node.in_token); + v.visit_expr_mut(&mut *node.expr); + v.visit_block_mut(&mut node.body); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_group_mut<V>(v: &mut V, node: &mut crate::ExprGroup) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.group_token); + v.visit_expr_mut(&mut *node.expr); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_if_mut<V>(v: &mut V, node: &mut crate::ExprIf) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.if_token); + v.visit_expr_mut(&mut *node.cond); + v.visit_block_mut(&mut node.then_branch); + if let Some(it) = &mut node.else_branch { + skip!((it).0); + v.visit_expr_mut(&mut *(it).1); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_index_mut<V>(v: &mut V, node: &mut crate::ExprIndex) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.expr); + skip!(node.bracket_token); + v.visit_expr_mut(&mut *node.index); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_infer_mut<V>(v: &mut V, node: &mut crate::ExprInfer) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.underscore_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_let_mut<V>(v: &mut V, node: &mut crate::ExprLet) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.let_token); + v.visit_pat_mut(&mut *node.pat); + skip!(node.eq_token); + v.visit_expr_mut(&mut *node.expr); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_lit_mut<V>(v: &mut V, node: &mut crate::ExprLit) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_lit_mut(&mut node.lit); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_loop_mut<V>(v: &mut V, node: &mut crate::ExprLoop) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.label { + v.visit_label_mut(it); + } + skip!(node.loop_token); + v.visit_block_mut(&mut node.body); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_macro_mut<V>(v: &mut V, node: &mut crate::ExprMacro) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_macro_mut(&mut node.mac); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_match_mut<V>(v: &mut V, node: &mut crate::ExprMatch) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.match_token); + v.visit_expr_mut(&mut *node.expr); + skip!(node.brace_token); + for it in &mut node.arms { + v.visit_arm_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_method_call_mut<V>(v: &mut V, node: &mut crate::ExprMethodCall) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.receiver); + skip!(node.dot_token); + v.visit_ident_mut(&mut node.method); + if let Some(it) = &mut node.turbofish { + v.visit_angle_bracketed_generic_arguments_mut(it); + } + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.args) { + let it = el.value_mut(); + v.visit_expr_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_paren_mut<V>(v: &mut V, node: &mut crate::ExprParen) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.paren_token); + v.visit_expr_mut(&mut *node.expr); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_path_mut<V>(v: &mut V, node: &mut crate::ExprPath) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.qself { + v.visit_qself_mut(it); + } + v.visit_path_mut(&mut node.path); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_range_mut<V>(v: &mut V, node: &mut crate::ExprRange) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.start { + v.visit_expr_mut(&mut **it); + } + v.visit_range_limits_mut(&mut node.limits); + if let Some(it) = &mut node.end { + v.visit_expr_mut(&mut **it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_raw_addr_mut<V>(v: &mut V, node: &mut crate::ExprRawAddr) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.and_token); + skip!(node.raw); + v.visit_pointer_mutability_mut(&mut node.mutability); + v.visit_expr_mut(&mut *node.expr); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_reference_mut<V>(v: &mut V, node: &mut crate::ExprReference) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.and_token); + skip!(node.mutability); + v.visit_expr_mut(&mut *node.expr); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_repeat_mut<V>(v: &mut V, node: &mut crate::ExprRepeat) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.bracket_token); + v.visit_expr_mut(&mut *node.expr); + skip!(node.semi_token); + v.visit_expr_mut(&mut *node.len); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_return_mut<V>(v: &mut V, node: &mut crate::ExprReturn) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.return_token); + if let Some(it) = &mut node.expr { + v.visit_expr_mut(&mut **it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_struct_mut<V>(v: &mut V, node: &mut crate::ExprStruct) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.qself { + v.visit_qself_mut(it); + } + v.visit_path_mut(&mut node.path); + skip!(node.brace_token); + for mut el in Punctuated::pairs_mut(&mut node.fields) { + let it = el.value_mut(); + v.visit_field_value_mut(it); + } + skip!(node.dot2_token); + if let Some(it) = &mut node.rest { + v.visit_expr_mut(&mut **it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_try_mut<V>(v: &mut V, node: &mut crate::ExprTry) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_expr_mut(&mut *node.expr); + skip!(node.question_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_try_block_mut<V>(v: &mut V, node: &mut crate::ExprTryBlock) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.try_token); + v.visit_block_mut(&mut node.block); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_tuple_mut<V>(v: &mut V, node: &mut crate::ExprTuple) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.elems) { + let it = el.value_mut(); + v.visit_expr_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_expr_unary_mut<V>(v: &mut V, node: &mut crate::ExprUnary) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_un_op_mut(&mut node.op); + v.visit_expr_mut(&mut *node.expr); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_unsafe_mut<V>(v: &mut V, node: &mut crate::ExprUnsafe) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.unsafe_token); + v.visit_block_mut(&mut node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_while_mut<V>(v: &mut V, node: &mut crate::ExprWhile) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.label { + v.visit_label_mut(it); + } + skip!(node.while_token); + v.visit_expr_mut(&mut *node.cond); + v.visit_block_mut(&mut node.body); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_expr_yield_mut<V>(v: &mut V, node: &mut crate::ExprYield) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.yield_token); + if let Some(it) = &mut node.expr { + v.visit_expr_mut(&mut **it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_field_mut<V>(v: &mut V, node: &mut crate::Field) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + v.visit_field_mutability_mut(&mut node.mutability); + if let Some(it) = &mut node.ident { + v.visit_ident_mut(it); + } + skip!(node.colon_token); + v.visit_type_mut(&mut node.ty); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_field_mutability_mut<V>(v: &mut V, node: &mut crate::FieldMutability) +where + V: VisitMut + ?Sized, +{ + match node { + crate::FieldMutability::None => {} + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_field_pat_mut<V>(v: &mut V, node: &mut crate::FieldPat) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_member_mut(&mut node.member); + skip!(node.colon_token); + v.visit_pat_mut(&mut *node.pat); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_field_value_mut<V>(v: &mut V, node: &mut crate::FieldValue) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_member_mut(&mut node.member); + skip!(node.colon_token); + v.visit_expr_mut(&mut node.expr); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_fields_mut<V>(v: &mut V, node: &mut crate::Fields) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Fields::Named(_binding_0) => { + v.visit_fields_named_mut(_binding_0); + } + crate::Fields::Unnamed(_binding_0) => { + v.visit_fields_unnamed_mut(_binding_0); + } + crate::Fields::Unit => {} + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_fields_named_mut<V>(v: &mut V, node: &mut crate::FieldsNamed) +where + V: VisitMut + ?Sized, +{ + skip!(node.brace_token); + for mut el in Punctuated::pairs_mut(&mut node.named) { + let it = el.value_mut(); + v.visit_field_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_fields_unnamed_mut<V>(v: &mut V, node: &mut crate::FieldsUnnamed) +where + V: VisitMut + ?Sized, +{ + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.unnamed) { + let it = el.value_mut(); + v.visit_field_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_file_mut<V>(v: &mut V, node: &mut crate::File) +where + V: VisitMut + ?Sized, +{ + skip!(node.shebang); + v.visit_attributes_mut(&mut node.attrs); + for it in &mut node.items { + v.visit_item_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_fn_arg_mut<V>(v: &mut V, node: &mut crate::FnArg) +where + V: VisitMut + ?Sized, +{ + match node { + crate::FnArg::Receiver(_binding_0) => { + v.visit_receiver_mut(_binding_0); + } + crate::FnArg::Typed(_binding_0) => { + v.visit_pat_type_mut(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_mut<V>(v: &mut V, node: &mut crate::ForeignItem) +where + V: VisitMut + ?Sized, +{ + match node { + crate::ForeignItem::Fn(_binding_0) => { + v.visit_foreign_item_fn_mut(_binding_0); + } + crate::ForeignItem::Static(_binding_0) => { + v.visit_foreign_item_static_mut(_binding_0); + } + crate::ForeignItem::Type(_binding_0) => { + v.visit_foreign_item_type_mut(_binding_0); + } + crate::ForeignItem::Macro(_binding_0) => { + v.visit_foreign_item_macro_mut(_binding_0); + } + crate::ForeignItem::Verbatim(_binding_0) => { + v.visit_token_stream_mut(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_fn_mut<V>(v: &mut V, node: &mut crate::ForeignItemFn) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + v.visit_signature_mut(&mut node.sig); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_macro_mut<V>(v: &mut V, node: &mut crate::ForeignItemMacro) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_macro_mut(&mut node.mac); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_static_mut<V>(v: &mut V, node: &mut crate::ForeignItemStatic) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.static_token); + v.visit_static_mutability_mut(&mut node.mutability); + v.visit_ident_mut(&mut node.ident); + skip!(node.colon_token); + v.visit_type_mut(&mut *node.ty); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_foreign_item_type_mut<V>(v: &mut V, node: &mut crate::ForeignItemType) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.type_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.semi_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_generic_argument_mut<V>(v: &mut V, node: &mut crate::GenericArgument) +where + V: VisitMut + ?Sized, +{ + match node { + crate::GenericArgument::Lifetime(_binding_0) => { + v.visit_lifetime_mut(_binding_0); + } + crate::GenericArgument::Type(_binding_0) => { + v.visit_type_mut(_binding_0); + } + crate::GenericArgument::Const(_binding_0) => { + v.visit_expr_mut(_binding_0); + } + crate::GenericArgument::AssocType(_binding_0) => { + v.visit_assoc_type_mut(_binding_0); + } + crate::GenericArgument::AssocConst(_binding_0) => { + v.visit_assoc_const_mut(_binding_0); + } + crate::GenericArgument::Constraint(_binding_0) => { + v.visit_constraint_mut(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_generic_param_mut<V>(v: &mut V, node: &mut crate::GenericParam) +where + V: VisitMut + ?Sized, +{ + match node { + crate::GenericParam::Lifetime(_binding_0) => { + v.visit_lifetime_param_mut(_binding_0); + } + crate::GenericParam::Type(_binding_0) => { + v.visit_type_param_mut(_binding_0); + } + crate::GenericParam::Const(_binding_0) => { + v.visit_const_param_mut(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_generics_mut<V>(v: &mut V, node: &mut crate::Generics) +where + V: VisitMut + ?Sized, +{ + skip!(node.lt_token); + for mut el in Punctuated::pairs_mut(&mut node.params) { + let it = el.value_mut(); + v.visit_generic_param_mut(it); + } + skip!(node.gt_token); + if let Some(it) = &mut node.where_clause { + v.visit_where_clause_mut(it); + } +} +pub fn visit_ident_mut<V>(v: &mut V, node: &mut proc_macro2::Ident) +where + V: VisitMut + ?Sized, +{ + let mut span = node.span(); + v.visit_span_mut(&mut span); + node.set_span(span); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_mut<V>(v: &mut V, node: &mut crate::ImplItem) +where + V: VisitMut + ?Sized, +{ + match node { + crate::ImplItem::Const(_binding_0) => { + v.visit_impl_item_const_mut(_binding_0); + } + crate::ImplItem::Fn(_binding_0) => { + v.visit_impl_item_fn_mut(_binding_0); + } + crate::ImplItem::Type(_binding_0) => { + v.visit_impl_item_type_mut(_binding_0); + } + crate::ImplItem::Macro(_binding_0) => { + v.visit_impl_item_macro_mut(_binding_0); + } + crate::ImplItem::Verbatim(_binding_0) => { + v.visit_token_stream_mut(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_const_mut<V>(v: &mut V, node: &mut crate::ImplItemConst) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.defaultness); + skip!(node.const_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.colon_token); + v.visit_type_mut(&mut node.ty); + skip!(node.eq_token); + v.visit_expr_mut(&mut node.expr); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_fn_mut<V>(v: &mut V, node: &mut crate::ImplItemFn) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.defaultness); + v.visit_signature_mut(&mut node.sig); + v.visit_block_mut(&mut node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_macro_mut<V>(v: &mut V, node: &mut crate::ImplItemMacro) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_macro_mut(&mut node.mac); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_item_type_mut<V>(v: &mut V, node: &mut crate::ImplItemType) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.defaultness); + skip!(node.type_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.eq_token); + v.visit_type_mut(&mut node.ty); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_impl_restriction_mut<V>(v: &mut V, node: &mut crate::ImplRestriction) +where + V: VisitMut + ?Sized, +{ + match *node {} +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_index_mut<V>(v: &mut V, node: &mut crate::Index) +where + V: VisitMut + ?Sized, +{ + skip!(node.index); + v.visit_span_mut(&mut node.span); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_mut<V>(v: &mut V, node: &mut crate::Item) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Item::Const(_binding_0) => { + v.visit_item_const_mut(_binding_0); + } + crate::Item::Enum(_binding_0) => { + v.visit_item_enum_mut(_binding_0); + } + crate::Item::ExternCrate(_binding_0) => { + v.visit_item_extern_crate_mut(_binding_0); + } + crate::Item::Fn(_binding_0) => { + v.visit_item_fn_mut(_binding_0); + } + crate::Item::ForeignMod(_binding_0) => { + v.visit_item_foreign_mod_mut(_binding_0); + } + crate::Item::Impl(_binding_0) => { + v.visit_item_impl_mut(_binding_0); + } + crate::Item::Macro(_binding_0) => { + v.visit_item_macro_mut(_binding_0); + } + crate::Item::Mod(_binding_0) => { + v.visit_item_mod_mut(_binding_0); + } + crate::Item::Static(_binding_0) => { + v.visit_item_static_mut(_binding_0); + } + crate::Item::Struct(_binding_0) => { + v.visit_item_struct_mut(_binding_0); + } + crate::Item::Trait(_binding_0) => { + v.visit_item_trait_mut(_binding_0); + } + crate::Item::TraitAlias(_binding_0) => { + v.visit_item_trait_alias_mut(_binding_0); + } + crate::Item::Type(_binding_0) => { + v.visit_item_type_mut(_binding_0); + } + crate::Item::Union(_binding_0) => { + v.visit_item_union_mut(_binding_0); + } + crate::Item::Use(_binding_0) => { + v.visit_item_use_mut(_binding_0); + } + crate::Item::Verbatim(_binding_0) => { + v.visit_token_stream_mut(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_const_mut<V>(v: &mut V, node: &mut crate::ItemConst) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.const_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.colon_token); + v.visit_type_mut(&mut *node.ty); + skip!(node.eq_token); + v.visit_expr_mut(&mut *node.expr); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_enum_mut<V>(v: &mut V, node: &mut crate::ItemEnum) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.enum_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.brace_token); + for mut el in Punctuated::pairs_mut(&mut node.variants) { + let it = el.value_mut(); + v.visit_variant_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_extern_crate_mut<V>(v: &mut V, node: &mut crate::ItemExternCrate) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.extern_token); + skip!(node.crate_token); + v.visit_ident_mut(&mut node.ident); + if let Some(it) = &mut node.rename { + skip!((it).0); + v.visit_ident_mut(&mut (it).1); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_fn_mut<V>(v: &mut V, node: &mut crate::ItemFn) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + v.visit_signature_mut(&mut node.sig); + v.visit_block_mut(&mut *node.block); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_foreign_mod_mut<V>(v: &mut V, node: &mut crate::ItemForeignMod) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.unsafety); + v.visit_abi_mut(&mut node.abi); + skip!(node.brace_token); + for it in &mut node.items { + v.visit_foreign_item_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_impl_mut<V>(v: &mut V, node: &mut crate::ItemImpl) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.defaultness); + skip!(node.unsafety); + skip!(node.impl_token); + v.visit_generics_mut(&mut node.generics); + if let Some(it) = &mut node.trait_ { + skip!((it).0); + v.visit_path_mut(&mut (it).1); + skip!((it).2); + } + v.visit_type_mut(&mut *node.self_ty); + skip!(node.brace_token); + for it in &mut node.items { + v.visit_impl_item_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_macro_mut<V>(v: &mut V, node: &mut crate::ItemMacro) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.ident { + v.visit_ident_mut(it); + } + v.visit_macro_mut(&mut node.mac); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_mod_mut<V>(v: &mut V, node: &mut crate::ItemMod) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.unsafety); + skip!(node.mod_token); + v.visit_ident_mut(&mut node.ident); + if let Some(it) = &mut node.content { + skip!((it).0); + for it in &mut (it).1 { + v.visit_item_mut(it); + } + } + skip!(node.semi); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_static_mut<V>(v: &mut V, node: &mut crate::ItemStatic) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.static_token); + v.visit_static_mutability_mut(&mut node.mutability); + v.visit_ident_mut(&mut node.ident); + skip!(node.colon_token); + v.visit_type_mut(&mut *node.ty); + skip!(node.eq_token); + v.visit_expr_mut(&mut *node.expr); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_struct_mut<V>(v: &mut V, node: &mut crate::ItemStruct) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.struct_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + v.visit_fields_mut(&mut node.fields); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_trait_mut<V>(v: &mut V, node: &mut crate::ItemTrait) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.unsafety); + skip!(node.auto_token); + if let Some(it) = &mut node.restriction { + v.visit_impl_restriction_mut(it); + } + skip!(node.trait_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.colon_token); + for mut el in Punctuated::pairs_mut(&mut node.supertraits) { + let it = el.value_mut(); + v.visit_type_param_bound_mut(it); + } + skip!(node.brace_token); + for it in &mut node.items { + v.visit_trait_item_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_trait_alias_mut<V>(v: &mut V, node: &mut crate::ItemTraitAlias) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.trait_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.eq_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_type_param_bound_mut(it); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_type_mut<V>(v: &mut V, node: &mut crate::ItemType) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.type_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.eq_token); + v.visit_type_mut(&mut *node.ty); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_union_mut<V>(v: &mut V, node: &mut crate::ItemUnion) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.union_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + v.visit_fields_named_mut(&mut node.fields); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_item_use_mut<V>(v: &mut V, node: &mut crate::ItemUse) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_visibility_mut(&mut node.vis); + skip!(node.use_token); + skip!(node.leading_colon); + v.visit_use_tree_mut(&mut node.tree); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_label_mut<V>(v: &mut V, node: &mut crate::Label) +where + V: VisitMut + ?Sized, +{ + v.visit_lifetime_mut(&mut node.name); + skip!(node.colon_token); +} +pub fn visit_lifetime_mut<V>(v: &mut V, node: &mut crate::Lifetime) +where + V: VisitMut + ?Sized, +{ + v.visit_span_mut(&mut node.apostrophe); + v.visit_ident_mut(&mut node.ident); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_lifetime_param_mut<V>(v: &mut V, node: &mut crate::LifetimeParam) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_lifetime_mut(&mut node.lifetime); + skip!(node.colon_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_lifetime_mut(it); + } +} +pub fn visit_lit_mut<V>(v: &mut V, node: &mut crate::Lit) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Lit::Str(_binding_0) => { + v.visit_lit_str_mut(_binding_0); + } + crate::Lit::ByteStr(_binding_0) => { + v.visit_lit_byte_str_mut(_binding_0); + } + crate::Lit::CStr(_binding_0) => { + v.visit_lit_cstr_mut(_binding_0); + } + crate::Lit::Byte(_binding_0) => { + v.visit_lit_byte_mut(_binding_0); + } + crate::Lit::Char(_binding_0) => { + v.visit_lit_char_mut(_binding_0); + } + crate::Lit::Int(_binding_0) => { + v.visit_lit_int_mut(_binding_0); + } + crate::Lit::Float(_binding_0) => { + v.visit_lit_float_mut(_binding_0); + } + crate::Lit::Bool(_binding_0) => { + v.visit_lit_bool_mut(_binding_0); + } + crate::Lit::Verbatim(_binding_0) => { + skip!(_binding_0); + } + } +} +pub fn visit_lit_bool_mut<V>(v: &mut V, node: &mut crate::LitBool) +where + V: VisitMut + ?Sized, +{ + skip!(node.value); + v.visit_span_mut(&mut node.span); +} +pub fn visit_lit_byte_mut<V>(v: &mut V, node: &mut crate::LitByte) +where + V: VisitMut + ?Sized, +{} +pub fn visit_lit_byte_str_mut<V>(v: &mut V, node: &mut crate::LitByteStr) +where + V: VisitMut + ?Sized, +{} +pub fn visit_lit_cstr_mut<V>(v: &mut V, node: &mut crate::LitCStr) +where + V: VisitMut + ?Sized, +{} +pub fn visit_lit_char_mut<V>(v: &mut V, node: &mut crate::LitChar) +where + V: VisitMut + ?Sized, +{} +pub fn visit_lit_float_mut<V>(v: &mut V, node: &mut crate::LitFloat) +where + V: VisitMut + ?Sized, +{} +pub fn visit_lit_int_mut<V>(v: &mut V, node: &mut crate::LitInt) +where + V: VisitMut + ?Sized, +{} +pub fn visit_lit_str_mut<V>(v: &mut V, node: &mut crate::LitStr) +where + V: VisitMut + ?Sized, +{} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_local_mut<V>(v: &mut V, node: &mut crate::Local) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.let_token); + v.visit_pat_mut(&mut node.pat); + if let Some(it) = &mut node.init { + v.visit_local_init_mut(it); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_local_init_mut<V>(v: &mut V, node: &mut crate::LocalInit) +where + V: VisitMut + ?Sized, +{ + skip!(node.eq_token); + v.visit_expr_mut(&mut *node.expr); + if let Some(it) = &mut node.diverge { + skip!((it).0); + v.visit_expr_mut(&mut *(it).1); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_macro_mut<V>(v: &mut V, node: &mut crate::Macro) +where + V: VisitMut + ?Sized, +{ + v.visit_path_mut(&mut node.path); + skip!(node.bang_token); + v.visit_macro_delimiter_mut(&mut node.delimiter); + v.visit_token_stream_mut(&mut node.tokens); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_macro_delimiter_mut<V>(v: &mut V, node: &mut crate::MacroDelimiter) +where + V: VisitMut + ?Sized, +{ + match node { + crate::MacroDelimiter::Paren(_binding_0) => { + skip!(_binding_0); + } + crate::MacroDelimiter::Brace(_binding_0) => { + skip!(_binding_0); + } + crate::MacroDelimiter::Bracket(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_member_mut<V>(v: &mut V, node: &mut crate::Member) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Member::Named(_binding_0) => { + v.visit_ident_mut(_binding_0); + } + crate::Member::Unnamed(_binding_0) => { + v.visit_index_mut(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_meta_mut<V>(v: &mut V, node: &mut crate::Meta) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Meta::Path(_binding_0) => { + v.visit_path_mut(_binding_0); + } + crate::Meta::List(_binding_0) => { + v.visit_meta_list_mut(_binding_0); + } + crate::Meta::NameValue(_binding_0) => { + v.visit_meta_name_value_mut(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_meta_list_mut<V>(v: &mut V, node: &mut crate::MetaList) +where + V: VisitMut + ?Sized, +{ + v.visit_path_mut(&mut node.path); + v.visit_macro_delimiter_mut(&mut node.delimiter); + v.visit_token_stream_mut(&mut node.tokens); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_meta_name_value_mut<V>(v: &mut V, node: &mut crate::MetaNameValue) +where + V: VisitMut + ?Sized, +{ + v.visit_path_mut(&mut node.path); + skip!(node.eq_token); + v.visit_expr_mut(&mut node.value); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_parenthesized_generic_arguments_mut<V>( + v: &mut V, + node: &mut crate::ParenthesizedGenericArguments, +) +where + V: VisitMut + ?Sized, +{ + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.inputs) { + let it = el.value_mut(); + v.visit_type_mut(it); + } + v.visit_return_type_mut(&mut node.output); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_mut<V>(v: &mut V, node: &mut crate::Pat) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Pat::Const(_binding_0) => { + v.visit_expr_const_mut(_binding_0); + } + crate::Pat::Ident(_binding_0) => { + v.visit_pat_ident_mut(_binding_0); + } + crate::Pat::Lit(_binding_0) => { + v.visit_expr_lit_mut(_binding_0); + } + crate::Pat::Macro(_binding_0) => { + v.visit_expr_macro_mut(_binding_0); + } + crate::Pat::Or(_binding_0) => { + v.visit_pat_or_mut(_binding_0); + } + crate::Pat::Paren(_binding_0) => { + v.visit_pat_paren_mut(_binding_0); + } + crate::Pat::Path(_binding_0) => { + v.visit_expr_path_mut(_binding_0); + } + crate::Pat::Range(_binding_0) => { + v.visit_expr_range_mut(_binding_0); + } + crate::Pat::Reference(_binding_0) => { + v.visit_pat_reference_mut(_binding_0); + } + crate::Pat::Rest(_binding_0) => { + v.visit_pat_rest_mut(_binding_0); + } + crate::Pat::Slice(_binding_0) => { + v.visit_pat_slice_mut(_binding_0); + } + crate::Pat::Struct(_binding_0) => { + v.visit_pat_struct_mut(_binding_0); + } + crate::Pat::Tuple(_binding_0) => { + v.visit_pat_tuple_mut(_binding_0); + } + crate::Pat::TupleStruct(_binding_0) => { + v.visit_pat_tuple_struct_mut(_binding_0); + } + crate::Pat::Type(_binding_0) => { + v.visit_pat_type_mut(_binding_0); + } + crate::Pat::Verbatim(_binding_0) => { + v.visit_token_stream_mut(_binding_0); + } + crate::Pat::Wild(_binding_0) => { + v.visit_pat_wild_mut(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_ident_mut<V>(v: &mut V, node: &mut crate::PatIdent) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.by_ref); + skip!(node.mutability); + v.visit_ident_mut(&mut node.ident); + if let Some(it) = &mut node.subpat { + skip!((it).0); + v.visit_pat_mut(&mut *(it).1); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_or_mut<V>(v: &mut V, node: &mut crate::PatOr) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.leading_vert); + for mut el in Punctuated::pairs_mut(&mut node.cases) { + let it = el.value_mut(); + v.visit_pat_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_paren_mut<V>(v: &mut V, node: &mut crate::PatParen) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.paren_token); + v.visit_pat_mut(&mut *node.pat); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_reference_mut<V>(v: &mut V, node: &mut crate::PatReference) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.and_token); + skip!(node.mutability); + v.visit_pat_mut(&mut *node.pat); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_rest_mut<V>(v: &mut V, node: &mut crate::PatRest) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.dot2_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_slice_mut<V>(v: &mut V, node: &mut crate::PatSlice) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.bracket_token); + for mut el in Punctuated::pairs_mut(&mut node.elems) { + let it = el.value_mut(); + v.visit_pat_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_struct_mut<V>(v: &mut V, node: &mut crate::PatStruct) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.qself { + v.visit_qself_mut(it); + } + v.visit_path_mut(&mut node.path); + skip!(node.brace_token); + for mut el in Punctuated::pairs_mut(&mut node.fields) { + let it = el.value_mut(); + v.visit_field_pat_mut(it); + } + if let Some(it) = &mut node.rest { + v.visit_pat_rest_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_tuple_mut<V>(v: &mut V, node: &mut crate::PatTuple) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.elems) { + let it = el.value_mut(); + v.visit_pat_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_tuple_struct_mut<V>(v: &mut V, node: &mut crate::PatTupleStruct) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.qself { + v.visit_qself_mut(it); + } + v.visit_path_mut(&mut node.path); + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.elems) { + let it = el.value_mut(); + v.visit_pat_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_type_mut<V>(v: &mut V, node: &mut crate::PatType) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_pat_mut(&mut *node.pat); + skip!(node.colon_token); + v.visit_type_mut(&mut *node.ty); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pat_wild_mut<V>(v: &mut V, node: &mut crate::PatWild) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.underscore_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_path_mut<V>(v: &mut V, node: &mut crate::Path) +where + V: VisitMut + ?Sized, +{ + skip!(node.leading_colon); + for mut el in Punctuated::pairs_mut(&mut node.segments) { + let it = el.value_mut(); + v.visit_path_segment_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_path_arguments_mut<V>(v: &mut V, node: &mut crate::PathArguments) +where + V: VisitMut + ?Sized, +{ + match node { + crate::PathArguments::None => {} + crate::PathArguments::AngleBracketed(_binding_0) => { + v.visit_angle_bracketed_generic_arguments_mut(_binding_0); + } + crate::PathArguments::Parenthesized(_binding_0) => { + v.visit_parenthesized_generic_arguments_mut(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_path_segment_mut<V>(v: &mut V, node: &mut crate::PathSegment) +where + V: VisitMut + ?Sized, +{ + v.visit_ident_mut(&mut node.ident); + v.visit_path_arguments_mut(&mut node.arguments); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_pointer_mutability_mut<V>(v: &mut V, node: &mut crate::PointerMutability) +where + V: VisitMut + ?Sized, +{ + match node { + crate::PointerMutability::Const(_binding_0) => { + skip!(_binding_0); + } + crate::PointerMutability::Mut(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_precise_capture_mut<V>(v: &mut V, node: &mut crate::PreciseCapture) +where + V: VisitMut + ?Sized, +{ + skip!(node.use_token); + skip!(node.lt_token); + for mut el in Punctuated::pairs_mut(&mut node.params) { + let it = el.value_mut(); + v.visit_captured_param_mut(it); + } + skip!(node.gt_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_predicate_lifetime_mut<V>(v: &mut V, node: &mut crate::PredicateLifetime) +where + V: VisitMut + ?Sized, +{ + v.visit_lifetime_mut(&mut node.lifetime); + skip!(node.colon_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_lifetime_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_predicate_type_mut<V>(v: &mut V, node: &mut crate::PredicateType) +where + V: VisitMut + ?Sized, +{ + if let Some(it) = &mut node.lifetimes { + v.visit_bound_lifetimes_mut(it); + } + v.visit_type_mut(&mut node.bounded_ty); + skip!(node.colon_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_type_param_bound_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_qself_mut<V>(v: &mut V, node: &mut crate::QSelf) +where + V: VisitMut + ?Sized, +{ + skip!(node.lt_token); + v.visit_type_mut(&mut *node.ty); + skip!(node.position); + skip!(node.as_token); + skip!(node.gt_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_range_limits_mut<V>(v: &mut V, node: &mut crate::RangeLimits) +where + V: VisitMut + ?Sized, +{ + match node { + crate::RangeLimits::HalfOpen(_binding_0) => { + skip!(_binding_0); + } + crate::RangeLimits::Closed(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_receiver_mut<V>(v: &mut V, node: &mut crate::Receiver) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.reference { + skip!((it).0); + if let Some(it) = &mut (it).1 { + v.visit_lifetime_mut(it); + } + } + skip!(node.mutability); + skip!(node.self_token); + skip!(node.colon_token); + v.visit_type_mut(&mut *node.ty); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_return_type_mut<V>(v: &mut V, node: &mut crate::ReturnType) +where + V: VisitMut + ?Sized, +{ + match node { + crate::ReturnType::Default => {} + crate::ReturnType::Type(_binding_0, _binding_1) => { + skip!(_binding_0); + v.visit_type_mut(&mut **_binding_1); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_signature_mut<V>(v: &mut V, node: &mut crate::Signature) +where + V: VisitMut + ?Sized, +{ + skip!(node.constness); + skip!(node.asyncness); + skip!(node.unsafety); + if let Some(it) = &mut node.abi { + v.visit_abi_mut(it); + } + skip!(node.fn_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.inputs) { + let it = el.value_mut(); + v.visit_fn_arg_mut(it); + } + if let Some(it) = &mut node.variadic { + v.visit_variadic_mut(it); + } + v.visit_return_type_mut(&mut node.output); +} +pub fn visit_span_mut<V>(v: &mut V, node: &mut proc_macro2::Span) +where + V: VisitMut + ?Sized, +{} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_static_mutability_mut<V>(v: &mut V, node: &mut crate::StaticMutability) +where + V: VisitMut + ?Sized, +{ + match node { + crate::StaticMutability::Mut(_binding_0) => { + skip!(_binding_0); + } + crate::StaticMutability::None => {} + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_stmt_mut<V>(v: &mut V, node: &mut crate::Stmt) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Stmt::Local(_binding_0) => { + v.visit_local_mut(_binding_0); + } + crate::Stmt::Item(_binding_0) => { + v.visit_item_mut(_binding_0); + } + crate::Stmt::Expr(_binding_0, _binding_1) => { + v.visit_expr_mut(_binding_0); + skip!(_binding_1); + } + crate::Stmt::Macro(_binding_0) => { + v.visit_stmt_macro_mut(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_stmt_macro_mut<V>(v: &mut V, node: &mut crate::StmtMacro) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_macro_mut(&mut node.mac); + skip!(node.semi_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_trait_bound_mut<V>(v: &mut V, node: &mut crate::TraitBound) +where + V: VisitMut + ?Sized, +{ + skip!(node.paren_token); + v.visit_trait_bound_modifier_mut(&mut node.modifier); + if let Some(it) = &mut node.lifetimes { + v.visit_bound_lifetimes_mut(it); + } + v.visit_path_mut(&mut node.path); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_trait_bound_modifier_mut<V>(v: &mut V, node: &mut crate::TraitBoundModifier) +where + V: VisitMut + ?Sized, +{ + match node { + crate::TraitBoundModifier::None => {} + crate::TraitBoundModifier::Maybe(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_mut<V>(v: &mut V, node: &mut crate::TraitItem) +where + V: VisitMut + ?Sized, +{ + match node { + crate::TraitItem::Const(_binding_0) => { + v.visit_trait_item_const_mut(_binding_0); + } + crate::TraitItem::Fn(_binding_0) => { + v.visit_trait_item_fn_mut(_binding_0); + } + crate::TraitItem::Type(_binding_0) => { + v.visit_trait_item_type_mut(_binding_0); + } + crate::TraitItem::Macro(_binding_0) => { + v.visit_trait_item_macro_mut(_binding_0); + } + crate::TraitItem::Verbatim(_binding_0) => { + v.visit_token_stream_mut(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_const_mut<V>(v: &mut V, node: &mut crate::TraitItemConst) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.const_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.colon_token); + v.visit_type_mut(&mut node.ty); + if let Some(it) = &mut node.default { + skip!((it).0); + v.visit_expr_mut(&mut (it).1); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_fn_mut<V>(v: &mut V, node: &mut crate::TraitItemFn) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_signature_mut(&mut node.sig); + if let Some(it) = &mut node.default { + v.visit_block_mut(it); + } + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_macro_mut<V>(v: &mut V, node: &mut crate::TraitItemMacro) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_macro_mut(&mut node.mac); + skip!(node.semi_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_trait_item_type_mut<V>(v: &mut V, node: &mut crate::TraitItemType) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + skip!(node.type_token); + v.visit_ident_mut(&mut node.ident); + v.visit_generics_mut(&mut node.generics); + skip!(node.colon_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_type_param_bound_mut(it); + } + if let Some(it) = &mut node.default { + skip!((it).0); + v.visit_type_mut(&mut (it).1); + } + skip!(node.semi_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_mut<V>(v: &mut V, node: &mut crate::Type) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Type::Array(_binding_0) => { + v.visit_type_array_mut(_binding_0); + } + crate::Type::BareFn(_binding_0) => { + v.visit_type_bare_fn_mut(_binding_0); + } + crate::Type::Group(_binding_0) => { + v.visit_type_group_mut(_binding_0); + } + crate::Type::ImplTrait(_binding_0) => { + v.visit_type_impl_trait_mut(_binding_0); + } + crate::Type::Infer(_binding_0) => { + v.visit_type_infer_mut(_binding_0); + } + crate::Type::Macro(_binding_0) => { + v.visit_type_macro_mut(_binding_0); + } + crate::Type::Never(_binding_0) => { + v.visit_type_never_mut(_binding_0); + } + crate::Type::Paren(_binding_0) => { + v.visit_type_paren_mut(_binding_0); + } + crate::Type::Path(_binding_0) => { + v.visit_type_path_mut(_binding_0); + } + crate::Type::Ptr(_binding_0) => { + v.visit_type_ptr_mut(_binding_0); + } + crate::Type::Reference(_binding_0) => { + v.visit_type_reference_mut(_binding_0); + } + crate::Type::Slice(_binding_0) => { + v.visit_type_slice_mut(_binding_0); + } + crate::Type::TraitObject(_binding_0) => { + v.visit_type_trait_object_mut(_binding_0); + } + crate::Type::Tuple(_binding_0) => { + v.visit_type_tuple_mut(_binding_0); + } + crate::Type::Verbatim(_binding_0) => { + v.visit_token_stream_mut(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_array_mut<V>(v: &mut V, node: &mut crate::TypeArray) +where + V: VisitMut + ?Sized, +{ + skip!(node.bracket_token); + v.visit_type_mut(&mut *node.elem); + skip!(node.semi_token); + v.visit_expr_mut(&mut node.len); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_bare_fn_mut<V>(v: &mut V, node: &mut crate::TypeBareFn) +where + V: VisitMut + ?Sized, +{ + if let Some(it) = &mut node.lifetimes { + v.visit_bound_lifetimes_mut(it); + } + skip!(node.unsafety); + if let Some(it) = &mut node.abi { + v.visit_abi_mut(it); + } + skip!(node.fn_token); + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.inputs) { + let it = el.value_mut(); + v.visit_bare_fn_arg_mut(it); + } + if let Some(it) = &mut node.variadic { + v.visit_bare_variadic_mut(it); + } + v.visit_return_type_mut(&mut node.output); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_group_mut<V>(v: &mut V, node: &mut crate::TypeGroup) +where + V: VisitMut + ?Sized, +{ + skip!(node.group_token); + v.visit_type_mut(&mut *node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_impl_trait_mut<V>(v: &mut V, node: &mut crate::TypeImplTrait) +where + V: VisitMut + ?Sized, +{ + skip!(node.impl_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_type_param_bound_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_infer_mut<V>(v: &mut V, node: &mut crate::TypeInfer) +where + V: VisitMut + ?Sized, +{ + skip!(node.underscore_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_macro_mut<V>(v: &mut V, node: &mut crate::TypeMacro) +where + V: VisitMut + ?Sized, +{ + v.visit_macro_mut(&mut node.mac); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_never_mut<V>(v: &mut V, node: &mut crate::TypeNever) +where + V: VisitMut + ?Sized, +{ + skip!(node.bang_token); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_param_mut<V>(v: &mut V, node: &mut crate::TypeParam) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_ident_mut(&mut node.ident); + skip!(node.colon_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_type_param_bound_mut(it); + } + skip!(node.eq_token); + if let Some(it) = &mut node.default { + v.visit_type_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_param_bound_mut<V>(v: &mut V, node: &mut crate::TypeParamBound) +where + V: VisitMut + ?Sized, +{ + match node { + crate::TypeParamBound::Trait(_binding_0) => { + v.visit_trait_bound_mut(_binding_0); + } + crate::TypeParamBound::Lifetime(_binding_0) => { + v.visit_lifetime_mut(_binding_0); + } + crate::TypeParamBound::PreciseCapture(_binding_0) => { + full!(v.visit_precise_capture_mut(_binding_0)); + } + crate::TypeParamBound::Verbatim(_binding_0) => { + v.visit_token_stream_mut(_binding_0); + } + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_paren_mut<V>(v: &mut V, node: &mut crate::TypeParen) +where + V: VisitMut + ?Sized, +{ + skip!(node.paren_token); + v.visit_type_mut(&mut *node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_path_mut<V>(v: &mut V, node: &mut crate::TypePath) +where + V: VisitMut + ?Sized, +{ + if let Some(it) = &mut node.qself { + v.visit_qself_mut(it); + } + v.visit_path_mut(&mut node.path); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_ptr_mut<V>(v: &mut V, node: &mut crate::TypePtr) +where + V: VisitMut + ?Sized, +{ + skip!(node.star_token); + skip!(node.const_token); + skip!(node.mutability); + v.visit_type_mut(&mut *node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_reference_mut<V>(v: &mut V, node: &mut crate::TypeReference) +where + V: VisitMut + ?Sized, +{ + skip!(node.and_token); + if let Some(it) = &mut node.lifetime { + v.visit_lifetime_mut(it); + } + skip!(node.mutability); + v.visit_type_mut(&mut *node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_slice_mut<V>(v: &mut V, node: &mut crate::TypeSlice) +where + V: VisitMut + ?Sized, +{ + skip!(node.bracket_token); + v.visit_type_mut(&mut *node.elem); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_trait_object_mut<V>(v: &mut V, node: &mut crate::TypeTraitObject) +where + V: VisitMut + ?Sized, +{ + skip!(node.dyn_token); + for mut el in Punctuated::pairs_mut(&mut node.bounds) { + let it = el.value_mut(); + v.visit_type_param_bound_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_type_tuple_mut<V>(v: &mut V, node: &mut crate::TypeTuple) +where + V: VisitMut + ?Sized, +{ + skip!(node.paren_token); + for mut el in Punctuated::pairs_mut(&mut node.elems) { + let it = el.value_mut(); + v.visit_type_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_un_op_mut<V>(v: &mut V, node: &mut crate::UnOp) +where + V: VisitMut + ?Sized, +{ + match node { + crate::UnOp::Deref(_binding_0) => { + skip!(_binding_0); + } + crate::UnOp::Not(_binding_0) => { + skip!(_binding_0); + } + crate::UnOp::Neg(_binding_0) => { + skip!(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_glob_mut<V>(v: &mut V, node: &mut crate::UseGlob) +where + V: VisitMut + ?Sized, +{ + skip!(node.star_token); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_group_mut<V>(v: &mut V, node: &mut crate::UseGroup) +where + V: VisitMut + ?Sized, +{ + skip!(node.brace_token); + for mut el in Punctuated::pairs_mut(&mut node.items) { + let it = el.value_mut(); + v.visit_use_tree_mut(it); + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_name_mut<V>(v: &mut V, node: &mut crate::UseName) +where + V: VisitMut + ?Sized, +{ + v.visit_ident_mut(&mut node.ident); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_path_mut<V>(v: &mut V, node: &mut crate::UsePath) +where + V: VisitMut + ?Sized, +{ + v.visit_ident_mut(&mut node.ident); + skip!(node.colon2_token); + v.visit_use_tree_mut(&mut *node.tree); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_rename_mut<V>(v: &mut V, node: &mut crate::UseRename) +where + V: VisitMut + ?Sized, +{ + v.visit_ident_mut(&mut node.ident); + skip!(node.as_token); + v.visit_ident_mut(&mut node.rename); +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_use_tree_mut<V>(v: &mut V, node: &mut crate::UseTree) +where + V: VisitMut + ?Sized, +{ + match node { + crate::UseTree::Path(_binding_0) => { + v.visit_use_path_mut(_binding_0); + } + crate::UseTree::Name(_binding_0) => { + v.visit_use_name_mut(_binding_0); + } + crate::UseTree::Rename(_binding_0) => { + v.visit_use_rename_mut(_binding_0); + } + crate::UseTree::Glob(_binding_0) => { + v.visit_use_glob_mut(_binding_0); + } + crate::UseTree::Group(_binding_0) => { + v.visit_use_group_mut(_binding_0); + } + } +} +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub fn visit_variadic_mut<V>(v: &mut V, node: &mut crate::Variadic) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + if let Some(it) = &mut node.pat { + v.visit_pat_mut(&mut *(it).0); + skip!((it).1); + } + skip!(node.dots); + skip!(node.comma); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_variant_mut<V>(v: &mut V, node: &mut crate::Variant) +where + V: VisitMut + ?Sized, +{ + v.visit_attributes_mut(&mut node.attrs); + v.visit_ident_mut(&mut node.ident); + v.visit_fields_mut(&mut node.fields); + if let Some(it) = &mut node.discriminant { + skip!((it).0); + v.visit_expr_mut(&mut (it).1); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_vis_restricted_mut<V>(v: &mut V, node: &mut crate::VisRestricted) +where + V: VisitMut + ?Sized, +{ + skip!(node.pub_token); + skip!(node.paren_token); + skip!(node.in_token); + v.visit_path_mut(&mut *node.path); +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_visibility_mut<V>(v: &mut V, node: &mut crate::Visibility) +where + V: VisitMut + ?Sized, +{ + match node { + crate::Visibility::Public(_binding_0) => { + skip!(_binding_0); + } + crate::Visibility::Restricted(_binding_0) => { + v.visit_vis_restricted_mut(_binding_0); + } + crate::Visibility::Inherited => {} + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_where_clause_mut<V>(v: &mut V, node: &mut crate::WhereClause) +where + V: VisitMut + ?Sized, +{ + skip!(node.where_token); + for mut el in Punctuated::pairs_mut(&mut node.predicates) { + let it = el.value_mut(); + v.visit_where_predicate_mut(it); + } +} +#[cfg(any(feature = "derive", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "derive", feature = "full"))))] +pub fn visit_where_predicate_mut<V>(v: &mut V, node: &mut crate::WherePredicate) +where + V: VisitMut + ?Sized, +{ + match node { + crate::WherePredicate::Lifetime(_binding_0) => { + v.visit_predicate_lifetime_mut(_binding_0); + } + crate::WherePredicate::Type(_binding_0) => { + v.visit_predicate_type_mut(_binding_0); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/generics.rs b/third_party/crates/syn-2.0.119/src/generics.rs new file mode 100644 index 0000000..5a7f1c3 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/generics.rs @@ -0,0 +1,1485 @@ +use crate::attr::Attribute; +use crate::expr::Expr; +use crate::ident::Ident; +use crate::lifetime::Lifetime; +use crate::path::Path; +use crate::punctuated::{Iter, IterMut, Punctuated}; +use crate::token; +use crate::ty::Type; +use alloc::vec::Vec; +#[cfg(all(feature = "printing", feature = "extra-traits"))] +use core::fmt::{self, Debug}; +#[cfg(all(feature = "printing", feature = "extra-traits"))] +use core::hash::{Hash, Hasher}; +use proc_macro2::TokenStream; + +ast_struct! { + /// Lifetimes and type parameters attached to a declaration of a function, + /// enum, trait, etc. + /// + /// This struct represents two distinct optional syntactic elements, + /// [generic parameters] and [where clause]. In some locations of the + /// grammar, there may be other tokens in between these two things. + /// + /// [generic parameters]: https://doc.rust-lang.org/stable/reference/items/generics.html#generic-parameters + /// [where clause]: https://doc.rust-lang.org/stable/reference/items/generics.html#where-clauses + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Generics { + pub lt_token: Option<Token![<]>, + pub params: Punctuated<GenericParam, Token![,]>, + pub gt_token: Option<Token![>]>, + pub where_clause: Option<WhereClause>, + } +} + +ast_enum_of_structs! { + /// A generic type parameter, lifetime, or const generic: `T: Into<String>`, + /// `'a: 'b`, `const LEN: usize`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum GenericParam { + /// A lifetime parameter: `'a: 'b + 'c + 'd`. + Lifetime(LifetimeParam), + + /// A generic type parameter: `T: Into<String>`. + Type(TypeParam), + + /// A const generic parameter: `const LENGTH: usize`. + Const(ConstParam), + } +} + +ast_struct! { + /// A lifetime definition: `'a: 'b + 'c + 'd`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct LifetimeParam { + pub attrs: Vec<Attribute>, + pub lifetime: Lifetime, + pub colon_token: Option<Token![:]>, + pub bounds: Punctuated<Lifetime, Token![+]>, + } +} + +ast_struct! { + /// A generic type parameter: `T: Into<String>`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeParam { + pub attrs: Vec<Attribute>, + pub ident: Ident, + pub colon_token: Option<Token![:]>, + pub bounds: Punctuated<TypeParamBound, Token![+]>, + pub eq_token: Option<Token![=]>, + pub default: Option<Type>, + } +} + +ast_struct! { + /// A const generic parameter: `const LENGTH: usize`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ConstParam { + pub attrs: Vec<Attribute>, + pub const_token: Token![const], + pub ident: Ident, + pub colon_token: Token![:], + pub ty: Type, + pub eq_token: Option<Token![=]>, + pub default: Option<Expr>, + } +} + +impl Default for Generics { + fn default() -> Self { + Generics { + lt_token: None, + params: Punctuated::new(), + gt_token: None, + where_clause: None, + } + } +} + +impl Generics { + return_impl_trait! { + /// Iterator over the lifetime parameters in `self.params`. + pub fn lifetimes(&self) -> impl Iterator<Item = &LifetimeParam> [Lifetimes] { + Lifetimes(self.params.iter()) + } + } + + return_impl_trait! { + /// Iterator over the lifetime parameters in `self.params`. + pub fn lifetimes_mut(&mut self) -> impl Iterator<Item = &mut LifetimeParam> [LifetimesMut] { + LifetimesMut(self.params.iter_mut()) + } + } + + return_impl_trait! { + /// Iterator over the type parameters in `self.params`. + pub fn type_params(&self) -> impl Iterator<Item = &TypeParam> [TypeParams] { + TypeParams(self.params.iter()) + } + } + + return_impl_trait! { + /// Iterator over the type parameters in `self.params`. + pub fn type_params_mut(&mut self) -> impl Iterator<Item = &mut TypeParam> [TypeParamsMut] { + TypeParamsMut(self.params.iter_mut()) + } + } + + return_impl_trait! { + /// Iterator over the constant parameters in `self.params`. + pub fn const_params(&self) -> impl Iterator<Item = &ConstParam> [ConstParams] { + ConstParams(self.params.iter()) + } + } + + return_impl_trait! { + /// Iterator over the constant parameters in `self.params`. + pub fn const_params_mut(&mut self) -> impl Iterator<Item = &mut ConstParam> [ConstParamsMut] { + ConstParamsMut(self.params.iter_mut()) + } + } + + /// Initializes an empty `where`-clause if there is not one present already. + pub fn make_where_clause(&mut self) -> &mut WhereClause { + self.where_clause.get_or_insert_with(|| WhereClause { + where_token: <Token![where]>::default(), + predicates: Punctuated::new(), + }) + } + + /// Split a type's generics into the pieces required for impl'ing a trait + /// for that type. + /// + /// ``` + /// # use proc_macro2::{Span, Ident}; + /// # use quote::quote; + /// # + /// # let generics: syn::Generics = Default::default(); + /// # let name = Ident::new("MyType", Span::call_site()); + /// # + /// let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + /// quote! { + /// impl #impl_generics MyTrait for #name #ty_generics #where_clause { + /// // ... + /// } + /// } + /// # ; + /// ``` + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + pub fn split_for_impl(&self) -> (ImplGenerics, TypeGenerics, Option<&WhereClause>) { + ( + ImplGenerics(self), + TypeGenerics(self), + self.where_clause.as_ref(), + ) + } +} + +pub struct Lifetimes<'a>(Iter<'a, GenericParam>); + +impl<'a> Iterator for Lifetimes<'a> { + type Item = &'a LifetimeParam; + + fn next(&mut self) -> Option<Self::Item> { + if let GenericParam::Lifetime(lifetime) = self.0.next()? { + Some(lifetime) + } else { + self.next() + } + } +} + +pub struct LifetimesMut<'a>(IterMut<'a, GenericParam>); + +impl<'a> Iterator for LifetimesMut<'a> { + type Item = &'a mut LifetimeParam; + + fn next(&mut self) -> Option<Self::Item> { + if let GenericParam::Lifetime(lifetime) = self.0.next()? { + Some(lifetime) + } else { + self.next() + } + } +} + +pub struct TypeParams<'a>(Iter<'a, GenericParam>); + +impl<'a> Iterator for TypeParams<'a> { + type Item = &'a TypeParam; + + fn next(&mut self) -> Option<Self::Item> { + if let GenericParam::Type(type_param) = self.0.next()? { + Some(type_param) + } else { + self.next() + } + } +} + +pub struct TypeParamsMut<'a>(IterMut<'a, GenericParam>); + +impl<'a> Iterator for TypeParamsMut<'a> { + type Item = &'a mut TypeParam; + + fn next(&mut self) -> Option<Self::Item> { + if let GenericParam::Type(type_param) = self.0.next()? { + Some(type_param) + } else { + self.next() + } + } +} + +pub struct ConstParams<'a>(Iter<'a, GenericParam>); + +impl<'a> Iterator for ConstParams<'a> { + type Item = &'a ConstParam; + + fn next(&mut self) -> Option<Self::Item> { + if let GenericParam::Const(const_param) = self.0.next()? { + Some(const_param) + } else { + self.next() + } + } +} + +pub struct ConstParamsMut<'a>(IterMut<'a, GenericParam>); + +impl<'a> Iterator for ConstParamsMut<'a> { + type Item = &'a mut ConstParam; + + fn next(&mut self) -> Option<Self::Item> { + if let GenericParam::Const(const_param) = self.0.next()? { + Some(const_param) + } else { + self.next() + } + } +} + +/// Returned by `Generics::split_for_impl`. +#[cfg(feature = "printing")] +#[cfg_attr( + docsrs, + doc(cfg(all(any(feature = "full", feature = "derive"), feature = "printing"))) +)] +pub struct ImplGenerics<'a>(&'a Generics); + +/// Returned by `Generics::split_for_impl`. +#[cfg(feature = "printing")] +#[cfg_attr( + docsrs, + doc(cfg(all(any(feature = "full", feature = "derive"), feature = "printing"))) +)] +pub struct TypeGenerics<'a>(&'a Generics); + +/// Returned by `TypeGenerics::as_turbofish`. +#[cfg(feature = "printing")] +#[cfg_attr( + docsrs, + doc(cfg(all(any(feature = "full", feature = "derive"), feature = "printing"))) +)] +pub struct Turbofish<'a>(&'a Generics); + +#[cfg(feature = "printing")] +macro_rules! generics_wrapper_impls { + ($ty:ident) => { + #[cfg(feature = "clone-impls")] + #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] + impl<'a> Clone for $ty<'a> { + fn clone(&self) -> Self { + $ty(self.0) + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl<'a> Debug for $ty<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter + .debug_tuple(stringify!($ty)) + .field(self.0) + .finish() + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl<'a> Eq for $ty<'a> {} + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl<'a> PartialEq for $ty<'a> { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl<'a> Hash for $ty<'a> { + fn hash<H: Hasher>(&self, state: &mut H) { + self.0.hash(state); + } + } + }; +} + +#[cfg(feature = "printing")] +generics_wrapper_impls!(ImplGenerics); +#[cfg(feature = "printing")] +generics_wrapper_impls!(TypeGenerics); +#[cfg(feature = "printing")] +generics_wrapper_impls!(Turbofish); + +#[cfg(feature = "printing")] +impl<'a> TypeGenerics<'a> { + /// Turn a type's generics like `<X, Y>` into a turbofish like `::<X, Y>`. + pub fn as_turbofish(&self) -> Turbofish<'a> { + Turbofish(self.0) + } +} + +ast_struct! { + /// A set of bound lifetimes: `for<'a, 'b, 'c>`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct BoundLifetimes { + pub for_token: Token![for], + pub lt_token: Token![<], + pub lifetimes: Punctuated<GenericParam, Token![,]>, + pub gt_token: Token![>], + } +} + +impl Default for BoundLifetimes { + fn default() -> Self { + BoundLifetimes { + for_token: Default::default(), + lt_token: Default::default(), + lifetimes: Punctuated::new(), + gt_token: Default::default(), + } + } +} + +impl LifetimeParam { + pub fn new(lifetime: Lifetime) -> Self { + LifetimeParam { + attrs: Vec::new(), + lifetime, + colon_token: None, + bounds: Punctuated::new(), + } + } +} + +impl From<Ident> for TypeParam { + fn from(ident: Ident) -> Self { + TypeParam { + attrs: Vec::new(), + ident, + colon_token: None, + bounds: Punctuated::new(), + eq_token: None, + default: None, + } + } +} + +ast_enum_of_structs! { + /// A trait or lifetime used as a bound on a type parameter. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum TypeParamBound { + Trait(TraitBound), + Lifetime(Lifetime), + PreciseCapture(PreciseCapture), + Verbatim(TokenStream), + } +} + +ast_struct! { + /// A trait used as a bound on a type parameter. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TraitBound { + pub paren_token: Option<token::Paren>, + pub modifier: TraitBoundModifier, + /// The `for<'a>` in `for<'a> Foo<&'a T>` + pub lifetimes: Option<BoundLifetimes>, + /// The `Foo<&'a T>` in `for<'a> Foo<&'a T>` + pub path: Path, + } +} + +ast_enum! { + /// A modifier on a trait bound, currently only used for the `?` in + /// `?Sized`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum TraitBoundModifier { + None, + Maybe(Token![?]), + } +} + +ast_struct! { + /// Precise capturing bound: the 'use<…>' in `impl Trait + + /// use<'a, T>`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PreciseCapture #full { + pub use_token: Token![use], + pub lt_token: Token![<], + pub params: Punctuated<CapturedParam, Token![,]>, + pub gt_token: Token![>], + } +} + +#[cfg(feature = "full")] +ast_enum! { + /// Single parameter in a precise capturing bound. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + #[non_exhaustive] + pub enum CapturedParam { + /// A lifetime parameter in precise capturing bound: `fn f<'a>() -> impl + /// Trait + use<'a>`. + Lifetime(Lifetime), + /// A type parameter or const generic parameter in precise capturing + /// bound: `fn f<T>() -> impl Trait + use<T>` or `fn f<const K: T>() -> + /// impl Trait + use<K>`. + Ident(Ident), + } +} + +ast_struct! { + /// A `where` clause in a definition: `where T: Deserialize<'de>, D: + /// 'static`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct WhereClause { + pub where_token: Token![where], + pub predicates: Punctuated<WherePredicate, Token![,]>, + } +} + +ast_enum_of_structs! { + /// A single predicate in a `where` clause: `T: Deserialize<'de>`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum WherePredicate { + /// A lifetime predicate in a `where` clause: `'a: 'b + 'c`. + Lifetime(PredicateLifetime), + + /// A type predicate in a `where` clause: `for<'c> Foo<'c>: Trait<'c>`. + Type(PredicateType), + } +} + +ast_struct! { + /// A lifetime predicate in a `where` clause: `'a: 'b + 'c`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct PredicateLifetime { + pub lifetime: Lifetime, + pub colon_token: Token![:], + pub bounds: Punctuated<Lifetime, Token![+]>, + } +} + +ast_struct! { + /// A type predicate in a `where` clause: `for<'c> Foo<'c>: Trait<'c>`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct PredicateType { + /// Any lifetimes from a `for` binding + pub lifetimes: Option<BoundLifetimes>, + /// The type being bounded + pub bounded_ty: Type, + pub colon_token: Token![:], + /// Trait and lifetime bounds (`Clone+Send+'static`) + pub bounds: Punctuated<TypeParamBound, Token![+]>, + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + #[cfg(feature = "full")] + use crate::error; + use crate::error::{Error, Result}; + use crate::ext::IdentExt as _; + use crate::generics::{ + BoundLifetimes, ConstParam, GenericParam, Generics, LifetimeParam, PredicateLifetime, + PredicateType, TraitBound, TraitBoundModifier, TypeParam, TypeParamBound, WhereClause, + WherePredicate, + }; + #[cfg(feature = "full")] + use crate::generics::{CapturedParam, PreciseCapture}; + use crate::ident::Ident; + use crate::lifetime::Lifetime; + use crate::parse::{Parse, ParseStream}; + use crate::path::{self, ParenthesizedGenericArguments, Path, PathArguments}; + use crate::punctuated::Punctuated; + use crate::token; + use crate::ty::Type; + use crate::verbatim; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Generics { + fn parse(input: ParseStream) -> Result<Self> { + if !input.peek(Token![<]) { + return Ok(Generics::default()); + } + + let lt_token: Token![<] = input.parse()?; + + let mut params = Punctuated::new(); + loop { + if input.peek(Token![>]) { + break; + } + + let attrs = input.call(Attribute::parse_outer)?; + let lookahead = input.lookahead1(); + if lookahead.peek(Lifetime) { + params.push_value(GenericParam::Lifetime(LifetimeParam { + attrs, + ..input.parse()? + })); + } else if lookahead.peek(Ident) { + params.push_value(GenericParam::Type(TypeParam { + attrs, + ..input.parse()? + })); + } else if lookahead.peek(Token![const]) { + params.push_value(GenericParam::Const(ConstParam { + attrs, + ..input.parse()? + })); + } else if input.peek(Token![_]) { + params.push_value(GenericParam::Type(TypeParam { + attrs, + ident: input.call(Ident::parse_any)?, + colon_token: None, + bounds: Punctuated::new(), + eq_token: None, + default: None, + })); + } else { + return Err(lookahead.error()); + } + + if input.peek(Token![>]) { + break; + } + let punct = input.parse()?; + params.push_punct(punct); + } + + let gt_token: Token![>] = input.parse()?; + + Ok(Generics { + lt_token: Some(lt_token), + params, + gt_token: Some(gt_token), + where_clause: None, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for GenericParam { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + + let lookahead = input.lookahead1(); + if lookahead.peek(Ident) { + Ok(GenericParam::Type(TypeParam { + attrs, + ..input.parse()? + })) + } else if lookahead.peek(Lifetime) { + Ok(GenericParam::Lifetime(LifetimeParam { + attrs, + ..input.parse()? + })) + } else if lookahead.peek(Token![const]) { + Ok(GenericParam::Const(ConstParam { + attrs, + ..input.parse()? + })) + } else { + Err(lookahead.error()) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LifetimeParam { + fn parse(input: ParseStream) -> Result<Self> { + let has_colon; + Ok(LifetimeParam { + attrs: input.call(Attribute::parse_outer)?, + lifetime: input.parse()?, + colon_token: { + if input.peek(Token![:]) { + has_colon = true; + Some(input.parse()?) + } else { + has_colon = false; + None + } + }, + bounds: { + let mut bounds = Punctuated::new(); + if has_colon { + loop { + if input.is_empty() || input.peek(Token![,]) || input.peek(Token![>]) { + break; + } + let value = input.parse()?; + bounds.push_value(value); + if !input.peek(Token![+]) { + break; + } + let punct = input.parse()?; + bounds.push_punct(punct); + } + } + bounds + }, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for BoundLifetimes { + fn parse(input: ParseStream) -> Result<Self> { + Ok(BoundLifetimes { + for_token: input.parse()?, + lt_token: input.parse()?, + lifetimes: { + let mut lifetimes = Punctuated::new(); + while !input.peek(Token![>]) { + lifetimes.push_value(input.parse()?); + if input.peek(Token![>]) { + break; + } + lifetimes.push_punct(input.parse()?); + } + lifetimes + }, + gt_token: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Option<BoundLifetimes> { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Token![for]) { + input.parse().map(Some) + } else { + Ok(None) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeParam { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let ident: Ident = input.parse()?; + let colon_token: Option<Token![:]> = input.parse()?; + + let mut bounds = Punctuated::new(); + if colon_token.is_some() { + loop { + if input.is_empty() + || input.peek(Token![,]) + || input.peek(Token![>]) + || input.peek(Token![=]) + { + break; + } + bounds.push_value({ + let allow_precise_capture = false; + let allow_const = true; + TypeParamBound::parse_single(input, allow_precise_capture, allow_const)? + }); + if !input.peek(Token![+]) { + break; + } + let punct: Token![+] = input.parse()?; + bounds.push_punct(punct); + } + } + + let eq_token: Option<Token![=]> = input.parse()?; + let default = if eq_token.is_some() { + Some(input.parse::<Type>()?) + } else { + None + }; + + Ok(TypeParam { + attrs, + ident, + colon_token, + bounds, + eq_token, + default, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeParamBound { + fn parse(input: ParseStream) -> Result<Self> { + let allow_precise_capture = true; + let allow_const = true; + Self::parse_single(input, allow_precise_capture, allow_const) + } + } + + impl TypeParamBound { + pub(crate) fn parse_single( + input: ParseStream, + #[cfg_attr(not(feature = "full"), allow(unused_variables))] allow_precise_capture: bool, + allow_const: bool, + ) -> Result<Self> { + if input.peek(Lifetime) { + return input.parse().map(TypeParamBound::Lifetime); + } + + #[cfg(feature = "full")] + { + if input.peek(Token![use]) { + let precise_capture: PreciseCapture = input.parse()?; + return if allow_precise_capture { + Ok(TypeParamBound::PreciseCapture(precise_capture)) + } else { + let msg = "`use<...>` precise capturing syntax is not allowed here"; + Err(error::new2( + precise_capture.use_token.span, + precise_capture.gt_token.span, + msg, + )) + }; + } + } + + let begin = input.cursor(); + + let content; + let (paren_token, content) = if input.peek(token::Paren) { + (Some(parenthesized!(content in input)), &content) + } else { + (None, input) + }; + + if let Some(mut bound) = TraitBound::do_parse(content, allow_const)? { + bound.paren_token = paren_token; + Ok(TypeParamBound::Trait(bound)) + } else { + Ok(TypeParamBound::Verbatim(verbatim::between( + begin, + input.cursor(), + ))) + } + } + + pub(crate) fn parse_multiple( + input: ParseStream, + allow_plus: bool, + allow_precise_capture: bool, + allow_const: bool, + ) -> Result<Punctuated<Self, Token![+]>> { + let mut bounds = Punctuated::new(); + loop { + let bound = Self::parse_single(input, allow_precise_capture, allow_const)?; + bounds.push_value(bound); + if !(allow_plus && input.peek(Token![+])) { + break; + } + bounds.push_punct(input.parse()?); + if !(input.peek(Ident::peek_any) + || input.peek(Token![::]) + || input.peek(Token![?]) + || input.peek(Lifetime) + || input.peek(token::Paren) + || (allow_const && (input.peek(token::Bracket) || input.peek(Token![const])))) + { + break; + } + } + Ok(bounds) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TraitBound { + fn parse(input: ParseStream) -> Result<Self> { + let allow_const = false; + Self::do_parse(input, allow_const).map(Option::unwrap) + } + } + + impl TraitBound { + fn do_parse(input: ParseStream, allow_const: bool) -> Result<Option<Self>> { + let mut lifetimes: Option<BoundLifetimes> = input.parse()?; + + let is_conditionally_const = cfg!(feature = "full") && input.peek(token::Bracket); + let is_unconditionally_const = cfg!(feature = "full") && input.peek(Token![const]); + if is_conditionally_const { + let conditionally_const; + let bracket_token = bracketed!(conditionally_const in input); + conditionally_const.parse::<Token![const]>()?; + if !allow_const { + let msg = "`[const]` is not allowed here"; + return Err(Error::new(bracket_token.span.join(), msg)); + } + } else if is_unconditionally_const { + let const_token: Token![const] = input.parse()?; + if !allow_const { + let msg = "`const` is not allowed here"; + return Err(Error::new(const_token.span, msg)); + } + } + + let modifier: TraitBoundModifier = input.parse()?; + if lifetimes.is_none() && matches!(modifier, TraitBoundModifier::Maybe(_)) { + lifetimes = input.parse()?; + } + + let mut path: Path = input.parse()?; + if path.segments.last().unwrap().arguments.is_empty() + && (input.peek(token::Paren) || input.peek(Token![::]) && input.peek3(token::Paren)) + { + input.parse::<Option<Token![::]>>()?; + let args: ParenthesizedGenericArguments = input.parse()?; + let parenthesized = PathArguments::Parenthesized(args); + path.segments.last_mut().unwrap().arguments = parenthesized; + } + + if lifetimes.is_some() { + match modifier { + TraitBoundModifier::None => {} + TraitBoundModifier::Maybe(maybe) => { + let msg = "`for<...>` binder not allowed with `?` trait polarity modifier"; + return Err(Error::new(maybe.span, msg)); + } + } + } + + if is_conditionally_const || is_unconditionally_const { + Ok(None) + } else { + Ok(Some(TraitBound { + paren_token: None, + modifier, + lifetimes, + path, + })) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TraitBoundModifier { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Token![?]) { + input.parse().map(TraitBoundModifier::Maybe) + } else { + Ok(TraitBoundModifier::None) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ConstParam { + fn parse(input: ParseStream) -> Result<Self> { + let mut default = None; + Ok(ConstParam { + attrs: input.call(Attribute::parse_outer)?, + const_token: input.parse()?, + ident: input.parse()?, + colon_token: input.parse()?, + ty: input.parse()?, + eq_token: { + if input.peek(Token![=]) { + let eq_token = input.parse()?; + default = Some(path::parsing::const_argument(input)?); + Some(eq_token) + } else { + None + } + }, + default, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for WhereClause { + fn parse(input: ParseStream) -> Result<Self> { + let where_token: Token![where] = input.parse()?; + + if choose_generics_over_qpath(input) { + return Err(input + .error("generic parameters on `where` clauses are reserved for future use")); + } + + Ok(WhereClause { + where_token, + predicates: { + let mut predicates = Punctuated::new(); + loop { + if input.is_empty() + || input.peek(token::Brace) + || input.peek(Token![,]) + || input.peek(Token![;]) + || input.peek(Token![:]) && !input.peek(Token![::]) + || input.peek(Token![=]) + { + break; + } + let value = input.parse()?; + predicates.push_value(value); + if !input.peek(Token![,]) { + break; + } + let punct = input.parse()?; + predicates.push_punct(punct); + } + predicates + }, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Option<WhereClause> { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Token![where]) { + input.parse().map(Some) + } else { + Ok(None) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for WherePredicate { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Lifetime) && input.peek2(Token![:]) { + Ok(WherePredicate::Lifetime(PredicateLifetime { + lifetime: input.parse()?, + colon_token: input.parse()?, + bounds: { + let mut bounds = Punctuated::new(); + loop { + if input.is_empty() + || input.peek(token::Brace) + || input.peek(Token![,]) + || input.peek(Token![;]) + || input.peek(Token![:]) + || input.peek(Token![=]) + { + break; + } + let value = input.parse()?; + bounds.push_value(value); + if !input.peek(Token![+]) { + break; + } + let punct = input.parse()?; + bounds.push_punct(punct); + } + bounds + }, + })) + } else { + Ok(WherePredicate::Type(PredicateType { + lifetimes: input.parse()?, + bounded_ty: input.parse()?, + colon_token: input.parse()?, + bounds: { + let mut bounds = Punctuated::new(); + loop { + if input.is_empty() + || input.peek(token::Brace) + || input.peek(Token![,]) + || input.peek(Token![;]) + || input.peek(Token![:]) && !input.peek(Token![::]) + || input.peek(Token![=]) + { + break; + } + bounds.push_value({ + let allow_precise_capture = false; + let allow_const = true; + TypeParamBound::parse_single( + input, + allow_precise_capture, + allow_const, + )? + }); + if !input.peek(Token![+]) { + break; + } + let punct = input.parse()?; + bounds.push_punct(punct); + } + bounds + }, + })) + } + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for PreciseCapture { + fn parse(input: ParseStream) -> Result<Self> { + let use_token: Token![use] = input.parse()?; + let lt_token: Token![<] = input.parse()?; + let mut params = Punctuated::new(); + loop { + let lookahead = input.lookahead1(); + params.push_value( + if lookahead.peek(Lifetime) || lookahead.peek(Ident) || input.peek(Token![Self]) + { + input.parse::<CapturedParam>()? + } else if lookahead.peek(Token![>]) { + break; + } else { + return Err(lookahead.error()); + }, + ); + let lookahead = input.lookahead1(); + params.push_punct(if lookahead.peek(Token![,]) { + input.parse::<Token![,]>()? + } else if lookahead.peek(Token![>]) { + break; + } else { + return Err(lookahead.error()); + }); + } + let gt_token: Token![>] = input.parse()?; + Ok(PreciseCapture { + use_token, + lt_token, + params, + gt_token, + }) + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for CapturedParam { + fn parse(input: ParseStream) -> Result<Self> { + let lookahead = input.lookahead1(); + if lookahead.peek(Lifetime) { + input.parse().map(CapturedParam::Lifetime) + } else if lookahead.peek(Ident) || input.peek(Token![Self]) { + input.call(Ident::parse_any).map(CapturedParam::Ident) + } else { + Err(lookahead.error()) + } + } + } + + pub(crate) fn choose_generics_over_qpath(input: ParseStream) -> bool { + // Rust syntax has an ambiguity between generic parameters and qualified + // paths. In `impl <T> :: Thing<T, U> {}` this may either be a generic + // inherent impl `impl<T> ::Thing<T, U>` or a non-generic inherent impl + // for an associated type `impl <T>::Thing<T, U>`. + // + // After `<` the following continuations can only begin generics, not a + // qualified path: + // + // `<` `>` - empty generic parameters + // `<` `#` - generic parameters with attribute + // `<` LIFETIME `>` - single lifetime parameter + // `<` (LIFETIME|IDENT) `,` - first generic parameter in a list + // `<` (LIFETIME|IDENT) `:` - generic parameter with bounds + // `<` (LIFETIME|IDENT) `=` - generic parameter with a default + // `<` const - generic const parameter + // + // The only truly ambiguous case is: + // + // `<` IDENT `>` `::` IDENT ... + // + // which we disambiguate in favor of generics because this is almost + // always the expected one in the context of real-world code. + input.peek(Token![<]) + && (input.peek2(Token![>]) + || input.peek2(Token![#]) + || (input.peek2(Lifetime) || input.peek2(Ident)) + && (input.peek3(Token![>]) + || input.peek3(Token![,]) + || input.peek3(Token![:]) && !input.peek3(Token![::]) + || input.peek3(Token![=])) + || input.peek2(Token![const])) + } + + #[cfg(feature = "full")] + pub(crate) fn choose_generics_over_qpath_after_keyword(input: ParseStream) -> bool { + let input = input.fork(); + input.call(Ident::parse_any).unwrap(); // `impl` or `for` or `where` + choose_generics_over_qpath(&input) + } +} + +#[cfg(feature = "printing")] +pub(crate) mod printing { + use crate::attr::FilterAttrs; + #[cfg(feature = "full")] + use crate::expr; + use crate::expr::Expr; + #[cfg(feature = "full")] + use crate::fixup::FixupContext; + use crate::generics::{ + BoundLifetimes, ConstParam, GenericParam, Generics, ImplGenerics, LifetimeParam, + PredicateLifetime, PredicateType, TraitBound, TraitBoundModifier, Turbofish, TypeGenerics, + TypeParam, WhereClause, + }; + #[cfg(feature = "full")] + use crate::generics::{CapturedParam, PreciseCapture}; + use crate::print::TokensOrDefault; + use crate::token; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Generics { + fn to_tokens(&self, tokens: &mut TokenStream) { + if self.params.is_empty() { + return; + } + + TokensOrDefault(&self.lt_token).to_tokens(tokens); + + // Print lifetimes before types and consts, regardless of their + // order in self.params. + let mut trailing_or_empty = true; + for param in self.params.pairs() { + if let GenericParam::Lifetime(_) = **param.value() { + param.to_tokens(tokens); + trailing_or_empty = param.punct().is_some(); + } + } + for param in self.params.pairs() { + match param.value() { + GenericParam::Type(_) | GenericParam::Const(_) => { + if !trailing_or_empty { + <Token![,]>::default().to_tokens(tokens); + trailing_or_empty = true; + } + param.to_tokens(tokens); + } + GenericParam::Lifetime(_) => {} + } + } + + TokensOrDefault(&self.gt_token).to_tokens(tokens); + } + } + + impl<'a> ToTokens for ImplGenerics<'a> { + fn to_tokens(&self, tokens: &mut TokenStream) { + if self.0.params.is_empty() { + return; + } + + TokensOrDefault(&self.0.lt_token).to_tokens(tokens); + + // Print lifetimes before types and consts, regardless of their + // order in self.params. + let mut trailing_or_empty = true; + for param in self.0.params.pairs() { + if let GenericParam::Lifetime(_) = **param.value() { + param.to_tokens(tokens); + trailing_or_empty = param.punct().is_some(); + } + } + for param in self.0.params.pairs() { + if let GenericParam::Lifetime(_) = **param.value() { + continue; + } + if !trailing_or_empty { + <Token![,]>::default().to_tokens(tokens); + trailing_or_empty = true; + } + match param.value() { + GenericParam::Lifetime(_) => unreachable!(), + GenericParam::Type(param) => { + // Leave off the type parameter defaults + tokens.append_all(param.attrs.outer()); + param.ident.to_tokens(tokens); + if !param.bounds.is_empty() { + TokensOrDefault(¶m.colon_token).to_tokens(tokens); + param.bounds.to_tokens(tokens); + } + } + GenericParam::Const(param) => { + // Leave off the const parameter defaults + tokens.append_all(param.attrs.outer()); + param.const_token.to_tokens(tokens); + param.ident.to_tokens(tokens); + param.colon_token.to_tokens(tokens); + param.ty.to_tokens(tokens); + } + } + param.punct().to_tokens(tokens); + } + + TokensOrDefault(&self.0.gt_token).to_tokens(tokens); + } + } + + impl<'a> ToTokens for TypeGenerics<'a> { + fn to_tokens(&self, tokens: &mut TokenStream) { + if self.0.params.is_empty() { + return; + } + + TokensOrDefault(&self.0.lt_token).to_tokens(tokens); + + // Print lifetimes before types and consts, regardless of their + // order in self.params. + let mut trailing_or_empty = true; + for param in self.0.params.pairs() { + if let GenericParam::Lifetime(def) = *param.value() { + // Leave off the lifetime bounds and attributes + def.lifetime.to_tokens(tokens); + param.punct().to_tokens(tokens); + trailing_or_empty = param.punct().is_some(); + } + } + for param in self.0.params.pairs() { + if let GenericParam::Lifetime(_) = **param.value() { + continue; + } + if !trailing_or_empty { + <Token![,]>::default().to_tokens(tokens); + trailing_or_empty = true; + } + match param.value() { + GenericParam::Lifetime(_) => unreachable!(), + GenericParam::Type(param) => { + // Leave off the type parameter defaults + param.ident.to_tokens(tokens); + } + GenericParam::Const(param) => { + // Leave off the const parameter defaults + param.ident.to_tokens(tokens); + } + } + param.punct().to_tokens(tokens); + } + + TokensOrDefault(&self.0.gt_token).to_tokens(tokens); + } + } + + impl<'a> ToTokens for Turbofish<'a> { + fn to_tokens(&self, tokens: &mut TokenStream) { + if !self.0.params.is_empty() { + <Token![::]>::default().to_tokens(tokens); + TypeGenerics(self.0).to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for BoundLifetimes { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.for_token.to_tokens(tokens); + self.lt_token.to_tokens(tokens); + self.lifetimes.to_tokens(tokens); + self.gt_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LifetimeParam { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.lifetime.to_tokens(tokens); + if !self.bounds.is_empty() { + TokensOrDefault(&self.colon_token).to_tokens(tokens); + self.bounds.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeParam { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.ident.to_tokens(tokens); + if !self.bounds.is_empty() { + TokensOrDefault(&self.colon_token).to_tokens(tokens); + self.bounds.to_tokens(tokens); + } + if let Some(default) = &self.default { + TokensOrDefault(&self.eq_token).to_tokens(tokens); + default.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TraitBound { + fn to_tokens(&self, tokens: &mut TokenStream) { + let to_tokens = |tokens: &mut TokenStream| { + self.modifier.to_tokens(tokens); + self.lifetimes.to_tokens(tokens); + self.path.to_tokens(tokens); + }; + match &self.paren_token { + Some(paren) => paren.surround(tokens, to_tokens), + None => to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TraitBoundModifier { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + TraitBoundModifier::None => {} + TraitBoundModifier::Maybe(t) => t.to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ConstParam { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.const_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + if let Some(default) = &self.default { + TokensOrDefault(&self.eq_token).to_tokens(tokens); + print_const_argument(default, tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for WhereClause { + fn to_tokens(&self, tokens: &mut TokenStream) { + if !self.predicates.is_empty() { + self.where_token.to_tokens(tokens); + self.predicates.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PredicateLifetime { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.lifetime.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.bounds.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PredicateType { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.lifetimes.to_tokens(tokens); + self.bounded_ty.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.bounds.to_tokens(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PreciseCapture { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.use_token.to_tokens(tokens); + self.lt_token.to_tokens(tokens); + + // Print lifetimes before types and consts, regardless of their + // order in self.params. + let mut trailing_or_empty = true; + for param in self.params.pairs() { + if let CapturedParam::Lifetime(_) = **param.value() { + param.to_tokens(tokens); + trailing_or_empty = param.punct().is_some(); + } + } + for param in self.params.pairs() { + if let CapturedParam::Ident(_) = **param.value() { + if !trailing_or_empty { + <Token![,]>::default().to_tokens(tokens); + trailing_or_empty = true; + } + param.to_tokens(tokens); + } + } + + self.gt_token.to_tokens(tokens); + } + } + + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for CapturedParam { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + CapturedParam::Lifetime(lifetime) => lifetime.to_tokens(tokens), + CapturedParam::Ident(ident) => ident.to_tokens(tokens), + } + } + } + + pub(crate) fn print_const_argument(expr: &Expr, tokens: &mut TokenStream) { + match expr { + Expr::Lit(expr) => expr.to_tokens(tokens), + + Expr::Path(expr) + if expr.attrs.is_empty() + && expr.qself.is_none() + && expr.path.get_ident().is_some() => + { + expr.to_tokens(tokens); + } + + #[cfg(feature = "full")] + Expr::Block(expr) => expr.to_tokens(tokens), + + #[cfg(not(feature = "full"))] + Expr::Verbatim(expr) => expr.to_tokens(tokens), + + // ERROR CORRECTION: Add braces to make sure that the + // generated code is valid. + _ => token::Brace::default().surround(tokens, |tokens| { + #[cfg(feature = "full")] + expr::printing::print_expr(expr, tokens, FixupContext::new_stmt()); + + #[cfg(not(feature = "full"))] + expr.to_tokens(tokens); + }), + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/group.rs b/third_party/crates/syn-2.0.119/src/group.rs new file mode 100644 index 0000000..56667b6 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/group.rs @@ -0,0 +1,294 @@ +use crate::error::Result; +use crate::parse::ParseBuffer; +use crate::token; +use proc_macro2::extra::DelimSpan; +use proc_macro2::Delimiter; + +// Not public API. +#[doc(hidden)] +pub struct Parens<'a> { + #[doc(hidden)] + pub token: token::Paren, + #[doc(hidden)] + pub content: ParseBuffer<'a>, +} + +// Not public API. +#[doc(hidden)] +pub struct Braces<'a> { + #[doc(hidden)] + pub token: token::Brace, + #[doc(hidden)] + pub content: ParseBuffer<'a>, +} + +// Not public API. +#[doc(hidden)] +pub struct Brackets<'a> { + #[doc(hidden)] + pub token: token::Bracket, + #[doc(hidden)] + pub content: ParseBuffer<'a>, +} + +// Not public API. +#[cfg(any(feature = "full", feature = "derive"))] +#[doc(hidden)] +pub struct Group<'a> { + #[doc(hidden)] + pub token: token::Group, + #[doc(hidden)] + pub content: ParseBuffer<'a>, +} + +// Not public API. +#[doc(hidden)] +pub fn parse_parens<'a>(input: &ParseBuffer<'a>) -> Result<Parens<'a>> { + parse_delimited(input, Delimiter::Parenthesis).map(|(span, content)| Parens { + token: token::Paren(span), + content, + }) +} + +// Not public API. +#[doc(hidden)] +pub fn parse_braces<'a>(input: &ParseBuffer<'a>) -> Result<Braces<'a>> { + parse_delimited(input, Delimiter::Brace).map(|(span, content)| Braces { + token: token::Brace(span), + content, + }) +} + +// Not public API. +#[doc(hidden)] +pub fn parse_brackets<'a>(input: &ParseBuffer<'a>) -> Result<Brackets<'a>> { + parse_delimited(input, Delimiter::Bracket).map(|(span, content)| Brackets { + token: token::Bracket(span), + content, + }) +} + +#[cfg(any(feature = "full", feature = "derive"))] +pub(crate) fn parse_group<'a>(input: &ParseBuffer<'a>) -> Result<Group<'a>> { + parse_delimited(input, Delimiter::None).map(|(span, content)| Group { + token: token::Group(span.join()), + content, + }) +} + +fn parse_delimited<'a>( + input: &ParseBuffer<'a>, + delimiter: Delimiter, +) -> Result<(DelimSpan, ParseBuffer<'a>)> { + input.step(|cursor| { + if let Some((content, span, rest)) = cursor.group(delimiter) { + let scope = span.close(); + let nested = crate::parse::advance_step_cursor(cursor, content); + let unexpected = crate::parse::get_unexpected(input); + let content = crate::parse::new_parse_buffer(scope, nested, unexpected); + Ok(((span, content), rest)) + } else { + let message = match delimiter { + Delimiter::Parenthesis => "expected parentheses", + Delimiter::Brace => "expected curly braces", + Delimiter::Bracket => "expected square brackets", + Delimiter::None => "expected invisible group", + }; + Err(cursor.error(message)) + } + }) +} + +/// Parse a set of parentheses and expose their content to subsequent parsers. +/// +/// # Example +/// +/// ``` +/// # use quote::quote; +/// # +/// use syn::{parenthesized, token, Ident, Result, Token, Type}; +/// use syn::parse::{Parse, ParseStream}; +/// use syn::punctuated::Punctuated; +/// +/// // Parse a simplified tuple struct syntax like: +/// // +/// // struct S(A, B); +/// struct TupleStruct { +/// struct_token: Token![struct], +/// ident: Ident, +/// paren_token: token::Paren, +/// fields: Punctuated<Type, Token![,]>, +/// semi_token: Token![;], +/// } +/// +/// impl Parse for TupleStruct { +/// fn parse(input: ParseStream) -> Result<Self> { +/// let content; +/// Ok(TupleStruct { +/// struct_token: input.parse()?, +/// ident: input.parse()?, +/// paren_token: parenthesized!(content in input), +/// fields: content.parse_terminated(Type::parse, Token![,])?, +/// semi_token: input.parse()?, +/// }) +/// } +/// } +/// # +/// # fn main() { +/// # let input = quote! { +/// # struct S(A, B); +/// # }; +/// # syn::parse2::<TupleStruct>(input).unwrap(); +/// # } +/// ``` +#[macro_export] +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +macro_rules! parenthesized { + ($content:ident in $cursor:expr) => { + match $crate::__private::parse_parens(&$cursor) { + $crate::__private::Ok(parens) => { + $content = parens.content; + _ = $content; + parens.token + } + $crate::__private::Err(error) => { + return $crate::__private::Err(error); + } + } + }; +} + +/// Parse a set of curly braces and expose their content to subsequent parsers. +/// +/// # Example +/// +/// ``` +/// # use quote::quote; +/// # +/// use syn::{braced, token, Ident, Result, Token, Type}; +/// use syn::parse::{Parse, ParseStream}; +/// use syn::punctuated::Punctuated; +/// +/// // Parse a simplified struct syntax like: +/// // +/// // struct S { +/// // a: A, +/// // b: B, +/// // } +/// struct Struct { +/// struct_token: Token![struct], +/// ident: Ident, +/// brace_token: token::Brace, +/// fields: Punctuated<Field, Token![,]>, +/// } +/// +/// struct Field { +/// name: Ident, +/// colon_token: Token![:], +/// ty: Type, +/// } +/// +/// impl Parse for Struct { +/// fn parse(input: ParseStream) -> Result<Self> { +/// let content; +/// Ok(Struct { +/// struct_token: input.parse()?, +/// ident: input.parse()?, +/// brace_token: braced!(content in input), +/// fields: content.parse_terminated(Field::parse, Token![,])?, +/// }) +/// } +/// } +/// +/// impl Parse for Field { +/// fn parse(input: ParseStream) -> Result<Self> { +/// Ok(Field { +/// name: input.parse()?, +/// colon_token: input.parse()?, +/// ty: input.parse()?, +/// }) +/// } +/// } +/// # +/// # fn main() { +/// # let input = quote! { +/// # struct S { +/// # a: A, +/// # b: B, +/// # } +/// # }; +/// # syn::parse2::<Struct>(input).unwrap(); +/// # } +/// ``` +#[macro_export] +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +macro_rules! braced { + ($content:ident in $cursor:expr) => { + match $crate::__private::parse_braces(&$cursor) { + $crate::__private::Ok(braces) => { + $content = braces.content; + _ = $content; + braces.token + } + $crate::__private::Err(error) => { + return $crate::__private::Err(error); + } + } + }; +} + +/// Parse a set of square brackets and expose their content to subsequent +/// parsers. +/// +/// # Example +/// +/// ``` +/// # use quote::quote; +/// # +/// use proc_macro2::TokenStream; +/// use syn::{bracketed, token, Result, Token}; +/// use syn::parse::{Parse, ParseStream}; +/// +/// // Parse an outer attribute like: +/// // +/// // #[repr(C, packed)] +/// struct OuterAttribute { +/// pound_token: Token![#], +/// bracket_token: token::Bracket, +/// content: TokenStream, +/// } +/// +/// impl Parse for OuterAttribute { +/// fn parse(input: ParseStream) -> Result<Self> { +/// let content; +/// Ok(OuterAttribute { +/// pound_token: input.parse()?, +/// bracket_token: bracketed!(content in input), +/// content: content.parse()?, +/// }) +/// } +/// } +/// # +/// # fn main() { +/// # let input = quote! { +/// # #[repr(C, packed)] +/// # }; +/// # syn::parse2::<OuterAttribute>(input).unwrap(); +/// # } +/// ``` +#[macro_export] +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +macro_rules! bracketed { + ($content:ident in $cursor:expr) => { + match $crate::__private::parse_brackets(&$cursor) { + $crate::__private::Ok(brackets) => { + $content = brackets.content; + _ = $content; + brackets.token + } + $crate::__private::Err(error) => { + return $crate::__private::Err(error); + } + } + }; +} diff --git a/third_party/crates/syn-2.0.119/src/ident.rs b/third_party/crates/syn-2.0.119/src/ident.rs new file mode 100644 index 0000000..923fe19 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/ident.rs @@ -0,0 +1,109 @@ +#[cfg(feature = "parsing")] +use crate::lookahead; + +pub use proc_macro2::Ident; + +#[cfg(feature = "parsing")] +pub_if_not_doc! { + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn Ident(marker: lookahead::TokenMarker) -> Ident { + match marker {} + } +} + +macro_rules! ident_from_token { + ($token:ident) => { + impl From<Token![$token]> for Ident { + fn from(token: Token![$token]) -> Ident { + Ident::new(stringify!($token), token.span) + } + } + }; +} + +ident_from_token!(self); +ident_from_token!(Self); +ident_from_token!(super); +ident_from_token!(crate); +ident_from_token!(extern); + +impl From<Token![_]> for Ident { + fn from(token: Token![_]) -> Ident { + Ident::new("_", token.span) + } +} + +pub(crate) fn xid_ok(symbol: &str) -> bool { + let mut chars = symbol.chars(); + let first = chars.next().unwrap(); + if !(first == '_' || unicode_ident::is_xid_start(first)) { + return false; + } + for ch in chars { + if !unicode_ident::is_xid_continue(ch) { + return false; + } + } + true +} + +#[cfg(feature = "parsing")] +mod parsing { + use crate::buffer::Cursor; + use crate::error::Result; + use crate::parse::{Parse, ParseStream}; + use crate::token::Token; + use alloc::string::ToString; + use proc_macro2::Ident; + + fn accept_as_ident(ident: &Ident) -> bool { + match ident.to_string().as_str() { + "_" | + // Based on https://doc.rust-lang.org/1.65.0/reference/keywords.html + "abstract" | "as" | "async" | "await" | "become" | "box" | "break" | + "const" | "continue" | "crate" | "do" | "dyn" | "else" | "enum" | + "extern" | "false" | "final" | "fn" | "for" | "if" | "impl" | "in" | + "let" | "loop" | "macro" | "match" | "mod" | "move" | "mut" | + "override" | "priv" | "pub" | "ref" | "return" | "Self" | "self" | + "static" | "struct" | "super" | "trait" | "true" | "try" | "type" | + "typeof" | "unsafe" | "unsized" | "use" | "virtual" | "where" | + "while" | "yield" => false, + _ => true, + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Ident { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| { + if let Some((ident, rest)) = cursor.ident() { + if accept_as_ident(&ident) { + Ok((ident, rest)) + } else { + Err(cursor.error(format_args!( + "expected identifier, found keyword `{}`", + ident, + ))) + } + } else { + Err(cursor.error("expected identifier")) + } + }) + } + } + + impl Token for Ident { + fn peek(cursor: Cursor) -> bool { + if let Some((ident, _rest)) = cursor.ident() { + accept_as_ident(&ident) + } else { + false + } + } + + fn display() -> &'static str { + "identifier" + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/item.rs b/third_party/crates/syn-2.0.119/src/item.rs new file mode 100644 index 0000000..d4df89b --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/item.rs @@ -0,0 +1,3539 @@ +use crate::attr::Attribute; +use crate::data::{Fields, FieldsNamed, Variant}; +use crate::derive::{Data, DataEnum, DataStruct, DataUnion, DeriveInput}; +use crate::expr::Expr; +use crate::generics::{Generics, TypeParamBound}; +use crate::ident::Ident; +use crate::lifetime::Lifetime; +use crate::mac::Macro; +use crate::pat::{Pat, PatType}; +use crate::path::Path; +use crate::punctuated::Punctuated; +use crate::restriction::Visibility; +use crate::stmt::Block; +use crate::token; +use crate::ty::{Abi, ReturnType, Type}; +use alloc::boxed::Box; +use alloc::vec::Vec; +#[cfg(feature = "parsing")] +use core::mem; +use proc_macro2::TokenStream; + +ast_enum_of_structs! { + /// Things that can appear directly inside of a module or scope. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + #[non_exhaustive] + pub enum Item { + /// A constant item: `const MAX: u16 = 65535`. + Const(ItemConst), + + /// An enum definition: `enum Foo<A, B> { A(A), B(B) }`. + Enum(ItemEnum), + + /// An `extern crate` item: `extern crate serde`. + ExternCrate(ItemExternCrate), + + /// A free-standing function: `fn process(n: usize) -> Result<()> { ... + /// }`. + Fn(ItemFn), + + /// A block of foreign items: `extern "C" { ... }`. + ForeignMod(ItemForeignMod), + + /// An impl block providing trait or associated items: `impl<A> Trait + /// for Data<A> { ... }`. + Impl(ItemImpl), + + /// A macro invocation, which includes `macro_rules!` definitions. + Macro(ItemMacro), + + /// A module or module declaration: `mod m` or `mod m { ... }`. + Mod(ItemMod), + + /// A static item: `static BIKE: Shed = Shed(42)`. + Static(ItemStatic), + + /// A struct definition: `struct Foo<A> { x: A }`. + Struct(ItemStruct), + + /// A trait definition: `pub trait Iterator { ... }`. + Trait(ItemTrait), + + /// A trait alias: `pub trait SharableIterator = Iterator + Sync`. + TraitAlias(ItemTraitAlias), + + /// A type alias: `type Result<T> = core::result::Result<T, MyError>`. + Type(ItemType), + + /// A union definition: `union Foo<A, B> { x: A, y: B }`. + Union(ItemUnion), + + /// A use declaration: `use alloc::collections::HashMap`. + Use(ItemUse), + + /// Tokens forming an item not interpreted by Syn. + Verbatim(TokenStream), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match item { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // Item::Const(item) => {...} + // Item::Enum(item) => {...} + // ... + // Item::Verbatim(item) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// A constant item: `const MAX: u16 = 65535`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemConst { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub const_token: Token![const], + pub ident: Ident, + pub generics: Generics, + pub colon_token: Token![:], + pub ty: Box<Type>, + pub eq_token: Token![=], + pub expr: Box<Expr>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// An enum definition: `enum Foo<A, B> { A(A), B(B) }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemEnum { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub enum_token: Token![enum], + pub ident: Ident, + pub generics: Generics, + pub brace_token: token::Brace, + pub variants: Punctuated<Variant, Token![,]>, + } +} + +ast_struct! { + /// An `extern crate` item: `extern crate serde`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemExternCrate { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub extern_token: Token![extern], + pub crate_token: Token![crate], + pub ident: Ident, + pub rename: Option<(Token![as], Ident)>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A free-standing function: `fn process(n: usize) -> Result<()> { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemFn { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub sig: Signature, + pub block: Box<Block>, + } +} + +ast_struct! { + /// A block of foreign items: `extern "C" { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemForeignMod { + pub attrs: Vec<Attribute>, + pub unsafety: Option<Token![unsafe]>, + pub abi: Abi, + pub brace_token: token::Brace, + pub items: Vec<ForeignItem>, + } +} + +ast_struct! { + /// An impl block providing trait or associated items: `impl<A> Trait + /// for Data<A> { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemImpl { + pub attrs: Vec<Attribute>, + pub defaultness: Option<Token![default]>, + pub unsafety: Option<Token![unsafe]>, + pub impl_token: Token![impl], + pub generics: Generics, + /// Trait this impl implements. + pub trait_: Option<(Option<Token![!]>, Path, Token![for])>, + /// The Self type of the impl. + pub self_ty: Box<Type>, + pub brace_token: token::Brace, + pub items: Vec<ImplItem>, + } +} + +ast_struct! { + /// A macro invocation, which includes `macro_rules!` definitions. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemMacro { + pub attrs: Vec<Attribute>, + /// The `example` in `macro_rules! example { ... }`. + pub ident: Option<Ident>, + pub mac: Macro, + pub semi_token: Option<Token![;]>, + } +} + +ast_struct! { + /// A module or module declaration: `mod m` or `mod m { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemMod { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub unsafety: Option<Token![unsafe]>, + pub mod_token: Token![mod], + pub ident: Ident, + pub content: Option<(token::Brace, Vec<Item>)>, + pub semi: Option<Token![;]>, + } +} + +ast_struct! { + /// A static item: `static BIKE: Shed = Shed(42)`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemStatic { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub static_token: Token![static], + pub mutability: StaticMutability, + pub ident: Ident, + pub colon_token: Token![:], + pub ty: Box<Type>, + pub eq_token: Token![=], + pub expr: Box<Expr>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A struct definition: `struct Foo<A> { x: A }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemStruct { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub struct_token: Token![struct], + pub ident: Ident, + pub generics: Generics, + pub fields: Fields, + pub semi_token: Option<Token![;]>, + } +} + +ast_struct! { + /// A trait definition: `pub trait Iterator { ... }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemTrait { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub unsafety: Option<Token![unsafe]>, + pub auto_token: Option<Token![auto]>, + pub restriction: Option<ImplRestriction>, + pub trait_token: Token![trait], + pub ident: Ident, + pub generics: Generics, + pub colon_token: Option<Token![:]>, + pub supertraits: Punctuated<TypeParamBound, Token![+]>, + pub brace_token: token::Brace, + pub items: Vec<TraitItem>, + } +} + +ast_struct! { + /// A trait alias: `pub trait SharableIterator = Iterator + Sync`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemTraitAlias { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub trait_token: Token![trait], + pub ident: Ident, + pub generics: Generics, + pub eq_token: Token![=], + pub bounds: Punctuated<TypeParamBound, Token![+]>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A type alias: `type Result<T> = core::result::Result<T, MyError>`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemType { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub type_token: Token![type], + pub ident: Ident, + pub generics: Generics, + pub eq_token: Token![=], + pub ty: Box<Type>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A union definition: `union Foo<A, B> { x: A, y: B }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemUnion { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub union_token: Token![union], + pub ident: Ident, + pub generics: Generics, + pub fields: FieldsNamed, + } +} + +ast_struct! { + /// A use declaration: `use alloc::collections::HashMap`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ItemUse { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub use_token: Token![use], + pub leading_colon: Option<Token![::]>, + pub tree: UseTree, + pub semi_token: Token![;], + } +} + +impl Item { + #[cfg(feature = "parsing")] + pub(crate) fn replace_attrs(&mut self, new: Vec<Attribute>) -> Vec<Attribute> { + match self { + Item::Const(ItemConst { attrs, .. }) + | Item::Enum(ItemEnum { attrs, .. }) + | Item::ExternCrate(ItemExternCrate { attrs, .. }) + | Item::Fn(ItemFn { attrs, .. }) + | Item::ForeignMod(ItemForeignMod { attrs, .. }) + | Item::Impl(ItemImpl { attrs, .. }) + | Item::Macro(ItemMacro { attrs, .. }) + | Item::Mod(ItemMod { attrs, .. }) + | Item::Static(ItemStatic { attrs, .. }) + | Item::Struct(ItemStruct { attrs, .. }) + | Item::Trait(ItemTrait { attrs, .. }) + | Item::TraitAlias(ItemTraitAlias { attrs, .. }) + | Item::Type(ItemType { attrs, .. }) + | Item::Union(ItemUnion { attrs, .. }) + | Item::Use(ItemUse { attrs, .. }) => mem::replace(attrs, new), + Item::Verbatim(_) => Vec::new(), + } + } +} + +impl From<DeriveInput> for Item { + fn from(input: DeriveInput) -> Item { + match input.data { + Data::Struct(data) => Item::Struct(ItemStruct { + attrs: input.attrs, + vis: input.vis, + struct_token: data.struct_token, + ident: input.ident, + generics: input.generics, + fields: data.fields, + semi_token: data.semi_token, + }), + Data::Enum(data) => Item::Enum(ItemEnum { + attrs: input.attrs, + vis: input.vis, + enum_token: data.enum_token, + ident: input.ident, + generics: input.generics, + brace_token: data.brace_token, + variants: data.variants, + }), + Data::Union(data) => Item::Union(ItemUnion { + attrs: input.attrs, + vis: input.vis, + union_token: data.union_token, + ident: input.ident, + generics: input.generics, + fields: data.fields, + }), + } + } +} + +impl From<ItemStruct> for DeriveInput { + fn from(input: ItemStruct) -> DeriveInput { + DeriveInput { + attrs: input.attrs, + vis: input.vis, + ident: input.ident, + generics: input.generics, + data: Data::Struct(DataStruct { + struct_token: input.struct_token, + fields: input.fields, + semi_token: input.semi_token, + }), + } + } +} + +impl From<ItemEnum> for DeriveInput { + fn from(input: ItemEnum) -> DeriveInput { + DeriveInput { + attrs: input.attrs, + vis: input.vis, + ident: input.ident, + generics: input.generics, + data: Data::Enum(DataEnum { + enum_token: input.enum_token, + brace_token: input.brace_token, + variants: input.variants, + }), + } + } +} + +impl From<ItemUnion> for DeriveInput { + fn from(input: ItemUnion) -> DeriveInput { + DeriveInput { + attrs: input.attrs, + vis: input.vis, + ident: input.ident, + generics: input.generics, + data: Data::Union(DataUnion { + union_token: input.union_token, + fields: input.fields, + }), + } + } +} + +ast_enum_of_structs! { + /// A suffix of an import tree in a `use` item: `Type as Renamed` or `*`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub enum UseTree { + /// A path prefix of imports in a `use` item: `core::...`. + Path(UsePath), + + /// An identifier imported by a `use` item: `HashMap`. + Name(UseName), + + /// An renamed identifier imported by a `use` item: `HashMap as Map`. + Rename(UseRename), + + /// A glob import in a `use` item: `*`. + Glob(UseGlob), + + /// A braced group of imports in a `use` item: `{A, B, C}`. + Group(UseGroup), + } +} + +ast_struct! { + /// A path prefix of imports in a `use` item: `core::...`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct UsePath { + pub ident: Ident, + pub colon2_token: Token![::], + pub tree: Box<UseTree>, + } +} + +ast_struct! { + /// An identifier imported by a `use` item: `HashMap`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct UseName { + pub ident: Ident, + } +} + +ast_struct! { + /// An renamed identifier imported by a `use` item: `HashMap as Map`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct UseRename { + pub ident: Ident, + pub as_token: Token![as], + pub rename: Ident, + } +} + +ast_struct! { + /// A glob import in a `use` item: `*`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct UseGlob { + pub star_token: Token![*], + } +} + +ast_struct! { + /// A braced group of imports in a `use` item: `{A, B, C}`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct UseGroup { + pub brace_token: token::Brace, + pub items: Punctuated<UseTree, Token![,]>, + } +} + +ast_enum_of_structs! { + /// An item within an `extern` block. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + #[non_exhaustive] + pub enum ForeignItem { + /// A foreign function in an `extern` block. + Fn(ForeignItemFn), + + /// A foreign static item in an `extern` block: `static ext: u8`. + Static(ForeignItemStatic), + + /// A foreign type in an `extern` block: `type void`. + Type(ForeignItemType), + + /// A macro invocation within an extern block. + Macro(ForeignItemMacro), + + /// Tokens in an `extern` block not interpreted by Syn. + Verbatim(TokenStream), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match item { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // ForeignItem::Fn(item) => {...} + // ForeignItem::Static(item) => {...} + // ... + // ForeignItem::Verbatim(item) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// A foreign function in an `extern` block. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ForeignItemFn { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub sig: Signature, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A foreign static item in an `extern` block: `static ext: u8`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ForeignItemStatic { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub static_token: Token![static], + pub mutability: StaticMutability, + pub ident: Ident, + pub colon_token: Token![:], + pub ty: Box<Type>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A foreign type in an `extern` block: `type void`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ForeignItemType { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub type_token: Token![type], + pub ident: Ident, + pub generics: Generics, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A macro invocation within an extern block. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ForeignItemMacro { + pub attrs: Vec<Attribute>, + pub mac: Macro, + pub semi_token: Option<Token![;]>, + } +} + +ast_enum_of_structs! { + /// An item declaration within the definition of a trait. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + #[non_exhaustive] + pub enum TraitItem { + /// An associated constant within the definition of a trait. + Const(TraitItemConst), + + /// An associated function within the definition of a trait. + Fn(TraitItemFn), + + /// An associated type within the definition of a trait. + Type(TraitItemType), + + /// A macro invocation within the definition of a trait. + Macro(TraitItemMacro), + + /// Tokens within the definition of a trait not interpreted by Syn. + Verbatim(TokenStream), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match item { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // TraitItem::Const(item) => {...} + // TraitItem::Fn(item) => {...} + // ... + // TraitItem::Verbatim(item) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// An associated constant within the definition of a trait. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct TraitItemConst { + pub attrs: Vec<Attribute>, + pub const_token: Token![const], + pub ident: Ident, + pub generics: Generics, + pub colon_token: Token![:], + pub ty: Type, + pub default: Option<(Token![=], Expr)>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// An associated function within the definition of a trait. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct TraitItemFn { + pub attrs: Vec<Attribute>, + pub sig: Signature, + pub default: Option<Block>, + pub semi_token: Option<Token![;]>, + } +} + +ast_struct! { + /// An associated type within the definition of a trait. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct TraitItemType { + pub attrs: Vec<Attribute>, + pub type_token: Token![type], + pub ident: Ident, + pub generics: Generics, + pub colon_token: Option<Token![:]>, + pub bounds: Punctuated<TypeParamBound, Token![+]>, + pub default: Option<(Token![=], Type)>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A macro invocation within the definition of a trait. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct TraitItemMacro { + pub attrs: Vec<Attribute>, + pub mac: Macro, + pub semi_token: Option<Token![;]>, + } +} + +ast_enum_of_structs! { + /// An item within an impl block. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + #[non_exhaustive] + pub enum ImplItem { + /// An associated constant within an impl block. + Const(ImplItemConst), + + /// An associated function within an impl block. + Fn(ImplItemFn), + + /// An associated type within an impl block. + Type(ImplItemType), + + /// A macro invocation within an impl block. + Macro(ImplItemMacro), + + /// Tokens within an impl block not interpreted by Syn. + Verbatim(TokenStream), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match item { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // ImplItem::Const(item) => {...} + // ImplItem::Fn(item) => {...} + // ... + // ImplItem::Verbatim(item) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// An associated constant within an impl block. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ImplItemConst { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub defaultness: Option<Token![default]>, + pub const_token: Token![const], + pub ident: Ident, + pub generics: Generics, + pub colon_token: Token![:], + pub ty: Type, + pub eq_token: Token![=], + pub expr: Expr, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// An associated function within an impl block. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ImplItemFn { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub defaultness: Option<Token![default]>, + pub sig: Signature, + pub block: Block, + } +} + +ast_struct! { + /// An associated type within an impl block. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ImplItemType { + pub attrs: Vec<Attribute>, + pub vis: Visibility, + pub defaultness: Option<Token![default]>, + pub type_token: Token![type], + pub ident: Ident, + pub generics: Generics, + pub eq_token: Token![=], + pub ty: Type, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// A macro invocation within an impl block. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct ImplItemMacro { + pub attrs: Vec<Attribute>, + pub mac: Macro, + pub semi_token: Option<Token![;]>, + } +} + +ast_struct! { + /// A function signature in a trait or implementation: `unsafe fn + /// initialize(&self)`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct Signature { + pub constness: Option<Token![const]>, + pub asyncness: Option<Token![async]>, + pub unsafety: Option<Token![unsafe]>, + pub abi: Option<Abi>, + pub fn_token: Token![fn], + pub ident: Ident, + pub generics: Generics, + pub paren_token: token::Paren, + pub inputs: Punctuated<FnArg, Token![,]>, + pub variadic: Option<Variadic>, + pub output: ReturnType, + } +} + +impl Signature { + /// A method's `self` receiver, such as `&self` or `self: Box<Self>`. + pub fn receiver(&self) -> Option<&Receiver> { + let arg = self.inputs.first()?; + match arg { + FnArg::Receiver(receiver) => Some(receiver), + FnArg::Typed(_) => None, + } + } +} + +ast_enum_of_structs! { + /// An argument in a function signature: the `n: usize` in `fn f(n: usize)`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub enum FnArg { + /// The `self` argument of an associated method. + Receiver(Receiver), + + /// A function argument accepted by pattern and type. + Typed(PatType), + } +} + +ast_struct! { + /// The `self` argument of an associated method. + /// + /// If `colon_token` is present, the receiver is written with an explicit + /// type such as `self: Box<Self>`. If `colon_token` is absent, the receiver + /// is written in shorthand such as `self` or `&self` or `&mut self`. In the + /// shorthand case, the type in `ty` is reconstructed as one of `Self`, + /// `&Self`, or `&mut Self`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct Receiver { + pub attrs: Vec<Attribute>, + pub reference: Option<(Token![&], Option<Lifetime>)>, + pub mutability: Option<Token![mut]>, + pub self_token: Token![self], + pub colon_token: Option<Token![:]>, + pub ty: Box<Type>, + } +} + +impl Receiver { + pub fn lifetime(&self) -> Option<&Lifetime> { + self.reference.as_ref()?.1.as_ref() + } +} + +ast_struct! { + /// The variadic argument of a foreign function. + /// + /// ```rust + /// # struct c_char; + /// # struct c_int; + /// # + /// extern "C" { + /// fn printf(format: *const c_char, ...) -> c_int; + /// // ^^^ + /// } + /// ``` + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct Variadic { + pub attrs: Vec<Attribute>, + pub pat: Option<(Box<Pat>, Token![:])>, + pub dots: Token![...], + pub comma: Option<Token![,]>, + } +} + +ast_enum! { + /// The mutability of an `Item::Static` or `ForeignItem::Static`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + #[non_exhaustive] + pub enum StaticMutability { + Mut(Token![mut]), + None, + } +} + +ast_enum! { + /// Unused, but reserved for RFC 3323 restrictions. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + #[non_exhaustive] + pub enum ImplRestriction {} + + + // TODO: https://rust-lang.github.io/rfcs/3323-restrictions.html + // + // pub struct ImplRestriction { + // pub impl_token: Token![impl], + // pub paren_token: token::Paren, + // pub in_token: Option<Token![in]>, + // pub path: Box<Path>, + // } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::{self, Attribute}; + use crate::buffer::Cursor; + use crate::derive; + use crate::error::{Error, Result}; + use crate::expr::Expr; + use crate::ext::IdentExt as _; + use crate::generics::{self, Generics, TypeParamBound}; + use crate::ident::Ident; + use crate::item::{ + FnArg, ForeignItem, ForeignItemFn, ForeignItemMacro, ForeignItemStatic, ForeignItemType, + ImplItem, ImplItemConst, ImplItemFn, ImplItemMacro, ImplItemType, Item, ItemConst, + ItemEnum, ItemExternCrate, ItemFn, ItemForeignMod, ItemImpl, ItemMacro, ItemMod, + ItemStatic, ItemStruct, ItemTrait, ItemTraitAlias, ItemType, ItemUnion, ItemUse, Receiver, + Signature, StaticMutability, TraitItem, TraitItemConst, TraitItemFn, TraitItemMacro, + TraitItemType, UseGlob, UseGroup, UseName, UsePath, UseRename, UseTree, Variadic, + }; + use crate::lifetime::Lifetime; + use crate::lit::LitStr; + use crate::mac::{self, Macro}; + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::pat::{Pat, PatType, PatWild}; + use crate::path::Path; + use crate::punctuated::Punctuated; + use crate::restriction::Visibility; + use crate::stmt::Block; + use crate::token; + use crate::ty::{Abi, ReturnType, Type, TypePath, TypeReference}; + use crate::verbatim; + use alloc::boxed::Box; + use alloc::vec::Vec; + use proc_macro2::TokenStream; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Item { + fn parse(input: ParseStream) -> Result<Self> { + let begin = input.cursor(); + let attrs = input.call(Attribute::parse_outer)?; + parse_rest_of_item(begin, attrs, input) + } + } + + pub(crate) fn parse_rest_of_item( + begin: Cursor, + mut attrs: Vec<Attribute>, + input: ParseStream, + ) -> Result<Item> { + let ahead = input.fork(); + let vis: Visibility = ahead.parse()?; + + let lookahead = ahead.lookahead1(); + let allow_safe = false; + let mut item = if lookahead.peek(Token![fn]) || peek_signature(&ahead, allow_safe) { + let vis: Visibility = input.parse()?; + let sig: Signature = input.parse()?; + if input.peek(Token![;]) { + input.parse::<Token![;]>()?; + Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))) + } else { + parse_rest_of_fn(input, Vec::new(), vis, sig).map(Item::Fn) + } + } else if lookahead.peek(Token![extern]) { + ahead.parse::<Token![extern]>()?; + let lookahead = ahead.lookahead1(); + if lookahead.peek(Token![crate]) { + input.parse().map(Item::ExternCrate) + } else if lookahead.peek(token::Brace) { + input.parse().map(Item::ForeignMod) + } else if lookahead.peek(LitStr) { + ahead.parse::<LitStr>()?; + let lookahead = ahead.lookahead1(); + if lookahead.peek(token::Brace) { + input.parse().map(Item::ForeignMod) + } else { + Err(lookahead.error()) + } + } else { + Err(lookahead.error()) + } + } else if lookahead.peek(Token![use]) { + let allow_crate_root_in_path = true; + match parse_item_use(input, allow_crate_root_in_path)? { + Some(item_use) => Ok(Item::Use(item_use)), + None => Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))), + } + } else if lookahead.peek(Token![static]) { + let vis = input.parse()?; + let static_token = input.parse()?; + let mutability = input.parse()?; + let ident = input.parse()?; + if input.peek(Token![=]) { + input.parse::<Token![=]>()?; + input.parse::<Expr>()?; + input.parse::<Token![;]>()?; + Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))) + } else { + let colon_token = input.parse()?; + let ty = input.parse()?; + if input.peek(Token![;]) { + input.parse::<Token![;]>()?; + Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))) + } else { + Ok(Item::Static(ItemStatic { + attrs: Vec::new(), + vis, + static_token, + mutability, + ident, + colon_token, + ty, + eq_token: input.parse()?, + expr: input.parse()?, + semi_token: input.parse()?, + })) + } + } + } else if lookahead.peek(Token![const]) { + let vis = input.parse()?; + let const_token: Token![const] = input.parse()?; + let lookahead = input.lookahead1(); + let ident = if lookahead.peek(Ident) || lookahead.peek(Token![_]) { + input.call(Ident::parse_any)? + } else { + return Err(lookahead.error()); + }; + let mut generics: Generics = input.parse()?; + let colon_token = input.parse()?; + let ty = input.parse()?; + let value = if let Some(eq_token) = input.parse::<Option<Token![=]>>()? { + let expr: Expr = input.parse()?; + Some((eq_token, expr)) + } else { + None + }; + generics.where_clause = input.parse()?; + let semi_token: Token![;] = input.parse()?; + match value { + Some((eq_token, expr)) + if generics.lt_token.is_none() && generics.where_clause.is_none() => + { + Ok(Item::Const(ItemConst { + attrs: Vec::new(), + vis, + const_token, + ident, + generics, + colon_token, + ty, + eq_token, + expr: Box::new(expr), + semi_token, + })) + } + _ => Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))), + } + } else if lookahead.peek(Token![unsafe]) { + ahead.parse::<Token![unsafe]>()?; + let lookahead = ahead.lookahead1(); + if lookahead.peek(Token![trait]) + || lookahead.peek(Token![auto]) && ahead.peek2(Token![trait]) + { + input.parse().map(Item::Trait) + } else if lookahead.peek(Token![impl]) { + let allow_verbatim_impl = true; + if let Some(item) = parse_impl(input, allow_verbatim_impl)? { + Ok(Item::Impl(item)) + } else { + Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))) + } + } else if lookahead.peek(Token![extern]) { + input.parse().map(Item::ForeignMod) + } else if lookahead.peek(Token![mod]) { + input.parse().map(Item::Mod) + } else { + Err(lookahead.error()) + } + } else if lookahead.peek(Token![mod]) { + input.parse().map(Item::Mod) + } else if lookahead.peek(Token![type]) { + parse_item_type(begin, input) + } else if lookahead.peek(Token![struct]) { + input.parse().map(Item::Struct) + } else if lookahead.peek(Token![enum]) { + input.parse().map(Item::Enum) + } else if lookahead.peek(Token![union]) && ahead.peek2(Ident) { + input.parse().map(Item::Union) + } else if lookahead.peek(Token![trait]) { + input.call(parse_trait_or_trait_alias) + } else if lookahead.peek(Token![auto]) && ahead.peek2(Token![trait]) { + input.parse().map(Item::Trait) + } else if lookahead.peek(Token![impl]) + || lookahead.peek(Token![default]) && !ahead.peek2(Token![!]) + { + let allow_verbatim_impl = true; + if let Some(item) = parse_impl(input, allow_verbatim_impl)? { + Ok(Item::Impl(item)) + } else { + Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))) + } + } else if lookahead.peek(Token![macro]) { + input.advance_to(&ahead); + parse_macro2(begin, vis, input) + } else if vis.is_inherited() + && (lookahead.peek(Ident) + || lookahead.peek(Token![self]) + || lookahead.peek(Token![super]) + || lookahead.peek(Token![crate]) + || lookahead.peek(Token![::])) + { + input.parse().map(Item::Macro) + } else { + Err(lookahead.error()) + }?; + + attrs.extend(item.replace_attrs(Vec::new())); + item.replace_attrs(attrs); + Ok(item) + } + + struct FlexibleItemType { + vis: Visibility, + defaultness: Option<Token![default]>, + type_token: Token![type], + ident: Ident, + generics: Generics, + colon_token: Option<Token![:]>, + bounds: Punctuated<TypeParamBound, Token![+]>, + ty: Option<(Token![=], Type)>, + semi_token: Token![;], + } + + enum TypeDefaultness { + Optional, + Disallowed, + } + + enum WhereClauseLocation { + // type Ty<T> where T: 'static = T; + BeforeEq, + // type Ty<T> = T where T: 'static; + AfterEq, + // TODO: goes away once the migration period on rust-lang/rust#89122 is over + Both, + } + + impl FlexibleItemType { + fn parse( + input: ParseStream, + allow_defaultness: TypeDefaultness, + where_clause_location: WhereClauseLocation, + ) -> Result<Self> { + let vis: Visibility = input.parse()?; + let defaultness: Option<Token![default]> = match allow_defaultness { + TypeDefaultness::Optional => input.parse()?, + TypeDefaultness::Disallowed => None, + }; + let type_token: Token![type] = input.parse()?; + let ident: Ident = input.parse()?; + let mut generics: Generics = input.parse()?; + let (colon_token, bounds) = Self::parse_optional_bounds(input)?; + + match where_clause_location { + WhereClauseLocation::BeforeEq | WhereClauseLocation::Both => { + generics.where_clause = input.parse()?; + } + WhereClauseLocation::AfterEq => {} + } + + let ty = Self::parse_optional_definition(input)?; + + match where_clause_location { + WhereClauseLocation::AfterEq | WhereClauseLocation::Both + if generics.where_clause.is_none() => + { + generics.where_clause = input.parse()?; + } + _ => {} + } + + let semi_token: Token![;] = input.parse()?; + + Ok(FlexibleItemType { + vis, + defaultness, + type_token, + ident, + generics, + colon_token, + bounds, + ty, + semi_token, + }) + } + + fn parse_optional_bounds( + input: ParseStream, + ) -> Result<(Option<Token![:]>, Punctuated<TypeParamBound, Token![+]>)> { + let colon_token: Option<Token![:]> = input.parse()?; + + let mut bounds = Punctuated::new(); + if colon_token.is_some() { + loop { + if input.peek(Token![where]) || input.peek(Token![=]) || input.peek(Token![;]) { + break; + } + bounds.push_value({ + let allow_precise_capture = false; + let allow_const = true; + TypeParamBound::parse_single(input, allow_precise_capture, allow_const)? + }); + if input.peek(Token![where]) || input.peek(Token![=]) || input.peek(Token![;]) { + break; + } + bounds.push_punct(input.parse::<Token![+]>()?); + } + } + + Ok((colon_token, bounds)) + } + + fn parse_optional_definition(input: ParseStream) -> Result<Option<(Token![=], Type)>> { + let eq_token: Option<Token![=]> = input.parse()?; + if let Some(eq_token) = eq_token { + let definition: Type = input.parse()?; + Ok(Some((eq_token, definition))) + } else { + Ok(None) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemMacro { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let path = input.call(Path::parse_mod_style)?; + let bang_token: Token![!] = input.parse()?; + let ident: Option<Ident> = if input.peek(Token![try]) { + input.call(Ident::parse_any).map(Some) + } else { + input.parse() + }?; + let (delimiter, tokens) = input.call(mac::parse_delimiter)?; + let semi_token: Option<Token![;]> = if !delimiter.is_brace() { + Some(input.parse()?) + } else { + None + }; + Ok(ItemMacro { + attrs, + ident, + mac: Macro { + path, + bang_token, + delimiter, + tokens, + }, + semi_token, + }) + } + } + + fn parse_macro2(begin: Cursor, _vis: Visibility, input: ParseStream) -> Result<Item> { + input.parse::<Token![macro]>()?; + input.parse::<Ident>()?; + + let mut lookahead = input.lookahead1(); + if lookahead.peek(token::Paren) { + let paren_content; + parenthesized!(paren_content in input); + paren_content.parse::<TokenStream>()?; + lookahead = input.lookahead1(); + } + + if lookahead.peek(token::Brace) { + let brace_content; + braced!(brace_content in input); + brace_content.parse::<TokenStream>()?; + } else { + return Err(lookahead.error()); + } + + Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemExternCrate { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ItemExternCrate { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + extern_token: input.parse()?, + crate_token: input.parse()?, + ident: { + if input.peek(Token![self]) { + input.call(Ident::parse_any)? + } else { + input.parse()? + } + }, + rename: { + if input.peek(Token![as]) { + let as_token: Token![as] = input.parse()?; + let rename: Ident = if input.peek(Token![_]) { + Ident::from(input.parse::<Token![_]>()?) + } else { + input.parse()? + }; + Some((as_token, rename)) + } else { + None + } + }, + semi_token: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemUse { + fn parse(input: ParseStream) -> Result<Self> { + let allow_crate_root_in_path = false; + parse_item_use(input, allow_crate_root_in_path).map(Option::unwrap) + } + } + + fn parse_item_use( + input: ParseStream, + allow_crate_root_in_path: bool, + ) -> Result<Option<ItemUse>> { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let use_token: Token![use] = input.parse()?; + let leading_colon: Option<Token![::]> = input.parse()?; + let tree = parse_use_tree(input, allow_crate_root_in_path && leading_colon.is_none())?; + let semi_token: Token![;] = input.parse()?; + + let tree = match tree { + Some(tree) => tree, + None => return Ok(None), + }; + + Ok(Some(ItemUse { + attrs, + vis, + use_token, + leading_colon, + tree, + semi_token, + })) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for UseTree { + fn parse(input: ParseStream) -> Result<UseTree> { + let allow_crate_root_in_path = false; + parse_use_tree(input, allow_crate_root_in_path).map(Option::unwrap) + } + } + + fn parse_use_tree( + input: ParseStream, + allow_crate_root_in_path: bool, + ) -> Result<Option<UseTree>> { + let lookahead = input.lookahead1(); + if lookahead.peek(Ident) + || lookahead.peek(Token![self]) + || lookahead.peek(Token![super]) + || lookahead.peek(Token![crate]) + || lookahead.peek(Token![try]) + { + let ident = input.call(Ident::parse_any)?; + if input.peek(Token![::]) { + Ok(Some(UseTree::Path(UsePath { + ident, + colon2_token: input.parse()?, + tree: Box::new(input.parse()?), + }))) + } else if input.peek(Token![as]) { + Ok(Some(UseTree::Rename(UseRename { + ident, + as_token: input.parse()?, + rename: { + if input.peek(Ident) { + input.parse()? + } else if input.peek(Token![_]) { + Ident::from(input.parse::<Token![_]>()?) + } else { + return Err(input.error("expected identifier or underscore")); + } + }, + }))) + } else { + Ok(Some(UseTree::Name(UseName { ident }))) + } + } else if lookahead.peek(Token![*]) { + Ok(Some(UseTree::Glob(UseGlob { + star_token: input.parse()?, + }))) + } else if lookahead.peek(token::Brace) { + let content; + let brace_token = braced!(content in input); + let mut items = Punctuated::new(); + let mut has_any_crate_root_in_path = false; + loop { + if content.is_empty() { + break; + } + let this_tree_starts_with_crate_root = + allow_crate_root_in_path && content.parse::<Option<Token![::]>>()?.is_some(); + has_any_crate_root_in_path |= this_tree_starts_with_crate_root; + match parse_use_tree( + &content, + allow_crate_root_in_path && !this_tree_starts_with_crate_root, + )? { + Some(tree) if !has_any_crate_root_in_path => items.push_value(tree), + _ => has_any_crate_root_in_path = true, + } + if content.is_empty() { + break; + } + let comma: Token![,] = content.parse()?; + if !has_any_crate_root_in_path { + items.push_punct(comma); + } + } + if has_any_crate_root_in_path { + Ok(None) + } else { + Ok(Some(UseTree::Group(UseGroup { brace_token, items }))) + } + } else { + Err(lookahead.error()) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemStatic { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ItemStatic { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + static_token: input.parse()?, + mutability: input.parse()?, + ident: input.parse()?, + colon_token: input.parse()?, + ty: input.parse()?, + eq_token: input.parse()?, + expr: input.parse()?, + semi_token: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemConst { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let const_token: Token![const] = input.parse()?; + + let lookahead = input.lookahead1(); + let ident = if lookahead.peek(Ident) || lookahead.peek(Token![_]) { + input.call(Ident::parse_any)? + } else { + return Err(lookahead.error()); + }; + + let colon_token: Token![:] = input.parse()?; + let ty: Type = input.parse()?; + let eq_token: Token![=] = input.parse()?; + let expr: Expr = input.parse()?; + let semi_token: Token![;] = input.parse()?; + + Ok(ItemConst { + attrs, + vis, + const_token, + ident, + generics: Generics::default(), + colon_token, + ty: Box::new(ty), + eq_token, + expr: Box::new(expr), + semi_token, + }) + } + } + + fn peek_signature(input: ParseStream, allow_safe: bool) -> bool { + let fork = input.fork(); + fork.parse::<Option<Token![const]>>().is_ok() + && fork.parse::<Option<Token![async]>>().is_ok() + && ((allow_safe + && token::parsing::peek_keyword(fork.cursor(), "safe") + && token::parsing::keyword(&fork, "safe").is_ok()) + || fork.parse::<Option<Token![unsafe]>>().is_ok()) + && fork.parse::<Option<Abi>>().is_ok() + && fork.peek(Token![fn]) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Signature { + fn parse(input: ParseStream) -> Result<Self> { + let allow_safe = false; + parse_signature(input, allow_safe).map(Option::unwrap) + } + } + + fn parse_signature(input: ParseStream, allow_safe: bool) -> Result<Option<Signature>> { + let constness: Option<Token![const]> = input.parse()?; + let asyncness: Option<Token![async]> = input.parse()?; + let unsafety: Option<Token![unsafe]> = input.parse()?; + let safe = allow_safe + && unsafety.is_none() + && token::parsing::peek_keyword(input.cursor(), "safe"); + if safe { + token::parsing::keyword(input, "safe")?; + } + let abi: Option<Abi> = input.parse()?; + let fn_token: Token![fn] = input.parse()?; + let ident: Ident = input.parse()?; + let mut generics: Generics = input.parse()?; + + let content; + let paren_token = parenthesized!(content in input); + let (inputs, variadic) = parse_fn_args(&content)?; + + let output: ReturnType = input.parse()?; + generics.where_clause = input.parse()?; + + Ok(if safe { + None + } else { + Some(Signature { + constness, + asyncness, + unsafety, + abi, + fn_token, + ident, + generics, + paren_token, + inputs, + variadic, + output, + }) + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemFn { + fn parse(input: ParseStream) -> Result<Self> { + let outer_attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let sig: Signature = input.parse()?; + parse_rest_of_fn(input, outer_attrs, vis, sig) + } + } + + fn parse_rest_of_fn( + input: ParseStream, + mut attrs: Vec<Attribute>, + vis: Visibility, + sig: Signature, + ) -> Result<ItemFn> { + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + let stmts = content.call(Block::parse_within)?; + + Ok(ItemFn { + attrs, + vis, + sig, + block: Box::new(Block { brace_token, stmts }), + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for FnArg { + fn parse(input: ParseStream) -> Result<Self> { + let allow_variadic = false; + let attrs = input.call(Attribute::parse_outer)?; + match parse_fn_arg_or_variadic(input, attrs, allow_variadic)? { + FnArgOrVariadic::FnArg(arg) => Ok(arg), + FnArgOrVariadic::Variadic(_) => unreachable!(), + } + } + } + + enum FnArgOrVariadic { + FnArg(FnArg), + Variadic(Variadic), + } + + fn parse_fn_arg_or_variadic( + input: ParseStream, + attrs: Vec<Attribute>, + allow_variadic: bool, + ) -> Result<FnArgOrVariadic> { + let ahead = input.fork(); + if let Ok((reference, mutability, self_token)) = parse_receiver_begin(&ahead) { + input.advance_to(&ahead); + let mut receiver = parse_rest_of_receiver(reference, mutability, self_token, input)?; + receiver.attrs = attrs; + return Ok(FnArgOrVariadic::FnArg(FnArg::Receiver(receiver))); + } + + // Hack to parse pre-2018 syntax in + // test/ui/rfc-2565-param-attrs/param-attrs-pretty.rs + // because the rest of the test case is valuable. + if input.peek(Ident) && input.peek2(Token![<]) { + let span = input.span(); + return Ok(FnArgOrVariadic::FnArg(FnArg::Typed(PatType { + attrs, + pat: Box::new(Pat::Wild(PatWild { + attrs: Vec::new(), + underscore_token: Token![_](span), + })), + colon_token: Token![:](span), + ty: input.parse()?, + }))); + } + + let pat = Box::new(Pat::parse_single(input)?); + let colon_token: Token![:] = input.parse()?; + + if allow_variadic { + if let Some(dots) = input.parse::<Option<Token![...]>>()? { + return Ok(FnArgOrVariadic::Variadic(Variadic { + attrs, + pat: Some((pat, colon_token)), + dots, + comma: None, + })); + } + } + + Ok(FnArgOrVariadic::FnArg(FnArg::Typed(PatType { + attrs, + pat, + colon_token, + ty: input.parse()?, + }))) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Receiver { + fn parse(input: ParseStream) -> Result<Self> { + let (reference, mutability, self_token) = parse_receiver_begin(input)?; + parse_rest_of_receiver(reference, mutability, self_token, input) + } + } + + fn parse_receiver_begin( + input: ParseStream, + ) -> Result<( + Option<(Token![&], Option<Lifetime>)>, + Option<Token![mut]>, + Token![self], + )> { + let reference = if input.peek(Token![&]) { + let ampersand: Token![&] = input.parse()?; + let lifetime: Option<Lifetime> = input.parse()?; + Some((ampersand, lifetime)) + } else { + None + }; + let mutability: Option<Token![mut]> = input.parse()?; + let self_token: Token![self] = input.parse()?; + if input.peek(Token![::]) { + return Err(input.error("expected `:`")); + } + Ok((reference, mutability, self_token)) + } + + fn parse_rest_of_receiver( + reference: Option<(Token![&], Option<Lifetime>)>, + mutability: Option<Token![mut]>, + self_token: Token![self], + input: ParseStream, + ) -> Result<Receiver> { + let colon_token: Option<Token![:]> = if reference.is_some() { + None + } else { + input.parse()? + }; + let ty: Type = if colon_token.is_some() { + input.parse()? + } else { + let mut ty = Type::Path(TypePath { + qself: None, + path: Path::from(Ident::new("Self", self_token.span)), + }); + if let Some((ampersand, lifetime)) = reference.as_ref() { + ty = Type::Reference(TypeReference { + and_token: Token![&](ampersand.span), + lifetime: lifetime.clone(), + mutability: mutability.as_ref().map(|m| Token![mut](m.span)), + elem: Box::new(ty), + }); + } + ty + }; + Ok(Receiver { + attrs: Vec::new(), + reference, + mutability, + self_token, + colon_token, + ty: Box::new(ty), + }) + } + + fn parse_fn_args( + input: ParseStream, + ) -> Result<(Punctuated<FnArg, Token![,]>, Option<Variadic>)> { + let mut args = Punctuated::new(); + let mut variadic = None; + let mut has_receiver = false; + + while !input.is_empty() { + let attrs = input.call(Attribute::parse_outer)?; + + if let Some(dots) = input.parse::<Option<Token![...]>>()? { + variadic = Some(Variadic { + attrs, + pat: None, + dots, + comma: if input.is_empty() { + None + } else { + Some(input.parse()?) + }, + }); + break; + } + + let allow_variadic = true; + let arg = match parse_fn_arg_or_variadic(input, attrs, allow_variadic)? { + FnArgOrVariadic::FnArg(arg) => arg, + FnArgOrVariadic::Variadic(arg) => { + variadic = Some(Variadic { + comma: if input.is_empty() { + None + } else { + Some(input.parse()?) + }, + ..arg + }); + break; + } + }; + + match &arg { + FnArg::Receiver(receiver) if has_receiver => { + return Err(Error::new( + receiver.self_token.span, + "unexpected second method receiver", + )); + } + FnArg::Receiver(receiver) if !args.is_empty() => { + return Err(Error::new( + receiver.self_token.span, + "unexpected method receiver", + )); + } + FnArg::Receiver(_) => has_receiver = true, + FnArg::Typed(_) => {} + } + args.push_value(arg); + + if input.is_empty() { + break; + } + + let comma: Token![,] = input.parse()?; + args.push_punct(comma); + } + + Ok((args, variadic)) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemMod { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let unsafety: Option<Token![unsafe]> = input.parse()?; + let mod_token: Token![mod] = input.parse()?; + let ident: Ident = if input.peek(Token![try]) { + input.call(Ident::parse_any) + } else { + input.parse() + }?; + + let lookahead = input.lookahead1(); + if lookahead.peek(Token![;]) { + Ok(ItemMod { + attrs, + vis, + unsafety, + mod_token, + ident, + content: None, + semi: Some(input.parse()?), + }) + } else if lookahead.peek(token::Brace) { + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + + let mut items = Vec::new(); + while !content.is_empty() { + items.push(content.parse()?); + } + + Ok(ItemMod { + attrs, + vis, + unsafety, + mod_token, + ident, + content: Some((brace_token, items)), + semi: None, + }) + } else { + Err(lookahead.error()) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemForeignMod { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = input.call(Attribute::parse_outer)?; + let unsafety: Option<Token![unsafe]> = input.parse()?; + let abi: Abi = input.parse()?; + + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + let mut items = Vec::new(); + while !content.is_empty() { + items.push(content.parse()?); + } + + Ok(ItemForeignMod { + attrs, + unsafety, + abi, + brace_token, + items, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ForeignItem { + fn parse(input: ParseStream) -> Result<Self> { + let begin = input.cursor(); + let mut attrs = input.call(Attribute::parse_outer)?; + let ahead = input.fork(); + let vis: Visibility = ahead.parse()?; + + let lookahead = ahead.lookahead1(); + let allow_safe = true; + let mut item = if lookahead.peek(Token![fn]) || peek_signature(&ahead, allow_safe) { + let vis: Visibility = input.parse()?; + let sig = parse_signature(input, allow_safe)?; + let has_safe = sig.is_none(); + let has_body = input.peek(token::Brace); + let semi_token: Option<Token![;]> = if has_body { + let content; + braced!(content in input); + content.call(Attribute::parse_inner)?; + content.call(Block::parse_within)?; + None + } else { + Some(input.parse()?) + }; + if has_safe || has_body { + Ok(ForeignItem::Verbatim(verbatim::between( + begin, + input.cursor(), + ))) + } else { + Ok(ForeignItem::Fn(ForeignItemFn { + attrs: Vec::new(), + vis, + sig: sig.unwrap(), + semi_token: semi_token.unwrap(), + })) + } + } else if lookahead.peek(Token![static]) + || ((ahead.peek(Token![unsafe]) + || token::parsing::peek_keyword(ahead.cursor(), "safe")) + && ahead.peek2(Token![static])) + { + let vis = input.parse()?; + let unsafety: Option<Token![unsafe]> = input.parse()?; + let safe = + unsafety.is_none() && token::parsing::peek_keyword(input.cursor(), "safe"); + if safe { + token::parsing::keyword(input, "safe")?; + } + let static_token = input.parse()?; + let mutability = input.parse()?; + let ident = input.parse()?; + let colon_token = input.parse()?; + let ty = input.parse()?; + let has_value = input.peek(Token![=]); + if has_value { + input.parse::<Token![=]>()?; + input.parse::<Expr>()?; + } + let semi_token: Token![;] = input.parse()?; + if unsafety.is_some() || safe || has_value { + Ok(ForeignItem::Verbatim(verbatim::between( + begin, + input.cursor(), + ))) + } else { + Ok(ForeignItem::Static(ForeignItemStatic { + attrs: Vec::new(), + vis, + static_token, + mutability, + ident, + colon_token, + ty, + semi_token, + })) + } + } else if lookahead.peek(Token![type]) { + parse_foreign_item_type(begin, input) + } else if vis.is_inherited() + && (lookahead.peek(Ident) + || lookahead.peek(Token![self]) + || lookahead.peek(Token![super]) + || lookahead.peek(Token![crate]) + || lookahead.peek(Token![::])) + { + input.parse().map(ForeignItem::Macro) + } else { + Err(lookahead.error()) + }?; + + let item_attrs = match &mut item { + ForeignItem::Fn(item) => &mut item.attrs, + ForeignItem::Static(item) => &mut item.attrs, + ForeignItem::Type(item) => &mut item.attrs, + ForeignItem::Macro(item) => &mut item.attrs, + ForeignItem::Verbatim(_) => return Ok(item), + }; + attrs.append(item_attrs); + *item_attrs = attrs; + + Ok(item) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ForeignItemFn { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let sig: Signature = input.parse()?; + let semi_token: Token![;] = input.parse()?; + Ok(ForeignItemFn { + attrs, + vis, + sig, + semi_token, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ForeignItemStatic { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ForeignItemStatic { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + static_token: input.parse()?, + mutability: input.parse()?, + ident: input.parse()?, + colon_token: input.parse()?, + ty: input.parse()?, + semi_token: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ForeignItemType { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ForeignItemType { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + type_token: input.parse()?, + ident: input.parse()?, + generics: { + let mut generics: Generics = input.parse()?; + generics.where_clause = input.parse()?; + generics + }, + semi_token: input.parse()?, + }) + } + } + + fn parse_foreign_item_type(begin: Cursor, input: ParseStream) -> Result<ForeignItem> { + let FlexibleItemType { + vis, + defaultness: _, + type_token, + ident, + generics, + colon_token, + bounds: _, + ty, + semi_token, + } = FlexibleItemType::parse( + input, + TypeDefaultness::Disallowed, + WhereClauseLocation::Both, + )?; + + if colon_token.is_some() || ty.is_some() { + Ok(ForeignItem::Verbatim(verbatim::between( + begin, + input.cursor(), + ))) + } else { + Ok(ForeignItem::Type(ForeignItemType { + attrs: Vec::new(), + vis, + type_token, + ident, + generics, + semi_token, + })) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ForeignItemMacro { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let mac: Macro = input.parse()?; + let semi_token: Option<Token![;]> = if mac.delimiter.is_brace() { + None + } else { + Some(input.parse()?) + }; + Ok(ForeignItemMacro { + attrs, + mac, + semi_token, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemType { + fn parse(input: ParseStream) -> Result<Self> { + Ok(ItemType { + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + type_token: input.parse()?, + ident: input.parse()?, + generics: { + let mut generics: Generics = input.parse()?; + generics.where_clause = input.parse()?; + generics + }, + eq_token: input.parse()?, + ty: input.parse()?, + semi_token: input.parse()?, + }) + } + } + + fn parse_item_type(begin: Cursor, input: ParseStream) -> Result<Item> { + let FlexibleItemType { + vis, + defaultness: _, + type_token, + ident, + generics, + colon_token, + bounds: _, + ty, + semi_token, + } = FlexibleItemType::parse( + input, + TypeDefaultness::Disallowed, + WhereClauseLocation::BeforeEq, + )?; + + let (eq_token, ty) = match ty { + Some(ty) if colon_token.is_none() => ty, + _ => return Ok(Item::Verbatim(verbatim::between(begin, input.cursor()))), + }; + + Ok(Item::Type(ItemType { + attrs: Vec::new(), + vis, + type_token, + ident, + generics, + eq_token, + ty: Box::new(ty), + semi_token, + })) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemStruct { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse::<Visibility>()?; + let struct_token = input.parse::<Token![struct]>()?; + let ident = input.parse::<Ident>()?; + let generics = input.parse::<Generics>()?; + let (where_clause, fields, semi_token) = derive::parsing::data_struct(input)?; + Ok(ItemStruct { + attrs, + vis, + struct_token, + ident, + generics: Generics { + where_clause, + ..generics + }, + fields, + semi_token, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemEnum { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse::<Visibility>()?; + let enum_token = input.parse::<Token![enum]>()?; + let ident = input.parse::<Ident>()?; + let generics = input.parse::<Generics>()?; + let (where_clause, brace_token, variants) = derive::parsing::data_enum(input)?; + Ok(ItemEnum { + attrs, + vis, + enum_token, + ident, + generics: Generics { + where_clause, + ..generics + }, + brace_token, + variants, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemUnion { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse::<Visibility>()?; + let union_token = input.parse::<Token![union]>()?; + let ident = input.parse::<Ident>()?; + let generics = input.parse::<Generics>()?; + let (where_clause, fields) = derive::parsing::data_union(input)?; + Ok(ItemUnion { + attrs, + vis, + union_token, + ident, + generics: Generics { + where_clause, + ..generics + }, + fields, + }) + } + } + + fn parse_trait_or_trait_alias(input: ParseStream) -> Result<Item> { + let (attrs, vis, trait_token, ident, generics) = parse_start_of_trait_alias(input)?; + let lookahead = input.lookahead1(); + if lookahead.peek(token::Brace) + || lookahead.peek(Token![:]) + || lookahead.peek(Token![where]) + { + let unsafety = None; + let auto_token = None; + parse_rest_of_trait( + input, + attrs, + vis, + unsafety, + auto_token, + trait_token, + ident, + generics, + ) + .map(Item::Trait) + } else if lookahead.peek(Token![=]) { + parse_rest_of_trait_alias(input, attrs, vis, trait_token, ident, generics) + .map(Item::TraitAlias) + } else { + Err(lookahead.error()) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemTrait { + fn parse(input: ParseStream) -> Result<Self> { + let outer_attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let unsafety: Option<Token![unsafe]> = input.parse()?; + let auto_token: Option<Token![auto]> = input.parse()?; + let trait_token: Token![trait] = input.parse()?; + let ident: Ident = input.parse()?; + let generics: Generics = input.parse()?; + parse_rest_of_trait( + input, + outer_attrs, + vis, + unsafety, + auto_token, + trait_token, + ident, + generics, + ) + } + } + + fn parse_rest_of_trait( + input: ParseStream, + mut attrs: Vec<Attribute>, + vis: Visibility, + unsafety: Option<Token![unsafe]>, + auto_token: Option<Token![auto]>, + trait_token: Token![trait], + ident: Ident, + mut generics: Generics, + ) -> Result<ItemTrait> { + let colon_token: Option<Token![:]> = input.parse()?; + + let mut supertraits = Punctuated::new(); + if colon_token.is_some() { + loop { + if input.peek(Token![where]) || input.peek(token::Brace) { + break; + } + supertraits.push_value({ + let allow_precise_capture = false; + let allow_const = true; + TypeParamBound::parse_single(input, allow_precise_capture, allow_const)? + }); + if input.peek(Token![where]) || input.peek(token::Brace) { + break; + } + supertraits.push_punct(input.parse()?); + } + } + + generics.where_clause = input.parse()?; + + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + let mut items = Vec::new(); + while !content.is_empty() { + items.push(content.parse()?); + } + + Ok(ItemTrait { + attrs, + vis, + unsafety, + auto_token, + restriction: None, + trait_token, + ident, + generics, + colon_token, + supertraits, + brace_token, + items, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemTraitAlias { + fn parse(input: ParseStream) -> Result<Self> { + let (attrs, vis, trait_token, ident, generics) = parse_start_of_trait_alias(input)?; + parse_rest_of_trait_alias(input, attrs, vis, trait_token, ident, generics) + } + } + + fn parse_start_of_trait_alias( + input: ParseStream, + ) -> Result<(Vec<Attribute>, Visibility, Token![trait], Ident, Generics)> { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let trait_token: Token![trait] = input.parse()?; + let ident: Ident = input.parse()?; + let generics: Generics = input.parse()?; + Ok((attrs, vis, trait_token, ident, generics)) + } + + fn parse_rest_of_trait_alias( + input: ParseStream, + attrs: Vec<Attribute>, + vis: Visibility, + trait_token: Token![trait], + ident: Ident, + mut generics: Generics, + ) -> Result<ItemTraitAlias> { + let eq_token: Token![=] = input.parse()?; + + let mut bounds = Punctuated::new(); + loop { + if input.peek(Token![where]) || input.peek(Token![;]) { + break; + } + bounds.push_value({ + let allow_precise_capture = false; + let allow_const = false; + TypeParamBound::parse_single(input, allow_precise_capture, allow_const)? + }); + if input.peek(Token![where]) || input.peek(Token![;]) { + break; + } + bounds.push_punct(input.parse()?); + } + + generics.where_clause = input.parse()?; + let semi_token: Token![;] = input.parse()?; + + Ok(ItemTraitAlias { + attrs, + vis, + trait_token, + ident, + generics, + eq_token, + bounds, + semi_token, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TraitItem { + fn parse(input: ParseStream) -> Result<Self> { + let begin = input.cursor(); + let mut attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let defaultness: Option<Token![default]> = input.parse()?; + let ahead = input.fork(); + + let lookahead = ahead.lookahead1(); + let allow_safe = false; + let mut item = if lookahead.peek(Token![fn]) || peek_signature(&ahead, allow_safe) { + input.parse().map(TraitItem::Fn) + } else if lookahead.peek(Token![const]) { + let const_token: Token![const] = ahead.parse()?; + let lookahead = ahead.lookahead1(); + if lookahead.peek(Ident) || lookahead.peek(Token![_]) { + input.advance_to(&ahead); + let ident = input.call(Ident::parse_any)?; + let mut generics: Generics = input.parse()?; + let colon_token: Token![:] = input.parse()?; + let ty: Type = input.parse()?; + let default = if let Some(eq_token) = input.parse::<Option<Token![=]>>()? { + let expr: Expr = input.parse()?; + Some((eq_token, expr)) + } else { + None + }; + generics.where_clause = input.parse()?; + let semi_token: Token![;] = input.parse()?; + if generics.lt_token.is_none() && generics.where_clause.is_none() { + Ok(TraitItem::Const(TraitItemConst { + attrs: Vec::new(), + const_token, + ident, + generics, + colon_token, + ty, + default, + semi_token, + })) + } else { + return Ok(TraitItem::Verbatim(verbatim::between( + begin, + input.cursor(), + ))); + } + } else if lookahead.peek(Token![async]) + || lookahead.peek(Token![unsafe]) + || lookahead.peek(Token![extern]) + || lookahead.peek(Token![fn]) + { + input.parse().map(TraitItem::Fn) + } else { + Err(lookahead.error()) + } + } else if lookahead.peek(Token![type]) { + parse_trait_item_type(begin, input) + } else if vis.is_inherited() + && defaultness.is_none() + && (lookahead.peek(Ident) + || lookahead.peek(Token![self]) + || lookahead.peek(Token![super]) + || lookahead.peek(Token![crate]) + || lookahead.peek(Token![::])) + { + input.parse().map(TraitItem::Macro) + } else { + Err(lookahead.error()) + }?; + + match (vis, defaultness) { + (Visibility::Inherited, None) => {} + _ => { + return Ok(TraitItem::Verbatim(verbatim::between( + begin, + input.cursor(), + ))) + } + } + + let item_attrs = match &mut item { + TraitItem::Const(item) => &mut item.attrs, + TraitItem::Fn(item) => &mut item.attrs, + TraitItem::Type(item) => &mut item.attrs, + TraitItem::Macro(item) => &mut item.attrs, + TraitItem::Verbatim(_) => unreachable!(), + }; + attrs.append(item_attrs); + *item_attrs = attrs; + Ok(item) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TraitItemConst { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let const_token: Token![const] = input.parse()?; + + let lookahead = input.lookahead1(); + let ident = if lookahead.peek(Ident) || lookahead.peek(Token![_]) { + input.call(Ident::parse_any)? + } else { + return Err(lookahead.error()); + }; + + let colon_token: Token![:] = input.parse()?; + let ty: Type = input.parse()?; + let default = if input.peek(Token![=]) { + let eq_token: Token![=] = input.parse()?; + let default: Expr = input.parse()?; + Some((eq_token, default)) + } else { + None + }; + let semi_token: Token![;] = input.parse()?; + + Ok(TraitItemConst { + attrs, + const_token, + ident, + generics: Generics::default(), + colon_token, + ty, + default, + semi_token, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TraitItemFn { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = input.call(Attribute::parse_outer)?; + let sig: Signature = input.parse()?; + + let lookahead = input.lookahead1(); + let (brace_token, stmts, semi_token) = if lookahead.peek(token::Brace) { + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + let stmts = content.call(Block::parse_within)?; + (Some(brace_token), stmts, None) + } else if lookahead.peek(Token![;]) { + let semi_token: Token![;] = input.parse()?; + (None, Vec::new(), Some(semi_token)) + } else { + return Err(lookahead.error()); + }; + + Ok(TraitItemFn { + attrs, + sig, + default: brace_token.map(|brace_token| Block { brace_token, stmts }), + semi_token, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TraitItemType { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let type_token: Token![type] = input.parse()?; + let ident: Ident = input.parse()?; + let mut generics: Generics = input.parse()?; + let (colon_token, bounds) = FlexibleItemType::parse_optional_bounds(input)?; + let default = FlexibleItemType::parse_optional_definition(input)?; + generics.where_clause = input.parse()?; + let semi_token: Token![;] = input.parse()?; + Ok(TraitItemType { + attrs, + type_token, + ident, + generics, + colon_token, + bounds, + default, + semi_token, + }) + } + } + + fn parse_trait_item_type(begin: Cursor, input: ParseStream) -> Result<TraitItem> { + let FlexibleItemType { + vis, + defaultness: _, + type_token, + ident, + generics, + colon_token, + bounds, + ty, + semi_token, + } = FlexibleItemType::parse( + input, + TypeDefaultness::Disallowed, + WhereClauseLocation::AfterEq, + )?; + + if vis.is_some() { + Ok(TraitItem::Verbatim(verbatim::between( + begin, + input.cursor(), + ))) + } else { + Ok(TraitItem::Type(TraitItemType { + attrs: Vec::new(), + type_token, + ident, + generics, + colon_token, + bounds, + default: ty, + semi_token, + })) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TraitItemMacro { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let mac: Macro = input.parse()?; + let semi_token: Option<Token![;]> = if mac.delimiter.is_brace() { + None + } else { + Some(input.parse()?) + }; + Ok(TraitItemMacro { + attrs, + mac, + semi_token, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ItemImpl { + fn parse(input: ParseStream) -> Result<Self> { + let allow_verbatim_impl = false; + parse_impl(input, allow_verbatim_impl).map(Option::unwrap) + } + } + + fn parse_impl(input: ParseStream, allow_verbatim_impl: bool) -> Result<Option<ItemImpl>> { + let mut attrs = input.call(Attribute::parse_outer)?; + let has_visibility = allow_verbatim_impl && input.parse::<Visibility>()?.is_some(); + let defaultness: Option<Token![default]> = input.parse()?; + let unsafety: Option<Token![unsafe]> = input.parse()?; + let impl_token: Token![impl] = input.parse()?; + + let has_generics = generics::parsing::choose_generics_over_qpath(input); + let mut generics: Generics = if has_generics { + input.parse()? + } else { + Generics::default() + }; + + let is_const_impl = allow_verbatim_impl + && (input.peek(Token![const]) || input.peek(Token![?]) && input.peek2(Token![const])); + if is_const_impl { + input.parse::<Option<Token![?]>>()?; + input.parse::<Token![const]>()?; + } + + let polarity = if input.peek(Token![!]) && !input.peek2(token::Brace) { + Some(input.parse::<Token![!]>()?) + } else { + None + }; + + #[cfg(not(feature = "printing"))] + let first_ty_span = input.span(); + let mut first_ty: Type = input.parse()?; + let self_ty: Type; + let trait_; + + let is_impl_for = input.peek(Token![for]); + if is_impl_for { + let for_token: Token![for] = input.parse()?; + let mut first_ty_ref = &first_ty; + while let Type::Group(ty) = first_ty_ref { + first_ty_ref = &ty.elem; + } + if let Type::Path(TypePath { qself: None, .. }) = first_ty_ref { + while let Type::Group(ty) = first_ty { + first_ty = *ty.elem; + } + if let Type::Path(TypePath { qself: None, path }) = first_ty { + trait_ = Some((polarity, path, for_token)); + } else { + unreachable!(); + } + } else if !allow_verbatim_impl { + #[cfg(feature = "printing")] + return Err(Error::new_spanned(first_ty_ref, "expected trait path")); + #[cfg(not(feature = "printing"))] + return Err(Error::new(first_ty_span, "expected trait path")); + } else { + trait_ = None; + } + self_ty = input.parse()?; + } else if let Some(polarity) = polarity { + return Err(Error::new( + polarity.span, + "inherent impls cannot be negative", + )); + } else { + trait_ = None; + self_ty = first_ty; + } + + generics.where_clause = input.parse()?; + + let content; + let brace_token = braced!(content in input); + attr::parsing::parse_inner(&content, &mut attrs)?; + + let mut items = Vec::new(); + while !content.is_empty() { + items.push(content.parse()?); + } + + if has_visibility || is_const_impl || is_impl_for && trait_.is_none() { + Ok(None) + } else { + Ok(Some(ItemImpl { + attrs, + defaultness, + unsafety, + impl_token, + generics, + trait_, + self_ty: Box::new(self_ty), + brace_token, + items, + })) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ImplItem { + fn parse(input: ParseStream) -> Result<Self> { + let begin = input.cursor(); + let mut attrs = input.call(Attribute::parse_outer)?; + let ahead = input.fork(); + let vis: Visibility = ahead.parse()?; + + let mut lookahead = ahead.lookahead1(); + let defaultness = if lookahead.peek(Token![default]) && !ahead.peek2(Token![!]) { + let defaultness: Token![default] = ahead.parse()?; + lookahead = ahead.lookahead1(); + Some(defaultness) + } else { + None + }; + + let allow_safe = false; + let mut item = if lookahead.peek(Token![fn]) || peek_signature(&ahead, allow_safe) { + let allow_omitted_body = true; + if let Some(item) = parse_impl_item_fn(input, allow_omitted_body)? { + Ok(ImplItem::Fn(item)) + } else { + Ok(ImplItem::Verbatim(verbatim::between(begin, input.cursor()))) + } + } else if lookahead.peek(Token![const]) { + input.advance_to(&ahead); + let const_token: Token![const] = input.parse()?; + let lookahead = input.lookahead1(); + let ident = if lookahead.peek(Ident) || lookahead.peek(Token![_]) { + input.call(Ident::parse_any)? + } else { + return Err(lookahead.error()); + }; + let mut generics: Generics = input.parse()?; + let colon_token: Token![:] = input.parse()?; + let ty: Type = input.parse()?; + let value = if let Some(eq_token) = input.parse::<Option<Token![=]>>()? { + let expr: Expr = input.parse()?; + Some((eq_token, expr)) + } else { + None + }; + generics.where_clause = input.parse()?; + let semi_token: Token![;] = input.parse()?; + return match value { + Some((eq_token, expr)) + if generics.lt_token.is_none() && generics.where_clause.is_none() => + { + Ok(ImplItem::Const(ImplItemConst { + attrs, + vis, + defaultness, + const_token, + ident, + generics, + colon_token, + ty, + eq_token, + expr, + semi_token, + })) + } + _ => Ok(ImplItem::Verbatim(verbatim::between(begin, input.cursor()))), + }; + } else if lookahead.peek(Token![type]) { + parse_impl_item_type(begin, input) + } else if vis.is_inherited() + && defaultness.is_none() + && (lookahead.peek(Ident) + || lookahead.peek(Token![self]) + || lookahead.peek(Token![super]) + || lookahead.peek(Token![crate]) + || lookahead.peek(Token![::])) + { + input.parse().map(ImplItem::Macro) + } else { + Err(lookahead.error()) + }?; + + { + let item_attrs = match &mut item { + ImplItem::Const(item) => &mut item.attrs, + ImplItem::Fn(item) => &mut item.attrs, + ImplItem::Type(item) => &mut item.attrs, + ImplItem::Macro(item) => &mut item.attrs, + ImplItem::Verbatim(_) => return Ok(item), + }; + attrs.append(item_attrs); + *item_attrs = attrs; + } + + Ok(item) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ImplItemConst { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let defaultness: Option<Token![default]> = input.parse()?; + let const_token: Token![const] = input.parse()?; + + let lookahead = input.lookahead1(); + let ident = if lookahead.peek(Ident) || lookahead.peek(Token![_]) { + input.call(Ident::parse_any)? + } else { + return Err(lookahead.error()); + }; + + let colon_token: Token![:] = input.parse()?; + let ty: Type = input.parse()?; + let eq_token: Token![=] = input.parse()?; + let expr: Expr = input.parse()?; + let semi_token: Token![;] = input.parse()?; + + Ok(ImplItemConst { + attrs, + vis, + defaultness, + const_token, + ident, + generics: Generics::default(), + colon_token, + ty, + eq_token, + expr, + semi_token, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ImplItemFn { + fn parse(input: ParseStream) -> Result<Self> { + let allow_omitted_body = false; + parse_impl_item_fn(input, allow_omitted_body).map(Option::unwrap) + } + } + + fn parse_impl_item_fn( + input: ParseStream, + allow_omitted_body: bool, + ) -> Result<Option<ImplItemFn>> { + let mut attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let defaultness: Option<Token![default]> = input.parse()?; + let sig: Signature = input.parse()?; + + // Accept functions without a body in an impl block because rustc's + // *parser* does not reject them (the compilation error is emitted later + // than parsing) and it can be useful for macro DSLs. + if allow_omitted_body && input.parse::<Option<Token![;]>>()?.is_some() { + return Ok(None); + } + + let content; + let brace_token = braced!(content in input); + attrs.extend(content.call(Attribute::parse_inner)?); + let block = Block { + brace_token, + stmts: content.call(Block::parse_within)?, + }; + + Ok(Some(ImplItemFn { + attrs, + vis, + defaultness, + sig, + block, + })) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ImplItemType { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + let defaultness: Option<Token![default]> = input.parse()?; + let type_token: Token![type] = input.parse()?; + let ident: Ident = input.parse()?; + let mut generics: Generics = input.parse()?; + let eq_token: Token![=] = input.parse()?; + let ty: Type = input.parse()?; + generics.where_clause = input.parse()?; + let semi_token: Token![;] = input.parse()?; + Ok(ImplItemType { + attrs, + vis, + defaultness, + type_token, + ident, + generics, + eq_token, + ty, + semi_token, + }) + } + } + + fn parse_impl_item_type(begin: Cursor, input: ParseStream) -> Result<ImplItem> { + let FlexibleItemType { + vis, + defaultness, + type_token, + ident, + generics, + colon_token, + bounds: _, + ty, + semi_token, + } = FlexibleItemType::parse( + input, + TypeDefaultness::Optional, + WhereClauseLocation::AfterEq, + )?; + + let (eq_token, ty) = match ty { + Some(ty) if colon_token.is_none() => ty, + _ => return Ok(ImplItem::Verbatim(verbatim::between(begin, input.cursor()))), + }; + + Ok(ImplItem::Type(ImplItemType { + attrs: Vec::new(), + vis, + defaultness, + type_token, + ident, + generics, + eq_token, + ty, + semi_token, + })) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ImplItemMacro { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let mac: Macro = input.parse()?; + let semi_token: Option<Token![;]> = if mac.delimiter.is_brace() { + None + } else { + Some(input.parse()?) + }; + Ok(ImplItemMacro { + attrs, + mac, + semi_token, + }) + } + } + + impl Visibility { + fn is_inherited(&self) -> bool { + match self { + Visibility::Inherited => true, + _ => false, + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for StaticMutability { + fn parse(input: ParseStream) -> Result<Self> { + let mut_token: Option<Token![mut]> = input.parse()?; + Ok(mut_token.map_or(StaticMutability::None, StaticMutability::Mut)) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::FilterAttrs; + use crate::data::Fields; + use crate::item::{ + ForeignItemFn, ForeignItemMacro, ForeignItemStatic, ForeignItemType, ImplItemConst, + ImplItemFn, ImplItemMacro, ImplItemType, ItemConst, ItemEnum, ItemExternCrate, ItemFn, + ItemForeignMod, ItemImpl, ItemMacro, ItemMod, ItemStatic, ItemStruct, ItemTrait, + ItemTraitAlias, ItemType, ItemUnion, ItemUse, Receiver, Signature, StaticMutability, + TraitItemConst, TraitItemFn, TraitItemMacro, TraitItemType, UseGlob, UseGroup, UseName, + UsePath, UseRename, Variadic, + }; + use crate::mac::MacroDelimiter; + use crate::path; + use crate::path::printing::PathStyle; + use crate::print::TokensOrDefault; + use crate::ty::Type; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemExternCrate { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.extern_token.to_tokens(tokens); + self.crate_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + if let Some((as_token, rename)) = &self.rename { + as_token.to_tokens(tokens); + rename.to_tokens(tokens); + } + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemUse { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.use_token.to_tokens(tokens); + self.leading_colon.to_tokens(tokens); + self.tree.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemStatic { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.static_token.to_tokens(tokens); + self.mutability.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + self.expr.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemConst { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.const_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + self.expr.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemFn { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.sig.to_tokens(tokens); + self.block.brace_token.surround(tokens, |tokens| { + tokens.append_all(self.attrs.inner()); + tokens.append_all(&self.block.stmts); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemMod { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.unsafety.to_tokens(tokens); + self.mod_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + if let Some((brace, items)) = &self.content { + brace.surround(tokens, |tokens| { + tokens.append_all(self.attrs.inner()); + tokens.append_all(items); + }); + } else { + TokensOrDefault(&self.semi).to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemForeignMod { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.unsafety.to_tokens(tokens); + self.abi.to_tokens(tokens); + self.brace_token.surround(tokens, |tokens| { + tokens.append_all(self.attrs.inner()); + tokens.append_all(&self.items); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemType { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.type_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemEnum { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.enum_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + self.brace_token.surround(tokens, |tokens| { + self.variants.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemStruct { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.struct_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + match &self.fields { + Fields::Named(fields) => { + self.generics.where_clause.to_tokens(tokens); + fields.to_tokens(tokens); + } + Fields::Unnamed(fields) => { + fields.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&self.semi_token).to_tokens(tokens); + } + Fields::Unit => { + self.generics.where_clause.to_tokens(tokens); + TokensOrDefault(&self.semi_token).to_tokens(tokens); + } + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemUnion { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.union_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + self.fields.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemTrait { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.unsafety.to_tokens(tokens); + self.auto_token.to_tokens(tokens); + self.trait_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + if !self.supertraits.is_empty() { + TokensOrDefault(&self.colon_token).to_tokens(tokens); + self.supertraits.to_tokens(tokens); + } + self.generics.where_clause.to_tokens(tokens); + self.brace_token.surround(tokens, |tokens| { + tokens.append_all(self.attrs.inner()); + tokens.append_all(&self.items); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemTraitAlias { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.trait_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + self.bounds.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemImpl { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.defaultness.to_tokens(tokens); + self.unsafety.to_tokens(tokens); + self.impl_token.to_tokens(tokens); + self.generics.to_tokens(tokens); + if let Some((polarity, path, for_token)) = &self.trait_ { + polarity.to_tokens(tokens); + path.to_tokens(tokens); + for_token.to_tokens(tokens); + } + self.self_ty.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + self.brace_token.surround(tokens, |tokens| { + tokens.append_all(self.attrs.inner()); + tokens.append_all(&self.items); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ItemMacro { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + path::printing::print_path(tokens, &self.mac.path, PathStyle::Mod); + self.mac.bang_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + match &self.mac.delimiter { + MacroDelimiter::Paren(paren) => { + paren.surround(tokens, |tokens| self.mac.tokens.to_tokens(tokens)); + } + MacroDelimiter::Brace(brace) => { + brace.surround(tokens, |tokens| self.mac.tokens.to_tokens(tokens)); + } + MacroDelimiter::Bracket(bracket) => { + bracket.surround(tokens, |tokens| self.mac.tokens.to_tokens(tokens)); + } + } + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for UsePath { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.ident.to_tokens(tokens); + self.colon2_token.to_tokens(tokens); + self.tree.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for UseName { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.ident.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for UseRename { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.ident.to_tokens(tokens); + self.as_token.to_tokens(tokens); + self.rename.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for UseGlob { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.star_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for UseGroup { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.brace_token.surround(tokens, |tokens| { + self.items.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TraitItemConst { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.const_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + if let Some((eq_token, default)) = &self.default { + eq_token.to_tokens(tokens); + default.to_tokens(tokens); + } + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TraitItemFn { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.sig.to_tokens(tokens); + match &self.default { + Some(block) => { + block.brace_token.surround(tokens, |tokens| { + tokens.append_all(self.attrs.inner()); + tokens.append_all(&block.stmts); + }); + } + None => { + TokensOrDefault(&self.semi_token).to_tokens(tokens); + } + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TraitItemType { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.type_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + if !self.bounds.is_empty() { + TokensOrDefault(&self.colon_token).to_tokens(tokens); + self.bounds.to_tokens(tokens); + } + if let Some((eq_token, default)) = &self.default { + eq_token.to_tokens(tokens); + default.to_tokens(tokens); + } + self.generics.where_clause.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TraitItemMacro { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.mac.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ImplItemConst { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.defaultness.to_tokens(tokens); + self.const_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + self.expr.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ImplItemFn { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.defaultness.to_tokens(tokens); + self.sig.to_tokens(tokens); + self.block.brace_token.surround(tokens, |tokens| { + tokens.append_all(self.attrs.inner()); + tokens.append_all(&self.block.stmts); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ImplItemType { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.defaultness.to_tokens(tokens); + self.type_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ImplItemMacro { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.mac.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ForeignItemFn { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.sig.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ForeignItemStatic { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.static_token.to_tokens(tokens); + self.mutability.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ForeignItemType { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.vis.to_tokens(tokens); + self.type_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ForeignItemMacro { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.mac.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Signature { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.constness.to_tokens(tokens); + self.asyncness.to_tokens(tokens); + self.unsafety.to_tokens(tokens); + self.abi.to_tokens(tokens); + self.fn_token.to_tokens(tokens); + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.paren_token.surround(tokens, |tokens| { + self.inputs.to_tokens(tokens); + if let Some(variadic) = &self.variadic { + if !self.inputs.empty_or_trailing() { + <Token![,]>::default().to_tokens(tokens); + } + variadic.to_tokens(tokens); + } + }); + self.output.to_tokens(tokens); + self.generics.where_clause.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Receiver { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + if let Some((ampersand, lifetime)) = &self.reference { + ampersand.to_tokens(tokens); + lifetime.to_tokens(tokens); + } + self.mutability.to_tokens(tokens); + self.self_token.to_tokens(tokens); + if let Some(colon_token) = &self.colon_token { + colon_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + } else { + let consistent = match (&self.reference, &self.mutability, &*self.ty) { + (Some(_), mutability, Type::Reference(ty)) => { + mutability.is_some() == ty.mutability.is_some() + && match &*ty.elem { + Type::Path(ty) => ty.qself.is_none() && ty.path.is_ident("Self"), + _ => false, + } + } + (None, _, Type::Path(ty)) => ty.qself.is_none() && ty.path.is_ident("Self"), + _ => false, + }; + if !consistent { + <Token![:]>::default().to_tokens(tokens); + self.ty.to_tokens(tokens); + } + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Variadic { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + if let Some((pat, colon)) = &self.pat { + pat.to_tokens(tokens); + colon.to_tokens(tokens); + } + self.dots.to_tokens(tokens); + self.comma.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for StaticMutability { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + StaticMutability::None => {} + StaticMutability::Mut(mut_token) => mut_token.to_tokens(tokens), + } + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/lib.rs b/third_party/crates/syn-2.0.119/src/lib.rs new file mode 100644 index 0000000..aeb74a2 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/lib.rs @@ -0,0 +1,1016 @@ +//! [![github]](https://github.com/dtolnay/syn) [![crates-io]](https://crates.io/crates/syn) [![docs-rs]](crate) +//! +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [crates-io]: https://img.shields.io/badge/crates.io-fc8d62?style=for-the-badge&labelColor=555555&logo=rust +//! [docs-rs]: https://img.shields.io/badge/docs.rs-66c2a5?style=for-the-badge&labelColor=555555&logo=docs.rs +//! +//! <br> +//! +//! Syn is a parsing library for parsing a stream of Rust tokens into a syntax +//! tree of Rust source code. +//! +//! Currently this library is geared toward use in Rust procedural macros, but +//! contains some APIs that may be useful more generally. +//! +//! - **Data structures** — Syn provides a complete syntax tree that can +//! represent any valid Rust source code. The syntax tree is rooted at +//! [`syn::File`] which represents a full source file, but there are other +//! entry points that may be useful to procedural macros including +//! [`syn::Item`], [`syn::Expr`] and [`syn::Type`]. +//! +//! - **Derives** — Of particular interest to derive macros is +//! [`syn::DeriveInput`] which is any of the three legal input items to a +//! derive macro. An example below shows using this type in a library that can +//! derive implementations of a user-defined trait. +//! +//! - **Parsing** — Parsing in Syn is built around [parser functions] with the +//! signature `fn(ParseStream) -> Result<T>`. Every syntax tree node defined +//! by Syn is individually parsable and may be used as a building block for +//! custom syntaxes, or you may dream up your own brand new syntax without +//! involving any of our syntax tree types. +//! +//! - **Location information** — Every token parsed by Syn is associated with a +//! `Span` that tracks line and column information back to the source of that +//! token. These spans allow a procedural macro to display detailed error +//! messages pointing to all the right places in the user's code. There is an +//! example of this below. +//! +//! - **Feature flags** — Functionality is aggressively feature gated so your +//! procedural macros enable only what they need, and do not pay in compile +//! time for all the rest. +//! +//! [`syn::File`]: File +//! [`syn::Item`]: Item +//! [`syn::Expr`]: Expr +//! [`syn::Type`]: Type +//! [`syn::DeriveInput`]: DeriveInput +//! [parser functions]: mod@parse +//! +//! <br> +//! +//! # Example of a derive macro +//! +//! The canonical derive macro using Syn looks like this. We write an ordinary +//! Rust function tagged with a `proc_macro_derive` attribute and the name of +//! the trait we are deriving. Any time that derive appears in the user's code, +//! the Rust compiler passes their data structure as tokens into our macro. We +//! get to execute arbitrary Rust code to figure out what to do with those +//! tokens, then hand some tokens back to the compiler to compile into the +//! user's crate. +//! +//! [`TokenStream`]: proc_macro::TokenStream +//! +//! ```toml +//! [dependencies] +//! syn = "2.0" +//! quote = "1.0" +//! +//! [lib] +//! proc-macro = true +//! ``` +//! +//! ``` +//! # extern crate proc_macro; +//! # +//! use proc_macro::TokenStream; +//! use quote::quote; +//! use syn::{parse_macro_input, DeriveInput}; +//! +//! # const IGNORE_TOKENS: &str = stringify! { +//! #[proc_macro_derive(MyMacro)] +//! # }; +//! pub fn my_macro(input: TokenStream) -> TokenStream { +//! // Parse the input tokens into a syntax tree +//! let input = parse_macro_input!(input as DeriveInput); +//! +//! // Build the output, possibly using quasi-quotation +//! let expanded = quote! { +//! // ... +//! }; +//! +//! // Hand the output tokens back to the compiler +//! TokenStream::from(expanded) +//! } +//! ``` +//! +//! The [`heapsize`] example directory shows a complete working implementation +//! of a derive macro. The example derives a `HeapSize` trait which computes an +//! estimate of the amount of heap memory owned by a value. +//! +//! [`heapsize`]: https://github.com/dtolnay/syn/tree/master/examples/heapsize +//! +//! ``` +//! pub trait HeapSize { +//! /// Total number of bytes of heap memory owned by `self`. +//! fn heap_size_of_children(&self) -> usize; +//! } +//! ``` +//! +//! The derive macro allows users to write `#[derive(HeapSize)]` on data +//! structures in their program. +//! +//! ``` +//! # const IGNORE_TOKENS: &str = stringify! { +//! #[derive(HeapSize)] +//! # }; +//! struct Demo<'a, T: ?Sized> { +//! a: Box<T>, +//! b: u8, +//! c: &'a str, +//! d: String, +//! } +//! ``` +//! +//! <p><br></p> +//! +//! # Spans and error reporting +//! +//! The token-based procedural macro API provides great control over where the +//! compiler's error messages are displayed in user code. Consider the error the +//! user sees if one of their field types does not implement `HeapSize`. +//! +//! ``` +//! # const IGNORE_TOKENS: &str = stringify! { +//! #[derive(HeapSize)] +//! # }; +//! struct Broken { +//! ok: String, +//! bad: std::thread::Thread, +//! } +//! ``` +//! +//! By tracking span information all the way through the expansion of a +//! procedural macro as shown in the `heapsize` example, token-based macros in +//! Syn are able to trigger errors that directly pinpoint the source of the +//! problem. +//! +//! ```text +//! error[E0277]: the trait bound `std::thread::Thread: HeapSize` is not satisfied +//! --> src/main.rs:7:5 +//! | +//! 7 | bad: std::thread::Thread, +//! | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `HeapSize` is not implemented for `Thread` +//! ``` +//! +//! <br> +//! +//! # Parsing a custom syntax +//! +//! The [`lazy-static`] example directory shows the implementation of a +//! `functionlike!(...)` procedural macro in which the input tokens are parsed +//! using Syn's parsing API. +//! +//! [`lazy-static`]: https://github.com/dtolnay/syn/tree/master/examples/lazy-static +//! +//! The example reimplements the popular `lazy_static` crate from crates.io as a +//! procedural macro. +//! +//! ``` +//! # macro_rules! lazy_static { +//! # ($($tt:tt)*) => {} +//! # } +//! # +//! lazy_static! { +//! static ref USERNAME: Regex = Regex::new("^[a-z0-9_-]{3,16}$").unwrap(); +//! } +//! ``` +//! +//! The implementation shows how to trigger custom warnings and error messages +//! on the macro input. +//! +//! ```text +//! warning: come on, pick a more creative name +//! --> src/main.rs:10:16 +//! | +//! 10 | static ref FOO: String = "lazy_static".to_owned(); +//! | ^^^ +//! ``` +//! +//! <br> +//! +//! # Testing +//! +//! When testing macros, we often care not just that the macro can be used +//! successfully but also that when the macro is provided with invalid input it +//! produces maximally helpful error messages. Consider using the [`trybuild`] +//! crate to write tests for errors that are emitted by your macro or errors +//! detected by the Rust compiler in the expanded code following misuse of the +//! macro. Such tests help avoid regressions from later refactors that +//! mistakenly make an error no longer trigger or be less helpful than it used +//! to be. +//! +//! [`trybuild`]: https://github.com/dtolnay/trybuild +//! +//! <br> +//! +//! # Debugging +//! +//! When developing a procedural macro it can be helpful to look at what the +//! generated code looks like. Use `cargo rustc -- -Zunstable-options +//! -Zunpretty=expanded` or the [`cargo expand`] subcommand. +//! +//! [`cargo expand`]: https://github.com/dtolnay/cargo-expand +//! +//! To show the expanded code for some crate that uses your procedural macro, +//! run `cargo expand` from that crate. To show the expanded code for one of +//! your own test cases, run `cargo expand --test the_test_case` where the last +//! argument is the name of the test file without the `.rs` extension. +//! +//! This write-up by Brandon W Maister discusses debugging in more detail: +//! [Debugging Rust's new Custom Derive system][debugging]. +//! +//! [debugging]: https://quodlibetor.github.io/posts/debugging-rusts-new-custom-derive-system/ +//! +//! <br> +//! +//! # Optional features +//! +//! Syn puts a lot of functionality behind optional features in order to +//! optimize compile time for the most common use cases. The following features +//! are available. +//! +//! - **`derive`** *(enabled by default)* — Data structures for representing the +//! possible input to a derive macro, including structs and enums and types. +//! - **`full`** — Data structures for representing the syntax tree of all valid +//! Rust source code, including items and expressions. +//! - **`parsing`** *(enabled by default)* — Ability to parse input tokens into +//! a syntax tree node of a chosen type. +//! - **`printing`** *(enabled by default)* — Ability to print a syntax tree +//! node as tokens of Rust source code. +//! - **`visit`** — Trait for traversing a syntax tree. +//! - **`visit-mut`** — Trait for traversing and mutating in place a syntax +//! tree. +//! - **`fold`** — Trait for transforming an owned syntax tree. +//! - **`clone-impls`** *(enabled by default)* — Clone impls for all syntax tree +//! types. +//! - **`extra-traits`** — Debug, Eq, PartialEq, Hash impls for all syntax tree +//! types. +//! - **`proc-macro`** *(enabled by default)* — Runtime dependency on the +//! dynamic library libproc_macro from rustc toolchain. + +#![no_std] +#![doc(html_root_url = "https://docs.rs/syn/2.0.119")] +#![cfg_attr(docsrs, feature(doc_cfg), doc(auto_cfg = false))] +#![deny(unsafe_op_in_unsafe_fn)] +#![allow(non_camel_case_types)] +#![cfg_attr(not(check_cfg), allow(unexpected_cfgs))] +#![allow( + clippy::bool_to_int_with_if, + clippy::cast_lossless, + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_ptr_alignment, + clippy::default_trait_access, + clippy::derivable_impls, + clippy::diverging_sub_expression, + clippy::doc_markdown, + clippy::elidable_lifetime_names, + clippy::enum_glob_use, + clippy::expl_impl_clone_on_copy, + clippy::explicit_auto_deref, + clippy::fn_params_excessive_bools, + clippy::if_not_else, + clippy::inherent_to_string, + clippy::into_iter_without_iter, + clippy::items_after_statements, + clippy::large_enum_variant, + clippy::let_underscore_untyped, // https://github.com/rust-lang/rust-clippy/issues/10410 + clippy::manual_assert, + clippy::manual_let_else, + clippy::manual_map, + clippy::match_like_matches_macro, + clippy::match_same_arms, + clippy::match_wildcard_for_single_variants, // clippy bug: https://github.com/rust-lang/rust-clippy/issues/6984 + clippy::missing_errors_doc, + clippy::missing_panics_doc, + clippy::module_name_repetitions, + clippy::must_use_candidate, + clippy::needless_doctest_main, + clippy::needless_lifetimes, + clippy::needless_pass_by_value, + clippy::needless_update, + clippy::never_loop, + clippy::range_plus_one, + clippy::redundant_else, + clippy::ref_option, + clippy::return_self_not_must_use, + clippy::similar_names, + clippy::single_match_else, + clippy::struct_excessive_bools, + clippy::too_many_arguments, + clippy::too_many_lines, + clippy::trivially_copy_pass_by_ref, + clippy::type_complexity, + clippy::unconditional_recursion, // https://github.com/rust-lang/rust-clippy/issues/12133 + clippy::uninhabited_references, + clippy::uninlined_format_args, + clippy::unnecessary_box_returns, + clippy::unnecessary_unwrap, + clippy::used_underscore_binding, + clippy::wildcard_imports, +)] +#![allow(unknown_lints, mismatched_lifetime_syntaxes)] + +extern crate alloc; +extern crate std; + +extern crate self as syn; + +#[cfg(feature = "proc-macro")] +extern crate proc_macro; + +#[macro_use] +mod macros; + +#[cfg(feature = "parsing")] +#[macro_use] +mod group; + +#[macro_use] +pub mod token; + +#[cfg(any(feature = "full", feature = "derive"))] +mod attr; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::attr::{AttrStyle, Attribute, Meta, MetaList, MetaNameValue}; + +mod bigint; + +#[cfg(feature = "parsing")] +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +pub mod buffer; + +#[cfg(any( + all(feature = "parsing", feature = "full"), + all(feature = "printing", any(feature = "full", feature = "derive")), +))] +mod classify; + +mod custom_keyword; + +mod custom_punctuation; + +#[cfg(any(feature = "full", feature = "derive"))] +mod data; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::data::{Field, Fields, FieldsNamed, FieldsUnnamed, Variant}; + +#[cfg(any(feature = "full", feature = "derive"))] +mod derive; +#[cfg(feature = "derive")] +#[cfg_attr(docsrs, doc(cfg(feature = "derive")))] +pub use crate::derive::{Data, DataEnum, DataStruct, DataUnion, DeriveInput}; + +mod drops; + +mod error; +pub use crate::error::{Error, Result}; + +#[cfg(any(feature = "full", feature = "derive"))] +mod expr; +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub use crate::expr::{Arm, Label, PointerMutability, RangeLimits}; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::expr::{ + Expr, ExprBinary, ExprCall, ExprCast, ExprField, ExprIndex, ExprLit, ExprMacro, ExprMethodCall, + ExprParen, ExprPath, ExprReference, ExprStruct, ExprUnary, FieldValue, Index, Member, +}; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub use crate::expr::{ + ExprArray, ExprAssign, ExprAsync, ExprAwait, ExprBlock, ExprBreak, ExprClosure, ExprConst, + ExprContinue, ExprForLoop, ExprGroup, ExprIf, ExprInfer, ExprLet, ExprLoop, ExprMatch, + ExprRange, ExprRawAddr, ExprRepeat, ExprReturn, ExprTry, ExprTryBlock, ExprTuple, ExprUnsafe, + ExprWhile, ExprYield, +}; + +pub mod ext; + +#[cfg(feature = "full")] +mod file; +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub use crate::file::File; + +#[cfg(all(any(feature = "full", feature = "derive"), feature = "printing"))] +mod fixup; + +#[cfg(any(feature = "full", feature = "derive"))] +mod generics; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::generics::{ + BoundLifetimes, ConstParam, GenericParam, Generics, LifetimeParam, PredicateLifetime, + PredicateType, TraitBound, TraitBoundModifier, TypeParam, TypeParamBound, WhereClause, + WherePredicate, +}; +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub use crate::generics::{CapturedParam, PreciseCapture}; +#[cfg(all(any(feature = "full", feature = "derive"), feature = "printing"))] +#[cfg_attr( + docsrs, + doc(cfg(all(any(feature = "full", feature = "derive"), feature = "printing"))) +)] +pub use crate::generics::{ImplGenerics, Turbofish, TypeGenerics}; + +mod ident; +#[doc(inline)] +pub use crate::ident::Ident; + +#[cfg(feature = "full")] +mod item; +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub use crate::item::{ + FnArg, ForeignItem, ForeignItemFn, ForeignItemMacro, ForeignItemStatic, ForeignItemType, + ImplItem, ImplItemConst, ImplItemFn, ImplItemMacro, ImplItemType, ImplRestriction, Item, + ItemConst, ItemEnum, ItemExternCrate, ItemFn, ItemForeignMod, ItemImpl, ItemMacro, ItemMod, + ItemStatic, ItemStruct, ItemTrait, ItemTraitAlias, ItemType, ItemUnion, ItemUse, Receiver, + Signature, StaticMutability, TraitItem, TraitItemConst, TraitItemFn, TraitItemMacro, + TraitItemType, UseGlob, UseGroup, UseName, UsePath, UseRename, UseTree, Variadic, +}; + +mod lifetime; +#[doc(inline)] +pub use crate::lifetime::Lifetime; + +mod lit; +#[doc(hidden)] // https://github.com/dtolnay/syn/issues/1566 +pub use crate::lit::StrStyle; +#[doc(inline)] +pub use crate::lit::{ + Lit, LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitInt, LitStr, +}; + +#[cfg(feature = "parsing")] +mod lookahead; + +#[cfg(any(feature = "full", feature = "derive"))] +mod mac; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::mac::{Macro, MacroDelimiter}; + +#[cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))] +#[cfg_attr( + docsrs, + doc(cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))) +)] +pub mod meta; + +#[cfg(any(feature = "full", feature = "derive"))] +mod op; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::op::{BinOp, UnOp}; + +#[cfg(feature = "parsing")] +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +pub mod parse; + +#[cfg(all(feature = "parsing", feature = "proc-macro"))] +mod parse_macro_input; + +#[cfg(all(feature = "parsing", feature = "printing"))] +mod parse_quote; + +#[cfg(feature = "full")] +mod pat; +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub use crate::pat::{ + FieldPat, Pat, PatConst, PatIdent, PatLit, PatMacro, PatOr, PatParen, PatPath, PatRange, + PatReference, PatRest, PatSlice, PatStruct, PatTuple, PatTupleStruct, PatType, PatWild, +}; + +#[cfg(any(feature = "full", feature = "derive"))] +mod path; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::path::{ + AngleBracketedGenericArguments, AssocConst, AssocType, Constraint, GenericArgument, + ParenthesizedGenericArguments, Path, PathArguments, PathSegment, QSelf, +}; + +#[cfg(all( + any(feature = "full", feature = "derive"), + any(feature = "parsing", feature = "printing") +))] +mod precedence; + +#[cfg(all(any(feature = "full", feature = "derive"), feature = "printing"))] +mod print; + +pub mod punctuated; + +#[cfg(any(feature = "full", feature = "derive"))] +mod restriction; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::restriction::{FieldMutability, VisRestricted, Visibility}; + +mod sealed; + +#[cfg(all(feature = "parsing", feature = "derive", not(feature = "full")))] +mod scan_expr; + +mod span; + +#[cfg(all(feature = "parsing", feature = "printing"))] +#[cfg_attr(docsrs, doc(cfg(all(feature = "parsing", feature = "printing"))))] +pub mod spanned; + +#[cfg(feature = "full")] +mod stmt; +#[cfg(feature = "full")] +#[cfg_attr(docsrs, doc(cfg(feature = "full")))] +pub use crate::stmt::{Block, Local, LocalInit, Stmt, StmtMacro}; + +mod thread; + +#[cfg(all(any(feature = "full", feature = "derive"), feature = "extra-traits"))] +mod tt; + +#[cfg(any(feature = "full", feature = "derive"))] +mod ty; +#[cfg(any(feature = "full", feature = "derive"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] +pub use crate::ty::{ + Abi, BareFnArg, BareVariadic, ReturnType, Type, TypeArray, TypeBareFn, TypeGroup, + TypeImplTrait, TypeInfer, TypeMacro, TypeNever, TypeParen, TypePath, TypePtr, TypeReference, + TypeSlice, TypeTraitObject, TypeTuple, +}; + +#[cfg(all(any(feature = "full", feature = "derive"), feature = "parsing"))] +mod verbatim; + +#[cfg(all(feature = "parsing", feature = "full"))] +mod whitespace; + +#[rustfmt::skip] // https://github.com/rust-lang/rustfmt/issues/6176 +mod gen { + /// Syntax tree traversal to transform the nodes of an owned syntax tree. + /// + /// Each method of the [`Fold`] trait is a hook that can be overridden to + /// customize the behavior when transforming the corresponding type of node. + /// By default, every method recursively visits the substructure of the + /// input by invoking the right visitor method of each of its fields. + /// + /// [`Fold`]: fold::Fold + /// + /// ``` + /// # use syn::{Attribute, BinOp, Expr, ExprBinary}; + /// # + /// pub trait Fold { + /// /* ... */ + /// + /// fn fold_expr_binary(&mut self, node: ExprBinary) -> ExprBinary { + /// fold_expr_binary(self, node) + /// } + /// + /// /* ... */ + /// # fn fold_attribute(&mut self, node: Attribute) -> Attribute; + /// # fn fold_expr(&mut self, node: Expr) -> Expr; + /// # fn fold_bin_op(&mut self, node: BinOp) -> BinOp; + /// } + /// + /// pub fn fold_expr_binary<V>(v: &mut V, node: ExprBinary) -> ExprBinary + /// where + /// V: Fold + ?Sized, + /// { + /// ExprBinary { + /// attrs: node + /// .attrs + /// .into_iter() + /// .map(|attr| v.fold_attribute(attr)) + /// .collect(), + /// left: Box::new(v.fold_expr(*node.left)), + /// op: v.fold_bin_op(node.op), + /// right: Box::new(v.fold_expr(*node.right)), + /// } + /// } + /// + /// /* ... */ + /// ``` + /// + /// <br> + /// + /// # Example + /// + /// This fold inserts parentheses to fully parenthesizes any expression. + /// + /// ``` + /// // [dependencies] + /// // quote = "1.0" + /// // syn = { version = "2.0", features = ["fold", "full"] } + /// + /// use quote::quote; + /// use syn::fold::{fold_expr, Fold}; + /// use syn::{token, Expr, ExprParen}; + /// + /// struct ParenthesizeEveryExpr; + /// + /// impl Fold for ParenthesizeEveryExpr { + /// fn fold_expr(&mut self, expr: Expr) -> Expr { + /// Expr::Paren(ExprParen { + /// attrs: Vec::new(), + /// expr: Box::new(fold_expr(self, expr)), + /// paren_token: token::Paren::default(), + /// }) + /// } + /// } + /// + /// fn main() { + /// let code = quote! { a() + b(1) * c.d }; + /// let expr: Expr = syn::parse2(code).unwrap(); + /// let parenthesized = ParenthesizeEveryExpr.fold_expr(expr); + /// println!("{}", quote!(#parenthesized)); + /// + /// // Output: (((a)()) + (((b)((1))) * ((c).d))) + /// } + /// ``` + #[cfg(feature = "fold")] + #[cfg_attr(docsrs, doc(cfg(feature = "fold")))] + #[rustfmt::skip] + pub mod fold; + + /// Syntax tree traversal to walk a shared borrow of a syntax tree. + /// + /// Each method of the [`Visit`] trait is a hook that can be overridden to + /// customize the behavior when visiting the corresponding type of node. By + /// default, every method recursively visits the substructure of the input + /// by invoking the right visitor method of each of its fields. + /// + /// [`Visit`]: visit::Visit + /// + /// ``` + /// # use syn::{Attribute, BinOp, Expr, ExprBinary}; + /// # + /// pub trait Visit<'ast> { + /// /* ... */ + /// + /// fn visit_expr_binary(&mut self, node: &'ast ExprBinary) { + /// visit_expr_binary(self, node); + /// } + /// + /// /* ... */ + /// # fn visit_attribute(&mut self, node: &'ast Attribute); + /// # fn visit_expr(&mut self, node: &'ast Expr); + /// # fn visit_bin_op(&mut self, node: &'ast BinOp); + /// } + /// + /// pub fn visit_expr_binary<'ast, V>(v: &mut V, node: &'ast ExprBinary) + /// where + /// V: Visit<'ast> + ?Sized, + /// { + /// for attr in &node.attrs { + /// v.visit_attribute(attr); + /// } + /// v.visit_expr(&*node.left); + /// v.visit_bin_op(&node.op); + /// v.visit_expr(&*node.right); + /// } + /// + /// /* ... */ + /// ``` + /// + /// <br> + /// + /// # Example + /// + /// This visitor will print the name of every freestanding function in the + /// syntax tree, including nested functions. + /// + /// ``` + /// // [dependencies] + /// // quote = "1.0" + /// // syn = { version = "2.0", features = ["full", "visit"] } + /// + /// use quote::quote; + /// use syn::visit::{self, Visit}; + /// use syn::{File, ItemFn}; + /// + /// struct FnVisitor; + /// + /// impl<'ast> Visit<'ast> for FnVisitor { + /// fn visit_item_fn(&mut self, node: &'ast ItemFn) { + /// println!("Function with name={}", node.sig.ident); + /// + /// // Delegate to the default impl to visit any nested functions. + /// visit::visit_item_fn(self, node); + /// } + /// } + /// + /// fn main() { + /// let code = quote! { + /// pub fn f() { + /// fn g() {} + /// } + /// }; + /// + /// let syntax_tree: File = syn::parse2(code).unwrap(); + /// FnVisitor.visit_file(&syntax_tree); + /// } + /// ``` + /// + /// The `'ast` lifetime on the input references means that the syntax tree + /// outlives the complete recursive visit call, so the visitor is allowed to + /// hold on to references into the syntax tree. + /// + /// ``` + /// use quote::quote; + /// use syn::visit::{self, Visit}; + /// use syn::{File, ItemFn}; + /// + /// struct FnVisitor<'ast> { + /// functions: Vec<&'ast ItemFn>, + /// } + /// + /// impl<'ast> Visit<'ast> for FnVisitor<'ast> { + /// fn visit_item_fn(&mut self, node: &'ast ItemFn) { + /// self.functions.push(node); + /// visit::visit_item_fn(self, node); + /// } + /// } + /// + /// fn main() { + /// let code = quote! { + /// pub fn f() { + /// fn g() {} + /// } + /// }; + /// + /// let syntax_tree: File = syn::parse2(code).unwrap(); + /// let mut visitor = FnVisitor { functions: Vec::new() }; + /// visitor.visit_file(&syntax_tree); + /// for f in visitor.functions { + /// println!("Function with name={}", f.sig.ident); + /// } + /// } + /// ``` + #[cfg(feature = "visit")] + #[cfg_attr(docsrs, doc(cfg(feature = "visit")))] + #[rustfmt::skip] + pub mod visit; + + /// Syntax tree traversal to mutate an exclusive borrow of a syntax tree in + /// place. + /// + /// Each method of the [`VisitMut`] trait is a hook that can be overridden + /// to customize the behavior when mutating the corresponding type of node. + /// By default, every method recursively visits the substructure of the + /// input by invoking the right visitor method of each of its fields. + /// + /// [`VisitMut`]: visit_mut::VisitMut + /// + /// ``` + /// # use syn::{Attribute, BinOp, Expr, ExprBinary}; + /// # + /// pub trait VisitMut { + /// /* ... */ + /// + /// fn visit_expr_binary_mut(&mut self, node: &mut ExprBinary) { + /// visit_expr_binary_mut(self, node); + /// } + /// + /// /* ... */ + /// # fn visit_attribute_mut(&mut self, node: &mut Attribute); + /// # fn visit_expr_mut(&mut self, node: &mut Expr); + /// # fn visit_bin_op_mut(&mut self, node: &mut BinOp); + /// } + /// + /// pub fn visit_expr_binary_mut<V>(v: &mut V, node: &mut ExprBinary) + /// where + /// V: VisitMut + ?Sized, + /// { + /// for attr in &mut node.attrs { + /// v.visit_attribute_mut(attr); + /// } + /// v.visit_expr_mut(&mut *node.left); + /// v.visit_bin_op_mut(&mut node.op); + /// v.visit_expr_mut(&mut *node.right); + /// } + /// + /// /* ... */ + /// ``` + /// + /// <br> + /// + /// # Example + /// + /// This mut visitor replace occurrences of u256 suffixed integer literals + /// like `999u256` with a macro invocation `bigint::u256!(999)`. + /// + /// ``` + /// // [dependencies] + /// // quote = "1.0" + /// // syn = { version = "2.0", features = ["full", "visit-mut"] } + /// + /// use quote::quote; + /// use syn::visit_mut::{self, VisitMut}; + /// use syn::{parse_quote, Expr, File, Lit, LitInt}; + /// + /// struct BigintReplace; + /// + /// impl VisitMut for BigintReplace { + /// fn visit_expr_mut(&mut self, node: &mut Expr) { + /// if let Expr::Lit(expr) = &node { + /// if let Lit::Int(int) = &expr.lit { + /// if int.suffix() == "u256" { + /// let digits = int.base10_digits(); + /// let unsuffixed: LitInt = syn::parse_str(digits).unwrap(); + /// *node = parse_quote!(bigint::u256!(#unsuffixed)); + /// return; + /// } + /// } + /// } + /// + /// // Delegate to the default impl to visit nested expressions. + /// visit_mut::visit_expr_mut(self, node); + /// } + /// } + /// + /// fn main() { + /// let code = quote! { + /// fn main() { + /// let _ = 999u256; + /// } + /// }; + /// + /// let mut syntax_tree: File = syn::parse2(code).unwrap(); + /// BigintReplace.visit_file_mut(&mut syntax_tree); + /// println!("{}", quote!(#syntax_tree)); + /// } + /// ``` + #[cfg(feature = "visit-mut")] + #[cfg_attr(docsrs, doc(cfg(feature = "visit-mut")))] + #[rustfmt::skip] + pub mod visit_mut; + + #[cfg(feature = "clone-impls")] + #[rustfmt::skip] + mod clone; + + #[cfg(feature = "extra-traits")] + #[rustfmt::skip] + mod debug; + + #[cfg(feature = "extra-traits")] + #[rustfmt::skip] + mod eq; + + #[cfg(feature = "extra-traits")] + #[rustfmt::skip] + mod hash; +} + +#[cfg(feature = "fold")] +#[cfg_attr(docsrs, doc(cfg(feature = "fold")))] +pub use crate::gen::fold; + +#[cfg(feature = "visit")] +#[cfg_attr(docsrs, doc(cfg(feature = "visit")))] +pub use crate::gen::visit; + +#[cfg(feature = "visit-mut")] +#[cfg_attr(docsrs, doc(cfg(feature = "visit-mut")))] +pub use crate::gen::visit_mut; + +// Not public API. +#[doc(hidden)] +#[path = "export.rs"] +pub mod __private; + +#[cfg(all(feature = "parsing", feature = "full"))] +use alloc::string::ToString; + +/// Parse tokens of source code into the chosen syntax tree node. +/// +/// This is preferred over parsing a string because tokens are able to preserve +/// information about where in the user's code they were originally written (the +/// "span" of the token), possibly allowing the compiler to produce better error +/// messages. +/// +/// This function parses a `proc_macro::TokenStream` which is the type used for +/// interop with the compiler in a procedural macro. To parse a +/// `proc_macro2::TokenStream`, use [`syn::parse2`] instead. +/// +/// [`syn::parse2`]: parse2 +/// +/// This function enforces that the input is fully parsed. If there are any +/// unparsed tokens at the end of the stream, an error is returned. +#[cfg(all(feature = "parsing", feature = "proc-macro"))] +#[cfg_attr(docsrs, doc(cfg(all(feature = "parsing", feature = "proc-macro"))))] +pub fn parse<T: parse::Parse>(tokens: proc_macro::TokenStream) -> Result<T> { + parse::Parser::parse(T::parse, tokens) +} + +/// Parse a proc-macro2 token stream into the chosen syntax tree node. +/// +/// This function parses a `proc_macro2::TokenStream` which is commonly useful +/// when the input comes from a node of the Syn syntax tree, for example the +/// body tokens of a [`Macro`] node. When in a procedural macro parsing the +/// `proc_macro::TokenStream` provided by the compiler, use [`syn::parse`] +/// instead. +/// +/// [`syn::parse`]: parse() +/// +/// This function enforces that the input is fully parsed. If there are any +/// unparsed tokens at the end of the stream, an error is returned. +#[cfg(feature = "parsing")] +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +pub fn parse2<T: parse::Parse>(tokens: proc_macro2::TokenStream) -> Result<T> { + parse::Parser::parse2(T::parse, tokens) +} + +/// Parse a string of Rust code into the chosen syntax tree node. +/// +/// This function enforces that the input is fully parsed. If there are any +/// unparsed tokens at the end of the stream, an error is returned. +/// +/// # Hygiene +/// +/// Every span in the resulting syntax tree will be set to resolve at the macro +/// call site. +/// +/// # Examples +/// +/// ``` +/// use syn::{Expr, Result}; +/// +/// fn run() -> Result<()> { +/// let code = "assert_eq!(u8::max_value(), 255)"; +/// let expr = syn::parse_str::<Expr>(code)?; +/// println!("{:#?}", expr); +/// Ok(()) +/// } +/// # +/// # run().unwrap(); +/// ``` +#[cfg(feature = "parsing")] +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +pub fn parse_str<T: parse::Parse>(s: &str) -> Result<T> { + parse::Parser::parse_str(T::parse, s) +} + +/// Parse the content of a file of Rust code. +/// +/// This is different from `syn::parse_str::<File>(content)` in two ways: +/// +/// - It discards a leading byte order mark `\u{FEFF}` if the file has one. +/// - It preserves the shebang line of the file, such as `#!/usr/bin/env rustx`. +/// +/// If present, either of these would be an error using `from_str`. +/// +/// # Examples +/// +/// ```no_run +/// use std::error::Error; +/// use std::fs; +/// use std::io::Read; +/// +/// fn run() -> Result<(), Box<dyn Error>> { +/// let content = fs::read_to_string("path/to/code.rs")?; +/// let ast = syn::parse_file(&content)?; +/// if let Some(shebang) = ast.shebang { +/// println!("{}", shebang); +/// } +/// println!("{} items", ast.items.len()); +/// +/// Ok(()) +/// } +/// # +/// # run().unwrap(); +/// ``` +#[cfg(all(feature = "parsing", feature = "full"))] +#[cfg_attr(docsrs, doc(cfg(all(feature = "parsing", feature = "full"))))] +pub fn parse_file(mut content: &str) -> Result<File> { + // Strip the BOM if it is present + const BOM: &str = "\u{feff}"; + if content.starts_with(BOM) { + content = &content[BOM.len()..]; + } + + let mut shebang = None; + if content.starts_with("#!") { + let rest = whitespace::skip(&content[2..]); + if !rest.starts_with('[') { + if let Some(idx) = content.find('\n') { + shebang = Some(content[..idx].to_string()); + content = &content[idx..]; + } else { + shebang = Some(content.to_string()); + content = ""; + } + } + } + + let mut file: File = parse_str(content)?; + file.shebang = shebang; + Ok(file) +} diff --git a/third_party/crates/syn-2.0.119/src/lifetime.rs b/third_party/crates/syn-2.0.119/src/lifetime.rs new file mode 100644 index 0000000..eeaa398 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/lifetime.rs @@ -0,0 +1,155 @@ +#[cfg(feature = "parsing")] +use crate::lookahead; +use core::cmp::Ordering; +use core::fmt::{self, Display}; +use core::hash::{Hash, Hasher}; +use proc_macro2::{Ident, Span}; + +/// A Rust lifetime: `'a`. +/// +/// Lifetime names must conform to the following rules: +/// +/// - Must start with an apostrophe. +/// - Must not consist of just an apostrophe: `'`. +/// - Character after the apostrophe must be `_` or a Unicode code point with +/// the XID_Start property. +/// - All following characters must be Unicode code points with the XID_Continue +/// property. +pub struct Lifetime { + pub apostrophe: Span, + pub ident: Ident, +} + +impl Lifetime { + /// # Panics + /// + /// Panics if the lifetime does not conform to the bulleted rules above. + /// + /// # Invocation + /// + /// ``` + /// # use proc_macro2::Span; + /// # use syn::Lifetime; + /// # + /// # fn f() -> Lifetime { + /// Lifetime::new("'a", Span::call_site()) + /// # } + /// ``` + pub fn new(symbol: &str, span: Span) -> Self { + if !symbol.starts_with('\'') { + panic!( + "lifetime name must start with apostrophe as in \"'a\", got {:?}", + symbol + ); + } + + if symbol == "'" { + panic!("lifetime name must not be empty"); + } + + if !crate::ident::xid_ok(&symbol[1..]) { + panic!("{:?} is not a valid lifetime name", symbol); + } + + Lifetime { + apostrophe: span, + ident: Ident::new(&symbol[1..], span), + } + } + + pub fn span(&self) -> Span { + self.apostrophe + .join(self.ident.span()) + .unwrap_or(self.apostrophe) + } + + pub fn set_span(&mut self, span: Span) { + self.apostrophe = span; + self.ident.set_span(span); + } +} + +impl Display for Lifetime { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + "'".fmt(formatter)?; + self.ident.fmt(formatter) + } +} + +impl Clone for Lifetime { + fn clone(&self) -> Self { + Lifetime { + apostrophe: self.apostrophe, + ident: self.ident.clone(), + } + } +} + +impl PartialEq for Lifetime { + fn eq(&self, other: &Lifetime) -> bool { + self.ident.eq(&other.ident) + } +} + +impl Eq for Lifetime {} + +impl PartialOrd for Lifetime { + fn partial_cmp(&self, other: &Lifetime) -> Option<Ordering> { + Some(self.cmp(other)) + } +} + +impl Ord for Lifetime { + fn cmp(&self, other: &Lifetime) -> Ordering { + self.ident.cmp(&other.ident) + } +} + +impl Hash for Lifetime { + fn hash<H: Hasher>(&self, h: &mut H) { + self.ident.hash(h); + } +} + +#[cfg(feature = "parsing")] +pub_if_not_doc! { + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn Lifetime(marker: lookahead::TokenMarker) -> Lifetime { + match marker {} + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::error::Result; + use crate::lifetime::Lifetime; + use crate::parse::{Parse, ParseStream}; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Lifetime { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| { + cursor + .lifetime() + .ok_or_else(|| cursor.error("expected lifetime")) + }) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::ext::PunctExt as _; + use crate::lifetime::Lifetime; + use proc_macro2::{Punct, Spacing, TokenStream}; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Lifetime { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Punct::new_spanned('\'', Spacing::Joint, self.apostrophe)); + self.ident.to_tokens(tokens); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/lit.rs b/third_party/crates/syn-2.0.119/src/lit.rs new file mode 100644 index 0000000..403d8ba --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/lit.rs @@ -0,0 +1,1928 @@ +#[cfg(feature = "parsing")] +use crate::ext::TokenStreamExt as _; +#[cfg(feature = "parsing")] +use crate::lookahead; +#[cfg(feature = "parsing")] +use crate::parse::{Parse, Parser}; +use crate::{Error, Result}; +use alloc::boxed::Box; +use alloc::ffi::CString; +#[cfg(feature = "parsing")] +use alloc::format; +use alloc::string::{String, ToString}; +use alloc::vec::Vec; +use core::ffi::CStr; +use core::fmt::{self, Display}; +#[cfg(feature = "extra-traits")] +use core::hash::{Hash, Hasher}; +use core::str::{self, FromStr}; +use proc_macro2::{Ident, Literal, Span}; +#[cfg(feature = "parsing")] +use proc_macro2::{TokenStream, TokenTree}; + +ast_enum_of_structs! { + /// A Rust literal such as a string or integer or boolean. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[non_exhaustive] + pub enum Lit { + /// A UTF-8 string literal: `"foo"`. + Str(LitStr), + + /// A byte string literal: `b"foo"`. + ByteStr(LitByteStr), + + /// A nul-terminated C-string literal: `c"foo"`. + CStr(LitCStr), + + /// A byte literal: `b'f'`. + Byte(LitByte), + + /// A character literal: `'a'`. + Char(LitChar), + + /// An integer literal: `1` or `1u16`. + Int(LitInt), + + /// A floating point literal: `1f64` or `1.0e10f64`. + /// + /// Must be finite. May not be infinite or NaN. + Float(LitFloat), + + /// A boolean literal: `true` or `false`. + Bool(LitBool), + + /// A raw token literal not interpreted by Syn. + Verbatim(Literal), + } +} + +ast_struct! { + /// A UTF-8 string literal: `"foo"`. + pub struct LitStr { + repr: Box<LitRepr>, + } +} + +ast_struct! { + /// A byte string literal: `b"foo"`. + pub struct LitByteStr { + repr: Box<LitRepr>, + } +} + +ast_struct! { + /// A nul-terminated C-string literal: `c"foo"`. + pub struct LitCStr { + repr: Box<LitRepr>, + } +} + +ast_struct! { + /// A byte literal: `b'f'`. + pub struct LitByte { + repr: Box<LitRepr>, + } +} + +ast_struct! { + /// A character literal: `'a'`. + pub struct LitChar { + repr: Box<LitRepr>, + } +} + +struct LitRepr { + token: Literal, + suffix: Box<str>, +} + +ast_struct! { + /// An integer literal: `1` or `1u16`. + pub struct LitInt { + repr: Box<LitIntRepr>, + } +} + +struct LitIntRepr { + token: Literal, + digits: Box<str>, + suffix: Box<str>, +} + +ast_struct! { + /// A floating point literal: `1f64` or `1.0e10f64`. + /// + /// Must be finite. May not be infinite or NaN. + pub struct LitFloat { + repr: Box<LitFloatRepr>, + } +} + +struct LitFloatRepr { + token: Literal, + digits: Box<str>, + suffix: Box<str>, +} + +ast_struct! { + /// A boolean literal: `true` or `false`. + pub struct LitBool { + pub value: bool, + pub span: Span, + } +} + +impl LitStr { + pub fn new(value: &str, span: Span) -> Self { + let mut token = Literal::string(value); + token.set_span(span); + LitStr { + repr: Box::new(LitRepr { + token, + suffix: Box::<str>::default(), + }), + } + } + + pub fn value(&self) -> String { + let repr = self.repr.token.to_string(); + let (value, _suffix) = value::parse_lit_str(&repr).unwrap(); + String::from(value) + } + + /// Parse a syntax tree node from the content of this string literal. + /// + /// All spans in the syntax tree will point to the span of this `LitStr`. + /// + /// # Example + /// + /// ``` + /// use syn::{Attribute, Error, Expr, Lit, Meta, Path, Result}; + /// + /// // Parses the path from an attribute that looks like: + /// // + /// // #[path = "a::b::c"] + /// // + /// // or returns `None` if the input is some other attribute. + /// fn get_path(attr: &Attribute) -> Result<Option<Path>> { + /// if !attr.path().is_ident("path") { + /// return Ok(None); + /// } + /// + /// if let Meta::NameValue(meta) = &attr.meta { + /// if let Expr::Lit(expr) = &meta.value { + /// if let Lit::Str(lit_str) = &expr.lit { + /// return lit_str.parse().map(Some); + /// } + /// } + /// } + /// + /// let message = "expected #[path = \"...\"]"; + /// Err(Error::new_spanned(attr, message)) + /// } + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse<T: Parse>(&self) -> Result<T> { + self.parse_with(T::parse) + } + + /// Invoke parser on the content of this string literal. + /// + /// All spans in the syntax tree will point to the span of this `LitStr`. + /// + /// # Example + /// + /// ``` + /// # use proc_macro2::Span; + /// # use syn::{LitStr, Result}; + /// # + /// # fn main() -> Result<()> { + /// # let lit_str = LitStr::new("a::b::c", Span::call_site()); + /// # + /// # const IGNORE: &str = stringify! { + /// let lit_str: LitStr = /* ... */; + /// # }; + /// + /// // Parse a string literal like "a::b::c" into a Path, not allowing + /// // generic arguments on any of the path segments. + /// let basic_path = lit_str.parse_with(syn::Path::parse_mod_style)?; + /// # + /// # Ok(()) + /// # } + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_with<F: Parser>(&self, parser: F) -> Result<F::Output> { + use proc_macro2::Group; + + // Token stream with every span replaced by the given one. + fn respan_token_stream(stream: TokenStream, span: Span) -> TokenStream { + let mut tokens = TokenStream::new(); + for token in stream { + tokens.append(respan_token_tree(token, span)); + } + tokens + } + + // Token tree with every span replaced by the given one. + fn respan_token_tree(mut token: TokenTree, span: Span) -> TokenTree { + match &mut token { + TokenTree::Group(g) => { + let stream = respan_token_stream(g.stream(), span); + *g = Group::new(g.delimiter(), stream); + g.set_span(span); + } + other => other.set_span(span), + } + token + } + + // Parse string literal into a token stream with every span equal to the + // original literal's span. + let span = self.span(); + let mut tokens = TokenStream::from_str(&self.value())?; + tokens = respan_token_stream(tokens, span); + + let result = crate::parse::parse_scoped(parser, span, tokens)?; + + let suffix = self.suffix(); + if !suffix.is_empty() { + return Err(Error::new( + self.span(), + format!("unexpected suffix `{}` on string literal", suffix), + )); + } + + Ok(result) + } + + pub fn span(&self) -> Span { + self.repr.token.span() + } + + pub fn set_span(&mut self, span: Span) { + self.repr.token.set_span(span); + } + + pub fn suffix(&self) -> &str { + &self.repr.suffix + } + + pub fn token(&self) -> Literal { + self.repr.token.clone() + } +} + +impl LitByteStr { + pub fn new(value: &[u8], span: Span) -> Self { + let mut token = Literal::byte_string(value); + token.set_span(span); + LitByteStr { + repr: Box::new(LitRepr { + token, + suffix: Box::<str>::default(), + }), + } + } + + pub fn value(&self) -> Vec<u8> { + let repr = self.repr.token.to_string(); + let (value, _suffix) = value::parse_lit_byte_str(&repr).unwrap(); + value + } + + pub fn span(&self) -> Span { + self.repr.token.span() + } + + pub fn set_span(&mut self, span: Span) { + self.repr.token.set_span(span); + } + + pub fn suffix(&self) -> &str { + &self.repr.suffix + } + + pub fn token(&self) -> Literal { + self.repr.token.clone() + } +} + +impl LitCStr { + pub fn new(value: &CStr, span: Span) -> Self { + let mut token = Literal::c_string(value); + token.set_span(span); + LitCStr { + repr: Box::new(LitRepr { + token, + suffix: Box::<str>::default(), + }), + } + } + + pub fn value(&self) -> CString { + let repr = self.repr.token.to_string(); + let (value, _suffix) = value::parse_lit_c_str(&repr).unwrap(); + value + } + + pub fn span(&self) -> Span { + self.repr.token.span() + } + + pub fn set_span(&mut self, span: Span) { + self.repr.token.set_span(span); + } + + pub fn suffix(&self) -> &str { + &self.repr.suffix + } + + pub fn token(&self) -> Literal { + self.repr.token.clone() + } +} + +impl LitByte { + pub fn new(value: u8, span: Span) -> Self { + let mut token = Literal::byte_character(value); + token.set_span(span); + LitByte { + repr: Box::new(LitRepr { + token, + suffix: Box::<str>::default(), + }), + } + } + + pub fn value(&self) -> u8 { + let repr = self.repr.token.to_string(); + let (value, _suffix) = value::parse_lit_byte(&repr).unwrap(); + value + } + + pub fn span(&self) -> Span { + self.repr.token.span() + } + + pub fn set_span(&mut self, span: Span) { + self.repr.token.set_span(span); + } + + pub fn suffix(&self) -> &str { + &self.repr.suffix + } + + pub fn token(&self) -> Literal { + self.repr.token.clone() + } +} + +impl LitChar { + pub fn new(value: char, span: Span) -> Self { + let mut token = Literal::character(value); + token.set_span(span); + LitChar { + repr: Box::new(LitRepr { + token, + suffix: Box::<str>::default(), + }), + } + } + + pub fn value(&self) -> char { + let repr = self.repr.token.to_string(); + let (value, _suffix) = value::parse_lit_char(&repr).unwrap(); + value + } + + pub fn span(&self) -> Span { + self.repr.token.span() + } + + pub fn set_span(&mut self, span: Span) { + self.repr.token.set_span(span); + } + + pub fn suffix(&self) -> &str { + &self.repr.suffix + } + + pub fn token(&self) -> Literal { + self.repr.token.clone() + } +} + +impl LitInt { + #[track_caller] + pub fn new(repr: &str, span: Span) -> Self { + let (digits, suffix) = match value::parse_lit_int(repr) { + Some(parse) => parse, + None => panic!("not an integer literal: `{}`", repr), + }; + + let mut token: Literal = repr.parse().unwrap(); + token.set_span(span); + LitInt { + repr: Box::new(LitIntRepr { + token, + digits, + suffix, + }), + } + } + + pub fn base10_digits(&self) -> &str { + &self.repr.digits + } + + /// Parses the literal into a selected number type. + /// + /// This is equivalent to `lit.base10_digits().parse()` except that the + /// resulting errors will be correctly spanned to point to the literal token + /// in the macro input. + /// + /// ``` + /// use syn::LitInt; + /// use syn::parse::{Parse, ParseStream, Result}; + /// + /// struct Port { + /// value: u16, + /// } + /// + /// impl Parse for Port { + /// fn parse(input: ParseStream) -> Result<Self> { + /// let lit: LitInt = input.parse()?; + /// let value = lit.base10_parse::<u16>()?; + /// Ok(Port { value }) + /// } + /// } + /// ``` + pub fn base10_parse<N>(&self) -> Result<N> + where + N: FromStr, + N::Err: Display, + { + self.base10_digits() + .parse() + .map_err(|err| Error::new(self.span(), err)) + } + + pub fn suffix(&self) -> &str { + &self.repr.suffix + } + + pub fn span(&self) -> Span { + self.repr.token.span() + } + + pub fn set_span(&mut self, span: Span) { + self.repr.token.set_span(span); + } + + pub fn token(&self) -> Literal { + self.repr.token.clone() + } +} + +impl From<Literal> for LitInt { + #[track_caller] + fn from(token: Literal) -> Self { + let repr = token.to_string(); + if let Some((digits, suffix)) = value::parse_lit_int(&repr) { + LitInt { + repr: Box::new(LitIntRepr { + token, + digits, + suffix, + }), + } + } else { + panic!("not an integer literal: `{}`", repr); + } + } +} + +impl Display for LitInt { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.repr.token.fmt(formatter) + } +} + +impl LitFloat { + #[track_caller] + pub fn new(repr: &str, span: Span) -> Self { + let (digits, suffix) = match value::parse_lit_float(repr) { + Some(parse) => parse, + None => panic!("not a float literal: `{}`", repr), + }; + + let mut token: Literal = repr.parse().unwrap(); + token.set_span(span); + LitFloat { + repr: Box::new(LitFloatRepr { + token, + digits, + suffix, + }), + } + } + + pub fn base10_digits(&self) -> &str { + &self.repr.digits + } + + pub fn base10_parse<N>(&self) -> Result<N> + where + N: FromStr, + N::Err: Display, + { + self.base10_digits() + .parse() + .map_err(|err| Error::new(self.span(), err)) + } + + pub fn suffix(&self) -> &str { + &self.repr.suffix + } + + pub fn span(&self) -> Span { + self.repr.token.span() + } + + pub fn set_span(&mut self, span: Span) { + self.repr.token.set_span(span); + } + + pub fn token(&self) -> Literal { + self.repr.token.clone() + } +} + +impl From<Literal> for LitFloat { + #[track_caller] + fn from(token: Literal) -> Self { + let repr = token.to_string(); + if let Some((digits, suffix)) = value::parse_lit_float(&repr) { + LitFloat { + repr: Box::new(LitFloatRepr { + token, + digits, + suffix, + }), + } + } else { + panic!("not a float literal: `{}`", repr); + } + } +} + +impl Display for LitFloat { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.repr.token.fmt(formatter) + } +} + +impl LitBool { + pub fn new(value: bool, span: Span) -> Self { + LitBool { value, span } + } + + pub fn value(&self) -> bool { + self.value + } + + pub fn span(&self) -> Span { + self.span + } + + pub fn set_span(&mut self, span: Span) { + self.span = span; + } + + pub fn token(&self) -> Ident { + let s = if self.value { "true" } else { "false" }; + Ident::new(s, self.span) + } +} + +#[cfg(feature = "extra-traits")] +mod debug_impls { + use crate::lit::{LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitInt, LitStr}; + use core::fmt::{self, Debug}; + + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for LitStr { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "LitStr") + } + } + + impl LitStr { + pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + formatter + .debug_struct(name) + .field("token", &format_args!("{}", self.repr.token)) + .finish() + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for LitByteStr { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "LitByteStr") + } + } + + impl LitByteStr { + pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + formatter + .debug_struct(name) + .field("token", &format_args!("{}", self.repr.token)) + .finish() + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for LitCStr { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "LitCStr") + } + } + + impl LitCStr { + pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + formatter + .debug_struct(name) + .field("token", &format_args!("{}", self.repr.token)) + .finish() + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for LitByte { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "LitByte") + } + } + + impl LitByte { + pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + formatter + .debug_struct(name) + .field("token", &format_args!("{}", self.repr.token)) + .finish() + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for LitChar { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "LitChar") + } + } + + impl LitChar { + pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + formatter + .debug_struct(name) + .field("token", &format_args!("{}", self.repr.token)) + .finish() + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for LitInt { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "LitInt") + } + } + + impl LitInt { + pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + formatter + .debug_struct(name) + .field("token", &format_args!("{}", self.repr.token)) + .finish() + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for LitFloat { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "LitFloat") + } + } + + impl LitFloat { + pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + formatter + .debug_struct(name) + .field("token", &format_args!("{}", self.repr.token)) + .finish() + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for LitBool { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.debug(formatter, "LitBool") + } + } + + impl LitBool { + pub(crate) fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + formatter + .debug_struct(name) + .field("value", &self.value) + .finish() + } + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for LitRepr { + fn clone(&self) -> Self { + LitRepr { + token: self.token.clone(), + suffix: self.suffix.clone(), + } + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for LitIntRepr { + fn clone(&self) -> Self { + LitIntRepr { + token: self.token.clone(), + digits: self.digits.clone(), + suffix: self.suffix.clone(), + } + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for LitFloatRepr { + fn clone(&self) -> Self { + LitFloatRepr { + token: self.token.clone(), + digits: self.digits.clone(), + suffix: self.suffix.clone(), + } + } +} + +macro_rules! lit_extra_traits { + ($ty:ident) => { + #[cfg(feature = "clone-impls")] + #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] + impl Clone for $ty { + fn clone(&self) -> Self { + $ty { + repr: self.repr.clone(), + } + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl PartialEq for $ty { + fn eq(&self, other: &Self) -> bool { + self.repr.token.to_string() == other.repr.token.to_string() + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Hash for $ty { + fn hash<H>(&self, state: &mut H) + where + H: Hasher, + { + self.repr.token.to_string().hash(state); + } + } + + #[cfg(feature = "parsing")] + pub_if_not_doc! { + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn $ty(marker: lookahead::TokenMarker) -> $ty { + match marker {} + } + } + }; +} + +lit_extra_traits!(LitStr); +lit_extra_traits!(LitByteStr); +lit_extra_traits!(LitCStr); +lit_extra_traits!(LitByte); +lit_extra_traits!(LitChar); +lit_extra_traits!(LitInt); +lit_extra_traits!(LitFloat); + +#[cfg(feature = "parsing")] +pub_if_not_doc! { + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn LitBool(marker: lookahead::TokenMarker) -> LitBool { + match marker {} + } +} + +/// The style of a string literal, either plain quoted or a raw string like +/// `r##"data"##`. +#[doc(hidden)] // https://github.com/dtolnay/syn/issues/1566 +pub enum StrStyle { + /// An ordinary string like `"data"`. + Cooked, + /// A raw string like `r##"data"##`. + /// + /// The unsigned integer is the number of `#` symbols used. + Raw(usize), +} + +#[cfg(feature = "parsing")] +pub_if_not_doc! { + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn Lit(marker: lookahead::TokenMarker) -> Lit { + match marker {} + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::buffer::Cursor; + use crate::error::Result; + use crate::lit::{ + value, Lit, LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitFloatRepr, LitInt, + LitIntRepr, LitStr, + }; + use crate::parse::{Parse, ParseStream, Unexpected}; + use crate::token::{self, Token}; + use alloc::boxed::Box; + use alloc::rc::Rc; + use alloc::string::ToString; + use core::cell::Cell; + use proc_macro2::{Literal, Punct, Span}; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Lit { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| { + if let Some((lit, rest)) = cursor.literal() { + return Ok((Lit::new(lit), rest)); + } + + if let Some((ident, rest)) = cursor.ident() { + let value = ident == "true"; + if value || ident == "false" { + let lit_bool = LitBool { + value, + span: ident.span(), + }; + return Ok((Lit::Bool(lit_bool), rest)); + } + } + + if let Some((punct, rest)) = cursor.punct() { + if punct.as_char() == '-' { + if let Some((lit, rest)) = parse_negative_lit(punct, rest) { + return Ok((lit, rest)); + } + } + } + + Err(cursor.error("expected literal")) + }) + } + } + + fn parse_negative_lit(neg: Punct, cursor: Cursor) -> Option<(Lit, Cursor)> { + let (lit, rest) = cursor.literal()?; + + let mut span = neg.span(); + span = span.join(lit.span()).unwrap_or(span); + + let mut repr = lit.to_string(); + repr.insert(0, '-'); + + if let Some((digits, suffix)) = value::parse_lit_int(&repr) { + let mut token: Literal = repr.parse().unwrap(); + token.set_span(span); + return Some(( + Lit::Int(LitInt { + repr: Box::new(LitIntRepr { + token, + digits, + suffix, + }), + }), + rest, + )); + } + + let (digits, suffix) = value::parse_lit_float(&repr)?; + let mut token: Literal = repr.parse().unwrap(); + token.set_span(span); + Some(( + Lit::Float(LitFloat { + repr: Box::new(LitFloatRepr { + token, + digits, + suffix, + }), + }), + rest, + )) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LitStr { + fn parse(input: ParseStream) -> Result<Self> { + let head = input.fork(); + match input.parse() { + Ok(Lit::Str(lit)) => Ok(lit), + _ => Err(head.error("expected string literal")), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LitByteStr { + fn parse(input: ParseStream) -> Result<Self> { + let head = input.fork(); + match input.parse() { + Ok(Lit::ByteStr(lit)) => Ok(lit), + _ => Err(head.error("expected byte string literal")), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LitCStr { + fn parse(input: ParseStream) -> Result<Self> { + let head = input.fork(); + match input.parse() { + Ok(Lit::CStr(lit)) => Ok(lit), + _ => Err(head.error("expected C string literal")), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LitByte { + fn parse(input: ParseStream) -> Result<Self> { + let head = input.fork(); + match input.parse() { + Ok(Lit::Byte(lit)) => Ok(lit), + _ => Err(head.error("expected byte literal")), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LitChar { + fn parse(input: ParseStream) -> Result<Self> { + let head = input.fork(); + match input.parse() { + Ok(Lit::Char(lit)) => Ok(lit), + _ => Err(head.error("expected character literal")), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LitInt { + fn parse(input: ParseStream) -> Result<Self> { + let head = input.fork(); + match input.parse() { + Ok(Lit::Int(lit)) => Ok(lit), + _ => Err(head.error("expected integer literal")), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LitFloat { + fn parse(input: ParseStream) -> Result<Self> { + let head = input.fork(); + match input.parse() { + Ok(Lit::Float(lit)) => Ok(lit), + _ => Err(head.error("expected floating point literal")), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for LitBool { + fn parse(input: ParseStream) -> Result<Self> { + let head = input.fork(); + match input.parse() { + Ok(Lit::Bool(lit)) => Ok(lit), + _ => Err(head.error("expected boolean literal")), + } + } + } + + fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool { + let scope = Span::call_site(); + let unexpected = Rc::new(Cell::new(Unexpected::None)); + let buffer = crate::parse::new_parse_buffer(scope, cursor, unexpected); + peek(&buffer) + } + + macro_rules! impl_token { + ($display:literal $name:ty) => { + impl Token for $name { + fn peek(cursor: Cursor) -> bool { + fn peek(input: ParseStream) -> bool { + <$name as Parse>::parse(input).is_ok() + } + peek_impl(cursor, peek) + } + + fn display() -> &'static str { + $display + } + } + + impl token::private::Sealed for $name {} + }; + } + + impl_token!("literal" Lit); + impl_token!("string literal" LitStr); + impl_token!("byte string literal" LitByteStr); + impl_token!("C-string literal" LitCStr); + impl_token!("byte literal" LitByte); + impl_token!("character literal" LitChar); + impl_token!("integer literal" LitInt); + impl_token!("floating point literal" LitFloat); + impl_token!("boolean literal" LitBool); +} + +#[cfg(feature = "printing")] +mod printing { + use crate::lit::{LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitInt, LitStr}; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LitStr { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.repr.token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LitByteStr { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.repr.token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LitCStr { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.repr.token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LitByte { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.repr.token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LitChar { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.repr.token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LitInt { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.repr.token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LitFloat { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.repr.token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for LitBool { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(self.token()); + } + } +} + +mod value { + use crate::bigint::BigInt; + use crate::lit::{ + Lit, LitBool, LitByte, LitByteStr, LitCStr, LitChar, LitFloat, LitFloatRepr, LitInt, + LitIntRepr, LitRepr, LitStr, + }; + use alloc::borrow::ToOwned; + use alloc::boxed::Box; + use alloc::ffi::CString; + use alloc::string::{String, ToString}; + use alloc::vec::Vec; + use core::char; + use core::ops::{Index, RangeFrom}; + use proc_macro2::{Literal, Span}; + + impl Lit { + /// Interpret a Syn literal from a proc-macro2 literal. + pub fn new(token: Literal) -> Self { + let repr = token.to_string(); + Lit::from_str(token, &repr) + } + + #[cfg(fuzzing)] + #[doc(hidden)] + pub fn from_str_for_fuzzing(repr: &str) -> Self { + let token = Literal::u8_unsuffixed(0); + Lit::from_str(token, repr) + } + + fn from_str(token: Literal, repr: &str) -> Self { + match byte(repr, 0) { + // "...", r"...", r#"..."# + b'"' | b'r' => { + if let Some((_, suffix)) = parse_lit_str(repr) { + return Lit::Str(LitStr { + repr: Box::new(LitRepr { token, suffix }), + }); + } + } + b'b' => match byte(repr, 1) { + // b"...", br"...", br#"...#" + b'"' | b'r' => { + if let Some((_, suffix)) = parse_lit_byte_str(repr) { + return Lit::ByteStr(LitByteStr { + repr: Box::new(LitRepr { token, suffix }), + }); + } + } + // b'...' + b'\'' => { + if let Some((_, suffix)) = parse_lit_byte(repr) { + return Lit::Byte(LitByte { + repr: Box::new(LitRepr { token, suffix }), + }); + } + } + _ => {} + }, + b'c' => match byte(repr, 1) { + // c"...", cr"...", cr#"..."# + b'"' | b'r' => { + if let Some((_, suffix)) = parse_lit_c_str(repr) { + return Lit::CStr(LitCStr { + repr: Box::new(LitRepr { token, suffix }), + }); + } + } + _ => {} + }, + // '...' + b'\'' => { + if let Some((_, suffix)) = parse_lit_char(repr) { + return Lit::Char(LitChar { + repr: Box::new(LitRepr { token, suffix }), + }); + } + } + b'0'..=b'9' | b'-' => { + // 0, 123, 0xFF, 0o77, 0b11 + if let Some((digits, suffix)) = parse_lit_int(repr) { + return Lit::Int(LitInt { + repr: Box::new(LitIntRepr { + token, + digits, + suffix, + }), + }); + } + // 1.0, 1e-1, 1e+1 + if let Some((digits, suffix)) = parse_lit_float(repr) { + return Lit::Float(LitFloat { + repr: Box::new(LitFloatRepr { + token, + digits, + suffix, + }), + }); + } + } + // true, false + b't' | b'f' if repr == "true" || repr == "false" => { + return Lit::Bool(LitBool { + value: repr == "true", + span: token.span(), + }); + } + b'(' if repr == "(/*ERROR*/)" => return Lit::Verbatim(token), + _ => {} + } + + Lit::Verbatim(token) + } + + pub fn suffix(&self) -> &str { + match self { + Lit::Str(lit) => lit.suffix(), + Lit::ByteStr(lit) => lit.suffix(), + Lit::CStr(lit) => lit.suffix(), + Lit::Byte(lit) => lit.suffix(), + Lit::Char(lit) => lit.suffix(), + Lit::Int(lit) => lit.suffix(), + Lit::Float(lit) => lit.suffix(), + Lit::Bool(_) | Lit::Verbatim(_) => "", + } + } + + pub fn span(&self) -> Span { + match self { + Lit::Str(lit) => lit.span(), + Lit::ByteStr(lit) => lit.span(), + Lit::CStr(lit) => lit.span(), + Lit::Byte(lit) => lit.span(), + Lit::Char(lit) => lit.span(), + Lit::Int(lit) => lit.span(), + Lit::Float(lit) => lit.span(), + Lit::Bool(lit) => lit.span, + Lit::Verbatim(lit) => lit.span(), + } + } + + pub fn set_span(&mut self, span: Span) { + match self { + Lit::Str(lit) => lit.set_span(span), + Lit::ByteStr(lit) => lit.set_span(span), + Lit::CStr(lit) => lit.set_span(span), + Lit::Byte(lit) => lit.set_span(span), + Lit::Char(lit) => lit.set_span(span), + Lit::Int(lit) => lit.set_span(span), + Lit::Float(lit) => lit.set_span(span), + Lit::Bool(lit) => lit.span = span, + Lit::Verbatim(lit) => lit.set_span(span), + } + } + } + + /// Get the byte at offset idx, or a default of `b'\0'` if we're looking + /// past the end of the input buffer. + pub(crate) fn byte<S: AsRef<[u8]> + ?Sized>(s: &S, idx: usize) -> u8 { + let s = s.as_ref(); + if idx < s.len() { + s[idx] + } else { + 0 + } + } + + fn next_chr(s: &str) -> char { + s.chars().next().unwrap_or('\0') + } + + // Returns (content, suffix). + pub(crate) fn parse_lit_str(s: &str) -> Option<(Box<str>, Box<str>)> { + match byte(s, 0) { + b'"' => parse_lit_str_cooked(s), + b'r' => parse_lit_str_raw(s), + _ => unreachable!(), + } + } + + fn parse_lit_str_cooked(mut s: &str) -> Option<(Box<str>, Box<str>)> { + assert_eq!(byte(s, 0), b'"'); + s = &s[1..]; + + let mut content = String::new(); + 'outer: loop { + let ch = match byte(s, 0) { + b'"' => break, + b'\\' => { + let b = byte(s, 1); + s = s.get(2..)?; + match b { + b'x' => { + let (byte, rest) = backslash_x(s)?; + s = rest; + if byte > 0x7F { + // invalid \x byte in string literal + return None; + } + char::from(byte) + } + b'u' => { + let (ch, rest) = backslash_u(s)?; + s = rest; + ch + } + b'n' => '\n', + b'r' => '\r', + b't' => '\t', + b'\\' => '\\', + b'0' => '\0', + b'\'' => '\'', + b'"' => '"', + b'\r' | b'\n' => loop { + let b = byte(s, 0); + match b { + b' ' | b'\t' | b'\n' | b'\r' => s = &s[1..], + _ => continue 'outer, + } + }, + _ => { + // unexpected byte after backslash + return None; + } + } + } + b'\r' => { + if byte(s, 1) != b'\n' { + // bare carriage return not allowed in string + return None; + } + s = &s[2..]; + '\n' + } + _ => { + let ch = next_chr(s); + s = s.get(ch.len_utf8()..)?; + ch + } + }; + content.push(ch); + } + + assert!(s.starts_with('"')); + let content = content.into_boxed_str(); + let suffix = s[1..].to_owned().into_boxed_str(); + Some((content, suffix)) + } + + fn parse_lit_str_raw(mut s: &str) -> Option<(Box<str>, Box<str>)> { + assert_eq!(byte(s, 0), b'r'); + s = &s[1..]; + + let mut pounds = 0; + loop { + match byte(s, pounds) { + b'#' => pounds += 1, + b'"' => break, + _ => return None, + } + } + let close = s.rfind('"').unwrap(); + for end in s.get(close + 1..close + 1 + pounds)?.bytes() { + if end != b'#' { + return None; + } + } + + let content = s.get(pounds + 1..close)?.to_owned().into_boxed_str(); + let suffix = s[close + 1 + pounds..].to_owned().into_boxed_str(); + Some((content, suffix)) + } + + // Returns (content, suffix). + pub(crate) fn parse_lit_byte_str(s: &str) -> Option<(Vec<u8>, Box<str>)> { + assert_eq!(byte(s, 0), b'b'); + match byte(s, 1) { + b'"' => parse_lit_byte_str_cooked(s), + b'r' => parse_lit_byte_str_raw(s), + _ => unreachable!(), + } + } + + fn parse_lit_byte_str_cooked(mut s: &str) -> Option<(Vec<u8>, Box<str>)> { + assert_eq!(byte(s, 0), b'b'); + assert_eq!(byte(s, 1), b'"'); + s = &s[2..]; + + // We're going to want to have slices which don't respect codepoint boundaries. + let mut v = s.as_bytes(); + + let mut out = Vec::new(); + 'outer: loop { + let byte = match byte(v, 0) { + b'"' => break, + b'\\' => { + let b = byte(v, 1); + v = v.get(2..)?; + match b { + b'x' => { + let (b, rest) = backslash_x(v)?; + v = rest; + b + } + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + b'\\' => b'\\', + b'0' => b'\0', + b'\'' => b'\'', + b'"' => b'"', + b'\r' | b'\n' => loop { + let byte = byte(v, 0); + if matches!(byte, b' ' | b'\t' | b'\n' | b'\r') { + v = &v[1..]; + } else { + continue 'outer; + } + }, + _ => { + // unexpected byte after backslash + return None; + } + } + } + b'\r' => { + if byte(v, 1) != b'\n' { + // bare carriage return not allowed in string + return None; + } + v = &v[2..]; + b'\n' + } + b => { + v = v.get(1..)?; + b + } + }; + out.push(byte); + } + + assert_eq!(byte(v, 0), b'"'); + let suffix = s[s.len() - v.len() + 1..].to_owned().into_boxed_str(); + Some((out, suffix)) + } + + fn parse_lit_byte_str_raw(s: &str) -> Option<(Vec<u8>, Box<str>)> { + assert_eq!(byte(s, 0), b'b'); + let (value, suffix) = parse_lit_str_raw(&s[1..])?; + Some((String::from(value).into_bytes(), suffix)) + } + + // Returns (content, suffix). + pub(crate) fn parse_lit_c_str(s: &str) -> Option<(CString, Box<str>)> { + assert_eq!(byte(s, 0), b'c'); + match byte(s, 1) { + b'"' => parse_lit_c_str_cooked(s), + b'r' => parse_lit_c_str_raw(s), + _ => unreachable!(), + } + } + + fn parse_lit_c_str_cooked(mut s: &str) -> Option<(CString, Box<str>)> { + assert_eq!(byte(s, 0), b'c'); + assert_eq!(byte(s, 1), b'"'); + s = &s[2..]; + + // We're going to want to have slices which don't respect codepoint boundaries. + let mut v = s.as_bytes(); + + let mut out = Vec::new(); + 'outer: loop { + let byte = match byte(v, 0) { + b'"' => break, + b'\\' => { + let b = byte(v, 1); + v = v.get(2..)?; + match b { + b'x' => { + let (b, rest) = backslash_x(v)?; + if b == 0 { + // \x00 is not allowed in C-string literal + return None; + } + v = rest; + b + } + b'u' => { + let (ch, rest) = backslash_u(v)?; + if ch == '\0' { + // \u{0} is not allowed in C-string literal + return None; + } + v = rest; + out.extend_from_slice(ch.encode_utf8(&mut [0u8; 4]).as_bytes()); + continue 'outer; + } + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + b'\\' => b'\\', + b'\'' => b'\'', + b'"' => b'"', + b'\r' | b'\n' => loop { + let byte = byte(v, 0); + if matches!(byte, b' ' | b'\t' | b'\n' | b'\r') { + v = &v[1..]; + } else { + continue 'outer; + } + }, + _ => { + // unexpected byte after backslash + return None; + } + } + } + b'\r' => { + if byte(v, 1) != b'\n' { + // bare carriage return not allowed in string + return None; + } + v = &v[2..]; + b'\n' + } + b => { + v = v.get(1..)?; + b + } + }; + out.push(byte); + } + + assert_eq!(byte(v, 0), b'"'); + let suffix = s[s.len() - v.len() + 1..].to_owned().into_boxed_str(); + let cstring = CString::new(out).ok()?; + Some((cstring, suffix)) + } + + fn parse_lit_c_str_raw(s: &str) -> Option<(CString, Box<str>)> { + assert_eq!(byte(s, 0), b'c'); + let (value, suffix) = parse_lit_str_raw(&s[1..])?; + let cstring = CString::new(String::from(value)).ok()?; + Some((cstring, suffix)) + } + + // Returns (value, suffix). + pub(crate) fn parse_lit_byte(s: &str) -> Option<(u8, Box<str>)> { + assert_eq!(byte(s, 0), b'b'); + assert_eq!(byte(s, 1), b'\''); + + // We're going to want to have slices which don't respect codepoint boundaries. + let mut v = &s.as_bytes()[2..]; + + let b = match byte(v, 0) { + b'\\' => { + let b = byte(v, 1); + v = v.get(2..)?; + match b { + b'x' => { + let (b, rest) = backslash_x(v)?; + v = rest; + b + } + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + b'\\' => b'\\', + b'0' => b'\0', + b'\'' => b'\'', + b'"' => b'"', + _ => { + // unexpected byte after backslash + return None; + } + } + } + b => { + v = v.get(1..)?; + b + } + }; + + if byte(v, 0) != b'\'' { + return None; + } + + let suffix = s[s.len() - v.len() + 1..].to_owned().into_boxed_str(); + Some((b, suffix)) + } + + // Returns (value, suffix). + pub(crate) fn parse_lit_char(mut s: &str) -> Option<(char, Box<str>)> { + assert_eq!(byte(s, 0), b'\''); + s = &s[1..]; + + let ch = match byte(s, 0) { + b'\\' => { + let b = byte(s, 1); + s = s.get(2..)?; + match b { + b'x' => { + let (byte, rest) = backslash_x(s)?; + s = rest; + if byte > 0x7F { + // invalid \x byte in character literal + return None; + } + char::from(byte) + } + b'u' => { + let (ch, rest) = backslash_u(s)?; + s = rest; + ch + } + b'n' => '\n', + b'r' => '\r', + b't' => '\t', + b'\\' => '\\', + b'0' => '\0', + b'\'' => '\'', + b'"' => '"', + _ => { + // unexpected byte after backslash + return None; + } + } + } + _ => { + let ch = next_chr(s); + s = s.get(ch.len_utf8()..)?; + ch + } + }; + + if byte(s, 0) != b'\'' { + return None; + } + + let suffix = s[1..].to_owned().into_boxed_str(); + Some((ch, suffix)) + } + + fn backslash_x<S>(s: &S) -> Option<(u8, &S)> + where + S: Index<RangeFrom<usize>, Output = S> + AsRef<[u8]> + ?Sized, + { + let mut ch = 0; + let b0 = byte(s, 0); + let b1 = byte(s, 1); + ch += 0x10 + * match b0 { + b'0'..=b'9' => b0 - b'0', + b'a'..=b'f' => 10 + (b0 - b'a'), + b'A'..=b'F' => 10 + (b0 - b'A'), + _ => return None, + }; + ch += match b1 { + b'0'..=b'9' => b1 - b'0', + b'a'..=b'f' => 10 + (b1 - b'a'), + b'A'..=b'F' => 10 + (b1 - b'A'), + _ => return None, + }; + Some((ch, &s[2..])) + } + + fn backslash_u<S>(mut s: &S) -> Option<(char, &S)> + where + S: Index<RangeFrom<usize>, Output = S> + AsRef<[u8]> + ?Sized, + { + if byte(s, 0) != b'{' { + return None; + } + s = &s[1..]; + + let mut ch = 0; + let mut digits = 0; + loop { + let b = byte(s, 0); + let digit = match b { + b'0'..=b'9' => b - b'0', + b'a'..=b'f' => 10 + b - b'a', + b'A'..=b'F' => 10 + b - b'A', + b'_' if digits > 0 => { + s = &s[1..]; + continue; + } + b'}' if digits == 0 => return None, + b'}' => break, + _ => return None, + }; + if digits == 6 { + return None; + } + ch *= 0x10; + ch += u32::from(digit); + digits += 1; + s = &s[1..]; + } + if byte(s, 0) != b'}' { + return None; + } + s = &s[1..]; + + let ch = char::from_u32(ch)?; + Some((ch, s)) + } + + // Returns base 10 digits and suffix. + pub(crate) fn parse_lit_int(mut s: &str) -> Option<(Box<str>, Box<str>)> { + let negative = byte(s, 0) == b'-'; + if negative { + s = &s[1..]; + } + + let base = match (byte(s, 0), byte(s, 1)) { + (b'0', b'x') => { + s = &s[2..]; + 16 + } + (b'0', b'o') => { + s = &s[2..]; + 8 + } + (b'0', b'b') => { + s = &s[2..]; + 2 + } + (b'0'..=b'9', _) => 10, + _ => return None, + }; + + let mut value = BigInt::new(); + let mut has_digit = false; + 'outer: loop { + let b = byte(s, 0); + let digit = match b { + b'0'..=b'9' => b - b'0', + b'a'..=b'f' if base > 10 => b - b'a' + 10, + b'A'..=b'F' if base > 10 => b - b'A' + 10, + b'_' => { + s = &s[1..]; + continue; + } + // If looking at a floating point literal, we don't want to + // consider it an integer. + b'.' if base == 10 => return None, + b'e' | b'E' if base == 10 => { + let mut has_exp = false; + for (i, b) in s[1..].bytes().enumerate() { + match b { + b'_' => {} + b'-' | b'+' => return None, + b'0'..=b'9' => has_exp = true, + _ => { + let suffix = &s[1 + i..]; + if has_exp && crate::ident::xid_ok(suffix) { + return None; + } else { + break 'outer; + } + } + } + } + if has_exp { + return None; + } else { + break; + } + } + _ => break, + }; + + if digit >= base { + return None; + } + + has_digit = true; + value *= base; + value += digit; + s = &s[1..]; + } + + if !has_digit { + return None; + } + + let suffix = s; + if suffix.is_empty() || crate::ident::xid_ok(suffix) { + let mut repr = value.to_string(); + if negative { + repr.insert(0, '-'); + } + Some((repr.into_boxed_str(), suffix.to_owned().into_boxed_str())) + } else { + None + } + } + + // Returns base 10 digits and suffix. + pub(crate) fn parse_lit_float(input: &str) -> Option<(Box<str>, Box<str>)> { + // Rust's floating point literals are very similar to the ones parsed by + // the standard library, except that rust's literals can contain + // ignorable underscores. Let's remove those underscores. + + let mut bytes = input.to_owned().into_bytes(); + + let start = (*bytes.first()? == b'-') as usize; + match bytes.get(start)? { + b'0'..=b'9' => {} + _ => return None, + } + + let mut read = start; + let mut write = start; + let mut has_dot = false; + let mut has_e = false; + let mut has_sign = false; + let mut has_exponent = false; + while read < bytes.len() { + match bytes[read] { + b'_' => { + // Don't increase write + read += 1; + continue; + } + b'0'..=b'9' => { + if has_e { + has_exponent = true; + } + bytes[write] = bytes[read]; + } + b'.' => { + if has_e || has_dot { + return None; + } + has_dot = true; + bytes[write] = b'.'; + } + b'e' | b'E' => { + match bytes[read + 1..] + .iter() + .find(|b| **b != b'_') + .unwrap_or(&b'\0') + { + b'-' | b'+' | b'0'..=b'9' => {} + _ => break, + } + if has_e { + if has_exponent { + break; + } else { + return None; + } + } + has_e = true; + bytes[write] = b'e'; + } + b'-' | b'+' => { + if has_sign || has_exponent || !has_e { + return None; + } + has_sign = true; + if bytes[read] == b'-' { + bytes[write] = bytes[read]; + } else { + // Omit '+' + read += 1; + continue; + } + } + _ => break, + } + read += 1; + write += 1; + } + + if has_e && !has_exponent { + return None; + } + + let mut digits = String::from_utf8(bytes).unwrap(); + let suffix = digits.split_off(read); + digits.truncate(write); + if suffix.is_empty() || crate::ident::xid_ok(&suffix) { + Some((digits.into_boxed_str(), suffix.into_boxed_str())) + } else { + None + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/lookahead.rs b/third_party/crates/syn-2.0.119/src/lookahead.rs new file mode 100644 index 0000000..7e7e031 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/lookahead.rs @@ -0,0 +1,350 @@ +use crate::buffer::Cursor; +use crate::error::{self, Error}; +use crate::sealed::lookahead::Sealed; +use crate::span::IntoSpans; +use crate::token::{CustomToken, Token}; +use alloc::format; +use alloc::vec::Vec; +use core::cell::RefCell; +use core::fmt::{self, Display}; +use proc_macro2::{Delimiter, Span}; + +/// Support for checking the next token in a stream to decide how to parse. +/// +/// An important advantage over [`ParseStream::peek`] is that here we +/// automatically construct an appropriate error message based on the token +/// alternatives that get peeked. If you are producing your own error message, +/// go ahead and use `ParseStream::peek` instead. +/// +/// Use [`ParseStream::lookahead1`] to construct this object. +/// +/// [`ParseStream::peek`]: crate::parse::ParseBuffer::peek +/// [`ParseStream::lookahead1`]: crate::parse::ParseBuffer::lookahead1 +/// +/// Consuming tokens from the source stream after constructing a lookahead +/// object does not also advance the lookahead object. +/// +/// # Example +/// +/// ``` +/// use syn::{ConstParam, Ident, Lifetime, LifetimeParam, Result, Token, TypeParam}; +/// use syn::parse::{Parse, ParseStream}; +/// +/// // A generic parameter, a single one of the comma-separated elements inside +/// // angle brackets in: +/// // +/// // fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... } +/// // +/// // On invalid input, lookahead gives us a reasonable error message. +/// // +/// // error: expected one of: identifier, lifetime, `const` +/// // | +/// // 5 | fn f<!Sized>() {} +/// // | ^ +/// enum GenericParam { +/// Type(TypeParam), +/// Lifetime(LifetimeParam), +/// Const(ConstParam), +/// } +/// +/// impl Parse for GenericParam { +/// fn parse(input: ParseStream) -> Result<Self> { +/// let lookahead = input.lookahead1(); +/// if lookahead.peek(Ident) { +/// input.parse().map(GenericParam::Type) +/// } else if lookahead.peek(Lifetime) { +/// input.parse().map(GenericParam::Lifetime) +/// } else if lookahead.peek(Token![const]) { +/// input.parse().map(GenericParam::Const) +/// } else { +/// Err(lookahead.error()) +/// } +/// } +/// } +/// ``` +pub struct Lookahead1<'a> { + scope: Span, + cursor: Cursor<'a>, + comparisons: RefCell<Vec<&'static str>>, +} + +pub(crate) fn new(scope: Span, cursor: Cursor) -> Lookahead1 { + Lookahead1 { + scope, + cursor, + comparisons: RefCell::new(Vec::new()), + } +} + +fn peek_impl( + lookahead: &Lookahead1, + peek: fn(Cursor) -> bool, + display: fn() -> &'static str, +) -> bool { + if peek(lookahead.cursor) { + return true; + } + lookahead.comparisons.borrow_mut().push(display()); + false +} + +impl<'a> Lookahead1<'a> { + /// Looks at the next token in the parse stream to determine whether it + /// matches the requested type of token. + /// + /// # Syntax + /// + /// Note that this method does not use turbofish syntax. Pass the peek type + /// inside of parentheses. + /// + /// - `input.peek(Token![struct])` + /// - `input.peek(Token![==])` + /// - `input.peek(Ident)` *(does not accept keywords)* + /// - `input.peek(Ident::peek_any)` + /// - `input.peek(Lifetime)` + /// - `input.peek(token::Brace)` + pub fn peek<T: Peek>(&self, token: T) -> bool { + let _ = token; + peek_impl(self, T::Token::peek, T::Token::display) + } + + /// Triggers an error at the current position of the parse stream. + /// + /// The error message will identify all of the expected token types that + /// have been peeked against this lookahead instance. + pub fn error(self) -> Error { + let mut comparisons = self.comparisons.into_inner(); + comparisons.retain_mut(|display| { + if *display == "`)`" { + *display = match self.cursor.scope_delimiter() { + Delimiter::Parenthesis => "`)`", + Delimiter::Brace => "`}`", + Delimiter::Bracket => "`]`", + Delimiter::None => return false, + } + } + true + }); + match comparisons.len() { + 0 => { + if self.cursor.eof() { + Error::new(self.scope, "unexpected end of input") + } else { + Error::new(self.cursor.span(), "unexpected token") + } + } + 1 => { + let message = format!("expected {}", comparisons[0]); + error::new_at(self.scope, self.cursor, message) + } + 2 => { + let message = format!("expected {} or {}", comparisons[0], comparisons[1]); + error::new_at(self.scope, self.cursor, message) + } + _ => { + let message = format!("expected one of: {}", CommaSeparated(&comparisons)); + error::new_at(self.scope, self.cursor, message) + } + } + } +} + +struct CommaSeparated<'a>(&'a [&'a str]); + +impl<'a> Display for CommaSeparated<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut first = true; + for &s in self.0 { + if !first { + f.write_str(", ")?; + } + f.write_str(s)?; + first = false; + } + Ok(()) + } +} + +/// Types that can be parsed by looking at just one token. +/// +/// Use [`ParseStream::peek`] to peek one of these types in a parse stream +/// without consuming it from the stream. +/// +/// This trait is sealed and cannot be implemented for types outside of Syn. +/// +/// [`ParseStream::peek`]: crate::parse::ParseBuffer::peek +pub trait Peek: Sealed { + // Not public API. + #[doc(hidden)] + type Token: Token; +} + +/// Pseudo-token used for peeking the end of a parse stream. +/// +/// This type is only useful as an argument to one of the following functions: +/// +/// - [`ParseStream::peek`][crate::parse::ParseBuffer::peek] +/// - [`ParseStream::peek2`][crate::parse::ParseBuffer::peek2] +/// - [`ParseStream::peek3`][crate::parse::ParseBuffer::peek3] +/// - [`Lookahead1::peek`] +/// +/// The peek will return `true` if there are no remaining tokens after that +/// point in the parse stream. +/// +/// # Example +/// +/// Suppose we are parsing attributes containing core::fmt inspired formatting +/// arguments: +/// +/// - `#[fmt("simple example")]` +/// - `#[fmt("interpolation e{}ample", self.x)]` +/// - `#[fmt("interpolation e{x}ample")]` +/// +/// and we want to recognize the cases where no interpolation occurs so that +/// more efficient code can be generated. +/// +/// The following implementation uses `input.peek(Token![,]) && +/// input.peek2(End)` to recognize the case of a trailing comma without +/// consuming the comma from the parse stream, because if it isn't a trailing +/// comma, that same comma needs to be parsed as part of `args`. +/// +/// ``` +/// use proc_macro2::TokenStream; +/// use quote::quote; +/// use syn::parse::{End, Parse, ParseStream, Result}; +/// use syn::{parse_quote, Attribute, LitStr, Token}; +/// +/// struct FormatArgs { +/// template: LitStr, // "...{}..." +/// args: TokenStream, // , self.x +/// } +/// +/// impl Parse for FormatArgs { +/// fn parse(input: ParseStream) -> Result<Self> { +/// let template: LitStr = input.parse()?; +/// +/// let args = if input.is_empty() +/// || input.peek(Token![,]) && input.peek2(End) +/// { +/// input.parse::<Option<Token![,]>>()?; +/// TokenStream::new() +/// } else { +/// input.parse()? +/// }; +/// +/// Ok(FormatArgs { +/// template, +/// args, +/// }) +/// } +/// } +/// +/// fn main() -> Result<()> { +/// let attrs: Vec<Attribute> = parse_quote! { +/// #[fmt("simple example")] +/// #[fmt("interpolation e{}ample", self.x)] +/// #[fmt("interpolation e{x}ample")] +/// }; +/// +/// for attr in &attrs { +/// let FormatArgs { template, args } = attr.parse_args()?; +/// let requires_fmt_machinery = +/// !args.is_empty() || template.value().contains(['{', '}']); +/// let out = if requires_fmt_machinery { +/// quote! { +/// ::core::write!(__formatter, #template #args) +/// } +/// } else { +/// quote! { +/// __formatter.write_str(#template) +/// } +/// }; +/// println!("{}", out); +/// } +/// Ok(()) +/// } +/// ``` +/// +/// Implementing this parsing logic without `peek2(End)` is more clumsy because +/// we'd need a parse stream actually advanced past the comma before being able +/// to find out whether there is anything after it. It would look something +/// like: +/// +/// ``` +/// # use proc_macro2::TokenStream; +/// # use syn::parse::{ParseStream, Result}; +/// # use syn::Token; +/// # +/// # fn parse(input: ParseStream) -> Result<()> { +/// use syn::parse::discouraged::Speculative as _; +/// +/// let ahead = input.fork(); +/// ahead.parse::<Option<Token![,]>>()?; +/// let args = if ahead.is_empty() { +/// input.advance_to(&ahead); +/// TokenStream::new() +/// } else { +/// input.parse()? +/// }; +/// # Ok(()) +/// # } +/// ``` +/// +/// or: +/// +/// ``` +/// # use proc_macro2::TokenStream; +/// # use syn::parse::{ParseStream, Result}; +/// # use syn::Token; +/// # +/// # fn parse(input: ParseStream) -> Result<()> { +/// use quote::ToTokens as _; +/// +/// let comma: Option<Token![,]> = input.parse()?; +/// let mut args = TokenStream::new(); +/// if !input.is_empty() { +/// comma.to_tokens(&mut args); +/// input.parse::<TokenStream>()?.to_tokens(&mut args); +/// } +/// # Ok(()) +/// # } +/// ``` +pub struct End; + +impl Copy for End {} + +impl Clone for End { + fn clone(&self) -> Self { + *self + } +} + +impl Peek for End { + type Token = Self; +} + +impl CustomToken for End { + fn peek(cursor: Cursor) -> bool { + cursor.eof() + } + + fn display() -> &'static str { + "`)`" // Lookahead1 error message will fill in the expected close delimiter + } +} + +impl<F: Copy + FnOnce(TokenMarker) -> T, T: Token> Peek for F { + type Token = T; +} + +pub enum TokenMarker {} + +impl<S> IntoSpans<S> for TokenMarker { + fn into_spans(self) -> S { + match self {} + } +} + +impl<F: Copy + FnOnce(TokenMarker) -> T, T: Token> Sealed for F {} + +impl Sealed for End {} diff --git a/third_party/crates/syn-2.0.119/src/mac.rs b/third_party/crates/syn-2.0.119/src/mac.rs new file mode 100644 index 0000000..1510780 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/mac.rs @@ -0,0 +1,225 @@ +#[cfg(feature = "parsing")] +use crate::error::Result; +#[cfg(feature = "parsing")] +use crate::parse::{Parse, ParseStream, Parser}; +use crate::path::Path; +use crate::token::{Brace, Bracket, Paren}; +use proc_macro2::extra::DelimSpan; +#[cfg(feature = "parsing")] +use proc_macro2::Delimiter; +use proc_macro2::TokenStream; +#[cfg(feature = "parsing")] +use proc_macro2::TokenTree; + +ast_struct! { + /// A macro invocation: `println!("{}", mac)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Macro { + pub path: Path, + pub bang_token: Token![!], + pub delimiter: MacroDelimiter, + pub tokens: TokenStream, + } +} + +ast_enum! { + /// A grouping token that surrounds a macro body: `m!(...)` or `m!{...}` or `m![...]`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum MacroDelimiter { + Paren(Paren), + Brace(Brace), + Bracket(Bracket), + } +} + +impl MacroDelimiter { + pub fn span(&self) -> &DelimSpan { + match self { + MacroDelimiter::Paren(token) => &token.span, + MacroDelimiter::Brace(token) => &token.span, + MacroDelimiter::Bracket(token) => &token.span, + } + } + + #[cfg(all(feature = "full", any(feature = "parsing", feature = "printing")))] + pub(crate) fn is_brace(&self) -> bool { + match self { + MacroDelimiter::Brace(_) => true, + MacroDelimiter::Paren(_) | MacroDelimiter::Bracket(_) => false, + } + } +} + +impl Macro { + /// Parse the tokens within the macro invocation's delimiters into a syntax + /// tree. + /// + /// This is equivalent to `syn::parse2::<T>(mac.tokens)` except that it + /// produces a more useful span when `tokens` is empty. + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Expr, ExprLit, Ident, Lit, LitStr, Macro, Token}; + /// use syn::ext::IdentExt; + /// use syn::parse::{Error, Parse, ParseStream, Result}; + /// use syn::punctuated::Punctuated; + /// + /// // The arguments expected by libcore's format_args macro, and as a + /// // result most other formatting and printing macros like println. + /// // + /// // println!("{} is {number:.prec$}", "x", prec=5, number=0.01) + /// struct FormatArgs { + /// format_string: Expr, + /// positional_args: Vec<Expr>, + /// named_args: Vec<(Ident, Expr)>, + /// } + /// + /// impl Parse for FormatArgs { + /// fn parse(input: ParseStream) -> Result<Self> { + /// let format_string: Expr; + /// let mut positional_args = Vec::new(); + /// let mut named_args = Vec::new(); + /// + /// format_string = input.parse()?; + /// while !input.is_empty() { + /// input.parse::<Token![,]>()?; + /// if input.is_empty() { + /// break; + /// } + /// if input.peek(Ident::peek_any) && input.peek2(Token![=]) { + /// while !input.is_empty() { + /// let name: Ident = input.call(Ident::parse_any)?; + /// input.parse::<Token![=]>()?; + /// let value: Expr = input.parse()?; + /// named_args.push((name, value)); + /// if input.is_empty() { + /// break; + /// } + /// input.parse::<Token![,]>()?; + /// } + /// break; + /// } + /// positional_args.push(input.parse()?); + /// } + /// + /// Ok(FormatArgs { + /// format_string, + /// positional_args, + /// named_args, + /// }) + /// } + /// } + /// + /// // Extract the first argument, the format string literal, from an + /// // invocation of a formatting or printing macro. + /// fn get_format_string(m: &Macro) -> Result<LitStr> { + /// let args: FormatArgs = m.parse_body()?; + /// match args.format_string { + /// Expr::Lit(ExprLit { lit: Lit::Str(lit), .. }) => Ok(lit), + /// other => { + /// // First argument was not a string literal expression. + /// // Maybe something like: println!(concat!(...), ...) + /// Err(Error::new_spanned(other, "format string must be a string literal")) + /// } + /// } + /// } + /// + /// fn main() { + /// let invocation = parse_quote! { + /// println!("{:?}", Instant::now()) + /// }; + /// let lit = get_format_string(&invocation).unwrap(); + /// assert_eq!(lit.value(), "{:?}"); + /// } + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_body<T: Parse>(&self) -> Result<T> { + self.parse_body_with(T::parse) + } + + /// Parse the tokens within the macro invocation's delimiters using the + /// given parser. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_body_with<F: Parser>(&self, parser: F) -> Result<F::Output> { + let scope = self.delimiter.span().close(); + crate::parse::parse_scoped(parser, scope, self.tokens.clone()) + } +} + +#[cfg(feature = "parsing")] +pub(crate) fn parse_delimiter(input: ParseStream) -> Result<(MacroDelimiter, TokenStream)> { + input.step(|cursor| { + if let Some((TokenTree::Group(g), rest)) = cursor.token_tree() { + let span = g.delim_span(); + let delimiter = match g.delimiter() { + Delimiter::Parenthesis => MacroDelimiter::Paren(Paren(span)), + Delimiter::Brace => MacroDelimiter::Brace(Brace(span)), + Delimiter::Bracket => MacroDelimiter::Bracket(Bracket(span)), + Delimiter::None => { + return Err(cursor.error("expected delimiter")); + } + }; + Ok(((delimiter, g.stream()), rest)) + } else { + Err(cursor.error("expected delimiter")) + } + }) +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::error::Result; + use crate::mac::{parse_delimiter, Macro}; + use crate::parse::{Parse, ParseStream}; + use crate::path::Path; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Macro { + fn parse(input: ParseStream) -> Result<Self> { + let tokens; + Ok(Macro { + path: input.call(Path::parse_mod_style)?, + bang_token: input.parse()?, + delimiter: { + let (delimiter, content) = parse_delimiter(input)?; + tokens = content; + delimiter + }, + tokens, + }) + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::mac::{Macro, MacroDelimiter}; + use crate::path; + use crate::path::printing::PathStyle; + use crate::token; + use proc_macro2::{Delimiter, TokenStream}; + use quote::ToTokens; + + impl MacroDelimiter { + pub(crate) fn surround(&self, tokens: &mut TokenStream, inner: TokenStream) { + let (delim, span) = match self { + MacroDelimiter::Paren(paren) => (Delimiter::Parenthesis, paren.span), + MacroDelimiter::Brace(brace) => (Delimiter::Brace, brace.span), + MacroDelimiter::Bracket(bracket) => (Delimiter::Bracket, bracket.span), + }; + token::printing::delim(delim, span.join(), tokens, inner); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Macro { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + self.bang_token.to_tokens(tokens); + self.delimiter.surround(tokens, self.tokens.clone()); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/macros.rs b/third_party/crates/syn-2.0.119/src/macros.rs new file mode 100644 index 0000000..c404ae1 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/macros.rs @@ -0,0 +1,182 @@ +#[cfg_attr( + not(any(feature = "full", feature = "derive")), + allow(unknown_lints, unused_macro_rules) +)] +macro_rules! ast_struct { + ( + $(#[$attr:meta])* + $pub:ident $struct:ident $name:ident #full $body:tt + ) => { + check_keyword_matches!(pub $pub); + check_keyword_matches!(struct $struct); + + #[cfg(feature = "full")] + $(#[$attr])* $pub $struct $name $body + + #[cfg(not(feature = "full"))] + $(#[$attr])* $pub $struct $name { + _noconstruct: ::core::marker::PhantomData<::proc_macro2::Span>, + } + + #[cfg(all(not(feature = "full"), feature = "printing"))] + impl ::quote::ToTokens for $name { + fn to_tokens(&self, _: &mut ::proc_macro2::TokenStream) { + unreachable!() + } + } + }; + + ( + $(#[$attr:meta])* + $pub:ident $struct:ident $name:ident $body:tt + ) => { + check_keyword_matches!(pub $pub); + check_keyword_matches!(struct $struct); + + $(#[$attr])* $pub $struct $name $body + }; +} + +#[cfg(any(feature = "full", feature = "derive"))] +macro_rules! ast_enum { + ( + $(#[$enum_attr:meta])* + $pub:ident $enum:ident $name:ident $body:tt + ) => { + check_keyword_matches!(pub $pub); + check_keyword_matches!(enum $enum); + + $(#[$enum_attr])* $pub $enum $name $body + }; +} + +macro_rules! ast_enum_of_structs { + ( + $(#[$enum_attr:meta])* + $pub:ident $enum:ident $name:ident $body:tt + ) => { + check_keyword_matches!(pub $pub); + check_keyword_matches!(enum $enum); + + $(#[$enum_attr])* $pub $enum $name $body + + ast_enum_of_structs_impl!($name $body); + + #[cfg(feature = "printing")] + generate_to_tokens!(() tokens $name $body); + }; +} + +macro_rules! ast_enum_of_structs_impl { + ( + $name:ident { + $( + $(#[cfg $cfg_attr:tt])* + $(#[doc $($doc_attr:tt)*])* + $variant:ident $( ($member:ident) )*, + )* + } + ) => { + $($( + ast_enum_from_struct!($name::$variant, $member); + )*)* + }; +} + +macro_rules! ast_enum_from_struct { + // No From<TokenStream> for verbatim variants. + ($name:ident::Verbatim, $member:ident) => {}; + + ($name:ident::$variant:ident, $member:ident) => { + impl From<$member> for $name { + fn from(e: $member) -> $name { + $name::$variant(e) + } + } + }; +} + +#[cfg(feature = "printing")] +macro_rules! generate_to_tokens { + ( + ($($arms:tt)*) $tokens:ident $name:ident { + $(#[cfg $cfg_attr:tt])* + $(#[doc $($doc_attr:tt)*])* + $variant:ident, + $($next:tt)* + } + ) => { + generate_to_tokens!( + ($($arms)* $(#[cfg $cfg_attr])* $name::$variant => {}) + $tokens $name { $($next)* } + ); + }; + + ( + ($($arms:tt)*) $tokens:ident $name:ident { + $(#[cfg $cfg_attr:tt])* + $(#[doc $($doc_attr:tt)*])* + $variant:ident($member:ident), + $($next:tt)* + } + ) => { + generate_to_tokens!( + ($($arms)* $(#[cfg $cfg_attr])* $name::$variant(_e) => _e.to_tokens($tokens),) + $tokens $name { $($next)* } + ); + }; + + (($($arms:tt)*) $tokens:ident $name:ident {}) => { + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ::quote::ToTokens for $name { + fn to_tokens(&self, $tokens: &mut ::proc_macro2::TokenStream) { + match self { + $($arms)* + } + } + } + }; +} + +// Rustdoc bug: does not respect the doc(hidden) on some items. +#[cfg(all(doc, feature = "parsing"))] +macro_rules! pub_if_not_doc { + ($(#[$m:meta])* $pub:ident $($item:tt)*) => { + check_keyword_matches!(pub $pub); + + $(#[$m])* + $pub(crate) $($item)* + }; +} + +#[cfg(all(not(doc), feature = "parsing"))] +macro_rules! pub_if_not_doc { + ($(#[$m:meta])* $pub:ident $($item:tt)*) => { + check_keyword_matches!(pub $pub); + + $(#[$m])* + $pub $($item)* + }; +} + +macro_rules! check_keyword_matches { + (enum enum) => {}; + (pub pub) => {}; + (struct struct) => {}; +} + +#[cfg(any(feature = "full", feature = "derive"))] +macro_rules! return_impl_trait { + ( + $(#[$attr:meta])* + $vis:vis fn $name:ident $args:tt -> $impl_trait:ty [$concrete:ty] $body:block + ) => { + #[cfg(not(docsrs))] + $(#[$attr])* + $vis fn $name $args -> $concrete $body + + #[cfg(docsrs)] + $(#[$attr])* + $vis fn $name $args -> $impl_trait $body + }; +} diff --git a/third_party/crates/syn-2.0.119/src/meta.rs b/third_party/crates/syn-2.0.119/src/meta.rs new file mode 100644 index 0000000..3112db7 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/meta.rs @@ -0,0 +1,427 @@ +//! Facility for interpreting structured content inside of an `Attribute`. + +use crate::error::{Error, Result}; +use crate::ext::IdentExt as _; +use crate::lit::Lit; +use crate::parse::{ParseStream, Parser}; +use crate::path::{Path, PathSegment}; +use crate::punctuated::Punctuated; +use core::fmt::Display; +use proc_macro2::Ident; + +/// Make a parser that is usable with `parse_macro_input!` in a +/// `#[proc_macro_attribute]` macro. +/// +/// *Warning:* When parsing attribute args **other than** the +/// `proc_macro::TokenStream` input of a `proc_macro_attribute`, you do **not** +/// need this function. In several cases your callers will get worse error +/// messages if you use this function, because the surrounding delimiter's span +/// is concealed from attribute macros by rustc. Use +/// [`Attribute::parse_nested_meta`] instead. +/// +/// [`Attribute::parse_nested_meta`]: crate::Attribute::parse_nested_meta +/// +/// # Example +/// +/// This example implements an attribute macro whose invocations look like this: +/// +/// ``` +/// # const IGNORE: &str = stringify! { +/// #[tea(kind = "EarlGrey", hot)] +/// struct Picard {...} +/// # }; +/// ``` +/// +/// The "parameters" supported by the attribute are: +/// +/// - `kind = "..."` +/// - `hot` +/// - `with(sugar, milk, ...)`, a comma-separated list of ingredients +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::{parse_macro_input, LitStr, Path}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn tea(args: TokenStream, input: TokenStream) -> TokenStream { +/// let mut kind: Option<LitStr> = None; +/// let mut hot: bool = false; +/// let mut with: Vec<Path> = Vec::new(); +/// let tea_parser = syn::meta::parser(|meta| { +/// if meta.path.is_ident("kind") { +/// kind = Some(meta.value()?.parse()?); +/// Ok(()) +/// } else if meta.path.is_ident("hot") { +/// hot = true; +/// Ok(()) +/// } else if meta.path.is_ident("with") { +/// meta.parse_nested_meta(|meta| { +/// with.push(meta.path); +/// Ok(()) +/// }) +/// } else { +/// Err(meta.error("unsupported tea property")) +/// } +/// }); +/// +/// parse_macro_input!(args with tea_parser); +/// eprintln!("kind={kind:?} hot={hot} with={with:?}"); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// ``` +/// +/// The `syn::meta` library will take care of dealing with the commas including +/// trailing commas, and producing sensible error messages on unexpected input. +/// +/// ```console +/// error: expected `,` +/// --> src/main.rs:3:37 +/// | +/// 3 | #[tea(kind = "EarlGrey", with(sugar = "lol", milk))] +/// | ^ +/// ``` +/// +/// # Example +/// +/// Same as above but we factor out most of the logic into a separate function. +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::meta::ParseNestedMeta; +/// use syn::parse::{Parser, Result}; +/// use syn::{parse_macro_input, LitStr, Path}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn tea(args: TokenStream, input: TokenStream) -> TokenStream { +/// let mut attrs = TeaAttributes::default(); +/// let tea_parser = syn::meta::parser(|meta| attrs.parse(meta)); +/// parse_macro_input!(args with tea_parser); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// +/// #[derive(Default)] +/// struct TeaAttributes { +/// kind: Option<LitStr>, +/// hot: bool, +/// with: Vec<Path>, +/// } +/// +/// impl TeaAttributes { +/// fn parse(&mut self, meta: ParseNestedMeta) -> Result<()> { +/// if meta.path.is_ident("kind") { +/// self.kind = Some(meta.value()?.parse()?); +/// Ok(()) +/// } else /* just like in last example */ +/// # { unimplemented!() } +/// +/// } +/// } +/// ``` +pub fn parser(logic: impl FnMut(ParseNestedMeta) -> Result<()>) -> impl Parser<Output = ()> { + |input: ParseStream| { + if input.is_empty() { + Ok(()) + } else { + parse_nested_meta(input, logic) + } + } +} + +/// Context for parsing a single property in the conventional syntax for +/// structured attributes. +/// +/// # Examples +/// +/// Refer to usage examples on the following two entry-points: +/// +/// - [`Attribute::parse_nested_meta`] if you have an entire `Attribute` to +/// parse. Always use this if possible. Generally this is able to produce +/// better error messages because `Attribute` holds span information for all +/// of the delimiters therein. +/// +/// - [`syn::meta::parser`] if you are implementing a `proc_macro_attribute` +/// macro and parsing the arguments to the attribute macro, i.e. the ones +/// written in the same attribute that dispatched the macro invocation. Rustc +/// does not pass span information for the surrounding delimiters into the +/// attribute macro invocation in this situation, so error messages might be +/// less precise. +/// +/// [`Attribute::parse_nested_meta`]: crate::Attribute::parse_nested_meta +/// [`syn::meta::parser`]: crate::meta::parser +#[non_exhaustive] +pub struct ParseNestedMeta<'a> { + pub path: Path, + pub input: ParseStream<'a>, +} + +impl<'a> ParseNestedMeta<'a> { + /// Used when parsing `key = "value"` syntax. + /// + /// All it does is advance `meta.input` past the `=` sign in the input. You + /// could accomplish the same effect by writing + /// `meta.parse::<Token![=]>()?`, so at most it is a minor convenience to + /// use `meta.value()?`. + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute, LitStr}; + /// + /// let attr: Attribute = parse_quote! { + /// #[tea(kind = "EarlGrey")] + /// }; + /// // conceptually: + /// if attr.path().is_ident("tea") { // this parses the `tea` + /// attr.parse_nested_meta(|meta| { // this parses the `(` + /// if meta.path.is_ident("kind") { // this parses the `kind` + /// let value = meta.value()?; // this parses the `=` + /// let s: LitStr = value.parse()?; // this parses `"EarlGrey"` + /// if s.value() == "EarlGrey" { + /// // ... + /// } + /// Ok(()) + /// } else { + /// Err(meta.error("unsupported attribute")) + /// } + /// })?; + /// } + /// # anyhow::Ok(()) + /// ``` + pub fn value(&self) -> Result<ParseStream<'a>> { + self.input.parse::<Token![=]>()?; + Ok(self.input) + } + + /// Used when parsing `list(...)` syntax **if** the content inside the + /// nested parentheses is also expected to conform to Rust's structured + /// attribute convention. + /// + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute}; + /// + /// let attr: Attribute = parse_quote! { + /// #[tea(with(sugar, milk))] + /// }; + /// + /// if attr.path().is_ident("tea") { + /// attr.parse_nested_meta(|meta| { + /// if meta.path.is_ident("with") { + /// meta.parse_nested_meta(|meta| { // <--- + /// if meta.path.is_ident("sugar") { + /// // Here we can go even deeper if needed. + /// Ok(()) + /// } else if meta.path.is_ident("milk") { + /// Ok(()) + /// } else { + /// Err(meta.error("unsupported ingredient")) + /// } + /// }) + /// } else { + /// Err(meta.error("unsupported tea property")) + /// } + /// })?; + /// } + /// # anyhow::Ok(()) + /// ``` + /// + /// # Counterexample + /// + /// If you don't need `parse_nested_meta`'s help in parsing the content + /// written within the nested parentheses, keep in mind that you can always + /// just parse it yourself from the exposed ParseStream. Rust syntax permits + /// arbitrary tokens within those parentheses so for the crazier stuff, + /// `parse_nested_meta` is not what you want. + /// + /// ``` + /// use syn::{parenthesized, parse_quote, Attribute, LitInt}; + /// + /// let attr: Attribute = parse_quote! { + /// #[repr(align(32))] + /// }; + /// + /// let mut align: Option<LitInt> = None; + /// if attr.path().is_ident("repr") { + /// attr.parse_nested_meta(|meta| { + /// if meta.path.is_ident("align") { + /// let content; + /// parenthesized!(content in meta.input); + /// align = Some(content.parse()?); + /// Ok(()) + /// } else { + /// Err(meta.error("unsupported repr")) + /// } + /// })?; + /// } + /// # anyhow::Ok(()) + /// ``` + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + let content; + parenthesized!(content in self.input); + parse_nested_meta(&content, logic) + } + + /// Report that the attribute's content did not conform to expectations. + /// + /// The span of the resulting error will cover `meta.path` *and* everything + /// that has been parsed so far since it. + /// + /// There are 2 ways you might call this. First, if `meta.path` is not + /// something you recognize: + /// + /// ``` + /// # use syn::Attribute; + /// # + /// # fn example(attr: &Attribute) -> syn::Result<()> { + /// attr.parse_nested_meta(|meta| { + /// if meta.path.is_ident("kind") { + /// // ... + /// Ok(()) + /// } else { + /// Err(meta.error("unsupported tea property")) + /// } + /// })?; + /// # Ok(()) + /// # } + /// ``` + /// + /// In this case, it behaves exactly like + /// `syn::Error::new_spanned(&meta.path, "message...")`. + /// + /// ```console + /// error: unsupported tea property + /// --> src/main.rs:3:26 + /// | + /// 3 | #[tea(kind = "EarlGrey", wat = "foo")] + /// | ^^^ + /// ``` + /// + /// More usefully, the second place is if you've already parsed a value but + /// have decided not to accept the value: + /// + /// ``` + /// # use syn::Attribute; + /// # + /// # fn example(attr: &Attribute) -> syn::Result<()> { + /// use syn::Expr; + /// + /// attr.parse_nested_meta(|meta| { + /// if meta.path.is_ident("kind") { + /// let expr: Expr = meta.value()?.parse()?; + /// match expr { + /// Expr::Lit(expr) => /* ... */ + /// # unimplemented!(), + /// Expr::Path(expr) => /* ... */ + /// # unimplemented!(), + /// Expr::Macro(expr) => /* ... */ + /// # unimplemented!(), + /// _ => Err(meta.error("tea kind must be a string literal, path, or macro")), + /// } + /// } else /* as above */ + /// # { unimplemented!() } + /// + /// })?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ```console + /// error: tea kind must be a string literal, path, or macro + /// --> src/main.rs:3:7 + /// | + /// 3 | #[tea(kind = async { replicator.await })] + /// | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + /// ``` + /// + /// Often you may want to use `syn::Error::new_spanned` even in this + /// situation. In the above code, that would be: + /// + /// ``` + /// # use syn::{Error, Expr}; + /// # + /// # fn example(expr: Expr) -> syn::Result<()> { + /// match expr { + /// Expr::Lit(expr) => /* ... */ + /// # unimplemented!(), + /// Expr::Path(expr) => /* ... */ + /// # unimplemented!(), + /// Expr::Macro(expr) => /* ... */ + /// # unimplemented!(), + /// _ => Err(Error::new_spanned(expr, "unsupported expression type for `kind`")), + /// } + /// # } + /// ``` + /// + /// ```console + /// error: unsupported expression type for `kind` + /// --> src/main.rs:3:14 + /// | + /// 3 | #[tea(kind = async { replicator.await })] + /// | ^^^^^^^^^^^^^^^^^^^^^^^^^^ + /// ``` + pub fn error(&self, msg: impl Display) -> Error { + let start_span = self.path.segments[0].ident.span(); + let end_span = self.input.cursor().prev_span(); + crate::error::new2(start_span, end_span, msg) + } +} + +pub(crate) fn parse_nested_meta( + input: ParseStream, + mut logic: impl FnMut(ParseNestedMeta) -> Result<()>, +) -> Result<()> { + loop { + let path = input.call(parse_meta_path)?; + logic(ParseNestedMeta { path, input })?; + if input.is_empty() { + return Ok(()); + } + input.parse::<Token![,]>()?; + if input.is_empty() { + return Ok(()); + } + } +} + +// Like Path::parse_mod_style, but accepts keywords in the path. +fn parse_meta_path(input: ParseStream) -> Result<Path> { + Ok(Path { + leading_colon: input.parse()?, + segments: { + let mut segments = Punctuated::new(); + if input.peek(Ident::peek_any) { + let ident = Ident::parse_any(input)?; + segments.push_value(PathSegment::from(ident)); + } else if input.is_empty() { + return Err(input.error("expected nested attribute")); + } else if input.peek(Lit) { + return Err(input.error("unexpected literal in nested attribute, expected ident")); + } else { + return Err(input.error("unexpected token in nested attribute, expected ident")); + } + while input.peek(Token![::]) { + let punct = input.parse()?; + segments.push_punct(punct); + let ident = Ident::parse_any(input)?; + segments.push_value(PathSegment::from(ident)); + } + segments + }, + }) +} diff --git a/third_party/crates/syn-2.0.119/src/op.rs b/third_party/crates/syn-2.0.119/src/op.rs new file mode 100644 index 0000000..575d9fa --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/op.rs @@ -0,0 +1,219 @@ +ast_enum! { + /// A binary operator: `+`, `+=`, `&`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum BinOp { + /// The `+` operator (addition) + Add(Token![+]), + /// The `-` operator (subtraction) + Sub(Token![-]), + /// The `*` operator (multiplication) + Mul(Token![*]), + /// The `/` operator (division) + Div(Token![/]), + /// The `%` operator (modulus) + Rem(Token![%]), + /// The `&&` operator (logical and) + And(Token![&&]), + /// The `||` operator (logical or) + Or(Token![||]), + /// The `^` operator (bitwise xor) + BitXor(Token![^]), + /// The `&` operator (bitwise and) + BitAnd(Token![&]), + /// The `|` operator (bitwise or) + BitOr(Token![|]), + /// The `<<` operator (shift left) + Shl(Token![<<]), + /// The `>>` operator (shift right) + Shr(Token![>>]), + /// The `==` operator (equality) + Eq(Token![==]), + /// The `<` operator (less than) + Lt(Token![<]), + /// The `<=` operator (less than or equal to) + Le(Token![<=]), + /// The `!=` operator (not equal to) + Ne(Token![!=]), + /// The `>=` operator (greater than or equal to) + Ge(Token![>=]), + /// The `>` operator (greater than) + Gt(Token![>]), + /// The `+=` operator + AddAssign(Token![+=]), + /// The `-=` operator + SubAssign(Token![-=]), + /// The `*=` operator + MulAssign(Token![*=]), + /// The `/=` operator + DivAssign(Token![/=]), + /// The `%=` operator + RemAssign(Token![%=]), + /// The `^=` operator + BitXorAssign(Token![^=]), + /// The `&=` operator + BitAndAssign(Token![&=]), + /// The `|=` operator + BitOrAssign(Token![|=]), + /// The `<<=` operator + ShlAssign(Token![<<=]), + /// The `>>=` operator + ShrAssign(Token![>>=]), + } +} + +ast_enum! { + /// A unary operator: `*`, `!`, `-`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum UnOp { + /// The `*` operator for dereferencing + Deref(Token![*]), + /// The `!` operator for logical inversion + Not(Token![!]), + /// The `-` operator for negation + Neg(Token![-]), + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::error::Result; + use crate::op::{BinOp, UnOp}; + use crate::parse::{Parse, ParseStream}; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for BinOp { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Token![+=]) { + input.parse().map(BinOp::AddAssign) + } else if input.peek(Token![-=]) { + input.parse().map(BinOp::SubAssign) + } else if input.peek(Token![*=]) { + input.parse().map(BinOp::MulAssign) + } else if input.peek(Token![/=]) { + input.parse().map(BinOp::DivAssign) + } else if input.peek(Token![%=]) { + input.parse().map(BinOp::RemAssign) + } else if input.peek(Token![^=]) { + input.parse().map(BinOp::BitXorAssign) + } else if input.peek(Token![&=]) { + input.parse().map(BinOp::BitAndAssign) + } else if input.peek(Token![|=]) { + input.parse().map(BinOp::BitOrAssign) + } else if input.peek(Token![<<=]) { + input.parse().map(BinOp::ShlAssign) + } else if input.peek(Token![>>=]) { + input.parse().map(BinOp::ShrAssign) + } else if input.peek(Token![&&]) { + input.parse().map(BinOp::And) + } else if input.peek(Token![||]) { + input.parse().map(BinOp::Or) + } else if input.peek(Token![<<]) { + input.parse().map(BinOp::Shl) + } else if input.peek(Token![>>]) { + input.parse().map(BinOp::Shr) + } else if input.peek(Token![==]) { + input.parse().map(BinOp::Eq) + } else if input.peek(Token![<=]) { + input.parse().map(BinOp::Le) + } else if input.peek(Token![!=]) { + input.parse().map(BinOp::Ne) + } else if input.peek(Token![>=]) { + input.parse().map(BinOp::Ge) + } else if input.peek(Token![+]) { + input.parse().map(BinOp::Add) + } else if input.peek(Token![-]) { + input.parse().map(BinOp::Sub) + } else if input.peek(Token![*]) { + input.parse().map(BinOp::Mul) + } else if input.peek(Token![/]) { + input.parse().map(BinOp::Div) + } else if input.peek(Token![%]) { + input.parse().map(BinOp::Rem) + } else if input.peek(Token![^]) { + input.parse().map(BinOp::BitXor) + } else if input.peek(Token![&]) { + input.parse().map(BinOp::BitAnd) + } else if input.peek(Token![|]) { + input.parse().map(BinOp::BitOr) + } else if input.peek(Token![<]) { + input.parse().map(BinOp::Lt) + } else if input.peek(Token![>]) { + input.parse().map(BinOp::Gt) + } else { + Err(input.error("expected binary operator")) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for UnOp { + fn parse(input: ParseStream) -> Result<Self> { + let lookahead = input.lookahead1(); + if lookahead.peek(Token![*]) { + input.parse().map(UnOp::Deref) + } else if lookahead.peek(Token![!]) { + input.parse().map(UnOp::Not) + } else if lookahead.peek(Token![-]) { + input.parse().map(UnOp::Neg) + } else { + Err(lookahead.error()) + } + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::op::{BinOp, UnOp}; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for BinOp { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + BinOp::Add(t) => t.to_tokens(tokens), + BinOp::Sub(t) => t.to_tokens(tokens), + BinOp::Mul(t) => t.to_tokens(tokens), + BinOp::Div(t) => t.to_tokens(tokens), + BinOp::Rem(t) => t.to_tokens(tokens), + BinOp::And(t) => t.to_tokens(tokens), + BinOp::Or(t) => t.to_tokens(tokens), + BinOp::BitXor(t) => t.to_tokens(tokens), + BinOp::BitAnd(t) => t.to_tokens(tokens), + BinOp::BitOr(t) => t.to_tokens(tokens), + BinOp::Shl(t) => t.to_tokens(tokens), + BinOp::Shr(t) => t.to_tokens(tokens), + BinOp::Eq(t) => t.to_tokens(tokens), + BinOp::Lt(t) => t.to_tokens(tokens), + BinOp::Le(t) => t.to_tokens(tokens), + BinOp::Ne(t) => t.to_tokens(tokens), + BinOp::Ge(t) => t.to_tokens(tokens), + BinOp::Gt(t) => t.to_tokens(tokens), + BinOp::AddAssign(t) => t.to_tokens(tokens), + BinOp::SubAssign(t) => t.to_tokens(tokens), + BinOp::MulAssign(t) => t.to_tokens(tokens), + BinOp::DivAssign(t) => t.to_tokens(tokens), + BinOp::RemAssign(t) => t.to_tokens(tokens), + BinOp::BitXorAssign(t) => t.to_tokens(tokens), + BinOp::BitAndAssign(t) => t.to_tokens(tokens), + BinOp::BitOrAssign(t) => t.to_tokens(tokens), + BinOp::ShlAssign(t) => t.to_tokens(tokens), + BinOp::ShrAssign(t) => t.to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for UnOp { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + UnOp::Deref(t) => t.to_tokens(tokens), + UnOp::Not(t) => t.to_tokens(tokens), + UnOp::Neg(t) => t.to_tokens(tokens), + } + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/parse.rs b/third_party/crates/syn-2.0.119/src/parse.rs new file mode 100644 index 0000000..9e10b4c --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/parse.rs @@ -0,0 +1,1420 @@ +//! Parsing interface for parsing a token stream into a syntax tree node. +//! +//! Parsing in Syn is built on parser functions that take in a [`ParseStream`] +//! and produce a [`Result<T>`] where `T` is some syntax tree node. Underlying +//! these parser functions is a lower level mechanism built around the +//! [`Cursor`] type. `Cursor` is a cheaply copyable cursor over a range of +//! tokens in a token stream. +//! +//! [`Result<T>`]: Result +//! [`Cursor`]: crate::buffer::Cursor +//! +//! # Example +//! +//! Here is a snippet of parsing code to get a feel for the style of the +//! library. We define data structures for a subset of Rust syntax including +//! enums (not shown) and structs, then provide implementations of the [`Parse`] +//! trait to parse these syntax tree data structures from a token stream. +//! +//! Once `Parse` impls have been defined, they can be called conveniently from a +//! procedural macro through [`parse_macro_input!`] as shown at the bottom of +//! the snippet. If the caller provides syntactically invalid input to the +//! procedural macro, they will receive a helpful compiler error message +//! pointing out the exact token that triggered the failure to parse. +//! +//! [`parse_macro_input!`]: crate::parse_macro_input! +//! +//! ``` +//! # extern crate proc_macro; +//! # +//! use proc_macro::TokenStream; +//! use syn::{braced, parse_macro_input, token, Field, Ident, Result, Token}; +//! use syn::parse::{Parse, ParseStream}; +//! use syn::punctuated::Punctuated; +//! +//! enum Item { +//! Struct(ItemStruct), +//! Enum(ItemEnum), +//! } +//! +//! struct ItemStruct { +//! struct_token: Token![struct], +//! ident: Ident, +//! brace_token: token::Brace, +//! fields: Punctuated<Field, Token![,]>, +//! } +//! # +//! # enum ItemEnum {} +//! +//! impl Parse for Item { +//! fn parse(input: ParseStream) -> Result<Self> { +//! let lookahead = input.lookahead1(); +//! if lookahead.peek(Token![struct]) { +//! input.parse().map(Item::Struct) +//! } else if lookahead.peek(Token![enum]) { +//! input.parse().map(Item::Enum) +//! } else { +//! Err(lookahead.error()) +//! } +//! } +//! } +//! +//! impl Parse for ItemStruct { +//! fn parse(input: ParseStream) -> Result<Self> { +//! let content; +//! Ok(ItemStruct { +//! struct_token: input.parse()?, +//! ident: input.parse()?, +//! brace_token: braced!(content in input), +//! fields: content.parse_terminated(Field::parse_named, Token![,])?, +//! }) +//! } +//! } +//! # +//! # impl Parse for ItemEnum { +//! # fn parse(input: ParseStream) -> Result<Self> { +//! # unimplemented!() +//! # } +//! # } +//! +//! # const IGNORE: &str = stringify! { +//! #[proc_macro] +//! # }; +//! pub fn my_macro(tokens: TokenStream) -> TokenStream { +//! let input = parse_macro_input!(tokens as Item); +//! +//! /* ... */ +//! # TokenStream::new() +//! } +//! ``` +//! +//! # The `syn::parse*` functions +//! +//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve +//! as an entry point for parsing syntax tree nodes that can be parsed in an +//! obvious default way. These functions can return any syntax tree node that +//! implements the [`Parse`] trait, which includes most types in Syn. +//! +//! [`syn::parse`]: crate::parse() +//! [`syn::parse2`]: crate::parse2() +//! [`syn::parse_str`]: crate::parse_str() +//! +//! ``` +//! use syn::Type; +//! +//! # fn run_parser() -> syn::Result<()> { +//! let t: Type = syn::parse_str("alloc::collections::HashMap<String, Value>")?; +//! # Ok(()) +//! # } +//! # +//! # run_parser().unwrap(); +//! ``` +//! +//! The [`parse_quote!`] macro also uses this approach. +//! +//! [`parse_quote!`]: crate::parse_quote! +//! +//! # The `Parser` trait +//! +//! Some types can be parsed in several ways depending on context. For example +//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like +//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`] +//! may or may not allow trailing punctuation, and parsing it the wrong way +//! would either reject valid input or accept invalid input. +//! +//! [`Attribute`]: crate::Attribute +//! [`Punctuated`]: crate::punctuated +//! +//! The `Parse` trait is not implemented in these cases because there is no good +//! behavior to consider the default. +//! +//! ```compile_fail +//! # extern crate proc_macro; +//! # +//! # use syn::punctuated::Punctuated; +//! # use syn::{PathSegment, Result, Token}; +//! # +//! # fn f(tokens: proc_macro::TokenStream) -> Result<()> { +//! # +//! // Can't parse `Punctuated` without knowing whether trailing punctuation +//! // should be allowed in this context. +//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?; +//! # +//! # Ok(()) +//! # } +//! ``` +//! +//! In these cases the types provide a choice of parser functions rather than a +//! single `Parse` implementation, and those parser functions can be invoked +//! through the [`Parser`] trait. +//! +//! +//! ``` +//! # extern crate proc_macro; +//! # +//! use proc_macro::TokenStream; +//! use syn::parse::Parser; +//! use syn::punctuated::Punctuated; +//! use syn::{Attribute, Expr, PathSegment, Result, Token}; +//! +//! fn call_some_parser_methods(input: TokenStream) -> Result<()> { +//! // Parse a nonempty sequence of path segments separated by `::` punctuation +//! // with no trailing punctuation. +//! let tokens = input.clone(); +//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty; +//! let _path = parser.parse(tokens)?; +//! +//! // Parse a possibly empty sequence of expressions terminated by commas with +//! // an optional trailing punctuation. +//! let tokens = input.clone(); +//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated; +//! let _args = parser.parse(tokens)?; +//! +//! // Parse zero or more outer attributes but not inner attributes. +//! let tokens = input.clone(); +//! let parser = Attribute::parse_outer; +//! let _attrs = parser.parse(tokens)?; +//! +//! Ok(()) +//! } +//! ``` + +#[path = "discouraged.rs"] +pub mod discouraged; + +use crate::buffer::{Cursor, TokenBuffer}; +use crate::error; +use crate::lookahead; +use crate::punctuated::Punctuated; +use crate::token::Token; +use alloc::boxed::Box; +use alloc::rc::Rc; +use core::cell::Cell; +use core::fmt::{self, Debug, Display}; +#[cfg(feature = "extra-traits")] +use core::hash::{Hash, Hasher}; +use core::marker::PhantomData; +use core::mem; +use core::ops::Deref; +use core::panic::{RefUnwindSafe, UnwindSafe}; +use core::str::FromStr; +use proc_macro2::{Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree}; +#[cfg(feature = "printing")] +use quote::ToTokens; + +pub use crate::error::{Error, Result}; +pub use crate::lookahead::{End, Lookahead1, Peek}; + +/// Parsing interface implemented by all types that can be parsed in a default +/// way from a token stream. +/// +/// Refer to the [module documentation] for details about implementing and using +/// the `Parse` trait. +/// +/// [module documentation]: self +pub trait Parse: Sized { + fn parse(input: ParseStream) -> Result<Self>; +} + +/// Input to a Syn parser function. +/// +/// See the methods of this type under the documentation of [`ParseBuffer`]. For +/// an overview of parsing in Syn, refer to the [module documentation]. +/// +/// [module documentation]: self +pub type ParseStream<'a> = &'a ParseBuffer<'a>; + +/// Cursor position within a buffered token stream. +/// +/// This type is more commonly used through the type alias [`ParseStream`] which +/// is an alias for `&ParseBuffer`. +/// +/// `ParseStream` is the input type for all parser functions in Syn. They have +/// the signature `fn(ParseStream) -> Result<T>`. +/// +/// ## Calling a parser function +/// +/// There is no public way to construct a `ParseBuffer`. Instead, if you are +/// looking to invoke a parser function that requires `ParseStream` as input, +/// you will need to go through one of the public parsing entry points. +/// +/// - The [`parse_macro_input!`] macro if parsing input of a procedural macro; +/// - One of [the `syn::parse*` functions][syn-parse]; or +/// - A method of the [`Parser`] trait. +/// +/// [`parse_macro_input!`]: crate::parse_macro_input! +/// [syn-parse]: self#the-synparse-functions +pub struct ParseBuffer<'a> { + scope: Span, + // Instead of Cell<Cursor<'a>> so that ParseBuffer<'a> is covariant in 'a. + // The rest of the code in this module needs to be careful that only a + // cursor derived from this `cell` is ever assigned to this `cell`. + // + // Cell<Cursor<'a>> cannot be covariant in 'a because then we could take a + // ParseBuffer<'a>, upcast to ParseBuffer<'short> for some lifetime shorter + // than 'a, and then assign a Cursor<'short> into the Cell. + // + // By extension, it would not be safe to expose an API that accepts a + // Cursor<'a> and trusts that it lives as long as the cursor currently in + // the cell. + cell: Cell<Cursor<'static>>, + marker: PhantomData<Cursor<'a>>, + unexpected: Cell<Option<Rc<Cell<Unexpected>>>>, +} + +impl<'a> Drop for ParseBuffer<'a> { + fn drop(&mut self) { + if let Some((unexpected_span, delimiter)) = span_of_unexpected_ignoring_nones(self.cursor()) + { + let (inner, old_span) = inner_unexpected(self); + if old_span.is_none() { + inner.set(Unexpected::Some(unexpected_span, delimiter)); + } + } + } +} + +impl<'a> Display for ParseBuffer<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Display::fmt(&self.cursor().token_stream(), f) + } +} + +impl<'a> Debug for ParseBuffer<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(&self.cursor().token_stream(), f) + } +} + +impl<'a> UnwindSafe for ParseBuffer<'a> {} +impl<'a> RefUnwindSafe for ParseBuffer<'a> {} + +/// Cursor state associated with speculative parsing. +/// +/// This type is the input of the closure provided to [`ParseStream::step`]. +/// +/// [`ParseStream::step`]: ParseBuffer::step +/// +/// # Example +/// +/// ``` +/// use proc_macro2::TokenTree; +/// use syn::Result; +/// use syn::parse::ParseStream; +/// +/// // This function advances the stream past the next occurrence of `@`. If +/// // no `@` is present in the stream, the stream position is unchanged and +/// // an error is returned. +/// fn skip_past_next_at(input: ParseStream) -> Result<()> { +/// input.step(|cursor| { +/// let mut rest = *cursor; +/// while let Some((tt, next)) = rest.token_tree() { +/// match &tt { +/// TokenTree::Punct(punct) if punct.as_char() == '@' => { +/// return Ok(((), next)); +/// } +/// _ => rest = next, +/// } +/// } +/// Err(cursor.error("no `@` was found after this point")) +/// }) +/// } +/// # +/// # fn remainder_after_skipping_past_next_at( +/// # input: ParseStream, +/// # ) -> Result<proc_macro2::TokenStream> { +/// # skip_past_next_at(input)?; +/// # input.parse() +/// # } +/// # +/// # use syn::parse::Parser; +/// # let remainder = remainder_after_skipping_past_next_at +/// # .parse_str("a @ b c") +/// # .unwrap(); +/// # assert_eq!(remainder.to_string(), "b c"); +/// ``` +pub struct StepCursor<'c, 'a> { + scope: Span, + // This field is covariant in 'c. + cursor: Cursor<'c>, + // This field is contravariant in 'c. Together these make StepCursor + // invariant in 'c. Also covariant in 'a. The user cannot cast 'c to a + // different lifetime but can upcast into a StepCursor with a shorter + // lifetime 'a. + // + // As long as we only ever construct a StepCursor for which 'c outlives 'a, + // this means if ever a StepCursor<'c, 'a> exists we are guaranteed that 'c + // outlives 'a. + marker: PhantomData<fn(Cursor<'c>) -> Cursor<'a>>, +} + +impl<'c, 'a> Deref for StepCursor<'c, 'a> { + type Target = Cursor<'c>; + + fn deref(&self) -> &Self::Target { + &self.cursor + } +} + +impl<'c, 'a> Copy for StepCursor<'c, 'a> {} + +impl<'c, 'a> Clone for StepCursor<'c, 'a> { + fn clone(&self) -> Self { + *self + } +} + +impl<'c, 'a> StepCursor<'c, 'a> { + /// Triggers an error at the current position of the parse stream. + /// + /// The `ParseStream::step` invocation will return this same error without + /// advancing the stream state. + pub fn error<T: Display>(self, message: T) -> Error { + error::new_at(self.scope, self.cursor, message) + } +} + +pub(crate) fn advance_step_cursor<'c, 'a>(proof: StepCursor<'c, 'a>, to: Cursor<'c>) -> Cursor<'a> { + // Refer to the comments within the StepCursor definition. We use the + // fact that a StepCursor<'c, 'a> exists as proof that 'c outlives 'a. + // Cursor is covariant in its lifetime parameter so we can cast a + // Cursor<'c> to one with the shorter lifetime Cursor<'a>. + let _ = proof; + unsafe { mem::transmute::<Cursor<'c>, Cursor<'a>>(to) } +} + +pub(crate) fn new_parse_buffer( + scope: Span, + cursor: Cursor, + unexpected: Rc<Cell<Unexpected>>, +) -> ParseBuffer { + ParseBuffer { + scope, + // See comment on `cell` in the struct definition. + cell: Cell::new(unsafe { mem::transmute::<Cursor, Cursor<'static>>(cursor) }), + marker: PhantomData, + unexpected: Cell::new(Some(unexpected)), + } +} + +pub(crate) enum Unexpected { + None, + Some(Span, Delimiter), + Chain(Rc<Cell<Unexpected>>), +} + +impl Default for Unexpected { + fn default() -> Self { + Unexpected::None + } +} + +impl Clone for Unexpected { + fn clone(&self) -> Self { + match self { + Unexpected::None => Unexpected::None, + Unexpected::Some(span, delimiter) => Unexpected::Some(*span, *delimiter), + Unexpected::Chain(next) => Unexpected::Chain(next.clone()), + } + } +} + +// We call this on Cell<Unexpected> and Cell<Option<T>> where temporarily +// swapping in a None is cheap. +fn cell_clone<T: Default + Clone>(cell: &Cell<T>) -> T { + let prev = cell.take(); + let ret = prev.clone(); + cell.set(prev); + ret +} + +fn inner_unexpected(buffer: &ParseBuffer) -> (Rc<Cell<Unexpected>>, Option<(Span, Delimiter)>) { + let mut unexpected = get_unexpected(buffer); + loop { + match cell_clone(&unexpected) { + Unexpected::None => return (unexpected, None), + Unexpected::Some(span, delimiter) => return (unexpected, Some((span, delimiter))), + Unexpected::Chain(next) => unexpected = next, + } + } +} + +pub(crate) fn get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Unexpected>> { + cell_clone(&buffer.unexpected).unwrap() +} + +fn span_of_unexpected_ignoring_nones(mut cursor: Cursor) -> Option<(Span, Delimiter)> { + if cursor.eof() { + return None; + } + while let Some((inner, _span, rest)) = cursor.group(Delimiter::None) { + if let Some(unexpected) = span_of_unexpected_ignoring_nones(inner) { + return Some(unexpected); + } + cursor = rest; + } + if cursor.eof() { + None + } else { + Some((cursor.span(), cursor.scope_delimiter())) + } +} + +impl<'a> ParseBuffer<'a> { + /// Parses a syntax tree node of type `T`, advancing the position of our + /// parse stream past it. + pub fn parse<T: Parse>(&self) -> Result<T> { + T::parse(self) + } + + /// Calls the given parser function to parse a syntax tree node of type `T` + /// from this stream. + /// + /// # Example + /// + /// The parser below invokes [`Attribute::parse_outer`] to parse a vector of + /// zero or more outer attributes. + /// + /// [`Attribute::parse_outer`]: crate::Attribute::parse_outer + /// + /// ``` + /// use syn::{Attribute, Ident, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parses a unit struct with attributes. + /// // + /// // #[path = "s.tmpl"] + /// // struct S; + /// struct UnitStruct { + /// attrs: Vec<Attribute>, + /// struct_token: Token![struct], + /// name: Ident, + /// semi_token: Token![;], + /// } + /// + /// impl Parse for UnitStruct { + /// fn parse(input: ParseStream) -> Result<Self> { + /// Ok(UnitStruct { + /// attrs: input.call(Attribute::parse_outer)?, + /// struct_token: input.parse()?, + /// name: input.parse()?, + /// semi_token: input.parse()?, + /// }) + /// } + /// } + /// ``` + pub fn call<T>(&'a self, function: fn(ParseStream<'a>) -> Result<T>) -> Result<T> { + function(self) + } + + /// Looks at the next token in the parse stream to determine whether it + /// matches the requested type of token. + /// + /// Does not advance the position of the parse stream. + /// + /// # Syntax + /// + /// Note that this method does not use turbofish syntax. Pass the peek type + /// inside of parentheses. + /// + /// - `input.peek(Token![struct])` + /// - `input.peek(Token![==])` + /// - `input.peek(syn::Ident)` *(does not accept keywords)* + /// - `input.peek(syn::Ident::peek_any)` + /// - `input.peek(Lifetime)` + /// - `input.peek(token::Brace)` + /// + /// # Example + /// + /// In this example we finish parsing the list of supertraits when the next + /// token in the input is either `where` or an opening curly brace. + /// + /// ``` + /// use syn::{braced, token, Generics, Ident, Result, Token, TypeParamBound}; + /// use syn::parse::{Parse, ParseStream}; + /// use syn::punctuated::Punctuated; + /// + /// // Parses a trait definition containing no associated items. + /// // + /// // trait Marker<'de, T>: A + B<'de> where Box<T>: Clone {} + /// struct MarkerTrait { + /// trait_token: Token![trait], + /// ident: Ident, + /// generics: Generics, + /// colon_token: Option<Token![:]>, + /// supertraits: Punctuated<TypeParamBound, Token![+]>, + /// brace_token: token::Brace, + /// } + /// + /// impl Parse for MarkerTrait { + /// fn parse(input: ParseStream) -> Result<Self> { + /// let trait_token: Token![trait] = input.parse()?; + /// let ident: Ident = input.parse()?; + /// let mut generics: Generics = input.parse()?; + /// let colon_token: Option<Token![:]> = input.parse()?; + /// + /// let mut supertraits = Punctuated::new(); + /// if colon_token.is_some() { + /// loop { + /// supertraits.push_value(input.parse()?); + /// if input.peek(Token![where]) || input.peek(token::Brace) { + /// break; + /// } + /// supertraits.push_punct(input.parse()?); + /// } + /// } + /// + /// generics.where_clause = input.parse()?; + /// let content; + /// let empty_brace_token = braced!(content in input); + /// + /// Ok(MarkerTrait { + /// trait_token, + /// ident, + /// generics, + /// colon_token, + /// supertraits, + /// brace_token: empty_brace_token, + /// }) + /// } + /// } + /// ``` + pub fn peek<T: Peek>(&self, token: T) -> bool { + let _ = token; + T::Token::peek(self.cursor()) + } + + /// Looks at the second-next token in the parse stream. + /// + /// This is commonly useful as a way to implement contextual keywords. + /// + /// # Example + /// + /// This example needs to use `peek2` because the symbol `union` is not a + /// keyword in Rust. We can't use just `peek` and decide to parse a union if + /// the very next token is `union`, because someone is free to write a `mod + /// union` and a macro invocation that looks like `union::some_macro! { ... + /// }`. In other words `union` is a contextual keyword. + /// + /// ``` + /// use syn::{Ident, ItemUnion, Macro, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parses either a union or a macro invocation. + /// enum UnionOrMacro { + /// // union MaybeUninit<T> { uninit: (), value: T } + /// Union(ItemUnion), + /// // lazy_static! { ... } + /// Macro(Macro), + /// } + /// + /// impl Parse for UnionOrMacro { + /// fn parse(input: ParseStream) -> Result<Self> { + /// if input.peek(Token![union]) && input.peek2(Ident) { + /// input.parse().map(UnionOrMacro::Union) + /// } else { + /// input.parse().map(UnionOrMacro::Macro) + /// } + /// } + /// } + /// ``` + pub fn peek2<T: Peek>(&self, token: T) -> bool { + fn peek2(buffer: &ParseBuffer, peek: fn(Cursor) -> bool) -> bool { + buffer.cursor().skip().is_some_and(peek) + } + + let _ = token; + peek2(self, T::Token::peek) + } + + /// Looks at the third-next token in the parse stream. + pub fn peek3<T: Peek>(&self, token: T) -> bool { + fn peek3(buffer: &ParseBuffer, peek: fn(Cursor) -> bool) -> bool { + buffer + .cursor() + .skip() + .and_then(Cursor::skip) + .is_some_and(peek) + } + + let _ = token; + peek3(self, T::Token::peek) + } + + /// Parses zero or more occurrences of `T` separated by punctuation of type + /// `P`, with optional trailing punctuation. + /// + /// Parsing continues until the end of this parse stream. The entire content + /// of this parse stream must consist of `T` and `P`. + /// + /// # Example + /// + /// ``` + /// # use quote::quote; + /// # + /// use syn::{parenthesized, token, Ident, Result, Token, Type}; + /// use syn::parse::{Parse, ParseStream}; + /// use syn::punctuated::Punctuated; + /// + /// // Parse a simplified tuple struct syntax like: + /// // + /// // struct S(A, B); + /// struct TupleStruct { + /// struct_token: Token![struct], + /// ident: Ident, + /// paren_token: token::Paren, + /// fields: Punctuated<Type, Token![,]>, + /// semi_token: Token![;], + /// } + /// + /// impl Parse for TupleStruct { + /// fn parse(input: ParseStream) -> Result<Self> { + /// let content; + /// Ok(TupleStruct { + /// struct_token: input.parse()?, + /// ident: input.parse()?, + /// paren_token: parenthesized!(content in input), + /// fields: content.parse_terminated(Type::parse, Token![,])?, + /// semi_token: input.parse()?, + /// }) + /// } + /// } + /// # + /// # let input = quote! { + /// # struct S(A, B); + /// # }; + /// # syn::parse2::<TupleStruct>(input).unwrap(); + /// ``` + /// + /// # See also + /// + /// If your separator is anything more complicated than an invocation of the + /// `Token!` macro, this method won't be applicable and you can instead + /// directly use `Punctuated`'s parser functions: [`parse_terminated`], + /// [`parse_separated_nonempty`] etc. + /// + /// [`parse_terminated`]: Punctuated::parse_terminated + /// [`parse_separated_nonempty`]: Punctuated::parse_separated_nonempty + /// + /// ``` + /// use syn::{custom_keyword, Expr, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// use syn::punctuated::Punctuated; + /// + /// mod kw { + /// syn::custom_keyword!(fin); + /// } + /// + /// struct Fin(kw::fin, Token![;]); + /// + /// impl Parse for Fin { + /// fn parse(input: ParseStream) -> Result<Self> { + /// Ok(Self(input.parse()?, input.parse()?)) + /// } + /// } + /// + /// struct Thing { + /// steps: Punctuated<Expr, Fin>, + /// } + /// + /// impl Parse for Thing { + /// fn parse(input: ParseStream) -> Result<Self> { + /// # if true { + /// Ok(Thing { + /// steps: Punctuated::parse_terminated(input)?, + /// }) + /// # } else { + /// // or equivalently, this means the same thing: + /// # Ok(Thing { + /// steps: input.call(Punctuated::parse_terminated)?, + /// # }) + /// # } + /// } + /// } + /// ``` + pub fn parse_terminated<T, P>( + &'a self, + parser: fn(ParseStream<'a>) -> Result<T>, + separator: P, + ) -> Result<Punctuated<T, P::Token>> + where + P: Peek, + P::Token: Parse, + { + let _ = separator; + Punctuated::parse_terminated_with(self, parser) + } + + /// Returns whether there are no more tokens remaining to be parsed from + /// this stream. + /// + /// This method returns true upon reaching the end of the content within a + /// set of delimiters, as well as at the end of the tokens provided to the + /// outermost parsing entry point. + /// + /// This is equivalent to + /// <code>.<a href="#method.peek">peek</a>(<a href="struct.End.html">syn::parse::End</a>)</code>. + /// Use `.peek2(End)` or `.peek3(End)` to look for the end of a parse stream + /// further ahead than the current position. + /// + /// # Example + /// + /// ``` + /// use syn::{braced, token, Ident, Item, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parses a Rust `mod m { ... }` containing zero or more items. + /// struct Mod { + /// mod_token: Token![mod], + /// name: Ident, + /// brace_token: token::Brace, + /// items: Vec<Item>, + /// } + /// + /// impl Parse for Mod { + /// fn parse(input: ParseStream) -> Result<Self> { + /// let content; + /// Ok(Mod { + /// mod_token: input.parse()?, + /// name: input.parse()?, + /// brace_token: braced!(content in input), + /// items: { + /// let mut items = Vec::new(); + /// while !content.is_empty() { + /// items.push(content.parse()?); + /// } + /// items + /// }, + /// }) + /// } + /// } + /// ``` + pub fn is_empty(&self) -> bool { + self.cursor().eof() + } + + /// Constructs a helper for peeking at the next token in this stream and + /// building an error message if it is not one of a set of expected tokens. + /// + /// # Example + /// + /// ``` + /// use syn::{ConstParam, Ident, Lifetime, LifetimeParam, Result, Token, TypeParam}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // A generic parameter, a single one of the comma-separated elements inside + /// // angle brackets in: + /// // + /// // fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... } + /// // + /// // On invalid input, lookahead gives us a reasonable error message. + /// // + /// // error: expected one of: identifier, lifetime, `const` + /// // | + /// // 5 | fn f<!Sized>() {} + /// // | ^ + /// enum GenericParam { + /// Type(TypeParam), + /// Lifetime(LifetimeParam), + /// Const(ConstParam), + /// } + /// + /// impl Parse for GenericParam { + /// fn parse(input: ParseStream) -> Result<Self> { + /// let lookahead = input.lookahead1(); + /// if lookahead.peek(Ident) { + /// input.parse().map(GenericParam::Type) + /// } else if lookahead.peek(Lifetime) { + /// input.parse().map(GenericParam::Lifetime) + /// } else if lookahead.peek(Token![const]) { + /// input.parse().map(GenericParam::Const) + /// } else { + /// Err(lookahead.error()) + /// } + /// } + /// } + /// ``` + pub fn lookahead1(&self) -> Lookahead1<'a> { + lookahead::new(self.scope, self.cursor()) + } + + /// Forks a parse stream so that parsing tokens out of either the original + /// or the fork does not advance the position of the other. + /// + /// # Performance + /// + /// Forking a parse stream is a cheap fixed amount of work and does not + /// involve copying token buffers. Where you might hit performance problems + /// is if your macro ends up parsing a large amount of content more than + /// once. + /// + /// ``` + /// # use syn::{Expr, Result}; + /// # use syn::parse::ParseStream; + /// # + /// # fn bad(input: ParseStream) -> Result<Expr> { + /// // Do not do this. + /// if input.fork().parse::<Expr>().is_ok() { + /// return input.parse::<Expr>(); + /// } + /// # unimplemented!() + /// # } + /// ``` + /// + /// As a rule, avoid parsing an unbounded amount of tokens out of a forked + /// parse stream. Only use a fork when the amount of work performed against + /// the fork is small and bounded. + /// + /// When complex speculative parsing against the forked stream is + /// unavoidable, use [`parse::discouraged::Speculative`] to advance the + /// original stream once the fork's parse is determined to have been + /// successful. + /// + /// For a lower level way to perform speculative parsing at the token level, + /// consider using [`ParseStream::step`] instead. + /// + /// [`parse::discouraged::Speculative`]: discouraged::Speculative + /// [`ParseStream::step`]: ParseBuffer::step + /// + /// # Example + /// + /// The parse implementation shown here parses possibly restricted `pub` + /// visibilities. + /// + /// - `pub` + /// - `pub(crate)` + /// - `pub(self)` + /// - `pub(super)` + /// - `pub(in some::path)` + /// + /// To handle the case of visibilities inside of tuple structs, the parser + /// needs to distinguish parentheses that specify visibility restrictions + /// from parentheses that form part of a tuple type. + /// + /// ``` + /// # struct A; + /// # struct B; + /// # struct C; + /// # + /// struct S(pub(crate) A, pub (B, C)); + /// ``` + /// + /// In this example input the first tuple struct element of `S` has + /// `pub(crate)` visibility while the second tuple struct element has `pub` + /// visibility; the parentheses around `(B, C)` are part of the type rather + /// than part of a visibility restriction. + /// + /// The parser uses a forked parse stream to check the first token inside of + /// parentheses after the `pub` keyword. This is a small bounded amount of + /// work performed against the forked parse stream. + /// + /// ``` + /// use syn::{parenthesized, token, Ident, Path, Result, Token}; + /// use syn::ext::IdentExt; + /// use syn::parse::{Parse, ParseStream}; + /// + /// struct PubVisibility { + /// pub_token: Token![pub], + /// restricted: Option<Restricted>, + /// } + /// + /// struct Restricted { + /// paren_token: token::Paren, + /// in_token: Option<Token![in]>, + /// path: Path, + /// } + /// + /// impl Parse for PubVisibility { + /// fn parse(input: ParseStream) -> Result<Self> { + /// let pub_token: Token![pub] = input.parse()?; + /// + /// if input.peek(token::Paren) { + /// let ahead = input.fork(); + /// let mut content; + /// parenthesized!(content in ahead); + /// + /// if content.peek(Token![crate]) + /// || content.peek(Token![self]) + /// || content.peek(Token![super]) + /// { + /// return Ok(PubVisibility { + /// pub_token, + /// restricted: Some(Restricted { + /// paren_token: parenthesized!(content in input), + /// in_token: None, + /// path: Path::from(content.call(Ident::parse_any)?), + /// }), + /// }); + /// } else if content.peek(Token![in]) { + /// return Ok(PubVisibility { + /// pub_token, + /// restricted: Some(Restricted { + /// paren_token: parenthesized!(content in input), + /// in_token: Some(content.parse()?), + /// path: content.call(Path::parse_mod_style)?, + /// }), + /// }); + /// } + /// } + /// + /// Ok(PubVisibility { + /// pub_token, + /// restricted: None, + /// }) + /// } + /// } + /// ``` + pub fn fork(&self) -> Self { + ParseBuffer { + scope: self.scope, + cell: self.cell.clone(), + marker: PhantomData, + // Not the parent's unexpected. Nothing cares whether the clone + // parses all the way unless we `advance_to`. + unexpected: Cell::new(Some(Rc::new(Cell::new(Unexpected::None)))), + } + } + + /// Triggers an error at the current position of the parse stream. + /// + /// # Example + /// + /// ``` + /// use syn::{Expr, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Some kind of loop: `while` or `for` or `loop`. + /// struct Loop { + /// expr: Expr, + /// } + /// + /// impl Parse for Loop { + /// fn parse(input: ParseStream) -> Result<Self> { + /// if input.peek(Token![while]) + /// || input.peek(Token![for]) + /// || input.peek(Token![loop]) + /// { + /// Ok(Loop { + /// expr: input.parse()?, + /// }) + /// } else { + /// Err(input.error("expected some kind of loop")) + /// } + /// } + /// } + /// ``` + pub fn error<T: Display>(&self, message: T) -> Error { + error::new_at(self.scope, self.cursor(), message) + } + + /// Speculatively parses tokens from this parse stream, advancing the + /// position of this stream only if parsing succeeds. + /// + /// This is a powerful low-level API used for defining the `Parse` impls of + /// the basic built-in token types. It is not something that will be used + /// widely outside of the Syn codebase. + /// + /// # Example + /// + /// ``` + /// use proc_macro2::TokenTree; + /// use syn::Result; + /// use syn::parse::ParseStream; + /// + /// // This function advances the stream past the next occurrence of `@`. If + /// // no `@` is present in the stream, the stream position is unchanged and + /// // an error is returned. + /// fn skip_past_next_at(input: ParseStream) -> Result<()> { + /// input.step(|cursor| { + /// let mut rest = *cursor; + /// while let Some((tt, next)) = rest.token_tree() { + /// match &tt { + /// TokenTree::Punct(punct) if punct.as_char() == '@' => { + /// return Ok(((), next)); + /// } + /// _ => rest = next, + /// } + /// } + /// Err(cursor.error("no `@` was found after this point")) + /// }) + /// } + /// # + /// # fn remainder_after_skipping_past_next_at( + /// # input: ParseStream, + /// # ) -> Result<proc_macro2::TokenStream> { + /// # skip_past_next_at(input)?; + /// # input.parse() + /// # } + /// # + /// # use syn::parse::Parser; + /// # let remainder = remainder_after_skipping_past_next_at + /// # .parse_str("a @ b c") + /// # .unwrap(); + /// # assert_eq!(remainder.to_string(), "b c"); + /// ``` + pub fn step<F, R>(&self, function: F) -> Result<R> + where + F: for<'c> FnOnce(StepCursor<'c, 'a>) -> Result<(R, Cursor<'c>)>, + { + // Since the user's function is required to work for any 'c, we know + // that the Cursor<'c> they return is either derived from the input + // StepCursor<'c, 'a> or from a Cursor<'static>. + // + // It would not be legal to write this function without the invariant + // lifetime 'c in StepCursor<'c, 'a>. If this function were written only + // in terms of 'a, the user could take our ParseBuffer<'a>, upcast it to + // a ParseBuffer<'short> which some shorter lifetime than 'a, invoke + // `step` on their ParseBuffer<'short> with a closure that returns + // Cursor<'short>, and we would wrongly write that Cursor<'short> into + // the Cell intended to hold Cursor<'a>. + // + // In some cases it may be necessary for R to contain a Cursor<'a>. + // Within Syn we solve this using `advance_step_cursor` which uses the + // existence of a StepCursor<'c, 'a> as proof that it is safe to cast + // from Cursor<'c> to Cursor<'a>. If needed outside of Syn, it would be + // safe to expose that API as a method on StepCursor. + let (node, rest) = function(StepCursor { + scope: self.scope, + cursor: self.cell.get(), + marker: PhantomData, + })?; + self.cell.set(rest); + Ok(node) + } + + /// Returns the `Span` of the next token in the parse stream, or + /// `Span::call_site()` if this parse stream has completely exhausted its + /// input `TokenStream`. + pub fn span(&self) -> Span { + let cursor = self.cursor(); + if cursor.eof() { + self.scope + } else { + crate::buffer::open_span_of_group(cursor) + } + } + + /// Provides low-level access to the token representation underlying this + /// parse stream. + /// + /// Cursors are immutable so no operations you perform against the cursor + /// will affect the state of this parse stream. + /// + /// # Example + /// + /// ``` + /// use proc_macro2::TokenStream; + /// use syn::buffer::Cursor; + /// use syn::parse::{ParseStream, Result}; + /// + /// // Run a parser that returns T, but get its output as TokenStream instead of T. + /// // This works without T needing to implement ToTokens. + /// fn recognize_token_stream<T>( + /// recognizer: fn(ParseStream) -> Result<T>, + /// ) -> impl Fn(ParseStream) -> Result<TokenStream> { + /// move |input| { + /// let begin = input.cursor(); + /// recognizer(input)?; + /// let end = input.cursor(); + /// Ok(tokens_between(begin, end)) + /// } + /// } + /// + /// // Collect tokens between two cursors as a TokenStream. + /// fn tokens_between(begin: Cursor, end: Cursor) -> TokenStream { + /// assert!(begin <= end); + /// + /// let mut cursor = begin; + /// let mut tokens = TokenStream::new(); + /// while cursor < end { + /// let (token, next) = cursor.token_tree().unwrap(); + /// tokens.extend(core::iter::once(token)); + /// cursor = next; + /// } + /// tokens + /// } + /// + /// fn main() { + /// use quote::quote; + /// use syn::parse::{Parse, Parser}; + /// use syn::Token; + /// + /// // Parse syn::Type as a TokenStream, surrounded by angle brackets. + /// fn example(input: ParseStream) -> Result<TokenStream> { + /// let _langle: Token![<] = input.parse()?; + /// let ty = recognize_token_stream(syn::Type::parse)(input)?; + /// let _rangle: Token![>] = input.parse()?; + /// Ok(ty) + /// } + /// + /// let tokens = quote! { <fn() -> u8> }; + /// println!("{}", example.parse2(tokens).unwrap()); + /// } + /// ``` + pub fn cursor(&self) -> Cursor<'a> { + self.cell.get() + } + + fn check_unexpected(&self) -> Result<()> { + match inner_unexpected(self).1 { + Some((span, delimiter)) => Err(err_unexpected_token(span, delimiter)), + None => Ok(()), + } + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +impl<T: Parse> Parse for Box<T> { + fn parse(input: ParseStream) -> Result<Self> { + input.parse().map(Box::new) + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +impl<T: Parse + Token> Parse for Option<T> { + fn parse(input: ParseStream) -> Result<Self> { + if T::peek(input.cursor()) { + Ok(Some(input.parse()?)) + } else { + Ok(None) + } + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +impl Parse for TokenStream { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| Ok((cursor.token_stream(), Cursor::empty()))) + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +impl Parse for TokenTree { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| match cursor.token_tree() { + Some((tt, rest)) => Ok((tt, rest)), + None => Err(cursor.error("expected token tree")), + }) + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +impl Parse for Group { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| { + if let Some((group, rest)) = cursor.any_group_token() { + if group.delimiter() != Delimiter::None { + return Ok((group, rest)); + } + } + Err(cursor.error("expected group token")) + }) + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +impl Parse for Punct { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| match cursor.punct() { + Some((punct, rest)) => Ok((punct, rest)), + None => Err(cursor.error("expected punctuation token")), + }) + } +} + +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +impl Parse for Literal { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| match cursor.literal() { + Some((literal, rest)) => Ok((literal, rest)), + None => Err(cursor.error("expected literal token")), + }) + } +} + +/// Parser that can parse Rust tokens into a particular syntax tree node. +/// +/// Refer to the [module documentation] for details about parsing in Syn. +/// +/// [module documentation]: self +pub trait Parser: Sized { + type Output; + + /// Parse a proc-macro2 token stream into the chosen syntax tree node. + /// + /// This function enforces that the input is fully parsed. If there are any + /// unparsed tokens at the end of the stream, an error is returned. + fn parse2(self, tokens: TokenStream) -> Result<Self::Output>; + + /// Parse tokens of source code into the chosen syntax tree node. + /// + /// This function enforces that the input is fully parsed. If there are any + /// unparsed tokens at the end of the stream, an error is returned. + #[cfg(feature = "proc-macro")] + #[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] + fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> { + self.parse2(proc_macro2::TokenStream::from(tokens)) + } + + /// Parse a string of Rust code into the chosen syntax tree node. + /// + /// This function enforces that the input is fully parsed. If there are any + /// unparsed tokens at the end of the string, an error is returned. + /// + /// # Hygiene + /// + /// Every span in the resulting syntax tree will be set to resolve at the + /// macro call site. + fn parse_str(self, s: &str) -> Result<Self::Output> { + self.parse2(proc_macro2::TokenStream::from_str(s)?) + } + + // Not public API. + #[doc(hidden)] + fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> { + let _ = scope; + self.parse2(tokens) + } +} + +fn tokens_to_parse_buffer(tokens: &TokenBuffer) -> ParseBuffer { + let scope = Span::call_site(); + let cursor = tokens.begin(); + let unexpected = Rc::new(Cell::new(Unexpected::None)); + new_parse_buffer(scope, cursor, unexpected) +} + +impl<F, T> Parser for F +where + F: FnOnce(ParseStream) -> Result<T>, +{ + type Output = T; + + fn parse2(self, tokens: TokenStream) -> Result<T> { + let buf = TokenBuffer::new2(tokens); + let state = tokens_to_parse_buffer(&buf); + let node = self(&state)?; + state.check_unexpected()?; + if let Some((unexpected_span, delimiter)) = + span_of_unexpected_ignoring_nones(state.cursor()) + { + Err(err_unexpected_token(unexpected_span, delimiter)) + } else { + Ok(node) + } + } + + fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> { + let buf = TokenBuffer::new2(tokens); + let cursor = buf.begin(); + let unexpected = Rc::new(Cell::new(Unexpected::None)); + let state = new_parse_buffer(scope, cursor, unexpected); + let node = self(&state)?; + state.check_unexpected()?; + if let Some((unexpected_span, delimiter)) = + span_of_unexpected_ignoring_nones(state.cursor()) + { + Err(err_unexpected_token(unexpected_span, delimiter)) + } else { + Ok(node) + } + } +} + +pub(crate) fn parse_scoped<F: Parser>(f: F, scope: Span, tokens: TokenStream) -> Result<F::Output> { + f.__parse_scoped(scope, tokens) +} + +fn err_unexpected_token(span: Span, delimiter: Delimiter) -> Error { + let msg = match delimiter { + Delimiter::Parenthesis => "unexpected token, expected `)`", + Delimiter::Brace => "unexpected token, expected `}`", + Delimiter::Bracket => "unexpected token, expected `]`", + Delimiter::None => "unexpected token", + }; + Error::new(span, msg) +} + +/// An empty syntax tree node that consumes no tokens when parsed. +/// +/// This is useful for attribute macros that want to ensure they are not +/// provided any attribute args. +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::parse_macro_input; +/// use syn::parse::Nothing; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { +/// parse_macro_input!(args as Nothing); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// ``` +/// +/// ```text +/// error: unexpected token +/// --> src/main.rs:3:19 +/// | +/// 3 | #[my_attr(asdf)] +/// | ^^^^ +/// ``` +pub struct Nothing; + +impl Parse for Nothing { + fn parse(_input: ParseStream) -> Result<Self> { + Ok(Nothing) + } +} + +#[cfg(feature = "printing")] +#[cfg_attr(docsrs, doc(cfg(feature = "printing")))] +impl ToTokens for Nothing { + fn to_tokens(&self, tokens: &mut TokenStream) { + let _ = tokens; + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for Nothing { + fn clone(&self) -> Self { + *self + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Copy for Nothing {} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for Nothing { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("Nothing") + } +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Eq for Nothing {} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for Nothing { + fn eq(&self, _other: &Self) -> bool { + true + } +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for Nothing { + fn hash<H: Hasher>(&self, _state: &mut H) {} +} diff --git a/third_party/crates/syn-2.0.119/src/parse_macro_input.rs b/third_party/crates/syn-2.0.119/src/parse_macro_input.rs new file mode 100644 index 0000000..f0660ae --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/parse_macro_input.rs @@ -0,0 +1,128 @@ +/// Parse the input TokenStream of a macro, triggering a compile error if the +/// tokens fail to parse. +/// +/// Refer to the [`parse` module] documentation for more details about parsing +/// in Syn. +/// +/// [`parse` module]: mod@crate::parse +/// +/// <br> +/// +/// # Intended usage +/// +/// This macro must be called from a function that returns +/// `proc_macro::TokenStream`. Usually this will be your proc macro entry point, +/// the function that has the #\[proc_macro\] / #\[proc_macro_derive\] / +/// #\[proc_macro_attribute\] attribute. +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::{parse_macro_input, Result}; +/// use syn::parse::{Parse, ParseStream}; +/// +/// struct MyMacroInput { +/// /* ... */ +/// } +/// +/// impl Parse for MyMacroInput { +/// fn parse(input: ParseStream) -> Result<Self> { +/// /* ... */ +/// # Ok(MyMacroInput {}) +/// } +/// } +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro] +/// # }; +/// pub fn my_macro(tokens: TokenStream) -> TokenStream { +/// let input = parse_macro_input!(tokens as MyMacroInput); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// ``` +/// +/// <br> +/// +/// # Usage with Parser +/// +/// This macro can also be used with the [`Parser` trait] for types that have +/// multiple ways that they can be parsed. +/// +/// [`Parser` trait]: crate::parse::Parser +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// # use proc_macro::TokenStream; +/// # use syn::{parse_macro_input, Result}; +/// # use syn::parse::ParseStream; +/// # +/// # struct MyMacroInput {} +/// # +/// impl MyMacroInput { +/// fn parse_alternate(input: ParseStream) -> Result<Self> { +/// /* ... */ +/// # Ok(MyMacroInput {}) +/// } +/// } +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro] +/// # }; +/// pub fn my_macro(tokens: TokenStream) -> TokenStream { +/// let input = parse_macro_input!(tokens with MyMacroInput::parse_alternate); +/// +/// /* ... */ +/// # TokenStream::new() +/// } +/// ``` +/// +/// <br> +/// +/// # Expansion +/// +/// `parse_macro_input!($variable as $Type)` expands to something like: +/// +/// ```no_run +/// # extern crate proc_macro; +/// # +/// # macro_rules! doc_test { +/// # ($variable:ident as $Type:ty) => { +/// match syn::parse::<$Type>($variable) { +/// Ok(syntax_tree) => syntax_tree, +/// Err(err) => return proc_macro::TokenStream::from(err.to_compile_error()), +/// } +/// # }; +/// # } +/// # +/// # fn test(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +/// # let _ = doc_test!(input as syn::Ident); +/// # proc_macro::TokenStream::new() +/// # } +/// ``` +#[macro_export] +#[cfg_attr(docsrs, doc(cfg(all(feature = "parsing", feature = "proc-macro"))))] +macro_rules! parse_macro_input { + ($tokenstream:ident as $ty:ty) => { + match $crate::parse::<$ty>($tokenstream) { + $crate::__private::Ok(data) => data, + $crate::__private::Err(err) => { + return $crate::__private::TokenStream::from(err.to_compile_error()); + } + } + }; + ($tokenstream:ident with $parser:path) => { + match $crate::parse::Parser::parse($parser, $tokenstream) { + $crate::__private::Ok(data) => data, + $crate::__private::Err(err) => { + return $crate::__private::TokenStream::from(err.to_compile_error()); + } + } + }; + ($tokenstream:ident) => { + $crate::parse_macro_input!($tokenstream as _) + }; +} diff --git a/third_party/crates/syn-2.0.119/src/parse_quote.rs b/third_party/crates/syn-2.0.119/src/parse_quote.rs new file mode 100644 index 0000000..3e96b2e --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/parse_quote.rs @@ -0,0 +1,244 @@ +/// Quasi-quotation macro that accepts input like the [`quote!`] macro but uses +/// type inference to figure out a return type for those tokens. +/// +/// [`quote!`]: https://docs.rs/quote/1.0/quote/index.html +/// +/// The return type can be any syntax tree node that implements the [`Parse`] +/// trait. +/// +/// [`Parse`]: crate::parse::Parse +/// +/// ``` +/// use quote::quote; +/// use syn::{parse_quote, Stmt}; +/// +/// fn main() { +/// let name = quote!(v); +/// let ty = quote!(u8); +/// +/// let stmt: Stmt = parse_quote! { +/// let #name: #ty = Default::default(); +/// }; +/// +/// println!("{:#?}", stmt); +/// } +/// ``` +/// +/// *This macro is available only if Syn is built with both the `"parsing"` and +/// `"printing"` features.* +/// +/// # Example +/// +/// The following helper function adds a bound `T: HeapSize` to every type +/// parameter `T` in the input generics. +/// +/// ``` +/// use syn::{parse_quote, Generics, GenericParam}; +/// +/// // Add a bound `T: HeapSize` to every type parameter T. +/// fn add_trait_bounds(mut generics: Generics) -> Generics { +/// for param in &mut generics.params { +/// if let GenericParam::Type(type_param) = param { +/// type_param.bounds.push(parse_quote!(HeapSize)); +/// } +/// } +/// generics +/// } +/// ``` +/// +/// # Special cases +/// +/// This macro can parse the following additional types as a special case even +/// though they do not implement the `Parse` trait. +/// +/// - [`Attribute`] — parses one attribute, allowing either outer like `#[...]` +/// or inner like `#![...]` +/// - [`Vec<Attribute>`] — parses multiple attributes, including mixed kinds in +/// any order +/// - [`Punctuated<T, P>`] — parses zero or more `T` separated by punctuation +/// `P` with optional trailing punctuation +/// - [`Vec<Arm>`] — parses arms separated by optional commas according to the +/// same grammar as the inside of a `match` expression +/// - [`Vec<Stmt>`] — parses the same as `Block::parse_within` +/// - [`Pat`], [`Box<Pat>`] — parses the same as +/// `Pat::parse_multi_with_leading_vert` +/// - [`Field`] — parses a named or unnamed struct field +/// +/// [`Vec<Attribute>`]: Attribute +/// [`Vec<Arm>`]: Arm +/// [`Vec<Stmt>`]: Block::parse_within +/// [`Pat`]: Pat::parse_multi_with_leading_vert +/// [`Box<Pat>`]: Pat::parse_multi_with_leading_vert +/// +/// # Panics +/// +/// Panics if the tokens fail to parse as the expected syntax tree type. The +/// caller is responsible for ensuring that the input tokens are syntactically +/// valid. +#[cfg_attr(docsrs, doc(cfg(all(feature = "parsing", feature = "printing"))))] +#[macro_export] +macro_rules! parse_quote { + ($($tt:tt)*) => { + $crate::__private::parse_quote($crate::__private::quote::quote!($($tt)*)) + }; +} + +/// This macro is [`parse_quote!`] + [`quote_spanned!`][quote::quote_spanned]. +/// +/// Please refer to each of their documentation. +/// +/// # Example +/// +/// ``` +/// use quote::{quote, quote_spanned}; +/// use syn::spanned::Spanned; +/// use syn::{parse_quote_spanned, ReturnType, Signature}; +/// +/// // Changes `fn()` to `fn() -> Pin<Box<dyn Future<Output = ()>>>`, +/// // and `fn() -> T` to `fn() -> Pin<Box<dyn Future<Output = T>>>`, +/// // without introducing any call_site() spans. +/// fn make_ret_pinned_future(sig: &mut Signature) { +/// let ret = match &sig.output { +/// ReturnType::Default => quote_spanned!(sig.paren_token.span=> ()), +/// ReturnType::Type(_, ret) => quote!(#ret), +/// }; +/// sig.output = parse_quote_spanned! {ret.span()=> +/// -> ::core::pin::Pin<::alloc::boxed::Box<dyn ::core::future::Future<Output = #ret>>> +/// }; +/// } +/// ``` +#[cfg_attr(docsrs, doc(cfg(all(feature = "parsing", feature = "printing"))))] +#[macro_export] +macro_rules! parse_quote_spanned { + ($span:expr=> $($tt:tt)*) => { + $crate::__private::parse_quote($crate::__private::quote::quote_spanned!($span=> $($tt)*)) + }; +} + +//////////////////////////////////////////////////////////////////////////////// +// Can parse any type that implements Parse. + +use crate::error::Result; +use crate::parse::{Parse, ParseStream, Parser}; +#[cfg(feature = "full")] +use alloc::boxed::Box; +#[cfg(any(feature = "full", feature = "derive"))] +use alloc::vec::Vec; +use proc_macro2::TokenStream; + +// Not public API. +#[doc(hidden)] +#[track_caller] +pub fn parse<T: ParseQuote>(token_stream: TokenStream) -> T { + let parser = T::parse; + match parser.parse2(token_stream) { + Ok(t) => t, + Err(err) => panic!("{}", err), + } +} + +#[doc(hidden)] +pub trait ParseQuote: Sized { + fn parse(input: ParseStream) -> Result<Self>; +} + +impl<T: Parse> ParseQuote for T { + fn parse(input: ParseStream) -> Result<Self> { + <T as Parse>::parse(input) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Any other types that we want `parse_quote!` to be able to parse. + +use crate::punctuated::Punctuated; +#[cfg(any(feature = "full", feature = "derive"))] +use crate::{attr, Attribute, Field, FieldMutability, Ident, Type, Visibility}; +#[cfg(feature = "full")] +use crate::{Arm, Block, Pat, Stmt}; + +#[cfg(any(feature = "full", feature = "derive"))] +impl ParseQuote for Attribute { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Token![#]) && input.peek2(Token![!]) { + attr::parsing::single_parse_inner(input) + } else { + attr::parsing::single_parse_outer(input) + } + } +} + +#[cfg(any(feature = "full", feature = "derive"))] +impl ParseQuote for Vec<Attribute> { + fn parse(input: ParseStream) -> Result<Self> { + let mut attrs = Vec::new(); + while !input.is_empty() { + attrs.push(ParseQuote::parse(input)?); + } + Ok(attrs) + } +} + +#[cfg(any(feature = "full", feature = "derive"))] +impl ParseQuote for Field { + fn parse(input: ParseStream) -> Result<Self> { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + + let ident: Option<Ident>; + let colon_token: Option<Token![:]>; + let is_named = input.peek(Ident) && input.peek2(Token![:]) && !input.peek2(Token![::]); + if is_named { + ident = Some(input.parse()?); + colon_token = Some(input.parse()?); + } else { + ident = None; + colon_token = None; + } + + let ty: Type = input.parse()?; + + Ok(Field { + attrs, + vis, + mutability: FieldMutability::None, + ident, + colon_token, + ty, + }) + } +} + +#[cfg(feature = "full")] +impl ParseQuote for Pat { + fn parse(input: ParseStream) -> Result<Self> { + Pat::parse_multi_with_leading_vert(input) + } +} + +#[cfg(feature = "full")] +impl ParseQuote for Box<Pat> { + fn parse(input: ParseStream) -> Result<Self> { + <Pat as ParseQuote>::parse(input).map(Box::new) + } +} + +impl<T: Parse, P: Parse> ParseQuote for Punctuated<T, P> { + fn parse(input: ParseStream) -> Result<Self> { + Self::parse_terminated(input) + } +} + +#[cfg(feature = "full")] +impl ParseQuote for Vec<Stmt> { + fn parse(input: ParseStream) -> Result<Self> { + Block::parse_within(input) + } +} + +#[cfg(feature = "full")] +impl ParseQuote for Vec<Arm> { + fn parse(input: ParseStream) -> Result<Self> { + Arm::parse_multiple(input) + } +} diff --git a/third_party/crates/syn-2.0.119/src/pat.rs b/third_party/crates/syn-2.0.119/src/pat.rs new file mode 100644 index 0000000..08f9c4c --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/pat.rs @@ -0,0 +1,960 @@ +use crate::attr::Attribute; +use crate::expr::Member; +use crate::ident::Ident; +use crate::path::{Path, QSelf}; +use crate::punctuated::Punctuated; +use crate::token; +use crate::ty::Type; +use alloc::boxed::Box; +use alloc::vec::Vec; +use proc_macro2::TokenStream; + +pub use crate::expr::{ + ExprConst as PatConst, ExprLit as PatLit, ExprMacro as PatMacro, ExprPath as PatPath, + ExprRange as PatRange, +}; + +ast_enum_of_structs! { + /// A pattern in a local binding, function signature, match expression, or + /// various other places. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + #[non_exhaustive] + pub enum Pat { + /// A const block: `const { ... }`. + Const(PatConst), + + /// A pattern that binds a new variable: `ref mut binding @ SUBPATTERN`. + Ident(PatIdent), + + /// A literal pattern: `0`. + Lit(PatLit), + + /// A macro in pattern position. + Macro(PatMacro), + + /// A pattern that matches any one of a set of cases. + Or(PatOr), + + /// A parenthesized pattern: `(A | B)`. + Paren(PatParen), + + /// A path pattern like `Color::Red`, optionally qualified with a + /// self-type. + /// + /// Unqualified path patterns can legally refer to variants, structs, + /// constants or associated constants. Qualified path patterns like + /// `<A>::B::C` and `<A as Trait>::B::C` can only legally refer to + /// associated constants. + Path(PatPath), + + /// A range pattern: `1..=2`. + Range(PatRange), + + /// A reference pattern: `&mut var`. + Reference(PatReference), + + /// The dots in a tuple or slice pattern: `[0, 1, ..]`. + Rest(PatRest), + + /// A dynamically sized slice pattern: `[a, b, ref i @ .., y, z]`. + Slice(PatSlice), + + /// A struct or struct variant pattern: `Variant { x, y, .. }`. + Struct(PatStruct), + + /// A tuple pattern: `(a, b)`. + Tuple(PatTuple), + + /// A tuple struct or tuple variant pattern: `Variant(x, y, .., z)`. + TupleStruct(PatTupleStruct), + + /// A type ascription pattern: `foo: f64`. + Type(PatType), + + /// Tokens in pattern position not interpreted by Syn. + Verbatim(TokenStream), + + /// A pattern that matches any value: `_`. + Wild(PatWild), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match pat { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // Pat::Box(pat) => {...} + // Pat::Ident(pat) => {...} + // ... + // Pat::Wild(pat) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// A pattern that binds a new variable: `ref mut binding @ SUBPATTERN`. + /// + /// It may also be a unit struct or struct variant (e.g. `None`), or a + /// constant; these cannot be distinguished syntactically. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatIdent { + pub attrs: Vec<Attribute>, + pub by_ref: Option<Token![ref]>, + pub mutability: Option<Token![mut]>, + pub ident: Ident, + pub subpat: Option<(Token![@], Box<Pat>)>, + } +} + +ast_struct! { + /// A pattern that matches any one of a set of cases. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatOr { + pub attrs: Vec<Attribute>, + pub leading_vert: Option<Token![|]>, + pub cases: Punctuated<Pat, Token![|]>, + } +} + +ast_struct! { + /// A parenthesized pattern: `(A | B)`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatParen { + pub attrs: Vec<Attribute>, + pub paren_token: token::Paren, + pub pat: Box<Pat>, + } +} + +ast_struct! { + /// A reference pattern: `&mut var`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatReference { + pub attrs: Vec<Attribute>, + pub and_token: Token![&], + pub mutability: Option<Token![mut]>, + pub pat: Box<Pat>, + } +} + +ast_struct! { + /// The dots in a tuple or slice pattern: `[0, 1, ..]`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatRest { + pub attrs: Vec<Attribute>, + pub dot2_token: Token![..], + } +} + +ast_struct! { + /// A dynamically sized slice pattern: `[a, b, ref i @ .., y, z]`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatSlice { + pub attrs: Vec<Attribute>, + pub bracket_token: token::Bracket, + pub elems: Punctuated<Pat, Token![,]>, + } +} + +ast_struct! { + /// A struct or struct variant pattern: `Variant { x, y, .. }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatStruct { + pub attrs: Vec<Attribute>, + pub qself: Option<QSelf>, + pub path: Path, + pub brace_token: token::Brace, + pub fields: Punctuated<FieldPat, Token![,]>, + pub rest: Option<PatRest>, + } +} + +ast_struct! { + /// A tuple pattern: `(a, b)`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatTuple { + pub attrs: Vec<Attribute>, + pub paren_token: token::Paren, + pub elems: Punctuated<Pat, Token![,]>, + } +} + +ast_struct! { + /// A tuple struct or tuple variant pattern: `Variant(x, y, .., z)`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatTupleStruct { + pub attrs: Vec<Attribute>, + pub qself: Option<QSelf>, + pub path: Path, + pub paren_token: token::Paren, + pub elems: Punctuated<Pat, Token![,]>, + } +} + +ast_struct! { + /// A type ascription pattern: `foo: f64`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatType { + pub attrs: Vec<Attribute>, + pub pat: Box<Pat>, + pub colon_token: Token![:], + pub ty: Box<Type>, + } +} + +ast_struct! { + /// A pattern that matches any value: `_`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct PatWild { + pub attrs: Vec<Attribute>, + pub underscore_token: Token![_], + } +} + +ast_struct! { + /// A single field in a struct pattern. + /// + /// Patterns like the fields of Foo `{ x, ref y, ref mut z }` are treated + /// the same as `x: x, y: ref y, z: ref mut z` but there is no colon token. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct FieldPat { + pub attrs: Vec<Attribute>, + pub member: Member, + pub colon_token: Option<Token![:]>, + pub pat: Box<Pat>, + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::buffer::Cursor; + use crate::error::{self, Result}; + use crate::expr::{ + Expr, ExprConst, ExprLit, ExprMacro, ExprPath, ExprRange, Member, RangeLimits, + }; + use crate::ext::IdentExt as _; + use crate::ident::Ident; + use crate::lit::Lit; + use crate::mac::{self, Macro}; + use crate::parse::{Parse, ParseStream}; + use crate::pat::{ + FieldPat, Pat, PatIdent, PatOr, PatParen, PatReference, PatRest, PatSlice, PatStruct, + PatTuple, PatTupleStruct, PatType, PatWild, + }; + use crate::path::{self, Path, QSelf}; + use crate::punctuated::Punctuated; + use crate::stmt::Block; + use crate::token; + use crate::verbatim; + use alloc::boxed::Box; + use alloc::vec::Vec; + use proc_macro2::TokenStream; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Pat { + /// Parse a pattern that does _not_ involve `|` at the top level. + /// + /// This parser matches the behavior of the `$:pat_param` macro_rules + /// matcher, and on editions prior to Rust 2021, the behavior of + /// `$:pat`. + /// + /// In Rust syntax, some examples of where this syntax would occur are + /// in the argument pattern of functions and closures. Patterns using + /// `|` are not allowed to occur in these positions. + /// + /// ```compile_fail + /// fn f(Some(_) | None: Option<T>) { + /// let _ = |Some(_) | None: Option<T>| {}; + /// // ^^^^^^^^^^^^^^^^^^^^^^^^^??? :( + /// } + /// ``` + /// + /// ```console + /// error: top-level or-patterns are not allowed in function parameters + /// --> src/main.rs:1:6 + /// | + /// 1 | fn f(Some(_) | None: Option<T>) { + /// | ^^^^^^^^^^^^^^ help: wrap the pattern in parentheses: `(Some(_) | None)` + /// ``` + pub fn parse_single(input: ParseStream) -> Result<Self> { + let begin = input.cursor(); + let lookahead = input.lookahead1(); + if lookahead.peek(Ident) + && (input.peek2(Token![::]) + || input.peek2(Token![!]) + || input.peek2(token::Brace) + || input.peek2(token::Paren) + || input.peek2(Token![..])) + || input.peek(Token![self]) && input.peek2(Token![::]) + || lookahead.peek(Token![::]) + || lookahead.peek(Token![<]) + || input.peek(Token![Self]) + || input.peek(Token![super]) + || input.peek(Token![crate]) + { + pat_path_or_macro_or_struct_or_range(input) + } else if lookahead.peek(Token![_]) { + input.call(pat_wild).map(Pat::Wild) + } else if input.peek(Token![box]) { + pat_box(begin, input) + } else if input.peek(Token![-]) || lookahead.peek(Lit) || lookahead.peek(Token![const]) + { + pat_lit_or_range(input) + } else if lookahead.peek(Token![ref]) + || lookahead.peek(Token![mut]) + || input.peek(Token![self]) + || input.peek(Ident) + { + input.call(pat_ident).map(Pat::Ident) + } else if lookahead.peek(Token![&]) { + input.call(pat_reference).map(Pat::Reference) + } else if lookahead.peek(token::Paren) { + input.call(pat_paren_or_tuple) + } else if lookahead.peek(token::Bracket) { + input.call(pat_slice).map(Pat::Slice) + } else if lookahead.peek(Token![..]) && !input.peek(Token![...]) { + pat_range_half_open(input) + } else if lookahead.peek(Token![const]) { + input.call(pat_const).map(Pat::Verbatim) + } else { + Err(lookahead.error()) + } + } + + /// Parse a pattern, possibly involving `|`, but not a leading `|`. + pub fn parse_multi(input: ParseStream) -> Result<Self> { + multi_pat_impl(input, None) + } + + /// Parse a pattern, possibly involving `|`, possibly including a + /// leading `|`. + /// + /// This parser matches the behavior of the Rust 2021 edition's `$:pat` + /// macro_rules matcher. + /// + /// In Rust syntax, an example of where this syntax would occur is in + /// the pattern of a `match` arm, where the language permits an optional + /// leading `|`, although it is not idiomatic to write one there in + /// handwritten code. + /// + /// ``` + /// # let wat = None; + /// match wat { + /// | None | Some(false) => {} + /// | Some(true) => {} + /// } + /// ``` + /// + /// The compiler accepts it only to facilitate some situations in + /// macro-generated code where a macro author might need to write: + /// + /// ``` + /// # macro_rules! doc { + /// # ($value:expr, ($($conditions1:pat),*), ($($conditions2:pat),*), $then:expr) => { + /// match $value { + /// $(| $conditions1)* $(| $conditions2)* => $then + /// } + /// # }; + /// # } + /// # + /// # doc!(true, (true), (false), {}); + /// # doc!(true, (), (true, false), {}); + /// # doc!(true, (true, false), (), {}); + /// ``` + /// + /// Expressing the same thing correctly in the case that either one (but + /// not both) of `$conditions1` and `$conditions2` might be empty, + /// without leading `|`, is complex. + /// + /// Use [`Pat::parse_multi`] instead if you are not intending to support + /// macro-generated macro input. + pub fn parse_multi_with_leading_vert(input: ParseStream) -> Result<Self> { + let leading_vert: Option<Token![|]> = input.parse()?; + multi_pat_impl(input, leading_vert) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for PatType { + fn parse(input: ParseStream) -> Result<Self> { + Ok(PatType { + attrs: Vec::new(), + pat: Box::new(Pat::parse_single(input)?), + colon_token: input.parse()?, + ty: input.parse()?, + }) + } + } + + fn multi_pat_impl(input: ParseStream, leading_vert: Option<Token![|]>) -> Result<Pat> { + let mut pat = Pat::parse_single(input)?; + if leading_vert.is_some() + || input.peek(Token![|]) && !input.peek(Token![||]) && !input.peek(Token![|=]) + { + let mut cases = Punctuated::new(); + cases.push_value(pat); + while input.peek(Token![|]) && !input.peek(Token![||]) && !input.peek(Token![|=]) { + let punct = input.parse()?; + cases.push_punct(punct); + let pat = Pat::parse_single(input)?; + cases.push_value(pat); + } + pat = Pat::Or(PatOr { + attrs: Vec::new(), + leading_vert, + cases, + }); + } + Ok(pat) + } + + fn pat_path_or_macro_or_struct_or_range(input: ParseStream) -> Result<Pat> { + let expr_style = true; + let (qself, path) = path::parsing::qpath(input, expr_style)?; + + if qself.is_none() + && input.peek(Token![!]) + && !input.peek(Token![!=]) + && path.is_mod_style() + { + let bang_token: Token![!] = input.parse()?; + let (delimiter, tokens) = mac::parse_delimiter(input)?; + return Ok(Pat::Macro(ExprMacro { + attrs: Vec::new(), + mac: Macro { + path, + bang_token, + delimiter, + tokens, + }, + })); + } + + if input.peek(token::Brace) { + pat_struct(input, qself, path).map(Pat::Struct) + } else if input.peek(token::Paren) { + pat_tuple_struct(input, qself, path).map(Pat::TupleStruct) + } else if input.peek(Token![..]) { + pat_range(input, qself, path) + } else { + Ok(Pat::Path(ExprPath { + attrs: Vec::new(), + qself, + path, + })) + } + } + + fn pat_wild(input: ParseStream) -> Result<PatWild> { + Ok(PatWild { + attrs: Vec::new(), + underscore_token: input.parse()?, + }) + } + + fn pat_box(begin: Cursor, input: ParseStream) -> Result<Pat> { + input.parse::<Token![box]>()?; + Pat::parse_single(input)?; + Ok(Pat::Verbatim(verbatim::between(begin, input.cursor()))) + } + + fn pat_ident(input: ParseStream) -> Result<PatIdent> { + Ok(PatIdent { + attrs: Vec::new(), + by_ref: input.parse()?, + mutability: input.parse()?, + ident: { + if input.peek(Token![self]) { + input.call(Ident::parse_any)? + } else { + input.parse()? + } + }, + subpat: { + if input.peek(Token![@]) { + let at_token: Token![@] = input.parse()?; + let subpat = Pat::parse_single(input)?; + Some((at_token, Box::new(subpat))) + } else { + None + } + }, + }) + } + + fn pat_tuple_struct( + input: ParseStream, + qself: Option<QSelf>, + path: Path, + ) -> Result<PatTupleStruct> { + let content; + let paren_token = parenthesized!(content in input); + + let mut elems = Punctuated::new(); + while !content.is_empty() { + let value = Pat::parse_multi_with_leading_vert(&content)?; + elems.push_value(value); + if content.is_empty() { + break; + } + let punct = content.parse()?; + elems.push_punct(punct); + } + + Ok(PatTupleStruct { + attrs: Vec::new(), + qself, + path, + paren_token, + elems, + }) + } + + fn pat_struct(input: ParseStream, qself: Option<QSelf>, path: Path) -> Result<PatStruct> { + let content; + let brace_token = braced!(content in input); + + let mut fields = Punctuated::new(); + let mut rest = None; + while !content.is_empty() { + let attrs = content.call(Attribute::parse_outer)?; + if content.peek(Token![..]) { + rest = Some(PatRest { + attrs, + dot2_token: content.parse()?, + }); + break; + } + let mut value = content.call(field_pat)?; + value.attrs = attrs; + fields.push_value(value); + if content.is_empty() { + break; + } + let punct: Token![,] = content.parse()?; + fields.push_punct(punct); + } + + Ok(PatStruct { + attrs: Vec::new(), + qself, + path, + brace_token, + fields, + rest, + }) + } + + fn field_pat(input: ParseStream) -> Result<FieldPat> { + let begin = input.cursor(); + let boxed: Option<Token![box]> = input.parse()?; + let by_ref: Option<Token![ref]> = input.parse()?; + let mutability: Option<Token![mut]> = input.parse()?; + + let member = if boxed.is_some() || by_ref.is_some() || mutability.is_some() { + input.parse().map(Member::Named) + } else { + input.parse() + }?; + + if boxed.is_none() && by_ref.is_none() && mutability.is_none() && input.peek(Token![:]) + || !member.is_named() + { + return Ok(FieldPat { + attrs: Vec::new(), + member, + colon_token: Some(input.parse()?), + pat: Box::new(Pat::parse_multi_with_leading_vert(input)?), + }); + } + + let ident = match member { + Member::Named(ident) => ident, + Member::Unnamed(_) => unreachable!(), + }; + + let pat = if boxed.is_some() { + Pat::Verbatim(verbatim::between(begin, input.cursor())) + } else { + Pat::Ident(PatIdent { + attrs: Vec::new(), + by_ref, + mutability, + ident: ident.clone(), + subpat: None, + }) + }; + + Ok(FieldPat { + attrs: Vec::new(), + member: Member::Named(ident), + colon_token: None, + pat: Box::new(pat), + }) + } + + fn pat_range(input: ParseStream, qself: Option<QSelf>, path: Path) -> Result<Pat> { + let limits = RangeLimits::parse_obsolete(input)?; + let end = input.call(pat_range_bound)?; + if let (RangeLimits::Closed(_), None) = (&limits, &end) { + return Err(input.error("expected range upper bound")); + } + Ok(Pat::Range(ExprRange { + attrs: Vec::new(), + start: Some(Box::new(Expr::Path(ExprPath { + attrs: Vec::new(), + qself, + path, + }))), + limits, + end: end.map(PatRangeBound::into_expr), + })) + } + + fn pat_range_half_open(input: ParseStream) -> Result<Pat> { + let limits: RangeLimits = input.parse()?; + let end = input.call(pat_range_bound)?; + if end.is_some() { + Ok(Pat::Range(ExprRange { + attrs: Vec::new(), + start: None, + limits, + end: end.map(PatRangeBound::into_expr), + })) + } else { + match limits { + RangeLimits::HalfOpen(dot2_token) => Ok(Pat::Rest(PatRest { + attrs: Vec::new(), + dot2_token, + })), + RangeLimits::Closed(_) => Err(input.error("expected range upper bound")), + } + } + } + + fn pat_paren_or_tuple(input: ParseStream) -> Result<Pat> { + let content; + let paren_token = parenthesized!(content in input); + + let mut elems = Punctuated::new(); + while !content.is_empty() { + let value = Pat::parse_multi_with_leading_vert(&content)?; + if content.is_empty() { + if elems.is_empty() && !matches!(value, Pat::Rest(_)) { + return Ok(Pat::Paren(PatParen { + attrs: Vec::new(), + paren_token, + pat: Box::new(value), + })); + } + elems.push_value(value); + break; + } + elems.push_value(value); + let punct = content.parse()?; + elems.push_punct(punct); + } + + Ok(Pat::Tuple(PatTuple { + attrs: Vec::new(), + paren_token, + elems, + })) + } + + fn pat_reference(input: ParseStream) -> Result<PatReference> { + Ok(PatReference { + attrs: Vec::new(), + and_token: input.parse()?, + mutability: input.parse()?, + pat: Box::new(Pat::parse_single(input)?), + }) + } + + fn pat_lit_or_range(input: ParseStream) -> Result<Pat> { + let start = input.call(pat_range_bound)?.unwrap(); + if input.peek(Token![..]) { + let limits = RangeLimits::parse_obsolete(input)?; + let end = input.call(pat_range_bound)?; + if let (RangeLimits::Closed(_), None) = (&limits, &end) { + return Err(input.error("expected range upper bound")); + } + Ok(Pat::Range(ExprRange { + attrs: Vec::new(), + start: Some(start.into_expr()), + limits, + end: end.map(PatRangeBound::into_expr), + })) + } else { + Ok(start.into_pat()) + } + } + + // Patterns that can appear on either side of a range pattern. + enum PatRangeBound { + Const(ExprConst), + Lit(ExprLit), + Path(ExprPath), + } + + impl PatRangeBound { + fn into_expr(self) -> Box<Expr> { + Box::new(match self { + PatRangeBound::Const(pat) => Expr::Const(pat), + PatRangeBound::Lit(pat) => Expr::Lit(pat), + PatRangeBound::Path(pat) => Expr::Path(pat), + }) + } + + fn into_pat(self) -> Pat { + match self { + PatRangeBound::Const(pat) => Pat::Const(pat), + PatRangeBound::Lit(pat) => Pat::Lit(pat), + PatRangeBound::Path(pat) => Pat::Path(pat), + } + } + } + + fn pat_range_bound(input: ParseStream) -> Result<Option<PatRangeBound>> { + if input.is_empty() + || input.peek(Token![|]) + || input.peek(Token![=]) + || input.peek(Token![:]) && !input.peek(Token![::]) + || input.peek(Token![,]) + || input.peek(Token![;]) + || input.peek(Token![if]) + { + return Ok(None); + } + + let lookahead = input.lookahead1(); + let expr = if lookahead.peek(Lit) { + PatRangeBound::Lit(input.parse()?) + } else if lookahead.peek(Ident) + || lookahead.peek(Token![::]) + || lookahead.peek(Token![<]) + || lookahead.peek(Token![self]) + || lookahead.peek(Token![Self]) + || lookahead.peek(Token![super]) + || lookahead.peek(Token![crate]) + { + PatRangeBound::Path(input.parse()?) + } else if lookahead.peek(Token![const]) { + PatRangeBound::Const(input.parse()?) + } else { + return Err(lookahead.error()); + }; + + Ok(Some(expr)) + } + + fn pat_slice(input: ParseStream) -> Result<PatSlice> { + let content; + let bracket_token = bracketed!(content in input); + + let mut elems = Punctuated::new(); + while !content.is_empty() { + let value = Pat::parse_multi_with_leading_vert(&content)?; + match value { + Pat::Range(pat) if pat.start.is_none() || pat.end.is_none() => { + let (start, end) = match pat.limits { + RangeLimits::HalfOpen(dot_dot) => (dot_dot.spans[0], dot_dot.spans[1]), + RangeLimits::Closed(dot_dot_eq) => { + (dot_dot_eq.spans[0], dot_dot_eq.spans[2]) + } + }; + let msg = "range pattern is not allowed unparenthesized inside slice pattern"; + return Err(error::new2(start, end, msg)); + } + _ => {} + } + elems.push_value(value); + if content.is_empty() { + break; + } + let punct = content.parse()?; + elems.push_punct(punct); + } + + Ok(PatSlice { + attrs: Vec::new(), + bracket_token, + elems, + }) + } + + fn pat_const(input: ParseStream) -> Result<TokenStream> { + let begin = input.cursor(); + input.parse::<Token![const]>()?; + + let content; + braced!(content in input); + content.call(Attribute::parse_inner)?; + content.call(Block::parse_within)?; + + Ok(verbatim::between(begin, input.cursor())) + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::FilterAttrs; + use crate::pat::{ + FieldPat, Pat, PatIdent, PatOr, PatParen, PatReference, PatRest, PatSlice, PatStruct, + PatTuple, PatTupleStruct, PatType, PatWild, + }; + use crate::path; + use crate::path::printing::PathStyle; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatIdent { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.by_ref.to_tokens(tokens); + self.mutability.to_tokens(tokens); + self.ident.to_tokens(tokens); + if let Some((at_token, subpat)) = &self.subpat { + at_token.to_tokens(tokens); + subpat.to_tokens(tokens); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatOr { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.leading_vert.to_tokens(tokens); + self.cases.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatParen { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.paren_token.surround(tokens, |tokens| { + self.pat.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatReference { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.and_token.to_tokens(tokens); + self.mutability.to_tokens(tokens); + self.pat.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatRest { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.dot2_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatSlice { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.bracket_token.surround(tokens, |tokens| { + self.elems.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatStruct { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + path::printing::print_qpath(tokens, &self.qself, &self.path, PathStyle::Expr); + self.brace_token.surround(tokens, |tokens| { + self.fields.to_tokens(tokens); + // NOTE: We need a comma before the dot2 token if it is present. + if !self.fields.empty_or_trailing() && self.rest.is_some() { + <Token![,]>::default().to_tokens(tokens); + } + self.rest.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatTuple { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.paren_token.surround(tokens, |tokens| { + self.elems.to_tokens(tokens); + // If there is only one element, a trailing comma is needed to + // distinguish PatTuple from PatParen, unless this is `(..)` + // which is a tuple pattern even without comma. + if self.elems.len() == 1 + && !self.elems.trailing_punct() + && !matches!(self.elems[0], Pat::Rest { .. }) + { + <Token![,]>::default().to_tokens(tokens); + } + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatTupleStruct { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + path::printing::print_qpath(tokens, &self.qself, &self.path, PathStyle::Expr); + self.paren_token.surround(tokens, |tokens| { + self.elems.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatType { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.pat.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PatWild { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + self.underscore_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for FieldPat { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + if let Some(colon_token) = &self.colon_token { + self.member.to_tokens(tokens); + colon_token.to_tokens(tokens); + } + self.pat.to_tokens(tokens); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/path.rs b/third_party/crates/syn-2.0.119/src/path.rs new file mode 100644 index 0000000..b8eac5a --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/path.rs @@ -0,0 +1,965 @@ +#[cfg(feature = "parsing")] +use crate::error::Result; +use crate::expr::Expr; +use crate::generics::TypeParamBound; +use crate::ident::Ident; +use crate::lifetime::Lifetime; +use crate::punctuated::Punctuated; +use crate::token; +use crate::ty::{ReturnType, Type}; +use alloc::boxed::Box; + +ast_struct! { + /// A path at which a named item is exported (e.g. `alloc::collections::HashMap`). + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Path { + pub leading_colon: Option<Token![::]>, + pub segments: Punctuated<PathSegment, Token![::]>, + } +} + +impl<T> From<T> for Path +where + T: Into<PathSegment>, +{ + fn from(segment: T) -> Self { + let mut path = Path { + leading_colon: None, + segments: Punctuated::new(), + }; + path.segments.push_value(segment.into()); + path + } +} + +impl Path { + /// Determines whether this is a path of length 1 equal to the given + /// ident. + /// + /// For them to compare equal, it must be the case that: + /// + /// - the path has no leading colon, + /// - the number of path segments is 1, + /// - the first path segment has no angle bracketed or parenthesized + /// path arguments, and + /// - the ident of the first path segment is equal to the given one. + /// + /// # Example + /// + /// ``` + /// use proc_macro2::TokenStream; + /// use syn::{Attribute, Error, Meta, Result}; + /// + /// fn get_serde_meta_item(attr: &Attribute) -> Result<Option<&TokenStream>> { + /// if attr.path().is_ident("serde") { + /// match &attr.meta { + /// Meta::List(meta) => Ok(Some(&meta.tokens)), + /// bad => Err(Error::new_spanned(bad, "unrecognized attribute")), + /// } + /// } else { + /// Ok(None) + /// } + /// } + /// ``` + pub fn is_ident<I>(&self, ident: &I) -> bool + where + I: ?Sized, + Ident: PartialEq<I>, + { + match self.get_ident() { + Some(id) => id == ident, + None => false, + } + } + + /// If this path consists of a single ident, returns the ident. + /// + /// A path is considered an ident if: + /// + /// - the path has no leading colon, + /// - the number of path segments is 1, and + /// - the first path segment has no angle bracketed or parenthesized + /// path arguments. + pub fn get_ident(&self) -> Option<&Ident> { + if self.leading_colon.is_none() + && self.segments.len() == 1 + && self.segments[0].arguments.is_none() + { + Some(&self.segments[0].ident) + } else { + None + } + } + + /// An error if this path is not a single ident, as defined in `get_ident`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn require_ident(&self) -> Result<&Ident> { + self.get_ident().ok_or_else(|| { + crate::error::new2( + self.segments.first().unwrap().ident.span(), + self.segments.last().unwrap().ident.span(), + "expected this path to be an identifier", + ) + }) + } +} + +ast_struct! { + /// A segment of a path together with any path arguments on that segment. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct PathSegment { + pub ident: Ident, + pub arguments: PathArguments, + } +} + +impl<T> From<T> for PathSegment +where + T: Into<Ident>, +{ + fn from(ident: T) -> Self { + PathSegment { + ident: ident.into(), + arguments: PathArguments::None, + } + } +} + +ast_enum! { + /// Angle bracketed or parenthesized arguments of a path segment. + /// + /// ## Angle bracketed + /// + /// The `<'a, T>` in `core::slice::iter<'a, T>`. + /// + /// ## Parenthesized + /// + /// The `(A, B) -> C` in `Fn(A, B) -> C`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum PathArguments { + None, + /// The `<'a, T>` in `core::slice::iter<'a, T>`. + AngleBracketed(AngleBracketedGenericArguments), + /// The `(A, B) -> C` in `Fn(A, B) -> C`. + Parenthesized(ParenthesizedGenericArguments), + } +} + +impl Default for PathArguments { + fn default() -> Self { + PathArguments::None + } +} + +impl PathArguments { + pub fn is_empty(&self) -> bool { + match self { + PathArguments::None => true, + PathArguments::AngleBracketed(bracketed) => bracketed.args.is_empty(), + PathArguments::Parenthesized(_) => false, + } + } + + pub fn is_none(&self) -> bool { + match self { + PathArguments::None => true, + PathArguments::AngleBracketed(_) | PathArguments::Parenthesized(_) => false, + } + } +} + +ast_enum! { + /// An individual generic argument, like `'a`, `T`, or `Item = T`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum GenericArgument { + /// A lifetime argument. + Lifetime(Lifetime), + /// A type argument. + Type(Type), + /// A const expression. Must be inside of a block. + /// + /// NOTE: Identity expressions are represented as Type arguments, as + /// they are indistinguishable syntactically. + Const(Expr), + /// A binding (equality constraint) on an associated type: the `Item = + /// u8` in `Iterator<Item = u8>`. + AssocType(AssocType), + /// An equality constraint on an associated constant: the `PANIC = + /// false` in `Trait<PANIC = false>`. + AssocConst(AssocConst), + /// An associated type bound: `Iterator<Item: Display>`. + Constraint(Constraint), + } +} + +ast_struct! { + /// Angle bracketed arguments of a path segment: the `<K, V>` in `HashMap<K, + /// V>`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct AngleBracketedGenericArguments { + pub colon2_token: Option<Token![::]>, + pub lt_token: Token![<], + pub args: Punctuated<GenericArgument, Token![,]>, + pub gt_token: Token![>], + } +} + +ast_struct! { + /// A binding (equality constraint) on an associated type: the `Item = u8` + /// in `Iterator<Item = u8>`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct AssocType { + pub ident: Ident, + pub generics: Option<AngleBracketedGenericArguments>, + pub eq_token: Token![=], + pub ty: Type, + } +} + +ast_struct! { + /// An equality constraint on an associated constant: the `PANIC = false` in + /// `Trait<PANIC = false>`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct AssocConst { + pub ident: Ident, + pub generics: Option<AngleBracketedGenericArguments>, + pub eq_token: Token![=], + pub value: Expr, + } +} + +ast_struct! { + /// An associated type bound: `Iterator<Item: Display>`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Constraint { + pub ident: Ident, + pub generics: Option<AngleBracketedGenericArguments>, + pub colon_token: Token![:], + pub bounds: Punctuated<TypeParamBound, Token![+]>, + } +} + +ast_struct! { + /// Arguments of a function path segment: the `(A, B) -> C` in `Fn(A,B) -> + /// C`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct ParenthesizedGenericArguments { + pub paren_token: token::Paren, + /// `(A, B)` + pub inputs: Punctuated<Type, Token![,]>, + /// `C` + pub output: ReturnType, + } +} + +ast_struct! { + /// The explicit Self type in a qualified path: the `T` in `<T as + /// Display>::fmt`. + /// + /// The actual path, including the trait and the associated item, is stored + /// separately. The `position` field represents the index of the associated + /// item qualified with this Self type. + /// + /// ```text + /// <Vec<T> as a::b::Trait>::AssociatedItem + /// ^~~~~~ ~~~~~~~~~~~~~~^ + /// ty position = 3 + /// + /// <Vec<T>>::AssociatedItem + /// ^~~~~~ ^ + /// ty position = 0 + /// ``` + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct QSelf { + pub lt_token: Token![<], + pub ty: Box<Type>, + pub position: usize, + pub as_token: Option<Token![as]>, + pub gt_token: Token![>], + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::error::Result; + #[cfg(feature = "full")] + use crate::expr::ExprBlock; + use crate::expr::{Expr, ExprPath}; + use crate::ext::IdentExt as _; + use crate::generics::TypeParamBound; + use crate::ident::Ident; + use crate::lifetime::Lifetime; + use crate::lit::Lit; + use crate::parse::{Parse, ParseStream}; + use crate::path::{ + AngleBracketedGenericArguments, AssocConst, AssocType, Constraint, GenericArgument, + ParenthesizedGenericArguments, Path, PathArguments, PathSegment, QSelf, + }; + use crate::punctuated::Punctuated; + use crate::token; + use crate::ty::{ReturnType, Type}; + #[cfg(not(feature = "full"))] + use crate::verbatim; + use alloc::boxed::Box; + use alloc::vec::Vec; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Path { + fn parse(input: ParseStream) -> Result<Self> { + Self::parse_helper(input, false) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for GenericArgument { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Lifetime) && !input.peek2(Token![+]) { + return Ok(GenericArgument::Lifetime(input.parse()?)); + } + + if input.peek(Lit) || input.peek(token::Brace) { + return const_argument(input).map(GenericArgument::Const); + } + + let mut argument: Type = input.parse()?; + + match argument { + Type::Path(mut ty) + if ty.qself.is_none() + && ty.path.leading_colon.is_none() + && ty.path.segments.len() == 1 + && match &ty.path.segments[0].arguments { + PathArguments::None | PathArguments::AngleBracketed(_) => true, + PathArguments::Parenthesized(_) => false, + } => + { + if let Some(eq_token) = input.parse::<Option<Token![=]>>()? { + let segment = ty.path.segments.pop().unwrap().into_value(); + let ident = segment.ident; + let generics = match segment.arguments { + PathArguments::None => None, + PathArguments::AngleBracketed(arguments) => Some(arguments), + PathArguments::Parenthesized(_) => unreachable!(), + }; + return if input.peek(Lit) || input.peek(token::Brace) { + Ok(GenericArgument::AssocConst(AssocConst { + ident, + generics, + eq_token, + value: const_argument(input)?, + })) + } else { + Ok(GenericArgument::AssocType(AssocType { + ident, + generics, + eq_token, + ty: input.parse()?, + })) + }; + } + + if let Some(colon_token) = input.parse::<Option<Token![:]>>()? { + let segment = ty.path.segments.pop().unwrap().into_value(); + return Ok(GenericArgument::Constraint(Constraint { + ident: segment.ident, + generics: match segment.arguments { + PathArguments::None => None, + PathArguments::AngleBracketed(arguments) => Some(arguments), + PathArguments::Parenthesized(_) => unreachable!(), + }, + colon_token, + bounds: { + let mut bounds = Punctuated::new(); + loop { + if input.peek(Token![,]) || input.peek(Token![>]) { + break; + } + bounds.push_value({ + let allow_precise_capture = false; + let allow_const = true; + TypeParamBound::parse_single( + input, + allow_precise_capture, + allow_const, + )? + }); + if !input.peek(Token![+]) { + break; + } + let punct: Token![+] = input.parse()?; + bounds.push_punct(punct); + } + bounds + }, + })); + } + + argument = Type::Path(ty); + } + _ => {} + } + + Ok(GenericArgument::Type(argument)) + } + } + + pub(crate) fn const_argument(input: ParseStream) -> Result<Expr> { + let lookahead = input.lookahead1(); + + if input.peek(Lit) { + let lit = input.parse()?; + return Ok(Expr::Lit(lit)); + } + + if input.peek(Ident) { + let ident: Ident = input.parse()?; + return Ok(Expr::Path(ExprPath { + attrs: Vec::new(), + qself: None, + path: Path::from(ident), + })); + } + + if input.peek(token::Brace) { + #[cfg(feature = "full")] + { + let block: ExprBlock = input.parse()?; + return Ok(Expr::Block(block)); + } + + #[cfg(not(feature = "full"))] + { + let begin = input.cursor(); + let content; + braced!(content in input); + content.parse::<Expr>()?; + let verbatim = verbatim::between(begin, input.cursor()); + return Ok(Expr::Verbatim(verbatim)); + } + } + + Err(lookahead.error()) + } + + impl AngleBracketedGenericArguments { + /// Parse `::<…>` with mandatory leading `::`. + /// + /// The ordinary [`Parse`] impl for `AngleBracketedGenericArguments` + /// parses optional leading `::`. + #[cfg(feature = "full")] + #[cfg_attr(docsrs, doc(cfg(all(feature = "parsing", feature = "full"))))] + pub fn parse_turbofish(input: ParseStream) -> Result<Self> { + let colon2_token: Token![::] = input.parse()?; + Self::do_parse(Some(colon2_token), input) + } + + pub(crate) fn do_parse( + colon2_token: Option<Token![::]>, + input: ParseStream, + ) -> Result<Self> { + Ok(AngleBracketedGenericArguments { + colon2_token, + lt_token: input.parse()?, + args: { + let mut args = Punctuated::new(); + loop { + if input.peek(Token![>]) { + break; + } + let value: GenericArgument = input.parse()?; + args.push_value(value); + if input.peek(Token![>]) { + break; + } + let punct: Token![,] = input.parse()?; + args.push_punct(punct); + } + args + }, + gt_token: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for AngleBracketedGenericArguments { + fn parse(input: ParseStream) -> Result<Self> { + let colon2_token: Option<Token![::]> = input.parse()?; + Self::do_parse(colon2_token, input) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ParenthesizedGenericArguments { + fn parse(input: ParseStream) -> Result<Self> { + let content; + Ok(ParenthesizedGenericArguments { + paren_token: parenthesized!(content in input), + inputs: content.parse_terminated(Type::parse, Token![,])?, + output: input.call(ReturnType::without_plus)?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for PathSegment { + fn parse(input: ParseStream) -> Result<Self> { + Self::parse_helper(input, false) + } + } + + impl PathSegment { + fn parse_helper(input: ParseStream, expr_style: bool) -> Result<Self> { + if input.peek(Token![super]) + || input.peek(Token![self]) + || input.peek(Token![crate]) + || cfg!(feature = "full") && input.peek(Token![try]) + { + let ident = input.call(Ident::parse_any)?; + return Ok(PathSegment::from(ident)); + } + + let ident = if input.peek(Token![Self]) { + input.call(Ident::parse_any)? + } else { + input.parse()? + }; + + if !expr_style + && input.peek(Token![<]) + && !input.peek(Token![<=]) + && !input.peek(Token![<<=]) + || input.peek(Token![::]) && input.peek3(Token![<]) + { + Ok(PathSegment { + ident, + arguments: PathArguments::AngleBracketed(input.parse()?), + }) + } else { + Ok(PathSegment::from(ident)) + } + } + } + + impl Path { + /// Parse a `Path` containing no path arguments on any of its segments. + /// + /// # Example + /// + /// ``` + /// use syn::{Path, Result, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // A simplified single `use` statement like: + /// // + /// // use alloc::collections::HashMap; + /// // + /// // Note that generic parameters are not allowed in a `use` statement + /// // so the following must not be accepted. + /// // + /// // use a::<b>::c; + /// struct SingleUse { + /// use_token: Token![use], + /// path: Path, + /// } + /// + /// impl Parse for SingleUse { + /// fn parse(input: ParseStream) -> Result<Self> { + /// Ok(SingleUse { + /// use_token: input.parse()?, + /// path: input.call(Path::parse_mod_style)?, + /// }) + /// } + /// } + /// ``` + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_mod_style(input: ParseStream) -> Result<Self> { + Ok(Path { + leading_colon: input.parse()?, + segments: { + let mut segments = Punctuated::new(); + loop { + if !input.peek(Ident) + && !input.peek(Token![super]) + && !input.peek(Token![self]) + && !input.peek(Token![Self]) + && !input.peek(Token![crate]) + { + break; + } + let ident = Ident::parse_any(input)?; + segments.push_value(PathSegment::from(ident)); + if !input.peek(Token![::]) { + break; + } + let punct = input.parse()?; + segments.push_punct(punct); + } + if segments.is_empty() { + return Err(input.parse::<Ident>().unwrap_err()); + } else if segments.trailing_punct() { + return Err(input.error("expected path segment after `::`")); + } + segments + }, + }) + } + + pub(crate) fn parse_helper(input: ParseStream, expr_style: bool) -> Result<Self> { + let mut path = Path { + leading_colon: input.parse()?, + segments: { + let mut segments = Punctuated::new(); + let value = PathSegment::parse_helper(input, expr_style)?; + segments.push_value(value); + segments + }, + }; + Path::parse_rest(input, &mut path, expr_style)?; + Ok(path) + } + + pub(crate) fn parse_rest( + input: ParseStream, + path: &mut Self, + expr_style: bool, + ) -> Result<()> { + while input.peek(Token![::]) && !input.peek3(token::Paren) { + let punct: Token![::] = input.parse()?; + path.segments.push_punct(punct); + let value = PathSegment::parse_helper(input, expr_style)?; + path.segments.push_value(value); + } + Ok(()) + } + + pub(crate) fn is_mod_style(&self) -> bool { + self.segments + .iter() + .all(|segment| segment.arguments.is_none()) + } + } + + pub(crate) fn qpath(input: ParseStream, expr_style: bool) -> Result<(Option<QSelf>, Path)> { + if input.peek(Token![<]) { + let lt_token: Token![<] = input.parse()?; + let this: Type = input.parse()?; + let path = if input.peek(Token![as]) { + let as_token: Token![as] = input.parse()?; + let path: Path = input.parse()?; + Some((as_token, path)) + } else { + None + }; + let gt_token: Token![>] = input.parse()?; + let colon2_token: Token![::] = input.parse()?; + let mut rest = Punctuated::new(); + loop { + let path = PathSegment::parse_helper(input, expr_style)?; + rest.push_value(path); + if !input.peek(Token![::]) { + break; + } + let punct: Token![::] = input.parse()?; + rest.push_punct(punct); + } + let (position, as_token, path) = match path { + Some((as_token, mut path)) => { + let pos = path.segments.len(); + path.segments.push_punct(colon2_token); + path.segments.extend(rest.into_pairs()); + (pos, Some(as_token), path) + } + None => { + let path = Path { + leading_colon: Some(colon2_token), + segments: rest, + }; + (0, None, path) + } + }; + let qself = QSelf { + lt_token, + ty: Box::new(this), + position, + as_token, + gt_token, + }; + Ok((Some(qself), path)) + } else { + let path = Path::parse_helper(input, expr_style)?; + Ok((None, path)) + } + } +} + +#[cfg(feature = "printing")] +pub(crate) mod printing { + use crate::generics; + use crate::path::{ + AngleBracketedGenericArguments, AssocConst, AssocType, Constraint, GenericArgument, + ParenthesizedGenericArguments, Path, PathArguments, PathSegment, QSelf, + }; + use crate::print::TokensOrDefault; + #[cfg(feature = "parsing")] + use crate::spanned::Spanned; + use core::cmp; + #[cfg(feature = "parsing")] + use proc_macro2::Span; + use proc_macro2::TokenStream; + use quote::ToTokens; + + pub(crate) enum PathStyle { + Expr, + Mod, + AsWritten, + } + + impl Copy for PathStyle {} + + impl Clone for PathStyle { + fn clone(&self) -> Self { + *self + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Path { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_path(tokens, self, PathStyle::AsWritten); + } + } + + pub(crate) fn print_path(tokens: &mut TokenStream, path: &Path, style: PathStyle) { + path.leading_colon.to_tokens(tokens); + for segment in path.segments.pairs() { + print_path_segment(tokens, segment.value(), style); + segment.punct().to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PathSegment { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_path_segment(tokens, self, PathStyle::AsWritten); + } + } + + fn print_path_segment(tokens: &mut TokenStream, segment: &PathSegment, style: PathStyle) { + segment.ident.to_tokens(tokens); + print_path_arguments(tokens, &segment.arguments, style); + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for PathArguments { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_path_arguments(tokens, self, PathStyle::AsWritten); + } + } + + fn print_path_arguments(tokens: &mut TokenStream, arguments: &PathArguments, style: PathStyle) { + match arguments { + PathArguments::None => {} + PathArguments::AngleBracketed(arguments) => { + print_angle_bracketed_generic_arguments(tokens, arguments, style); + } + PathArguments::Parenthesized(arguments) => { + print_parenthesized_generic_arguments(tokens, arguments, style); + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for GenericArgument { + #[allow(clippy::match_same_arms)] + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + GenericArgument::Lifetime(lt) => lt.to_tokens(tokens), + GenericArgument::Type(ty) => ty.to_tokens(tokens), + GenericArgument::Const(expr) => { + generics::printing::print_const_argument(expr, tokens); + } + GenericArgument::AssocType(assoc) => assoc.to_tokens(tokens), + GenericArgument::AssocConst(assoc) => assoc.to_tokens(tokens), + GenericArgument::Constraint(constraint) => constraint.to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for AngleBracketedGenericArguments { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_angle_bracketed_generic_arguments(tokens, self, PathStyle::AsWritten); + } + } + + pub(crate) fn print_angle_bracketed_generic_arguments( + tokens: &mut TokenStream, + arguments: &AngleBracketedGenericArguments, + style: PathStyle, + ) { + if let PathStyle::Mod = style { + return; + } + + conditionally_print_turbofish(tokens, &arguments.colon2_token, style); + arguments.lt_token.to_tokens(tokens); + + // Print lifetimes before types/consts/bindings, regardless of their + // order in args. + let mut trailing_or_empty = true; + for param in arguments.args.pairs() { + match param.value() { + GenericArgument::Lifetime(_) => { + param.to_tokens(tokens); + trailing_or_empty = param.punct().is_some(); + } + GenericArgument::Type(_) + | GenericArgument::Const(_) + | GenericArgument::AssocType(_) + | GenericArgument::AssocConst(_) + | GenericArgument::Constraint(_) => {} + } + } + for param in arguments.args.pairs() { + match param.value() { + GenericArgument::Type(_) + | GenericArgument::Const(_) + | GenericArgument::AssocType(_) + | GenericArgument::AssocConst(_) + | GenericArgument::Constraint(_) => { + if !trailing_or_empty { + <Token![,]>::default().to_tokens(tokens); + } + param.to_tokens(tokens); + trailing_or_empty = param.punct().is_some(); + } + GenericArgument::Lifetime(_) => {} + } + } + + arguments.gt_token.to_tokens(tokens); + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for AssocType { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + self.ty.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for AssocConst { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.eq_token.to_tokens(tokens); + generics::printing::print_const_argument(&self.value, tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Constraint { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.ident.to_tokens(tokens); + self.generics.to_tokens(tokens); + self.colon_token.to_tokens(tokens); + self.bounds.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ParenthesizedGenericArguments { + fn to_tokens(&self, tokens: &mut TokenStream) { + print_parenthesized_generic_arguments(tokens, self, PathStyle::AsWritten); + } + } + + fn print_parenthesized_generic_arguments( + tokens: &mut TokenStream, + arguments: &ParenthesizedGenericArguments, + style: PathStyle, + ) { + if let PathStyle::Mod = style { + return; + } + + conditionally_print_turbofish(tokens, &None, style); + arguments.paren_token.surround(tokens, |tokens| { + arguments.inputs.to_tokens(tokens); + }); + arguments.output.to_tokens(tokens); + } + + pub(crate) fn print_qpath( + tokens: &mut TokenStream, + qself: &Option<QSelf>, + path: &Path, + style: PathStyle, + ) { + let qself = match qself { + Some(qself) => qself, + None => { + print_path(tokens, path, style); + return; + } + }; + qself.lt_token.to_tokens(tokens); + qself.ty.to_tokens(tokens); + + let pos = cmp::min(qself.position, path.segments.len()); + let mut segments = path.segments.pairs(); + if pos > 0 { + TokensOrDefault(&qself.as_token).to_tokens(tokens); + path.leading_colon.to_tokens(tokens); + for (i, segment) in segments.by_ref().take(pos).enumerate() { + print_path_segment(tokens, segment.value(), PathStyle::AsWritten); + if i + 1 == pos { + qself.gt_token.to_tokens(tokens); + } + segment.punct().to_tokens(tokens); + } + } else { + qself.gt_token.to_tokens(tokens); + path.leading_colon.to_tokens(tokens); + } + for segment in segments { + print_path_segment(tokens, segment.value(), style); + segment.punct().to_tokens(tokens); + } + } + + fn conditionally_print_turbofish( + tokens: &mut TokenStream, + colon2_token: &Option<Token![::]>, + style: PathStyle, + ) { + match style { + PathStyle::Expr => TokensOrDefault(colon2_token).to_tokens(tokens), + PathStyle::Mod => unreachable!(), + PathStyle::AsWritten => colon2_token.to_tokens(tokens), + } + } + + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(all(feature = "parsing", feature = "printing"))))] + impl Spanned for QSelf { + fn span(&self) -> Span { + struct QSelfDelimiters<'a>(&'a QSelf); + + impl<'a> ToTokens for QSelfDelimiters<'a> { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.lt_token.to_tokens(tokens); + self.0.gt_token.to_tokens(tokens); + } + } + + QSelfDelimiters(self).span() + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/precedence.rs b/third_party/crates/syn-2.0.119/src/precedence.rs new file mode 100644 index 0000000..d225f7f --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/precedence.rs @@ -0,0 +1,210 @@ +#[cfg(all(feature = "printing", feature = "full"))] +use crate::attr::{AttrStyle, Attribute}; +#[cfg(feature = "printing")] +use crate::expr::Expr; +#[cfg(all(feature = "printing", feature = "full"))] +use crate::expr::{ + ExprArray, ExprAsync, ExprAwait, ExprBlock, ExprBreak, ExprCall, ExprConst, ExprContinue, + ExprField, ExprForLoop, ExprGroup, ExprIf, ExprIndex, ExprInfer, ExprLit, ExprLoop, ExprMacro, + ExprMatch, ExprMethodCall, ExprParen, ExprPath, ExprRepeat, ExprReturn, ExprStruct, ExprTry, + ExprTryBlock, ExprTuple, ExprUnsafe, ExprWhile, ExprYield, +}; +use crate::op::BinOp; +#[cfg(all(feature = "printing", feature = "full"))] +use crate::ty::ReturnType; +use core::cmp::Ordering; + +// Reference: https://doc.rust-lang.org/reference/expressions.html#expression-precedence +pub(crate) enum Precedence { + // return, break, closures + Jump, + // = += -= *= /= %= &= |= ^= <<= >>= + Assign, + // .. ..= + Range, + // || + Or, + // && + And, + // let + #[cfg(feature = "printing")] + Let, + // == != < > <= >= + Compare, + // | + BitOr, + // ^ + BitXor, + // & + BitAnd, + // << >> + Shift, + // + - + Sum, + // * / % + Product, + // as + Cast, + // unary - * ! & &mut + #[cfg(feature = "printing")] + Prefix, + // paths, loops, function calls, array indexing, field expressions, method calls + #[cfg(feature = "printing")] + Unambiguous, +} + +impl Precedence { + pub(crate) const MIN: Self = Precedence::Jump; + + pub(crate) fn of_binop(op: &BinOp) -> Self { + match op { + BinOp::Add(_) | BinOp::Sub(_) => Precedence::Sum, + BinOp::Mul(_) | BinOp::Div(_) | BinOp::Rem(_) => Precedence::Product, + BinOp::And(_) => Precedence::And, + BinOp::Or(_) => Precedence::Or, + BinOp::BitXor(_) => Precedence::BitXor, + BinOp::BitAnd(_) => Precedence::BitAnd, + BinOp::BitOr(_) => Precedence::BitOr, + BinOp::Shl(_) | BinOp::Shr(_) => Precedence::Shift, + + BinOp::Eq(_) + | BinOp::Lt(_) + | BinOp::Le(_) + | BinOp::Ne(_) + | BinOp::Ge(_) + | BinOp::Gt(_) => Precedence::Compare, + + BinOp::AddAssign(_) + | BinOp::SubAssign(_) + | BinOp::MulAssign(_) + | BinOp::DivAssign(_) + | BinOp::RemAssign(_) + | BinOp::BitXorAssign(_) + | BinOp::BitAndAssign(_) + | BinOp::BitOrAssign(_) + | BinOp::ShlAssign(_) + | BinOp::ShrAssign(_) => Precedence::Assign, + } + } + + #[cfg(feature = "printing")] + pub(crate) fn of(e: &Expr) -> Self { + #[cfg(feature = "full")] + fn prefix_attrs(attrs: &[Attribute]) -> Precedence { + for attr in attrs { + if let AttrStyle::Outer = attr.style { + return Precedence::Prefix; + } + } + Precedence::Unambiguous + } + + match e { + #[cfg(feature = "full")] + Expr::Closure(e) => match e.output { + ReturnType::Default => Precedence::Jump, + ReturnType::Type(..) => prefix_attrs(&e.attrs), + }, + + #[cfg(feature = "full")] + Expr::Break(ExprBreak { expr, .. }) + | Expr::Return(ExprReturn { expr, .. }) + | Expr::Yield(ExprYield { expr, .. }) => match expr { + Some(_) => Precedence::Jump, + None => Precedence::Unambiguous, + }, + + Expr::Assign(_) => Precedence::Assign, + Expr::Range(_) => Precedence::Range, + Expr::Binary(e) => Precedence::of_binop(&e.op), + Expr::Let(_) => Precedence::Let, + Expr::Cast(_) => Precedence::Cast, + Expr::RawAddr(_) | Expr::Reference(_) | Expr::Unary(_) => Precedence::Prefix, + + #[cfg(feature = "full")] + Expr::Array(ExprArray { attrs, .. }) + | Expr::Async(ExprAsync { attrs, .. }) + | Expr::Await(ExprAwait { attrs, .. }) + | Expr::Block(ExprBlock { attrs, .. }) + | Expr::Call(ExprCall { attrs, .. }) + | Expr::Const(ExprConst { attrs, .. }) + | Expr::Continue(ExprContinue { attrs, .. }) + | Expr::Field(ExprField { attrs, .. }) + | Expr::ForLoop(ExprForLoop { attrs, .. }) + | Expr::Group(ExprGroup { attrs, .. }) + | Expr::If(ExprIf { attrs, .. }) + | Expr::Index(ExprIndex { attrs, .. }) + | Expr::Infer(ExprInfer { attrs, .. }) + | Expr::Lit(ExprLit { attrs, .. }) + | Expr::Loop(ExprLoop { attrs, .. }) + | Expr::Macro(ExprMacro { attrs, .. }) + | Expr::Match(ExprMatch { attrs, .. }) + | Expr::MethodCall(ExprMethodCall { attrs, .. }) + | Expr::Paren(ExprParen { attrs, .. }) + | Expr::Path(ExprPath { attrs, .. }) + | Expr::Repeat(ExprRepeat { attrs, .. }) + | Expr::Struct(ExprStruct { attrs, .. }) + | Expr::Try(ExprTry { attrs, .. }) + | Expr::TryBlock(ExprTryBlock { attrs, .. }) + | Expr::Tuple(ExprTuple { attrs, .. }) + | Expr::Unsafe(ExprUnsafe { attrs, .. }) + | Expr::While(ExprWhile { attrs, .. }) => prefix_attrs(attrs), + + #[cfg(not(feature = "full"))] + Expr::Array(_) + | Expr::Async(_) + | Expr::Await(_) + | Expr::Block(_) + | Expr::Call(_) + | Expr::Const(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::ForLoop(_) + | Expr::Group(_) + | Expr::If(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Lit(_) + | Expr::Loop(_) + | Expr::Macro(_) + | Expr::Match(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Repeat(_) + | Expr::Struct(_) + | Expr::Try(_) + | Expr::TryBlock(_) + | Expr::Tuple(_) + | Expr::Unsafe(_) + | Expr::While(_) => Precedence::Unambiguous, + + Expr::Verbatim(_) => Precedence::Unambiguous, + + #[cfg(not(feature = "full"))] + Expr::Break(_) | Expr::Closure(_) | Expr::Return(_) | Expr::Yield(_) => unreachable!(), + } + } +} + +impl Copy for Precedence {} + +impl Clone for Precedence { + fn clone(&self) -> Self { + *self + } +} + +impl PartialEq for Precedence { + fn eq(&self, other: &Self) -> bool { + *self as u8 == *other as u8 + } +} + +impl PartialOrd for Precedence { + fn partial_cmp(&self, other: &Self) -> Option<Ordering> { + let this = *self as u8; + let other = *other as u8; + Some(this.cmp(&other)) + } +} diff --git a/third_party/crates/syn-2.0.119/src/print.rs b/third_party/crates/syn-2.0.119/src/print.rs new file mode 100644 index 0000000..0740993 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/print.rs @@ -0,0 +1,16 @@ +use proc_macro2::TokenStream; +use quote::ToTokens; + +pub(crate) struct TokensOrDefault<'a, T: 'a>(pub &'a Option<T>); + +impl<'a, T> ToTokens for TokensOrDefault<'a, T> +where + T: ToTokens + Default, +{ + fn to_tokens(&self, tokens: &mut TokenStream) { + match self.0 { + Some(t) => t.to_tokens(tokens), + None => T::default().to_tokens(tokens), + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/punctuated.rs b/third_party/crates/syn-2.0.119/src/punctuated.rs new file mode 100644 index 0000000..7545c66 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/punctuated.rs @@ -0,0 +1,1170 @@ +//! A punctuated sequence of syntax tree nodes separated by punctuation. +//! +//! Lots of things in Rust are punctuated sequences. +//! +//! - The fields of a struct are `Punctuated<Field, Token![,]>`. +//! - The segments of a path are `Punctuated<PathSegment, Token![::]>`. +//! - The bounds on a generic parameter are `Punctuated<TypeParamBound, +//! Token![+]>`. +//! - The arguments to a function call are `Punctuated<Expr, Token![,]>`. +//! +//! This module provides a common representation for these punctuated sequences +//! in the form of the [`Punctuated<T, P>`] type. We store a vector of pairs of +//! syntax tree node + punctuation, where every node in the sequence is followed +//! by punctuation except for possibly the final one. +//! +//! [`Punctuated<T, P>`]: Punctuated +//! +//! ```text +//! a_function_call(arg1, arg2, arg3); +//! ~~~~^ ~~~~^ ~~~~ +//! ``` + +use crate::drops::{NoDrop, TrivialDrop}; +#[cfg(feature = "parsing")] +use crate::error::Result; +#[cfg(feature = "parsing")] +use crate::parse::{Parse, ParseStream}; +#[cfg(feature = "parsing")] +use crate::token::Token; +use alloc::boxed::Box; +#[cfg(all(feature = "fold", any(feature = "full", feature = "derive")))] +use alloc::collections::VecDeque; +use alloc::vec::{self, Vec}; +#[cfg(feature = "extra-traits")] +use core::fmt::{self, Debug}; +#[cfg(feature = "extra-traits")] +use core::hash::{Hash, Hasher}; +#[cfg(any(feature = "full", feature = "derive"))] +use core::iter; +use core::ops::{Index, IndexMut}; +use core::option; +use core::slice; + +/// **A punctuated sequence of syntax tree nodes of type `T` separated by +/// punctuation of type `P`.** +/// +/// Refer to the [module documentation] for details about punctuated sequences. +/// +/// [module documentation]: self +pub struct Punctuated<T, P> { + inner: Vec<(T, P)>, + last: Option<Box<T>>, +} + +impl<T, P> Punctuated<T, P> { + /// Creates an empty punctuated sequence. + pub const fn new() -> Self { + Punctuated { + inner: Vec::new(), + last: None, + } + } + + /// Determines whether this punctuated sequence is empty, meaning it + /// contains no syntax tree nodes or punctuation. + pub fn is_empty(&self) -> bool { + self.inner.len() == 0 && self.last.is_none() + } + + /// Returns the number of syntax tree nodes in this punctuated sequence. + /// + /// This is the number of nodes of type `T`, not counting the punctuation of + /// type `P`. + pub fn len(&self) -> usize { + self.inner.len() + if self.last.is_some() { 1 } else { 0 } + } + + /// Borrows the first element in this sequence. + pub fn first(&self) -> Option<&T> { + self.iter().next() + } + + /// Mutably borrows the first element in this sequence. + pub fn first_mut(&mut self) -> Option<&mut T> { + self.iter_mut().next() + } + + /// Borrows the last element in this sequence. + pub fn last(&self) -> Option<&T> { + self.iter().next_back() + } + + /// Mutably borrows the last element in this sequence. + pub fn last_mut(&mut self) -> Option<&mut T> { + self.iter_mut().next_back() + } + + /// Borrows the element at the given index. + pub fn get(&self, index: usize) -> Option<&T> { + if let Some((value, _punct)) = self.inner.get(index) { + Some(value) + } else if index == self.inner.len() { + self.last.as_deref() + } else { + None + } + } + + /// Mutably borrows the element at the given index. + pub fn get_mut(&mut self, index: usize) -> Option<&mut T> { + let inner_len = self.inner.len(); + if let Some((value, _punct)) = self.inner.get_mut(index) { + Some(value) + } else if index == inner_len { + self.last.as_deref_mut() + } else { + None + } + } + + /// Returns an iterator over borrowed syntax tree nodes of type `&T`. + pub fn iter(&self) -> Iter<T> { + Iter { + inner: Box::new(NoDrop::new(PrivateIter { + inner: self.inner.iter(), + last: self.last.as_ref().map(Box::as_ref).into_iter(), + })), + } + } + + /// Returns an iterator over mutably borrowed syntax tree nodes of type + /// `&mut T`. + pub fn iter_mut(&mut self) -> IterMut<T> { + IterMut { + inner: Box::new(NoDrop::new(PrivateIterMut { + inner: self.inner.iter_mut(), + last: self.last.as_mut().map(Box::as_mut).into_iter(), + })), + } + } + + /// Returns an iterator over the contents of this sequence as borrowed + /// punctuated pairs. + pub fn pairs(&self) -> Pairs<T, P> { + Pairs { + inner: self.inner.iter(), + last: self.last.as_ref().map(Box::as_ref).into_iter(), + } + } + + /// Returns an iterator over the contents of this sequence as mutably + /// borrowed punctuated pairs. + pub fn pairs_mut(&mut self) -> PairsMut<T, P> { + PairsMut { + inner: self.inner.iter_mut(), + last: self.last.as_mut().map(Box::as_mut).into_iter(), + } + } + + /// Returns an iterator over the contents of this sequence as owned + /// punctuated pairs. + pub fn into_pairs(self) -> IntoPairs<T, P> { + IntoPairs { + inner: self.inner.into_iter(), + last: self.last.map(|t| *t).into_iter(), + } + } + + /// Appends a syntax tree node onto the end of this punctuated sequence. The + /// sequence must already have a trailing punctuation, or be empty. + /// + /// Use [`push`] instead if the punctuated sequence may or may not already + /// have trailing punctuation. + /// + /// [`push`]: Punctuated::push + /// + /// # Panics + /// + /// Panics if the sequence is nonempty and does not already have a trailing + /// punctuation. + pub fn push_value(&mut self, value: T) { + assert!( + self.empty_or_trailing(), + "Punctuated::push_value: cannot push value if Punctuated is missing trailing punctuation", + ); + + self.last = Some(Box::new(value)); + } + + /// Appends a trailing punctuation onto the end of this punctuated sequence. + /// The sequence must be non-empty and must not already have trailing + /// punctuation. + /// + /// # Panics + /// + /// Panics if the sequence is empty or already has a trailing punctuation. + pub fn push_punct(&mut self, punctuation: P) { + assert!( + self.last.is_some(), + "Punctuated::push_punct: cannot push punctuation if Punctuated is empty or already has trailing punctuation", + ); + + let last = self.last.take().unwrap(); + self.inner.push((*last, punctuation)); + } + + /// Removes the last punctuated pair from this sequence, or `None` if the + /// sequence is empty. + pub fn pop(&mut self) -> Option<Pair<T, P>> { + if self.last.is_some() { + self.last.take().map(|t| Pair::End(*t)) + } else { + self.inner.pop().map(|(t, p)| Pair::Punctuated(t, p)) + } + } + + /// Removes the trailing punctuation from this punctuated sequence, or + /// `None` if there isn't any. + pub fn pop_punct(&mut self) -> Option<P> { + if self.last.is_some() { + None + } else { + let (t, p) = self.inner.pop()?; + self.last = Some(Box::new(t)); + Some(p) + } + } + + /// Determines whether this punctuated sequence ends with a trailing + /// punctuation. + pub fn trailing_punct(&self) -> bool { + self.last.is_none() && !self.is_empty() + } + + /// Returns true if either this `Punctuated` is empty, or it has a trailing + /// punctuation. + /// + /// Equivalent to `punctuated.is_empty() || punctuated.trailing_punct()`. + pub fn empty_or_trailing(&self) -> bool { + self.last.is_none() + } + + /// Appends a syntax tree node onto the end of this punctuated sequence. + /// + /// If there is not a trailing punctuation in this sequence when this method + /// is called, the default value of punctuation type `P` is inserted before + /// the given value of type `T`. + pub fn push(&mut self, value: T) + where + P: Default, + { + if !self.empty_or_trailing() { + self.push_punct(Default::default()); + } + self.push_value(value); + } + + /// Inserts an element at position `index`. + /// + /// # Panics + /// + /// Panics if `index` is greater than the number of elements previously in + /// this punctuated sequence. + pub fn insert(&mut self, index: usize, value: T) + where + P: Default, + { + assert!( + index <= self.len(), + "Punctuated::insert: index out of range", + ); + + if index == self.len() { + self.push(value); + } else { + self.inner.insert(index, (value, Default::default())); + } + } + + /// Clears the sequence of all values and punctuation, making it empty. + pub fn clear(&mut self) { + self.inner.clear(); + self.last = None; + } + + /// Parses zero or more occurrences of `T` separated by punctuation of type + /// `P`, with optional trailing punctuation. + /// + /// Parsing continues until the end of this parse stream. The entire content + /// of this parse stream must consist of `T` and `P`. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_terminated(input: ParseStream) -> Result<Self> + where + T: Parse, + P: Parse, + { + Self::parse_terminated_with(input, T::parse) + } + + /// Parses zero or more occurrences of `T` using the given parse function, + /// separated by punctuation of type `P`, with optional trailing + /// punctuation. + /// + /// Like [`parse_terminated`], the entire content of this stream is expected + /// to be parsed. + /// + /// [`parse_terminated`]: Punctuated::parse_terminated + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_terminated_with<'a>( + input: ParseStream<'a>, + parser: fn(ParseStream<'a>) -> Result<T>, + ) -> Result<Self> + where + P: Parse, + { + let mut punctuated = Punctuated::new(); + + loop { + if input.is_empty() { + break; + } + let value = parser(input)?; + punctuated.push_value(value); + if input.is_empty() { + break; + } + let punct = input.parse()?; + punctuated.push_punct(punct); + } + + Ok(punctuated) + } + + /// Parses one or more occurrences of `T` separated by punctuation of type + /// `P`, not accepting trailing punctuation. + /// + /// Parsing continues as long as punctuation `P` is present at the head of + /// the stream. This method returns upon parsing a `T` and observing that it + /// is not followed by a `P`, even if there are remaining tokens in the + /// stream. + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_separated_nonempty(input: ParseStream) -> Result<Self> + where + T: Parse, + P: Token + Parse, + { + Self::parse_separated_nonempty_with(input, T::parse) + } + + /// Parses one or more occurrences of `T` using the given parse function, + /// separated by punctuation of type `P`, not accepting trailing + /// punctuation. + /// + /// Like [`parse_separated_nonempty`], may complete early without parsing + /// the entire content of this stream. + /// + /// [`parse_separated_nonempty`]: Punctuated::parse_separated_nonempty + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_separated_nonempty_with<'a>( + input: ParseStream<'a>, + parser: fn(ParseStream<'a>) -> Result<T>, + ) -> Result<Self> + where + P: Token + Parse, + { + let mut punctuated = Punctuated::new(); + + loop { + let value = parser(input)?; + punctuated.push_value(value); + if !P::peek(input.cursor()) { + break; + } + let punct = input.parse()?; + punctuated.push_punct(punct); + } + + Ok(punctuated) + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl<T, P> Clone for Punctuated<T, P> +where + T: Clone, + P: Clone, +{ + fn clone(&self) -> Self { + Punctuated { + inner: self.inner.clone(), + last: self.last.clone(), + } + } + + fn clone_from(&mut self, other: &Self) { + self.inner.clone_from(&other.inner); + self.last.clone_from(&other.last); + } +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl<T, P> Eq for Punctuated<T, P> +where + T: Eq, + P: Eq, +{ +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl<T, P> PartialEq for Punctuated<T, P> +where + T: PartialEq, + P: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + let Punctuated { inner, last } = self; + *inner == other.inner && *last == other.last + } +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl<T, P> Hash for Punctuated<T, P> +where + T: Hash, + P: Hash, +{ + fn hash<H: Hasher>(&self, state: &mut H) { + let Punctuated { inner, last } = self; + inner.hash(state); + last.hash(state); + } +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl<T: Debug, P: Debug> Debug for Punctuated<T, P> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let mut list = f.debug_list(); + for (t, p) in &self.inner { + list.entry(t); + list.entry(p); + } + if let Some(last) = &self.last { + list.entry(last); + } + list.finish() + } +} + +impl<T, P> FromIterator<T> for Punctuated<T, P> +where + P: Default, +{ + fn from_iter<I: IntoIterator<Item = T>>(i: I) -> Self { + let mut ret = Punctuated::new(); + ret.extend(i); + ret + } +} + +impl<T, P> Extend<T> for Punctuated<T, P> +where + P: Default, +{ + fn extend<I: IntoIterator<Item = T>>(&mut self, i: I) { + for value in i { + self.push(value); + } + } +} + +impl<T, P> FromIterator<Pair<T, P>> for Punctuated<T, P> { + fn from_iter<I: IntoIterator<Item = Pair<T, P>>>(i: I) -> Self { + let mut ret = Punctuated::new(); + do_extend(&mut ret, i.into_iter()); + ret + } +} + +impl<T, P> Extend<Pair<T, P>> for Punctuated<T, P> +where + P: Default, +{ + fn extend<I: IntoIterator<Item = Pair<T, P>>>(&mut self, i: I) { + if !self.empty_or_trailing() { + self.push_punct(P::default()); + } + do_extend(self, i.into_iter()); + } +} + +fn do_extend<T, P, I>(punctuated: &mut Punctuated<T, P>, i: I) +where + I: Iterator<Item = Pair<T, P>>, +{ + let mut nomore = false; + for pair in i { + if nomore { + panic!("punctuated extended with items after a Pair::End"); + } + match pair { + Pair::Punctuated(a, b) => punctuated.inner.push((a, b)), + Pair::End(a) => { + punctuated.last = Some(Box::new(a)); + nomore = true; + } + } + } +} + +impl<T, P> IntoIterator for Punctuated<T, P> { + type Item = T; + type IntoIter = IntoIter<T>; + + fn into_iter(self) -> Self::IntoIter { + let mut elements = Vec::with_capacity(self.len()); + + for (t, _) in self.inner { + elements.push(t); + } + if let Some(t) = self.last { + elements.push(*t); + } + + IntoIter { + inner: elements.into_iter(), + } + } +} + +impl<'a, T, P> IntoIterator for &'a Punctuated<T, P> { + type Item = &'a T; + type IntoIter = Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + Punctuated::iter(self) + } +} + +impl<'a, T, P> IntoIterator for &'a mut Punctuated<T, P> { + type Item = &'a mut T; + type IntoIter = IterMut<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + Punctuated::iter_mut(self) + } +} + +impl<T, P> Default for Punctuated<T, P> { + fn default() -> Self { + Punctuated::new() + } +} + +/// An iterator over borrowed pairs of type `Pair<&T, &P>`. +/// +/// Refer to the [module documentation] for details about punctuated sequences. +/// +/// [module documentation]: self +pub struct Pairs<'a, T: 'a, P: 'a> { + inner: slice::Iter<'a, (T, P)>, + last: option::IntoIter<&'a T>, +} + +impl<'a, T, P> Iterator for Pairs<'a, T, P> { + type Item = Pair<&'a T, &'a P>; + + fn next(&mut self) -> Option<Self::Item> { + self.inner + .next() + .map(|(t, p)| Pair::Punctuated(t, p)) + .or_else(|| self.last.next().map(Pair::End)) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + (self.len(), Some(self.len())) + } +} + +impl<'a, T, P> DoubleEndedIterator for Pairs<'a, T, P> { + fn next_back(&mut self) -> Option<Self::Item> { + self.last + .next() + .map(Pair::End) + .or_else(|| self.inner.next_back().map(|(t, p)| Pair::Punctuated(t, p))) + } +} + +impl<'a, T, P> ExactSizeIterator for Pairs<'a, T, P> { + fn len(&self) -> usize { + self.inner.len() + self.last.len() + } +} + +// No Clone bound on T or P. +impl<'a, T, P> Clone for Pairs<'a, T, P> { + fn clone(&self) -> Self { + Pairs { + inner: self.inner.clone(), + last: self.last.clone(), + } + } +} + +/// An iterator over mutably borrowed pairs of type `Pair<&mut T, &mut P>`. +/// +/// Refer to the [module documentation] for details about punctuated sequences. +/// +/// [module documentation]: self +pub struct PairsMut<'a, T: 'a, P: 'a> { + inner: slice::IterMut<'a, (T, P)>, + last: option::IntoIter<&'a mut T>, +} + +impl<'a, T, P> Iterator for PairsMut<'a, T, P> { + type Item = Pair<&'a mut T, &'a mut P>; + + fn next(&mut self) -> Option<Self::Item> { + self.inner + .next() + .map(|(t, p)| Pair::Punctuated(t, p)) + .or_else(|| self.last.next().map(Pair::End)) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + (self.len(), Some(self.len())) + } +} + +impl<'a, T, P> DoubleEndedIterator for PairsMut<'a, T, P> { + fn next_back(&mut self) -> Option<Self::Item> { + self.last + .next() + .map(Pair::End) + .or_else(|| self.inner.next_back().map(|(t, p)| Pair::Punctuated(t, p))) + } +} + +impl<'a, T, P> ExactSizeIterator for PairsMut<'a, T, P> { + fn len(&self) -> usize { + self.inner.len() + self.last.len() + } +} + +/// An iterator over owned pairs of type `Pair<T, P>`. +/// +/// Refer to the [module documentation] for details about punctuated sequences. +/// +/// [module documentation]: self +pub struct IntoPairs<T, P> { + inner: vec::IntoIter<(T, P)>, + last: option::IntoIter<T>, +} + +impl<T, P> Iterator for IntoPairs<T, P> { + type Item = Pair<T, P>; + + fn next(&mut self) -> Option<Self::Item> { + self.inner + .next() + .map(|(t, p)| Pair::Punctuated(t, p)) + .or_else(|| self.last.next().map(Pair::End)) + } + + fn size_hint(&self) -> (usize, Option<usize>) { + (self.len(), Some(self.len())) + } +} + +impl<T, P> DoubleEndedIterator for IntoPairs<T, P> { + fn next_back(&mut self) -> Option<Self::Item> { + self.last + .next() + .map(Pair::End) + .or_else(|| self.inner.next_back().map(|(t, p)| Pair::Punctuated(t, p))) + } +} + +impl<T, P> ExactSizeIterator for IntoPairs<T, P> { + fn len(&self) -> usize { + self.inner.len() + self.last.len() + } +} + +impl<T, P> Clone for IntoPairs<T, P> +where + T: Clone, + P: Clone, +{ + fn clone(&self) -> Self { + IntoPairs { + inner: self.inner.clone(), + last: self.last.clone(), + } + } +} + +/// An iterator over owned values of type `T`. +/// +/// Refer to the [module documentation] for details about punctuated sequences. +/// +/// [module documentation]: self +pub struct IntoIter<T> { + inner: vec::IntoIter<T>, +} + +impl<T> Iterator for IntoIter<T> { + type Item = T; + + fn next(&mut self) -> Option<Self::Item> { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option<usize>) { + (self.len(), Some(self.len())) + } +} + +impl<T> DoubleEndedIterator for IntoIter<T> { + fn next_back(&mut self) -> Option<Self::Item> { + self.inner.next_back() + } +} + +impl<T> ExactSizeIterator for IntoIter<T> { + fn len(&self) -> usize { + self.inner.len() + } +} + +impl<T> Clone for IntoIter<T> +where + T: Clone, +{ + fn clone(&self) -> Self { + IntoIter { + inner: self.inner.clone(), + } + } +} + +/// An iterator over borrowed values of type `&T`. +/// +/// Refer to the [module documentation] for details about punctuated sequences. +/// +/// [module documentation]: self +pub struct Iter<'a, T: 'a> { + inner: Box<NoDrop<dyn IterTrait<'a, T> + 'a>>, +} + +trait IterTrait<'a, T: 'a>: Iterator<Item = &'a T> + DoubleEndedIterator + ExactSizeIterator { + fn clone_box(&self) -> Box<NoDrop<dyn IterTrait<'a, T> + 'a>>; +} + +struct PrivateIter<'a, T: 'a, P: 'a> { + inner: slice::Iter<'a, (T, P)>, + last: option::IntoIter<&'a T>, +} + +impl<'a, T, P> TrivialDrop for PrivateIter<'a, T, P> +where + slice::Iter<'a, (T, P)>: TrivialDrop, + option::IntoIter<&'a T>: TrivialDrop, +{ +} + +#[cfg(any(feature = "full", feature = "derive"))] +pub(crate) fn empty_punctuated_iter<'a, T>() -> Iter<'a, T> { + Iter { + inner: Box::new(NoDrop::new(iter::empty())), + } +} + +// No Clone bound on T. +impl<'a, T> Clone for Iter<'a, T> { + fn clone(&self) -> Self { + Iter { + inner: self.inner.clone_box(), + } + } +} + +impl<'a, T> Iterator for Iter<'a, T> { + type Item = &'a T; + + fn next(&mut self) -> Option<Self::Item> { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option<usize>) { + (self.len(), Some(self.len())) + } +} + +impl<'a, T> DoubleEndedIterator for Iter<'a, T> { + fn next_back(&mut self) -> Option<Self::Item> { + self.inner.next_back() + } +} + +impl<'a, T> ExactSizeIterator for Iter<'a, T> { + fn len(&self) -> usize { + self.inner.len() + } +} + +impl<'a, T, P> Iterator for PrivateIter<'a, T, P> { + type Item = &'a T; + + fn next(&mut self) -> Option<Self::Item> { + self.inner + .next() + .map(|pair| &pair.0) + .or_else(|| self.last.next()) + } +} + +impl<'a, T, P> DoubleEndedIterator for PrivateIter<'a, T, P> { + fn next_back(&mut self) -> Option<Self::Item> { + self.last + .next() + .or_else(|| self.inner.next_back().map(|pair| &pair.0)) + } +} + +impl<'a, T, P> ExactSizeIterator for PrivateIter<'a, T, P> { + fn len(&self) -> usize { + self.inner.len() + self.last.len() + } +} + +// No Clone bound on T or P. +impl<'a, T, P> Clone for PrivateIter<'a, T, P> { + fn clone(&self) -> Self { + PrivateIter { + inner: self.inner.clone(), + last: self.last.clone(), + } + } +} + +impl<'a, T, I> IterTrait<'a, T> for I +where + T: 'a, + I: DoubleEndedIterator<Item = &'a T> + + ExactSizeIterator<Item = &'a T> + + Clone + + TrivialDrop + + 'a, +{ + fn clone_box(&self) -> Box<NoDrop<dyn IterTrait<'a, T> + 'a>> { + Box::new(NoDrop::new(self.clone())) + } +} + +/// An iterator over mutably borrowed values of type `&mut T`. +/// +/// Refer to the [module documentation] for details about punctuated sequences. +/// +/// [module documentation]: self +pub struct IterMut<'a, T: 'a> { + inner: Box<NoDrop<dyn IterMutTrait<'a, T, Item = &'a mut T> + 'a>>, +} + +trait IterMutTrait<'a, T: 'a>: + DoubleEndedIterator<Item = &'a mut T> + ExactSizeIterator<Item = &'a mut T> +{ +} + +struct PrivateIterMut<'a, T: 'a, P: 'a> { + inner: slice::IterMut<'a, (T, P)>, + last: option::IntoIter<&'a mut T>, +} + +impl<'a, T, P> TrivialDrop for PrivateIterMut<'a, T, P> +where + slice::IterMut<'a, (T, P)>: TrivialDrop, + option::IntoIter<&'a mut T>: TrivialDrop, +{ +} + +#[cfg(any(feature = "full", feature = "derive"))] +pub(crate) fn empty_punctuated_iter_mut<'a, T>() -> IterMut<'a, T> { + IterMut { + inner: Box::new(NoDrop::new(iter::empty())), + } +} + +impl<'a, T> Iterator for IterMut<'a, T> { + type Item = &'a mut T; + + fn next(&mut self) -> Option<Self::Item> { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option<usize>) { + (self.len(), Some(self.len())) + } +} + +impl<'a, T> DoubleEndedIterator for IterMut<'a, T> { + fn next_back(&mut self) -> Option<Self::Item> { + self.inner.next_back() + } +} + +impl<'a, T> ExactSizeIterator for IterMut<'a, T> { + fn len(&self) -> usize { + self.inner.len() + } +} + +impl<'a, T, P> Iterator for PrivateIterMut<'a, T, P> { + type Item = &'a mut T; + + fn next(&mut self) -> Option<Self::Item> { + self.inner + .next() + .map(|pair| &mut pair.0) + .or_else(|| self.last.next()) + } +} + +impl<'a, T, P> DoubleEndedIterator for PrivateIterMut<'a, T, P> { + fn next_back(&mut self) -> Option<Self::Item> { + self.last + .next() + .or_else(|| self.inner.next_back().map(|pair| &mut pair.0)) + } +} + +impl<'a, T, P> ExactSizeIterator for PrivateIterMut<'a, T, P> { + fn len(&self) -> usize { + self.inner.len() + self.last.len() + } +} + +impl<'a, T, I> IterMutTrait<'a, T> for I +where + T: 'a, + I: DoubleEndedIterator<Item = &'a mut T> + ExactSizeIterator<Item = &'a mut T> + 'a, +{ +} + +/// A single syntax tree node of type `T` followed by its trailing punctuation +/// of type `P` if any. +/// +/// Refer to the [module documentation] for details about punctuated sequences. +/// +/// [module documentation]: self +pub enum Pair<T, P> { + Punctuated(T, P), + End(T), +} + +impl<T, P> Pair<T, P> { + /// Extracts the syntax tree node from this punctuated pair, discarding the + /// following punctuation. + pub fn into_value(self) -> T { + match self { + Pair::Punctuated(t, _) | Pair::End(t) => t, + } + } + + /// Borrows the syntax tree node from this punctuated pair. + pub fn value(&self) -> &T { + match self { + Pair::Punctuated(t, _) | Pair::End(t) => t, + } + } + + /// Mutably borrows the syntax tree node from this punctuated pair. + pub fn value_mut(&mut self) -> &mut T { + match self { + Pair::Punctuated(t, _) | Pair::End(t) => t, + } + } + + /// Borrows the punctuation from this punctuated pair, unless this pair is + /// the final one and there is no trailing punctuation. + pub fn punct(&self) -> Option<&P> { + match self { + Pair::Punctuated(_, p) => Some(p), + Pair::End(_) => None, + } + } + + /// Mutably borrows the punctuation from this punctuated pair, unless the + /// pair is the final one and there is no trailing punctuation. + /// + /// # Example + /// + /// ``` + /// # use proc_macro2::Span; + /// # use syn::punctuated::Punctuated; + /// # use syn::{parse_quote, Token, TypeParamBound}; + /// # + /// # let mut punctuated = Punctuated::<TypeParamBound, Token![+]>::new(); + /// # let span = Span::call_site(); + /// # + /// punctuated.insert(0, parse_quote!('lifetime)); + /// if let Some(punct) = punctuated.pairs_mut().next().unwrap().punct_mut() { + /// punct.span = span; + /// } + /// ``` + pub fn punct_mut(&mut self) -> Option<&mut P> { + match self { + Pair::Punctuated(_, p) => Some(p), + Pair::End(_) => None, + } + } + + /// Creates a punctuated pair out of a syntax tree node and an optional + /// following punctuation. + pub fn new(t: T, p: Option<P>) -> Self { + match p { + Some(p) => Pair::Punctuated(t, p), + None => Pair::End(t), + } + } + + /// Produces this punctuated pair as a tuple of syntax tree node and + /// optional following punctuation. + pub fn into_tuple(self) -> (T, Option<P>) { + match self { + Pair::Punctuated(t, p) => (t, Some(p)), + Pair::End(t) => (t, None), + } + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl<T, P> Pair<&T, &P> { + pub fn cloned(self) -> Pair<T, P> + where + T: Clone, + P: Clone, + { + match self { + Pair::Punctuated(t, p) => Pair::Punctuated(t.clone(), p.clone()), + Pair::End(t) => Pair::End(t.clone()), + } + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl<T, P> Clone for Pair<T, P> +where + T: Clone, + P: Clone, +{ + fn clone(&self) -> Self { + match self { + Pair::Punctuated(t, p) => Pair::Punctuated(t.clone(), p.clone()), + Pair::End(t) => Pair::End(t.clone()), + } + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl<T, P> Copy for Pair<T, P> +where + T: Copy, + P: Copy, +{ +} + +impl<T, P> Index<usize> for Punctuated<T, P> { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + if index.checked_add(1) == Some(self.len()) { + match &self.last { + Some(t) => t, + None => &self.inner[index].0, + } + } else { + &self.inner[index].0 + } + } +} + +impl<T, P> IndexMut<usize> for Punctuated<T, P> { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + if index.checked_add(1) == Some(self.len()) { + match &mut self.last { + Some(t) => t, + None => &mut self.inner[index].0, + } + } else { + &mut self.inner[index].0 + } + } +} + +#[cfg(all(feature = "fold", any(feature = "full", feature = "derive")))] +pub(crate) fn fold<T, P, V, F>( + punctuated: Punctuated<T, P>, + fold: &mut V, + mut f: F, +) -> Punctuated<T, P> +where + V: ?Sized, + F: FnMut(&mut V, T) -> T, +{ + let Punctuated { inner, last } = punctuated; + + // Convert into VecDeque to prevent needing to allocate a new Vec<(T, P)> + // for the folded elements. + let mut inner = VecDeque::from(inner); + for _ in 0..inner.len() { + if let Some((t, p)) = inner.pop_front() { + inner.push_back((f(fold, t), p)); + } + } + + Punctuated { + inner: Vec::from(inner), + last: match last { + Some(t) => Some(Box::new(f(fold, *t))), + None => None, + }, + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::punctuated::{Pair, Punctuated}; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl<T, P> ToTokens for Punctuated<T, P> + where + T: ToTokens, + P: ToTokens, + { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.pairs()); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl<T, P> ToTokens for Pair<T, P> + where + T: ToTokens, + P: ToTokens, + { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Pair::Punctuated(a, b) => { + a.to_tokens(tokens); + b.to_tokens(tokens); + } + Pair::End(a) => a.to_tokens(tokens), + } + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/restriction.rs b/third_party/crates/syn-2.0.119/src/restriction.rs new file mode 100644 index 0000000..d0cdf8e --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/restriction.rs @@ -0,0 +1,180 @@ +use crate::path::Path; +use crate::token; +use alloc::boxed::Box; + +ast_enum! { + /// The visibility level of an item: inherited or `pub` or + /// `pub(restricted)`. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Visibility { + /// A public visibility level: `pub`. + Public(Token![pub]), + + /// A visibility level restricted to some path: `pub(self)` or + /// `pub(super)` or `pub(crate)` or `pub(in some::module)`. + Restricted(VisRestricted), + + /// An inherited visibility, which usually means private. + Inherited, + } +} + +ast_struct! { + /// A visibility level restricted to some path: `pub(self)` or + /// `pub(super)` or `pub(crate)` or `pub(in some::module)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct VisRestricted { + pub pub_token: Token![pub], + pub paren_token: token::Paren, + pub in_token: Option<Token![in]>, + pub path: Box<Path>, + } +} + +ast_enum! { + /// Unused, but reserved for RFC 3323 restrictions. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum FieldMutability { + None, + + // TODO: https://rust-lang.github.io/rfcs/3323-restrictions.html + // + // FieldMutability::Restricted(MutRestricted) + // + // pub struct MutRestricted { + // pub mut_token: Token![mut], + // pub paren_token: token::Paren, + // pub in_token: Option<Token![in]>, + // pub path: Box<Path>, + // } + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::error::Result; + use crate::ext::IdentExt as _; + use crate::ident::Ident; + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::path::Path; + use crate::restriction::{VisRestricted, Visibility}; + use crate::token; + use alloc::boxed::Box; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Visibility { + fn parse(input: ParseStream) -> Result<Self> { + // Recognize an empty None-delimited group, as produced by a $:vis + // matcher that matched no tokens. + if input.peek(token::Group) { + let ahead = input.fork(); + let group = crate::group::parse_group(&ahead)?; + if group.content.is_empty() { + input.advance_to(&ahead); + return Ok(Visibility::Inherited); + } + } + + if input.peek(Token![pub]) { + Self::parse_pub(input) + } else { + Ok(Visibility::Inherited) + } + } + } + + impl Visibility { + fn parse_pub(input: ParseStream) -> Result<Self> { + let pub_token = input.parse::<Token![pub]>()?; + + if input.peek(token::Paren) { + let ahead = input.fork(); + + let content; + let paren_token = parenthesized!(content in ahead); + if content.peek(Token![crate]) + || content.peek(Token![self]) + || content.peek(Token![super]) + { + let path = content.call(Ident::parse_any)?; + + // Ensure there are no additional tokens within `content`. + // Without explicitly checking, we may misinterpret a tuple + // field as a restricted visibility, causing a parse error. + // e.g. `pub (crate::A, crate::B)` (Issue #720). + if content.is_empty() { + input.advance_to(&ahead); + return Ok(Visibility::Restricted(VisRestricted { + pub_token, + paren_token, + in_token: None, + path: Box::new(Path::from(path)), + })); + } + } else if content.peek(Token![in]) { + let in_token: Token![in] = content.parse()?; + let path = content.call(Path::parse_mod_style)?; + + input.advance_to(&ahead); + return Ok(Visibility::Restricted(VisRestricted { + pub_token, + paren_token, + in_token: Some(in_token), + path: Box::new(path), + })); + } + } + + Ok(Visibility::Public(pub_token)) + } + + #[cfg(feature = "full")] + pub(crate) fn is_some(&self) -> bool { + match self { + Visibility::Inherited => false, + _ => true, + } + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::path; + use crate::path::printing::PathStyle; + use crate::restriction::{VisRestricted, Visibility}; + use proc_macro2::TokenStream; + use quote::ToTokens; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Visibility { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Visibility::Public(pub_token) => pub_token.to_tokens(tokens), + Visibility::Restricted(vis_restricted) => vis_restricted.to_tokens(tokens), + Visibility::Inherited => {} + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for VisRestricted { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pub_token.to_tokens(tokens); + self.paren_token.surround(tokens, |tokens| { + // TODO: If we have a path which is not "self" or "super" or + // "crate", automatically add the "in" token. + self.in_token.to_tokens(tokens); + path::printing::print_path(tokens, &self.path, PathStyle::Mod); + }); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/scan_expr.rs b/third_party/crates/syn-2.0.119/src/scan_expr.rs new file mode 100644 index 0000000..27775ae --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/scan_expr.rs @@ -0,0 +1,270 @@ +use self::{Action::*, Input::*}; +use proc_macro2::{Delimiter, Ident, Spacing, TokenTree}; +use syn::parse::{ParseStream, Result}; +#[allow(unused_imports)] +//#[cfg_attr(not(test), expect(unused_imports))] // Rust 1.81+ +use syn::Token; +use syn::{AngleBracketedGenericArguments, BinOp, Expr, ExprPath, Lifetime, Lit, Type}; + +enum Input { + Keyword(&'static str), + Punct(&'static str), + ConsumeAny, + ConsumeBinOp, + ConsumeBrace, + ConsumeDelimiter, + ConsumeIdent, + ConsumeLifetime, + ConsumeLiteral, + ConsumeNestedBrace, + ExpectPath, + ExpectTurbofish, + ExpectType, + CanBeginExpr, + Otherwise, + Empty, +} + +enum Action { + SetState(&'static [(Input, Action)]), + IncDepth, + DecDepth, + Finish, +} + +static INIT: [(Input, Action); 29] = [ + (ConsumeDelimiter, SetState(&POSTFIX)), + (Keyword("async"), SetState(&ASYNC)), + (Keyword("break"), SetState(&BREAK_LABEL)), + (Keyword("const"), SetState(&CONST)), + (Keyword("continue"), SetState(&CONTINUE)), + (Keyword("for"), SetState(&FOR)), + (Keyword("if"), IncDepth), + (Keyword("let"), SetState(&PATTERN)), + (Keyword("loop"), SetState(&BLOCK)), + (Keyword("match"), IncDepth), + (Keyword("move"), SetState(&CLOSURE)), + (Keyword("return"), SetState(&RETURN)), + (Keyword("static"), SetState(&CLOSURE)), + (Keyword("try"), SetState(&BLOCK)), + (Keyword("unsafe"), SetState(&BLOCK)), + (Keyword("while"), IncDepth), + (Keyword("yield"), SetState(&RETURN)), + (Keyword("_"), SetState(&POSTFIX)), + (Punct("!"), SetState(&INIT)), + (Punct("#"), SetState(&[(ConsumeDelimiter, SetState(&INIT))])), + (Punct("&"), SetState(&REFERENCE)), + (Punct("*"), SetState(&INIT)), + (Punct("-"), SetState(&INIT)), + (Punct("..="), SetState(&INIT)), + (Punct(".."), SetState(&RANGE)), + (Punct("|"), SetState(&CLOSURE_ARGS)), + (ConsumeLifetime, SetState(&[(Punct(":"), SetState(&INIT))])), + (ConsumeLiteral, SetState(&POSTFIX)), + (ExpectPath, SetState(&PATH)), +]; + +static POSTFIX: [(Input, Action); 10] = [ + (Keyword("as"), SetState(&[(ExpectType, SetState(&POSTFIX))])), + (Punct("..="), SetState(&INIT)), + (Punct(".."), SetState(&RANGE)), + (Punct("."), SetState(&DOT)), + (Punct("?"), SetState(&POSTFIX)), + (ConsumeBinOp, SetState(&INIT)), + (Punct("="), SetState(&INIT)), + (ConsumeNestedBrace, SetState(&IF_THEN)), + (ConsumeDelimiter, SetState(&POSTFIX)), + (Empty, Finish), +]; + +static ASYNC: [(Input, Action); 3] = [ + (Keyword("move"), SetState(&ASYNC)), + (Punct("|"), SetState(&CLOSURE_ARGS)), + (ConsumeBrace, SetState(&POSTFIX)), +]; + +static BLOCK: [(Input, Action); 1] = [(ConsumeBrace, SetState(&POSTFIX))]; + +static BREAK_LABEL: [(Input, Action); 2] = [ + (ConsumeLifetime, SetState(&BREAK_VALUE)), + (Otherwise, SetState(&BREAK_VALUE)), +]; + +static BREAK_VALUE: [(Input, Action); 3] = [ + (ConsumeNestedBrace, SetState(&IF_THEN)), + (CanBeginExpr, SetState(&INIT)), + (Otherwise, SetState(&POSTFIX)), +]; + +static CLOSURE: [(Input, Action); 7] = [ + (Keyword("async"), SetState(&CLOSURE)), + (Keyword("move"), SetState(&CLOSURE)), + (Punct(","), SetState(&CLOSURE)), + (Punct(">"), SetState(&CLOSURE)), + (Punct("|"), SetState(&CLOSURE_ARGS)), + (ConsumeLifetime, SetState(&CLOSURE)), + (ConsumeIdent, SetState(&CLOSURE)), +]; + +static CLOSURE_ARGS: [(Input, Action); 2] = [ + (Punct("|"), SetState(&CLOSURE_RET)), + (ConsumeAny, SetState(&CLOSURE_ARGS)), +]; + +static CLOSURE_RET: [(Input, Action); 2] = [ + (Punct("->"), SetState(&[(ExpectType, SetState(&BLOCK))])), + (Otherwise, SetState(&INIT)), +]; + +static CONST: [(Input, Action); 3] = [ + (Keyword("move"), SetState(&CLOSURE)), + (Punct("|"), SetState(&CLOSURE_ARGS)), + (ConsumeBrace, SetState(&POSTFIX)), +]; + +static CONTINUE: [(Input, Action); 2] = [ + (ConsumeLifetime, SetState(&POSTFIX)), + (Otherwise, SetState(&POSTFIX)), +]; + +static DOT: [(Input, Action); 3] = [ + (Keyword("await"), SetState(&POSTFIX)), + (ConsumeIdent, SetState(&METHOD)), + (ConsumeLiteral, SetState(&POSTFIX)), +]; + +static FOR: [(Input, Action); 2] = [ + (Punct("<"), SetState(&CLOSURE)), + (Otherwise, SetState(&PATTERN)), +]; + +static IF_ELSE: [(Input, Action); 2] = [(Keyword("if"), SetState(&INIT)), (ConsumeBrace, DecDepth)]; +static IF_THEN: [(Input, Action); 2] = + [(Keyword("else"), SetState(&IF_ELSE)), (Otherwise, DecDepth)]; + +static METHOD: [(Input, Action); 1] = [(ExpectTurbofish, SetState(&POSTFIX))]; + +static PATH: [(Input, Action); 4] = [ + (Punct("!="), SetState(&INIT)), + (Punct("!"), SetState(&INIT)), + (ConsumeNestedBrace, SetState(&IF_THEN)), + (Otherwise, SetState(&POSTFIX)), +]; + +static PATTERN: [(Input, Action); 15] = [ + (ConsumeDelimiter, SetState(&PATTERN)), + (Keyword("box"), SetState(&PATTERN)), + (Keyword("in"), IncDepth), + (Keyword("mut"), SetState(&PATTERN)), + (Keyword("ref"), SetState(&PATTERN)), + (Keyword("_"), SetState(&PATTERN)), + (Punct("!"), SetState(&PATTERN)), + (Punct("&"), SetState(&PATTERN)), + (Punct("..="), SetState(&PATTERN)), + (Punct(".."), SetState(&PATTERN)), + (Punct("="), SetState(&INIT)), + (Punct("@"), SetState(&PATTERN)), + (Punct("|"), SetState(&PATTERN)), + (ConsumeLiteral, SetState(&PATTERN)), + (ExpectPath, SetState(&PATTERN)), +]; + +static RANGE: [(Input, Action); 6] = [ + (Punct("..="), SetState(&INIT)), + (Punct(".."), SetState(&RANGE)), + (Punct("."), SetState(&DOT)), + (ConsumeNestedBrace, SetState(&IF_THEN)), + (Empty, Finish), + (Otherwise, SetState(&INIT)), +]; + +static RAW: [(Input, Action); 3] = [ + (Keyword("const"), SetState(&INIT)), + (Keyword("mut"), SetState(&INIT)), + (Otherwise, SetState(&POSTFIX)), +]; + +static REFERENCE: [(Input, Action); 3] = [ + (Keyword("mut"), SetState(&INIT)), + (Keyword("raw"), SetState(&RAW)), + (Otherwise, SetState(&INIT)), +]; + +static RETURN: [(Input, Action); 2] = [ + (CanBeginExpr, SetState(&INIT)), + (Otherwise, SetState(&POSTFIX)), +]; + +pub(crate) fn scan_expr(input: ParseStream) -> Result<()> { + let mut state = INIT.as_slice(); + let mut depth = 0usize; + 'table: loop { + for rule in state { + if match rule.0 { + Input::Keyword(expected) => input.step(|cursor| match cursor.ident() { + Some((ident, rest)) if ident == expected => Ok((true, rest)), + _ => Ok((false, *cursor)), + })?, + Input::Punct(expected) => input.step(|cursor| { + let begin = *cursor; + let mut cursor = begin; + for (i, ch) in expected.chars().enumerate() { + match cursor.punct() { + Some((punct, _)) if punct.as_char() != ch => break, + Some((_, rest)) if i == expected.len() - 1 => { + return Ok((true, rest)); + } + Some((punct, rest)) if punct.spacing() == Spacing::Joint => { + cursor = rest; + } + _ => break, + } + } + Ok((false, begin)) + })?, + Input::ConsumeAny => input.parse::<Option<TokenTree>>()?.is_some(), + Input::ConsumeBinOp => input.parse::<BinOp>().is_ok(), + Input::ConsumeBrace | Input::ConsumeNestedBrace => { + (matches!(rule.0, Input::ConsumeBrace) || depth > 0) + && input.step(|cursor| match cursor.group(Delimiter::Brace) { + Some((_inside, _span, rest)) => Ok((true, rest)), + None => Ok((false, *cursor)), + })? + } + Input::ConsumeDelimiter => input.step(|cursor| match cursor.any_group() { + Some((_inside, _delimiter, _span, rest)) => Ok((true, rest)), + None => Ok((false, *cursor)), + })?, + Input::ConsumeIdent => input.parse::<Option<Ident>>()?.is_some(), + Input::ConsumeLifetime => input.parse::<Option<Lifetime>>()?.is_some(), + Input::ConsumeLiteral => input.parse::<Option<Lit>>()?.is_some(), + Input::ExpectPath => { + input.parse::<ExprPath>()?; + true + } + Input::ExpectTurbofish => { + if input.peek(Token![::]) { + input.parse::<AngleBracketedGenericArguments>()?; + } + true + } + Input::ExpectType => { + Type::without_plus(input)?; + true + } + Input::CanBeginExpr => Expr::peek(input), + Input::Otherwise => true, + Input::Empty => input.is_empty() || input.peek(Token![,]), + } { + state = match rule.1 { + Action::SetState(next) => next, + Action::IncDepth => (depth += 1, &INIT).1, + Action::DecDepth => (depth -= 1, &POSTFIX).1, + Action::Finish => return if depth == 0 { Ok(()) } else { break }, + }; + continue 'table; + } + } + return Err(input.error("unsupported expression")); + } +} diff --git a/third_party/crates/syn-2.0.119/src/sealed.rs b/third_party/crates/syn-2.0.119/src/sealed.rs new file mode 100644 index 0000000..dc80474 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/sealed.rs @@ -0,0 +1,4 @@ +#[cfg(feature = "parsing")] +pub(crate) mod lookahead { + pub trait Sealed: Copy {} +} diff --git a/third_party/crates/syn-2.0.119/src/span.rs b/third_party/crates/syn-2.0.119/src/span.rs new file mode 100644 index 0000000..eb27794 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/span.rs @@ -0,0 +1,63 @@ +use proc_macro2::extra::DelimSpan; +use proc_macro2::{Delimiter, Group, Span, TokenStream}; + +#[doc(hidden)] +pub trait IntoSpans<S> { + fn into_spans(self) -> S; +} + +impl IntoSpans<Span> for Span { + fn into_spans(self) -> Span { + self + } +} + +impl IntoSpans<[Span; 1]> for Span { + fn into_spans(self) -> [Span; 1] { + [self] + } +} + +impl IntoSpans<[Span; 2]> for Span { + fn into_spans(self) -> [Span; 2] { + [self, self] + } +} + +impl IntoSpans<[Span; 3]> for Span { + fn into_spans(self) -> [Span; 3] { + [self, self, self] + } +} + +impl IntoSpans<[Span; 1]> for [Span; 1] { + fn into_spans(self) -> [Span; 1] { + self + } +} + +impl IntoSpans<[Span; 2]> for [Span; 2] { + fn into_spans(self) -> [Span; 2] { + self + } +} + +impl IntoSpans<[Span; 3]> for [Span; 3] { + fn into_spans(self) -> [Span; 3] { + self + } +} + +impl IntoSpans<DelimSpan> for Span { + fn into_spans(self) -> DelimSpan { + let mut group = Group::new(Delimiter::None, TokenStream::new()); + group.set_span(self); + group.delim_span() + } +} + +impl IntoSpans<DelimSpan> for DelimSpan { + fn into_spans(self) -> DelimSpan { + self + } +} diff --git a/third_party/crates/syn-2.0.119/src/spanned.rs b/third_party/crates/syn-2.0.119/src/spanned.rs new file mode 100644 index 0000000..3855144 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/spanned.rs @@ -0,0 +1,118 @@ +//! A trait that can provide the `Span` of the complete contents of a syntax +//! tree node. +//! +//! <br> +//! +//! # Example +//! +//! Suppose in a procedural macro we have a [`Type`] that we want to assert +//! implements the [`Sync`] trait. Maybe this is the type of one of the fields +//! of a struct for which we are deriving a trait implementation, and we need to +//! be able to pass a reference to one of those fields across threads. +//! +//! [`Type`]: crate::Type +//! [`Sync`]: core::marker::Sync +//! +//! If the field type does *not* implement `Sync` as required, we want the +//! compiler to report an error pointing out exactly which type it was. +//! +//! The following macro code takes a variable `ty` of type `Type` and produces a +//! static assertion that `Sync` is implemented for that type. +//! +//! ``` +//! # extern crate proc_macro; +//! # +//! use proc_macro::TokenStream; +//! use proc_macro2::Span; +//! use quote::quote_spanned; +//! use syn::Type; +//! use syn::spanned::Spanned; +//! +//! # const IGNORE_TOKENS: &str = stringify! { +//! #[proc_macro_derive(MyMacro)] +//! # }; +//! pub fn my_macro(input: TokenStream) -> TokenStream { +//! # let ty = get_a_type(); +//! /* ... */ +//! +//! let assert_sync = quote_spanned! {ty.span()=> +//! struct _AssertSync where #ty: Sync; +//! }; +//! +//! /* ... */ +//! # input +//! } +//! # +//! # fn get_a_type() -> Type { +//! # unimplemented!() +//! # } +//! ``` +//! +//! By inserting this `assert_sync` fragment into the output code generated by +//! our macro, the user's code will fail to compile if `ty` does not implement +//! `Sync`. The errors they would see look like the following. +//! +//! ```text +//! error[E0277]: the trait bound `*const i32: core::marker::Sync` is not satisfied +//! --> src/main.rs:10:21 +//! | +//! 10 | bad_field: *const i32, +//! | ^^^^^^^^^^ `*const i32` cannot be shared between threads safely +//! ``` +//! +//! In this technique, using the `Type`'s span for the error message makes the +//! error appear in the correct place underlining the right type. +//! +//! <br> +//! +//! # Limitations +//! +//! The underlying [`proc_macro::Span::join`] method is nightly-only. When +//! called from within a procedural macro in a nightly compiler, `Spanned` will +//! use `join` to produce the intended span. When not using a nightly compiler, +//! only the span of the *first token* of the syntax tree node is returned. +//! +//! In the common case of wanting to use the joined span as the span of a +//! `syn::Error`, consider instead using [`syn::Error::new_spanned`] which is +//! able to span the error correctly under the complete syntax tree node without +//! needing the unstable `join`. +//! +//! [`syn::Error::new_spanned`]: crate::Error::new_spanned + +use proc_macro2::Span; +use quote::spanned::Spanned as ToTokens; + +/// A trait that can provide the `Span` of the complete contents of a syntax +/// tree node. +/// +/// This trait is automatically implemented for all types that implement +/// [`ToTokens`] from the `quote` crate, as well as for `Span` itself. +/// +/// [`ToTokens`]: quote::ToTokens +/// +/// See the [module documentation] for an example. +/// +/// [module documentation]: self +pub trait Spanned: private::Sealed { + /// Returns a `Span` covering the complete contents of this syntax tree + /// node, or [`Span::call_site()`] if this node is empty. + /// + /// [`Span::call_site()`]: proc_macro2::Span::call_site + fn span(&self) -> Span; +} + +impl<T: ?Sized + ToTokens> Spanned for T { + fn span(&self) -> Span { + self.__span() + } +} + +mod private { + use crate::spanned::ToTokens; + + pub trait Sealed {} + impl<T: ?Sized + ToTokens> Sealed for T {} + + #[cfg(any(feature = "full", feature = "derive"))] + impl Sealed for crate::QSelf {} +} diff --git a/third_party/crates/syn-2.0.119/src/stmt.rs b/third_party/crates/syn-2.0.119/src/stmt.rs new file mode 100644 index 0000000..b2bb027 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/stmt.rs @@ -0,0 +1,504 @@ +use crate::attr::Attribute; +use crate::expr::Expr; +use crate::item::Item; +use crate::mac::Macro; +use crate::pat::Pat; +use crate::token; +use alloc::boxed::Box; +use alloc::vec::Vec; + +ast_struct! { + /// A braced block containing Rust statements. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct Block { + pub brace_token: token::Brace, + /// Statements in a block + pub stmts: Vec<Stmt>, + } +} + +ast_enum! { + /// A statement, usually ending in a semicolon. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub enum Stmt { + /// A local (let) binding. + Local(Local), + + /// An item definition. + Item(Item), + + /// Expression, with or without trailing semicolon. + Expr(Expr, Option<Token![;]>), + + /// A macro invocation in statement position. + /// + /// Syntactically it's ambiguous which other kind of statement this + /// macro would expand to. It can be any of local variable (`let`), + /// item, or expression. + Macro(StmtMacro), + } +} + +ast_struct! { + /// A local `let` binding: `let x: u64 = s.parse()?;`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct Local { + pub attrs: Vec<Attribute>, + pub let_token: Token![let], + pub pat: Pat, + pub init: Option<LocalInit>, + pub semi_token: Token![;], + } +} + +ast_struct! { + /// The expression assigned in a local `let` binding, including optional + /// diverging `else` block. + /// + /// `LocalInit` represents `= s.parse()?` in `let x: u64 = s.parse()?` and + /// `= r else { return }` in `let Ok(x) = r else { return }`. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct LocalInit { + pub eq_token: Token![=], + pub expr: Box<Expr>, + pub diverge: Option<(Token![else], Box<Expr>)>, + } +} + +ast_struct! { + /// A macro invocation in statement position. + /// + /// Syntactically it's ambiguous which other kind of statement this macro + /// would expand to. It can be any of local variable (`let`), item, or + /// expression. + #[cfg_attr(docsrs, doc(cfg(feature = "full")))] + pub struct StmtMacro { + pub attrs: Vec<Attribute>, + pub mac: Macro, + pub semi_token: Option<Token![;]>, + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::buffer::Cursor; + use crate::classify; + use crate::error::Result; + use crate::expr::{Expr, ExprBlock, ExprMacro}; + use crate::ident::Ident; + use crate::item; + use crate::mac::{self, Macro}; + use crate::parse::discouraged::Speculative as _; + use crate::parse::{Parse, ParseStream}; + use crate::pat::{Pat, PatType}; + use crate::path::Path; + use crate::stmt::{Block, Local, LocalInit, Stmt, StmtMacro}; + use crate::token; + use crate::ty::Type; + use crate::verbatim; + use alloc::boxed::Box; + use alloc::vec::Vec; + use core::mem; + use proc_macro2::TokenStream; + + struct AllowNoSemi(bool); + + impl Block { + /// Parse the body of a block as zero or more statements, possibly + /// including one trailing expression. + /// + /// # Example + /// + /// ``` + /// use syn::{braced, token, Attribute, Block, Ident, Result, Stmt, Token}; + /// use syn::parse::{Parse, ParseStream}; + /// + /// // Parse a function with no generics or parameter list. + /// // + /// // fn playground { + /// // let mut x = 1; + /// // x += 1; + /// // println!("{}", x); + /// // } + /// struct MiniFunction { + /// attrs: Vec<Attribute>, + /// fn_token: Token![fn], + /// name: Ident, + /// brace_token: token::Brace, + /// stmts: Vec<Stmt>, + /// } + /// + /// impl Parse for MiniFunction { + /// fn parse(input: ParseStream) -> Result<Self> { + /// let outer_attrs = input.call(Attribute::parse_outer)?; + /// let fn_token: Token![fn] = input.parse()?; + /// let name: Ident = input.parse()?; + /// + /// let content; + /// let brace_token = braced!(content in input); + /// let inner_attrs = content.call(Attribute::parse_inner)?; + /// let stmts = content.call(Block::parse_within)?; + /// + /// Ok(MiniFunction { + /// attrs: { + /// let mut attrs = outer_attrs; + /// attrs.extend(inner_attrs); + /// attrs + /// }, + /// fn_token, + /// name, + /// brace_token, + /// stmts, + /// }) + /// } + /// } + /// ``` + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn parse_within(input: ParseStream) -> Result<Vec<Stmt>> { + let mut stmts = Vec::new(); + loop { + while let semi @ Some(_) = input.parse()? { + stmts.push(Stmt::Expr(Expr::Verbatim(TokenStream::new()), semi)); + } + if input.is_empty() { + break; + } + let stmt = parse_stmt(input, AllowNoSemi(true))?; + let requires_semicolon = match &stmt { + Stmt::Expr(stmt, None) => classify::requires_semi_to_be_stmt(stmt), + Stmt::Macro(stmt) => { + stmt.semi_token.is_none() && !stmt.mac.delimiter.is_brace() + } + Stmt::Local(_) | Stmt::Item(_) | Stmt::Expr(_, Some(_)) => false, + }; + stmts.push(stmt); + if input.is_empty() { + break; + } else if requires_semicolon { + return Err(input.error("unexpected token, expected `;`")); + } + } + Ok(stmts) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Block { + fn parse(input: ParseStream) -> Result<Self> { + let content; + Ok(Block { + brace_token: braced!(content in input), + stmts: content.call(Block::parse_within)?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Stmt { + fn parse(input: ParseStream) -> Result<Self> { + let allow_nosemi = AllowNoSemi(false); + parse_stmt(input, allow_nosemi) + } + } + + fn parse_stmt(input: ParseStream, allow_nosemi: AllowNoSemi) -> Result<Stmt> { + let begin = input.cursor(); + let attrs = input.call(Attribute::parse_outer)?; + let attrs_end = input.cursor(); + + // brace-style macros; paren and bracket macros get parsed as + // expression statements. + let ahead = input.fork(); + let mut is_item_macro = false; + if let Ok(path) = ahead.call(Path::parse_mod_style) { + if ahead.peek(Token![!]) { + if ahead.peek2(Ident) || ahead.peek2(Token![try]) { + is_item_macro = true; + } else if ahead.peek2(token::Brace) + && !(ahead.peek3(Token![.]) && !ahead.peek3(Token![..]) + || ahead.peek3(Token![?])) + { + input.advance_to(&ahead); + return stmt_mac(input, attrs, path).map(Stmt::Macro); + } + } + } + + if input.peek(Token![let]) && !input.peek(token::Group) { + stmt_local(input, attrs).map(Stmt::Local) + } else if input.peek(Token![pub]) + || input.peek(Token![crate]) && !input.peek2(Token![::]) + || input.peek(Token![extern]) + || input.peek(Token![use]) + || input.peek(Token![static]) + && (input.peek2(Token![mut]) + || input.peek2(Ident) + && !(input.peek2(Token![async]) + && (input.peek3(Token![move]) || input.peek3(Token![|])))) + || input.peek(Token![const]) + && !(input.peek2(token::Brace) + || input.peek2(Token![static]) + || input.peek2(Token![async]) + && !(input.peek3(Token![unsafe]) + || input.peek3(Token![extern]) + || input.peek3(Token![fn])) + || input.peek2(Token![move]) + || input.peek2(Token![|])) + || input.peek(Token![unsafe]) && !input.peek2(token::Brace) + || input.peek(Token![async]) + && (input.peek2(Token![unsafe]) + || input.peek2(Token![extern]) + || input.peek2(Token![fn])) + || input.peek(Token![fn]) + || input.peek(Token![mod]) + || input.peek(Token![type]) + || input.peek(Token![struct]) + || input.peek(Token![enum]) + || input.peek(Token![union]) && input.peek2(Ident) + || input.peek(Token![auto]) && input.peek2(Token![trait]) + || input.peek(Token![trait]) + || input.peek(Token![default]) + && (input.peek2(Token![unsafe]) || input.peek2(Token![impl])) + || input.peek(Token![impl]) + || input.peek(Token![macro]) + || is_item_macro + { + let item = item::parsing::parse_rest_of_item(begin, attrs, input)?; + Ok(Stmt::Item(item)) + } else { + stmt_expr(begin, input, allow_nosemi, attrs, attrs_end) + } + } + + fn stmt_mac(input: ParseStream, attrs: Vec<Attribute>, path: Path) -> Result<StmtMacro> { + let bang_token: Token![!] = input.parse()?; + let (delimiter, tokens) = mac::parse_delimiter(input)?; + let semi_token: Option<Token![;]> = input.parse()?; + + Ok(StmtMacro { + attrs, + mac: Macro { + path, + bang_token, + delimiter, + tokens, + }, + semi_token, + }) + } + + fn stmt_local(input: ParseStream, attrs: Vec<Attribute>) -> Result<Local> { + let let_token: Token![let] = input.parse()?; + + let mut pat = Pat::parse_single(input)?; + if input.peek(Token![:]) { + let colon_token: Token![:] = input.parse()?; + let ty: Type = input.parse()?; + pat = Pat::Type(PatType { + attrs: Vec::new(), + pat: Box::new(pat), + colon_token, + ty: Box::new(ty), + }); + } + + let init = if let Some(eq_token) = input.parse()? { + let eq_token: Token![=] = eq_token; + let expr: Expr = input.parse()?; + + let diverge = if !classify::expr_trailing_brace(&expr) && input.peek(Token![else]) { + let else_token: Token![else] = input.parse()?; + let diverge = ExprBlock { + attrs: Vec::new(), + label: None, + block: input.parse()?, + }; + Some((else_token, Box::new(Expr::Block(diverge)))) + } else { + None + }; + + Some(LocalInit { + eq_token, + expr: Box::new(expr), + diverge, + }) + } else { + None + }; + + let semi_token: Token![;] = input.parse()?; + + Ok(Local { + attrs, + let_token, + pat, + init, + semi_token, + }) + } + + fn stmt_expr( + begin: Cursor, + input: ParseStream, + allow_nosemi: AllowNoSemi, + mut attrs: Vec<Attribute>, + attrs_end: Cursor, + ) -> Result<Stmt> { + let mut e = Expr::parse_with_earlier_boundary_rule(input)?; + + let mut attr_target = &mut e; + loop { + attr_target = match attr_target { + Expr::Assign(e) => &mut e.left, + Expr::Binary(e) => &mut e.left, + Expr::Cast(e) => &mut e.expr, + Expr::Array(_) + | Expr::Async(_) + | Expr::Await(_) + | Expr::Block(_) + | Expr::Break(_) + | Expr::Call(_) + | Expr::Closure(_) + | Expr::Const(_) + | Expr::Continue(_) + | Expr::Field(_) + | Expr::ForLoop(_) + | Expr::Group(_) + | Expr::If(_) + | Expr::Index(_) + | Expr::Infer(_) + | Expr::Let(_) + | Expr::Lit(_) + | Expr::Loop(_) + | Expr::Macro(_) + | Expr::Match(_) + | Expr::MethodCall(_) + | Expr::Paren(_) + | Expr::Path(_) + | Expr::Range(_) + | Expr::RawAddr(_) + | Expr::Reference(_) + | Expr::Repeat(_) + | Expr::Return(_) + | Expr::Struct(_) + | Expr::Try(_) + | Expr::TryBlock(_) + | Expr::Tuple(_) + | Expr::Unary(_) + | Expr::Unsafe(_) + | Expr::While(_) + | Expr::Yield(_) + | Expr::Verbatim(_) => break, + }; + } + + if !attrs.is_empty() { + if let Expr::Verbatim(expr_tokens) = attr_target { + let mut attr_tokens = verbatim::between(begin, attrs_end); + attr_tokens.extend(mem::replace(expr_tokens, TokenStream::new())); + *expr_tokens = attr_tokens; + } else { + let inner_attrs = attr_target.replace_attrs(Vec::new()); + attrs.extend(inner_attrs); + attr_target.replace_attrs(attrs); + } + } + + let semi_token: Option<Token![;]> = input.parse()?; + + match e { + Expr::Macro(ExprMacro { attrs, mac }) + if semi_token.is_some() || mac.delimiter.is_brace() => + { + return Ok(Stmt::Macro(StmtMacro { + attrs, + mac, + semi_token, + })); + } + _ => {} + } + + if semi_token.is_some() { + Ok(Stmt::Expr(e, semi_token)) + } else if allow_nosemi.0 || !classify::requires_semi_to_be_stmt(&e) { + Ok(Stmt::Expr(e, None)) + } else { + Err(input.error("expected semicolon")) + } + } +} + +#[cfg(feature = "printing")] +pub(crate) mod printing { + use crate::classify; + use crate::expr::{self, Expr}; + use crate::fixup::FixupContext; + use crate::stmt::{Block, Local, Stmt, StmtMacro}; + use crate::token; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Block { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.brace_token.surround(tokens, |tokens| { + tokens.append_all(&self.stmts); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Stmt { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Stmt::Local(local) => local.to_tokens(tokens), + Stmt::Item(item) => item.to_tokens(tokens), + Stmt::Expr(expr, semi) => { + expr::printing::print_expr(expr, tokens, FixupContext::new_stmt()); + semi.to_tokens(tokens); + } + Stmt::Macro(mac) => mac.to_tokens(tokens), + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Local { + fn to_tokens(&self, tokens: &mut TokenStream) { + expr::printing::outer_attrs_to_tokens(&self.attrs, tokens); + self.let_token.to_tokens(tokens); + self.pat.to_tokens(tokens); + if let Some(init) = &self.init { + init.eq_token.to_tokens(tokens); + expr::printing::print_subexpression( + &init.expr, + init.diverge.is_some() && classify::expr_trailing_brace(&init.expr), + tokens, + FixupContext::NONE, + ); + if let Some((else_token, diverge)) = &init.diverge { + else_token.to_tokens(tokens); + match &**diverge { + Expr::Block(diverge) => diverge.to_tokens(tokens), + _ => token::Brace::default().surround(tokens, |tokens| { + expr::printing::print_expr(diverge, tokens, FixupContext::new_stmt()); + }), + } + } + } + self.semi_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for StmtMacro { + fn to_tokens(&self, tokens: &mut TokenStream) { + expr::printing::outer_attrs_to_tokens(&self.attrs, tokens); + self.mac.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/thread.rs b/third_party/crates/syn-2.0.119/src/thread.rs new file mode 100644 index 0000000..82accac --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/thread.rs @@ -0,0 +1,60 @@ +use core::fmt::{self, Debug}; +use std::thread::{self, ThreadId}; + +/// ThreadBound is a Sync-maker and Send-maker that allows accessing a value +/// of type T only from the original thread on which the ThreadBound was +/// constructed. +pub(crate) struct ThreadBound<T> { + value: T, + thread_id: ThreadId, +} + +unsafe impl<T> Sync for ThreadBound<T> {} + +// Send bound requires Copy, as otherwise Drop could run in the wrong place. +// +// Today Copy and Drop are mutually exclusive so `T: Copy` implies `T: !Drop`. +// This impl needs to be revisited if that restriction is relaxed in the future. +unsafe impl<T: Copy> Send for ThreadBound<T> {} + +impl<T> ThreadBound<T> { + pub(crate) fn new(value: T) -> Self { + ThreadBound { + value, + thread_id: thread::current().id(), + } + } + + pub(crate) fn get(&self) -> Option<&T> { + if thread::current().id() == self.thread_id { + Some(&self.value) + } else { + None + } + } +} + +impl<T: Debug> Debug for ThreadBound<T> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match self.get() { + Some(value) => Debug::fmt(value, formatter), + None => formatter.write_str("unknown"), + } + } +} + +// Copy the bytes of T, even if the currently running thread is the "wrong" +// thread. This is fine as long as the original thread is not simultaneously +// mutating this value via interior mutability, which would be a data race. +// +// Currently `T: Copy` is sufficient to guarantee that T contains no interior +// mutability, because _all_ interior mutability in Rust is built on +// core::cell::UnsafeCell, which has no Copy impl. This impl needs to be +// revisited if that restriction is relaxed in the future. +impl<T: Copy> Copy for ThreadBound<T> {} + +impl<T: Copy> Clone for ThreadBound<T> { + fn clone(&self) -> Self { + *self + } +} diff --git a/third_party/crates/syn-2.0.119/src/token.rs b/third_party/crates/syn-2.0.119/src/token.rs new file mode 100644 index 0000000..32c5a01 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/token.rs @@ -0,0 +1,1104 @@ +//! Tokens representing Rust punctuation, keywords, and delimiters. +//! +//! The type names in this module can be difficult to keep straight, so we +//! prefer to use the [`Token!`] macro instead. This is a type-macro that +//! expands to the token type of the given token. +//! +//! [`Token!`]: crate::Token +//! +//! # Example +//! +//! The [`ItemStatic`] syntax tree node is defined like this. +//! +//! [`ItemStatic`]: crate::ItemStatic +//! +//! ``` +//! # use syn::{Attribute, Expr, Ident, Token, Type, Visibility}; +//! # +//! pub struct ItemStatic { +//! pub attrs: Vec<Attribute>, +//! pub vis: Visibility, +//! pub static_token: Token![static], +//! pub mutability: Option<Token![mut]>, +//! pub ident: Ident, +//! pub colon_token: Token![:], +//! pub ty: Box<Type>, +//! pub eq_token: Token![=], +//! pub expr: Box<Expr>, +//! pub semi_token: Token![;], +//! } +//! ``` +//! +//! # Parsing +//! +//! Keywords and punctuation can be parsed through the [`ParseStream::parse`] +//! method. Delimiter tokens are parsed using the [`parenthesized!`], +//! [`bracketed!`] and [`braced!`] macros. +//! +//! [`ParseStream::parse`]: crate::parse::ParseBuffer::parse() +//! [`parenthesized!`]: crate::parenthesized! +//! [`bracketed!`]: crate::bracketed! +//! [`braced!`]: crate::braced! +//! +//! ``` +//! use syn::{Attribute, Result}; +//! use syn::parse::{Parse, ParseStream}; +//! # +//! # enum ItemStatic {} +//! +//! // Parse the ItemStatic struct shown above. +//! impl Parse for ItemStatic { +//! fn parse(input: ParseStream) -> Result<Self> { +//! # use syn::ItemStatic; +//! # fn parse(input: ParseStream) -> Result<ItemStatic> { +//! Ok(ItemStatic { +//! attrs: input.call(Attribute::parse_outer)?, +//! vis: input.parse()?, +//! static_token: input.parse()?, +//! mutability: input.parse()?, +//! ident: input.parse()?, +//! colon_token: input.parse()?, +//! ty: input.parse()?, +//! eq_token: input.parse()?, +//! expr: input.parse()?, +//! semi_token: input.parse()?, +//! }) +//! # } +//! # unimplemented!() +//! } +//! } +//! ``` +//! +//! # Other operations +//! +//! Every keyword and punctuation token supports the following operations. +//! +//! - [Peeking] — `input.peek(Token![...])` +//! +//! - [Parsing] — `input.parse::<Token![...]>()?` +//! +//! - [Printing] — `quote!( ... #the_token ... )` +//! +//! - Construction from a [`Span`] — `let the_token = Token![...](sp)` +//! +//! - Field access to its span — `let sp = the_token.span` +//! +//! [Peeking]: crate::parse::ParseBuffer::peek() +//! [Parsing]: crate::parse::ParseBuffer::parse() +//! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html +//! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html + +#[cfg(feature = "parsing")] +pub(crate) use self::private::CustomToken; +use self::private::WithSpan; +#[cfg(feature = "parsing")] +use crate::buffer::Cursor; +#[cfg(feature = "parsing")] +use crate::error::Result; +#[cfg(feature = "parsing")] +use crate::lifetime::Lifetime; +#[cfg(feature = "parsing")] +use crate::parse::{Parse, ParseStream}; +use crate::span::IntoSpans; +#[cfg(feature = "extra-traits")] +use core::cmp; +#[cfg(feature = "extra-traits")] +use core::fmt::{self, Debug}; +#[cfg(feature = "extra-traits")] +use core::hash::{Hash, Hasher}; +use core::ops::{Deref, DerefMut}; +use proc_macro2::extra::DelimSpan; +use proc_macro2::Span; +#[cfg(feature = "printing")] +use proc_macro2::TokenStream; +#[cfg(any(feature = "parsing", feature = "printing"))] +use proc_macro2::{Delimiter, Ident}; +#[cfg(feature = "parsing")] +use proc_macro2::{Literal, Punct, TokenTree}; +#[cfg(feature = "printing")] +use quote::{ToTokens, TokenStreamExt as _}; + +/// Marker trait for types that represent single tokens. +/// +/// This trait is sealed and cannot be implemented for types outside of Syn. +#[cfg(feature = "parsing")] +pub trait Token: private::Sealed { + // Not public API. + #[doc(hidden)] + fn peek(cursor: Cursor) -> bool; + + // Not public API. + #[doc(hidden)] + fn display() -> &'static str; +} + +pub(crate) mod private { + #[cfg(feature = "parsing")] + use crate::buffer::Cursor; + use proc_macro2::Span; + + #[cfg(feature = "parsing")] + pub trait Sealed {} + + /// Support writing `token.span` rather than `token.spans[0]` on tokens that + /// hold a single span. + #[repr(transparent)] + #[allow( + unknown_lints, + renamed_and_removed_lints, + // False positive: https://github.com/rust-lang/rust/issues/115922 + repr_transparent_non_zst_fields, + )] + pub struct WithSpan { + pub span: Span, + } + + // Not public API. + #[doc(hidden)] + #[cfg(feature = "parsing")] + pub trait CustomToken { + fn peek(cursor: Cursor) -> bool; + fn display() -> &'static str; + } +} + +#[cfg(feature = "parsing")] +impl private::Sealed for Ident {} + +macro_rules! impl_low_level_token { + ($display:literal $($path:ident)::+ $get:ident) => { + #[cfg(feature = "parsing")] + impl Token for $($path)::+ { + fn peek(cursor: Cursor) -> bool { + cursor.$get().is_some() + } + + fn display() -> &'static str { + $display + } + } + + #[cfg(feature = "parsing")] + impl private::Sealed for $($path)::+ {} + }; +} + +impl_low_level_token!("punctuation token" Punct punct); +impl_low_level_token!("literal" Literal literal); +impl_low_level_token!("token" TokenTree token_tree); +impl_low_level_token!("group token" proc_macro2::Group any_group); +impl_low_level_token!("lifetime" Lifetime lifetime); + +#[cfg(feature = "parsing")] +impl<T: CustomToken> private::Sealed for T {} + +#[cfg(feature = "parsing")] +impl<T: CustomToken> Token for T { + fn peek(cursor: Cursor) -> bool { + <Self as CustomToken>::peek(cursor) + } + + fn display() -> &'static str { + <Self as CustomToken>::display() + } +} + +macro_rules! define_keywords { + ($($token:literal pub struct $name:ident)*) => { + $( + #[doc = concat!('`', $token, '`')] + /// + /// Don't try to remember the name of this type — use the + /// [`Token!`] macro instead. + /// + /// [`Token!`]: crate::token + pub struct $name { + pub span: Span, + } + + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn $name<S: IntoSpans<Span>>(span: S) -> $name { + $name { + span: span.into_spans(), + } + } + + impl core::default::Default for $name { + fn default() -> Self { + $name { + span: Span::call_site(), + } + } + } + + #[cfg(feature = "clone-impls")] + #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] + impl Copy for $name {} + + #[cfg(feature = "clone-impls")] + #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] + impl Clone for $name { + fn clone(&self) -> Self { + *self + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(stringify!($name)) + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl cmp::Eq for $name {} + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl PartialEq for $name { + fn eq(&self, _other: &$name) -> bool { + true + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Hash for $name { + fn hash<H: Hasher>(&self, _state: &mut H) {} + } + + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for $name { + fn to_tokens(&self, tokens: &mut TokenStream) { + printing::keyword($token, self.span, tokens); + } + } + + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for $name { + fn parse(input: ParseStream) -> Result<Self> { + Ok($name { + span: parsing::keyword(input, $token)?, + }) + } + } + + #[cfg(feature = "parsing")] + impl Token for $name { + fn peek(cursor: Cursor) -> bool { + parsing::peek_keyword(cursor, $token) + } + + fn display() -> &'static str { + concat!("`", $token, "`") + } + } + + #[cfg(feature = "parsing")] + impl private::Sealed for $name {} + )* + }; +} + +macro_rules! impl_deref_if_len_is_1 { + ($name:ident/1) => { + impl Deref for $name { + type Target = WithSpan; + + fn deref(&self) -> &Self::Target { + unsafe { &*(self as *const Self).cast::<WithSpan>() } + } + } + + impl DerefMut for $name { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { &mut *(self as *mut Self).cast::<WithSpan>() } + } + } + }; + + ($name:ident/$len:literal) => {}; +} + +macro_rules! define_punctuation_structs { + ($($token:literal pub struct $name:ident/$len:tt #[doc = $usage:literal])*) => { + $( + #[cfg_attr(not(doc), repr(transparent))] + #[allow( + unknown_lints, + renamed_and_removed_lints, + // False positive: https://github.com/rust-lang/rust/issues/115922 + repr_transparent_non_zst_fields, + )] + #[doc = concat!('`', $token, '`')] + /// + /// Usage: + #[doc = concat!($usage, '.')] + /// + /// Don't try to remember the name of this type — use the + /// [`Token!`] macro instead. + /// + /// [`Token!`]: crate::token + pub struct $name { + pub spans: [Span; $len], + } + + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn $name<S: IntoSpans<[Span; $len]>>(spans: S) -> $name { + $name { + spans: spans.into_spans(), + } + } + + impl core::default::Default for $name { + fn default() -> Self { + $name { + spans: [Span::call_site(); $len], + } + } + } + + #[cfg(feature = "clone-impls")] + #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] + impl Copy for $name {} + + #[cfg(feature = "clone-impls")] + #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] + impl Clone for $name { + fn clone(&self) -> Self { + *self + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(stringify!($name)) + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl cmp::Eq for $name {} + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl PartialEq for $name { + fn eq(&self, _other: &$name) -> bool { + true + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Hash for $name { + fn hash<H: Hasher>(&self, _state: &mut H) {} + } + + impl_deref_if_len_is_1!($name/$len); + )* + }; +} + +macro_rules! define_punctuation { + ($($token:literal pub struct $name:ident/$len:tt #[doc = $usage:literal])*) => { + $( + define_punctuation_structs! { + $token pub struct $name/$len #[doc = $usage] + } + + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for $name { + fn to_tokens(&self, tokens: &mut TokenStream) { + printing::punct($token, &self.spans, tokens); + } + } + + #[cfg(feature = "parsing")] + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for $name { + fn parse(input: ParseStream) -> Result<Self> { + Ok($name { + spans: parsing::punct(input, $token)?, + }) + } + } + + #[cfg(feature = "parsing")] + impl Token for $name { + fn peek(cursor: Cursor) -> bool { + parsing::peek_punct(cursor, $token) + } + + fn display() -> &'static str { + concat!("`", $token, "`") + } + } + + #[cfg(feature = "parsing")] + impl private::Sealed for $name {} + )* + }; +} + +macro_rules! define_delimiters { + ($($delim:ident pub struct $name:ident #[$doc:meta])*) => { + $( + #[$doc] + pub struct $name { + pub span: DelimSpan, + } + + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn $name<S: IntoSpans<DelimSpan>>(span: S) -> $name { + $name { + span: span.into_spans(), + } + } + + impl core::default::Default for $name { + fn default() -> Self { + $name(Span::call_site()) + } + } + + #[cfg(feature = "clone-impls")] + #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] + impl Copy for $name {} + + #[cfg(feature = "clone-impls")] + #[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] + impl Clone for $name { + fn clone(&self) -> Self { + *self + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(stringify!($name)) + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl cmp::Eq for $name {} + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl PartialEq for $name { + fn eq(&self, _other: &$name) -> bool { + true + } + } + + #[cfg(feature = "extra-traits")] + #[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] + impl Hash for $name { + fn hash<H: Hasher>(&self, _state: &mut H) {} + } + + impl $name { + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + pub fn surround<F>(&self, tokens: &mut TokenStream, f: F) + where + F: FnOnce(&mut TokenStream), + { + let mut inner = TokenStream::new(); + f(&mut inner); + printing::delim(Delimiter::$delim, self.span.join(), tokens, inner); + } + } + + #[cfg(feature = "parsing")] + impl private::Sealed for $name {} + )* + }; +} + +define_punctuation_structs! { + "_" pub struct Underscore/1 /// wildcard patterns, inferred types, unnamed items in constants, extern crates, use declarations, and destructuring assignment +} + +#[cfg(feature = "printing")] +#[cfg_attr(docsrs, doc(cfg(feature = "printing")))] +impl ToTokens for Underscore { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append(Ident::new("_", self.span)); + } +} + +#[cfg(feature = "parsing")] +#[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] +impl Parse for Underscore { + fn parse(input: ParseStream) -> Result<Self> { + input.step(|cursor| { + if let Some((ident, rest)) = cursor.ident() { + if ident == "_" { + return Ok((Underscore(ident.span()), rest)); + } + } + if let Some((punct, rest)) = cursor.punct() { + if punct.as_char() == '_' { + return Ok((Underscore(punct.span()), rest)); + } + } + Err(cursor.error("expected `_`")) + }) + } +} + +#[cfg(feature = "parsing")] +impl Token for Underscore { + fn peek(cursor: Cursor) -> bool { + if let Some((ident, _rest)) = cursor.ident() { + return ident == "_"; + } + if let Some((punct, _rest)) = cursor.punct() { + return punct.as_char() == '_'; + } + false + } + + fn display() -> &'static str { + "`_`" + } +} + +#[cfg(feature = "parsing")] +impl private::Sealed for Underscore {} + +/// None-delimited group +pub struct Group { + pub span: Span, +} + +#[doc(hidden)] +#[allow(non_snake_case)] +pub fn Group<S: IntoSpans<Span>>(span: S) -> Group { + Group { + span: span.into_spans(), + } +} + +impl core::default::Default for Group { + fn default() -> Self { + Group { + span: Span::call_site(), + } + } +} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Copy for Group {} + +#[cfg(feature = "clone-impls")] +#[cfg_attr(docsrs, doc(cfg(feature = "clone-impls")))] +impl Clone for Group { + fn clone(&self) -> Self { + *self + } +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Debug for Group { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("Group") + } +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl cmp::Eq for Group {} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl PartialEq for Group { + fn eq(&self, _other: &Group) -> bool { + true + } +} + +#[cfg(feature = "extra-traits")] +#[cfg_attr(docsrs, doc(cfg(feature = "extra-traits")))] +impl Hash for Group { + fn hash<H: Hasher>(&self, _state: &mut H) {} +} + +impl Group { + #[cfg(feature = "printing")] + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + pub fn surround<F>(&self, tokens: &mut TokenStream, f: F) + where + F: FnOnce(&mut TokenStream), + { + let mut inner = TokenStream::new(); + f(&mut inner); + printing::delim(Delimiter::None, self.span, tokens, inner); + } +} + +#[cfg(feature = "parsing")] +impl private::Sealed for Group {} + +#[cfg(feature = "parsing")] +impl Token for Paren { + fn peek(cursor: Cursor) -> bool { + cursor.group(Delimiter::Parenthesis).is_some() + } + + fn display() -> &'static str { + "parentheses" + } +} + +#[cfg(feature = "parsing")] +impl Token for Brace { + fn peek(cursor: Cursor) -> bool { + cursor.group(Delimiter::Brace).is_some() + } + + fn display() -> &'static str { + "curly braces" + } +} + +#[cfg(feature = "parsing")] +impl Token for Bracket { + fn peek(cursor: Cursor) -> bool { + cursor.group(Delimiter::Bracket).is_some() + } + + fn display() -> &'static str { + "square brackets" + } +} + +#[cfg(feature = "parsing")] +impl Token for Group { + fn peek(cursor: Cursor) -> bool { + cursor.group(Delimiter::None).is_some() + } + + fn display() -> &'static str { + "invisible group" + } +} + +define_keywords! { + "abstract" pub struct Abstract + "as" pub struct As + "async" pub struct Async + "auto" pub struct Auto + "await" pub struct Await + "become" pub struct Become + "box" pub struct Box + "break" pub struct Break + "const" pub struct Const + "continue" pub struct Continue + "crate" pub struct Crate + "default" pub struct Default + "do" pub struct Do + "dyn" pub struct Dyn + "else" pub struct Else + "enum" pub struct Enum + "extern" pub struct Extern + "final" pub struct Final + "fn" pub struct Fn + "for" pub struct For + "if" pub struct If + "impl" pub struct Impl + "in" pub struct In + "let" pub struct Let + "loop" pub struct Loop + "macro" pub struct Macro + "match" pub struct Match + "mod" pub struct Mod + "move" pub struct Move + "mut" pub struct Mut + "override" pub struct Override + "priv" pub struct Priv + "pub" pub struct Pub + "raw" pub struct Raw + "ref" pub struct Ref + "return" pub struct Return + "Self" pub struct SelfType + "self" pub struct SelfValue + "static" pub struct Static + "struct" pub struct Struct + "super" pub struct Super + "trait" pub struct Trait + "try" pub struct Try + "type" pub struct Type + "typeof" pub struct Typeof + "union" pub struct Union + "unsafe" pub struct Unsafe + "unsized" pub struct Unsized + "use" pub struct Use + "virtual" pub struct Virtual + "where" pub struct Where + "while" pub struct While + "yield" pub struct Yield +} + +define_punctuation! { + "&" pub struct And/1 /// bitwise and logical AND, borrow, references, reference patterns + "&&" pub struct AndAnd/2 /// lazy AND, borrow, references, reference patterns + "&=" pub struct AndEq/2 /// bitwise AND assignment + "@" pub struct At/1 /// subpattern binding + "^" pub struct Caret/1 /// bitwise and logical XOR + "^=" pub struct CaretEq/2 /// bitwise XOR assignment + ":" pub struct Colon/1 /// various separators + "," pub struct Comma/1 /// various separators + "$" pub struct Dollar/1 /// macros + "." pub struct Dot/1 /// field access, tuple index + ".." pub struct DotDot/2 /// range, struct expressions, patterns, range patterns + "..." pub struct DotDotDot/3 /// variadic functions, range patterns + "..=" pub struct DotDotEq/3 /// inclusive range, range patterns + "=" pub struct Eq/1 /// assignment, attributes, various type definitions + "==" pub struct EqEq/2 /// equal + "=>" pub struct FatArrow/2 /// match arms, macros + ">=" pub struct Ge/2 /// greater than or equal to, generics + ">" pub struct Gt/1 /// greater than, generics, paths + "<-" pub struct LArrow/2 /// unused + "<=" pub struct Le/2 /// less than or equal to + "<" pub struct Lt/1 /// less than, generics, paths + "-" pub struct Minus/1 /// subtraction, negation + "-=" pub struct MinusEq/2 /// subtraction assignment + "!=" pub struct Ne/2 /// not equal + "!" pub struct Not/1 /// bitwise and logical NOT, macro calls, inner attributes, never type, negative impls + "|" pub struct Or/1 /// bitwise and logical OR, closures, patterns in match, if let, and while let + "|=" pub struct OrEq/2 /// bitwise OR assignment + "||" pub struct OrOr/2 /// lazy OR, closures + "::" pub struct PathSep/2 /// path separator + "%" pub struct Percent/1 /// remainder + "%=" pub struct PercentEq/2 /// remainder assignment + "+" pub struct Plus/1 /// addition, trait bounds, macro Kleene matcher + "+=" pub struct PlusEq/2 /// addition assignment + "#" pub struct Pound/1 /// attributes + "?" pub struct Question/1 /// question mark operator, questionably sized, macro Kleene matcher + "->" pub struct RArrow/2 /// function return type, closure return type, function pointer type + ";" pub struct Semi/1 /// terminator for various items and statements, array types + "<<" pub struct Shl/2 /// shift left, nested generics + "<<=" pub struct ShlEq/3 /// shift left assignment + ">>" pub struct Shr/2 /// shift right, nested generics + ">>=" pub struct ShrEq/3 /// shift right assignment, nested generics + "/" pub struct Slash/1 /// division + "/=" pub struct SlashEq/2 /// division assignment + "*" pub struct Star/1 /// multiplication, dereference, raw pointers, macro Kleene matcher, use wildcards + "*=" pub struct StarEq/2 /// multiplication assignment + "~" pub struct Tilde/1 /// unused since before Rust 1.0 +} + +define_delimiters! { + Brace pub struct Brace /// `{`…`}` + Bracket pub struct Bracket /// `[`…`]` + Parenthesis pub struct Paren /// `(`…`)` +} + +/// A type-macro that expands to the name of the Rust type representation of a +/// given token. +/// +/// As a type, `Token!` is commonly used in the type of struct fields, the type +/// of a `let` statement, or in turbofish for a `parse` function. +/// +/// ``` +/// use syn::{Ident, Token}; +/// use syn::parse::{Parse, ParseStream, Result}; +/// +/// // `struct Foo;` +/// pub struct UnitStruct { +/// struct_token: Token![struct], +/// ident: Ident, +/// semi_token: Token![;], +/// } +/// +/// impl Parse for UnitStruct { +/// fn parse(input: ParseStream) -> Result<Self> { +/// let struct_token: Token![struct] = input.parse()?; +/// let ident: Ident = input.parse()?; +/// let semi_token = input.parse::<Token![;]>()?; +/// Ok(UnitStruct { struct_token, ident, semi_token }) +/// } +/// } +/// ``` +/// +/// As an expression, `Token!` is used for peeking tokens or instantiating +/// tokens from a span. +/// +/// ``` +/// # use syn::{Ident, Token}; +/// # use syn::parse::{Parse, ParseStream, Result}; +/// # +/// # struct UnitStruct { +/// # struct_token: Token![struct], +/// # ident: Ident, +/// # semi_token: Token![;], +/// # } +/// # +/// # impl Parse for UnitStruct { +/// # fn parse(input: ParseStream) -> Result<Self> { +/// # unimplemented!() +/// # } +/// # } +/// # +/// fn make_unit_struct(name: Ident) -> UnitStruct { +/// let span = name.span(); +/// UnitStruct { +/// struct_token: Token![struct](span), +/// ident: name, +/// semi_token: Token![;](span), +/// } +/// } +/// +/// # fn parse(input: ParseStream) -> Result<()> { +/// if input.peek(Token![struct]) { +/// let unit_struct: UnitStruct = input.parse()?; +/// /* ... */ +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// See the [token module] documentation for details and examples. +/// +/// [token module]: crate::token +#[macro_export] +macro_rules! Token { + [abstract] => { $crate::token::Abstract }; + [as] => { $crate::token::As }; + [async] => { $crate::token::Async }; + [auto] => { $crate::token::Auto }; + [await] => { $crate::token::Await }; + [become] => { $crate::token::Become }; + [box] => { $crate::token::Box }; + [break] => { $crate::token::Break }; + [const] => { $crate::token::Const }; + [continue] => { $crate::token::Continue }; + [crate] => { $crate::token::Crate }; + [default] => { $crate::token::Default }; + [do] => { $crate::token::Do }; + [dyn] => { $crate::token::Dyn }; + [else] => { $crate::token::Else }; + [enum] => { $crate::token::Enum }; + [extern] => { $crate::token::Extern }; + [final] => { $crate::token::Final }; + [fn] => { $crate::token::Fn }; + [for] => { $crate::token::For }; + [if] => { $crate::token::If }; + [impl] => { $crate::token::Impl }; + [in] => { $crate::token::In }; + [let] => { $crate::token::Let }; + [loop] => { $crate::token::Loop }; + [macro] => { $crate::token::Macro }; + [match] => { $crate::token::Match }; + [mod] => { $crate::token::Mod }; + [move] => { $crate::token::Move }; + [mut] => { $crate::token::Mut }; + [override] => { $crate::token::Override }; + [priv] => { $crate::token::Priv }; + [pub] => { $crate::token::Pub }; + [raw] => { $crate::token::Raw }; + [ref] => { $crate::token::Ref }; + [return] => { $crate::token::Return }; + [Self] => { $crate::token::SelfType }; + [self] => { $crate::token::SelfValue }; + [static] => { $crate::token::Static }; + [struct] => { $crate::token::Struct }; + [super] => { $crate::token::Super }; + [trait] => { $crate::token::Trait }; + [try] => { $crate::token::Try }; + [type] => { $crate::token::Type }; + [typeof] => { $crate::token::Typeof }; + [union] => { $crate::token::Union }; + [unsafe] => { $crate::token::Unsafe }; + [unsized] => { $crate::token::Unsized }; + [use] => { $crate::token::Use }; + [virtual] => { $crate::token::Virtual }; + [where] => { $crate::token::Where }; + [while] => { $crate::token::While }; + [yield] => { $crate::token::Yield }; + [&] => { $crate::token::And }; + [&&] => { $crate::token::AndAnd }; + [&=] => { $crate::token::AndEq }; + [@] => { $crate::token::At }; + [^] => { $crate::token::Caret }; + [^=] => { $crate::token::CaretEq }; + [:] => { $crate::token::Colon }; + [,] => { $crate::token::Comma }; + [$] => { $crate::token::Dollar }; + [.] => { $crate::token::Dot }; + [..] => { $crate::token::DotDot }; + [...] => { $crate::token::DotDotDot }; + [..=] => { $crate::token::DotDotEq }; + [=] => { $crate::token::Eq }; + [==] => { $crate::token::EqEq }; + [=>] => { $crate::token::FatArrow }; + [>=] => { $crate::token::Ge }; + [>] => { $crate::token::Gt }; + [<-] => { $crate::token::LArrow }; + [<=] => { $crate::token::Le }; + [<] => { $crate::token::Lt }; + [-] => { $crate::token::Minus }; + [-=] => { $crate::token::MinusEq }; + [!=] => { $crate::token::Ne }; + [!] => { $crate::token::Not }; + [|] => { $crate::token::Or }; + [|=] => { $crate::token::OrEq }; + [||] => { $crate::token::OrOr }; + [::] => { $crate::token::PathSep }; + [%] => { $crate::token::Percent }; + [%=] => { $crate::token::PercentEq }; + [+] => { $crate::token::Plus }; + [+=] => { $crate::token::PlusEq }; + [#] => { $crate::token::Pound }; + [?] => { $crate::token::Question }; + [->] => { $crate::token::RArrow }; + [;] => { $crate::token::Semi }; + [<<] => { $crate::token::Shl }; + [<<=] => { $crate::token::ShlEq }; + [>>] => { $crate::token::Shr }; + [>>=] => { $crate::token::ShrEq }; + [/] => { $crate::token::Slash }; + [/=] => { $crate::token::SlashEq }; + [*] => { $crate::token::Star }; + [*=] => { $crate::token::StarEq }; + [~] => { $crate::token::Tilde }; + [_] => { $crate::token::Underscore }; +} + +// Not public API. +#[doc(hidden)] +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::buffer::Cursor; + use crate::error::{Error, Result}; + use crate::parse::ParseStream; + use alloc::format; + use proc_macro2::{Spacing, Span}; + + pub(crate) fn keyword(input: ParseStream, token: &str) -> Result<Span> { + input.step(|cursor| { + if let Some((ident, rest)) = cursor.ident() { + if ident == token { + return Ok((ident.span(), rest)); + } + } + Err(cursor.error(format!("expected `{}`", token))) + }) + } + + pub(crate) fn peek_keyword(cursor: Cursor, token: &str) -> bool { + if let Some((ident, _rest)) = cursor.ident() { + ident == token + } else { + false + } + } + + #[doc(hidden)] + pub fn punct<const N: usize>(input: ParseStream, token: &str) -> Result<[Span; N]> { + let mut spans = [input.span(); N]; + punct_helper(input, token, &mut spans)?; + Ok(spans) + } + + fn punct_helper(input: ParseStream, token: &str, spans: &mut [Span]) -> Result<()> { + input.step(|cursor| { + let mut cursor = *cursor; + assert_eq!(token.len(), spans.len()); + + for (i, ch) in token.chars().enumerate() { + match cursor.punct() { + Some((punct, rest)) => { + spans[i] = punct.span(); + if punct.as_char() != ch { + break; + } else if i == token.len() - 1 { + return Ok(((), rest)); + } else if punct.spacing() != Spacing::Joint { + break; + } + cursor = rest; + } + None => break, + } + } + + Err(Error::new(spans[0], format!("expected `{}`", token))) + }) + } + + #[doc(hidden)] + pub fn peek_punct(mut cursor: Cursor, token: &str) -> bool { + for (i, ch) in token.chars().enumerate() { + match cursor.punct() { + Some((punct, rest)) => { + if punct.as_char() != ch { + break; + } else if i == token.len() - 1 { + return true; + } else if punct.spacing() != Spacing::Joint { + break; + } + cursor = rest; + } + None => break, + } + } + false + } +} + +// Not public API. +#[doc(hidden)] +#[cfg(feature = "printing")] +pub(crate) mod printing { + use crate::ext::PunctExt as _; + use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream}; + use quote::TokenStreamExt as _; + + #[doc(hidden)] + pub fn punct(s: &str, spans: &[Span], tokens: &mut TokenStream) { + assert_eq!(s.len(), spans.len()); + + let mut chars = s.chars(); + let mut spans = spans.iter(); + let ch = chars.next_back().unwrap(); + let span = spans.next_back().unwrap(); + for (ch, span) in chars.zip(spans) { + tokens.append(Punct::new_spanned(ch, Spacing::Joint, *span)); + } + + tokens.append(Punct::new_spanned(ch, Spacing::Alone, *span)); + } + + pub(crate) fn keyword(s: &str, span: Span, tokens: &mut TokenStream) { + tokens.append(Ident::new(s, span)); + } + + pub(crate) fn delim( + delim: Delimiter, + span: Span, + tokens: &mut TokenStream, + inner: TokenStream, + ) { + let mut g = Group::new(delim, inner); + g.set_span(span); + tokens.append(g); + } +} diff --git a/third_party/crates/syn-2.0.119/src/tt.rs b/third_party/crates/syn-2.0.119/src/tt.rs new file mode 100644 index 0000000..63a89ac --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/tt.rs @@ -0,0 +1,97 @@ +use alloc::string::ToString; +use core::hash::{Hash, Hasher}; +use proc_macro2::{Delimiter, Spacing, TokenStream, TokenTree}; + +pub(crate) struct TokenTreeHelper<'a>(pub &'a TokenTree); + +impl<'a> PartialEq for TokenTreeHelper<'a> { + fn eq(&self, other: &Self) -> bool { + match (self.0, other.0) { + (TokenTree::Group(g1), TokenTree::Group(g2)) => { + match (g1.delimiter(), g2.delimiter()) { + (Delimiter::Parenthesis, Delimiter::Parenthesis) + | (Delimiter::Brace, Delimiter::Brace) + | (Delimiter::Bracket, Delimiter::Bracket) + | (Delimiter::None, Delimiter::None) => {} + _ => return false, + } + + TokenStreamHelper(&g1.stream()) == TokenStreamHelper(&g2.stream()) + } + (TokenTree::Punct(o1), TokenTree::Punct(o2)) => { + o1.as_char() == o2.as_char() + && match (o1.spacing(), o2.spacing()) { + (Spacing::Alone, Spacing::Alone) | (Spacing::Joint, Spacing::Joint) => true, + _ => false, + } + } + (TokenTree::Literal(l1), TokenTree::Literal(l2)) => l1.to_string() == l2.to_string(), + (TokenTree::Ident(s1), TokenTree::Ident(s2)) => s1 == s2, + _ => false, + } + } +} + +impl<'a> Hash for TokenTreeHelper<'a> { + fn hash<H: Hasher>(&self, h: &mut H) { + match self.0 { + TokenTree::Group(g) => { + 0u8.hash(h); + match g.delimiter() { + Delimiter::Parenthesis => 0u8.hash(h), + Delimiter::Brace => 1u8.hash(h), + Delimiter::Bracket => 2u8.hash(h), + Delimiter::None => 3u8.hash(h), + } + + for item in g.stream() { + TokenTreeHelper(&item).hash(h); + } + 0xFFu8.hash(h); // terminator w/ a variant we don't normally hash + } + TokenTree::Punct(op) => { + 1u8.hash(h); + op.as_char().hash(h); + match op.spacing() { + Spacing::Alone => 0u8.hash(h), + Spacing::Joint => 1u8.hash(h), + } + } + TokenTree::Literal(lit) => (2u8, lit.to_string()).hash(h), + TokenTree::Ident(word) => (3u8, word).hash(h), + } + } +} + +pub(crate) struct TokenStreamHelper<'a>(pub &'a TokenStream); + +impl<'a> PartialEq for TokenStreamHelper<'a> { + fn eq(&self, other: &Self) -> bool { + let left = self.0.clone().into_iter(); + let mut right = other.0.clone().into_iter(); + + for item1 in left { + let item2 = match right.next() { + Some(item) => item, + None => return false, + }; + if TokenTreeHelper(&item1) != TokenTreeHelper(&item2) { + return false; + } + } + + right.next().is_none() + } +} + +impl<'a> Hash for TokenStreamHelper<'a> { + fn hash<H: Hasher>(&self, state: &mut H) { + let tokens = self.0.clone().into_iter(); + + tokens.clone().count().hash(state); + + for tt in tokens { + TokenTreeHelper(&tt).hash(state); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/ty.rs b/third_party/crates/syn-2.0.119/src/ty.rs new file mode 100644 index 0000000..aa141c5 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/ty.rs @@ -0,0 +1,1286 @@ +use crate::attr::Attribute; +use crate::expr::Expr; +use crate::generics::{BoundLifetimes, TypeParamBound}; +use crate::ident::Ident; +use crate::lifetime::Lifetime; +use crate::lit::LitStr; +use crate::mac::Macro; +use crate::path::{Path, QSelf}; +use crate::punctuated::Punctuated; +use crate::token; +use alloc::boxed::Box; +use alloc::vec::Vec; +use proc_macro2::TokenStream; + +ast_enum_of_structs! { + /// The possible types that a Rust value could have. + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: crate::expr::Expr#syntax-tree-enums + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + #[non_exhaustive] + pub enum Type { + /// A fixed size array type: `[T; n]`. + Array(TypeArray), + + /// A bare function type: `fn(usize) -> bool`. + BareFn(TypeBareFn), + + /// A type contained within invisible delimiters. + Group(TypeGroup), + + /// An `impl Bound1 + Bound2 + Bound3` type where `Bound` is a trait or + /// a lifetime. + ImplTrait(TypeImplTrait), + + /// Indication that a type should be inferred by the compiler: `_`. + Infer(TypeInfer), + + /// A macro in the type position. + Macro(TypeMacro), + + /// The never type: `!`. + Never(TypeNever), + + /// A parenthesized type equivalent to the inner type. + Paren(TypeParen), + + /// A path like `core::slice::Iter`, optionally qualified with a + /// self-type as in `<Vec<T> as SomeTrait>::Associated`. + Path(TypePath), + + /// A raw pointer type: `*const T` or `*mut T`. + Ptr(TypePtr), + + /// A reference type: `&'a T` or `&'a mut T`. + Reference(TypeReference), + + /// A dynamically sized slice type: `[T]`. + Slice(TypeSlice), + + /// A trait object type `dyn Bound1 + Bound2 + Bound3` where `Bound` is a + /// trait or a lifetime. + TraitObject(TypeTraitObject), + + /// A tuple type: `(A, B, C, String)`. + Tuple(TypeTuple), + + /// Tokens in type position not interpreted by Syn. + Verbatim(TokenStream), + + // For testing exhaustiveness in downstream code, use the following idiom: + // + // match ty { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // + // Type::Array(ty) => {...} + // Type::BareFn(ty) => {...} + // ... + // Type::Verbatim(ty) => {...} + // + // _ => { /* some sane fallback */ } + // } + // + // This way we fail your tests but don't break your library when adding + // a variant. You will be notified by a test failure when a variant is + // added, so that you can add code to handle it, but your library will + // continue to compile and work for downstream users in the interim. + } +} + +ast_struct! { + /// A fixed size array type: `[T; n]`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeArray { + pub bracket_token: token::Bracket, + pub elem: Box<Type>, + pub semi_token: Token![;], + pub len: Expr, + } +} + +ast_struct! { + /// A bare function type: `fn(usize) -> bool`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeBareFn { + pub lifetimes: Option<BoundLifetimes>, + pub unsafety: Option<Token![unsafe]>, + pub abi: Option<Abi>, + pub fn_token: Token![fn], + pub paren_token: token::Paren, + pub inputs: Punctuated<BareFnArg, Token![,]>, + pub variadic: Option<BareVariadic>, + pub output: ReturnType, + } +} + +ast_struct! { + /// A type contained within invisible delimiters. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeGroup { + pub group_token: token::Group, + pub elem: Box<Type>, + } +} + +ast_struct! { + /// An `impl Bound1 + Bound2 + Bound3` type where `Bound` is a trait or + /// a lifetime. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeImplTrait { + pub impl_token: Token![impl], + pub bounds: Punctuated<TypeParamBound, Token![+]>, + } +} + +ast_struct! { + /// Indication that a type should be inferred by the compiler: `_`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeInfer { + pub underscore_token: Token![_], + } +} + +ast_struct! { + /// A macro in the type position. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeMacro { + pub mac: Macro, + } +} + +ast_struct! { + /// The never type: `!`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeNever { + pub bang_token: Token![!], + } +} + +ast_struct! { + /// A parenthesized type equivalent to the inner type. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeParen { + pub paren_token: token::Paren, + pub elem: Box<Type>, + } +} + +ast_struct! { + /// A path like `core::slice::Iter`, optionally qualified with a + /// self-type as in `<Vec<T> as SomeTrait>::Associated`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypePath { + pub qself: Option<QSelf>, + pub path: Path, + } +} + +ast_struct! { + /// A raw pointer type: `*const T` or `*mut T`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypePtr { + pub star_token: Token![*], + pub const_token: Option<Token![const]>, + pub mutability: Option<Token![mut]>, + pub elem: Box<Type>, + } +} + +ast_struct! { + /// A reference type: `&'a T` or `&'a mut T`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeReference { + pub and_token: Token![&], + pub lifetime: Option<Lifetime>, + pub mutability: Option<Token![mut]>, + pub elem: Box<Type>, + } +} + +ast_struct! { + /// A dynamically sized slice type: `[T]`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeSlice { + pub bracket_token: token::Bracket, + pub elem: Box<Type>, + } +} + +ast_struct! { + /// A trait object type `dyn Bound1 + Bound2 + Bound3` where `Bound` is a + /// trait or a lifetime. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeTraitObject { + pub dyn_token: Option<Token![dyn]>, + pub bounds: Punctuated<TypeParamBound, Token![+]>, + } +} + +ast_struct! { + /// A tuple type: `(A, B, C, String)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct TypeTuple { + pub paren_token: token::Paren, + pub elems: Punctuated<Type, Token![,]>, + } +} + +ast_struct! { + /// The binary interface of a function: `extern "C"`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct Abi { + pub extern_token: Token![extern], + pub name: Option<LitStr>, + } +} + +ast_struct! { + /// An argument in a function type: the `usize` in `fn(usize) -> bool`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct BareFnArg { + pub attrs: Vec<Attribute>, + pub name: Option<(Ident, Token![:])>, + pub ty: Type, + } +} + +ast_struct! { + /// The variadic argument of a function pointer like `fn(usize, ...)`. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct BareVariadic { + pub attrs: Vec<Attribute>, + pub name: Option<(Ident, Token![:])>, + pub dots: Token![...], + pub comma: Option<Token![,]>, + } +} + +ast_enum! { + /// Return type of a function signature. + #[cfg_attr(docsrs, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum ReturnType { + /// Return type is not specified. + /// + /// Functions default to `()` and closures default to type inference. + Default, + /// A particular type is returned. + Type(Token![->], Box<Type>), + } +} + +#[cfg(feature = "parsing")] +pub(crate) mod parsing { + use crate::attr::Attribute; + use crate::error::{self, Result}; + use crate::ext::IdentExt as _; + use crate::generics::{BoundLifetimes, TraitBound, TraitBoundModifier, TypeParamBound}; + use crate::ident::Ident; + use crate::lifetime::Lifetime; + use crate::mac::{self, Macro}; + use crate::parse::{Parse, ParseStream}; + use crate::path; + use crate::path::{Path, PathArguments, QSelf}; + use crate::punctuated::Punctuated; + use crate::token; + use crate::ty::{ + Abi, BareFnArg, BareVariadic, ReturnType, Type, TypeArray, TypeBareFn, TypeGroup, + TypeImplTrait, TypeInfer, TypeMacro, TypeNever, TypeParen, TypePath, TypePtr, + TypeReference, TypeSlice, TypeTraitObject, TypeTuple, + }; + use crate::verbatim; + use alloc::boxed::Box; + use alloc::vec::Vec; + use proc_macro2::{Span, TokenStream}; + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Type { + fn parse(input: ParseStream) -> Result<Self> { + let allow_plus = true; + let allow_group_generic = true; + ambig_ty(input, allow_plus, allow_group_generic) + } + } + + impl Type { + /// In some positions, types may not contain the `+` character, to + /// disambiguate them. For example in the expression `1 as T`, T may not + /// contain a `+` character. + /// + /// This parser does not allow a `+`, while the default parser does. + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn without_plus(input: ParseStream) -> Result<Self> { + let allow_plus = false; + let allow_group_generic = true; + ambig_ty(input, allow_plus, allow_group_generic) + } + } + + pub(crate) fn ambig_ty( + input: ParseStream, + allow_plus: bool, + allow_group_generic: bool, + ) -> Result<Type> { + let begin = input.cursor(); + + if input.peek(token::Group) { + let mut group: TypeGroup = input.parse()?; + if input.peek(Token![::]) && input.peek3(Ident::peek_any) { + if let Type::Path(mut ty) = *group.elem { + Path::parse_rest(input, &mut ty.path, false)?; + return Ok(Type::Path(ty)); + } else { + return Ok(Type::Path(TypePath { + qself: Some(QSelf { + lt_token: Token![<](group.group_token.span), + position: 0, + as_token: None, + gt_token: Token![>](group.group_token.span), + ty: group.elem, + }), + path: Path::parse_helper(input, false)?, + })); + } + } else if input.peek(Token![<]) && allow_group_generic + || input.peek(Token![::]) && input.peek3(Token![<]) + { + if let Type::Path(mut ty) = *group.elem { + let arguments = &mut ty.path.segments.last_mut().unwrap().arguments; + if arguments.is_none() { + *arguments = PathArguments::AngleBracketed(input.parse()?); + Path::parse_rest(input, &mut ty.path, false)?; + return Ok(Type::Path(ty)); + } else { + *group.elem = Type::Path(ty); + } + } + } + return Ok(Type::Group(group)); + } + + let mut lifetimes = None::<BoundLifetimes>; + let mut lookahead = input.lookahead1(); + if lookahead.peek(Token![for]) { + lifetimes = input.parse()?; + lookahead = input.lookahead1(); + if !lookahead.peek(Ident) + && !lookahead.peek(Token![fn]) + && !lookahead.peek(Token![unsafe]) + && !lookahead.peek(Token![extern]) + && !lookahead.peek(Token![super]) + && !lookahead.peek(Token![self]) + && !lookahead.peek(Token![Self]) + && !lookahead.peek(Token![crate]) + || input.peek(Token![dyn]) + { + return Err(lookahead.error()); + } + } + + if lookahead.peek(token::Paren) { + let content; + let paren_token = parenthesized!(content in input); + if content.is_empty() { + return Ok(Type::Tuple(TypeTuple { + paren_token, + elems: Punctuated::new(), + })); + } + if content.peek(Lifetime) { + return Ok(Type::Paren(TypeParen { + paren_token, + elem: Box::new(Type::TraitObject(content.parse()?)), + })); + } + if content.peek(Token![?]) { + return Ok(Type::TraitObject(TypeTraitObject { + dyn_token: None, + bounds: { + let mut bounds = Punctuated::new(); + bounds.push_value(TypeParamBound::Trait(TraitBound { + paren_token: Some(paren_token), + ..content.parse()? + })); + while let Some(plus) = input.parse()? { + bounds.push_punct(plus); + bounds.push_value({ + let allow_precise_capture = false; + let allow_const = false; + TypeParamBound::parse_single( + input, + allow_precise_capture, + allow_const, + )? + }); + } + bounds + }, + })); + } + let mut first: Type = content.parse()?; + if content.peek(Token![,]) { + return Ok(Type::Tuple(TypeTuple { + paren_token, + elems: { + let mut elems = Punctuated::new(); + elems.push_value(first); + elems.push_punct(content.parse()?); + while !content.is_empty() { + elems.push_value(content.parse()?); + if content.is_empty() { + break; + } + elems.push_punct(content.parse()?); + } + elems + }, + })); + } + if allow_plus && input.peek(Token![+]) { + loop { + let first = match first { + Type::Path(TypePath { qself: None, path }) => { + TypeParamBound::Trait(TraitBound { + paren_token: Some(paren_token), + modifier: TraitBoundModifier::None, + lifetimes: None, + path, + }) + } + Type::TraitObject(TypeTraitObject { + dyn_token: None, + bounds, + }) => { + if bounds.len() > 1 || bounds.trailing_punct() { + first = Type::TraitObject(TypeTraitObject { + dyn_token: None, + bounds, + }); + break; + } + match bounds.into_iter().next().unwrap() { + TypeParamBound::Trait(trait_bound) => { + TypeParamBound::Trait(TraitBound { + paren_token: Some(paren_token), + ..trait_bound + }) + } + other @ (TypeParamBound::Lifetime(_) + | TypeParamBound::PreciseCapture(_) + | TypeParamBound::Verbatim(_)) => other, + } + } + _ => break, + }; + return Ok(Type::TraitObject(TypeTraitObject { + dyn_token: None, + bounds: { + let mut bounds = Punctuated::new(); + bounds.push_value(first); + while let Some(plus) = input.parse()? { + bounds.push_punct(plus); + bounds.push_value({ + let allow_precise_capture = false; + let allow_const = false; + TypeParamBound::parse_single( + input, + allow_precise_capture, + allow_const, + )? + }); + } + bounds + }, + })); + } + } + Ok(Type::Paren(TypeParen { + paren_token, + elem: Box::new(first), + })) + } else if lookahead.peek(Token![fn]) + || lookahead.peek(Token![unsafe]) + || lookahead.peek(Token![extern]) + { + let mut bare_fn: TypeBareFn = input.parse()?; + bare_fn.lifetimes = lifetimes; + Ok(Type::BareFn(bare_fn)) + } else if cfg!(feature = "full") + && token::parsing::peek_keyword(input.cursor(), "builtin") + && input.peek2(Token![#]) + { + token::parsing::keyword(input, "builtin")?; + input.parse::<Token![#]>()?; + input.parse::<Ident>()?; + let args; + parenthesized!(args in input); + args.parse::<TokenStream>()?; + Ok(Type::Verbatim(verbatim::between(begin, input.cursor()))) + } else if lookahead.peek(Ident) + || input.peek(Token![super]) + || input.peek(Token![self]) + || input.peek(Token![Self]) + || input.peek(Token![crate]) + || lookahead.peek(Token![::]) + || lookahead.peek(Token![<]) + { + let ty: TypePath = input.parse()?; + if ty.qself.is_some() { + return Ok(Type::Path(ty)); + } + + if input.peek(Token![!]) && !input.peek(Token![!=]) && ty.path.is_mod_style() { + let bang_token: Token![!] = input.parse()?; + let (delimiter, tokens) = mac::parse_delimiter(input)?; + return Ok(Type::Macro(TypeMacro { + mac: Macro { + path: ty.path, + bang_token, + delimiter, + tokens, + }, + })); + } + + if lifetimes.is_some() || allow_plus && input.peek(Token![+]) { + let mut bounds = Punctuated::new(); + bounds.push_value(TypeParamBound::Trait(TraitBound { + paren_token: None, + modifier: TraitBoundModifier::None, + lifetimes, + path: ty.path, + })); + if allow_plus { + while input.peek(Token![+]) { + bounds.push_punct(input.parse()?); + if !(input.peek(Ident::peek_any) + || input.peek(Token![::]) + || input.peek(Token![?]) + || input.peek(Lifetime) + || input.peek(token::Paren)) + { + break; + } + bounds.push_value({ + let allow_precise_capture = false; + let allow_const = false; + TypeParamBound::parse_single(input, allow_precise_capture, allow_const)? + }); + } + } + return Ok(Type::TraitObject(TypeTraitObject { + dyn_token: None, + bounds, + })); + } + + Ok(Type::Path(ty)) + } else if lookahead.peek(Token![dyn]) { + let dyn_token: Token![dyn] = input.parse()?; + let dyn_span = dyn_token.span; + let star_token: Option<Token![*]> = input.parse()?; + let bounds = TypeTraitObject::parse_bounds(dyn_span, input, allow_plus)?; + Ok(if star_token.is_some() { + Type::Verbatim(verbatim::between(begin, input.cursor())) + } else { + Type::TraitObject(TypeTraitObject { + dyn_token: Some(dyn_token), + bounds, + }) + }) + } else if lookahead.peek(token::Bracket) { + let content; + let bracket_token = bracketed!(content in input); + let elem: Type = content.parse()?; + if content.peek(Token![;]) { + Ok(Type::Array(TypeArray { + bracket_token, + elem: Box::new(elem), + semi_token: content.parse()?, + len: content.parse()?, + })) + } else { + Ok(Type::Slice(TypeSlice { + bracket_token, + elem: Box::new(elem), + })) + } + } else if lookahead.peek(Token![*]) { + input.parse().map(Type::Ptr) + } else if lookahead.peek(Token![&]) { + input.parse().map(Type::Reference) + } else if lookahead.peek(Token![!]) && !input.peek(Token![=]) { + input.parse().map(Type::Never) + } else if lookahead.peek(Token![impl]) { + TypeImplTrait::parse(input, allow_plus).map(Type::ImplTrait) + } else if lookahead.peek(Token![_]) { + input.parse().map(Type::Infer) + } else if lookahead.peek(Lifetime) { + input.parse().map(Type::TraitObject) + } else { + Err(lookahead.error()) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeSlice { + fn parse(input: ParseStream) -> Result<Self> { + let content; + Ok(TypeSlice { + bracket_token: bracketed!(content in input), + elem: content.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeArray { + fn parse(input: ParseStream) -> Result<Self> { + let content; + Ok(TypeArray { + bracket_token: bracketed!(content in input), + elem: content.parse()?, + semi_token: content.parse()?, + len: content.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypePtr { + fn parse(input: ParseStream) -> Result<Self> { + let star_token: Token![*] = input.parse()?; + + let lookahead = input.lookahead1(); + let (const_token, mutability) = if lookahead.peek(Token![const]) { + (Some(input.parse()?), None) + } else if lookahead.peek(Token![mut]) { + (None, Some(input.parse()?)) + } else { + return Err(lookahead.error()); + }; + + Ok(TypePtr { + star_token, + const_token, + mutability, + elem: Box::new(input.call(Type::without_plus)?), + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeReference { + fn parse(input: ParseStream) -> Result<Self> { + Ok(TypeReference { + and_token: input.parse()?, + lifetime: input.parse()?, + mutability: input.parse()?, + // & binds tighter than +, so we don't allow + here. + elem: Box::new(input.call(Type::without_plus)?), + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeBareFn { + fn parse(input: ParseStream) -> Result<Self> { + let args; + let mut variadic = None; + + Ok(TypeBareFn { + lifetimes: input.parse()?, + unsafety: input.parse()?, + abi: input.parse()?, + fn_token: input.parse()?, + paren_token: parenthesized!(args in input), + inputs: { + let mut inputs = Punctuated::new(); + + while !args.is_empty() { + let attrs = args.call(Attribute::parse_outer)?; + + if inputs.empty_or_trailing() + && (args.peek(Token![...]) + || (args.peek(Ident) || args.peek(Token![_])) + && args.peek2(Token![:]) + && args.peek3(Token![...])) + { + variadic = Some(parse_bare_variadic(&args, attrs)?); + break; + } + + let allow_self = inputs.is_empty(); + let arg = parse_bare_fn_arg(&args, allow_self)?; + inputs.push_value(BareFnArg { attrs, ..arg }); + if args.is_empty() { + break; + } + + let comma = args.parse()?; + inputs.push_punct(comma); + } + + inputs + }, + variadic, + output: input.call(ReturnType::without_plus)?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeNever { + fn parse(input: ParseStream) -> Result<Self> { + Ok(TypeNever { + bang_token: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeInfer { + fn parse(input: ParseStream) -> Result<Self> { + Ok(TypeInfer { + underscore_token: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeTuple { + fn parse(input: ParseStream) -> Result<Self> { + let content; + let paren_token = parenthesized!(content in input); + + if content.is_empty() { + return Ok(TypeTuple { + paren_token, + elems: Punctuated::new(), + }); + } + + let first: Type = content.parse()?; + Ok(TypeTuple { + paren_token, + elems: { + let mut elems = Punctuated::new(); + elems.push_value(first); + elems.push_punct(content.parse()?); + while !content.is_empty() { + elems.push_value(content.parse()?); + if content.is_empty() { + break; + } + elems.push_punct(content.parse()?); + } + elems + }, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeMacro { + fn parse(input: ParseStream) -> Result<Self> { + Ok(TypeMacro { + mac: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypePath { + fn parse(input: ParseStream) -> Result<Self> { + let expr_style = false; + let (qself, path) = path::parsing::qpath(input, expr_style)?; + Ok(TypePath { qself, path }) + } + } + + impl ReturnType { + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn without_plus(input: ParseStream) -> Result<Self> { + let allow_plus = false; + Self::parse(input, allow_plus) + } + + pub(crate) fn parse(input: ParseStream, allow_plus: bool) -> Result<Self> { + if input.peek(Token![->]) { + let arrow = input.parse()?; + let allow_group_generic = true; + let ty = ambig_ty(input, allow_plus, allow_group_generic)?; + Ok(ReturnType::Type(arrow, Box::new(ty))) + } else { + Ok(ReturnType::Default) + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for ReturnType { + fn parse(input: ParseStream) -> Result<Self> { + let allow_plus = true; + Self::parse(input, allow_plus) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeTraitObject { + fn parse(input: ParseStream) -> Result<Self> { + let allow_plus = true; + Self::parse(input, allow_plus) + } + } + + impl TypeTraitObject { + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn without_plus(input: ParseStream) -> Result<Self> { + let allow_plus = false; + Self::parse(input, allow_plus) + } + + // Only allow multiple trait references if allow_plus is true. + pub(crate) fn parse(input: ParseStream, allow_plus: bool) -> Result<Self> { + let dyn_token: Option<Token![dyn]> = input.parse()?; + let dyn_span = match &dyn_token { + Some(token) => token.span, + None => input.span(), + }; + let bounds = Self::parse_bounds(dyn_span, input, allow_plus)?; + Ok(TypeTraitObject { dyn_token, bounds }) + } + + fn parse_bounds( + dyn_span: Span, + input: ParseStream, + allow_plus: bool, + ) -> Result<Punctuated<TypeParamBound, Token![+]>> { + let allow_precise_capture = false; + let allow_const = false; + let bounds = TypeParamBound::parse_multiple( + input, + allow_plus, + allow_precise_capture, + allow_const, + )?; + let mut last_lifetime_span = None; + let mut at_least_one_trait = false; + for bound in &bounds { + match bound { + TypeParamBound::Trait(_) => { + at_least_one_trait = true; + break; + } + TypeParamBound::Lifetime(lifetime) => { + last_lifetime_span = Some(lifetime.ident.span()); + } + TypeParamBound::PreciseCapture(_) | TypeParamBound::Verbatim(_) => { + unreachable!() + } + } + } + // Just lifetimes like `'a + 'b` is not a TraitObject. + if !at_least_one_trait { + let msg = "at least one trait is required for an object type"; + return Err(error::new2(dyn_span, last_lifetime_span.unwrap(), msg)); + } + Ok(bounds) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeImplTrait { + fn parse(input: ParseStream) -> Result<Self> { + let allow_plus = true; + Self::parse(input, allow_plus) + } + } + + impl TypeImplTrait { + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + pub fn without_plus(input: ParseStream) -> Result<Self> { + let allow_plus = false; + Self::parse(input, allow_plus) + } + + pub(crate) fn parse(input: ParseStream, allow_plus: bool) -> Result<Self> { + let impl_token: Token![impl] = input.parse()?; + let allow_precise_capture = true; + let allow_const = true; + let bounds = TypeParamBound::parse_multiple( + input, + allow_plus, + allow_precise_capture, + allow_const, + )?; + let mut last_nontrait_span = None; + let mut at_least_one_trait = false; + for bound in &bounds { + match bound { + TypeParamBound::Trait(_) => { + at_least_one_trait = true; + break; + } + TypeParamBound::Lifetime(lifetime) => { + last_nontrait_span = Some(lifetime.ident.span()); + } + TypeParamBound::PreciseCapture(precise_capture) => { + #[cfg(feature = "full")] + { + last_nontrait_span = Some(precise_capture.gt_token.span); + } + #[cfg(not(feature = "full"))] + { + _ = precise_capture; + unreachable!(); + } + } + TypeParamBound::Verbatim(_) => { + // `[const] Trait` + at_least_one_trait = true; + break; + } + } + } + if !at_least_one_trait { + let msg = "at least one trait must be specified"; + return Err(error::new2( + impl_token.span, + last_nontrait_span.unwrap(), + msg, + )); + } + Ok(TypeImplTrait { impl_token, bounds }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeGroup { + fn parse(input: ParseStream) -> Result<Self> { + let group = crate::group::parse_group(input)?; + Ok(TypeGroup { + group_token: group.token, + elem: group.content.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for TypeParen { + fn parse(input: ParseStream) -> Result<Self> { + let allow_plus = false; + Self::parse(input, allow_plus) + } + } + + impl TypeParen { + fn parse(input: ParseStream, allow_plus: bool) -> Result<Self> { + let content; + Ok(TypeParen { + paren_token: parenthesized!(content in input), + elem: Box::new({ + let allow_group_generic = true; + ambig_ty(&content, allow_plus, allow_group_generic)? + }), + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for BareFnArg { + fn parse(input: ParseStream) -> Result<Self> { + let allow_self = false; + parse_bare_fn_arg(input, allow_self) + } + } + + fn parse_bare_fn_arg(input: ParseStream, allow_self: bool) -> Result<BareFnArg> { + let attrs = input.call(Attribute::parse_outer)?; + + let begin = input.cursor(); + + let has_mut_self = allow_self && input.peek(Token![mut]) && input.peek2(Token![self]); + if has_mut_self { + input.parse::<Token![mut]>()?; + } + + let mut has_self = false; + let mut name = if (input.peek(Ident) || input.peek(Token![_]) || { + has_self = allow_self && input.peek(Token![self]); + has_self + }) && input.peek2(Token![:]) + && !input.peek2(Token![::]) + { + let name = input.call(Ident::parse_any)?; + let colon: Token![:] = input.parse()?; + Some((name, colon)) + } else { + has_self = false; + None + }; + + let ty = if allow_self && !has_self && input.peek(Token![mut]) && input.peek2(Token![self]) + { + input.parse::<Token![mut]>()?; + input.parse::<Token![self]>()?; + None + } else if has_mut_self && name.is_none() { + input.parse::<Token![self]>()?; + None + } else { + Some(input.parse()?) + }; + + let ty = match ty { + Some(ty) if !has_mut_self => ty, + _ => { + name = None; + Type::Verbatim(verbatim::between(begin, input.cursor())) + } + }; + + Ok(BareFnArg { attrs, name, ty }) + } + + fn parse_bare_variadic(input: ParseStream, attrs: Vec<Attribute>) -> Result<BareVariadic> { + Ok(BareVariadic { + attrs, + name: if input.peek(Ident) || input.peek(Token![_]) { + let name = input.call(Ident::parse_any)?; + let colon: Token![:] = input.parse()?; + Some((name, colon)) + } else { + None + }, + dots: input.parse()?, + comma: input.parse()?, + }) + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Abi { + fn parse(input: ParseStream) -> Result<Self> { + Ok(Abi { + extern_token: input.parse()?, + name: input.parse()?, + }) + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "parsing")))] + impl Parse for Option<Abi> { + fn parse(input: ParseStream) -> Result<Self> { + if input.peek(Token![extern]) { + input.parse().map(Some) + } else { + Ok(None) + } + } + } +} + +#[cfg(feature = "printing")] +mod printing { + use crate::attr::FilterAttrs; + use crate::path; + use crate::path::printing::PathStyle; + use crate::print::TokensOrDefault; + use crate::ty::{ + Abi, BareFnArg, BareVariadic, ReturnType, TypeArray, TypeBareFn, TypeGroup, TypeImplTrait, + TypeInfer, TypeMacro, TypeNever, TypeParen, TypePath, TypePtr, TypeReference, TypeSlice, + TypeTraitObject, TypeTuple, + }; + use proc_macro2::TokenStream; + use quote::{ToTokens, TokenStreamExt as _}; + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeSlice { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.bracket_token.surround(tokens, |tokens| { + self.elem.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeArray { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.bracket_token.surround(tokens, |tokens| { + self.elem.to_tokens(tokens); + self.semi_token.to_tokens(tokens); + self.len.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypePtr { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.star_token.to_tokens(tokens); + match &self.mutability { + Some(tok) => tok.to_tokens(tokens), + None => { + TokensOrDefault(&self.const_token).to_tokens(tokens); + } + } + self.elem.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeReference { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.and_token.to_tokens(tokens); + self.lifetime.to_tokens(tokens); + self.mutability.to_tokens(tokens); + self.elem.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeBareFn { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.lifetimes.to_tokens(tokens); + self.unsafety.to_tokens(tokens); + self.abi.to_tokens(tokens); + self.fn_token.to_tokens(tokens); + self.paren_token.surround(tokens, |tokens| { + self.inputs.to_tokens(tokens); + if let Some(variadic) = &self.variadic { + if !self.inputs.empty_or_trailing() { + let span = variadic.dots.spans[0]; + Token![,](span).to_tokens(tokens); + } + variadic.to_tokens(tokens); + } + }); + self.output.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeNever { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.bang_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeTuple { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.paren_token.surround(tokens, |tokens| { + self.elems.to_tokens(tokens); + // If we only have one argument, we need a trailing comma to + // distinguish TypeTuple from TypeParen. + if self.elems.len() == 1 && !self.elems.trailing_punct() { + <Token![,]>::default().to_tokens(tokens); + } + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypePath { + fn to_tokens(&self, tokens: &mut TokenStream) { + path::printing::print_qpath(tokens, &self.qself, &self.path, PathStyle::AsWritten); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeTraitObject { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.dyn_token.to_tokens(tokens); + self.bounds.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeImplTrait { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.impl_token.to_tokens(tokens); + self.bounds.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeGroup { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.group_token.surround(tokens, |tokens| { + self.elem.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeParen { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.paren_token.surround(tokens, |tokens| { + self.elem.to_tokens(tokens); + }); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeInfer { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.underscore_token.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for TypeMacro { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.mac.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for ReturnType { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + ReturnType::Default => {} + ReturnType::Type(arrow, ty) => { + arrow.to_tokens(tokens); + ty.to_tokens(tokens); + } + } + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for BareFnArg { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + if let Some((name, colon)) = &self.name { + name.to_tokens(tokens); + colon.to_tokens(tokens); + } + self.ty.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for BareVariadic { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(self.attrs.outer()); + if let Some((name, colon)) = &self.name { + name.to_tokens(tokens); + colon.to_tokens(tokens); + } + self.dots.to_tokens(tokens); + self.comma.to_tokens(tokens); + } + } + + #[cfg_attr(docsrs, doc(cfg(feature = "printing")))] + impl ToTokens for Abi { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.extern_token.to_tokens(tokens); + self.name.to_tokens(tokens); + } + } +} diff --git a/third_party/crates/syn-2.0.119/src/verbatim.rs b/third_party/crates/syn-2.0.119/src/verbatim.rs new file mode 100644 index 0000000..7e16db3 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/verbatim.rs @@ -0,0 +1,32 @@ +use crate::buffer::Cursor; +use crate::ext::TokenStreamExt as _; +use core::cmp::Ordering; +use proc_macro2::{Delimiter, TokenStream}; + +pub(crate) fn between(begin: Cursor, end: Cursor) -> TokenStream { + let mut cursor = begin; + assert!(crate::buffer::same_buffer(end, cursor)); + + let mut tokens = TokenStream::new(); + while cursor != end { + let (tt, next) = cursor.token_tree().unwrap(); + + if crate::buffer::cmp_assuming_same_buffer(end, next) == Ordering::Less { + // A syntax node can cross the boundary of a None-delimited group + // due to such groups being transparent to the parser in most cases. + // Any time this occurs the group is known to be semantically + // irrelevant. https://github.com/dtolnay/syn/issues/1235 + if let Some((inside, _span, after)) = cursor.group(Delimiter::None) { + assert!(next == after); + cursor = inside; + continue; + } else { + panic!("verbatim end must not be inside a delimited group"); + } + } + + tokens.append(tt); + cursor = next; + } + tokens +} diff --git a/third_party/crates/syn-2.0.119/src/whitespace.rs b/third_party/crates/syn-2.0.119/src/whitespace.rs new file mode 100644 index 0000000..a50b506 --- /dev/null +++ b/third_party/crates/syn-2.0.119/src/whitespace.rs @@ -0,0 +1,65 @@ +pub(crate) fn skip(mut s: &str) -> &str { + 'skip: while !s.is_empty() { + let byte = s.as_bytes()[0]; + if byte == b'/' { + if s.starts_with("//") + && (!s.starts_with("///") || s.starts_with("////")) + && !s.starts_with("//!") + { + if let Some(i) = s.find('\n') { + s = &s[i + 1..]; + continue; + } else { + return ""; + } + } else if s.starts_with("/**/") { + s = &s[4..]; + continue; + } else if s.starts_with("/*") + && (!s.starts_with("/**") || s.starts_with("/***")) + && !s.starts_with("/*!") + { + let mut depth = 0; + let bytes = s.as_bytes(); + let mut i = 0; + let upper = bytes.len() - 1; + while i < upper { + if bytes[i] == b'/' && bytes[i + 1] == b'*' { + depth += 1; + i += 1; // eat '*' + } else if bytes[i] == b'*' && bytes[i + 1] == b'/' { + depth -= 1; + if depth == 0 { + s = &s[i + 2..]; + continue 'skip; + } + i += 1; // eat '/' + } + i += 1; + } + return s; + } + } + match byte { + b' ' | 0x09..=0x0D => { + s = &s[1..]; + continue; + } + b if b <= 0x7F => {} + _ => { + let ch = s.chars().next().unwrap(); + if is_whitespace(ch) { + s = &s[ch.len_utf8()..]; + continue; + } + } + } + return s; + } + s +} + +fn is_whitespace(ch: char) -> bool { + // Rust treats left-to-right mark and right-to-left mark as whitespace + ch.is_whitespace() || ch == '\u{200e}' || ch == '\u{200f}' +} diff --git a/third_party/crates/syn-2.0.119/tests/common/eq.rs b/third_party/crates/syn-2.0.119/tests/common/eq.rs new file mode 100644 index 0000000..7b7118b --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/common/eq.rs @@ -0,0 +1,926 @@ +#![allow(unused_macro_rules)] + +extern crate rustc_ast; +extern crate rustc_data_structures; +extern crate rustc_driver; +extern crate rustc_span; +extern crate thin_vec; + +use rustc_ast::ast::AngleBracketedArg; +use rustc_ast::ast::AngleBracketedArgs; +use rustc_ast::ast::AnonConst; +use rustc_ast::ast::Arm; +use rustc_ast::ast::AsmMacro; +use rustc_ast::ast::AssignOpKind; +use rustc_ast::ast::AssocItemConstraint; +use rustc_ast::ast::AssocItemConstraintKind; +use rustc_ast::ast::AssocItemKind; +use rustc_ast::ast::AttrArgs; +use rustc_ast::ast::AttrId; +use rustc_ast::ast::AttrItem; +use rustc_ast::ast::AttrItemKind; +use rustc_ast::ast::AttrKind; +use rustc_ast::ast::AttrStyle; +use rustc_ast::ast::Attribute; +use rustc_ast::ast::BinOpKind; +use rustc_ast::ast::BindingMode; +use rustc_ast::ast::Block; +use rustc_ast::ast::BlockCheckMode; +use rustc_ast::ast::BorrowKind; +use rustc_ast::ast::BoundAsyncness; +use rustc_ast::ast::BoundConstness; +use rustc_ast::ast::BoundPolarity; +use rustc_ast::ast::ByRef; +use rustc_ast::ast::CaptureBy; +use rustc_ast::ast::Closure; +use rustc_ast::ast::ClosureBinder; +use rustc_ast::ast::Const; +use rustc_ast::ast::ConstBlockItem; +use rustc_ast::ast::ConstItem; +use rustc_ast::ast::ConstItemRhsKind; +use rustc_ast::ast::CoroutineKind; +use rustc_ast::ast::Crate; +use rustc_ast::ast::Defaultness; +use rustc_ast::ast::Delegation; +use rustc_ast::ast::DelegationMac; +use rustc_ast::ast::DelegationSource; +use rustc_ast::ast::DelegationSuffixes; +use rustc_ast::ast::DelimArgs; +use rustc_ast::ast::EarlyParsedAttribute; +use rustc_ast::ast::EiiDecl; +use rustc_ast::ast::EiiImpl; +use rustc_ast::ast::EnumDef; +use rustc_ast::ast::Expr; +use rustc_ast::ast::ExprField; +use rustc_ast::ast::ExprKind; +use rustc_ast::ast::Extern; +use rustc_ast::ast::FieldDef; +use rustc_ast::ast::FloatTy; +use rustc_ast::ast::Fn; +use rustc_ast::ast::FnContract; +use rustc_ast::ast::FnDecl; +use rustc_ast::ast::FnHeader; +use rustc_ast::ast::FnPtrTy; +use rustc_ast::ast::FnRetTy; +use rustc_ast::ast::FnSig; +use rustc_ast::ast::ForLoopKind; +use rustc_ast::ast::ForeignItemKind; +use rustc_ast::ast::ForeignMod; +use rustc_ast::ast::FormatAlignment; +use rustc_ast::ast::FormatArgPosition; +use rustc_ast::ast::FormatArgPositionKind; +use rustc_ast::ast::FormatArgs; +use rustc_ast::ast::FormatArgsPiece; +use rustc_ast::ast::FormatArgument; +use rustc_ast::ast::FormatArgumentKind; +use rustc_ast::ast::FormatArguments; +use rustc_ast::ast::FormatCount; +use rustc_ast::ast::FormatDebugHex; +use rustc_ast::ast::FormatOptions; +use rustc_ast::ast::FormatPlaceholder; +use rustc_ast::ast::FormatSign; +use rustc_ast::ast::FormatTrait; +use rustc_ast::ast::GenBlockKind; +use rustc_ast::ast::GenericArg; +use rustc_ast::ast::GenericArgs; +use rustc_ast::ast::GenericBound; +use rustc_ast::ast::GenericParam; +use rustc_ast::ast::GenericParamKind; +use rustc_ast::ast::Generics; +use rustc_ast::ast::Guard; +use rustc_ast::ast::Impl; +use rustc_ast::ast::ImplPolarity; +use rustc_ast::ast::ImplRestriction; +use rustc_ast::ast::Inline; +use rustc_ast::ast::InlineAsm; +use rustc_ast::ast::InlineAsmOperand; +use rustc_ast::ast::InlineAsmOptions; +use rustc_ast::ast::InlineAsmRegOrRegClass; +use rustc_ast::ast::InlineAsmSym; +use rustc_ast::ast::InlineAsmTemplatePiece; +use rustc_ast::ast::IntTy; +use rustc_ast::ast::IsAuto; +use rustc_ast::ast::Item; +use rustc_ast::ast::ItemKind; +use rustc_ast::ast::Label; +use rustc_ast::ast::Lifetime; +use rustc_ast::ast::LitFloatType; +use rustc_ast::ast::LitIntType; +use rustc_ast::ast::LitKind; +use rustc_ast::ast::Local; +use rustc_ast::ast::LocalKind; +use rustc_ast::ast::MacCall; +use rustc_ast::ast::MacCallStmt; +use rustc_ast::ast::MacStmtStyle; +use rustc_ast::ast::MacroDef; +use rustc_ast::ast::MatchKind; +use rustc_ast::ast::MetaItem; +use rustc_ast::ast::MetaItemInner; +use rustc_ast::ast::MetaItemKind; +use rustc_ast::ast::MetaItemLit; +use rustc_ast::ast::MethodCall; +use rustc_ast::ast::ModKind; +use rustc_ast::ast::ModSpans; +use rustc_ast::ast::Movability; +use rustc_ast::ast::MutRestriction; +use rustc_ast::ast::MutTy; +use rustc_ast::ast::Mutability; +use rustc_ast::ast::NodeId; +use rustc_ast::ast::NormalAttr; +use rustc_ast::ast::Param; +use rustc_ast::ast::Parens; +use rustc_ast::ast::ParenthesizedArgs; +use rustc_ast::ast::Pat; +use rustc_ast::ast::PatField; +use rustc_ast::ast::PatFieldsRest; +use rustc_ast::ast::PatKind; +use rustc_ast::ast::Path; +use rustc_ast::ast::PathSegment; +use rustc_ast::ast::Pinnedness; +use rustc_ast::ast::PolyTraitRef; +use rustc_ast::ast::PreciseCapturingArg; +use rustc_ast::ast::QSelf; +use rustc_ast::ast::RangeEnd; +use rustc_ast::ast::RangeLimits; +use rustc_ast::ast::RangeSyntax; +use rustc_ast::ast::Recovered; +use rustc_ast::ast::RestrictionKind; +use rustc_ast::ast::Safety; +use rustc_ast::ast::StaticItem; +use rustc_ast::ast::Stmt; +use rustc_ast::ast::StmtKind; +use rustc_ast::ast::StrLit; +use rustc_ast::ast::StrStyle; +use rustc_ast::ast::StructExpr; +use rustc_ast::ast::StructRest; +use rustc_ast::ast::Term; +use rustc_ast::ast::Trait; +use rustc_ast::ast::TraitAlias; +use rustc_ast::ast::TraitBoundModifiers; +use rustc_ast::ast::TraitImplHeader; +use rustc_ast::ast::TraitObjectSyntax; +use rustc_ast::ast::TraitRef; +use rustc_ast::ast::Ty; +use rustc_ast::ast::TyAlias; +use rustc_ast::ast::TyKind; +use rustc_ast::ast::TyPat; +use rustc_ast::ast::TyPatKind; +use rustc_ast::ast::UintTy; +use rustc_ast::ast::UnOp; +use rustc_ast::ast::UnsafeBinderCastKind; +use rustc_ast::ast::UnsafeBinderTy; +use rustc_ast::ast::UnsafeSource; +use rustc_ast::ast::UseTree; +use rustc_ast::ast::UseTreeKind; +use rustc_ast::ast::Variant; +use rustc_ast::ast::VariantData; +use rustc_ast::ast::Visibility; +use rustc_ast::ast::VisibilityKind; +use rustc_ast::ast::WhereBoundPredicate; +use rustc_ast::ast::WhereClause; +use rustc_ast::ast::WhereEqPredicate; +use rustc_ast::ast::WherePredicate; +use rustc_ast::ast::WherePredicateKind; +use rustc_ast::ast::WhereRegionPredicate; +use rustc_ast::ast::YieldKind; +use rustc_ast::attr::data_structures::CfgEntry; +use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Lit, Token, TokenKind}; +use rustc_ast::tokenstream::{ + AttrTokenStream, AttrTokenTree, AttrsTarget, DelimSpacing, DelimSpan, LazyAttrTokenStream, + Spacing, TokenStream, TokenTree, +}; +use rustc_data_structures::packed::Pu128; +use rustc_span::symbol::{sym, ByteSymbol, Ident, Symbol}; +use rustc_span::{ErrorGuaranteed, LocalExpnId, Span, Spanned, SyntaxContext, DUMMY_SP}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::hash::{BuildHasher, Hash}; +use std::sync::Arc; +use thin_vec::ThinVec; + +pub trait SpanlessEq { + fn eq(&self, other: &Self) -> bool; +} + +impl<T: ?Sized + SpanlessEq> SpanlessEq for Box<T> { + fn eq(&self, other: &Self) -> bool { + SpanlessEq::eq(&**self, &**other) + } +} + +impl<T: ?Sized + SpanlessEq> SpanlessEq for Arc<T> { + fn eq(&self, other: &Self) -> bool { + SpanlessEq::eq(&**self, &**other) + } +} + +impl<T: SpanlessEq> SpanlessEq for Option<T> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (None, None) => true, + (Some(this), Some(other)) => SpanlessEq::eq(this, other), + _ => false, + } + } +} + +impl<T: SpanlessEq, E: SpanlessEq> SpanlessEq for Result<T, E> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Ok(this), Ok(other)) => SpanlessEq::eq(this, other), + (Err(this), Err(other)) => SpanlessEq::eq(this, other), + _ => false, + } + } +} + +impl<T: SpanlessEq> SpanlessEq for [T] { + fn eq(&self, other: &Self) -> bool { + self.len() == other.len() && self.iter().zip(other).all(|(a, b)| SpanlessEq::eq(a, b)) + } +} + +impl<T: SpanlessEq> SpanlessEq for Vec<T> { + fn eq(&self, other: &Self) -> bool { + <[T] as SpanlessEq>::eq(self, other) + } +} + +impl<T: SpanlessEq> SpanlessEq for ThinVec<T> { + fn eq(&self, other: &Self) -> bool { + self.len() == other.len() + && self + .iter() + .zip(other.iter()) + .all(|(a, b)| SpanlessEq::eq(a, b)) + } +} + +impl<K: Eq + Hash, V: SpanlessEq, S: BuildHasher> SpanlessEq for HashMap<K, V, S> { + fn eq(&self, other: &Self) -> bool { + self.len() == other.len() + && self.iter().all(|(key, this_v)| { + other + .get(key) + .is_some_and(|other_v| SpanlessEq::eq(this_v, other_v)) + }) + } +} + +impl<'a, T: ?Sized + ToOwned + SpanlessEq> SpanlessEq for Cow<'a, T> { + fn eq(&self, other: &Self) -> bool { + <T as SpanlessEq>::eq(self, other) + } +} + +impl<T: SpanlessEq> SpanlessEq for Spanned<T> { + fn eq(&self, other: &Self) -> bool { + SpanlessEq::eq(&self.node, &other.node) + } +} + +impl<A: SpanlessEq, B: SpanlessEq> SpanlessEq for (A, B) { + fn eq(&self, other: &Self) -> bool { + SpanlessEq::eq(&self.0, &other.0) && SpanlessEq::eq(&self.1, &other.1) + } +} + +impl<A: SpanlessEq, B: SpanlessEq, C: SpanlessEq> SpanlessEq for (A, B, C) { + fn eq(&self, other: &Self) -> bool { + SpanlessEq::eq(&self.0, &other.0) + && SpanlessEq::eq(&self.1, &other.1) + && SpanlessEq::eq(&self.2, &other.2) + } +} + +macro_rules! spanless_eq_true { + ($name:ty) => { + impl SpanlessEq for $name { + fn eq(&self, _other: &Self) -> bool { + true + } + } + }; +} + +spanless_eq_true!(Span); +spanless_eq_true!(DelimSpan); +spanless_eq_true!(AttrId); +spanless_eq_true!(NodeId); +spanless_eq_true!(SyntaxContext); +spanless_eq_true!(Spacing); +spanless_eq_true!(LocalExpnId); + +macro_rules! spanless_eq_partial_eq { + ($name:ty) => { + impl SpanlessEq for $name { + fn eq(&self, other: &Self) -> bool { + PartialEq::eq(self, other) + } + } + }; +} + +spanless_eq_partial_eq!(()); +spanless_eq_partial_eq!(bool); +spanless_eq_partial_eq!(u8); +spanless_eq_partial_eq!(u16); +spanless_eq_partial_eq!(u32); +spanless_eq_partial_eq!(u128); +spanless_eq_partial_eq!(usize); +spanless_eq_partial_eq!(char); +spanless_eq_partial_eq!(str); +spanless_eq_partial_eq!(String); +spanless_eq_partial_eq!(Pu128); +spanless_eq_partial_eq!(Symbol); +spanless_eq_partial_eq!(ByteSymbol); +spanless_eq_partial_eq!(CommentKind); +spanless_eq_partial_eq!(Delimiter); +spanless_eq_partial_eq!(InlineAsmOptions); +spanless_eq_partial_eq!(token::LitKind); +spanless_eq_partial_eq!(ErrorGuaranteed); +spanless_eq_partial_eq!(CfgEntry); + +macro_rules! spanless_eq_struct { + { + $($name:ident)::+ $(<$param:ident>)? + $([$field:tt $this:ident $other:ident])* + $(![$ignore:tt])*; + } => { + impl $(<$param: SpanlessEq>)* SpanlessEq for $($name)::+ $(<$param>)* { + fn eq(&self, other: &Self) -> bool { + let $($name)::+ { $($field: $this,)* $($ignore: _,)* } = self; + let $($name)::+ { $($field: $other,)* $($ignore: _,)* } = other; + true $(&& SpanlessEq::eq($this, $other))* + } + } + }; + + { + $($name:ident)::+ $(<$param:ident>)? + $([$field:tt $this:ident $other:ident])* + $(![$ignore:tt])*; + !$next:tt + $($rest:tt)* + } => { + spanless_eq_struct! { + $($name)::+ $(<$param>)* + $([$field $this $other])* + $(![$ignore])* + ![$next]; + $($rest)* + } + }; + + { + $($name:ident)::+ $(<$param:ident>)? + $([$field:tt $this:ident $other:ident])* + $(![$ignore:tt])*; + $next:tt + $($rest:tt)* + } => { + spanless_eq_struct! { + $($name)::+ $(<$param>)* + $([$field $this $other])* + [$next this other] + $(![$ignore])*; + $($rest)* + } + }; +} + +macro_rules! spanless_eq_enum { + { + $($name:ident)::+; + $([$($variant:ident)::+; $([$field:tt $this:ident $other:ident])* $(![$ignore:tt])*])* + } => { + impl SpanlessEq for $($name)::+ { + fn eq(&self, other: &Self) -> bool { + match self { + $( + $($variant)::+ { .. } => {} + )* + } + #[allow(unreachable_patterns)] + match (self, other) { + $( + ( + $($variant)::+ { $($field: $this,)* $($ignore: _,)* }, + $($variant)::+ { $($field: $other,)* $($ignore: _,)* }, + ) => { + true $(&& SpanlessEq::eq($this, $other))* + } + )* + _ => false, + } + } + } + }; + + { + $($name:ident)::+; + $([$($variant:ident)::+; $($fields:tt)*])* + $next:ident [$([$($named:tt)*])* $(![$ignore:tt])*] (!$i:tt $($field:tt)*) + $($rest:tt)* + } => { + spanless_eq_enum! { + $($name)::+; + $([$($variant)::+; $($fields)*])* + $next [$([$($named)*])* $(![$ignore])* ![$i]] ($($field)*) + $($rest)* + } + }; + + { + $($name:ident)::+; + $([$($variant:ident)::+; $($fields:tt)*])* + $next:ident [$([$($named:tt)*])* $(![$ignore:tt])*] ($i:tt $($field:tt)*) + $($rest:tt)* + } => { + spanless_eq_enum! { + $($name)::+; + $([$($variant)::+; $($fields)*])* + $next [$([$($named)*])* [$i this other] $(![$ignore])*] ($($field)*) + $($rest)* + } + }; + + { + $($name:ident)::+; + $([$($variant:ident)::+; $($fields:tt)*])* + $next:ident [$($named:tt)*] () + $($rest:tt)* + } => { + spanless_eq_enum! { + $($name)::+; + $([$($variant)::+; $($fields)*])* + [$($name)::+::$next; $($named)*] + $($rest)* + } + }; + + { + $($name:ident)::+; + $([$($variant:ident)::+; $($fields:tt)*])* + $next:ident ($($field:tt)*) + $($rest:tt)* + } => { + spanless_eq_enum! { + $($name)::+; + $([$($variant)::+; $($fields)*])* + $next [] ($($field)*) + $($rest)* + } + }; + + { + $($name:ident)::+; + $([$($variant:ident)::+; $($fields:tt)*])* + $next:ident + $($rest:tt)* + } => { + spanless_eq_enum! { + $($name)::+; + $([$($variant)::+; $($fields)*])* + [$($name)::+::$next;] + $($rest)* + } + }; +} + +spanless_eq_struct!(AngleBracketedArgs; span args); +spanless_eq_struct!(AnonConst; id value); +spanless_eq_struct!(Arm; attrs pat guard body span id is_placeholder); +spanless_eq_struct!(AssocItemConstraint; id ident gen_args kind span); +spanless_eq_struct!(AttrItem; unsafety path args); +spanless_eq_struct!(AttrTokenStream; 0); +spanless_eq_struct!(Attribute; kind id style span); +spanless_eq_struct!(AttrsTarget; attrs tokens); +spanless_eq_struct!(BindingMode; 0 1); +spanless_eq_struct!(Block; stmts id rules span); +spanless_eq_struct!(Closure; binder capture_clause constness coroutine_kind movability fn_decl body !fn_decl_span !fn_arg_span); +spanless_eq_struct!(ConstBlockItem; id span block); +spanless_eq_struct!(ConstItem; defaultness ident generics ty rhs_kind define_opaque); +spanless_eq_struct!(Crate; attrs items spans id is_placeholder); +spanless_eq_struct!(Delegation; id qself path ident rename body source); +spanless_eq_struct!(DelegationMac; qself prefix suffixes body); +spanless_eq_struct!(DelimArgs; dspan delim tokens); +spanless_eq_struct!(DelimSpacing; open close); +spanless_eq_struct!(EiiDecl; foreign_item impl_unsafe); +spanless_eq_struct!(EiiImpl; node_id eii_macro_path known_eii_macro_resolution impl_safety span inner_span is_default); +spanless_eq_struct!(EnumDef; variants); +spanless_eq_struct!(Expr; id kind span attrs !tokens); +spanless_eq_struct!(ExprField; attrs id span ident expr is_shorthand is_placeholder); +spanless_eq_struct!(FieldDef; attrs id span vis mut_restriction safety ident ty default is_placeholder); +spanless_eq_struct!(Fn; defaultness ident generics sig contract define_opaque body eii_impls); +spanless_eq_struct!(FnContract; declarations requires ensures); +spanless_eq_struct!(FnDecl; inputs output); +spanless_eq_struct!(FnHeader; constness coroutine_kind safety ext); +spanless_eq_struct!(FnPtrTy; safety ext generic_params decl decl_span); +spanless_eq_struct!(FnSig; header decl span); +spanless_eq_struct!(ForeignMod; extern_span safety abi items); +spanless_eq_struct!(FormatArgPosition; index kind span); +spanless_eq_struct!(FormatArgs; span template arguments uncooked_fmt_str is_source_literal); +spanless_eq_struct!(FormatArgument; kind expr); +spanless_eq_struct!(FormatOptions; width precision alignment fill sign alternate zero_pad debug_hex); +spanless_eq_struct!(FormatPlaceholder; argument span format_trait format_options); +spanless_eq_struct!(GenericParam; id ident attrs bounds is_placeholder kind !colon_span); +spanless_eq_struct!(Generics; params where_clause span); +spanless_eq_struct!(Guard; cond span_with_leading_if); +spanless_eq_struct!(Impl; generics constness of_trait self_ty items); +spanless_eq_struct!(ImplRestriction; kind span); +spanless_eq_struct!(InlineAsm; asm_macro template template_strs operands clobber_abis options line_spans); +spanless_eq_struct!(InlineAsmSym; id qself path); +spanless_eq_struct!(Item<K>; attrs id span vis kind !tokens); +spanless_eq_struct!(Label; ident); +spanless_eq_struct!(Lifetime; id ident); +spanless_eq_struct!(Lit; kind symbol suffix); +spanless_eq_struct!(Local; id super_ pat ty kind span colon_sp attrs !tokens); +spanless_eq_struct!(MacCall; path args); +spanless_eq_struct!(MacCallStmt; mac style attrs tokens); +spanless_eq_struct!(MacroDef; body macro_rules eii_declaration); +spanless_eq_struct!(MetaItem; unsafety path kind span); +spanless_eq_struct!(MetaItemLit; symbol suffix kind span); +spanless_eq_struct!(MethodCall; seg receiver args !span); +spanless_eq_struct!(ModSpans; !inner_span !inject_use_span); +spanless_eq_struct!(MutRestriction; kind span); +spanless_eq_struct!(MutTy; ty mutbl); +spanless_eq_struct!(NormalAttr; item tokens); +spanless_eq_struct!(ParenthesizedArgs; span inputs inputs_span output); +spanless_eq_struct!(Pat; id kind span); +spanless_eq_struct!(PatField; ident pat is_shorthand attrs id span is_placeholder); +spanless_eq_struct!(Path; span segments); +spanless_eq_struct!(PathSegment; ident id args); +spanless_eq_struct!(PolyTraitRef; bound_generic_params modifiers trait_ref span parens); +spanless_eq_struct!(QSelf; ty path_span position); +spanless_eq_struct!(StaticItem; ident ty safety mutability expr define_opaque eii_impls); +spanless_eq_struct!(Stmt; id kind span); +spanless_eq_struct!(StrLit; symbol suffix symbol_unescaped style span); +spanless_eq_struct!(StructExpr; qself path fields rest); +spanless_eq_struct!(Token; kind span); +spanless_eq_struct!(Trait; constness safety is_auto impl_restriction ident generics bounds items); +spanless_eq_struct!(TraitAlias; constness ident generics bounds); +spanless_eq_struct!(TraitBoundModifiers; constness asyncness polarity); +spanless_eq_struct!(TraitImplHeader; defaultness safety polarity trait_ref); +spanless_eq_struct!(TraitRef; path ref_id); +spanless_eq_struct!(Ty; id kind span); +spanless_eq_struct!(TyAlias; defaultness ident generics after_where_clause bounds ty); +spanless_eq_struct!(TyPat; id kind span); +spanless_eq_struct!(UnsafeBinderTy; generic_params inner_ty); +spanless_eq_struct!(UseTree; prefix kind); +spanless_eq_struct!(Variant; attrs id span !vis ident data disr_expr is_placeholder); +spanless_eq_struct!(Visibility; kind span); +spanless_eq_struct!(WhereBoundPredicate; bound_generic_params bounded_ty bounds); +spanless_eq_struct!(WhereClause; has_where_token predicates span); +spanless_eq_struct!(WhereEqPredicate; lhs_ty rhs_ty); +spanless_eq_struct!(WherePredicate; attrs kind id span is_placeholder); +spanless_eq_struct!(WhereRegionPredicate; lifetime bounds); +spanless_eq_enum!(AngleBracketedArg; Arg(0) Constraint(0)); +spanless_eq_enum!(AsmMacro; Asm GlobalAsm NakedAsm); +spanless_eq_enum!(AssocItemConstraintKind; Equality(term) Bound(bounds)); +spanless_eq_enum!(AssocItemKind; Const(0) Fn(0) Type(0) MacCall(0) Delegation(0) DelegationMac(0)); +spanless_eq_enum!(AttrArgs; Empty Delimited(0) Eq(eq_span expr)); +spanless_eq_enum!(AttrItemKind; Parsed(0) Unparsed(0)); +spanless_eq_enum!(AttrStyle; Outer Inner); +spanless_eq_enum!(AttrTokenTree; Token(0 1) Delimited(0 1 2 3) AttrsTarget(0)); +spanless_eq_enum!(BinOpKind; Add Sub Mul Div Rem And Or BitXor BitAnd BitOr Shl Shr Eq Lt Le Ne Ge Gt); +spanless_eq_enum!(BlockCheckMode; Default Unsafe(0)); +spanless_eq_enum!(BorrowKind; Ref Raw Pin); +spanless_eq_enum!(BoundAsyncness; Normal Async(0)); +spanless_eq_enum!(BoundConstness; Never Always(0) Maybe(0)); +spanless_eq_enum!(BoundPolarity; Positive Negative(0) Maybe(0)); +spanless_eq_enum!(ByRef; Yes(0 1) No); +spanless_eq_enum!(CaptureBy; Value(move_kw) Ref Use(use_kw)); +spanless_eq_enum!(ClosureBinder; NotPresent For(span generic_params)); +spanless_eq_enum!(Const; Yes(0) No); +spanless_eq_enum!(ConstItemRhsKind; Body(rhs) TypeConst(rhs)); +spanless_eq_enum!(Defaultness; Implicit Default(0) Final(0)); +spanless_eq_enum!(DelegationSource; Single List(0) Glob); +spanless_eq_enum!(DelegationSuffixes; List(0) Glob(0)); +spanless_eq_enum!(EarlyParsedAttribute; CfgTrace(0) CfgAttrTrace); +spanless_eq_enum!(Extern; None Implicit(0) Explicit(0 1)); +spanless_eq_enum!(FloatTy; F16 F32 F64 F128); +spanless_eq_enum!(FnRetTy; Default(0) Ty(0)); +spanless_eq_enum!(ForLoopKind; For ForAwait); +spanless_eq_enum!(ForeignItemKind; Static(0) Fn(0) TyAlias(0) MacCall(0)); +spanless_eq_enum!(FormatAlignment; Left Right Center); +spanless_eq_enum!(FormatArgPositionKind; Implicit Number Named); +spanless_eq_enum!(FormatArgsPiece; Literal(0) Placeholder(0)); +spanless_eq_enum!(FormatArgumentKind; Normal Named(0) Captured(0)); +spanless_eq_enum!(FormatCount; Literal(0) Argument(0)); +spanless_eq_enum!(FormatDebugHex; Lower Upper); +spanless_eq_enum!(FormatSign; Plus Minus); +spanless_eq_enum!(FormatTrait; Display Debug LowerExp UpperExp Octal Pointer Binary LowerHex UpperHex); +spanless_eq_enum!(GenBlockKind; Async Gen AsyncGen); +spanless_eq_enum!(GenericArg; Lifetime(0) Type(0) Const(0)); +spanless_eq_enum!(GenericArgs; AngleBracketed(0) Parenthesized(0) ParenthesizedElided(0)); +spanless_eq_enum!(GenericBound; Trait(0) Outlives(0) Use(0 1)); +spanless_eq_enum!(GenericParamKind; Lifetime Type(default) Const(ty span default)); +spanless_eq_enum!(ImplPolarity; Positive Negative(0)); +spanless_eq_enum!(Inline; Yes No(had_parse_error)); +spanless_eq_enum!(InlineAsmRegOrRegClass; Reg(0) RegClass(0)); +spanless_eq_enum!(InlineAsmTemplatePiece; String(0) Placeholder(operand_idx modifier span)); +spanless_eq_enum!(IntTy; Isize I8 I16 I32 I64 I128); +spanless_eq_enum!(IsAuto; Yes No); +spanless_eq_enum!(LitFloatType; Suffixed(0) Unsuffixed); +spanless_eq_enum!(LitIntType; Signed(0) Unsigned(0) Unsuffixed); +spanless_eq_enum!(LocalKind; Decl Init(0) InitElse(0 1)); +spanless_eq_enum!(MacStmtStyle; Semicolon Braces NoBraces); +spanless_eq_enum!(MatchKind; Prefix Postfix); +spanless_eq_enum!(MetaItemInner; MetaItem(0) Lit(0)); +spanless_eq_enum!(MetaItemKind; Word List(0) NameValue(0)); +spanless_eq_enum!(ModKind; Loaded(0 1 2) Unloaded); +spanless_eq_enum!(Movability; Static Movable); +spanless_eq_enum!(Mutability; Mut Not); +spanless_eq_enum!(Parens; Yes No); +spanless_eq_enum!(PatFieldsRest; Rest(0) Recovered(0) None); +spanless_eq_enum!(Pinnedness; Not Pinned); +spanless_eq_enum!(PreciseCapturingArg; Lifetime(0) Arg(0 1)); +spanless_eq_enum!(RangeEnd; Included(0) Excluded); +spanless_eq_enum!(RangeLimits; HalfOpen Closed); +spanless_eq_enum!(Recovered; No Yes(0)); +spanless_eq_enum!(RestrictionKind; Unrestricted Restricted(path id shorthand)); +spanless_eq_enum!(Safety; Unsafe(0) Safe(0) Default); +spanless_eq_enum!(StmtKind; Let(0) Item(0) Expr(0) Semi(0) Empty MacCall(0)); +spanless_eq_enum!(StrStyle; Cooked Raw(0)); +spanless_eq_enum!(StructRest; Base(0) Rest(0) None NoneWithError(0)); +spanless_eq_enum!(Term; Ty(0) Const(0)); +spanless_eq_enum!(TokenTree; Token(0 1) Delimited(0 1 2 3)); +spanless_eq_enum!(TraitObjectSyntax; Dyn None); +spanless_eq_enum!(TyPatKind; Range(0 1 2) NotNull Or(0) Err(0)); +spanless_eq_enum!(UintTy; Usize U8 U16 U32 U64 U128); +spanless_eq_enum!(UnOp; Deref Not Neg); +spanless_eq_enum!(UnsafeBinderCastKind; Wrap Unwrap); +spanless_eq_enum!(UnsafeSource; CompilerGenerated UserProvided); +spanless_eq_enum!(UseTreeKind; Simple(0) Nested(items span) Glob(0)); +spanless_eq_enum!(VariantData; Struct(fields recovered) Tuple(0 1) Unit(0)); +spanless_eq_enum!(VisibilityKind; Public Restricted(path id shorthand) Inherited); +spanless_eq_enum!(WherePredicateKind; BoundPredicate(0) RegionPredicate(0)); +spanless_eq_enum!(YieldKind; Prefix(0) Postfix(0)); +spanless_eq_enum!(AssignOpKind; AddAssign SubAssign MulAssign DivAssign + RemAssign BitXorAssign BitAndAssign BitOrAssign ShlAssign ShrAssign); +spanless_eq_enum!(CoroutineKind; Async(span closure_id return_impl_trait_id) + Gen(span closure_id return_impl_trait_id) + AsyncGen(span closure_id return_impl_trait_id)); +spanless_eq_enum!(ExprKind; Array(0) ConstBlock(0) Call(0 1) MethodCall(0) + Tup(0) Binary(0 1 2) Unary(0 1) Move(0 1) Lit(0) Cast(0 1) Type(0 1) + Let(0 1 2 3) If(0 1 2) While(0 1 2) ForLoop(pat iter body label kind) + Loop(0 1 2) Match(0 1 2) Closure(0) Block(0 1) Gen(0 1 2 3) Await(0 1) + Use(0 1) TryBlock(0 1) Assign(0 1 2) AssignOp(0 1 2) Field(0 1) Index(0 1 2) + Underscore Range(0 1 2) Path(0 1) AddrOf(0 1 2) Break(0 1) Continue(0) + Ret(0) InlineAsm(0) OffsetOf(0 1) MacCall(0) Struct(0) Repeat(0 1) Paren(0) + Try(0) Yield(0) Yeet(0) Become(0) IncludedBytes(0) FormatArgs(0) + UnsafeBinderCast(0 1 2) DirectConstArg(0) Err(0) Dummy); +spanless_eq_enum!(InlineAsmOperand; In(reg expr) Out(reg late expr) + InOut(reg late expr) SplitInOut(reg late in_expr out_expr) Const(anon_const) + Sym(sym) Label(block)); +spanless_eq_enum!(ItemKind; ExternCrate(0 1) Use(0) Static(0) Const(0) + ConstBlock(0) Fn(0) Mod(0 1 2) ForeignMod(0) GlobalAsm(0) TyAlias(0) + Enum(0 1 2) Struct(0 1 2) Union(0 1 2) Trait(0) TraitAlias(0) Impl(0) + MacCall(0) MacroDef(0 1) Delegation(0) DelegationMac(0)); +spanless_eq_enum!(LitKind; Str(0 1) ByteStr(0 1) CStr(0 1) Byte(0) Char(0) + Int(0 1) Float(0 1) Bool(0) Err(0)); +spanless_eq_enum!(PatKind; Missing Wild Ident(0 1 2) Struct(0 1 2 3) + TupleStruct(0 1 2) Or(0) Path(0 1) Tuple(0) Box(0) Deref(0) Ref(0 1 2) + Expr(0) Range(0 1 2) Slice(0) Rest Never Guard(0 1) Paren(0) MacCall(0) + Err(0)); +spanless_eq_enum!(TyKind; Slice(0) Array(0 1) Ptr(0) Ref(0 1) PinnedRef(0 1) + FnPtr(0) UnsafeBinder(0) Never Tup(0) Path(0 1) TraitObject(0 1) + ImplTrait(0 1) Paren(0) Infer ImplicitSelf MacCall(0) CVarArgs Pat(0 1) + FieldOf(0 1 2) View(0 1) DirectConstArg(0) Dummy Err(0)); + +impl SpanlessEq for Ident { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl SpanlessEq for RangeSyntax { + fn eq(&self, _other: &Self) -> bool { + match self { + RangeSyntax::DotDotDot | RangeSyntax::DotDotEq => true, + } + } +} + +impl SpanlessEq for Param { + fn eq(&self, other: &Self) -> bool { + let Param { + attrs, + ty, + pat, + id, + span: _, + is_placeholder, + } = self; + let Param { + attrs: attrs2, + ty: ty2, + pat: pat2, + id: id2, + span: _, + is_placeholder: is_placeholder2, + } = other; + SpanlessEq::eq(id, id2) + && SpanlessEq::eq(is_placeholder, is_placeholder2) + && (matches!(ty.kind, TyKind::Err(_)) + || matches!(ty2.kind, TyKind::Err(_)) + || SpanlessEq::eq(attrs, attrs2) + && SpanlessEq::eq(ty, ty2) + && SpanlessEq::eq(pat, pat2)) + } +} + +impl SpanlessEq for TokenKind { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (TokenKind::Literal(this), TokenKind::Literal(other)) => SpanlessEq::eq(this, other), + (TokenKind::DotDotEq | TokenKind::DotDotDot, _) => match other { + TokenKind::DotDotEq | TokenKind::DotDotDot => true, + _ => false, + }, + _ => self == other, + } + } +} + +impl SpanlessEq for TokenStream { + fn eq(&self, other: &Self) -> bool { + let mut this_trees = self.iter(); + let mut other_trees = other.iter(); + loop { + let Some(this) = this_trees.next() else { + return other_trees.next().is_none(); + }; + let Some(other) = other_trees.next() else { + return false; + }; + if SpanlessEq::eq(this, other) { + continue; + } + if let (TokenTree::Token(this, _), TokenTree::Token(other, _)) = (this, other) { + if match (&this.kind, &other.kind) { + (TokenKind::Literal(this), TokenKind::Literal(other)) => { + SpanlessEq::eq(this, other) + } + (TokenKind::DocComment(_kind, style, symbol), TokenKind::Pound) => { + doc_comment(*style, *symbol, &mut other_trees) + } + (TokenKind::Pound, TokenKind::DocComment(_kind, style, symbol)) => { + doc_comment(*style, *symbol, &mut this_trees) + } + _ => false, + } { + continue; + } + } + return false; + } + } +} + +fn doc_comment<'a>( + style: AttrStyle, + unescaped: Symbol, + trees: &mut impl Iterator<Item = &'a TokenTree>, +) -> bool { + if match style { + AttrStyle::Outer => false, + AttrStyle::Inner => true, + } { + match trees.next() { + Some(TokenTree::Token( + Token { + kind: TokenKind::Bang, + span: _, + }, + _spacing, + )) => {} + _ => return false, + } + } + let Some(TokenTree::Delimited(_span, _spacing, Delimiter::Bracket, stream)) = trees.next() + else { + return false; + }; + let mut trees = stream.iter(); + match trees.next() { + Some(TokenTree::Token( + Token { + kind: TokenKind::Ident(symbol, IdentIsRaw::No), + span: _, + }, + _spacing, + )) if *symbol == sym::doc => {} + _ => return false, + } + match trees.next() { + Some(TokenTree::Token( + Token { + kind: TokenKind::Eq, + span: _, + }, + _spacing, + )) => {} + _ => return false, + } + match trees.next() { + Some(TokenTree::Token(token, _spacing)) => { + is_escaped_literal_token(token, unescaped) && trees.next().is_none() + } + _ => false, + } +} + +fn is_escaped_literal_token(token: &Token, unescaped: Symbol) -> bool { + match token { + Token { + kind: TokenKind::Literal(lit), + span: _, + } => match MetaItemLit::from_token_lit(*lit, DUMMY_SP) { + Ok(lit) => is_escaped_literal_meta_item_lit(&lit, unescaped), + Err(_) => false, + }, + _ => false, + } +} + +fn is_escaped_literal_meta_item_lit(lit: &MetaItemLit, unescaped: Symbol) -> bool { + match lit { + MetaItemLit { + symbol: _, + suffix: None, + kind, + span: _, + } => is_escaped_lit_kind(kind, unescaped), + _ => false, + } +} + +fn is_escaped_lit(lit: &Lit, unescaped: Symbol) -> bool { + match lit { + Lit { + kind: token::LitKind::Str, + symbol: _, + suffix: None, + } => match LitKind::from_token_lit(*lit) { + Ok(lit_kind) => is_escaped_lit_kind(&lit_kind, unescaped), + _ => false, + }, + _ => false, + } +} + +fn is_escaped_lit_kind(kind: &LitKind, unescaped: Symbol) -> bool { + match kind { + LitKind::Str(symbol, StrStyle::Cooked) => { + symbol.as_str().replace('\r', "") == unescaped.as_str().replace('\r', "") + } + _ => false, + } +} + +impl SpanlessEq for LazyAttrTokenStream { + fn eq(&self, other: &Self) -> bool { + let this = self.to_attr_token_stream(); + let other = other.to_attr_token_stream(); + SpanlessEq::eq(&this, &other) + } +} + +impl SpanlessEq for AttrKind { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (AttrKind::Normal(normal), AttrKind::Normal(normal2)) => { + SpanlessEq::eq(normal, normal2) + } + (AttrKind::DocComment(kind, symbol), AttrKind::DocComment(kind2, symbol2)) => { + SpanlessEq::eq(kind, kind2) && SpanlessEq::eq(symbol, symbol2) + } + (AttrKind::DocComment(kind, unescaped), AttrKind::Normal(normal2)) => { + match kind { + CommentKind::Line | CommentKind::Block => {} + } + let path = Path::from_ident(Ident::with_dummy_span(sym::doc)); + SpanlessEq::eq(&path, &normal2.item.path) + && match &normal2.item.args { + AttrItemKind::Parsed(_) + | AttrItemKind::Unparsed(AttrArgs::Empty | AttrArgs::Delimited(_)) => false, + AttrItemKind::Unparsed(AttrArgs::Eq { eq_span: _, expr }) => { + match &expr.kind { + ExprKind::Lit(lit) => is_escaped_lit(lit, *unescaped), + _ => false, + } + } + } + } + (AttrKind::Normal(_), AttrKind::DocComment(..)) => SpanlessEq::eq(other, self), + } + } +} + +impl SpanlessEq for FormatArguments { + fn eq(&self, other: &Self) -> bool { + SpanlessEq::eq(self.all_args(), other.all_args()) + } +} diff --git a/third_party/crates/syn-2.0.119/tests/common/mod.rs b/third_party/crates/syn-2.0.119/tests/common/mod.rs new file mode 100644 index 0000000..ead830f --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/common/mod.rs @@ -0,0 +1,6 @@ +#![allow(dead_code)] +#![allow(clippy::module_name_repetitions, clippy::shadow_unrelated)] + +pub mod eq; +pub mod parse; +pub mod visit; diff --git a/third_party/crates/syn-2.0.119/tests/common/parse.rs b/third_party/crates/syn-2.0.119/tests/common/parse.rs new file mode 100644 index 0000000..75217e1 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/common/parse.rs @@ -0,0 +1,51 @@ +extern crate rustc_ast; +extern crate rustc_driver; +extern crate rustc_expand; +extern crate rustc_parse; +extern crate rustc_session; +extern crate rustc_span; + +use rustc_ast::ast; +use rustc_parse::lexer::StripTokens; +use rustc_session::parse::ParseSess; +use rustc_span::FileName; +use std::panic; + +pub fn librustc_expr(input: &str) -> Option<Box<ast::Expr>> { + match panic::catch_unwind(|| { + let sess = ParseSess::new(); + let name = FileName::Custom("test_precedence".to_string()); + let mut parser = rustc_parse::new_parser_from_source_str( + &sess, + name, + input.to_string(), + StripTokens::ShebangAndFrontmatter, + ) + .unwrap(); + let presult = parser.parse_expr(); + match presult { + Ok(expr) => Some(expr), + Err(diagnostic) => { + diagnostic.emit(); + None + } + } + }) { + Ok(Some(e)) => Some(e), + Ok(None) => None, + Err(_) => { + errorf!("librustc panicked\n"); + None + } + } +} + +pub fn syn_expr(input: &str) -> Option<syn::Expr> { + match syn::parse_str(input) { + Ok(e) => Some(e), + Err(msg) => { + errorf!("syn failed to parse\n{:?}\n", msg); + None + } + } +} diff --git a/third_party/crates/syn-2.0.119/tests/common/visit.rs b/third_party/crates/syn-2.0.119/tests/common/visit.rs new file mode 100644 index 0000000..2d2a6c5 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/common/visit.rs @@ -0,0 +1,119 @@ +use proc_macro2::{Delimiter, Group, TokenStream, TokenTree}; +use std::mem; +use syn::visit_mut::{self, VisitMut}; +use syn::{Expr, File, Generics, LifetimeParam, MacroDelimiter, Stmt, StmtMacro, TypeParam}; + +pub struct FlattenParens { + discard_paren_attrs: bool, +} + +impl FlattenParens { + pub fn discard_attrs() -> Self { + FlattenParens { + discard_paren_attrs: true, + } + } + + pub fn combine_attrs() -> Self { + FlattenParens { + discard_paren_attrs: false, + } + } + + pub fn visit_token_stream_mut(tokens: &mut TokenStream) { + *tokens = mem::take(tokens) + .into_iter() + .flat_map(|tt| { + if let TokenTree::Group(group) = tt { + let delimiter = group.delimiter(); + let mut content = group.stream(); + Self::visit_token_stream_mut(&mut content); + if let Delimiter::Parenthesis = delimiter { + content + } else { + TokenStream::from(TokenTree::Group(Group::new(delimiter, content))) + } + } else { + TokenStream::from(tt) + } + }) + .collect(); + } +} + +impl VisitMut for FlattenParens { + fn visit_expr_mut(&mut self, e: &mut Expr) { + while let Expr::Paren(paren) = e { + let paren_attrs = mem::take(&mut paren.attrs); + *e = mem::replace(&mut *paren.expr, Expr::PLACEHOLDER); + if !paren_attrs.is_empty() && !self.discard_paren_attrs { + let nested_attrs = match e { + Expr::Assign(e) => &mut e.attrs, + Expr::Binary(e) => &mut e.attrs, + Expr::Cast(e) => &mut e.attrs, + _ => unimplemented!(), + }; + assert!(nested_attrs.is_empty()); + *nested_attrs = paren_attrs; + } + } + visit_mut::visit_expr_mut(self, e); + } +} + +pub struct AsIfPrinted; + +impl VisitMut for AsIfPrinted { + fn visit_file_mut(&mut self, file: &mut File) { + file.shebang = None; + visit_mut::visit_file_mut(self, file); + } + + fn visit_generics_mut(&mut self, generics: &mut Generics) { + if generics.params.is_empty() { + generics.lt_token = None; + generics.gt_token = None; + } + if let Some(where_clause) = &generics.where_clause { + if where_clause.predicates.is_empty() { + generics.where_clause = None; + } + } + visit_mut::visit_generics_mut(self, generics); + } + + fn visit_lifetime_param_mut(&mut self, param: &mut LifetimeParam) { + if param.bounds.is_empty() { + param.colon_token = None; + } + visit_mut::visit_lifetime_param_mut(self, param); + } + + fn visit_stmt_mut(&mut self, stmt: &mut Stmt) { + if let Stmt::Expr(expr, semi) = stmt { + if let Expr::Macro(e) = expr { + if match e.mac.delimiter { + MacroDelimiter::Brace(_) => true, + MacroDelimiter::Paren(_) | MacroDelimiter::Bracket(_) => semi.is_some(), + } { + let Expr::Macro(expr) = mem::replace(expr, Expr::PLACEHOLDER) else { + unreachable!(); + }; + *stmt = Stmt::Macro(StmtMacro { + attrs: expr.attrs, + mac: expr.mac, + semi_token: *semi, + }); + } + } + } + visit_mut::visit_stmt_mut(self, stmt); + } + + fn visit_type_param_mut(&mut self, param: &mut TypeParam) { + if param.bounds.is_empty() { + param.colon_token = None; + } + visit_mut::visit_type_param_mut(self, param); + } +} diff --git a/third_party/crates/syn-2.0.119/tests/debug/gen.rs b/third_party/crates/syn-2.0.119/tests/debug/gen.rs new file mode 100644 index 0000000..b74d3a6 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/debug/gen.rs @@ -0,0 +1,5238 @@ +// This file is @generated by syn-internal-codegen. +// It is not intended for manual editing. + +#![allow(clippy::match_wildcard_for_single_variants)] +use super::{Lite, Present}; +use core::fmt::{self, Debug, Display}; +use ref_cast::RefCast; +impl Debug for Lite<syn::Abi> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Abi"); + if let Some(val) = &self.value.name { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::LitStr); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("name", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::AngleBracketedGenericArguments> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("AngleBracketedGenericArguments"); + if self.value.colon2_token.is_some() { + formatter.field("colon2_token", &Present); + } + if !self.value.args.is_empty() { + formatter.field("args", Lite(&self.value.args)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Arm> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Arm"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("pat", Lite(&self.value.pat)); + if let Some(val) = &self.value.guard { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::If, Box<syn::Expr>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("guard", Print::ref_cast(val)); + } + formatter.field("body", Lite(&self.value.body)); + if self.value.comma.is_some() { + formatter.field("comma", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::AssocConst> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("AssocConst"); + formatter.field("ident", Lite(&self.value.ident)); + if let Some(val) = &self.value.generics { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::AngleBracketedGenericArguments); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("generics", Print::ref_cast(val)); + } + formatter.field("value", Lite(&self.value.value)); + formatter.finish() + } +} +impl Debug for Lite<syn::AssocType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("AssocType"); + formatter.field("ident", Lite(&self.value.ident)); + if let Some(val) = &self.value.generics { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::AngleBracketedGenericArguments); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("generics", Print::ref_cast(val)); + } + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::AttrStyle> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::AttrStyle::Outer => formatter.write_str("AttrStyle::Outer"), + syn::AttrStyle::Inner(_val) => { + formatter.write_str("AttrStyle::Inner")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::Attribute> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Attribute"); + formatter.field("style", Lite(&self.value.style)); + formatter.field("meta", Lite(&self.value.meta)); + formatter.finish() + } +} +impl Debug for Lite<syn::BareFnArg> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("BareFnArg"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.name { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((proc_macro2::Ident, syn::token::Colon)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("name", Print::ref_cast(val)); + } + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::BareVariadic> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("BareVariadic"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.name { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((proc_macro2::Ident, syn::token::Colon)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("name", Print::ref_cast(val)); + } + if self.value.comma.is_some() { + formatter.field("comma", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::BinOp> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::BinOp::Add(_val) => { + formatter.write_str("BinOp::Add")?; + Ok(()) + } + syn::BinOp::Sub(_val) => { + formatter.write_str("BinOp::Sub")?; + Ok(()) + } + syn::BinOp::Mul(_val) => { + formatter.write_str("BinOp::Mul")?; + Ok(()) + } + syn::BinOp::Div(_val) => { + formatter.write_str("BinOp::Div")?; + Ok(()) + } + syn::BinOp::Rem(_val) => { + formatter.write_str("BinOp::Rem")?; + Ok(()) + } + syn::BinOp::And(_val) => { + formatter.write_str("BinOp::And")?; + Ok(()) + } + syn::BinOp::Or(_val) => { + formatter.write_str("BinOp::Or")?; + Ok(()) + } + syn::BinOp::BitXor(_val) => { + formatter.write_str("BinOp::BitXor")?; + Ok(()) + } + syn::BinOp::BitAnd(_val) => { + formatter.write_str("BinOp::BitAnd")?; + Ok(()) + } + syn::BinOp::BitOr(_val) => { + formatter.write_str("BinOp::BitOr")?; + Ok(()) + } + syn::BinOp::Shl(_val) => { + formatter.write_str("BinOp::Shl")?; + Ok(()) + } + syn::BinOp::Shr(_val) => { + formatter.write_str("BinOp::Shr")?; + Ok(()) + } + syn::BinOp::Eq(_val) => { + formatter.write_str("BinOp::Eq")?; + Ok(()) + } + syn::BinOp::Lt(_val) => { + formatter.write_str("BinOp::Lt")?; + Ok(()) + } + syn::BinOp::Le(_val) => { + formatter.write_str("BinOp::Le")?; + Ok(()) + } + syn::BinOp::Ne(_val) => { + formatter.write_str("BinOp::Ne")?; + Ok(()) + } + syn::BinOp::Ge(_val) => { + formatter.write_str("BinOp::Ge")?; + Ok(()) + } + syn::BinOp::Gt(_val) => { + formatter.write_str("BinOp::Gt")?; + Ok(()) + } + syn::BinOp::AddAssign(_val) => { + formatter.write_str("BinOp::AddAssign")?; + Ok(()) + } + syn::BinOp::SubAssign(_val) => { + formatter.write_str("BinOp::SubAssign")?; + Ok(()) + } + syn::BinOp::MulAssign(_val) => { + formatter.write_str("BinOp::MulAssign")?; + Ok(()) + } + syn::BinOp::DivAssign(_val) => { + formatter.write_str("BinOp::DivAssign")?; + Ok(()) + } + syn::BinOp::RemAssign(_val) => { + formatter.write_str("BinOp::RemAssign")?; + Ok(()) + } + syn::BinOp::BitXorAssign(_val) => { + formatter.write_str("BinOp::BitXorAssign")?; + Ok(()) + } + syn::BinOp::BitAndAssign(_val) => { + formatter.write_str("BinOp::BitAndAssign")?; + Ok(()) + } + syn::BinOp::BitOrAssign(_val) => { + formatter.write_str("BinOp::BitOrAssign")?; + Ok(()) + } + syn::BinOp::ShlAssign(_val) => { + formatter.write_str("BinOp::ShlAssign")?; + Ok(()) + } + syn::BinOp::ShrAssign(_val) => { + formatter.write_str("BinOp::ShrAssign")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::Block> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Block"); + formatter.field("stmts", Lite(&self.value.stmts)); + formatter.finish() + } +} +impl Debug for Lite<syn::BoundLifetimes> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("BoundLifetimes"); + if !self.value.lifetimes.is_empty() { + formatter.field("lifetimes", Lite(&self.value.lifetimes)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::CapturedParam> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::CapturedParam::Lifetime(_val) => { + formatter.write_str("CapturedParam::Lifetime")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::CapturedParam::Ident(_val) => { + formatter.write_str("CapturedParam::Ident")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::ConstParam> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ConstParam"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("ty", Lite(&self.value.ty)); + if self.value.eq_token.is_some() { + formatter.field("eq_token", &Present); + } + if let Some(val) = &self.value.default { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Expr); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("default", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Constraint> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Constraint"); + formatter.field("ident", Lite(&self.value.ident)); + if let Some(val) = &self.value.generics { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::AngleBracketedGenericArguments); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("generics", Print::ref_cast(val)); + } + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Data> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Data::Struct(_val) => { + let mut formatter = formatter.debug_struct("Data::Struct"); + formatter.field("fields", Lite(&_val.fields)); + if _val.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } + syn::Data::Enum(_val) => { + let mut formatter = formatter.debug_struct("Data::Enum"); + if !_val.variants.is_empty() { + formatter.field("variants", Lite(&_val.variants)); + } + formatter.finish() + } + syn::Data::Union(_val) => { + let mut formatter = formatter.debug_struct("Data::Union"); + formatter.field("fields", Lite(&_val.fields)); + formatter.finish() + } + } + } +} +impl Debug for Lite<syn::DataEnum> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("DataEnum"); + if !self.value.variants.is_empty() { + formatter.field("variants", Lite(&self.value.variants)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::DataStruct> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("DataStruct"); + formatter.field("fields", Lite(&self.value.fields)); + if self.value.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::DataUnion> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("DataUnion"); + formatter.field("fields", Lite(&self.value.fields)); + formatter.finish() + } +} +impl Debug for Lite<syn::DeriveInput> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("DeriveInput"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.field("data", Lite(&self.value.data)); + formatter.finish() + } +} +impl Debug for Lite<syn::Expr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Expr::Array(_val) => { + let mut formatter = formatter.debug_struct("Expr::Array"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if !_val.elems.is_empty() { + formatter.field("elems", Lite(&_val.elems)); + } + formatter.finish() + } + syn::Expr::Assign(_val) => { + let mut formatter = formatter.debug_struct("Expr::Assign"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("left", Lite(&_val.left)); + formatter.field("right", Lite(&_val.right)); + formatter.finish() + } + syn::Expr::Async(_val) => { + let mut formatter = formatter.debug_struct("Expr::Async"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if _val.capture.is_some() { + formatter.field("capture", &Present); + } + formatter.field("block", Lite(&_val.block)); + formatter.finish() + } + syn::Expr::Await(_val) => { + let mut formatter = formatter.debug_struct("Expr::Await"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("base", Lite(&_val.base)); + formatter.finish() + } + syn::Expr::Binary(_val) => { + let mut formatter = formatter.debug_struct("Expr::Binary"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("left", Lite(&_val.left)); + formatter.field("op", Lite(&_val.op)); + formatter.field("right", Lite(&_val.right)); + formatter.finish() + } + syn::Expr::Block(_val) => { + let mut formatter = formatter.debug_struct("Expr::Block"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Label); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.field("block", Lite(&_val.block)); + formatter.finish() + } + syn::Expr::Break(_val) => { + let mut formatter = formatter.debug_struct("Expr::Break"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Lifetime); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + if let Some(val) = &_val.expr { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("expr", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Expr::Call(_val) => { + let mut formatter = formatter.debug_struct("Expr::Call"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("func", Lite(&_val.func)); + if !_val.args.is_empty() { + formatter.field("args", Lite(&_val.args)); + } + formatter.finish() + } + syn::Expr::Cast(_val) => { + let mut formatter = formatter.debug_struct("Expr::Cast"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("expr", Lite(&_val.expr)); + formatter.field("ty", Lite(&_val.ty)); + formatter.finish() + } + syn::Expr::Closure(_val) => { + let mut formatter = formatter.debug_struct("Expr::Closure"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.lifetimes { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::BoundLifetimes); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("lifetimes", Print::ref_cast(val)); + } + if _val.constness.is_some() { + formatter.field("constness", &Present); + } + if _val.movability.is_some() { + formatter.field("movability", &Present); + } + if _val.asyncness.is_some() { + formatter.field("asyncness", &Present); + } + if _val.capture.is_some() { + formatter.field("capture", &Present); + } + if !_val.inputs.is_empty() { + formatter.field("inputs", Lite(&_val.inputs)); + } + formatter.field("output", Lite(&_val.output)); + formatter.field("body", Lite(&_val.body)); + formatter.finish() + } + syn::Expr::Const(_val) => { + let mut formatter = formatter.debug_struct("Expr::Const"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("block", Lite(&_val.block)); + formatter.finish() + } + syn::Expr::Continue(_val) => { + let mut formatter = formatter.debug_struct("Expr::Continue"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Lifetime); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Expr::Field(_val) => { + let mut formatter = formatter.debug_struct("Expr::Field"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("base", Lite(&_val.base)); + formatter.field("member", Lite(&_val.member)); + formatter.finish() + } + syn::Expr::ForLoop(_val) => { + let mut formatter = formatter.debug_struct("Expr::ForLoop"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Label); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.field("pat", Lite(&_val.pat)); + formatter.field("expr", Lite(&_val.expr)); + formatter.field("body", Lite(&_val.body)); + formatter.finish() + } + syn::Expr::Group(_val) => { + let mut formatter = formatter.debug_struct("Expr::Group"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Expr::If(_val) => { + let mut formatter = formatter.debug_struct("Expr::If"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("cond", Lite(&_val.cond)); + formatter.field("then_branch", Lite(&_val.then_branch)); + if let Some(val) = &_val.else_branch { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Else, Box<syn::Expr>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("else_branch", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Expr::Index(_val) => { + let mut formatter = formatter.debug_struct("Expr::Index"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("expr", Lite(&_val.expr)); + formatter.field("index", Lite(&_val.index)); + formatter.finish() + } + syn::Expr::Infer(_val) => { + let mut formatter = formatter.debug_struct("Expr::Infer"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.finish() + } + syn::Expr::Let(_val) => { + let mut formatter = formatter.debug_struct("Expr::Let"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("pat", Lite(&_val.pat)); + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Expr::Lit(_val) => { + let mut formatter = formatter.debug_struct("Expr::Lit"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("lit", Lite(&_val.lit)); + formatter.finish() + } + syn::Expr::Loop(_val) => { + let mut formatter = formatter.debug_struct("Expr::Loop"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Label); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.field("body", Lite(&_val.body)); + formatter.finish() + } + syn::Expr::Macro(_val) => { + let mut formatter = formatter.debug_struct("Expr::Macro"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("mac", Lite(&_val.mac)); + formatter.finish() + } + syn::Expr::Match(_val) => { + let mut formatter = formatter.debug_struct("Expr::Match"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("expr", Lite(&_val.expr)); + if !_val.arms.is_empty() { + formatter.field("arms", Lite(&_val.arms)); + } + formatter.finish() + } + syn::Expr::MethodCall(_val) => { + let mut formatter = formatter.debug_struct("Expr::MethodCall"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("receiver", Lite(&_val.receiver)); + formatter.field("method", Lite(&_val.method)); + if let Some(val) = &_val.turbofish { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::AngleBracketedGenericArguments); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("turbofish", Print::ref_cast(val)); + } + if !_val.args.is_empty() { + formatter.field("args", Lite(&_val.args)); + } + formatter.finish() + } + syn::Expr::Paren(_val) => { + let mut formatter = formatter.debug_struct("Expr::Paren"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Expr::Path(_val) => { + let mut formatter = formatter.debug_struct("Expr::Path"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&_val.path)); + formatter.finish() + } + syn::Expr::Range(_val) => { + let mut formatter = formatter.debug_struct("Expr::Range"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.start { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("start", Print::ref_cast(val)); + } + formatter.field("limits", Lite(&_val.limits)); + if let Some(val) = &_val.end { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("end", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Expr::RawAddr(_val) => { + let mut formatter = formatter.debug_struct("Expr::RawAddr"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("mutability", Lite(&_val.mutability)); + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Expr::Reference(_val) => { + let mut formatter = formatter.debug_struct("Expr::Reference"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if _val.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Expr::Repeat(_val) => { + let mut formatter = formatter.debug_struct("Expr::Repeat"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("expr", Lite(&_val.expr)); + formatter.field("len", Lite(&_val.len)); + formatter.finish() + } + syn::Expr::Return(_val) => { + let mut formatter = formatter.debug_struct("Expr::Return"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.expr { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("expr", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Expr::Struct(_val) => { + let mut formatter = formatter.debug_struct("Expr::Struct"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&_val.path)); + if !_val.fields.is_empty() { + formatter.field("fields", Lite(&_val.fields)); + } + if _val.dot2_token.is_some() { + formatter.field("dot2_token", &Present); + } + if let Some(val) = &_val.rest { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("rest", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Expr::Try(_val) => { + let mut formatter = formatter.debug_struct("Expr::Try"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Expr::TryBlock(_val) => { + let mut formatter = formatter.debug_struct("Expr::TryBlock"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("block", Lite(&_val.block)); + formatter.finish() + } + syn::Expr::Tuple(_val) => { + let mut formatter = formatter.debug_struct("Expr::Tuple"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if !_val.elems.is_empty() { + formatter.field("elems", Lite(&_val.elems)); + } + formatter.finish() + } + syn::Expr::Unary(_val) => { + let mut formatter = formatter.debug_struct("Expr::Unary"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("op", Lite(&_val.op)); + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Expr::Unsafe(_val) => { + let mut formatter = formatter.debug_struct("Expr::Unsafe"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("block", Lite(&_val.block)); + formatter.finish() + } + syn::Expr::Verbatim(_val) => { + formatter.write_str("Expr::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + syn::Expr::While(_val) => { + let mut formatter = formatter.debug_struct("Expr::While"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Label); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.field("cond", Lite(&_val.cond)); + formatter.field("body", Lite(&_val.body)); + formatter.finish() + } + syn::Expr::Yield(_val) => { + let mut formatter = formatter.debug_struct("Expr::Yield"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.expr { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("expr", Print::ref_cast(val)); + } + formatter.finish() + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::ExprArray> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprArray"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if !self.value.elems.is_empty() { + formatter.field("elems", Lite(&self.value.elems)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprAssign> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprAssign"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("left", Lite(&self.value.left)); + formatter.field("right", Lite(&self.value.right)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprAsync> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprAsync"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if self.value.capture.is_some() { + formatter.field("capture", &Present); + } + formatter.field("block", Lite(&self.value.block)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprAwait> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprAwait"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("base", Lite(&self.value.base)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprBinary> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprBinary"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("left", Lite(&self.value.left)); + formatter.field("op", Lite(&self.value.op)); + formatter.field("right", Lite(&self.value.right)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprBlock> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprBlock"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Label); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.field("block", Lite(&self.value.block)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprBreak> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprBreak"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Lifetime); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + if let Some(val) = &self.value.expr { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("expr", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprCall> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprCall"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("func", Lite(&self.value.func)); + if !self.value.args.is_empty() { + formatter.field("args", Lite(&self.value.args)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprCast> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprCast"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("expr", Lite(&self.value.expr)); + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprClosure> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprClosure"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.lifetimes { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::BoundLifetimes); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("lifetimes", Print::ref_cast(val)); + } + if self.value.constness.is_some() { + formatter.field("constness", &Present); + } + if self.value.movability.is_some() { + formatter.field("movability", &Present); + } + if self.value.asyncness.is_some() { + formatter.field("asyncness", &Present); + } + if self.value.capture.is_some() { + formatter.field("capture", &Present); + } + if !self.value.inputs.is_empty() { + formatter.field("inputs", Lite(&self.value.inputs)); + } + formatter.field("output", Lite(&self.value.output)); + formatter.field("body", Lite(&self.value.body)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprConst> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprConst"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("block", Lite(&self.value.block)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprContinue> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprContinue"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Lifetime); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprField> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprField"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("base", Lite(&self.value.base)); + formatter.field("member", Lite(&self.value.member)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprForLoop> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprForLoop"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Label); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.field("pat", Lite(&self.value.pat)); + formatter.field("expr", Lite(&self.value.expr)); + formatter.field("body", Lite(&self.value.body)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprGroup> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprGroup"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprIf> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprIf"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("cond", Lite(&self.value.cond)); + formatter.field("then_branch", Lite(&self.value.then_branch)); + if let Some(val) = &self.value.else_branch { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Else, Box<syn::Expr>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("else_branch", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprIndex> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprIndex"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("expr", Lite(&self.value.expr)); + formatter.field("index", Lite(&self.value.index)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprInfer> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprInfer"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprLet> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprLet"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("pat", Lite(&self.value.pat)); + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprLit> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprLit"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("lit", Lite(&self.value.lit)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprLoop> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprLoop"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Label); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.field("body", Lite(&self.value.body)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprMacro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprMacro"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("mac", Lite(&self.value.mac)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprMatch> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprMatch"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("expr", Lite(&self.value.expr)); + if !self.value.arms.is_empty() { + formatter.field("arms", Lite(&self.value.arms)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprMethodCall> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprMethodCall"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("receiver", Lite(&self.value.receiver)); + formatter.field("method", Lite(&self.value.method)); + if let Some(val) = &self.value.turbofish { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::AngleBracketedGenericArguments); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("turbofish", Print::ref_cast(val)); + } + if !self.value.args.is_empty() { + formatter.field("args", Lite(&self.value.args)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprParen> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprParen"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprPath> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprPath"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&self.value.path)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprRange> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprRange"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.start { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("start", Print::ref_cast(val)); + } + formatter.field("limits", Lite(&self.value.limits)); + if let Some(val) = &self.value.end { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("end", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprRawAddr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprRawAddr"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("mutability", Lite(&self.value.mutability)); + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprReference> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprReference"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if self.value.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprRepeat> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprRepeat"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("expr", Lite(&self.value.expr)); + formatter.field("len", Lite(&self.value.len)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprReturn> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprReturn"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.expr { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("expr", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprStruct> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprStruct"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&self.value.path)); + if !self.value.fields.is_empty() { + formatter.field("fields", Lite(&self.value.fields)); + } + if self.value.dot2_token.is_some() { + formatter.field("dot2_token", &Present); + } + if let Some(val) = &self.value.rest { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("rest", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprTry> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprTry"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprTryBlock> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprTryBlock"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("block", Lite(&self.value.block)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprTuple> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprTuple"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if !self.value.elems.is_empty() { + formatter.field("elems", Lite(&self.value.elems)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ExprUnary> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprUnary"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("op", Lite(&self.value.op)); + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprUnsafe> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprUnsafe"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("block", Lite(&self.value.block)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprWhile> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprWhile"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.label { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Label); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("label", Print::ref_cast(val)); + } + formatter.field("cond", Lite(&self.value.cond)); + formatter.field("body", Lite(&self.value.body)); + formatter.finish() + } +} +impl Debug for Lite<syn::ExprYield> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ExprYield"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.expr { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Box<syn::Expr>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("expr", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Field> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Field"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + match self.value.mutability { + syn::FieldMutability::None => {} + _ => { + formatter.field("mutability", Lite(&self.value.mutability)); + } + } + if let Some(val) = &self.value.ident { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(proc_macro2::Ident); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("ident", Print::ref_cast(val)); + } + if self.value.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::FieldMutability> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::FieldMutability::None => formatter.write_str("FieldMutability::None"), + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::FieldPat> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("FieldPat"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("member", Lite(&self.value.member)); + if self.value.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + formatter.field("pat", Lite(&self.value.pat)); + formatter.finish() + } +} +impl Debug for Lite<syn::FieldValue> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("FieldValue"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("member", Lite(&self.value.member)); + if self.value.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::Fields> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Fields::Named(_val) => { + let mut formatter = formatter.debug_struct("Fields::Named"); + if !_val.named.is_empty() { + formatter.field("named", Lite(&_val.named)); + } + formatter.finish() + } + syn::Fields::Unnamed(_val) => { + let mut formatter = formatter.debug_struct("Fields::Unnamed"); + if !_val.unnamed.is_empty() { + formatter.field("unnamed", Lite(&_val.unnamed)); + } + formatter.finish() + } + syn::Fields::Unit => formatter.write_str("Fields::Unit"), + } + } +} +impl Debug for Lite<syn::FieldsNamed> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("FieldsNamed"); + if !self.value.named.is_empty() { + formatter.field("named", Lite(&self.value.named)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::FieldsUnnamed> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("FieldsUnnamed"); + if !self.value.unnamed.is_empty() { + formatter.field("unnamed", Lite(&self.value.unnamed)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::File> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("File"); + if let Some(val) = &self.value.shebang { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(String); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("shebang", Print::ref_cast(val)); + } + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if !self.value.items.is_empty() { + formatter.field("items", Lite(&self.value.items)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::FnArg> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::FnArg::Receiver(_val) => { + formatter.write_str("FnArg::Receiver")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::FnArg::Typed(_val) => { + formatter.write_str("FnArg::Typed")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::ForeignItem> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::ForeignItem::Fn(_val) => { + let mut formatter = formatter.debug_struct("ForeignItem::Fn"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("sig", Lite(&_val.sig)); + formatter.finish() + } + syn::ForeignItem::Static(_val) => { + let mut formatter = formatter.debug_struct("ForeignItem::Static"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + match _val.mutability { + syn::StaticMutability::None => {} + _ => { + formatter.field("mutability", Lite(&_val.mutability)); + } + } + formatter.field("ident", Lite(&_val.ident)); + formatter.field("ty", Lite(&_val.ty)); + formatter.finish() + } + syn::ForeignItem::Type(_val) => { + let mut formatter = formatter.debug_struct("ForeignItem::Type"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + formatter.finish() + } + syn::ForeignItem::Macro(_val) => { + let mut formatter = formatter.debug_struct("ForeignItem::Macro"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("mac", Lite(&_val.mac)); + if _val.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } + syn::ForeignItem::Verbatim(_val) => { + formatter.write_str("ForeignItem::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::ForeignItemFn> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ForeignItemFn"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("sig", Lite(&self.value.sig)); + formatter.finish() + } +} +impl Debug for Lite<syn::ForeignItemMacro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ForeignItemMacro"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("mac", Lite(&self.value.mac)); + if self.value.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ForeignItemStatic> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ForeignItemStatic"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + match self.value.mutability { + syn::StaticMutability::None => {} + _ => { + formatter.field("mutability", Lite(&self.value.mutability)); + } + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::ForeignItemType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ForeignItemType"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.finish() + } +} +impl Debug for Lite<syn::GenericArgument> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::GenericArgument::Lifetime(_val) => { + formatter.write_str("GenericArgument::Lifetime")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::GenericArgument::Type(_val) => { + formatter.write_str("GenericArgument::Type")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::GenericArgument::Const(_val) => { + formatter.write_str("GenericArgument::Const")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::GenericArgument::AssocType(_val) => { + formatter.write_str("GenericArgument::AssocType")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::GenericArgument::AssocConst(_val) => { + formatter.write_str("GenericArgument::AssocConst")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::GenericArgument::Constraint(_val) => { + formatter.write_str("GenericArgument::Constraint")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::GenericParam> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::GenericParam::Lifetime(_val) => { + formatter.write_str("GenericParam::Lifetime")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::GenericParam::Type(_val) => { + formatter.write_str("GenericParam::Type")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::GenericParam::Const(_val) => { + formatter.write_str("GenericParam::Const")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::Generics> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Generics"); + if self.value.lt_token.is_some() { + formatter.field("lt_token", &Present); + } + if !self.value.params.is_empty() { + formatter.field("params", Lite(&self.value.params)); + } + if self.value.gt_token.is_some() { + formatter.field("gt_token", &Present); + } + if let Some(val) = &self.value.where_clause { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::WhereClause); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("where_clause", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ImplItem> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::ImplItem::Const(_val) => { + let mut formatter = formatter.debug_struct("ImplItem::Const"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + if _val.defaultness.is_some() { + formatter.field("defaultness", &Present); + } + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + formatter.field("ty", Lite(&_val.ty)); + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::ImplItem::Fn(_val) => { + let mut formatter = formatter.debug_struct("ImplItem::Fn"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + if _val.defaultness.is_some() { + formatter.field("defaultness", &Present); + } + formatter.field("sig", Lite(&_val.sig)); + formatter.field("block", Lite(&_val.block)); + formatter.finish() + } + syn::ImplItem::Type(_val) => { + let mut formatter = formatter.debug_struct("ImplItem::Type"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + if _val.defaultness.is_some() { + formatter.field("defaultness", &Present); + } + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + formatter.field("ty", Lite(&_val.ty)); + formatter.finish() + } + syn::ImplItem::Macro(_val) => { + let mut formatter = formatter.debug_struct("ImplItem::Macro"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("mac", Lite(&_val.mac)); + if _val.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } + syn::ImplItem::Verbatim(_val) => { + formatter.write_str("ImplItem::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::ImplItemConst> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ImplItemConst"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + if self.value.defaultness.is_some() { + formatter.field("defaultness", &Present); + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.field("ty", Lite(&self.value.ty)); + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ImplItemFn> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ImplItemFn"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + if self.value.defaultness.is_some() { + formatter.field("defaultness", &Present); + } + formatter.field("sig", Lite(&self.value.sig)); + formatter.field("block", Lite(&self.value.block)); + formatter.finish() + } +} +impl Debug for Lite<syn::ImplItemMacro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ImplItemMacro"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("mac", Lite(&self.value.mac)); + if self.value.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ImplItemType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ImplItemType"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + if self.value.defaultness.is_some() { + formatter.field("defaultness", &Present); + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::ImplRestriction> { + fn fmt(&self, _formatter: &mut fmt::Formatter) -> fmt::Result { + unreachable!() + } +} +impl Debug for Lite<syn::Index> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Index"); + formatter.field("index", Lite(&self.value.index)); + formatter.finish() + } +} +impl Debug for Lite<syn::Item> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Item::Const(_val) => { + let mut formatter = formatter.debug_struct("Item::Const"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + formatter.field("ty", Lite(&_val.ty)); + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Item::Enum(_val) => { + let mut formatter = formatter.debug_struct("Item::Enum"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + if !_val.variants.is_empty() { + formatter.field("variants", Lite(&_val.variants)); + } + formatter.finish() + } + syn::Item::ExternCrate(_val) => { + let mut formatter = formatter.debug_struct("Item::ExternCrate"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("ident", Lite(&_val.ident)); + if let Some(val) = &_val.rename { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::As, proc_macro2::Ident)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("rename", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Item::Fn(_val) => { + let mut formatter = formatter.debug_struct("Item::Fn"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("sig", Lite(&_val.sig)); + formatter.field("block", Lite(&_val.block)); + formatter.finish() + } + syn::Item::ForeignMod(_val) => { + let mut formatter = formatter.debug_struct("Item::ForeignMod"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if _val.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + formatter.field("abi", Lite(&_val.abi)); + if !_val.items.is_empty() { + formatter.field("items", Lite(&_val.items)); + } + formatter.finish() + } + syn::Item::Impl(_val) => { + let mut formatter = formatter.debug_struct("Item::Impl"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if _val.defaultness.is_some() { + formatter.field("defaultness", &Present); + } + if _val.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + formatter.field("generics", Lite(&_val.generics)); + if let Some(val) = &_val.trait_ { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((Option<syn::token::Not>, syn::Path, syn::token::For)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt( + &( + &super::Option { + present: self.0.0.is_some(), + }, + Lite(&self.0.1), + ), + formatter, + )?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("trait_", Print::ref_cast(val)); + } + formatter.field("self_ty", Lite(&_val.self_ty)); + if !_val.items.is_empty() { + formatter.field("items", Lite(&_val.items)); + } + formatter.finish() + } + syn::Item::Macro(_val) => { + let mut formatter = formatter.debug_struct("Item::Macro"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.ident { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(proc_macro2::Ident); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("ident", Print::ref_cast(val)); + } + formatter.field("mac", Lite(&_val.mac)); + if _val.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } + syn::Item::Mod(_val) => { + let mut formatter = formatter.debug_struct("Item::Mod"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + if _val.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + formatter.field("ident", Lite(&_val.ident)); + if let Some(val) = &_val.content { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Brace, Vec<syn::Item>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("content", Print::ref_cast(val)); + } + if _val.semi.is_some() { + formatter.field("semi", &Present); + } + formatter.finish() + } + syn::Item::Static(_val) => { + let mut formatter = formatter.debug_struct("Item::Static"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + match _val.mutability { + syn::StaticMutability::None => {} + _ => { + formatter.field("mutability", Lite(&_val.mutability)); + } + } + formatter.field("ident", Lite(&_val.ident)); + formatter.field("ty", Lite(&_val.ty)); + formatter.field("expr", Lite(&_val.expr)); + formatter.finish() + } + syn::Item::Struct(_val) => { + let mut formatter = formatter.debug_struct("Item::Struct"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + formatter.field("fields", Lite(&_val.fields)); + if _val.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } + syn::Item::Trait(_val) => { + let mut formatter = formatter.debug_struct("Item::Trait"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + if _val.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + if _val.auto_token.is_some() { + formatter.field("auto_token", &Present); + } + if let Some(val) = &_val.restriction { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::ImplRestriction); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("restriction", Print::ref_cast(val)); + } + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + if _val.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + if !_val.supertraits.is_empty() { + formatter.field("supertraits", Lite(&_val.supertraits)); + } + if !_val.items.is_empty() { + formatter.field("items", Lite(&_val.items)); + } + formatter.finish() + } + syn::Item::TraitAlias(_val) => { + let mut formatter = formatter.debug_struct("Item::TraitAlias"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + if !_val.bounds.is_empty() { + formatter.field("bounds", Lite(&_val.bounds)); + } + formatter.finish() + } + syn::Item::Type(_val) => { + let mut formatter = formatter.debug_struct("Item::Type"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + formatter.field("ty", Lite(&_val.ty)); + formatter.finish() + } + syn::Item::Union(_val) => { + let mut formatter = formatter.debug_struct("Item::Union"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + formatter.field("fields", Lite(&_val.fields)); + formatter.finish() + } + syn::Item::Use(_val) => { + let mut formatter = formatter.debug_struct("Item::Use"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("vis", Lite(&_val.vis)); + if _val.leading_colon.is_some() { + formatter.field("leading_colon", &Present); + } + formatter.field("tree", Lite(&_val.tree)); + formatter.finish() + } + syn::Item::Verbatim(_val) => { + formatter.write_str("Item::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::ItemConst> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemConst"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.field("ty", Lite(&self.value.ty)); + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ItemEnum> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemEnum"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + if !self.value.variants.is_empty() { + formatter.field("variants", Lite(&self.value.variants)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemExternCrate> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemExternCrate"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + if let Some(val) = &self.value.rename { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::As, proc_macro2::Ident)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("rename", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemFn> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemFn"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("sig", Lite(&self.value.sig)); + formatter.field("block", Lite(&self.value.block)); + formatter.finish() + } +} +impl Debug for Lite<syn::ItemForeignMod> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemForeignMod"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if self.value.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + formatter.field("abi", Lite(&self.value.abi)); + if !self.value.items.is_empty() { + formatter.field("items", Lite(&self.value.items)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemImpl> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemImpl"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if self.value.defaultness.is_some() { + formatter.field("defaultness", &Present); + } + if self.value.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + formatter.field("generics", Lite(&self.value.generics)); + if let Some(val) = &self.value.trait_ { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((Option<syn::token::Not>, syn::Path, syn::token::For)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt( + &( + &super::Option { + present: self.0.0.is_some(), + }, + Lite(&self.0.1), + ), + formatter, + )?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("trait_", Print::ref_cast(val)); + } + formatter.field("self_ty", Lite(&self.value.self_ty)); + if !self.value.items.is_empty() { + formatter.field("items", Lite(&self.value.items)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemMacro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemMacro"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.ident { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(proc_macro2::Ident); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("ident", Print::ref_cast(val)); + } + formatter.field("mac", Lite(&self.value.mac)); + if self.value.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemMod> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemMod"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + if self.value.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + formatter.field("ident", Lite(&self.value.ident)); + if let Some(val) = &self.value.content { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Brace, Vec<syn::Item>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("content", Print::ref_cast(val)); + } + if self.value.semi.is_some() { + formatter.field("semi", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemStatic> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemStatic"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + match self.value.mutability { + syn::StaticMutability::None => {} + _ => { + formatter.field("mutability", Lite(&self.value.mutability)); + } + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("ty", Lite(&self.value.ty)); + formatter.field("expr", Lite(&self.value.expr)); + formatter.finish() + } +} +impl Debug for Lite<syn::ItemStruct> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemStruct"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.field("fields", Lite(&self.value.fields)); + if self.value.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemTrait> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemTrait"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + if self.value.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + if self.value.auto_token.is_some() { + formatter.field("auto_token", &Present); + } + if let Some(val) = &self.value.restriction { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::ImplRestriction); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("restriction", Print::ref_cast(val)); + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + if self.value.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + if !self.value.supertraits.is_empty() { + formatter.field("supertraits", Lite(&self.value.supertraits)); + } + if !self.value.items.is_empty() { + formatter.field("items", Lite(&self.value.items)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemTraitAlias> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemTraitAlias"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::ItemType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemType"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::ItemUnion> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemUnion"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.field("fields", Lite(&self.value.fields)); + formatter.finish() + } +} +impl Debug for Lite<syn::ItemUse> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ItemUse"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("vis", Lite(&self.value.vis)); + if self.value.leading_colon.is_some() { + formatter.field("leading_colon", &Present); + } + formatter.field("tree", Lite(&self.value.tree)); + formatter.finish() + } +} +impl Debug for Lite<syn::Label> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Label"); + formatter.field("name", Lite(&self.value.name)); + formatter.finish() + } +} +impl Debug for Lite<syn::Lifetime> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Lifetime"); + formatter.field("ident", Lite(&self.value.ident)); + formatter.finish() + } +} +impl Debug for Lite<syn::LifetimeParam> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("LifetimeParam"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("lifetime", Lite(&self.value.lifetime)); + if self.value.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Lit> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Lit::Str(_val) => write!(formatter, "{:?}", _val.value()), + syn::Lit::ByteStr(_val) => write!(formatter, "{:?}", _val.value()), + syn::Lit::CStr(_val) => write!(formatter, "{:?}", _val.value()), + syn::Lit::Byte(_val) => write!(formatter, "{:?}", _val.value()), + syn::Lit::Char(_val) => write!(formatter, "{:?}", _val.value()), + syn::Lit::Int(_val) => write!(formatter, "{}", _val), + syn::Lit::Float(_val) => write!(formatter, "{}", _val), + syn::Lit::Bool(_val) => { + let mut formatter = formatter.debug_struct("Lit::Bool"); + formatter.field("value", Lite(&_val.value)); + formatter.finish() + } + syn::Lit::Verbatim(_val) => { + formatter.write_str("Lit::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::LitBool> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("LitBool"); + formatter.field("value", Lite(&self.value.value)); + formatter.finish() + } +} +impl Debug for Lite<syn::LitByte> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{:?}", self.value.value()) + } +} +impl Debug for Lite<syn::LitByteStr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{:?}", self.value.value()) + } +} +impl Debug for Lite<syn::LitCStr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{:?}", self.value.value()) + } +} +impl Debug for Lite<syn::LitChar> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{:?}", self.value.value()) + } +} +impl Debug for Lite<syn::LitFloat> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{}", self.value) + } +} +impl Debug for Lite<syn::LitInt> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{}", self.value) + } +} +impl Debug for Lite<syn::LitStr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{:?}", self.value.value()) + } +} +impl Debug for Lite<syn::Local> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Local"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("pat", Lite(&self.value.pat)); + if let Some(val) = &self.value.init { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::LocalInit); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("init", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::LocalInit> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("LocalInit"); + formatter.field("expr", Lite(&self.value.expr)); + if let Some(val) = &self.value.diverge { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Else, Box<syn::Expr>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("diverge", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Macro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Macro"); + formatter.field("path", Lite(&self.value.path)); + formatter.field("delimiter", Lite(&self.value.delimiter)); + formatter.field("tokens", Lite(&self.value.tokens)); + formatter.finish() + } +} +impl Debug for Lite<syn::MacroDelimiter> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::MacroDelimiter::Paren(_val) => { + formatter.write_str("MacroDelimiter::Paren")?; + Ok(()) + } + syn::MacroDelimiter::Brace(_val) => { + formatter.write_str("MacroDelimiter::Brace")?; + Ok(()) + } + syn::MacroDelimiter::Bracket(_val) => { + formatter.write_str("MacroDelimiter::Bracket")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::Member> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Member::Named(_val) => { + formatter.write_str("Member::Named")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::Member::Unnamed(_val) => { + formatter.write_str("Member::Unnamed")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::Meta> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Meta::Path(_val) => { + let mut formatter = formatter.debug_struct("Meta::Path"); + if _val.leading_colon.is_some() { + formatter.field("leading_colon", &Present); + } + if !_val.segments.is_empty() { + formatter.field("segments", Lite(&_val.segments)); + } + formatter.finish() + } + syn::Meta::List(_val) => { + let mut formatter = formatter.debug_struct("Meta::List"); + formatter.field("path", Lite(&_val.path)); + formatter.field("delimiter", Lite(&_val.delimiter)); + formatter.field("tokens", Lite(&_val.tokens)); + formatter.finish() + } + syn::Meta::NameValue(_val) => { + let mut formatter = formatter.debug_struct("Meta::NameValue"); + formatter.field("path", Lite(&_val.path)); + formatter.field("value", Lite(&_val.value)); + formatter.finish() + } + } + } +} +impl Debug for Lite<syn::MetaList> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("MetaList"); + formatter.field("path", Lite(&self.value.path)); + formatter.field("delimiter", Lite(&self.value.delimiter)); + formatter.field("tokens", Lite(&self.value.tokens)); + formatter.finish() + } +} +impl Debug for Lite<syn::MetaNameValue> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("MetaNameValue"); + formatter.field("path", Lite(&self.value.path)); + formatter.field("value", Lite(&self.value.value)); + formatter.finish() + } +} +impl Debug for Lite<syn::ParenthesizedGenericArguments> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("ParenthesizedGenericArguments"); + if !self.value.inputs.is_empty() { + formatter.field("inputs", Lite(&self.value.inputs)); + } + formatter.field("output", Lite(&self.value.output)); + formatter.finish() + } +} +impl Debug for Lite<syn::Pat> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Pat::Const(_val) => { + formatter.write_str("Pat::Const")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::Pat::Ident(_val) => { + let mut formatter = formatter.debug_struct("Pat::Ident"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if _val.by_ref.is_some() { + formatter.field("by_ref", &Present); + } + if _val.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("ident", Lite(&_val.ident)); + if let Some(val) = &_val.subpat { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::At, Box<syn::Pat>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("subpat", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Pat::Lit(_val) => { + formatter.write_str("Pat::Lit")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::Pat::Macro(_val) => { + formatter.write_str("Pat::Macro")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::Pat::Or(_val) => { + let mut formatter = formatter.debug_struct("Pat::Or"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if _val.leading_vert.is_some() { + formatter.field("leading_vert", &Present); + } + if !_val.cases.is_empty() { + formatter.field("cases", Lite(&_val.cases)); + } + formatter.finish() + } + syn::Pat::Paren(_val) => { + let mut formatter = formatter.debug_struct("Pat::Paren"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("pat", Lite(&_val.pat)); + formatter.finish() + } + syn::Pat::Path(_val) => { + formatter.write_str("Pat::Path")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::Pat::Range(_val) => { + formatter.write_str("Pat::Range")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::Pat::Reference(_val) => { + let mut formatter = formatter.debug_struct("Pat::Reference"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if _val.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("pat", Lite(&_val.pat)); + formatter.finish() + } + syn::Pat::Rest(_val) => { + let mut formatter = formatter.debug_struct("Pat::Rest"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.finish() + } + syn::Pat::Slice(_val) => { + let mut formatter = formatter.debug_struct("Pat::Slice"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if !_val.elems.is_empty() { + formatter.field("elems", Lite(&_val.elems)); + } + formatter.finish() + } + syn::Pat::Struct(_val) => { + let mut formatter = formatter.debug_struct("Pat::Struct"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&_val.path)); + if !_val.fields.is_empty() { + formatter.field("fields", Lite(&_val.fields)); + } + if let Some(val) = &_val.rest { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::PatRest); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("rest", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Pat::Tuple(_val) => { + let mut formatter = formatter.debug_struct("Pat::Tuple"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if !_val.elems.is_empty() { + formatter.field("elems", Lite(&_val.elems)); + } + formatter.finish() + } + syn::Pat::TupleStruct(_val) => { + let mut formatter = formatter.debug_struct("Pat::TupleStruct"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + if let Some(val) = &_val.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&_val.path)); + if !_val.elems.is_empty() { + formatter.field("elems", Lite(&_val.elems)); + } + formatter.finish() + } + syn::Pat::Type(_val) => { + let mut formatter = formatter.debug_struct("Pat::Type"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("pat", Lite(&_val.pat)); + formatter.field("ty", Lite(&_val.ty)); + formatter.finish() + } + syn::Pat::Verbatim(_val) => { + formatter.write_str("Pat::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + syn::Pat::Wild(_val) => { + let mut formatter = formatter.debug_struct("Pat::Wild"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.finish() + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::PatIdent> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatIdent"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if self.value.by_ref.is_some() { + formatter.field("by_ref", &Present); + } + if self.value.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("ident", Lite(&self.value.ident)); + if let Some(val) = &self.value.subpat { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::At, Box<syn::Pat>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("subpat", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PatOr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatOr"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if self.value.leading_vert.is_some() { + formatter.field("leading_vert", &Present); + } + if !self.value.cases.is_empty() { + formatter.field("cases", Lite(&self.value.cases)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PatParen> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatParen"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("pat", Lite(&self.value.pat)); + formatter.finish() + } +} +impl Debug for Lite<syn::PatReference> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatReference"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if self.value.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("pat", Lite(&self.value.pat)); + formatter.finish() + } +} +impl Debug for Lite<syn::PatRest> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatRest"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PatSlice> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatSlice"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if !self.value.elems.is_empty() { + formatter.field("elems", Lite(&self.value.elems)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PatStruct> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatStruct"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&self.value.path)); + if !self.value.fields.is_empty() { + formatter.field("fields", Lite(&self.value.fields)); + } + if let Some(val) = &self.value.rest { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::PatRest); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("rest", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PatTuple> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatTuple"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if !self.value.elems.is_empty() { + formatter.field("elems", Lite(&self.value.elems)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PatTupleStruct> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatTupleStruct"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&self.value.path)); + if !self.value.elems.is_empty() { + formatter.field("elems", Lite(&self.value.elems)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PatType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatType"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("pat", Lite(&self.value.pat)); + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::PatWild> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PatWild"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Path> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Path"); + if self.value.leading_colon.is_some() { + formatter.field("leading_colon", &Present); + } + if !self.value.segments.is_empty() { + formatter.field("segments", Lite(&self.value.segments)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PathArguments> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::PathArguments::None => formatter.write_str("PathArguments::None"), + syn::PathArguments::AngleBracketed(_val) => { + let mut formatter = formatter + .debug_struct("PathArguments::AngleBracketed"); + if _val.colon2_token.is_some() { + formatter.field("colon2_token", &Present); + } + if !_val.args.is_empty() { + formatter.field("args", Lite(&_val.args)); + } + formatter.finish() + } + syn::PathArguments::Parenthesized(_val) => { + let mut formatter = formatter + .debug_struct("PathArguments::Parenthesized"); + if !_val.inputs.is_empty() { + formatter.field("inputs", Lite(&_val.inputs)); + } + formatter.field("output", Lite(&_val.output)); + formatter.finish() + } + } + } +} +impl Debug for Lite<syn::PathSegment> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PathSegment"); + formatter.field("ident", Lite(&self.value.ident)); + match self.value.arguments { + syn::PathArguments::None => {} + _ => { + formatter.field("arguments", Lite(&self.value.arguments)); + } + } + formatter.finish() + } +} +impl Debug for Lite<syn::PointerMutability> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::PointerMutability::Const(_val) => { + formatter.write_str("PointerMutability::Const")?; + Ok(()) + } + syn::PointerMutability::Mut(_val) => { + formatter.write_str("PointerMutability::Mut")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::PreciseCapture> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PreciseCapture"); + if !self.value.params.is_empty() { + formatter.field("params", Lite(&self.value.params)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PredicateLifetime> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PredicateLifetime"); + formatter.field("lifetime", Lite(&self.value.lifetime)); + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::PredicateType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("PredicateType"); + if let Some(val) = &self.value.lifetimes { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::BoundLifetimes); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("lifetimes", Print::ref_cast(val)); + } + formatter.field("bounded_ty", Lite(&self.value.bounded_ty)); + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::QSelf> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("QSelf"); + formatter.field("ty", Lite(&self.value.ty)); + formatter.field("position", Lite(&self.value.position)); + if self.value.as_token.is_some() { + formatter.field("as_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::RangeLimits> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::RangeLimits::HalfOpen(_val) => { + formatter.write_str("RangeLimits::HalfOpen")?; + Ok(()) + } + syn::RangeLimits::Closed(_val) => { + formatter.write_str("RangeLimits::Closed")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::Receiver> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Receiver"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.reference { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::And, Option<syn::Lifetime>)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt( + { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Option<syn::Lifetime>); + impl Debug for Print { + fn fmt( + &self, + formatter: &mut fmt::Formatter, + ) -> fmt::Result { + match &self.0 { + Some(_val) => { + formatter.write_str("Some(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + None => formatter.write_str("None"), + } + } + } + Print::ref_cast(&self.0.1) + }, + formatter, + )?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("reference", Print::ref_cast(val)); + } + if self.value.mutability.is_some() { + formatter.field("mutability", &Present); + } + if self.value.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + formatter.field("ty", Lite(&self.value.ty)); + formatter.finish() + } +} +impl Debug for Lite<syn::ReturnType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::ReturnType::Default => formatter.write_str("ReturnType::Default"), + syn::ReturnType::Type(_v0, _v1) => { + let mut formatter = formatter.debug_tuple("ReturnType::Type"); + formatter.field(Lite(_v1)); + formatter.finish() + } + } + } +} +impl Debug for Lite<syn::Signature> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Signature"); + if self.value.constness.is_some() { + formatter.field("constness", &Present); + } + if self.value.asyncness.is_some() { + formatter.field("asyncness", &Present); + } + if self.value.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + if let Some(val) = &self.value.abi { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Abi); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("abi", Print::ref_cast(val)); + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + if !self.value.inputs.is_empty() { + formatter.field("inputs", Lite(&self.value.inputs)); + } + if let Some(val) = &self.value.variadic { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Variadic); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("variadic", Print::ref_cast(val)); + } + formatter.field("output", Lite(&self.value.output)); + formatter.finish() + } +} +impl Debug for Lite<syn::StaticMutability> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::StaticMutability::Mut(_val) => { + formatter.write_str("StaticMutability::Mut")?; + Ok(()) + } + syn::StaticMutability::None => formatter.write_str("StaticMutability::None"), + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::Stmt> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Stmt::Local(_val) => { + let mut formatter = formatter.debug_struct("Stmt::Local"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("pat", Lite(&_val.pat)); + if let Some(val) = &_val.init { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::LocalInit); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("init", Print::ref_cast(val)); + } + formatter.finish() + } + syn::Stmt::Item(_val) => { + formatter.write_str("Stmt::Item")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::Stmt::Expr(_v0, _v1) => { + let mut formatter = formatter.debug_tuple("Stmt::Expr"); + formatter.field(Lite(_v0)); + formatter + .field( + &super::Option { + present: _v1.is_some(), + }, + ); + formatter.finish() + } + syn::Stmt::Macro(_val) => { + let mut formatter = formatter.debug_struct("Stmt::Macro"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("mac", Lite(&_val.mac)); + if _val.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } + } + } +} +impl Debug for Lite<syn::StmtMacro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("StmtMacro"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("mac", Lite(&self.value.mac)); + if self.value.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::TraitBound> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TraitBound"); + if self.value.paren_token.is_some() { + formatter.field("paren_token", &Present); + } + match self.value.modifier { + syn::TraitBoundModifier::None => {} + _ => { + formatter.field("modifier", Lite(&self.value.modifier)); + } + } + if let Some(val) = &self.value.lifetimes { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::BoundLifetimes); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("lifetimes", Print::ref_cast(val)); + } + formatter.field("path", Lite(&self.value.path)); + formatter.finish() + } +} +impl Debug for Lite<syn::TraitBoundModifier> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::TraitBoundModifier::None => { + formatter.write_str("TraitBoundModifier::None") + } + syn::TraitBoundModifier::Maybe(_val) => { + formatter.write_str("TraitBoundModifier::Maybe")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::TraitItem> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::TraitItem::Const(_val) => { + let mut formatter = formatter.debug_struct("TraitItem::Const"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + formatter.field("ty", Lite(&_val.ty)); + if let Some(val) = &_val.default { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Eq, syn::Expr)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("default", Print::ref_cast(val)); + } + formatter.finish() + } + syn::TraitItem::Fn(_val) => { + let mut formatter = formatter.debug_struct("TraitItem::Fn"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("sig", Lite(&_val.sig)); + if let Some(val) = &_val.default { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Block); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("default", Print::ref_cast(val)); + } + if _val.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } + syn::TraitItem::Type(_val) => { + let mut formatter = formatter.debug_struct("TraitItem::Type"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("ident", Lite(&_val.ident)); + formatter.field("generics", Lite(&_val.generics)); + if _val.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + if !_val.bounds.is_empty() { + formatter.field("bounds", Lite(&_val.bounds)); + } + if let Some(val) = &_val.default { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Eq, syn::Type)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("default", Print::ref_cast(val)); + } + formatter.finish() + } + syn::TraitItem::Macro(_val) => { + let mut formatter = formatter.debug_struct("TraitItem::Macro"); + if !_val.attrs.is_empty() { + formatter.field("attrs", Lite(&_val.attrs)); + } + formatter.field("mac", Lite(&_val.mac)); + if _val.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } + syn::TraitItem::Verbatim(_val) => { + formatter.write_str("TraitItem::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::TraitItemConst> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TraitItemConst"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + formatter.field("ty", Lite(&self.value.ty)); + if let Some(val) = &self.value.default { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Eq, syn::Expr)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("default", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::TraitItemFn> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TraitItemFn"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("sig", Lite(&self.value.sig)); + if let Some(val) = &self.value.default { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Block); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("default", Print::ref_cast(val)); + } + if self.value.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::TraitItemMacro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TraitItemMacro"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("mac", Lite(&self.value.mac)); + if self.value.semi_token.is_some() { + formatter.field("semi_token", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::TraitItemType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TraitItemType"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("generics", Lite(&self.value.generics)); + if self.value.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + if let Some(val) = &self.value.default { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Eq, syn::Type)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("default", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Type> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Type::Array(_val) => { + let mut formatter = formatter.debug_struct("Type::Array"); + formatter.field("elem", Lite(&_val.elem)); + formatter.field("len", Lite(&_val.len)); + formatter.finish() + } + syn::Type::BareFn(_val) => { + let mut formatter = formatter.debug_struct("Type::BareFn"); + if let Some(val) = &_val.lifetimes { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::BoundLifetimes); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("lifetimes", Print::ref_cast(val)); + } + if _val.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + if let Some(val) = &_val.abi { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Abi); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("abi", Print::ref_cast(val)); + } + if !_val.inputs.is_empty() { + formatter.field("inputs", Lite(&_val.inputs)); + } + if let Some(val) = &_val.variadic { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::BareVariadic); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("variadic", Print::ref_cast(val)); + } + formatter.field("output", Lite(&_val.output)); + formatter.finish() + } + syn::Type::Group(_val) => { + let mut formatter = formatter.debug_struct("Type::Group"); + formatter.field("elem", Lite(&_val.elem)); + formatter.finish() + } + syn::Type::ImplTrait(_val) => { + let mut formatter = formatter.debug_struct("Type::ImplTrait"); + if !_val.bounds.is_empty() { + formatter.field("bounds", Lite(&_val.bounds)); + } + formatter.finish() + } + syn::Type::Infer(_val) => { + let mut formatter = formatter.debug_struct("Type::Infer"); + formatter.finish() + } + syn::Type::Macro(_val) => { + let mut formatter = formatter.debug_struct("Type::Macro"); + formatter.field("mac", Lite(&_val.mac)); + formatter.finish() + } + syn::Type::Never(_val) => { + let mut formatter = formatter.debug_struct("Type::Never"); + formatter.finish() + } + syn::Type::Paren(_val) => { + let mut formatter = formatter.debug_struct("Type::Paren"); + formatter.field("elem", Lite(&_val.elem)); + formatter.finish() + } + syn::Type::Path(_val) => { + let mut formatter = formatter.debug_struct("Type::Path"); + if let Some(val) = &_val.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&_val.path)); + formatter.finish() + } + syn::Type::Ptr(_val) => { + let mut formatter = formatter.debug_struct("Type::Ptr"); + if _val.const_token.is_some() { + formatter.field("const_token", &Present); + } + if _val.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("elem", Lite(&_val.elem)); + formatter.finish() + } + syn::Type::Reference(_val) => { + let mut formatter = formatter.debug_struct("Type::Reference"); + if let Some(val) = &_val.lifetime { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Lifetime); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("lifetime", Print::ref_cast(val)); + } + if _val.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("elem", Lite(&_val.elem)); + formatter.finish() + } + syn::Type::Slice(_val) => { + let mut formatter = formatter.debug_struct("Type::Slice"); + formatter.field("elem", Lite(&_val.elem)); + formatter.finish() + } + syn::Type::TraitObject(_val) => { + let mut formatter = formatter.debug_struct("Type::TraitObject"); + if _val.dyn_token.is_some() { + formatter.field("dyn_token", &Present); + } + if !_val.bounds.is_empty() { + formatter.field("bounds", Lite(&_val.bounds)); + } + formatter.finish() + } + syn::Type::Tuple(_val) => { + let mut formatter = formatter.debug_struct("Type::Tuple"); + if !_val.elems.is_empty() { + formatter.field("elems", Lite(&_val.elems)); + } + formatter.finish() + } + syn::Type::Verbatim(_val) => { + formatter.write_str("Type::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::TypeArray> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeArray"); + formatter.field("elem", Lite(&self.value.elem)); + formatter.field("len", Lite(&self.value.len)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeBareFn> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeBareFn"); + if let Some(val) = &self.value.lifetimes { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::BoundLifetimes); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("lifetimes", Print::ref_cast(val)); + } + if self.value.unsafety.is_some() { + formatter.field("unsafety", &Present); + } + if let Some(val) = &self.value.abi { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Abi); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("abi", Print::ref_cast(val)); + } + if !self.value.inputs.is_empty() { + formatter.field("inputs", Lite(&self.value.inputs)); + } + if let Some(val) = &self.value.variadic { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::BareVariadic); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("variadic", Print::ref_cast(val)); + } + formatter.field("output", Lite(&self.value.output)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeGroup> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeGroup"); + formatter.field("elem", Lite(&self.value.elem)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeImplTrait> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeImplTrait"); + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::TypeInfer> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeInfer"); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeMacro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeMacro"); + formatter.field("mac", Lite(&self.value.mac)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeNever> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeNever"); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeParam> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeParam"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("ident", Lite(&self.value.ident)); + if self.value.colon_token.is_some() { + formatter.field("colon_token", &Present); + } + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + if self.value.eq_token.is_some() { + formatter.field("eq_token", &Present); + } + if let Some(val) = &self.value.default { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Type); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("default", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::TypeParamBound> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::TypeParamBound::Trait(_val) => { + formatter.write_str("TypeParamBound::Trait")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::TypeParamBound::Lifetime(_val) => { + let mut formatter = formatter.debug_struct("TypeParamBound::Lifetime"); + formatter.field("ident", Lite(&_val.ident)); + formatter.finish() + } + syn::TypeParamBound::PreciseCapture(_val) => { + formatter.write_str("TypeParamBound::PreciseCapture")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::TypeParamBound::Verbatim(_val) => { + formatter.write_str("TypeParamBound::Verbatim")?; + formatter.write_str("(`")?; + Display::fmt(_val, formatter)?; + formatter.write_str("`)")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::TypeParen> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeParen"); + formatter.field("elem", Lite(&self.value.elem)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypePath> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypePath"); + if let Some(val) = &self.value.qself { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::QSelf); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("qself", Print::ref_cast(val)); + } + formatter.field("path", Lite(&self.value.path)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypePtr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypePtr"); + if self.value.const_token.is_some() { + formatter.field("const_token", &Present); + } + if self.value.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("elem", Lite(&self.value.elem)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeReference> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeReference"); + if let Some(val) = &self.value.lifetime { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(syn::Lifetime); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("lifetime", Print::ref_cast(val)); + } + if self.value.mutability.is_some() { + formatter.field("mutability", &Present); + } + formatter.field("elem", Lite(&self.value.elem)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeSlice> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeSlice"); + formatter.field("elem", Lite(&self.value.elem)); + formatter.finish() + } +} +impl Debug for Lite<syn::TypeTraitObject> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeTraitObject"); + if self.value.dyn_token.is_some() { + formatter.field("dyn_token", &Present); + } + if !self.value.bounds.is_empty() { + formatter.field("bounds", Lite(&self.value.bounds)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::TypeTuple> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("TypeTuple"); + if !self.value.elems.is_empty() { + formatter.field("elems", Lite(&self.value.elems)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::UnOp> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::UnOp::Deref(_val) => { + formatter.write_str("UnOp::Deref")?; + Ok(()) + } + syn::UnOp::Not(_val) => { + formatter.write_str("UnOp::Not")?; + Ok(()) + } + syn::UnOp::Neg(_val) => { + formatter.write_str("UnOp::Neg")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::UseGlob> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UseGlob"); + formatter.finish() + } +} +impl Debug for Lite<syn::UseGroup> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UseGroup"); + if !self.value.items.is_empty() { + formatter.field("items", Lite(&self.value.items)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::UseName> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UseName"); + formatter.field("ident", Lite(&self.value.ident)); + formatter.finish() + } +} +impl Debug for Lite<syn::UsePath> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UsePath"); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("tree", Lite(&self.value.tree)); + formatter.finish() + } +} +impl Debug for Lite<syn::UseRename> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("UseRename"); + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("rename", Lite(&self.value.rename)); + formatter.finish() + } +} +impl Debug for Lite<syn::UseTree> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::UseTree::Path(_val) => { + formatter.write_str("UseTree::Path")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::UseTree::Name(_val) => { + formatter.write_str("UseTree::Name")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::UseTree::Rename(_val) => { + formatter.write_str("UseTree::Rename")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::UseTree::Glob(_val) => { + formatter.write_str("UseTree::Glob")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::UseTree::Group(_val) => { + formatter.write_str("UseTree::Group")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + } +} +impl Debug for Lite<syn::Variadic> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Variadic"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + if let Some(val) = &self.value.pat { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((Box<syn::Pat>, syn::token::Colon)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.0), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("pat", Print::ref_cast(val)); + } + if self.value.comma.is_some() { + formatter.field("comma", &Present); + } + formatter.finish() + } +} +impl Debug for Lite<syn::Variant> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("Variant"); + if !self.value.attrs.is_empty() { + formatter.field("attrs", Lite(&self.value.attrs)); + } + formatter.field("ident", Lite(&self.value.ident)); + formatter.field("fields", Lite(&self.value.fields)); + if let Some(val) = &self.value.discriminant { + #[derive(RefCast)] + #[repr(transparent)] + struct Print((syn::token::Eq, syn::Expr)); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(Lite(&self.0.1), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + } + formatter.field("discriminant", Print::ref_cast(val)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::VisRestricted> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("VisRestricted"); + if self.value.in_token.is_some() { + formatter.field("in_token", &Present); + } + formatter.field("path", Lite(&self.value.path)); + formatter.finish() + } +} +impl Debug for Lite<syn::Visibility> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::Visibility::Public(_val) => { + formatter.write_str("Visibility::Public")?; + Ok(()) + } + syn::Visibility::Restricted(_val) => { + let mut formatter = formatter.debug_struct("Visibility::Restricted"); + if _val.in_token.is_some() { + formatter.field("in_token", &Present); + } + formatter.field("path", Lite(&_val.path)); + formatter.finish() + } + syn::Visibility::Inherited => formatter.write_str("Visibility::Inherited"), + } + } +} +impl Debug for Lite<syn::WhereClause> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut formatter = formatter.debug_struct("WhereClause"); + if !self.value.predicates.is_empty() { + formatter.field("predicates", Lite(&self.value.predicates)); + } + formatter.finish() + } +} +impl Debug for Lite<syn::WherePredicate> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.value { + syn::WherePredicate::Lifetime(_val) => { + formatter.write_str("WherePredicate::Lifetime")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + syn::WherePredicate::Type(_val) => { + formatter.write_str("WherePredicate::Type")?; + formatter.write_str("(")?; + Debug::fmt(Lite(_val), formatter)?; + formatter.write_str(")")?; + Ok(()) + } + _ => unreachable!(), + } + } +} +impl Debug for Lite<syn::token::Abstract> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![abstract]") + } +} +impl Debug for Lite<syn::token::And> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![&]") + } +} +impl Debug for Lite<syn::token::AndAnd> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![&&]") + } +} +impl Debug for Lite<syn::token::AndEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![&=]") + } +} +impl Debug for Lite<syn::token::As> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![as]") + } +} +impl Debug for Lite<syn::token::Async> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![async]") + } +} +impl Debug for Lite<syn::token::At> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![@]") + } +} +impl Debug for Lite<syn::token::Auto> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![auto]") + } +} +impl Debug for Lite<syn::token::Await> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![await]") + } +} +impl Debug for Lite<syn::token::Become> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![become]") + } +} +impl Debug for Lite<syn::token::Box> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![box]") + } +} +impl Debug for Lite<syn::token::Break> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![break]") + } +} +impl Debug for Lite<syn::token::Caret> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![^]") + } +} +impl Debug for Lite<syn::token::CaretEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![^=]") + } +} +impl Debug for Lite<syn::token::Colon> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![:]") + } +} +impl Debug for Lite<syn::token::Comma> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![,]") + } +} +impl Debug for Lite<syn::token::Const> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![const]") + } +} +impl Debug for Lite<syn::token::Continue> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![continue]") + } +} +impl Debug for Lite<syn::token::Crate> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![crate]") + } +} +impl Debug for Lite<syn::token::Default> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![default]") + } +} +impl Debug for Lite<syn::token::Do> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![do]") + } +} +impl Debug for Lite<syn::token::Dollar> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![$]") + } +} +impl Debug for Lite<syn::token::Dot> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![.]") + } +} +impl Debug for Lite<syn::token::DotDot> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![..]") + } +} +impl Debug for Lite<syn::token::DotDotDot> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![...]") + } +} +impl Debug for Lite<syn::token::DotDotEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![..=]") + } +} +impl Debug for Lite<syn::token::Dyn> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![dyn]") + } +} +impl Debug for Lite<syn::token::Else> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![else]") + } +} +impl Debug for Lite<syn::token::Enum> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![enum]") + } +} +impl Debug for Lite<syn::token::Eq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![=]") + } +} +impl Debug for Lite<syn::token::EqEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![==]") + } +} +impl Debug for Lite<syn::token::Extern> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![extern]") + } +} +impl Debug for Lite<syn::token::FatArrow> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![=>]") + } +} +impl Debug for Lite<syn::token::Final> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![final]") + } +} +impl Debug for Lite<syn::token::Fn> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![fn]") + } +} +impl Debug for Lite<syn::token::For> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![for]") + } +} +impl Debug for Lite<syn::token::Ge> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![>=]") + } +} +impl Debug for Lite<syn::token::Gt> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![>]") + } +} +impl Debug for Lite<syn::token::If> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![if]") + } +} +impl Debug for Lite<syn::token::Impl> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![impl]") + } +} +impl Debug for Lite<syn::token::In> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![in]") + } +} +impl Debug for Lite<syn::token::LArrow> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![<-]") + } +} +impl Debug for Lite<syn::token::Le> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![<=]") + } +} +impl Debug for Lite<syn::token::Let> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![let]") + } +} +impl Debug for Lite<syn::token::Loop> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![loop]") + } +} +impl Debug for Lite<syn::token::Lt> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![<]") + } +} +impl Debug for Lite<syn::token::Macro> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![macro]") + } +} +impl Debug for Lite<syn::token::Match> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![match]") + } +} +impl Debug for Lite<syn::token::Minus> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![-]") + } +} +impl Debug for Lite<syn::token::MinusEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![-=]") + } +} +impl Debug for Lite<syn::token::Mod> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![mod]") + } +} +impl Debug for Lite<syn::token::Move> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![move]") + } +} +impl Debug for Lite<syn::token::Mut> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![mut]") + } +} +impl Debug for Lite<syn::token::Ne> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![!=]") + } +} +impl Debug for Lite<syn::token::Not> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![!]") + } +} +impl Debug for Lite<syn::token::Or> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![|]") + } +} +impl Debug for Lite<syn::token::OrEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![|=]") + } +} +impl Debug for Lite<syn::token::OrOr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![||]") + } +} +impl Debug for Lite<syn::token::Override> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![override]") + } +} +impl Debug for Lite<syn::token::PathSep> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![::]") + } +} +impl Debug for Lite<syn::token::Percent> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![%]") + } +} +impl Debug for Lite<syn::token::PercentEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![%=]") + } +} +impl Debug for Lite<syn::token::Plus> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![+]") + } +} +impl Debug for Lite<syn::token::PlusEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![+=]") + } +} +impl Debug for Lite<syn::token::Pound> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![#]") + } +} +impl Debug for Lite<syn::token::Priv> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![priv]") + } +} +impl Debug for Lite<syn::token::Pub> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![pub]") + } +} +impl Debug for Lite<syn::token::Question> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![?]") + } +} +impl Debug for Lite<syn::token::RArrow> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![->]") + } +} +impl Debug for Lite<syn::token::Raw> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![raw]") + } +} +impl Debug for Lite<syn::token::Ref> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![ref]") + } +} +impl Debug for Lite<syn::token::Return> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![return]") + } +} +impl Debug for Lite<syn::token::SelfType> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![Self]") + } +} +impl Debug for Lite<syn::token::SelfValue> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![self]") + } +} +impl Debug for Lite<syn::token::Semi> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![;]") + } +} +impl Debug for Lite<syn::token::Shl> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![<<]") + } +} +impl Debug for Lite<syn::token::ShlEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![<<=]") + } +} +impl Debug for Lite<syn::token::Shr> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![>>]") + } +} +impl Debug for Lite<syn::token::ShrEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![>>=]") + } +} +impl Debug for Lite<syn::token::Slash> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![/]") + } +} +impl Debug for Lite<syn::token::SlashEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![/=]") + } +} +impl Debug for Lite<syn::token::Star> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![*]") + } +} +impl Debug for Lite<syn::token::StarEq> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![*=]") + } +} +impl Debug for Lite<syn::token::Static> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![static]") + } +} +impl Debug for Lite<syn::token::Struct> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![struct]") + } +} +impl Debug for Lite<syn::token::Super> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![super]") + } +} +impl Debug for Lite<syn::token::Tilde> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![~]") + } +} +impl Debug for Lite<syn::token::Trait> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![trait]") + } +} +impl Debug for Lite<syn::token::Try> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![try]") + } +} +impl Debug for Lite<syn::token::Type> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![type]") + } +} +impl Debug for Lite<syn::token::Typeof> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![typeof]") + } +} +impl Debug for Lite<syn::token::Underscore> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![_]") + } +} +impl Debug for Lite<syn::token::Union> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![union]") + } +} +impl Debug for Lite<syn::token::Unsafe> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![unsafe]") + } +} +impl Debug for Lite<syn::token::Unsized> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![unsized]") + } +} +impl Debug for Lite<syn::token::Use> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![use]") + } +} +impl Debug for Lite<syn::token::Virtual> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![virtual]") + } +} +impl Debug for Lite<syn::token::Where> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![where]") + } +} +impl Debug for Lite<syn::token::While> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![while]") + } +} +impl Debug for Lite<syn::token::Yield> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Token![yield]") + } +} diff --git a/third_party/crates/syn-2.0.119/tests/debug/mod.rs b/third_party/crates/syn-2.0.119/tests/debug/mod.rs new file mode 100644 index 0000000..7ab2b79 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/debug/mod.rs @@ -0,0 +1,147 @@ +#![allow( + clippy::no_effect_underscore_binding, + clippy::too_many_lines, + clippy::used_underscore_binding +)] + +#[rustfmt::skip] +mod gen; + +use proc_macro2::{Ident, Literal, TokenStream}; +use ref_cast::RefCast; +use std::fmt::{self, Debug}; +use std::ops::Deref; +use syn::punctuated::Punctuated; + +#[derive(RefCast)] +#[repr(transparent)] +pub struct Lite<T: ?Sized> { + value: T, +} + +#[allow(non_snake_case)] +pub fn Lite<T: ?Sized>(value: &T) -> &Lite<T> { + Lite::ref_cast(value) +} + +impl<T: ?Sized> Deref for Lite<T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + +impl Debug for Lite<bool> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{}", self.value) + } +} + +impl Debug for Lite<u32> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{}", self.value) + } +} + +impl Debug for Lite<usize> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{}", self.value) + } +} + +impl Debug for Lite<String> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{:?}", self.value) + } +} + +impl Debug for Lite<Ident> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{:?}", self.value.to_string()) + } +} + +impl Debug for Lite<Literal> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "{}", self.value) + } +} + +impl Debug for Lite<TokenStream> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let string = self.value.to_string(); + if string.len() <= 80 { + write!(formatter, "TokenStream(`{}`)", self.value) + } else { + formatter + .debug_tuple("TokenStream") + .field(&format_args!("`{}`", string)) + .finish() + } + } +} + +impl<T> Debug for Lite<&T> +where + Lite<T>: Debug, +{ + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(Lite(self.value), formatter) + } +} + +impl<T> Debug for Lite<Box<T>> +where + Lite<T>: Debug, +{ + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + Debug::fmt(Lite(&*self.value), formatter) + } +} + +impl<T> Debug for Lite<Vec<T>> +where + Lite<T>: Debug, +{ + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter + .debug_list() + .entries(self.value.iter().map(Lite)) + .finish() + } +} + +impl<T, P> Debug for Lite<Punctuated<T, P>> +where + Lite<T>: Debug, + Lite<P>: Debug, +{ + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + let mut list = formatter.debug_list(); + for pair in self.pairs() { + let (node, punct) = pair.into_tuple(); + list.entry(Lite(node)); + list.entries(punct.map(Lite)); + } + list.finish() + } +} + +struct Present; + +impl Debug for Present { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some") + } +} + +struct Option { + present: bool, +} + +impl Debug for Option { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(if self.present { "Some" } else { "None" }) + } +} diff --git a/third_party/crates/syn-2.0.119/tests/macros/mod.rs b/third_party/crates/syn-2.0.119/tests/macros/mod.rs new file mode 100644 index 0000000..9c9a957 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/macros/mod.rs @@ -0,0 +1,7 @@ +macro_rules! errorf { + ($($tt:tt)*) => {{ + use ::std::io::Write; + let stderr = ::std::io::stderr(); + write!(stderr.lock(), $($tt)*).unwrap(); + }}; +} diff --git a/third_party/crates/syn-2.0.119/tests/regression.rs b/third_party/crates/syn-2.0.119/tests/regression.rs new file mode 100644 index 0000000..5c7fcdd --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/regression.rs @@ -0,0 +1,5 @@ +#![allow(clippy::let_underscore_untyped, clippy::uninlined_format_args)] + +mod regression { + automod::dir!("tests/regression"); +} diff --git a/third_party/crates/syn-2.0.119/tests/regression/issue1108.rs b/third_party/crates/syn-2.0.119/tests/regression/issue1108.rs new file mode 100644 index 0000000..11a82ad --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/regression/issue1108.rs @@ -0,0 +1,5 @@ +#[test] +fn issue1108() { + let data = "impl<x<>>::x for"; + let _ = syn::parse_file(data); +} diff --git a/third_party/crates/syn-2.0.119/tests/regression/issue1235.rs b/third_party/crates/syn-2.0.119/tests/regression/issue1235.rs new file mode 100644 index 0000000..8836030 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/regression/issue1235.rs @@ -0,0 +1,32 @@ +use proc_macro2::{Delimiter, Group}; +use quote::quote; + +#[test] +fn main() { + // Okay. Rustc allows top-level `static` with no value syntactically, but + // not semantically. Syn parses as Item::Verbatim. + let tokens = quote! { + pub static FOO: usize; + pub static BAR: usize; + }; + let file = syn::parse2::<syn::File>(tokens).unwrap(); + println!("{:#?}", file); + + // Okay. + let inner = Group::new( + Delimiter::None, + quote!(static FOO: usize = 0; pub static BAR: usize = 0), + ); + let tokens = quote!(pub #inner;); + let file = syn::parse2::<syn::File>(tokens).unwrap(); + println!("{:#?}", file); + + // Formerly parser crash. + let inner = Group::new( + Delimiter::None, + quote!(static FOO: usize; pub static BAR: usize), + ); + let tokens = quote!(pub #inner;); + let file = syn::parse2::<syn::File>(tokens).unwrap(); + println!("{:#?}", file); +} diff --git a/third_party/crates/syn-2.0.119/tests/repo/mod.rs b/third_party/crates/syn-2.0.119/tests/repo/mod.rs new file mode 100644 index 0000000..3c3657a --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/repo/mod.rs @@ -0,0 +1,1023 @@ +#![allow(clippy::manual_assert)] + +mod progress; + +use self::progress::Progress; +use anyhow::Result; +use flate2::read::GzDecoder; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use rayon::ThreadPoolBuilder; +use std::collections::BTreeSet; +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use tar::Archive; +use walkdir::{DirEntry, WalkDir}; + +// nightly-2026-07-13 +const REVISION: &str = "77cf889bc178ddb44d6a1c78e5a820b5abb31d8d"; + +#[rustfmt::skip] +static EXCLUDE_FILES: &[&str] = &[ + // TODO: const traits: `pub const trait Trait {}` + // https://github.com/dtolnay/syn/issues/1887 + "library/core/src/alloc/mod.rs", + "library/core/src/borrow.rs", + "library/core/src/clone.rs", + "library/core/src/cmp.rs", + "library/core/src/cmp/bytewise.rs", + "library/core/src/convert/mod.rs", + "library/core/src/default.rs", + "library/core/src/intrinsics/fallback.rs", + "library/core/src/iter/range.rs", + "library/core/src/iter/traits/accum.rs", + "library/core/src/iter/traits/collect.rs", + "library/core/src/iter/traits/double_ended.rs", + "library/core/src/iter/traits/iterator.rs", + "library/core/src/iter/traits/marker.rs", + "library/core/src/ops/arith.rs", + "library/core/src/ops/bit.rs", + "library/core/src/ops/deref.rs", + "library/core/src/ops/drop.rs", + "library/core/src/ops/function.rs", + "library/core/src/ops/index.rs", + "library/core/src/ops/range.rs", + "library/core/src/ops/try_trait.rs", + "library/core/src/pat.rs", + "library/core/src/slice/mod.rs", + "library/stdarch/crates/core_arch/src/loongarch64/simd.rs", + "src/tools/clippy/tests/ui/assign_ops.rs", + "src/tools/clippy/tests/ui/missing_const_for_fn/const_trait.rs", + "src/tools/clippy/tests/ui/trait_duplication_in_bounds.rs", + "src/tools/rust-analyzer/crates/test-utils/src/minicore.rs", + "src/tools/rustfmt/tests/target/const_trait.rs", + "tests/rustdoc-html/constant/const-effect-param.rs", + "tests/rustdoc-html/constant/const-trait-and-impl-methods.rs", + "tests/rustdoc-html/constant/rfc-2632-const-trait-impl.rs", + "tests/rustdoc-html/inline_cross/auxiliary/const-effect-param.rs", + "tests/rustdoc-json/attrs/stability/const_traits.rs", + "tests/ui/const-generics/const_trait_fn-issue-88433.rs", + "tests/ui/const-generics/issues/issue-88119.rs", + "tests/ui/consts/const-closure-in-trait-impl.rs", + "tests/ui/consts/trait_alias_method_call.rs", + "tests/ui/generic-const-items/const-trait-impl.rs", + "tests/ui/parser/impls-nested-within-fns-semantic-1.rs", + "tests/ui/traits/const-traits/assoc-type-const-bound-usage-0.rs", + "tests/ui/traits/const-traits/assoc-type-const-bound-usage-1.rs", + "tests/ui/traits/const-traits/auxiliary/associated-const-stability.rs", + "tests/ui/traits/const-traits/auxiliary/cross-crate.rs", + "tests/ui/traits/const-traits/auxiliary/minicore.rs", + "tests/ui/traits/const-traits/auxiliary/staged-api.rs", + "tests/ui/traits/const-traits/call-generic-in-impl.rs", + "tests/ui/traits/const-traits/conditionally-const-assoc-fn-in-trait-impl.rs", + "tests/ui/traits/const-traits/conditionally-const-inherent-assoc-const-fn.rs", + "tests/ui/traits/const-traits/conditionally-const-trait-bound-assoc-tys.rs", + "tests/ui/traits/const-traits/const-assoc-bound-in-trait-wc.rs", + "tests/ui/traits/const-traits/const-bound-in-host.rs", + "tests/ui/traits/const-traits/const-closure-inherited-const-condition.rs", + "tests/ui/traits/const-traits/const-closure-trait-method.rs", + "tests/ui/traits/const-traits/const-cond-for-rpitit.rs", + "tests/ui/traits/const-traits/const-impl-inherent-bounds.rs", + "tests/ui/traits/const-traits/const-impl-recovery.rs", + "tests/ui/traits/const-traits/const-impl-trait-not-impl-const-trait.rs", + "tests/ui/traits/const-traits/const-impl-trait.rs", + "tests/ui/traits/const-traits/const-in-closure.rs", + "tests/ui/traits/const-traits/const-via-item-bound.rs", + "tests/ui/traits/const-traits/default-method-body-is-const-with-staged-api.rs", + "tests/ui/traits/const-traits/do-not-const-check-override.rs", + "tests/ui/traits/const-traits/do-not-const-check.rs", + "tests/ui/traits/const-traits/dont-ice-on-const-pred-for-bounds.rs", + "tests/ui/traits/const-traits/dont-observe-host.rs", + "tests/ui/traits/const-traits/dont-prefer-param-env-for-infer-self-ty.rs", + "tests/ui/traits/const-traits/effect-param-infer.rs", + "tests/ui/traits/const-traits/function-pointer-does-not-require-const.rs", + "tests/ui/traits/const-traits/hir-const-check.rs", + "tests/ui/traits/const-traits/impl-with-default-fn-pass.rs", + "tests/ui/traits/const-traits/imply-always-const.rs", + "tests/ui/traits/const-traits/inherent-impl-const-bounds.rs", + "tests/ui/traits/const-traits/issue-100222.rs", + "tests/ui/traits/const-traits/issue-92230-wf-super-trait-env.rs", + "tests/ui/traits/const-traits/item-bound-entailment.rs", + "tests/ui/traits/const-traits/non-const-op-in-closure-in-const.rs", + "tests/ui/traits/const-traits/parse-const-unsafe-trait.rs", + "tests/ui/traits/const-traits/predicate-entailment-passes.rs", + "tests/ui/traits/const-traits/project.rs", + "tests/ui/traits/const-traits/specialization/const-default-const-specialized.rs", + "tests/ui/traits/const-traits/specialization/default-keyword.rs", + "tests/ui/traits/const-traits/specialization/issue-95187-same-trait-bound-different-constness.rs", + "tests/ui/traits/const-traits/specialization/non-const-default-const-specialized.rs", + "tests/ui/traits/const-traits/specialization/pass.rs", + "tests/ui/traits/const-traits/specialization/specialize-on-conditionally-const.rs", + "tests/ui/traits/const-traits/super-traits.rs", + "tests/ui/traits/const-traits/trait-where-clause-run.rs", + "tests/ui/traits/const-traits/trait-where-clause-self-referential.rs", + "tests/ui/traits/const-traits/unconstrained-var-specialization.rs", + "tests/ui/traits/next-solver/canonical/effect-var.rs", + + // TODO: const impls: `const impl Trait for T {}` + // https://github.com/dtolnay/syn/issues/1980 + "compiler/rustc_data_structures/src/unord.rs", + "compiler/rustc_middle/src/middle/codegen_fn_attrs.rs", + "library/alloc/src/alloc.rs", + "library/alloc/src/collections/btree/map.rs", + "library/alloc/src/collections/mod.rs", + "library/alloc/src/raw_vec/mod.rs", + "library/alloc/src/string.rs", + "library/alloc/src/vec/mod.rs", + "library/core/src/any.rs", + "library/core/src/array/drain.rs", + "library/core/src/array/equality.rs", + "library/core/src/array/mod.rs", + "library/core/src/bstr/mod.rs", + "library/core/src/cell.rs", + "library/core/src/cell/lazy.rs", + "library/core/src/cell/once.rs", + "library/core/src/char/convert.rs", + "library/core/src/ffi/c_str.rs", + "library/core/src/ffi/va_list.rs", + "library/core/src/hash/mod.rs", + "library/core/src/hash/sip.rs", + "library/core/src/iter/adapters/cycle.rs", + "library/core/src/iter/adapters/flatten.rs", + "library/core/src/iter/sources/empty.rs", + "library/core/src/marker.rs", + "library/core/src/mem/alignment.rs", + "library/core/src/mem/drop_guard.rs", + "library/core/src/mem/manually_drop.rs", + "library/core/src/net/ip_addr.rs", + "library/core/src/net/socket_addr.rs", + "library/core/src/num/error.rs", + "library/core/src/num/niche_types.rs", + "library/core/src/ops/control_flow.rs", + "library/core/src/option.rs", + "library/core/src/panic/unwind_safe.rs", + "library/core/src/pin.rs", + "library/core/src/pin/unsafe_pinned.rs", + "library/core/src/ptr/non_null.rs", + "library/core/src/ptr/unique.rs", + "library/core/src/range.rs", + "library/core/src/result.rs", + "library/core/src/slice/cmp.rs", + "library/core/src/slice/index.rs", + "library/core/src/str/mod.rs", + "library/core/src/str/traits.rs", + "library/core/src/sync/sync_view.rs", + "library/core/src/task/poll.rs", + "library/core/src/task/wake.rs", + "library/core/src/time.rs", + "library/coretests/tests/array.rs", + "library/coretests/tests/cmp.rs", + "library/coretests/tests/hint.rs", + "library/coretests/tests/manually_drop.rs", + "library/std/src/collections/hash/map.rs", + "library/std/src/collections/hash/set.rs", + "library/std/src/ffi/os_str.rs", + "library/std/src/hash/random.rs", + "library/std/src/path.rs", + "library/std/src/sync/lazy_lock.rs", + "library/std/src/sync/once_lock.rs", + "library/stdarch/crates/core_arch/src/simd.rs", + "src/tools/clippy/tests/ui/derivable_impls.rs", + "src/tools/clippy/tests/ui/derivable_impls_derive_const.rs", + "src/tools/clippy/tests/ui/equatable_if_let_const_cmp.rs", + "src/tools/rustfmt/tests/source/impls.rs", + "src/tools/rustfmt/tests/target/impls.rs", + "tests/ui/consts/const-try.rs", + "tests/ui/traits/const-traits/call-const-trait-method-pass.rs", + "tests/ui/traits/const-traits/call-generic-method-chain.rs", + "tests/ui/traits/const-traits/call-generic-method-dup-bound.rs", + "tests/ui/traits/const-traits/call-generic-method-pass.rs", + "tests/ui/traits/const-traits/const-drop.rs", + "tests/ui/traits/const-traits/const-impl-inherent.rs", + "tests/ui/traits/const-traits/const-unsafe-impl.rs", + "tests/ui/traits/const-traits/const_derives/derive-const-use.rs", + "tests/ui/traits/const-traits/drop-manually-drop.rs", + "tests/ui/traits/const-traits/enforce-deref-on-adjust.rs", + "tests/ui/traits/const-traits/generic-bound.rs", + "tests/ui/traits/const-traits/inherent-impl.rs", + "tests/ui/traits/const-traits/minicore-works.rs", + "tests/ui/traits/const-traits/rustc-impl-const-stability.rs", + "tests/ui/traits/const-traits/trait-default-body-stability.rs", + + // TODO: unsafe binders: `unsafe<'a> &'a T` + // https://github.com/dtolnay/syn/issues/1791 + "src/tools/rustfmt/tests/source/unsafe-binders.rs", + "src/tools/rustfmt/tests/target/unsafe-binders.rs", + "tests/debuginfo/unsafe-binders.rs", + "tests/mir-opt/gvn_on_unsafe_binder.rs", + "tests/rustdoc-html/auxiliary/unsafe-binder-dep.rs", + "tests/rustdoc-html/unsafe-binder.rs", + "tests/ui/unsafe-binders/binder-sized-crit.rs", + "tests/ui/unsafe-binders/cat-projection.rs", + "tests/ui/unsafe-binders/expr.rs", + "tests/ui/unsafe-binders/simple.rs", + "tests/ui/unsafe-binders/unsafe-binders-debuginfo.rs", + + // TODO: unsafe fields: `struct S { unsafe field: T }` + // https://github.com/dtolnay/syn/issues/1792 + "src/tools/clippy/tests/ui/derive.rs", + "src/tools/clippy/tests/ui/doc_unsafe.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/record_field_list.rs", + "src/tools/rustfmt/tests/source/unsafe-field.rs", + "src/tools/rustfmt/tests/target/unsafe-field.rs", + "tests/ui/unsafe-fields/auxiliary/unsafe-fields-crate-dep.rs", + + // TODO: guard patterns: `match expr { (A if f()) | (B if g()) => {} }` + // https://github.com/dtolnay/syn/issues/1793 + "src/tools/rustfmt/tests/target/guard_patterns.rs", + "tests/ui/lint/unused/unused_parens/parens-around-guard-patterns-not-unused.rs", + "tests/ui/pattern/rfc-3637-guard-patterns/liveness-ice-issue-146445.rs", + "tests/ui/pattern/rfc-3637-guard-patterns/only-gather-locals-once.rs", + "tests/ui/reachable/guard_read_for_never.rs", + + // TODO: struct field default: `struct S { field: i32 = 1 }` + // https://github.com/dtolnay/syn/issues/1774 + "compiler/rustc_ast_lowering/src/delegation/generics.rs", + "compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs", + "compiler/rustc_errors/src/markdown/parse.rs", + "compiler/rustc_hir/src/attrs/diagnostic.rs", + "compiler/rustc_hir_analysis/src/hir_wf_check.rs", + "compiler/rustc_middle/src/hir/map.rs", + "compiler/rustc_middle/src/ty/mod.rs", + "compiler/rustc_parse/src/parser/mod.rs", + "compiler/rustc_parse/src/parser/stmt.rs", + "compiler/rustc_privacy/src/lib.rs", + "compiler/rustc_resolve/src/imports.rs", + "compiler/rustc_resolve/src/lib.rs", + "compiler/rustc_session/src/config.rs", + "compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs", + "src/tools/clippy/tests/ui/exhaustive_items.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/record_field_default_values.rs", + "src/tools/rustfmt/tests/source/default-field-values.rs", + "src/tools/rustfmt/tests/target/default-field-values.rs", + "tests/ui/structs/default-field-values/auxiliary/struct_field_default.rs", + "tests/ui/structs/default-field-values/const-trait-default-field-value.rs", + "tests/ui/structs/default-field-values/field-references-param.rs", + "tests/ui/structs/default-field-values/support.rs", + "tests/ui/structs/default-field-values/use-normalized-ty-for-default-struct-value.rs", + + // TODO: final associated functions: `final fn` + // https://github.com/dtolnay/syn/issues/1981 + "library/core/src/io/size_hint.rs", + "src/tools/rustfmt/tests/target/final-kw.rs", + "tests/rustdoc-html/final-trait-method.rs", + "tests/ui/traits/final/dyn-compat.rs", + "tests/ui/traits/final/works.rs", + + // TODO: impl restrictions: `pub impl(self) trait Trait` + // https://github.com/dtolnay/syn/issues/1982 + "library/core/src/convert/num.rs", + "library/core/src/num/nonzero.rs", + "library/core/src/num/traits.rs", + "library/core/src/sync/atomic.rs", + "library/std/src/io/stdio.rs", + "library/std/src/os/darwin/fs.rs", + "library/std/src/os/freebsd/net.rs", + "library/std/src/os/illumos/net.rs", + "library/std/src/os/linux/process.rs", + "library/std/src/os/motor/ffi.rs", + "library/std/src/os/motor/process.rs", + "library/std/src/os/net/linux_ext/addr.rs", + "library/std/src/os/net/linux_ext/socket.rs", + "library/std/src/os/net/linux_ext/tcp.rs", + "library/std/src/os/netbsd/net.rs", + "library/std/src/os/solaris/net.rs", + "library/std/src/os/unix/ffi/os_str.rs", + "library/std/src/os/unix/fs.rs", + "library/std/src/os/unix/io/mod.rs", + "library/std/src/os/unix/process.rs", + "library/std/src/os/windows/ffi.rs", + "library/std/src/os/windows/fs.rs", + "library/std/src/os/windows/process.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/impl_restrictions.rs", + "src/tools/rustfmt/tests/source/impl-restriction.rs", + "src/tools/rustfmt/tests/target/impl-restriction.rs", + "tests/pretty/hir-impl-restriction.rs", + "tests/rustdoc-html/impl/impl-restriction-document-private.rs", + "tests/rustdoc-html/impl/impl-restriction.rs", + "tests/ui/impl-restriction/auxiliary/external-impl-restriction.rs", + + // TODO: return type notation: `where T: Trait<method(): Send>` and `where T::method(..): Send` + // https://github.com/dtolnay/syn/issues/1434 + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/return_type_syntax_in_path.rs", + "src/tools/rustfmt/tests/target/return-type-notation.rs", + "tests/rustdoc-html/jump-to-def/assoc-items-extra.rs", + "tests/rustdoc-html/return-type-notation.rs", + "tests/rustdoc-json/return-type-notation.rs", + "tests/ui/associated-type-bounds/all-generics-lookup.rs", + "tests/ui/associated-type-bounds/implied-from-self-where-clause.rs", + "tests/ui/associated-type-bounds/return-type-notation/basic.rs", + "tests/ui/associated-type-bounds/return-type-notation/higher-ranked-bound-works.rs", + "tests/ui/associated-type-bounds/return-type-notation/namespace-conflict.rs", + "tests/ui/associated-type-bounds/return-type-notation/path-constrained-in-method.rs", + "tests/ui/associated-type-bounds/return-type-notation/path-self-qself.rs", + "tests/ui/associated-type-bounds/return-type-notation/path-works.rs", + "tests/ui/associated-type-bounds/return-type-notation/trait-alias.rs", + "tests/ui/associated-type-bounds/return-type-notation/unpretty-parenthesized.rs", + "tests/ui/async-await/return-type-notation/issue-110963-late.rs", + "tests/ui/async-await/return-type-notation/normalizing-self-auto-trait-issue-109924.rs", + "tests/ui/async-await/return-type-notation/rtn-implied-in-supertrait.rs", + "tests/ui/async-await/return-type-notation/super-method-bound.rs", + "tests/ui/async-await/return-type-notation/supertrait-bound.rs", + "tests/ui/borrowck/alias-liveness/rtn-static.rs", + "tests/ui/feature-gates/feature-gate-return_type_notation.rs", + + // TODO: lazy type alias syntax with where-clause in trailing position + // https://github.com/dtolnay/syn/issues/1525 + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/type_item_where_clause.rs", + "src/tools/rustfmt/tests/source/type-alias-where-clauses-with-comments.rs", + "src/tools/rustfmt/tests/source/type-alias-where-clauses.rs", + "src/tools/rustfmt/tests/target/type-alias-where-clauses-with-comments.rs", + "src/tools/rustfmt/tests/target/type-alias-where-clauses.rs", + "tests/rustdoc-html/typedef-inner-variants-lazy_type_alias.rs", + "tests/ui/traits/next-solver/normalize/normalize-self-type-constrains-trait-args.rs", + + // TODO: gen blocks and functions + // https://github.com/dtolnay/syn/issues/1526 + "compiler/rustc_codegen_cranelift/example/gen_block_iterate.rs", + "compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs", + "compiler/rustc_metadata/src/rmeta/decoder.rs", + "compiler/rustc_middle/src/ty/closure.rs", + "compiler/rustc_middle/src/ty/context.rs", + "src/tools/clippy/tests/ui/infinite_loops.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/gen_blocks.rs", + "tests/ui/async-await/async-drop/assign-incompatible-types.rs", + "tests/ui/coroutine/async-gen-deduce-yield.rs", + "tests/ui/coroutine/async-gen-yield-ty-is-unit.rs", + "tests/ui/coroutine/async_gen_fn_iter.rs", + "tests/ui/coroutine/gen_block_is_fused_iter.rs", + "tests/ui/coroutine/gen_block_is_iter.rs", + "tests/ui/coroutine/gen_block_iterate.rs", + "tests/ui/coroutine/gen_fn_iter.rs", + "tests/ui/coroutine/gen_fn_lifetime_capture.rs", + "tests/ui/coroutine/other-attribute-on-gen.rs", + "tests/ui/coroutine/return-types-diverge.rs", + "tests/ui/higher-ranked/builtin-closure-like-bounds.rs", + "tests/ui/sanitizer/cfi/coroutine.rs", + + // TODO: postfix yield + // https://github.com/dtolnay/syn/issues/1890 + "tests/pretty/postfix-yield.rs", + "tests/ui/coroutine/postfix-yield.rs", + + // TODO: `!` as a pattern + // https://github.com/dtolnay/syn/issues/1546 + "tests/mir-opt/building/match/never_patterns.rs", + "tests/pretty/never-pattern.rs", + "tests/ui/reachable/never-pattern-closure-param-array.rs", + "tests/ui/rfcs/rfc-0000-never_patterns/always-read-in-closure-capture.rs", + "tests/ui/rfcs/rfc-0000-never_patterns/diverges.rs", + "tests/ui/rfcs/rfc-0000-never_patterns/use-bindings.rs", + + // TODO: async trait bounds: `impl async Fn()` + // https://github.com/dtolnay/syn/issues/1628 + "src/tools/miri/tests/pass/async-closure-captures.rs", + "src/tools/miri/tests/pass/async-closure-drop.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/async_trait_bound.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/for_binder_bound.rs", + "src/tools/rustfmt/tests/target/asyncness.rs", + "tests/coverage/async_closure.rs", + "tests/ui/async-await/async-closures/async-fn-mut-for-async-fn.rs", + "tests/ui/async-await/async-closures/async-fn-once-for-async-fn.rs", + "tests/ui/async-await/async-closures/auxiliary/foreign.rs", + "tests/ui/async-await/async-closures/body-check-on-non-fnmut.rs", + "tests/ui/async-await/async-closures/box-deref-in-debuginfo.rs", + "tests/ui/async-await/async-closures/brand.rs", + "tests/ui/async-await/async-closures/captures.rs", + "tests/ui/async-await/async-closures/clone-closure.rs", + "tests/ui/async-await/async-closures/constrained-but-no-upvars-yet.rs", + "tests/ui/async-await/async-closures/debuginfo-by-move-body.rs", + "tests/ui/async-await/async-closures/drop.rs", + "tests/ui/async-await/async-closures/force-move-due-to-inferred-kind.rs", + "tests/ui/async-await/async-closures/foreign.rs", + "tests/ui/async-await/async-closures/inline-body.rs", + "tests/ui/async-await/async-closures/mangle.rs", + "tests/ui/async-await/async-closures/moro-example.rs", + "tests/ui/async-await/async-closures/move-is-async-fn.rs", + "tests/ui/async-await/async-closures/mut-ref-reborrow.rs", + "tests/ui/async-await/async-closures/no-borrow-from-env.rs", + "tests/ui/async-await/async-closures/non-copy-arg-does-not-force-inner-move.rs", + "tests/ui/async-await/async-closures/overlapping-projs.rs", + "tests/ui/async-await/async-closures/precise-captures.rs", + "tests/ui/async-await/async-closures/refd.rs", + "tests/ui/async-await/async-closures/signature-deduction.rs", + "tests/ui/async-await/async-fn/edition-2015-not-async-bound.rs", + "tests/ui/async-await/async-fn/higher-ranked-async-fn.rs", + "tests/ui/async-await/async-fn/impl-trait.rs", + "tests/ui/async-await/async-fn/project.rs", + "tests/ui/async-await/async-fn/sugar.rs", + + // TODO: mutable by-reference bindings (mut ref) + // https://github.com/dtolnay/syn/issues/1629 + "src/tools/rustfmt/tests/source/mut_ref.rs", + "src/tools/rustfmt/tests/target/mut_ref.rs", + "tests/ui/mut/mut-ref.rs", + + // TODO: postfix match + // https://github.com/dtolnay/syn/issues/1630 + "src/tools/clippy/tests/ui/unnecessary_semicolon.rs", + "src/tools/rustfmt/tests/source/postfix-match/pf-match.rs", + "src/tools/rustfmt/tests/target/postfix-match/pf-match.rs", + "tests/pretty/postfix-match/simple-matches.rs", + "tests/ui/match/postfix-match/no-unused-parens.rs", + "tests/ui/match/postfix-match/pf-match-chain.rs", + "tests/ui/match/postfix-match/postfix-match.rs", + + // TODO: delegation: `reuse Trait::bar { Box::new(self.0) }` + // https://github.com/dtolnay/syn/issues/1580 + "src/tools/rustfmt/tests/target/issue_6513.rs", + "tests/incremental/delegation-ice-155729.rs", + "tests/pretty/delegation/delegation.rs", + "tests/pretty/delegation/generics.rs", + "tests/pretty/delegation/impl-reuse.rs", + "tests/pretty/delegation/inherit-attributes.rs", + "tests/pretty/delegation/inline-attribute.rs", + "tests/pretty/delegation/self-mapping-output.rs", + "tests/pretty/delegation/self-rename.rs", + "tests/pretty/hir-delegation.rs", + "tests/rustdoc-html/async/async-fn-delegation.rs", + "tests/ui/delegation/auxiliary/recursive-delegation-aux.rs", + "tests/ui/delegation/body-identity-glob.rs", + "tests/ui/delegation/body-identity-list.rs", + "tests/ui/delegation/explicit-paths-in-traits-pass.rs", + "tests/ui/delegation/explicit-paths-pass.rs", + "tests/ui/delegation/explicit-paths-signature-pass.rs", + "tests/ui/delegation/fn-header.rs", + "tests/ui/delegation/generics/const-type-ice-154334.rs", + "tests/ui/delegation/generics/free-fn-to-free-fn-pass.rs", + "tests/ui/delegation/generics/free-fn-to-trait-method-pass.rs", + "tests/ui/delegation/generics/generics-aux-pass.rs", + "tests/ui/delegation/generics/impl-to-free-fn-pass.rs", + "tests/ui/delegation/generics/impl-trait-to-trait-method-pass.rs", + "tests/ui/delegation/generics/inherent-impl-to-trait-method-pass.rs", + "tests/ui/delegation/generics/mapping/free-to-free-pass.rs", + "tests/ui/delegation/generics/mapping/free-to-trait-pass.rs", + "tests/ui/delegation/generics/mapping/impl-trait-to-free-pass.rs", + "tests/ui/delegation/generics/mapping/impl-trait-to-trait-pass.rs", + "tests/ui/delegation/generics/mapping/inherent-impl-to-free-pass.rs", + "tests/ui/delegation/generics/mapping/inherent-impl-to-trait-pass.rs", + "tests/ui/delegation/generics/mapping/trait-to-free-pass.rs", + "tests/ui/delegation/generics/mapping/trait-to-trait-pass.rs", + "tests/ui/delegation/generics/self-rename.rs", + "tests/ui/delegation/generics/stability-implications-non-local-defid.rs", + "tests/ui/delegation/generics/synth-params-ice-154780.rs", + "tests/ui/delegation/generics/trait-method-to-other-pass.rs", + "tests/ui/delegation/generics/unelided-lifetime-in-sig-ice-156848.rs", + "tests/ui/delegation/glob-glob.rs", + "tests/ui/delegation/glob-override.rs", + "tests/ui/delegation/glob.rs", + "tests/ui/delegation/impl-reuse-pass.rs", + "tests/ui/delegation/impl-trait.rs", + "tests/ui/delegation/list.rs", + "tests/ui/delegation/macro-inside-glob.rs", + "tests/ui/delegation/macro-inside-list.rs", + "tests/ui/delegation/method-call-priority.rs", + "tests/ui/delegation/parse.rs", + "tests/ui/delegation/recursive-delegation-pass.rs", + "tests/ui/delegation/rename.rs", + "tests/ui/delegation/self-coercion.rs", + "tests/ui/delegation/self-mapping-arguments.rs", + "tests/ui/delegation/target-expression-removal-pass.rs", + + // TODO: for await + // https://github.com/dtolnay/syn/issues/1631 + "tests/ui/async-await/for-await-2015.rs", + "tests/ui/async-await/for-await-passthrough.rs", + "tests/ui/async-await/for-await.rs", + + // TODO: unparenthesized half-open range pattern inside slice pattern: `[1..]` + // https://github.com/dtolnay/syn/issues/1769 + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/range_pat.rs", + "tests/ui/consts/miri_unleashed/const_refers_to_static_cross_crate.rs", + + // TODO: pinned type sugar: `&pin const Self` + // https://github.com/dtolnay/syn/issues/1770 + "src/tools/rustfmt/tests/source/pin_sugar.rs", + "src/tools/rustfmt/tests/target/pin_sugar.rs", + "tests/pretty/pin-ergonomics-hir.rs", + "tests/pretty/pin-ergonomics.rs", + "tests/ui/pin-ergonomics/borrow.rs", + "tests/ui/pin-ergonomics/pin-coercion.rs", + "tests/ui/pin-ergonomics/pinned-drop-sugar-no-core.rs", + "tests/ui/pin-ergonomics/pinned-drop-sugar.rs", + "tests/ui/pin-ergonomics/pinned-drop.rs", + "tests/ui/pin-ergonomics/sugar-self.rs", + "tests/ui/pin-ergonomics/sugar.rs", + "tests/ui/pin-ergonomics/user-type-projection.rs", + + // TODO: attributes on where-predicates + // https://github.com/dtolnay/syn/issues/1705 + "src/tools/rustfmt/tests/target/cfg_attribute_in_where.rs", + "tests/ui/where-clauses/cfg-attr-issue-138010-1.rs", + + // TODO: super let + // https://github.com/dtolnay/syn/issues/1889 + "library/std/src/macros/tests.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/let_stmt.rs", + "tests/ui/drop/if-let-super-let.rs", + "tests/ui/drop/super-let-tail-expr-drop-order.rs", + + // TODO: "ergonomic clones": `f(obj.use)`, `thread::spawn(use || f(obj))`, `async use` + // https://github.com/dtolnay/syn/issues/1802 + "src/tools/clippy/tests/ui/redundant_closure_call_early.rs", + "src/tools/clippy/tests/ui/redundant_closure_call_fixable.rs", + "src/tools/rustfmt/tests/source/ergonomic_clones.rs", + "src/tools/rustfmt/tests/target/ergonomic_clones.rs", + "tests/codegen-llvm/ergonomic-clones/closure.rs", + "tests/mir-opt/ergonomic-clones/closure.rs", + "tests/ui/ergonomic-clones/async/basic.rs", + "tests/ui/ergonomic-clones/closure/basic.rs", + "tests/ui/ergonomic-clones/closure/const-closure.rs", + "tests/ui/ergonomic-clones/closure/mutation.rs", + "tests/ui/ergonomic-clones/closure/nested.rs", + "tests/ui/ergonomic-clones/closure/once-move-out-on-heap.rs", + "tests/ui/ergonomic-clones/closure/with-binders.rs", + "tests/ui/ergonomic-clones/dotuse/basic.rs", + "tests/ui/ergonomic-clones/dotuse/block.rs", + + // TODO: move expression + // https://github.com/dtolnay/syn/issues/1983 + "tests/ui/move-expr/borrow-only.rs", + "tests/ui/move-expr/copy-type.rs", + "tests/ui/move-expr/name-resolution.rs", + "tests/ui/move-expr/nested-closures.rs", + "tests/ui/move-expr/nested-move-expr.rs", + "tests/ui/move-expr/parse-ambiguity.rs", + "tests/ui/move-expr/plain-closure.rs", + + // TODO: contracts + // https://github.com/dtolnay/syn/issues/1892 + "tests/ui/contracts/internal_machinery/contract-ast-extensions-nest.rs", + "tests/ui/contracts/internal_machinery/contract-ast-extensions-tail.rs", + "tests/ui/contracts/internal_machinery/contracts-lowering-ensures-is-not-inherited-when-nesting.rs", + "tests/ui/contracts/internal_machinery/contracts-lowering-requires-is-not-inherited-when-nesting.rs", + + // TODO: frontmatter + // https://github.com/dtolnay/syn/issues/1893 + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/frontmatter.rs", + "src/tools/rustfmt/tests/source/frontmatter_compact.rs", + "src/tools/rustfmt/tests/source/frontmatter_escaped.rs", + "src/tools/rustfmt/tests/source/frontmatter_spaced.rs", + "src/tools/rustfmt/tests/target/frontmatter_compact.rs", + "src/tools/rustfmt/tests/target/frontmatter_escaped.rs", + "src/tools/rustfmt/tests/target/frontmatter_spaced.rs", + "tests/rustdoc-html/source-code-pages/frontmatter.rs", + "tests/ui/frontmatter/auxiliary/expr.rs", + "tests/ui/frontmatter/auxiliary/lib.rs", + "tests/ui/frontmatter/content-contains-whitespace.rs", + "tests/ui/frontmatter/content-non-lexible-tokens.rs", + "tests/ui/frontmatter/escape-hyphens-leading.rs", + "tests/ui/frontmatter/escape-hyphens-nonleading-1.rs", + "tests/ui/frontmatter/escape-hyphens-nonleading-2.rs", + "tests/ui/frontmatter/escape-hyphens-nonleading-3.rs", + "tests/ui/frontmatter/fence-whitespace-trailing-1.rs", + "tests/ui/frontmatter/fence-whitespace-trailing-2.rs", + "tests/ui/frontmatter/frontmatter-crlf.rs", + "tests/ui/frontmatter/infostring-dot-nonleading.rs", + "tests/ui/frontmatter/infostring-hyphen-nonleading.rs", + "tests/ui/frontmatter/location-after-shebang.rs", + "tests/ui/include-macros/auxiliary/shebang-expr.rs", + "tests/ui/unpretty/frontmatter.rs", + + // TODO: `type const` associated items + // https://github.com/dtolnay/syn/issues/1984 + "src/tools/clippy/tests/ui/crashes/mgca-16691.rs", + "src/tools/clippy/tests/ui/trait_duplication_in_bounds_assoc_const_eq.rs", + "src/tools/rustfmt/tests/source/direct_const_arg.rs", + "src/tools/rustfmt/tests/target/direct_const_arg.rs", + "tests/debuginfo/associated-const-bindings.rs", + "tests/rustdoc-html/constant/ice-associated-const-equality-105952.rs", + "tests/rustdoc-html/inline_cross/auxiliary/assoc-const-equality.rs", + "tests/rustdoc-html/type-const-associated-const-no-body.rs", + "tests/rustdoc-html/type-const-free-in-array.rs", + "tests/rustdoc-html/type-const-inherent-with-body.rs", + "tests/ui/associated-consts/issue-110933.rs", + "tests/ui/associated-consts/type-const-in-array-len.rs", + "tests/ui/associated-type-bounds/duplicate-bound.rs", + "tests/ui/const-generics/associated-const-bindings/assoc-const.rs", + "tests/ui/const-generics/associated-const-bindings/bound-var-in-ty.rs", + "tests/ui/const-generics/associated-const-bindings/coexisting-with-type-binding.rs", + "tests/ui/const-generics/associated-const-bindings/const_evaluatable_unchecked.rs", + "tests/ui/const-generics/associated-const-bindings/dyn-compat-basic.rs", + "tests/ui/const-generics/associated-const-bindings/dyn-compat-self-const-projections-in-methods.rs", + "tests/ui/const-generics/associated-const-bindings/dyn-const-projection-escaping-bound-vars.rs", + "tests/ui/const-generics/associated-const-bindings/equality-unused-issue-126729.rs", + "tests/ui/const-generics/associated-const-bindings/equality_bound_with_infer.rs", + "tests/ui/const-generics/associated-const-bindings/normalization-via-param-env.rs", + "tests/ui/const-generics/associated-const-bindings/supertraits.rs", + "tests/ui/const-generics/gca/basic-different-definitions.rs", + "tests/ui/const-generics/gca/basic.rs", + "tests/ui/const-generics/generic_const_exprs/auxiliary/non_local_type_const.rs", + "tests/ui/const-generics/mgca/adt_expr_infers_from_value.rs", + "tests/ui/const-generics/mgca/array-expr-with-assoc-const.rs", + "tests/ui/const-generics/mgca/assoc-const-projection-in-bound.rs", + "tests/ui/const-generics/mgca/assoc-const.rs", + "tests/ui/const-generics/mgca/concrete-expr-with-generics-in-env.rs", + "tests/ui/const-generics/mgca/explicit_anon_consts_literals_hack.rs", + "tests/ui/const-generics/mgca/generic-args-on-enum-variant-segments.rs", + "tests/ui/const-generics/mgca/multi_braced_direct_const_args.rs", + "tests/ui/const-generics/mgca/tuple_ctor_arg_simple.rs", + "tests/ui/const-generics/mgca/tuple_expr_arg_simple.rs", + "tests/ui/const-generics/mgca/type-const-ctor-148953.rs", + "tests/ui/const-generics/mgca/type-const-used-in-trait.rs", + "tests/ui/const-generics/mgca/type_const-array-return.rs", + "tests/ui/const-generics/mgca/type_const-incemental-compile.rs", + "tests/ui/const-generics/mgca/type_const-pub.rs", + "tests/ui/const-generics/mgca/type_const-use.rs", + "tests/ui/const-generics/mgca/type_const_in_pattern.rs", + "tests/ui/generic-const-items/assoc-const-bindings.rs", + "tests/ui/generic-const-items/type-const-nested-assoc-const.rs", + "tests/ui/sanitizer/cfi/assoc-const-projection-issue-151878.rs", + "tests/ui/supertrait-shadowing/assoc-const.rs", + "tests/ui/supertrait-shadowing/out-of-scope.rs", + "tests/ui/supertrait-shadowing/type-dependent.rs", + + // TODO: const block in path argument: `T: Trait<const { ... }>` + // https://github.com/dtolnay/syn/issues/1985 + "tests/ui/const-generics/mgca/higher-ranked-lts-good.rs", + "tests/ui/traits/associated_type_bound/generic-const-args-default.rs", + + // TODO: item-level const blocks + // https://github.com/dtolnay/syn/issues/1986 + "src/tools/rustfmt/tests/source/const-block-items.rs", + "src/tools/rustfmt/tests/target/const-block-items.rs", + "tests/ui/consts/const-block-items/assert-pass.rs", + "tests/ui/consts/const-block-items/hir.rs", + "tests/ui/parser/const-block-items/attrs.rs", + "tests/ui/parser/const-block-items/mod-in-fn.rs", + "tests/ui/parser/const-block-items/pub.rs", + + // TODO: mut-restricted fields + // https://github.com/dtolnay/syn/issues/1987 + "src/tools/rustfmt/tests/source/mut-restriction.rs", + "src/tools/rustfmt/tests/target/mut-restriction.rs", + "tests/pretty/hir-mut-restriction.rs", + "tests/ui/mut-restriction/auxiliary/external-mut-restriction.rs", + + // TODO: heterogeneous try-blocks: `try bikeshed Option<_>` + // https://github.com/dtolnay/syn/issues/1988 + "src/tools/rustfmt/tests/source/try_blocks_heterogeneous.rs", + "src/tools/rustfmt/tests/target/try_blocks_heterogeneous.rs", + "tests/pretty/try-blocks.rs", + "tests/ui/try-block/try-block-heterogeneous.rs", + + // TODO: `|| .. .method()` + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/closure_range_method_call.rs", + "src/tools/rustfmt/tests/source/issue-4808.rs", + + // Negative inherent impl: `impl !Box<JoinHandle> {}` + "src/tools/rustfmt/tests/source/negative-impl.rs", + "src/tools/rustfmt/tests/target/negative-impl.rs", + + // Compile-fail expr parameter in const generic position: `f::<1 + 2>()` + "tests/ui/const-generics/early/closing-args-token.rs", + "tests/ui/const-generics/early/const-expression-parameter.rs", + + // Compile-fail variadics in not the last position of a function parameter list + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/fn_def_param.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/param_list_vararg.rs", + "tests/ui/parser/variadic-ffi-syntactic-pass.rs", + + // Need at least one trait in impl Trait, no such type as impl 'static + "tests/ui/type-alias-impl-trait/generic_type_does_not_live_long_enough.rs", + + // Negative polarity trait bound: `where T: !Copy` + "src/tools/rustfmt/tests/target/negative-bounds.rs", + "tests/ui/traits/negative-bounds/supertrait.rs", + + // Const impl that is not a trait impl: `impl ~const T {}` + "tests/ui/traits/const-traits/syntax.rs", + + // Lifetimes and types out of order in angle bracketed path arguments + "tests/ui/parser/constraints-before-generic-args-syntactic-pass.rs", + + // Deprecated anonymous parameter syntax in traits + "src/tools/rustfmt/tests/source/trait.rs", + "src/tools/rustfmt/tests/target/trait.rs", + "tests/pretty/hir-fn-params.rs", + "tests/rustdoc-html/anon-fn-params.rs", + "tests/rustdoc-html/auxiliary/ext-anon-fn-params.rs", + "tests/ui/anon-params/anon-params-trait-method-multiple.rs", + "tests/ui/fn/anonymous-parameters-trait-13105.rs", + "tests/ui/proc-macro/trait-fn-args-2015.rs", + "tests/ui/trait-bounds/anonymous-parameters-13775.rs", + + // Deprecated where-clause location + "src/tools/rustfmt/tests/source/issue_4257.rs", + "src/tools/rustfmt/tests/source/issue_4911.rs", + "src/tools/rustfmt/tests/target/issue_4257.rs", + "src/tools/rustfmt/tests/target/issue_4911.rs", + "tests/pretty/gat-bounds.rs", + + // Deprecated trait object syntax with parenthesized generic arguments and no dyn keyword + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/path_fn_trait_args.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/typepathfn_with_coloncolon.rs", + "src/tools/rustfmt/tests/source/attrib.rs", + "src/tools/rustfmt/tests/source/closure.rs", + "src/tools/rustfmt/tests/source/existential_type.rs", + "src/tools/rustfmt/tests/source/fn-simple.rs", + "src/tools/rustfmt/tests/source/fn_args_layout-vertical.rs", + "src/tools/rustfmt/tests/source/issue-4689/one.rs", + "src/tools/rustfmt/tests/source/issue-4689/two.rs", + "src/tools/rustfmt/tests/source/paths.rs", + "src/tools/rustfmt/tests/source/structs.rs", + "src/tools/rustfmt/tests/target/attrib.rs", + "src/tools/rustfmt/tests/target/closure.rs", + "src/tools/rustfmt/tests/target/existential_type.rs", + "src/tools/rustfmt/tests/target/fn-simple.rs", + "src/tools/rustfmt/tests/target/fn.rs", + "src/tools/rustfmt/tests/target/fn_args_layout-vertical.rs", + "src/tools/rustfmt/tests/target/issue-4689/one.rs", + "src/tools/rustfmt/tests/target/issue-4689/two.rs", + "src/tools/rustfmt/tests/target/paths.rs", + "src/tools/rustfmt/tests/target/structs.rs", + "tests/codegen-units/item-collection/non-generic-closures.rs", + "tests/debuginfo/recursive-enum.rs", + "tests/pretty/closure-reform-pretty.rs", + "tests/run-make/reproducible-build-2/reproducible-build.rs", + "tests/run-make/reproducible-build/reproducible-build.rs", + "tests/ui/impl-trait/generic-with-implicit-hrtb-without-dyn.rs", + "tests/ui/lifetimes/auxiliary/lifetime_bound_will_change_warning_lib.rs", + "tests/ui/lifetimes/bare-trait-object-borrowck.rs", + "tests/ui/lifetimes/bare-trait-object.rs", + "tests/ui/parser/bounds-obj-parens.rs", + + // Various extensions to Rust syntax made up by rust-analyzer + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/asm_piece_attr.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/assoc_type_bound.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/const_param_default_path.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/deref_pat.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/field_expr.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/generic_arg_bounds.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/global_asm.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/include_bytes.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/not_null_pat.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/pattern_type.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/record_mut_restrictions_after.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/record_mut_restrictions_before.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/ref_expr.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/try_block_expr.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/tuple_mut_restrictions.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/type_const.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/use_tree_abs_star.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0015_use_tree.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0019_enums.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0029_range_forms.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0051_parameter_attrs.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0055_dot_dot_dot.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0068_item_modifiers.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0031_block_inner_attrs.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0038_endless_inclusive_range.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0045_ambiguous_trait_object.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0046_mutable_const_item.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0224_dangling_dyn.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0261_dangling_impl_undeclared_lifetime.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/dangling_impl.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/dangling_impl_reference.rs", + "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/impl_trait_lifetime_only.rs", + + // Placeholder syntax for "throw expressions" + "compiler/rustc_expand/src/module.rs", + "compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs", + "src/tools/clippy/tests/ui/needless_return.rs", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/yeet_expr.rs", + "tests/pretty/yeet-expr.rs", + "tests/ui/contracts/contracts-ensures-early-fn-exit.rs", + "tests/ui/try-trait/yeet-for-option.rs", + "tests/ui/try-trait/yeet-for-result.rs", + + // Edition 2015 code using identifiers that are now keywords + // TODO: some of these we should probably parse + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/dyn_trait_type_weak.rs", + "src/tools/rustfmt/tests/source/configs/indent_style/block_call.rs", + "src/tools/rustfmt/tests/source/configs/use_try_shorthand/false.rs", + "src/tools/rustfmt/tests/source/configs/use_try_shorthand/true.rs", + "src/tools/rustfmt/tests/source/issue_1306.rs", + "src/tools/rustfmt/tests/source/try-conversion.rs", + "src/tools/rustfmt/tests/target/configs/indent_style/block_call.rs", + "src/tools/rustfmt/tests/target/configs/use_try_shorthand/false.rs", + "src/tools/rustfmt/tests/target/issue-1681.rs", + "src/tools/rustfmt/tests/target/issue_1306.rs", + "tests/ui/dyn-keyword/dyn-2015-no-warnings-without-lints.rs", + "tests/ui/editions/edition-keywords-2015-2015.rs", + "tests/ui/editions/edition-keywords-2015-2018.rs", + "tests/ui/lint/keyword-idents/auxiliary/multi_file_submod.rs", + "tests/ui/lint/lint_pre_expansion_extern_module_aux.rs", + "tests/ui/macros/macro-comma-support-rpass.rs", + "tests/ui/macros/try-macro.rs", + "tests/ui/parser/extern-crate-async.rs", + "tests/ui/try-block/try-is-identifier-edition2015.rs", + + // Excessive nesting + "tests/ui/expr/if/expr-stack-overflow.rs", + + // Testing tools on invalid syntax + "src/tools/clippy/tests/ui/non_expressive_names_error_recovery.rs", + "src/tools/rustfmt/tests/coverage/target/comments.rs", + "src/tools/rustfmt/tests/parser/issue-4126/invalid.rs", + "src/tools/rustfmt/tests/parser/issue_4418.rs", + "src/tools/rustfmt/tests/parser/stashed-diag.rs", + "src/tools/rustfmt/tests/parser/stashed-diag2.rs", + "src/tools/rustfmt/tests/parser/unclosed-delims/issue_4466.rs", + "src/tools/rustfmt/tests/source/configs/disable_all_formatting/true.rs", + "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/false.rs", + "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/true.rs", + "src/tools/rustfmt/tests/source/type.rs", + "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/false.rs", + "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/true.rs", + "src/tools/rustfmt/tests/target/type.rs", + "src/tools/rustfmt/tests/target/unsafe_extern_blocks.rs", + "tests/ui/generics/issue-94432-garbage-ice.rs", + + // Not actually test cases + "tests/ui/lint/expansion-time-include.rs", + "tests/ui/macros/auxiliary/macro-comma-support.rs", + "tests/ui/macros/auxiliary/macro-include-items-expr.rs", + "tests/ui/macros/include-single-expr-helper.rs", + "tests/ui/macros/include-single-expr-helper-1.rs", + "tests/ui/parser/issues/auxiliary/issue-21146-inc.rs", +]; + +#[rustfmt::skip] +static EXCLUDE_DIRS: &[&str] = &[ + // Inputs that intentionally do not parse + "src/tools/rust-analyzer/crates/parser/test_data/parser/err", + "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err", + + // Inputs that lex but do not necessarily parse + "src/tools/rust-analyzer/crates/parser/test_data/lexer", + + // Inputs that used to crash rust-analyzer, but aren't necessarily supposed to parse + "src/tools/rust-analyzer/crates/syntax/test_data/parser/fuzz-failures", + "src/tools/rust-analyzer/crates/syntax/test_data/reparse/fuzz-failures", + + // Inputs that crash rustc, making no claim about whether they are valid Rust + "tests/crashes", +]; + +// Directories in which a .stderr implies the corresponding .rs is not expected +// to work. +static UI_TEST_DIRS: &[&str] = &["tests/ui", "tests/rustdoc-ui"]; + +pub fn for_each_rust_file(for_each: impl Fn(&Path) + Sync + Send) { + let mut rs_files = BTreeSet::new(); + + let repo_dir = Path::new("tests/rust"); + for entry in WalkDir::new(repo_dir) + .into_iter() + .filter_entry(base_dir_filter) + { + let entry = entry.unwrap(); + if !entry.file_type().is_dir() { + rs_files.insert(entry.into_path()); + } + } + + for ui_test_dir in UI_TEST_DIRS { + for entry in WalkDir::new(repo_dir.join(ui_test_dir)) { + let mut path = entry.unwrap().into_path(); + if path.extension() == Some(OsStr::new("stderr")) { + loop { + rs_files.remove(&path.with_extension("rs")); + path = path.with_extension(""); + if path.extension().is_none() { + break; + } + } + } + } + } + + rs_files.par_iter().map(PathBuf::as_path).for_each(for_each); +} + +pub fn base_dir_filter(entry: &DirEntry) -> bool { + let path = entry.path(); + + let mut path_string = path.to_string_lossy(); + if cfg!(windows) { + path_string = path_string.replace('\\', "/").into(); + } + let path_string = if path_string == "tests/rust" { + return true; + } else if let Some(path) = path_string.strip_prefix("tests/rust/") { + path + } else { + panic!("unexpected path in Rust dist: {}", path_string); + }; + + if path.is_dir() { + return !EXCLUDE_DIRS.contains(&path_string); + } + + if path.extension() != Some(OsStr::new("rs")) { + return false; + } + + !EXCLUDE_FILES.contains(&path_string) +} + +#[allow(dead_code)] +pub fn edition(path: &Path) -> &'static str { + if path.ends_with("dyn-2015-no-warnings-without-lints.rs") { + "2015" + } else { + "2021" + } +} + +#[allow(dead_code)] +pub fn abort_after() -> usize { + match env::var("ABORT_AFTER_FAILURE") { + Ok(s) => s.parse().expect("failed to parse ABORT_AFTER_FAILURE"), + Err(_) => usize::MAX, + } +} + +pub fn rayon_init() { + let stack_size = match env::var("RUST_MIN_STACK") { + Ok(s) => s.parse().expect("failed to parse RUST_MIN_STACK"), + Err(_) => 1024 * 1024 * if cfg!(debug_assertions) { 40 } else { 20 }, + }; + ThreadPoolBuilder::new() + .stack_size(stack_size) + .build_global() + .unwrap(); +} + +pub fn clone_rust() { + let needs_clone = match fs::read_to_string("tests/rust/COMMIT") { + Err(_) => true, + Ok(contents) => contents.trim() != REVISION, + }; + if needs_clone { + download_and_unpack().unwrap(); + } + + let mut missing = String::new(); + let test_src = Path::new("tests/rust"); + + let mut exclude_files_set = BTreeSet::new(); + for exclude in EXCLUDE_FILES { + if !exclude_files_set.insert(exclude) { + panic!("duplicate path in EXCLUDE_FILES: {}", exclude); + } + for dir in EXCLUDE_DIRS { + if Path::new(exclude).starts_with(dir) { + panic!("excluded file {} is inside an excluded dir", exclude); + } + } + if !test_src.join(exclude).is_file() { + missing += "\ntests/rust/"; + missing += exclude; + } + } + + let mut exclude_dirs_set = BTreeSet::new(); + for exclude in EXCLUDE_DIRS { + if !exclude_dirs_set.insert(exclude) { + panic!("duplicate path in EXCLUDE_DIRS: {}", exclude); + } + if !test_src.join(exclude).is_dir() { + missing += "\ntests/rust/"; + missing += exclude; + missing += "/"; + } + } + + if !missing.is_empty() { + panic!("excluded test file does not exist:{}\n", missing); + } +} + +fn download_and_unpack() -> Result<()> { + let url = format!("https://github.com/rust-lang/rust/archive/{REVISION}.tar.gz"); + errorf!("downloading {url}\n"); + + let response = reqwest::blocking::get(url)?.error_for_status()?; + let progress = Progress::new(response); + let decoder = GzDecoder::new(progress); + let mut archive = Archive::new(decoder); + let prefix = format!("rust-{}", REVISION); + + let tests_rust = Path::new("tests/rust"); + if tests_rust.exists() { + fs::remove_dir_all(tests_rust)?; + } + + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.path()?; + if path == Path::new("pax_global_header") { + continue; + } + let relative = path.strip_prefix(&prefix)?; + let out = tests_rust.join(relative); + entry.unpack(&out)?; + } + + fs::write("tests/rust/COMMIT", REVISION)?; + Ok(()) +} diff --git a/third_party/crates/syn-2.0.119/tests/repo/progress.rs b/third_party/crates/syn-2.0.119/tests/repo/progress.rs new file mode 100644 index 0000000..cb2e87d --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/repo/progress.rs @@ -0,0 +1,37 @@ +use std::io::{Read, Result}; +use std::time::{Duration, Instant}; + +pub struct Progress<R> { + bytes: usize, + tick: Instant, + stream: R, +} + +impl<R> Progress<R> { + pub fn new(stream: R) -> Self { + Progress { + bytes: 0, + tick: Instant::now() + Duration::from_secs(2), + stream, + } + } +} + +impl<R: Read> Read for Progress<R> { + fn read(&mut self, buf: &mut [u8]) -> Result<usize> { + let num = self.stream.read(buf)?; + self.bytes += num; + let now = Instant::now(); + if now > self.tick { + self.tick = now + Duration::from_millis(500); + errorf!("downloading... {} bytes\n", self.bytes); + } + Ok(num) + } +} + +impl<R> Drop for Progress<R> { + fn drop(&mut self) { + errorf!("done ({} bytes)\n", self.bytes); + } +} diff --git a/third_party/crates/syn-2.0.119/tests/snapshot/mod.rs b/third_party/crates/syn-2.0.119/tests/snapshot/mod.rs new file mode 100644 index 0000000..98d2aeb --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/snapshot/mod.rs @@ -0,0 +1,68 @@ +#![allow(unused_macros, unused_macro_rules)] + +use std::str::FromStr; +use syn::parse::Result; + +macro_rules! snapshot { + ($($args:tt)*) => { + snapshot_impl!(() $($args)*) + }; +} + +macro_rules! snapshot_impl { + (($expr:ident) as $t:ty, @$snapshot:literal) => { + let tokens = crate::snapshot::TryIntoTokens::try_into_tokens($expr).unwrap(); + let $expr: $t = syn::parse_quote!(#tokens); + let debug = crate::debug::Lite(&$expr); + if !cfg!(miri) { + #[allow(clippy::needless_raw_string_hashes)] // https://github.com/mitsuhiko/insta/issues/389 + { + insta::assert_debug_snapshot!(debug, @$snapshot); + } + } + }; + (($($expr:tt)*) as $t:ty, @$snapshot:literal) => {{ + let tokens = crate::snapshot::TryIntoTokens::try_into_tokens($($expr)*).unwrap(); + let syntax_tree: $t = syn::parse_quote!(#tokens); + let debug = crate::debug::Lite(&syntax_tree); + if !cfg!(miri) { + #[allow(clippy::needless_raw_string_hashes)] + { + insta::assert_debug_snapshot!(debug, @$snapshot); + } + } + syntax_tree + }}; + (($($expr:tt)*) , @$snapshot:literal) => {{ + let syntax_tree = $($expr)*; + let debug = crate::debug::Lite(&syntax_tree); + if !cfg!(miri) { + #[allow(clippy::needless_raw_string_hashes)] + { + insta::assert_debug_snapshot!(debug, @$snapshot); + } + } + syntax_tree + }}; + (($($expr:tt)*) $next:tt $($rest:tt)*) => { + snapshot_impl!(($($expr)* $next) $($rest)*) + }; +} + +pub trait TryIntoTokens { + #[allow(dead_code)] + fn try_into_tokens(self) -> Result<proc_macro2::TokenStream>; +} + +impl TryIntoTokens for &str { + fn try_into_tokens(self) -> Result<proc_macro2::TokenStream> { + let tokens = proc_macro2::TokenStream::from_str(self)?; + Ok(tokens) + } +} + +impl TryIntoTokens for proc_macro2::TokenStream { + fn try_into_tokens(self) -> Result<proc_macro2::TokenStream> { + Ok(self) + } +} diff --git a/third_party/crates/syn-2.0.119/tests/test_asyncness.rs b/third_party/crates/syn-2.0.119/tests/test_asyncness.rs new file mode 100644 index 0000000..c7aee32 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_asyncness.rs @@ -0,0 +1,49 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use syn::{Expr, Item}; + +#[test] +fn test_async_fn() { + let input = "async fn process() {}"; + + snapshot!(input as Item, @r#" + Item::Fn { + vis: Visibility::Inherited, + sig: Signature { + asyncness: Some, + ident: "process", + generics: Generics, + output: ReturnType::Default, + }, + block: Block { + stmts: [], + }, + } + "#); +} + +#[test] +fn test_async_closure() { + let input = "async || {}"; + + snapshot!(input as Expr, @r#" + Expr::Closure { + asyncness: Some, + output: ReturnType::Default, + body: Expr::Block { + block: Block { + stmts: [], + }, + }, + } + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_attribute.rs b/third_party/crates/syn-2.0.119/tests/test_attribute.rs new file mode 100644 index 0000000..81c485e --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_attribute.rs @@ -0,0 +1,231 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use syn::parse::Parser; +use syn::{Attribute, Meta}; + +#[test] +fn test_meta_item_word() { + let meta = test("#[foo]"); + + snapshot!(meta, @r#" + Meta::Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + } + "#); +} + +#[test] +fn test_meta_item_name_value() { + let meta = test("#[foo = 5]"); + + snapshot!(meta, @r#" + Meta::NameValue { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + value: Expr::Lit { + lit: 5, + }, + } + "#); +} + +#[test] +fn test_meta_item_bool_value() { + let meta = test("#[foo = true]"); + + snapshot!(meta, @r#" + Meta::NameValue { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + value: Expr::Lit { + lit: Lit::Bool { + value: true, + }, + }, + } + "#); + + let meta = test("#[foo = false]"); + + snapshot!(meta, @r#" + Meta::NameValue { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + value: Expr::Lit { + lit: Lit::Bool { + value: false, + }, + }, + } + "#); +} + +#[test] +fn test_meta_item_list_lit() { + let meta = test("#[foo(5)]"); + + snapshot!(meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`5`), + } + "#); +} + +#[test] +fn test_meta_item_list_word() { + let meta = test("#[foo(bar)]"); + + snapshot!(meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`bar`), + } + "#); +} + +#[test] +fn test_meta_item_list_name_value() { + let meta = test("#[foo(bar = 5)]"); + + snapshot!(meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`bar = 5`), + } + "#); +} + +#[test] +fn test_meta_item_list_bool_value() { + let meta = test("#[foo(bar = true)]"); + + snapshot!(meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`bar = true`), + } + "#); +} + +#[test] +fn test_meta_item_multiple() { + let meta = test("#[foo(word, name = 5, list(name2 = 6), word2)]"); + + snapshot!(meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`word , name = 5 , list (name2 = 6) , word2`), + } + "#); +} + +#[test] +fn test_bool_lit() { + let meta = test("#[foo(true)]"); + + snapshot!(meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`true`), + } + "#); +} + +#[test] +fn test_negative_lit() { + let meta = test("#[form(min = -1, max = 200)]"); + + snapshot!(meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "form", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`min = - 1 , max = 200`), + } + "#); +} + +fn test(input: &str) -> Meta { + let attrs = Attribute::parse_outer.parse_str(input).unwrap(); + + assert_eq!(attrs.len(), 1); + let attr = attrs.into_iter().next().unwrap(); + + attr.meta +} diff --git a/third_party/crates/syn-2.0.119/tests/test_derive_input.rs b/third_party/crates/syn-2.0.119/tests/test_derive_input.rs new file mode 100644 index 0000000..790e279 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_derive_input.rs @@ -0,0 +1,785 @@ +#![allow( + clippy::assertions_on_result_states, + clippy::elidable_lifetime_names, + clippy::manual_let_else, + clippy::needless_lifetimes, + clippy::too_many_lines, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use quote::quote; +use syn::{Data, DeriveInput}; + +#[test] +fn test_unit() { + let input = quote! { + struct Unit; + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "Unit", + generics: Generics, + data: Data::Struct { + fields: Fields::Unit, + semi_token: Some, + }, + } + "#); +} + +#[test] +fn test_struct() { + let input = quote! { + #[derive(Debug, Clone)] + pub struct Item { + pub ident: Ident, + pub attrs: Vec<Attribute> + } + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "derive", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`Debug , Clone`), + }, + }, + ], + vis: Visibility::Public, + ident: "Item", + generics: Generics, + data: Data::Struct { + fields: Fields::Named { + named: [ + Field { + vis: Visibility::Public, + ident: Some("ident"), + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Ident", + }, + ], + }, + }, + }, + Token![,], + Field { + vis: Visibility::Public, + ident: Some("attrs"), + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Vec", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Type(Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Attribute", + }, + ], + }, + }), + ], + }, + }, + ], + }, + }, + }, + ], + }, + }, + } + "#); + + snapshot!(&input.attrs[0].meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "derive", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`Debug , Clone`), + } + "#); +} + +#[test] +fn test_union() { + let input = quote! { + union MaybeUninit<T> { + uninit: (), + value: T + } + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "MaybeUninit", + generics: Generics { + lt_token: Some, + params: [ + GenericParam::Type(TypeParam { + ident: "T", + }), + ], + gt_token: Some, + }, + data: Data::Union { + fields: FieldsNamed { + named: [ + Field { + vis: Visibility::Inherited, + ident: Some("uninit"), + colon_token: Some, + ty: Type::Tuple, + }, + Token![,], + Field { + vis: Visibility::Inherited, + ident: Some("value"), + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "T", + }, + ], + }, + }, + }, + ], + }, + }, + } + "#); +} + +#[test] +#[cfg(feature = "full")] +fn test_enum() { + let input = quote! { + /// See the std::result module documentation for details. + #[must_use] + pub enum Result<T, E> { + Ok(T), + Err(E), + Surprise = 0isize, + + // Smuggling data into a proc_macro_derive, + // in the style of https://github.com/dtolnay/proc-macro-hack + ProcMacroHack = (0, "data").0 + } + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::NameValue { + path: Path { + segments: [ + PathSegment { + ident: "doc", + }, + ], + }, + value: Expr::Lit { + lit: " See the std::result module documentation for details.", + }, + }, + }, + Attribute { + style: AttrStyle::Outer, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "must_use", + }, + ], + }, + }, + ], + vis: Visibility::Public, + ident: "Result", + generics: Generics { + lt_token: Some, + params: [ + GenericParam::Type(TypeParam { + ident: "T", + }), + Token![,], + GenericParam::Type(TypeParam { + ident: "E", + }), + ], + gt_token: Some, + }, + data: Data::Enum { + variants: [ + Variant { + ident: "Ok", + fields: Fields::Unnamed { + unnamed: [ + Field { + vis: Visibility::Inherited, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "T", + }, + ], + }, + }, + }, + ], + }, + }, + Token![,], + Variant { + ident: "Err", + fields: Fields::Unnamed { + unnamed: [ + Field { + vis: Visibility::Inherited, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "E", + }, + ], + }, + }, + }, + ], + }, + }, + Token![,], + Variant { + ident: "Surprise", + fields: Fields::Unit, + discriminant: Some(Expr::Lit { + lit: 0isize, + }), + }, + Token![,], + Variant { + ident: "ProcMacroHack", + fields: Fields::Unit, + discriminant: Some(Expr::Field { + base: Expr::Tuple { + elems: [ + Expr::Lit { + lit: 0, + }, + Token![,], + Expr::Lit { + lit: "data", + }, + ], + }, + member: Member::Unnamed(Index { + index: 0, + }), + }), + }, + ], + }, + } + "#); + + let meta_items: Vec<_> = input.attrs.into_iter().map(|attr| attr.meta).collect(); + + snapshot!(meta_items, @r#" + [ + Meta::NameValue { + path: Path { + segments: [ + PathSegment { + ident: "doc", + }, + ], + }, + value: Expr::Lit { + lit: " See the std::result module documentation for details.", + }, + }, + Meta::Path { + segments: [ + PathSegment { + ident: "must_use", + }, + ], + }, + ] + "#); +} + +#[test] +fn test_attr_with_non_mod_style_path() { + let input = quote! { + #[inert <T>] + struct S; + }; + + syn::parse2::<DeriveInput>(input).unwrap_err(); +} + +#[test] +fn test_attr_with_mod_style_path_with_self() { + let input = quote! { + #[foo::self] + struct S; + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "foo", + }, + Token![::], + PathSegment { + ident: "self", + }, + ], + }, + }, + ], + vis: Visibility::Inherited, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Unit, + semi_token: Some, + }, + } + "#); + + snapshot!(&input.attrs[0].meta, @r#" + Meta::Path { + segments: [ + PathSegment { + ident: "foo", + }, + Token![::], + PathSegment { + ident: "self", + }, + ], + } + "#); +} + +#[test] +fn test_pub_restricted() { + // Taken from tests/rust/src/test/ui/resolve/auxiliary/privacy-struct-ctor.rs + let input = quote! { + pub(in m) struct Z(pub(in m::n) u8); + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Restricted { + in_token: Some, + path: Path { + segments: [ + PathSegment { + ident: "m", + }, + ], + }, + }, + ident: "Z", + generics: Generics, + data: Data::Struct { + fields: Fields::Unnamed { + unnamed: [ + Field { + vis: Visibility::Restricted { + in_token: Some, + path: Path { + segments: [ + PathSegment { + ident: "m", + }, + Token![::], + PathSegment { + ident: "n", + }, + ], + }, + }, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "u8", + }, + ], + }, + }, + }, + ], + }, + semi_token: Some, + }, + } + "#); +} + +#[test] +fn test_pub_restricted_crate() { + let input = quote! { + pub(crate) struct S; + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Restricted { + path: Path { + segments: [ + PathSegment { + ident: "crate", + }, + ], + }, + }, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Unit, + semi_token: Some, + }, + } + "#); +} + +#[test] +fn test_pub_restricted_super() { + let input = quote! { + pub(super) struct S; + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Restricted { + path: Path { + segments: [ + PathSegment { + ident: "super", + }, + ], + }, + }, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Unit, + semi_token: Some, + }, + } + "#); +} + +#[test] +fn test_pub_restricted_in_super() { + let input = quote! { + pub(in super) struct S; + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Restricted { + in_token: Some, + path: Path { + segments: [ + PathSegment { + ident: "super", + }, + ], + }, + }, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Unit, + semi_token: Some, + }, + } + "#); +} + +#[test] +fn test_fields_on_unit_struct() { + let input = quote! { + struct S; + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Unit, + semi_token: Some, + }, + } + "#); + + let data = match input.data { + Data::Struct(data) => data, + _ => panic!("expected a struct"), + }; + + assert_eq!(0, data.fields.iter().count()); +} + +#[test] +fn test_fields_on_named_struct() { + let input = quote! { + struct S { + foo: i32, + pub bar: String, + } + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Named { + named: [ + Field { + vis: Visibility::Inherited, + ident: Some("foo"), + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "i32", + }, + ], + }, + }, + }, + Token![,], + Field { + vis: Visibility::Public, + ident: Some("bar"), + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "String", + }, + ], + }, + }, + }, + Token![,], + ], + }, + }, + } + "#); + + let data = match input.data { + Data::Struct(data) => data, + _ => panic!("expected a struct"), + }; + + snapshot!(data.fields.into_iter().collect::<Vec<_>>(), @r#" + [ + Field { + vis: Visibility::Inherited, + ident: Some("foo"), + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "i32", + }, + ], + }, + }, + }, + Field { + vis: Visibility::Public, + ident: Some("bar"), + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "String", + }, + ], + }, + }, + }, + ] + "#); +} + +#[test] +fn test_fields_on_tuple_struct() { + let input = quote! { + struct S(i32, pub String); + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Unnamed { + unnamed: [ + Field { + vis: Visibility::Inherited, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "i32", + }, + ], + }, + }, + }, + Token![,], + Field { + vis: Visibility::Public, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "String", + }, + ], + }, + }, + }, + ], + }, + semi_token: Some, + }, + } + "#); + + let data = match input.data { + Data::Struct(data) => data, + _ => panic!("expected a struct"), + }; + + snapshot!(data.fields.iter().collect::<Vec<_>>(), @r#" + [ + Field { + vis: Visibility::Inherited, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "i32", + }, + ], + }, + }, + }, + Field { + vis: Visibility::Public, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "String", + }, + ], + }, + }, + }, + ] + "#); +} + +#[test] +fn test_ambiguous_crate() { + let input = quote! { + // The field type is `(crate::X)` not `crate (::X)`. + struct S(crate::X); + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Unnamed { + unnamed: [ + Field { + vis: Visibility::Inherited, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "crate", + }, + Token![::], + PathSegment { + ident: "X", + }, + ], + }, + }, + }, + ], + }, + semi_token: Some, + }, + } + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_expr.rs b/third_party/crates/syn-2.0.119/tests/test_expr.rs new file mode 100644 index 0000000..e21373c --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_expr.rs @@ -0,0 +1,1702 @@ +#![cfg(not(miri))] +#![recursion_limit = "1024"] +#![feature(rustc_private)] +#![allow( + clippy::elidable_lifetime_names, + clippy::match_like_matches_macro, + clippy::needless_lifetimes, + clippy::single_element_loop, + clippy::too_many_lines, + clippy::uninlined_format_args, + clippy::unreadable_literal +)] + +#[macro_use] +mod macros; +#[macro_use] +mod snapshot; + +mod common; +mod debug; + +use crate::common::visit::{AsIfPrinted, FlattenParens}; +use proc_macro2::{Delimiter, Group, Ident, Span, TokenStream}; +use quote::{quote, ToTokens as _}; +use std::process::ExitCode; +use syn::punctuated::Punctuated; +use syn::visit_mut::VisitMut as _; +use syn::{ + parse_quote, token, AngleBracketedGenericArguments, Arm, BinOp, Block, Expr, ExprArray, + ExprAssign, ExprAsync, ExprAwait, ExprBinary, ExprBlock, ExprBreak, ExprCall, ExprCast, + ExprClosure, ExprConst, ExprContinue, ExprField, ExprForLoop, ExprIf, ExprIndex, ExprLet, + ExprLit, ExprLoop, ExprMacro, ExprMatch, ExprMethodCall, ExprPath, ExprRange, ExprRawAddr, + ExprReference, ExprReturn, ExprStruct, ExprTry, ExprTryBlock, ExprTuple, ExprUnary, ExprUnsafe, + ExprWhile, ExprYield, GenericArgument, Label, Lifetime, Lit, LitInt, Macro, MacroDelimiter, + Member, Pat, PatWild, Path, PathArguments, PathSegment, PointerMutability, QSelf, RangeLimits, + ReturnType, Stmt, Token, Type, TypePath, UnOp, +}; + +#[test] +fn test_expr_parse() { + let tokens = quote!(..100u32); + snapshot!(tokens as Expr, @r#" + Expr::Range { + limits: RangeLimits::HalfOpen, + end: Some(Expr::Lit { + lit: 100u32, + }), + } + "#); + + let tokens = quote!(..100u32); + snapshot!(tokens as ExprRange, @r#" + ExprRange { + limits: RangeLimits::HalfOpen, + end: Some(Expr::Lit { + lit: 100u32, + }), + } + "#); +} + +#[test] +fn test_await() { + // Must not parse as Expr::Field. + let tokens = quote!(fut.await); + + snapshot!(tokens as Expr, @r#" + Expr::Await { + base: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "fut", + }, + ], + }, + }, + } + "#); +} + +#[rustfmt::skip] +#[test] +fn test_tuple_multi_index() { + let expected = snapshot!("tuple.0.0" as Expr, @r#" + Expr::Field { + base: Expr::Field { + base: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "tuple", + }, + ], + }, + }, + member: Member::Unnamed(Index { + index: 0, + }), + }, + member: Member::Unnamed(Index { + index: 0, + }), + } + "#); + + for &input in &[ + "tuple .0.0", + "tuple. 0.0", + "tuple.0 .0", + "tuple.0. 0", + "tuple . 0 . 0", + ] { + assert_eq!(expected, syn::parse_str(input).unwrap()); + } + + for tokens in [ + quote!(tuple.0.0), + quote!(tuple .0.0), + quote!(tuple. 0.0), + quote!(tuple.0 .0), + quote!(tuple.0. 0), + quote!(tuple . 0 . 0), + ] { + assert_eq!(expected, syn::parse2(tokens).unwrap()); + } +} + +#[test] +fn test_macro_variable_func() { + // mimics the token stream corresponding to `$fn()` + let path = Group::new(Delimiter::None, quote!(f)); + let tokens = quote!(#path()); + + snapshot!(tokens as Expr, @r#" + Expr::Call { + func: Expr::Group { + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "f", + }, + ], + }, + }, + }, + } + "#); + + let path = Group::new(Delimiter::None, quote! { #[inside] f }); + let tokens = quote!(#[outside] #path()); + + snapshot!(tokens as Expr, @r#" + Expr::Call { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "outside", + }, + ], + }, + }, + ], + func: Expr::Group { + expr: Expr::Path { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "inside", + }, + ], + }, + }, + ], + path: Path { + segments: [ + PathSegment { + ident: "f", + }, + ], + }, + }, + }, + } + "#); +} + +#[test] +fn test_macro_variable_macro() { + // mimics the token stream corresponding to `$macro!()` + let mac = Group::new(Delimiter::None, quote!(m)); + let tokens = quote!(#mac!()); + + snapshot!(tokens as Expr, @r#" + Expr::Macro { + mac: Macro { + path: Path { + segments: [ + PathSegment { + ident: "m", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(``), + }, + } + "#); +} + +#[test] +fn test_macro_variable_struct() { + // mimics the token stream corresponding to `$struct {}` + let s = Group::new(Delimiter::None, quote! { S }); + let tokens = quote!(#s {}); + + snapshot!(tokens as Expr, @r#" + Expr::Struct { + path: Path { + segments: [ + PathSegment { + ident: "S", + }, + ], + }, + } + "#); +} + +#[test] +fn test_macro_variable_unary() { + // mimics the token stream corresponding to `$expr.method()` where expr is `&self` + let inner = Group::new(Delimiter::None, quote!(&self)); + let tokens = quote!(#inner.method()); + snapshot!(tokens as Expr, @r#" + Expr::MethodCall { + receiver: Expr::Group { + expr: Expr::Reference { + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "self", + }, + ], + }, + }, + }, + }, + method: "method", + } + "#); +} + +#[test] +fn test_macro_variable_match_arm() { + // mimics the token stream corresponding to `match v { _ => $expr }` + let expr = Group::new(Delimiter::None, quote! { #[a] () }); + let tokens = quote!(match v { _ => #expr }); + snapshot!(tokens as Expr, @r#" + Expr::Match { + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "v", + }, + ], + }, + }, + arms: [ + Arm { + pat: Pat::Wild, + body: Expr::Group { + expr: Expr::Tuple { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "a", + }, + ], + }, + }, + ], + }, + }, + }, + ], + } + "#); + + let expr = Group::new(Delimiter::None, quote!(loop {} + 1)); + let tokens = quote!(match v { _ => #expr }); + snapshot!(tokens as Expr, @r#" + Expr::Match { + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "v", + }, + ], + }, + }, + arms: [ + Arm { + pat: Pat::Wild, + body: Expr::Group { + expr: Expr::Binary { + left: Expr::Loop { + body: Block { + stmts: [], + }, + }, + op: BinOp::Add, + right: Expr::Lit { + lit: 1, + }, + }, + }, + }, + ], + } + "#); +} + +// https://github.com/dtolnay/syn/issues/1019 +#[test] +fn test_closure_vs_rangefull() { + #[rustfmt::skip] // rustfmt bug: https://github.com/rust-lang/rustfmt/issues/4808 + let tokens = quote!(|| .. .method()); + snapshot!(tokens as Expr, @r#" + Expr::MethodCall { + receiver: Expr::Closure { + output: ReturnType::Default, + body: Expr::Range { + limits: RangeLimits::HalfOpen, + }, + }, + method: "method", + } + "#); +} + +#[test] +fn test_postfix_operator_after_cast() { + syn::parse_str::<Expr>("|| &x as T[0]").unwrap_err(); + syn::parse_str::<Expr>("|| () as ()()").unwrap_err(); +} + +#[test] +fn test_range_kinds() { + syn::parse_str::<Expr>("..").unwrap(); + syn::parse_str::<Expr>("..hi").unwrap(); + syn::parse_str::<Expr>("lo..").unwrap(); + syn::parse_str::<Expr>("lo..hi").unwrap(); + + syn::parse_str::<Expr>("..=").unwrap_err(); + syn::parse_str::<Expr>("..=hi").unwrap(); + syn::parse_str::<Expr>("lo..=").unwrap_err(); + syn::parse_str::<Expr>("lo..=hi").unwrap(); + + syn::parse_str::<Expr>("...").unwrap_err(); + syn::parse_str::<Expr>("...hi").unwrap_err(); + syn::parse_str::<Expr>("lo...").unwrap_err(); + syn::parse_str::<Expr>("lo...hi").unwrap_err(); +} + +#[test] +fn test_range_precedence() { + snapshot!(".. .." as Expr, @r#" + Expr::Range { + limits: RangeLimits::HalfOpen, + end: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + }), + } + "#); + + snapshot!(".. .. ()" as Expr, @r#" + Expr::Range { + limits: RangeLimits::HalfOpen, + end: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + end: Some(Expr::Tuple), + }), + } + "#); + + snapshot!("() .. .." as Expr, @r#" + Expr::Range { + start: Some(Expr::Tuple), + limits: RangeLimits::HalfOpen, + end: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + }), + } + "#); + + snapshot!("() = .. + ()" as Expr, @r" + Expr::Binary { + left: Expr::Assign { + left: Expr::Tuple, + right: Expr::Range { + limits: RangeLimits::HalfOpen, + }, + }, + op: BinOp::Add, + right: Expr::Tuple, + } + "); + + // A range with a lower bound cannot be the upper bound of another range, + // and a range with an upper bound cannot be the lower bound of another + // range. + syn::parse_str::<Expr>(".. x ..").unwrap_err(); + syn::parse_str::<Expr>("x .. x ..").unwrap_err(); +} + +#[test] +fn test_range_attrs() { + // Attributes are not allowed on range expressions starting with `..` + syn::parse_str::<Expr>("#[allow()] ..").unwrap_err(); + syn::parse_str::<Expr>("#[allow()] .. hi").unwrap_err(); + + snapshot!("#[allow()] lo .. hi" as Expr, @r#" + Expr::Range { + start: Some(Expr::Path { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "allow", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(``), + }, + }, + ], + path: Path { + segments: [ + PathSegment { + ident: "lo", + }, + ], + }, + }), + limits: RangeLimits::HalfOpen, + end: Some(Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "hi", + }, + ], + }, + }), + } + "#); +} + +#[test] +fn test_ranges_bailout() { + syn::parse_str::<Expr>(".. ?").unwrap_err(); + syn::parse_str::<Expr>(".. .field").unwrap_err(); + + snapshot!("return .. ?" as Expr, @r" + Expr::Try { + expr: Expr::Return { + expr: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + }), + }, + } + "); + + snapshot!("break .. ?" as Expr, @r" + Expr::Try { + expr: Expr::Break { + expr: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + }), + }, + } + "); + + snapshot!("|| .. ?" as Expr, @r" + Expr::Try { + expr: Expr::Closure { + output: ReturnType::Default, + body: Expr::Range { + limits: RangeLimits::HalfOpen, + }, + }, + } + "); + + snapshot!("return .. .field" as Expr, @r#" + Expr::Field { + base: Expr::Return { + expr: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + }), + }, + member: Member::Named("field"), + } + "#); + + snapshot!("break .. .field" as Expr, @r#" + Expr::Field { + base: Expr::Break { + expr: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + }), + }, + member: Member::Named("field"), + } + "#); + + snapshot!("|| .. .field" as Expr, @r#" + Expr::Field { + base: Expr::Closure { + output: ReturnType::Default, + body: Expr::Range { + limits: RangeLimits::HalfOpen, + }, + }, + member: Member::Named("field"), + } + "#); + + snapshot!("return .. = ()" as Expr, @r" + Expr::Assign { + left: Expr::Return { + expr: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + }), + }, + right: Expr::Tuple, + } + "); + + snapshot!("return .. += ()" as Expr, @r" + Expr::Binary { + left: Expr::Return { + expr: Some(Expr::Range { + limits: RangeLimits::HalfOpen, + }), + }, + op: BinOp::AddAssign, + right: Expr::Tuple, + } + "); +} + +#[test] +fn test_ambiguous_label() { + for stmt in [ + quote! { + return 'label: loop { break 'label 42; }; + }, + quote! { + break ('label: loop { break 'label 42; }); + }, + quote! { + break 1 + 'label: loop { break 'label 42; }; + }, + quote! { + break 'outer 'inner: loop { break 'inner 42; }; + }, + ] { + syn::parse2::<Stmt>(stmt).unwrap(); + } + + for stmt in [ + // Parentheses required. See https://github.com/rust-lang/rust/pull/87026. + quote! { + break 'label: loop { break 'label 42; }; + }, + ] { + syn::parse2::<Stmt>(stmt).unwrap_err(); + } +} + +#[test] +fn test_extended_interpolated_path() { + let path = Group::new(Delimiter::None, quote!(a::b)); + + let tokens = quote!(if #path {}); + snapshot!(tokens as Expr, @r#" + Expr::If { + cond: Expr::Group { + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "a", + }, + Token![::], + PathSegment { + ident: "b", + }, + ], + }, + }, + }, + then_branch: Block { + stmts: [], + }, + } + "#); + + let tokens = quote!(#path {}); + snapshot!(tokens as Expr, @r#" + Expr::Struct { + path: Path { + segments: [ + PathSegment { + ident: "a", + }, + Token![::], + PathSegment { + ident: "b", + }, + ], + }, + } + "#); + + let tokens = quote!(#path :: c); + snapshot!(tokens as Expr, @r#" + Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "a", + }, + Token![::], + PathSegment { + ident: "b", + }, + Token![::], + PathSegment { + ident: "c", + }, + ], + }, + } + "#); + + let nested = Group::new(Delimiter::None, quote!(a::b || true)); + let tokens = quote!(if #nested && false {}); + snapshot!(tokens as Expr, @r#" + Expr::If { + cond: Expr::Binary { + left: Expr::Group { + expr: Expr::Binary { + left: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "a", + }, + Token![::], + PathSegment { + ident: "b", + }, + ], + }, + }, + op: BinOp::Or, + right: Expr::Lit { + lit: Lit::Bool { + value: true, + }, + }, + }, + }, + op: BinOp::And, + right: Expr::Lit { + lit: Lit::Bool { + value: false, + }, + }, + }, + then_branch: Block { + stmts: [], + }, + } + "#); +} + +#[test] +fn test_tuple_comma() { + let mut expr = ExprTuple { + attrs: Vec::new(), + paren_token: token::Paren::default(), + elems: Punctuated::new(), + }; + snapshot!(expr.to_token_stream() as Expr, @"Expr::Tuple"); + + expr.elems.push_value(parse_quote!(continue)); + // Must not parse to Expr::Paren + snapshot!(expr.to_token_stream() as Expr, @r#" + Expr::Tuple { + elems: [ + Expr::Continue, + Token![,], + ], + } + "#); + + expr.elems.push_punct(<Token![,]>::default()); + snapshot!(expr.to_token_stream() as Expr, @r#" + Expr::Tuple { + elems: [ + Expr::Continue, + Token![,], + ], + } + "#); + + expr.elems.push_value(parse_quote!(continue)); + snapshot!(expr.to_token_stream() as Expr, @r#" + Expr::Tuple { + elems: [ + Expr::Continue, + Token![,], + Expr::Continue, + ], + } + "#); + + expr.elems.push_punct(<Token![,]>::default()); + snapshot!(expr.to_token_stream() as Expr, @r#" + Expr::Tuple { + elems: [ + Expr::Continue, + Token![,], + Expr::Continue, + Token![,], + ], + } + "#); +} + +#[test] +fn test_binop_associativity() { + // Left to right. + snapshot!("() + () + ()" as Expr, @r#" + Expr::Binary { + left: Expr::Binary { + left: Expr::Tuple, + op: BinOp::Add, + right: Expr::Tuple, + }, + op: BinOp::Add, + right: Expr::Tuple, + } + "#); + + // Right to left. + snapshot!("() += () += ()" as Expr, @r#" + Expr::Binary { + left: Expr::Tuple, + op: BinOp::AddAssign, + right: Expr::Binary { + left: Expr::Tuple, + op: BinOp::AddAssign, + right: Expr::Tuple, + }, + } + "#); + + // Parenthesization is required. + syn::parse_str::<Expr>("() == () == ()").unwrap_err(); +} + +#[test] +fn test_assign_range_precedence() { + // Range has higher precedence as the right-hand of an assignment, but + // ambiguous precedence as the left-hand of an assignment. + snapshot!("() = () .. ()" as Expr, @r#" + Expr::Assign { + left: Expr::Tuple, + right: Expr::Range { + start: Some(Expr::Tuple), + limits: RangeLimits::HalfOpen, + end: Some(Expr::Tuple), + }, + } + "#); + + snapshot!("() += () .. ()" as Expr, @r#" + Expr::Binary { + left: Expr::Tuple, + op: BinOp::AddAssign, + right: Expr::Range { + start: Some(Expr::Tuple), + limits: RangeLimits::HalfOpen, + end: Some(Expr::Tuple), + }, + } + "#); + + syn::parse_str::<Expr>("() .. () = ()").unwrap_err(); + syn::parse_str::<Expr>("() .. () += ()").unwrap_err(); +} + +#[test] +fn test_chained_comparison() { + // https://github.com/dtolnay/syn/issues/1738 + let _ = syn::parse_str::<Expr>("a = a < a <"); + let _ = syn::parse_str::<Expr>("a = a .. a .."); + let _ = syn::parse_str::<Expr>("a = a .. a +="); + + let err = syn::parse_str::<Expr>("a < a < a").unwrap_err(); + assert_eq!("comparison operators cannot be chained", err.to_string()); + + let err = syn::parse_str::<Expr>("a .. a .. a").unwrap_err(); + assert_eq!("unexpected token", err.to_string()); + + let err = syn::parse_str::<Expr>("a .. a += a").unwrap_err(); + assert_eq!("unexpected token", err.to_string()); +} + +#[test] +fn test_fixup() { + for tokens in [ + quote! { 2 * (1 + 1) }, + quote! { 0 + (0 + 0) }, + quote! { (a = b) = c }, + quote! { (x as i32) < 0 }, + quote! { 1 + (x as i32) < 0 }, + quote! { (1 + 1).abs() }, + quote! { (lo..hi)[..] }, + quote! { (a..b)..(c..d) }, + quote! { (x > ..) > x }, + quote! { (&mut fut).await }, + quote! { &mut (x as i32) }, + quote! { -(x as i32) }, + quote! { if (S {}) == 1 {} }, + quote! { { (m! {}) - 1 } }, + quote! { match m { _ => ({}) - 1 } }, + quote! { if let _ = (a && b) && c {} }, + quote! { if let _ = (S {}) {} }, + quote! { if (S {}) == 0 && let Some(_) = x {} }, + quote! { break ('a: loop { break 'a 1 } + 1) }, + quote! { a + (|| b) + c }, + quote! { if let _ = ((break) - 1 || true) {} }, + quote! { if let _ = (break + 1 || true) {} }, + quote! { if break (break) {} }, + quote! { if break break {} {} }, + quote! { if return (..) {} }, + quote! { if return .. {} {} }, + quote! { if || (Struct {}) {} }, + quote! { if || (Struct {}).await {} }, + quote! { if break || Struct {}.await {} }, + quote! { if break 'outer 'block: {} {} }, + quote! { if ..'block: {} {} }, + quote! { if break ({}).await {} }, + quote! { (break)() }, + quote! { (..) = () }, + quote! { (..) += () }, + quote! { (1 < 2) == (3 < 4) }, + quote! { { (let _ = ()) } }, + quote! { (#[attr] thing).field }, + quote! { #[attr] (1 + 1) }, + quote! { #[attr] (x = 1) }, + quote! { #[attr] (x += 1) }, + quote! { #[attr] (1 as T) }, + quote! { (return #[attr] (x + ..)).field }, + quote! { (self.f)() }, + quote! { (return)..=return }, + quote! { 1 + (return)..=1 + return }, + quote! { .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. .. }, + ] { + let original: Expr = syn::parse2(tokens).unwrap(); + + let mut flat = original.clone(); + FlattenParens::combine_attrs().visit_expr_mut(&mut flat); + let reconstructed: Expr = match syn::parse2(flat.to_token_stream()) { + Ok(reconstructed) => reconstructed, + Err(err) => panic!("failed to parse `{}`: {}", flat.to_token_stream(), err), + }; + + assert!( + original == reconstructed, + "original: {}\n{:#?}\nreconstructed: {}\n{:#?}", + original.to_token_stream(), + crate::debug::Lite(&original), + reconstructed.to_token_stream(), + crate::debug::Lite(&reconstructed), + ); + } +} + +#[test] +fn test_permutations() -> ExitCode { + fn iter(depth: usize, f: &mut dyn FnMut(Expr)) { + let span = Span::call_site(); + + // Expr::Path + f(Expr::Path(ExprPath { + // `x` + attrs: Vec::new(), + qself: None, + path: Path::from(Ident::new("x", span)), + })); + if false { + f(Expr::Path(ExprPath { + // `x::<T>` + attrs: Vec::new(), + qself: None, + path: Path { + leading_colon: None, + segments: Punctuated::from_iter([PathSegment { + ident: Ident::new("x", span), + arguments: PathArguments::AngleBracketed(AngleBracketedGenericArguments { + colon2_token: Some(Token![::](span)), + lt_token: Token![<](span), + args: Punctuated::from_iter([GenericArgument::Type(Type::Path( + TypePath { + qself: None, + path: Path::from(Ident::new("T", span)), + }, + ))]), + gt_token: Token![>](span), + }), + }]), + }, + })); + f(Expr::Path(ExprPath { + // `<T as Trait>::CONST` + attrs: Vec::new(), + qself: Some(QSelf { + lt_token: Token![<](span), + ty: Box::new(Type::Path(TypePath { + qself: None, + path: Path::from(Ident::new("T", span)), + })), + position: 1, + as_token: Some(Token![as](span)), + gt_token: Token![>](span), + }), + path: Path { + leading_colon: None, + segments: Punctuated::from_iter([ + PathSegment::from(Ident::new("Trait", span)), + PathSegment::from(Ident::new("CONST", span)), + ]), + }, + })); + } + + let Some(depth) = depth.checked_sub(1) else { + return; + }; + + // Expr::Assign + iter(depth, &mut |expr| { + iter(0, &mut |simple| { + f(Expr::Assign(ExprAssign { + // `x = $expr` + attrs: Vec::new(), + left: Box::new(simple.clone()), + eq_token: Token![=](span), + right: Box::new(expr.clone()), + })); + f(Expr::Assign(ExprAssign { + // `$expr = x` + attrs: Vec::new(), + left: Box::new(expr.clone()), + eq_token: Token![=](span), + right: Box::new(simple), + })); + }); + }); + + // Expr::Binary + iter(depth, &mut |expr| { + iter(0, &mut |simple| { + for op in [ + BinOp::Add(Token![+](span)), + //BinOp::Sub(Token![-](span)), + //BinOp::Mul(Token![*](span)), + //BinOp::Div(Token![/](span)), + //BinOp::Rem(Token![%](span)), + //BinOp::And(Token![&&](span)), + //BinOp::Or(Token![||](span)), + //BinOp::BitXor(Token![^](span)), + //BinOp::BitAnd(Token![&](span)), + //BinOp::BitOr(Token![|](span)), + //BinOp::Shl(Token![<<](span)), + //BinOp::Shr(Token![>>](span)), + //BinOp::Eq(Token![==](span)), + BinOp::Lt(Token![<](span)), + //BinOp::Le(Token![<=](span)), + //BinOp::Ne(Token![!=](span)), + //BinOp::Ge(Token![>=](span)), + //BinOp::Gt(Token![>](span)), + BinOp::ShlAssign(Token![<<=](span)), + ] { + f(Expr::Binary(ExprBinary { + // `x + $expr` + attrs: Vec::new(), + left: Box::new(simple.clone()), + op, + right: Box::new(expr.clone()), + })); + f(Expr::Binary(ExprBinary { + // `$expr + x` + attrs: Vec::new(), + left: Box::new(expr.clone()), + op, + right: Box::new(simple.clone()), + })); + } + }); + }); + + // Expr::Block + f(Expr::Block(ExprBlock { + // `{}` + attrs: Vec::new(), + label: None, + block: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + + // Expr::Break + f(Expr::Break(ExprBreak { + // `break` + attrs: Vec::new(), + break_token: Token![break](span), + label: None, + expr: None, + })); + iter(depth, &mut |expr| { + f(Expr::Break(ExprBreak { + // `break $expr` + attrs: Vec::new(), + break_token: Token![break](span), + label: None, + expr: Some(Box::new(expr)), + })); + }); + + // Expr::Call + iter(depth, &mut |expr| { + f(Expr::Call(ExprCall { + // `$expr()` + attrs: Vec::new(), + func: Box::new(expr), + paren_token: token::Paren(span), + args: Punctuated::new(), + })); + }); + + // Expr::Cast + iter(depth, &mut |expr| { + f(Expr::Cast(ExprCast { + // `$expr as T` + attrs: Vec::new(), + expr: Box::new(expr), + as_token: Token![as](span), + ty: Box::new(Type::Path(TypePath { + qself: None, + path: Path::from(Ident::new("T", span)), + })), + })); + }); + + // Expr::Closure + iter(depth, &mut |expr| { + f(Expr::Closure(ExprClosure { + // `|| $expr` + attrs: Vec::new(), + lifetimes: None, + constness: None, + movability: None, + asyncness: None, + capture: None, + or1_token: Token![|](span), + inputs: Punctuated::new(), + or2_token: Token![|](span), + output: ReturnType::Default, + body: Box::new(expr), + })); + }); + + // Expr::Field + iter(depth, &mut |expr| { + f(Expr::Field(ExprField { + // `$expr.field` + attrs: Vec::new(), + base: Box::new(expr), + dot_token: Token![.](span), + member: Member::Named(Ident::new("field", span)), + })); + }); + + // Expr::If + iter(depth, &mut |expr| { + f(Expr::If(ExprIf { + // `if $expr {}` + attrs: Vec::new(), + if_token: Token![if](span), + cond: Box::new(expr), + then_branch: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + else_branch: None, + })); + }); + + // Expr::Let + iter(depth, &mut |expr| { + f(Expr::Let(ExprLet { + attrs: Vec::new(), + let_token: Token![let](span), + pat: Box::new(Pat::Wild(PatWild { + attrs: Vec::new(), + underscore_token: Token![_](span), + })), + eq_token: Token![=](span), + expr: Box::new(expr), + })); + }); + + // Expr::Range + f(Expr::Range(ExprRange { + // `..` + attrs: Vec::new(), + start: None, + limits: RangeLimits::HalfOpen(Token![..](span)), + end: None, + })); + iter(depth, &mut |expr| { + f(Expr::Range(ExprRange { + // `..$expr` + attrs: Vec::new(), + start: None, + limits: RangeLimits::HalfOpen(Token![..](span)), + end: Some(Box::new(expr.clone())), + })); + f(Expr::Range(ExprRange { + // `$expr..` + attrs: Vec::new(), + start: Some(Box::new(expr)), + limits: RangeLimits::HalfOpen(Token![..](span)), + end: None, + })); + }); + + // Expr::Reference + iter(depth, &mut |expr| { + f(Expr::Reference(ExprReference { + // `&$expr` + attrs: Vec::new(), + and_token: Token![&](span), + mutability: None, + expr: Box::new(expr), + })); + }); + + // Expr::Return + f(Expr::Return(ExprReturn { + // `return` + attrs: Vec::new(), + return_token: Token![return](span), + expr: None, + })); + iter(depth, &mut |expr| { + f(Expr::Return(ExprReturn { + // `return $expr` + attrs: Vec::new(), + return_token: Token![return](span), + expr: Some(Box::new(expr)), + })); + }); + + // Expr::Try + iter(depth, &mut |expr| { + f(Expr::Try(ExprTry { + // `$expr?` + attrs: Vec::new(), + expr: Box::new(expr), + question_token: Token![?](span), + })); + }); + + // Expr::Unary + iter(depth, &mut |expr| { + for op in [ + UnOp::Deref(Token![*](span)), + //UnOp::Not(Token![!](span)), + //UnOp::Neg(Token![-](span)), + ] { + f(Expr::Unary(ExprUnary { + // `*$expr` + attrs: Vec::new(), + op, + expr: Box::new(expr.clone()), + })); + } + }); + + if false { + // Expr::Array + f(Expr::Array(ExprArray { + // `[]` + attrs: Vec::new(), + bracket_token: token::Bracket(span), + elems: Punctuated::new(), + })); + + // Expr::Async + f(Expr::Async(ExprAsync { + // `async {}` + attrs: Vec::new(), + async_token: Token![async](span), + capture: None, + block: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + + // Expr::Await + iter(depth, &mut |expr| { + f(Expr::Await(ExprAwait { + // `$expr.await` + attrs: Vec::new(), + base: Box::new(expr), + dot_token: Token![.](span), + await_token: Token![await](span), + })); + }); + + // Expr::Block + f(Expr::Block(ExprBlock { + // `'a: {}` + attrs: Vec::new(), + label: Some(Label { + name: Lifetime::new("'a", span), + colon_token: Token![:](span), + }), + block: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + iter(depth, &mut |expr| { + f(Expr::Block(ExprBlock { + // `{ $expr }` + attrs: Vec::new(), + label: None, + block: Block { + brace_token: token::Brace(span), + stmts: Vec::from([Stmt::Expr(expr.clone(), None)]), + }, + })); + f(Expr::Block(ExprBlock { + // `{ $expr; }` + attrs: Vec::new(), + label: None, + block: Block { + brace_token: token::Brace(span), + stmts: Vec::from([Stmt::Expr(expr, Some(Token![;](span)))]), + }, + })); + }); + + // Expr::Break + f(Expr::Break(ExprBreak { + // `break 'a` + attrs: Vec::new(), + break_token: Token![break](span), + label: Some(Lifetime::new("'a", span)), + expr: None, + })); + iter(depth, &mut |expr| { + f(Expr::Break(ExprBreak { + // `break 'a $expr` + attrs: Vec::new(), + break_token: Token![break](span), + label: Some(Lifetime::new("'a", span)), + expr: Some(Box::new(expr)), + })); + }); + + // Expr::Closure + f(Expr::Closure(ExprClosure { + // `|| -> T {}` + attrs: Vec::new(), + lifetimes: None, + constness: None, + movability: None, + asyncness: None, + capture: None, + or1_token: Token![|](span), + inputs: Punctuated::new(), + or2_token: Token![|](span), + output: ReturnType::Type( + Token![->](span), + Box::new(Type::Path(TypePath { + qself: None, + path: Path::from(Ident::new("T", span)), + })), + ), + body: Box::new(Expr::Block(ExprBlock { + attrs: Vec::new(), + label: None, + block: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })), + })); + + // Expr::Const + f(Expr::Const(ExprConst { + // `const {}` + attrs: Vec::new(), + const_token: Token![const](span), + block: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + + // Expr::Continue + f(Expr::Continue(ExprContinue { + // `continue` + attrs: Vec::new(), + continue_token: Token![continue](span), + label: None, + })); + f(Expr::Continue(ExprContinue { + // `continue 'a` + attrs: Vec::new(), + continue_token: Token![continue](span), + label: Some(Lifetime::new("'a", span)), + })); + + // Expr::ForLoop + iter(depth, &mut |expr| { + f(Expr::ForLoop(ExprForLoop { + // `for _ in $expr {}` + attrs: Vec::new(), + label: None, + for_token: Token![for](span), + pat: Box::new(Pat::Wild(PatWild { + attrs: Vec::new(), + underscore_token: Token![_](span), + })), + in_token: Token![in](span), + expr: Box::new(expr.clone()), + body: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + f(Expr::ForLoop(ExprForLoop { + // `'a: for _ in $expr {}` + attrs: Vec::new(), + label: Some(Label { + name: Lifetime::new("'a", span), + colon_token: Token![:](span), + }), + for_token: Token![for](span), + pat: Box::new(Pat::Wild(PatWild { + attrs: Vec::new(), + underscore_token: Token![_](span), + })), + in_token: Token![in](span), + expr: Box::new(expr), + body: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + }); + + // Expr::Index + iter(depth, &mut |expr| { + f(Expr::Index(ExprIndex { + // `$expr[0]` + attrs: Vec::new(), + expr: Box::new(expr), + bracket_token: token::Bracket(span), + index: Box::new(Expr::Lit(ExprLit { + attrs: Vec::new(), + lit: Lit::Int(LitInt::new("0", span)), + })), + })); + }); + + // Expr::Loop + f(Expr::Loop(ExprLoop { + // `loop {}` + attrs: Vec::new(), + label: None, + loop_token: Token![loop](span), + body: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + f(Expr::Loop(ExprLoop { + // `'a: loop {}` + attrs: Vec::new(), + label: Some(Label { + name: Lifetime::new("'a", span), + colon_token: Token![:](span), + }), + loop_token: Token![loop](span), + body: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + + // Expr::Macro + f(Expr::Macro(ExprMacro { + // `m!()` + attrs: Vec::new(), + mac: Macro { + path: Path::from(Ident::new("m", span)), + bang_token: Token![!](span), + delimiter: MacroDelimiter::Paren(token::Paren(span)), + tokens: TokenStream::new(), + }, + })); + f(Expr::Macro(ExprMacro { + // `m! {}` + attrs: Vec::new(), + mac: Macro { + path: Path::from(Ident::new("m", span)), + bang_token: Token![!](span), + delimiter: MacroDelimiter::Brace(token::Brace(span)), + tokens: TokenStream::new(), + }, + })); + + // Expr::Match + iter(depth, &mut |expr| { + f(Expr::Match(ExprMatch { + // `match $expr {}` + attrs: Vec::new(), + match_token: Token![match](span), + expr: Box::new(expr.clone()), + brace_token: token::Brace(span), + arms: Vec::new(), + })); + f(Expr::Match(ExprMatch { + // `match x { _ => $expr }` + attrs: Vec::new(), + match_token: Token![match](span), + expr: Box::new(Expr::Path(ExprPath { + attrs: Vec::new(), + qself: None, + path: Path::from(Ident::new("x", span)), + })), + brace_token: token::Brace(span), + arms: Vec::from([Arm { + attrs: Vec::new(), + pat: Pat::Wild(PatWild { + attrs: Vec::new(), + underscore_token: Token![_](span), + }), + guard: None, + fat_arrow_token: Token![=>](span), + body: Box::new(expr.clone()), + comma: None, + }]), + })); + f(Expr::Match(ExprMatch { + // `match x { _ if $expr => {} }` + attrs: Vec::new(), + match_token: Token![match](span), + expr: Box::new(Expr::Path(ExprPath { + attrs: Vec::new(), + qself: None, + path: Path::from(Ident::new("x", span)), + })), + brace_token: token::Brace(span), + arms: Vec::from([Arm { + attrs: Vec::new(), + pat: Pat::Wild(PatWild { + attrs: Vec::new(), + underscore_token: Token![_](span), + }), + guard: Some((Token![if](span), Box::new(expr))), + fat_arrow_token: Token![=>](span), + body: Box::new(Expr::Block(ExprBlock { + attrs: Vec::new(), + label: None, + block: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })), + comma: None, + }]), + })); + }); + + // Expr::MethodCall + iter(depth, &mut |expr| { + f(Expr::MethodCall(ExprMethodCall { + // `$expr.method()` + attrs: Vec::new(), + receiver: Box::new(expr.clone()), + dot_token: Token![.](span), + method: Ident::new("method", span), + turbofish: None, + paren_token: token::Paren(span), + args: Punctuated::new(), + })); + f(Expr::MethodCall(ExprMethodCall { + // `$expr.method::<T>()` + attrs: Vec::new(), + receiver: Box::new(expr), + dot_token: Token![.](span), + method: Ident::new("method", span), + turbofish: Some(AngleBracketedGenericArguments { + colon2_token: Some(Token![::](span)), + lt_token: Token![<](span), + args: Punctuated::from_iter([GenericArgument::Type(Type::Path( + TypePath { + qself: None, + path: Path::from(Ident::new("T", span)), + }, + ))]), + gt_token: Token![>](span), + }), + paren_token: token::Paren(span), + args: Punctuated::new(), + })); + }); + + // Expr::RawAddr + iter(depth, &mut |expr| { + f(Expr::RawAddr(ExprRawAddr { + // `&raw const $expr` + attrs: Vec::new(), + and_token: Token![&](span), + raw: Token![raw](span), + mutability: PointerMutability::Const(Token![const](span)), + expr: Box::new(expr), + })); + }); + + // Expr::Struct + f(Expr::Struct(ExprStruct { + // `Struct {}` + attrs: Vec::new(), + qself: None, + path: Path::from(Ident::new("Struct", span)), + brace_token: token::Brace(span), + fields: Punctuated::new(), + dot2_token: None, + rest: None, + })); + + // Expr::TryBlock + f(Expr::TryBlock(ExprTryBlock { + // `try {}` + attrs: Vec::new(), + try_token: Token![try](span), + block: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + + // Expr::Unsafe + f(Expr::Unsafe(ExprUnsafe { + // `unsafe {}` + attrs: Vec::new(), + unsafe_token: Token![unsafe](span), + block: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + + // Expr::While + iter(depth, &mut |expr| { + f(Expr::While(ExprWhile { + // `while $expr {}` + attrs: Vec::new(), + label: None, + while_token: Token![while](span), + cond: Box::new(expr.clone()), + body: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + f(Expr::While(ExprWhile { + // `'a: while $expr {}` + attrs: Vec::new(), + label: Some(Label { + name: Lifetime::new("'a", span), + colon_token: Token![:](span), + }), + while_token: Token![while](span), + cond: Box::new(expr), + body: Block { + brace_token: token::Brace(span), + stmts: Vec::new(), + }, + })); + }); + + // Expr::Yield + f(Expr::Yield(ExprYield { + // `yield` + attrs: Vec::new(), + yield_token: Token![yield](span), + expr: None, + })); + iter(depth, &mut |expr| { + f(Expr::Yield(ExprYield { + // `yield $expr` + attrs: Vec::new(), + yield_token: Token![yield](span), + expr: Some(Box::new(expr)), + })); + }); + } + } + + let mut failures = 0; + macro_rules! fail { + ($($message:tt)*) => {{ + eprintln!($($message)*); + failures += 1; + return; + }}; + } + let mut assert = |mut original: Expr| { + let tokens = original.to_token_stream(); + let Ok(mut parsed) = syn::parse2::<Expr>(tokens.clone()) else { + fail!( + "failed to parse: {}\n{:#?}", + tokens, + crate::debug::Lite(&original), + ); + }; + AsIfPrinted.visit_expr_mut(&mut original); + FlattenParens::combine_attrs().visit_expr_mut(&mut parsed); + if original != parsed { + fail!( + "before: {}\n{:#?}\nafter: {}\n{:#?}", + tokens, + crate::debug::Lite(&original), + parsed.to_token_stream(), + crate::debug::Lite(&parsed), + ); + } + let mut tokens_no_paren = tokens.clone(); + FlattenParens::visit_token_stream_mut(&mut tokens_no_paren); + if tokens.to_string() != tokens_no_paren.to_string() { + if let Ok(mut parsed2) = syn::parse2::<Expr>(tokens_no_paren) { + FlattenParens::combine_attrs().visit_expr_mut(&mut parsed2); + if original == parsed2 { + fail!("redundant parens: {}", tokens); + } + } + } + }; + + iter(4, &mut assert); + if failures > 0 { + eprintln!("FAILURES: {failures}"); + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + } +} diff --git a/third_party/crates/syn-2.0.119/tests/test_generics.rs b/third_party/crates/syn-2.0.119/tests/test_generics.rs new file mode 100644 index 0000000..61bc8de --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_generics.rs @@ -0,0 +1,356 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::manual_let_else, + clippy::needless_lifetimes, + clippy::too_many_lines, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use quote::quote; +use syn::{ + parse_quote, DeriveInput, GenericParam, Generics, ItemFn, Lifetime, LifetimeParam, + TypeParamBound, WhereClause, WherePredicate, +}; + +#[test] +fn test_split_for_impl() { + let input = quote! { + struct S<'a, 'b: 'a, #[may_dangle] T: 'a = ()> where T: Debug; + }; + + snapshot!(input as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "S", + generics: Generics { + lt_token: Some, + params: [ + GenericParam::Lifetime(LifetimeParam { + lifetime: Lifetime { + ident: "a", + }, + }), + Token![,], + GenericParam::Lifetime(LifetimeParam { + lifetime: Lifetime { + ident: "b", + }, + colon_token: Some, + bounds: [ + Lifetime { + ident: "a", + }, + ], + }), + Token![,], + GenericParam::Type(TypeParam { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "may_dangle", + }, + ], + }, + }, + ], + ident: "T", + colon_token: Some, + bounds: [ + TypeParamBound::Lifetime { + ident: "a", + }, + ], + eq_token: Some, + default: Some(Type::Tuple), + }), + ], + gt_token: Some, + where_clause: Some(WhereClause { + predicates: [ + WherePredicate::Type(PredicateType { + bounded_ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "T", + }, + ], + }, + }, + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Debug", + }, + ], + }, + }), + ], + }), + ], + }), + }, + data: Data::Struct { + fields: Fields::Unit, + semi_token: Some, + }, + } + "#); + + let generics = input.generics; + let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + + let generated = quote! { + impl #impl_generics MyTrait for Test #ty_generics #where_clause {} + }; + let expected = quote! { + impl<'a, 'b: 'a, #[may_dangle] T: 'a> MyTrait + for Test<'a, 'b, T> + where + T: Debug + {} + }; + assert_eq!(generated.to_string(), expected.to_string()); + + let turbofish = ty_generics.as_turbofish(); + let generated = quote! { + Test #turbofish + }; + let expected = quote! { + Test::<'a, 'b, T> + }; + assert_eq!(generated.to_string(), expected.to_string()); +} + +#[test] +fn test_type_param_bound() { + let tokens = quote!('a); + snapshot!(tokens as TypeParamBound, @r#" + TypeParamBound::Lifetime { + ident: "a", + } + "#); + + let tokens = quote!('_); + snapshot!(tokens as TypeParamBound, @r#" + TypeParamBound::Lifetime { + ident: "_", + } + "#); + + let tokens = quote!(Debug); + snapshot!(tokens as TypeParamBound, @r#" + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Debug", + }, + ], + }, + }) + "#); + + let tokens = quote!(?Sized); + snapshot!(tokens as TypeParamBound, @r#" + TypeParamBound::Trait(TraitBound { + modifier: TraitBoundModifier::Maybe, + path: Path { + segments: [ + PathSegment { + ident: "Sized", + }, + ], + }, + }) + "#); + + let tokens = quote!(for<'a> Trait); + snapshot!(tokens as TypeParamBound, @r#" + TypeParamBound::Trait(TraitBound { + lifetimes: Some(BoundLifetimes { + lifetimes: [ + GenericParam::Lifetime(LifetimeParam { + lifetime: Lifetime { + ident: "a", + }, + }), + ], + }), + path: Path { + segments: [ + PathSegment { + ident: "Trait", + }, + ], + }, + }) + "#); + + let tokens = quote!(for<> ?Trait); + let err = syn::parse2::<TypeParamBound>(tokens).unwrap_err(); + assert_eq!( + "`for<...>` binder not allowed with `?` trait polarity modifier", + err.to_string(), + ); + + let tokens = quote!(?for<> Trait); + let err = syn::parse2::<TypeParamBound>(tokens).unwrap_err(); + assert_eq!( + "`for<...>` binder not allowed with `?` trait polarity modifier", + err.to_string(), + ); +} + +#[test] +fn test_fn_precedence_in_where_clause() { + // This should parse as two separate bounds, `FnOnce() -> i32` and `Send` - not + // `FnOnce() -> (i32 + Send)`. + let input = quote! { + fn f<G>() + where + G: FnOnce() -> i32 + Send, + { + } + }; + + snapshot!(input as ItemFn, @r#" + ItemFn { + vis: Visibility::Inherited, + sig: Signature { + ident: "f", + generics: Generics { + lt_token: Some, + params: [ + GenericParam::Type(TypeParam { + ident: "G", + }), + ], + gt_token: Some, + where_clause: Some(WhereClause { + predicates: [ + WherePredicate::Type(PredicateType { + bounded_ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "G", + }, + ], + }, + }, + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "FnOnce", + arguments: PathArguments::Parenthesized { + output: ReturnType::Type( + Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "i32", + }, + ], + }, + }, + ), + }, + }, + ], + }, + }), + Token![+], + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Send", + }, + ], + }, + }), + ], + }), + Token![,], + ], + }), + }, + output: ReturnType::Default, + }, + block: Block { + stmts: [], + }, + } + "#); + + let where_clause = input.sig.generics.where_clause.as_ref().unwrap(); + assert_eq!(where_clause.predicates.len(), 1); + + let predicate = match &where_clause.predicates[0] { + WherePredicate::Type(pred) => pred, + _ => panic!("wrong predicate kind"), + }; + + assert_eq!(predicate.bounds.len(), 2, "{:#?}", predicate.bounds); + + let first_bound = &predicate.bounds[0]; + assert_eq!(quote!(#first_bound).to_string(), "FnOnce () -> i32"); + + let second_bound = &predicate.bounds[1]; + assert_eq!(quote!(#second_bound).to_string(), "Send"); +} + +#[test] +fn test_where_clause_at_end_of_input() { + let input = quote! { + where + }; + + snapshot!(input as WhereClause, @"WhereClause"); + + assert_eq!(input.predicates.len(), 0); +} + +// Regression test for https://github.com/dtolnay/syn/issues/1718 +#[test] +#[allow(clippy::map_unwrap_or)] +fn no_opaque_drop() { + let mut generics = Generics::default(); + + let _ = generics + .lifetimes() + .next() + .map(|param| param.lifetime.clone()) + .unwrap_or_else(|| { + let lifetime: Lifetime = parse_quote!('a); + generics.params.insert( + 0, + GenericParam::Lifetime(LifetimeParam::new(lifetime.clone())), + ); + lifetime + }); +} + +#[test] +fn type_param_with_colon_and_no_bounds() { + let tokens = quote!(T:); + snapshot!(tokens as GenericParam, @r#" + GenericParam::Type(TypeParam { + ident: "T", + colon_token: Some, + }) + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_grouping.rs b/third_party/crates/syn-2.0.119/tests/test_grouping.rs new file mode 100644 index 0000000..b466c7e --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_grouping.rs @@ -0,0 +1,59 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::{Delimiter, Group, Literal, Punct, Spacing, TokenStream, TokenTree}; +use syn::Expr; + +#[test] +fn test_grouping() { + let tokens: TokenStream = TokenStream::from_iter([ + TokenTree::Literal(Literal::i32_suffixed(1)), + TokenTree::Punct(Punct::new('+', Spacing::Alone)), + TokenTree::Group(Group::new( + Delimiter::None, + TokenStream::from_iter([ + TokenTree::Literal(Literal::i32_suffixed(2)), + TokenTree::Punct(Punct::new('+', Spacing::Alone)), + TokenTree::Literal(Literal::i32_suffixed(3)), + ]), + )), + TokenTree::Punct(Punct::new('*', Spacing::Alone)), + TokenTree::Literal(Literal::i32_suffixed(4)), + ]); + + assert_eq!(tokens.to_string(), "1i32 + 2i32 + 3i32 * 4i32"); + + snapshot!(tokens as Expr, @r#" + Expr::Binary { + left: Expr::Lit { + lit: 1i32, + }, + op: BinOp::Add, + right: Expr::Binary { + left: Expr::Group { + expr: Expr::Binary { + left: Expr::Lit { + lit: 2i32, + }, + op: BinOp::Add, + right: Expr::Lit { + lit: 3i32, + }, + }, + }, + op: BinOp::Mul, + right: Expr::Lit { + lit: 4i32, + }, + }, + } + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_ident.rs b/third_party/crates/syn-2.0.119/tests/test_ident.rs new file mode 100644 index 0000000..10df0ad --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_ident.rs @@ -0,0 +1,87 @@ +use proc_macro2::{Ident, Span, TokenStream}; +use std::str::FromStr; +use syn::Result; + +#[track_caller] +fn parse(s: &str) -> Result<Ident> { + syn::parse2(TokenStream::from_str(s).unwrap()) +} + +#[track_caller] +fn new(s: &str) -> Ident { + Ident::new(s, Span::call_site()) +} + +#[test] +fn ident_parse() { + parse("String").unwrap(); +} + +#[test] +fn ident_parse_keyword() { + parse("abstract").unwrap_err(); +} + +#[test] +fn ident_parse_empty() { + parse("").unwrap_err(); +} + +#[test] +fn ident_parse_lifetime() { + parse("'static").unwrap_err(); +} + +#[test] +fn ident_parse_underscore() { + parse("_").unwrap_err(); +} + +#[test] +fn ident_parse_number() { + parse("255").unwrap_err(); +} + +#[test] +fn ident_parse_invalid() { + parse("a#").unwrap_err(); +} + +#[test] +fn ident_new() { + new("String"); +} + +#[test] +fn ident_new_keyword() { + new("abstract"); +} + +#[test] +#[should_panic(expected = "use Option<Ident>")] +fn ident_new_empty() { + new(""); +} + +#[test] +#[should_panic(expected = "not a valid Ident")] +fn ident_new_lifetime() { + new("'static"); +} + +#[test] +fn ident_new_underscore() { + new("_"); +} + +#[test] +#[should_panic(expected = "use Literal instead")] +fn ident_new_number() { + new("255"); +} + +#[test] +#[should_panic(expected = "\"a#\" is not a valid Ident")] +fn ident_new_invalid() { + new("a#"); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_item.rs b/third_party/crates/syn-2.0.119/tests/test_item.rs new file mode 100644 index 0000000..cb63ad4 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_item.rs @@ -0,0 +1,371 @@ +#![recursion_limit = "256"] +#![feature(negative_impls)] +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::{Delimiter, Group, Ident, Span, TokenStream, TokenTree}; +use quote::quote; +use syn::{Item, ItemTrait}; + +#[test] +fn test_macro_variable_attr() { + // mimics the token stream corresponding to `$attr fn f() {}` + let tokens = TokenStream::from_iter([ + TokenTree::Group(Group::new(Delimiter::None, quote! { #[test] })), + TokenTree::Ident(Ident::new("fn", Span::call_site())), + TokenTree::Ident(Ident::new("f", Span::call_site())), + TokenTree::Group(Group::new(Delimiter::Parenthesis, TokenStream::new())), + TokenTree::Group(Group::new(Delimiter::Brace, TokenStream::new())), + ]); + + snapshot!(tokens as Item, @r#" + Item::Fn { + attrs: [ + Attribute { + style: AttrStyle::Outer, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "test", + }, + ], + }, + }, + ], + vis: Visibility::Inherited, + sig: Signature { + ident: "f", + generics: Generics, + output: ReturnType::Default, + }, + block: Block { + stmts: [], + }, + } + "#); +} + +#[test] +fn test_negative_impl() { + #[cfg(any())] + impl ! {} + let tokens = quote! { + impl ! {} + }; + snapshot!(tokens as Item, @r#" + Item::Impl { + generics: Generics, + self_ty: Type::Never, + } + "#); + + let tokens = quote! { + impl !Trait {} + }; + let err = syn::parse2::<Item>(tokens).unwrap_err(); + assert_eq!(err.to_string(), "inherent impls cannot be negative"); + + #[cfg(any())] + impl !Trait for T {} + let tokens = quote! { + impl !Trait for T {} + }; + snapshot!(tokens as Item, @r#" + Item::Impl { + generics: Generics, + trait_: Some(( + Some, + Path { + segments: [ + PathSegment { + ident: "Trait", + }, + ], + }, + )), + self_ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "T", + }, + ], + }, + }, + } + "#); +} + +#[test] +fn test_macro_variable_impl() { + // mimics the token stream corresponding to `impl $trait for $ty {}` + let tokens = TokenStream::from_iter([ + TokenTree::Ident(Ident::new("impl", Span::call_site())), + TokenTree::Group(Group::new(Delimiter::None, quote!(Trait))), + TokenTree::Ident(Ident::new("for", Span::call_site())), + TokenTree::Group(Group::new(Delimiter::None, quote!(Type))), + TokenTree::Group(Group::new(Delimiter::Brace, TokenStream::new())), + ]); + + snapshot!(tokens as Item, @r#" + Item::Impl { + generics: Generics, + trait_: Some(( + None, + Path { + segments: [ + PathSegment { + ident: "Trait", + }, + ], + }, + )), + self_ty: Type::Group { + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Type", + }, + ], + }, + }, + }, + } + "#); +} + +#[test] +fn test_supertraits() { + // Rustc parses all of the following. + + #[rustfmt::skip] + let tokens = quote!(trait Trait where {}); + snapshot!(tokens as ItemTrait, @r#" + ItemTrait { + vis: Visibility::Inherited, + ident: "Trait", + generics: Generics { + where_clause: Some(WhereClause), + }, + } + "#); + + #[rustfmt::skip] + let tokens = quote!(trait Trait: where {}); + snapshot!(tokens as ItemTrait, @r#" + ItemTrait { + vis: Visibility::Inherited, + ident: "Trait", + generics: Generics { + where_clause: Some(WhereClause), + }, + colon_token: Some, + } + "#); + + #[rustfmt::skip] + let tokens = quote!(trait Trait: Sized where {}); + snapshot!(tokens as ItemTrait, @r#" + ItemTrait { + vis: Visibility::Inherited, + ident: "Trait", + generics: Generics { + where_clause: Some(WhereClause), + }, + colon_token: Some, + supertraits: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Sized", + }, + ], + }, + }), + ], + } + "#); + + #[rustfmt::skip] + let tokens = quote!(trait Trait: Sized + where {}); + snapshot!(tokens as ItemTrait, @r#" + ItemTrait { + vis: Visibility::Inherited, + ident: "Trait", + generics: Generics { + where_clause: Some(WhereClause), + }, + colon_token: Some, + supertraits: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Sized", + }, + ], + }, + }), + Token![+], + ], + } + "#); +} + +#[test] +fn test_type_empty_bounds() { + #[rustfmt::skip] + let tokens = quote! { + trait Foo { + type Bar: ; + } + }; + + snapshot!(tokens as ItemTrait, @r#" + ItemTrait { + vis: Visibility::Inherited, + ident: "Foo", + generics: Generics, + items: [ + TraitItem::Type { + ident: "Bar", + generics: Generics, + colon_token: Some, + }, + ], + } + "#); +} + +#[test] +fn test_impl_visibility() { + let tokens = quote! { + pub default unsafe impl union {} + }; + + snapshot!(tokens as Item, @"Item::Verbatim(`pub default unsafe impl union { }`)"); +} + +#[test] +fn test_impl_type_parameter_defaults() { + #[cfg(any())] + impl<T = ()> () {} + let tokens = quote! { + impl<T = ()> () {} + }; + snapshot!(tokens as Item, @r#" + Item::Impl { + generics: Generics { + lt_token: Some, + params: [ + GenericParam::Type(TypeParam { + ident: "T", + eq_token: Some, + default: Some(Type::Tuple), + }), + ], + gt_token: Some, + }, + self_ty: Type::Tuple, + } + "#); +} + +#[test] +fn test_impl_trait_trailing_plus() { + let tokens = quote! { + fn f() -> impl Sized + {} + }; + + snapshot!(tokens as Item, @r#" + Item::Fn { + vis: Visibility::Inherited, + sig: Signature { + ident: "f", + generics: Generics, + output: ReturnType::Type( + Type::ImplTrait { + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Sized", + }, + ], + }, + }), + Token![+], + ], + }, + ), + }, + block: Block { + stmts: [], + }, + } + "#); +} + +// Regression test for issue https://github.com/dtolnay/syn/issues/1967 +#[test] +fn test_nested_receiver_classification() { + let tokens = quote! { + fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + self: foo<{ fn foo( + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + )}> + ) {} + }; + + let _ = syn::parse2::<syn::File>(tokens); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_lit.rs b/third_party/crates/syn-2.0.119/tests/test_lit.rs new file mode 100644 index 0000000..f2367b4 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_lit.rs @@ -0,0 +1,335 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::float_cmp, + clippy::needless_lifetimes, + clippy::needless_raw_string_hashes, + clippy::non_ascii_literal, + clippy::single_match_else, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::{Delimiter, Group, Literal, Span, TokenStream, TokenTree}; +use quote::ToTokens; +use std::ffi::CStr; +use std::str::FromStr; +use syn::{Lit, LitFloat, LitInt, LitStr}; + +#[track_caller] +fn lit(s: &str) -> Lit { + let mut tokens = TokenStream::from_str(s).unwrap().into_iter(); + match tokens.next().unwrap() { + TokenTree::Literal(lit) => { + assert!(tokens.next().is_none()); + Lit::new(lit) + } + wrong => panic!("{:?}", wrong), + } +} + +#[test] +fn strings() { + #[track_caller] + fn test_string(s: &str, value: &str) { + let s = s.trim(); + match lit(s) { + Lit::Str(lit) => { + assert_eq!(lit.value(), value); + let again = lit.into_token_stream().to_string(); + if again != s { + test_string(&again, value); + } + } + wrong => panic!("{:?}", wrong), + } + } + + test_string(r#" "" "#, ""); + test_string(r#" "a" "#, "a"); + test_string(r#" "\n" "#, "\n"); + test_string(r#" "\r" "#, "\r"); + test_string(r#" "\t" "#, "\t"); + test_string(r#" "🐕" "#, "🐕"); // NOTE: This is an emoji + test_string(r#" "\"" "#, "\""); + test_string(r#" "'" "#, "'"); + test_string(r#" "\u{1F415}" "#, "\u{1F415}"); + test_string(r#" "\u{1_2__3_}" "#, "\u{123}"); + test_string( + "\"contains\nnewlines\\\nescaped newlines\"", + "contains\nnewlinesescaped newlines", + ); + test_string( + "\"escaped newline\\\n \x0C unsupported whitespace\"", + "escaped newline\x0C unsupported whitespace", + ); + test_string("r\"raw\nstring\\\nhere\"", "raw\nstring\\\nhere"); + test_string("\"...\"q", "..."); + test_string("r\"...\"q", "..."); + test_string("r##\"...\"##q", "..."); +} + +#[test] +fn byte_strings() { + #[track_caller] + fn test_byte_string(s: &str, value: &[u8]) { + let s = s.trim(); + match lit(s) { + Lit::ByteStr(lit) => { + assert_eq!(lit.value(), value); + let again = lit.into_token_stream().to_string(); + if again != s { + test_byte_string(&again, value); + } + } + wrong => panic!("{:?}", wrong), + } + } + + test_byte_string(r#" b"" "#, b""); + test_byte_string(r#" b"a" "#, b"a"); + test_byte_string(r#" b"\n" "#, b"\n"); + test_byte_string(r#" b"\r" "#, b"\r"); + test_byte_string(r#" b"\t" "#, b"\t"); + test_byte_string(r#" b"\"" "#, b"\""); + test_byte_string(r#" b"'" "#, b"'"); + test_byte_string( + "b\"contains\nnewlines\\\nescaped newlines\"", + b"contains\nnewlinesescaped newlines", + ); + test_byte_string("br\"raw\nstring\\\nhere\"", b"raw\nstring\\\nhere"); + test_byte_string("b\"...\"q", b"..."); + test_byte_string("br\"...\"q", b"..."); + test_byte_string("br##\"...\"##q", b"..."); +} + +#[test] +fn c_strings() { + #[track_caller] + fn test_c_string(s: &str, value: &CStr) { + let s = s.trim(); + match lit(s) { + Lit::CStr(lit) => { + assert_eq!(*lit.value(), *value); + let again = lit.into_token_stream().to_string(); + if again != s { + test_c_string(&again, value); + } + } + wrong => panic!("{:?}", wrong), + } + } + + test_c_string(r#" c"" "#, c""); + test_c_string(r#" c"a" "#, c"a"); + test_c_string(r#" c"\n" "#, c"\n"); + test_c_string(r#" c"\r" "#, c"\r"); + test_c_string(r#" c"\t" "#, c"\t"); + test_c_string(r#" c"\\" "#, c"\\"); + test_c_string(r#" c"\'" "#, c"'"); + test_c_string(r#" c"\"" "#, c"\""); + test_c_string( + "c\"contains\nnewlines\\\nescaped newlines\"", + c"contains\nnewlinesescaped newlines", + ); + test_c_string("cr\"raw\nstring\\\nhere\"", c"raw\nstring\\\nhere"); + test_c_string("c\"...\"q", c"..."); + test_c_string("cr\"...\"", c"..."); + test_c_string("cr##\"...\"##", c"..."); + test_c_string( + r#" c"hello\x80我叫\u{1F980}" "#, // from the RFC + c"hello\x80我叫\u{1F980}", + ); +} + +#[test] +fn bytes() { + #[track_caller] + fn test_byte(s: &str, value: u8) { + let s = s.trim(); + match lit(s) { + Lit::Byte(lit) => { + assert_eq!(lit.value(), value); + let again = lit.into_token_stream().to_string(); + assert_eq!(again, s); + } + wrong => panic!("{:?}", wrong), + } + } + + test_byte(r#" b'a' "#, b'a'); + test_byte(r#" b'\n' "#, b'\n'); + test_byte(r#" b'\r' "#, b'\r'); + test_byte(r#" b'\t' "#, b'\t'); + test_byte(r#" b'\'' "#, b'\''); + test_byte(r#" b'"' "#, b'"'); + test_byte(r#" b'a'q "#, b'a'); +} + +#[test] +fn chars() { + #[track_caller] + fn test_char(s: &str, value: char) { + let s = s.trim(); + match lit(s) { + Lit::Char(lit) => { + assert_eq!(lit.value(), value); + let again = lit.into_token_stream().to_string(); + if again != s { + test_char(&again, value); + } + } + wrong => panic!("{:?}", wrong), + } + } + + test_char(r#" 'a' "#, 'a'); + test_char(r#" '\n' "#, '\n'); + test_char(r#" '\r' "#, '\r'); + test_char(r#" '\t' "#, '\t'); + test_char(r#" '🐕' "#, '🐕'); // NOTE: This is an emoji + test_char(r#" '\'' "#, '\''); + test_char(r#" '"' "#, '"'); + test_char(r#" '\u{1F415}' "#, '\u{1F415}'); + test_char(r#" 'a'q "#, 'a'); +} + +#[test] +fn ints() { + #[track_caller] + fn test_int(s: &str, value: u64, suffix: &str) { + match lit(s) { + Lit::Int(lit) => { + assert_eq!(lit.base10_digits().parse::<u64>().unwrap(), value); + assert_eq!(lit.suffix(), suffix); + let again = lit.into_token_stream().to_string(); + if again != s { + test_int(&again, value, suffix); + } + } + wrong => panic!("{:?}", wrong), + } + } + + test_int("5", 5, ""); + test_int("5u32", 5, "u32"); + test_int("0E", 0, "E"); + test_int("0ECMA", 0, "ECMA"); + test_int("0o0A", 0, "A"); + test_int("5_0", 50, ""); + test_int("5_____0_____", 50, ""); + test_int("0x7f", 127, ""); + test_int("0x7F", 127, ""); + test_int("0b1001", 9, ""); + test_int("0o73", 59, ""); + test_int("0x7Fu8", 127, "u8"); + test_int("0b1001i8", 9, "i8"); + test_int("0o73u32", 59, "u32"); + test_int("0x__7___f_", 127, ""); + test_int("0x__7___F_", 127, ""); + test_int("0b_1_0__01", 9, ""); + test_int("0o_7__3", 59, ""); + test_int("0x_7F__u8", 127, "u8"); + test_int("0b__10__0_1i8", 9, "i8"); + test_int("0o__7__________________3u32", 59, "u32"); + test_int("0e1\u{5c5}", 0, "e1\u{5c5}"); +} + +#[test] +fn floats() { + #[track_caller] + fn test_float(s: &str, value: f64, suffix: &str) { + match lit(s) { + Lit::Float(lit) => { + assert_eq!(lit.base10_digits().parse::<f64>().unwrap(), value); + assert_eq!(lit.suffix(), suffix); + let again = lit.into_token_stream().to_string(); + if again != s { + test_float(&again, value, suffix); + } + } + wrong => panic!("{:?}", wrong), + } + } + + test_float("5.5", 5.5, ""); + test_float("5.5E12", 5.5e12, ""); + test_float("5.5e12", 5.5e12, ""); + test_float("1.0__3e-12", 1.03e-12, ""); + test_float("1.03e+12", 1.03e12, ""); + test_float("9e99e99", 9e99, "e99"); + test_float("1e_0", 1.0, ""); + test_float("0.0ECMA", 0.0, "ECMA"); +} + +#[test] +fn negative() { + let span = Span::call_site(); + assert_eq!("-1", LitInt::new("-1", span).to_string()); + assert_eq!("-1i8", LitInt::new("-1i8", span).to_string()); + assert_eq!("-1i16", LitInt::new("-1i16", span).to_string()); + assert_eq!("-1i32", LitInt::new("-1i32", span).to_string()); + assert_eq!("-1i64", LitInt::new("-1i64", span).to_string()); + assert_eq!("-1.5", LitFloat::new("-1.5", span).to_string()); + assert_eq!("-1.5f32", LitFloat::new("-1.5f32", span).to_string()); + assert_eq!("-1.5f64", LitFloat::new("-1.5f64", span).to_string()); +} + +#[test] +fn suffix() { + #[track_caller] + fn get_suffix(token: &str) -> String { + let lit = syn::parse_str::<Lit>(token).unwrap(); + match lit { + Lit::Str(lit) => lit.suffix().to_owned(), + Lit::ByteStr(lit) => lit.suffix().to_owned(), + Lit::CStr(lit) => lit.suffix().to_owned(), + Lit::Byte(lit) => lit.suffix().to_owned(), + Lit::Char(lit) => lit.suffix().to_owned(), + Lit::Int(lit) => lit.suffix().to_owned(), + Lit::Float(lit) => lit.suffix().to_owned(), + _ => unimplemented!(), + } + } + + assert_eq!(get_suffix("\"\"s"), "s"); + assert_eq!(get_suffix("r\"\"r"), "r"); + assert_eq!(get_suffix("r#\"\"#r"), "r"); + assert_eq!(get_suffix("b\"\"b"), "b"); + assert_eq!(get_suffix("br\"\"br"), "br"); + assert_eq!(get_suffix("br#\"\"#br"), "br"); + assert_eq!(get_suffix("c\"\"c"), "c"); + assert_eq!(get_suffix("cr\"\"cr"), "cr"); + assert_eq!(get_suffix("cr#\"\"#cr"), "cr"); + assert_eq!(get_suffix("'c'c"), "c"); + assert_eq!(get_suffix("b'b'b"), "b"); + assert_eq!(get_suffix("1i32"), "i32"); + assert_eq!(get_suffix("1_i32"), "i32"); + assert_eq!(get_suffix("1.0f32"), "f32"); + assert_eq!(get_suffix("1.0_f32"), "f32"); +} + +#[test] +fn test_deep_group_empty() { + let tokens = TokenStream::from_iter([TokenTree::Group(Group::new( + Delimiter::None, + TokenStream::from_iter([TokenTree::Group(Group::new( + Delimiter::None, + TokenStream::from_iter([TokenTree::Literal(Literal::string("hi"))]), + ))]), + ))]); + + snapshot!(tokens as Lit, @r#""hi""# ); +} + +#[test] +fn test_error() { + let err = syn::parse_str::<LitStr>("...").unwrap_err(); + assert_eq!("expected string literal", err.to_string()); + + let err = syn::parse_str::<LitStr>("5").unwrap_err(); + assert_eq!("expected string literal", err.to_string()); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_meta.rs b/third_party/crates/syn-2.0.119/tests/test_meta.rs new file mode 100644 index 0000000..4e1f9ca --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_meta.rs @@ -0,0 +1,180 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::shadow_unrelated, + clippy::too_many_lines, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use quote::quote; +use syn::parse::{ParseStream, Parser as _, Result}; +use syn::{Meta, MetaList, MetaNameValue, Token}; + +#[test] +fn test_parse_meta_item_word() { + let input = "hello"; + + snapshot!(input as Meta, @r#" + Meta::Path { + segments: [ + PathSegment { + ident: "hello", + }, + ], + } + "#); +} + +#[test] +fn test_parse_meta_name_value() { + let input = "foo = 5"; + let (inner, meta) = (input, input); + + snapshot!(inner as MetaNameValue, @r#" + MetaNameValue { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + value: Expr::Lit { + lit: 5, + }, + } + "#); + + snapshot!(meta as Meta, @r#" + Meta::NameValue { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + value: Expr::Lit { + lit: 5, + }, + } + "#); + + assert_eq!(meta, Meta::NameValue(inner)); +} + +#[test] +fn test_parse_meta_item_list_lit() { + let input = "foo(5)"; + let (inner, meta) = (input, input); + + snapshot!(inner as MetaList, @r#" + MetaList { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`5`), + } + "#); + + snapshot!(meta as Meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`5`), + } + "#); + + assert_eq!(meta, Meta::List(inner)); +} + +#[test] +fn test_parse_meta_item_multiple() { + let input = "foo(word, name = 5, list(name2 = 6), word2)"; + let (inner, meta) = (input, input); + + snapshot!(inner as MetaList, @r#" + MetaList { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`word , name = 5 , list (name2 = 6) , word2`), + } + "#); + + snapshot!(meta as Meta, @r#" + Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "foo", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`word , name = 5 , list (name2 = 6) , word2`), + } + "#); + + assert_eq!(meta, Meta::List(inner)); +} + +#[test] +fn test_parse_path() { + let input = "::serde::Serialize"; + snapshot!(input as Meta, @r#" + Meta::Path { + leading_colon: Some, + segments: [ + PathSegment { + ident: "serde", + }, + Token![::], + PathSegment { + ident: "Serialize", + }, + ], + } + "#); +} + +#[test] +fn test_fat_arrow_after_meta() { + fn parse(input: ParseStream) -> Result<()> { + while !input.is_empty() { + let _: Meta = input.parse()?; + let _: Token![=>] = input.parse()?; + let brace; + syn::braced!(brace in input); + } + Ok(()) + } + + let input = quote! { + target_os = "linux" => {} + windows => {} + }; + + parse.parse2(input).unwrap(); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_parse_buffer.rs b/third_party/crates/syn-2.0.119/tests/test_parse_buffer.rs new file mode 100644 index 0000000..f756578 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_parse_buffer.rs @@ -0,0 +1,104 @@ +#![allow(clippy::non_ascii_literal)] + +use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, TokenStream, TokenTree}; +use std::panic; +use syn::parse::discouraged::Speculative as _; +use syn::parse::{Parse, ParseStream, Parser, Result}; +use syn::{parenthesized, Token}; + +#[test] +#[should_panic(expected = "fork was not derived from the advancing parse stream")] +fn smuggled_speculative_cursor_between_sources() { + struct BreakRules; + impl Parse for BreakRules { + fn parse(input1: ParseStream) -> Result<Self> { + let nested = |input2: ParseStream| { + input1.advance_to(input2); + Ok(Self) + }; + nested.parse_str("") + } + } + + syn::parse_str::<BreakRules>("").unwrap(); +} + +#[test] +#[should_panic(expected = "fork was not derived from the advancing parse stream")] +fn smuggled_speculative_cursor_between_brackets() { + struct BreakRules; + impl Parse for BreakRules { + fn parse(input: ParseStream) -> Result<Self> { + let a; + let b; + parenthesized!(a in input); + parenthesized!(b in input); + a.advance_to(&b); + Ok(Self) + } + } + + syn::parse_str::<BreakRules>("()()").unwrap(); +} + +#[test] +#[should_panic(expected = "fork was not derived from the advancing parse stream")] +fn smuggled_speculative_cursor_into_brackets() { + struct BreakRules; + impl Parse for BreakRules { + fn parse(input: ParseStream) -> Result<Self> { + let a; + parenthesized!(a in input); + input.advance_to(&a); + Ok(Self) + } + } + + syn::parse_str::<BreakRules>("()").unwrap(); +} + +#[test] +fn trailing_empty_none_group() { + fn parse(input: ParseStream) -> Result<()> { + input.parse::<Token![+]>()?; + + let content; + parenthesized!(content in input); + content.parse::<Token![+]>()?; + + Ok(()) + } + + // `+ ( + «∅ ∅» ) «∅ «∅ ∅» ∅»` + let tokens = TokenStream::from_iter([ + TokenTree::Punct(Punct::new('+', Spacing::Alone)), + TokenTree::Group(Group::new( + Delimiter::Parenthesis, + TokenStream::from_iter([ + TokenTree::Punct(Punct::new('+', Spacing::Alone)), + TokenTree::Group(Group::new(Delimiter::None, TokenStream::new())), + ]), + )), + TokenTree::Group(Group::new(Delimiter::None, TokenStream::new())), + TokenTree::Group(Group::new( + Delimiter::None, + TokenStream::from_iter([TokenTree::Group(Group::new( + Delimiter::None, + TokenStream::new(), + ))]), + )), + ]); + + parse.parse2(tokens).unwrap(); +} + +#[test] +#[cfg_attr(miri, ignore)] // https://github.com/rust-lang/miri/issues/4793 +fn test_unwind_safe() { + fn parse(input: ParseStream) -> Result<Ident> { + let thread_result = panic::catch_unwind(|| input.parse()); + thread_result.unwrap() + } + + parse.parse_str("throw").unwrap(); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_parse_quote.rs b/third_party/crates/syn-2.0.119/tests/test_parse_quote.rs new file mode 100644 index 0000000..600870b --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_parse_quote.rs @@ -0,0 +1,172 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use syn::punctuated::Punctuated; +use syn::{parse_quote, Attribute, Field, Lit, Pat, Stmt, Token}; + +#[test] +fn test_attribute() { + let attr: Attribute = parse_quote!(#[test]); + snapshot!(attr, @r#" + Attribute { + style: AttrStyle::Outer, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "test", + }, + ], + }, + } + "#); + + let attr: Attribute = parse_quote!(#![no_std]); + snapshot!(attr, @r#" + Attribute { + style: AttrStyle::Inner, + meta: Meta::Path { + segments: [ + PathSegment { + ident: "no_std", + }, + ], + }, + } + "#); +} + +#[test] +fn test_field() { + let field: Field = parse_quote!(pub enabled: bool); + snapshot!(field, @r#" + Field { + vis: Visibility::Public, + ident: Some("enabled"), + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "bool", + }, + ], + }, + }, + } + "#); + + let field: Field = parse_quote!(primitive::bool); + snapshot!(field, @r#" + Field { + vis: Visibility::Inherited, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "primitive", + }, + Token![::], + PathSegment { + ident: "bool", + }, + ], + }, + }, + } + "#); +} + +#[test] +fn test_pat() { + let pat: Pat = parse_quote!(Some(false) | None); + snapshot!(&pat, @r#" + Pat::Or { + cases: [ + Pat::TupleStruct { + path: Path { + segments: [ + PathSegment { + ident: "Some", + }, + ], + }, + elems: [ + Pat::Lit(ExprLit { + lit: Lit::Bool { + value: false, + }, + }), + ], + }, + Token![|], + Pat::Ident { + ident: "None", + }, + ], + } + "#); + + let boxed_pat: Box<Pat> = parse_quote!(Some(false) | None); + assert_eq!(*boxed_pat, pat); +} + +#[test] +fn test_punctuated() { + let punctuated: Punctuated<Lit, Token![|]> = parse_quote!(true | true); + snapshot!(punctuated, @r#" + [ + Lit::Bool { + value: true, + }, + Token![|], + Lit::Bool { + value: true, + }, + ] + "#); + + let punctuated: Punctuated<Lit, Token![|]> = parse_quote!(true | true |); + snapshot!(punctuated, @r#" + [ + Lit::Bool { + value: true, + }, + Token![|], + Lit::Bool { + value: true, + }, + Token![|], + ] + "#); +} + +#[test] +fn test_vec_stmt() { + let stmts: Vec<Stmt> = parse_quote! { + let _; + true + }; + snapshot!(stmts, @r#" + [ + Stmt::Local { + pat: Pat::Wild, + }, + Stmt::Expr( + Expr::Lit { + lit: Lit::Bool { + value: true, + }, + }, + None, + ), + ] + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_parse_stream.rs b/third_party/crates/syn-2.0.119/tests/test_parse_stream.rs new file mode 100644 index 0000000..a650fc8 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_parse_stream.rs @@ -0,0 +1,187 @@ +#![allow(clippy::items_after_statements, clippy::let_underscore_untyped)] + +use proc_macro2::{Delimiter, Group, Punct, Spacing, Span, TokenStream, TokenTree}; +use quote::quote; +use syn::ext::IdentExt as _; +use syn::parse::discouraged::AnyDelimiter; +use syn::parse::{ParseStream, Parser as _, Result}; +use syn::{parenthesized, token, Ident, Lifetime, Token}; + +#[test] +fn test_peek_punct() { + let tokens = quote!(+= + =); + + fn assert(input: ParseStream) -> Result<()> { + assert!(input.peek(Token![+])); + assert!(input.peek(Token![+=])); + + let _: Token![+] = input.parse()?; + + assert!(input.peek(Token![=])); + assert!(!input.peek(Token![==])); + assert!(!input.peek(Token![+])); + + let _: Token![=] = input.parse()?; + + assert!(input.peek(Token![+])); + assert!(!input.peek(Token![+=])); + + let _: Token![+] = input.parse()?; + let _: Token![=] = input.parse()?; + Ok(()) + } + + assert.parse2(tokens).unwrap(); +} + +#[test] +fn test_peek_lifetime() { + // 'static ; + let tokens = TokenStream::from_iter([ + TokenTree::Punct(Punct::new('\'', Spacing::Joint)), + TokenTree::Ident(Ident::new("static", Span::call_site())), + TokenTree::Punct(Punct::new(';', Spacing::Alone)), + ]); + + fn assert(input: ParseStream) -> Result<()> { + assert!(input.peek(Lifetime)); + assert!(input.peek2(Token![;])); + assert!(!input.peek2(Token![static])); + + let _: Lifetime = input.parse()?; + + assert!(input.peek(Token![;])); + + let _: Token![;] = input.parse()?; + Ok(()) + } + + assert.parse2(tokens).unwrap(); +} + +#[test] +fn test_peek_not_lifetime() { + // ' static + let tokens = TokenStream::from_iter([ + TokenTree::Punct(Punct::new('\'', Spacing::Alone)), + TokenTree::Ident(Ident::new("static", Span::call_site())), + ]); + + fn assert(input: ParseStream) -> Result<()> { + assert!(!input.peek(Lifetime)); + assert!(input.parse::<Option<Punct>>()?.is_none()); + + let _: TokenTree = input.parse()?; + + assert!(input.peek(Token![static])); + + let _: Token![static] = input.parse()?; + Ok(()) + } + + assert.parse2(tokens).unwrap(); +} + +#[test] +fn test_peek_ident() { + let tokens = quote!(static var); + + fn assert(input: ParseStream) -> Result<()> { + assert!(!input.peek(Ident)); + assert!(input.peek(Ident::peek_any)); + assert!(input.peek(Token![static])); + + let _: Token![static] = input.parse()?; + + assert!(input.peek(Ident)); + assert!(input.peek(Ident::peek_any)); + + let _: Ident = input.parse()?; + Ok(()) + } + + assert.parse2(tokens).unwrap(); +} + +#[test] +fn test_peek_groups() { + // pub ( :: ) «∅ ! = ∅» static + let tokens = TokenStream::from_iter([ + TokenTree::Ident(Ident::new("pub", Span::call_site())), + TokenTree::Group(Group::new( + Delimiter::Parenthesis, + TokenStream::from_iter([ + TokenTree::Punct(Punct::new(':', Spacing::Joint)), + TokenTree::Punct(Punct::new(':', Spacing::Alone)), + ]), + )), + TokenTree::Group(Group::new( + Delimiter::None, + TokenStream::from_iter([ + TokenTree::Punct(Punct::new('!', Spacing::Alone)), + TokenTree::Punct(Punct::new('=', Spacing::Alone)), + ]), + )), + TokenTree::Ident(Ident::new("static", Span::call_site())), + ]); + + fn assert(input: ParseStream) -> Result<()> { + assert!(input.peek2(token::Paren)); + assert!(input.peek3(token::Group)); + assert!(input.peek3(Token![!])); + + let _: Token![pub] = input.parse()?; + + assert!(input.peek(token::Paren)); + assert!(!input.peek(Token![::])); + assert!(!input.peek2(Token![::])); + assert!(input.peek2(Token![!])); + assert!(input.peek2(token::Group)); + assert!(input.peek3(Token![=])); + assert!(!input.peek3(Token![static])); + + let content; + parenthesized!(content in input); + + assert!(content.peek(Token![::])); + assert!(content.peek2(Token![:])); + assert!(!content.peek3(token::Group)); + assert!(!content.peek3(Token![!])); + + assert!(input.peek(token::Group)); + assert!(input.peek(Token![!])); + + let _: Token![::] = content.parse()?; + + assert!(input.peek(token::Group)); + assert!(input.peek(Token![!])); + assert!(input.peek2(Token![=])); + assert!(input.peek3(Token![static])); + assert!(!input.peek2(Token![static])); + + let implicit = input.fork(); + let explicit = input.fork(); + + let _: Token![!] = implicit.parse()?; + assert!(implicit.peek(Token![=])); + assert!(implicit.peek2(Token![static])); + let _: Token![=] = implicit.parse()?; + assert!(implicit.peek(Token![static])); + + let (delimiter, _span, grouped) = explicit.parse_any_delimiter()?; + assert_eq!(delimiter, Delimiter::None); + assert!(grouped.peek(Token![!])); + assert!(grouped.peek2(Token![=])); + assert!(!grouped.peek3(Token![static])); + let _: Token![!] = grouped.parse()?; + assert!(grouped.peek(Token![=])); + assert!(!grouped.peek2(Token![static])); + let _: Token![=] = grouped.parse()?; + assert!(!grouped.peek(Token![static])); + + let _: TokenStream = input.parse()?; + Ok(()) + } + + assert.parse2(tokens).unwrap(); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_pat.rs b/third_party/crates/syn-2.0.119/tests/test_pat.rs new file mode 100644 index 0000000..f778928 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_pat.rs @@ -0,0 +1,158 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::{Delimiter, Group, TokenStream, TokenTree}; +use quote::{quote, ToTokens as _}; +use syn::parse::Parser; +use syn::punctuated::Punctuated; +use syn::{parse_quote, token, Item, Pat, PatTuple, Stmt, Token}; + +#[test] +fn test_pat_ident() { + match Pat::parse_single.parse2(quote!(self)).unwrap() { + Pat::Ident(_) => (), + value => panic!("expected PatIdent, got {:?}", value), + } +} + +#[test] +fn test_pat_path() { + match Pat::parse_single.parse2(quote!(self::CONST)).unwrap() { + Pat::Path(_) => (), + value => panic!("expected PatPath, got {:?}", value), + } +} + +#[test] +fn test_leading_vert() { + // https://github.com/rust-lang/rust/blob/1.43.0/src/test/ui/or-patterns/remove-leading-vert.rs + + syn::parse_str::<Item>("fn f() {}").unwrap(); + syn::parse_str::<Item>("fn fun1(| A: E) {}").unwrap_err(); + syn::parse_str::<Item>("fn fun2(|| A: E) {}").unwrap_err(); + + syn::parse_str::<Stmt>("let | () = ();").unwrap_err(); + syn::parse_str::<Stmt>("let (| A): E;").unwrap(); + syn::parse_str::<Stmt>("let (|| A): (E);").unwrap_err(); + syn::parse_str::<Stmt>("let (| A,): (E,);").unwrap(); + syn::parse_str::<Stmt>("let [| A]: [E; 1];").unwrap(); + syn::parse_str::<Stmt>("let [|| A]: [E; 1];").unwrap_err(); + syn::parse_str::<Stmt>("let TS(| A): TS;").unwrap(); + syn::parse_str::<Stmt>("let TS(|| A): TS;").unwrap_err(); + syn::parse_str::<Stmt>("let NS { f: | A }: NS;").unwrap(); + syn::parse_str::<Stmt>("let NS { f: || A }: NS;").unwrap_err(); +} + +#[test] +fn test_group() { + let group = Group::new(Delimiter::None, quote!(Some(_))); + let tokens = TokenStream::from_iter([TokenTree::Group(group)]); + let pat = Pat::parse_single.parse2(tokens).unwrap(); + + snapshot!(pat, @r#" + Pat::TupleStruct { + path: Path { + segments: [ + PathSegment { + ident: "Some", + }, + ], + }, + elems: [ + Pat::Wild, + ], + } + "#); +} + +#[test] +fn test_ranges() { + Pat::parse_single.parse_str("..").unwrap(); + Pat::parse_single.parse_str("..hi").unwrap(); + Pat::parse_single.parse_str("lo..").unwrap(); + Pat::parse_single.parse_str("lo..hi").unwrap(); + + Pat::parse_single.parse_str("..=").unwrap_err(); + Pat::parse_single.parse_str("..=hi").unwrap(); + Pat::parse_single.parse_str("lo..=").unwrap_err(); + Pat::parse_single.parse_str("lo..=hi").unwrap(); + + Pat::parse_single.parse_str("...").unwrap_err(); + Pat::parse_single.parse_str("...hi").unwrap_err(); + Pat::parse_single.parse_str("lo...").unwrap_err(); + Pat::parse_single.parse_str("lo...hi").unwrap(); + + Pat::parse_single.parse_str("[lo..]").unwrap_err(); + Pat::parse_single.parse_str("[..=hi]").unwrap_err(); + Pat::parse_single.parse_str("[(lo..)]").unwrap(); + Pat::parse_single.parse_str("[(..=hi)]").unwrap(); + Pat::parse_single.parse_str("[lo..=hi]").unwrap(); + + Pat::parse_single.parse_str("[_, lo.., _]").unwrap_err(); + Pat::parse_single.parse_str("[_, ..=hi, _]").unwrap_err(); + Pat::parse_single.parse_str("[_, (lo..), _]").unwrap(); + Pat::parse_single.parse_str("[_, (..=hi), _]").unwrap(); + Pat::parse_single.parse_str("[_, lo..=hi, _]").unwrap(); +} + +#[test] +fn test_tuple_comma() { + let mut expr = PatTuple { + attrs: Vec::new(), + paren_token: token::Paren::default(), + elems: Punctuated::new(), + }; + snapshot!(expr.to_token_stream() as Pat, @"Pat::Tuple"); + + expr.elems.push_value(parse_quote!(_)); + // Must not parse to Pat::Paren + snapshot!(expr.to_token_stream() as Pat, @r#" + Pat::Tuple { + elems: [ + Pat::Wild, + Token![,], + ], + } + "#); + + expr.elems.push_punct(<Token![,]>::default()); + snapshot!(expr.to_token_stream() as Pat, @r#" + Pat::Tuple { + elems: [ + Pat::Wild, + Token![,], + ], + } + "#); + + expr.elems.push_value(parse_quote!(_)); + snapshot!(expr.to_token_stream() as Pat, @r#" + Pat::Tuple { + elems: [ + Pat::Wild, + Token![,], + Pat::Wild, + ], + } + "#); + + expr.elems.push_punct(<Token![,]>::default()); + snapshot!(expr.to_token_stream() as Pat, @r#" + Pat::Tuple { + elems: [ + Pat::Wild, + Token![,], + Pat::Wild, + Token![,], + ], + } + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_path.rs b/third_party/crates/syn-2.0.119/tests/test_path.rs new file mode 100644 index 0000000..7f9e515 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_path.rs @@ -0,0 +1,116 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; +use quote::{quote, ToTokens}; +use syn::{parse_quote, Expr, Type, TypePath}; + +#[test] +fn parse_interpolated_leading_component() { + // mimics the token stream corresponding to `$mod::rest` + let tokens = TokenStream::from_iter([ + TokenTree::Group(Group::new(Delimiter::None, quote! { first })), + TokenTree::Punct(Punct::new(':', Spacing::Joint)), + TokenTree::Punct(Punct::new(':', Spacing::Alone)), + TokenTree::Ident(Ident::new("rest", Span::call_site())), + ]); + + snapshot!(tokens.clone() as Expr, @r#" + Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "first", + }, + Token![::], + PathSegment { + ident: "rest", + }, + ], + }, + } + "#); + + snapshot!(tokens as Type, @r#" + Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "first", + }, + Token![::], + PathSegment { + ident: "rest", + }, + ], + }, + } + "#); +} + +#[test] +fn print_incomplete_qpath() { + // qpath with `as` token + let mut ty: TypePath = parse_quote!(<Self as A>::Q); + snapshot!(ty.to_token_stream(), @"TokenStream(`< Self as A > :: Q`)"); + assert!(ty.path.segments.pop().is_some()); + snapshot!(ty.to_token_stream(), @"TokenStream(`< Self as A > ::`)"); + assert!(ty.path.segments.pop().is_some()); + snapshot!(ty.to_token_stream(), @"TokenStream(`< Self >`)"); + assert!(ty.path.segments.pop().is_none()); + + // qpath without `as` token + let mut ty: TypePath = parse_quote!(<Self>::A::B); + snapshot!(ty.to_token_stream(), @"TokenStream(`< Self > :: A :: B`)"); + assert!(ty.path.segments.pop().is_some()); + snapshot!(ty.to_token_stream(), @"TokenStream(`< Self > :: A ::`)"); + assert!(ty.path.segments.pop().is_some()); + snapshot!(ty.to_token_stream(), @"TokenStream(`< Self > ::`)"); + assert!(ty.path.segments.pop().is_none()); + + // normal path + let mut ty: TypePath = parse_quote!(Self::A::B); + snapshot!(ty.to_token_stream(), @"TokenStream(`Self :: A :: B`)"); + assert!(ty.path.segments.pop().is_some()); + snapshot!(ty.to_token_stream(), @"TokenStream(`Self :: A ::`)"); + assert!(ty.path.segments.pop().is_some()); + snapshot!(ty.to_token_stream(), @"TokenStream(`Self ::`)"); + assert!(ty.path.segments.pop().is_some()); + snapshot!(ty.to_token_stream(), @"TokenStream(``)"); + assert!(ty.path.segments.pop().is_none()); +} + +#[test] +fn parse_parenthesized_path_arguments_with_disambiguator() { + #[rustfmt::skip] + let tokens = quote!(dyn FnOnce::() -> !); + snapshot!(tokens as Type, @r#" + Type::TraitObject { + dyn_token: Some, + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "FnOnce", + arguments: PathArguments::Parenthesized { + output: ReturnType::Type( + Type::Never, + ), + }, + }, + ], + }, + }), + ], + } + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_precedence.rs b/third_party/crates/syn-2.0.119/tests/test_precedence.rs new file mode 100644 index 0000000..4950eec --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_precedence.rs @@ -0,0 +1,556 @@ +// This test does the following for every file in the rust-lang/rust repo: +// +// 1. Parse the file using syn into a syn::File. +// 2. Extract every syn::Expr from the file. +// 3. Print each expr to a string of source code. +// 4. Parse the source code using librustc_parse into a rustc_ast::Expr. +// 5. For both the syn::Expr and rustc_ast::Expr, crawl the syntax tree to +// insert parentheses surrounding every subexpression. +// 6. Serialize the fully parenthesized syn::Expr to a string of source code. +// 7. Parse the fully parenthesized source code using librustc_parse. +// 8. Compare the rustc_ast::Expr resulting from parenthesizing using rustc data +// structures vs syn data structures, ignoring spans. If they agree, rustc's +// parser and syn's parser have identical handling of expression precedence. + +#![cfg(not(syn_disable_nightly_tests))] +#![cfg(not(miri))] +#![recursion_limit = "1024"] +#![feature(rustc_private)] +#![allow( + clippy::blocks_in_conditions, + clippy::doc_markdown, + clippy::elidable_lifetime_names, + clippy::explicit_deref_methods, + clippy::let_underscore_untyped, + clippy::manual_assert, + clippy::manual_let_else, + clippy::match_like_matches_macro, + clippy::match_wildcard_for_single_variants, + clippy::needless_lifetimes, + clippy::too_many_lines, + clippy::uninlined_format_args, + clippy::unnecessary_box_returns +)] + +extern crate rustc_ast; +extern crate rustc_ast_pretty; +extern crate rustc_data_structures; +extern crate rustc_driver; +extern crate rustc_span; + +use crate::common::eq::SpanlessEq; +use crate::common::parse; +use quote::ToTokens; +use rustc_ast::ast; +use rustc_ast_pretty::pprust; +use rustc_span::edition::Edition; +use std::fs; +use std::mem; +use std::path::Path; +use std::process; +use std::sync::atomic::{AtomicUsize, Ordering}; +use syn::parse::Parser as _; + +#[macro_use] +mod macros; + +mod common; +mod repo; + +#[path = "../src/scan_expr.rs"] +mod scan_expr; + +#[test] +fn test_rustc_precedence() { + repo::rayon_init(); + repo::clone_rust(); + let abort_after = repo::abort_after(); + if abort_after == 0 { + panic!("skipping all precedence tests"); + } + + let passed = AtomicUsize::new(0); + let failed = AtomicUsize::new(0); + + repo::for_each_rust_file(|path| { + let content = fs::read_to_string(path).unwrap(); + + let (l_passed, l_failed) = match syn::parse_file(&content) { + Ok(file) => { + let edition = repo::edition(path).parse().unwrap(); + let exprs = collect_exprs(file); + let (l_passed, l_failed) = test_expressions(path, edition, exprs); + errorf!( + "=== {}: {} passed | {} failed\n", + path.display(), + l_passed, + l_failed, + ); + (l_passed, l_failed) + } + Err(msg) => { + errorf!("\nFAIL {} - syn failed to parse: {}\n", path.display(), msg); + (0, 1) + } + }; + + passed.fetch_add(l_passed, Ordering::Relaxed); + let prev_failed = failed.fetch_add(l_failed, Ordering::Relaxed); + + if prev_failed + l_failed >= abort_after { + process::exit(1); + } + }); + + let passed = passed.into_inner(); + let failed = failed.into_inner(); + + errorf!("\n===== Precedence Test Results =====\n"); + errorf!("{} passed | {} failed\n", passed, failed); + + if failed > 0 { + panic!("{} failures", failed); + } +} + +fn test_expressions(path: &Path, edition: Edition, exprs: Vec<syn::Expr>) -> (usize, usize) { + let mut passed = 0; + let mut failed = 0; + + rustc_span::create_session_if_not_set_then(edition, |_| { + for expr in exprs { + let expr_tokens = expr.to_token_stream(); + let source_code = expr_tokens.to_string(); + let librustc_ast = if let Some(e) = librustc_parse_and_rewrite(&source_code) { + e + } else { + failed += 1; + errorf!( + "\nFAIL {} - librustc failed to parse original\n", + path.display(), + ); + continue; + }; + + let syn_parenthesized_code = + syn_parenthesize(expr.clone()).to_token_stream().to_string(); + let syn_ast = if let Some(e) = parse::librustc_expr(&syn_parenthesized_code) { + e + } else { + failed += 1; + errorf!( + "\nFAIL {} - librustc failed to parse parenthesized\n", + path.display(), + ); + continue; + }; + + if !SpanlessEq::eq(&syn_ast, &librustc_ast) { + failed += 1; + let syn_pretty = pprust::expr_to_string(&syn_ast); + let librustc_pretty = pprust::expr_to_string(&librustc_ast); + errorf!( + "\nFAIL {}\n{}\nsyn != rustc\n{}\n", + path.display(), + syn_pretty, + librustc_pretty, + ); + continue; + } + + let expr_invisible = make_parens_invisible(expr); + let Ok(reparsed_expr_invisible) = syn::parse2(expr_invisible.to_token_stream()) else { + failed += 1; + errorf!( + "\nFAIL {} - syn failed to parse invisible delimiters\n{}\n", + path.display(), + source_code, + ); + continue; + }; + if expr_invisible != reparsed_expr_invisible { + failed += 1; + errorf!( + "\nFAIL {} - mismatch after parsing invisible delimiters\n{}\n", + path.display(), + source_code, + ); + continue; + } + + if scan_expr::scan_expr.parse2(expr_tokens).is_err() { + failed += 1; + errorf!( + "\nFAIL {} - failed to scan expr\n{}\n", + path.display(), + source_code, + ); + continue; + } + + passed += 1; + } + }); + + (passed, failed) +} + +fn librustc_parse_and_rewrite(input: &str) -> Option<Box<ast::Expr>> { + parse::librustc_expr(input).map(librustc_parenthesize) +} + +fn librustc_parenthesize(mut librustc_expr: Box<ast::Expr>) -> Box<ast::Expr> { + use rustc_ast::ast::{ + AssocItem, AssocItemKind, Attribute, BinOpKind, Block, BoundConstness, Expr, ExprField, + ExprKind, GenericArg, GenericBound, Local, LocalKind, Pat, PolyTraitRef, Stmt, StmtKind, + StructExpr, StructRest, TraitBoundModifiers, Ty, + }; + use rustc_ast::mut_visit::{walk_flat_map_assoc_item, MutVisitor}; + use rustc_ast::visit::{AssocCtxt, BoundKind}; + use rustc_data_structures::flat_map_in_place::FlatMapInPlace; + use rustc_data_structures::smallvec::SmallVec; + use rustc_data_structures::thin_vec::ThinVec; + use rustc_span::DUMMY_SP; + use std::ops::DerefMut; + + struct FullyParenthesize; + + fn contains_let_chain(expr: &Expr) -> bool { + match &expr.kind { + ExprKind::Let(..) => true, + ExprKind::Binary(binop, left, right) => { + binop.node == BinOpKind::And + && (contains_let_chain(left) || contains_let_chain(right)) + } + _ => false, + } + } + + fn flat_map_field<T: MutVisitor>(mut f: ExprField, vis: &mut T) -> Vec<ExprField> { + if f.is_shorthand { + noop_visit_expr(&mut f.expr, vis); + } else { + vis.visit_expr(&mut f.expr); + } + vec![f] + } + + fn flat_map_stmt<T: MutVisitor>(stmt: Stmt, vis: &mut T) -> Vec<Stmt> { + let kind = match stmt.kind { + // Don't wrap toplevel expressions in statements. + StmtKind::Expr(mut e) => { + noop_visit_expr(&mut e, vis); + StmtKind::Expr(e) + } + StmtKind::Semi(mut e) => { + noop_visit_expr(&mut e, vis); + StmtKind::Semi(e) + } + s => s, + }; + + vec![Stmt { kind, ..stmt }] + } + + fn noop_visit_expr<T: MutVisitor>(e: &mut Expr, vis: &mut T) { + match &mut e.kind { + ExprKind::Become(..) => {} + ExprKind::Struct(expr) => { + let StructExpr { + qself, + path, + fields, + rest, + } = expr.deref_mut(); + if let Some(qself) = qself { + vis.visit_qself(qself); + } + vis.visit_path(path); + fields.flat_map_in_place(|field| flat_map_field(field, vis)); + if let StructRest::Base(rest) = rest { + vis.visit_expr(rest); + } + } + _ => rustc_ast::mut_visit::walk_expr(vis, e), + } + } + + impl MutVisitor for FullyParenthesize { + fn visit_expr(&mut self, e: &mut Expr) { + noop_visit_expr(e, self); + match e.kind { + ExprKind::Block(..) | ExprKind::If(..) | ExprKind::Let(..) => {} + ExprKind::Binary(..) if contains_let_chain(e) => {} + _ => { + let inner = mem::replace(e, Expr::dummy()); + *e = Expr { + id: ast::DUMMY_NODE_ID, + kind: ExprKind::Paren(Box::new(inner)), + span: DUMMY_SP, + attrs: ThinVec::new(), + tokens: None, + }; + } + } + } + + fn visit_generic_arg(&mut self, arg: &mut GenericArg) { + match arg { + GenericArg::Lifetime(_lifetime) => {} + GenericArg::Type(arg) => self.visit_ty(arg), + // Don't wrap unbraced const generic arg as that's invalid syntax. + GenericArg::Const(anon_const) => { + if let ExprKind::Block(..) = &mut anon_const.value.kind { + noop_visit_expr(&mut anon_const.value, self); + } + } + } + } + + fn visit_param_bound(&mut self, bound: &mut GenericBound, _ctxt: BoundKind) { + match bound { + GenericBound::Trait(PolyTraitRef { + modifiers: + TraitBoundModifiers { + constness: BoundConstness::Maybe(_), + .. + }, + .. + }) + | GenericBound::Outlives(..) + | GenericBound::Use(..) => {} + GenericBound::Trait(ty) => self.visit_poly_trait_ref(ty), + } + } + + fn visit_block(&mut self, block: &mut Block) { + self.visit_id(&mut block.id); + block + .stmts + .flat_map_in_place(|stmt| flat_map_stmt(stmt, self)); + self.visit_span(&mut block.span); + } + + fn visit_local(&mut self, local: &mut Local) { + match &mut local.kind { + LocalKind::Decl => {} + LocalKind::Init(init) => { + self.visit_expr(init); + } + LocalKind::InitElse(init, els) => { + self.visit_expr(init); + self.visit_block(els); + } + } + } + + fn flat_map_assoc_item( + &mut self, + item: Box<AssocItem>, + ctxt: AssocCtxt, + ) -> SmallVec<[Box<AssocItem>; 1]> { + match &item.kind { + AssocItemKind::Const(const_item) + if !const_item.generics.params.is_empty() + || !const_item.generics.where_clause.predicates.is_empty() => + { + SmallVec::from([item]) + } + _ => walk_flat_map_assoc_item(self, item, ctxt), + } + } + + // We don't want to look at expressions that might appear in patterns or + // types yet. We'll look into comparing those in the future. For now + // focus on expressions appearing in other places. + fn visit_pat(&mut self, pat: &mut Pat) { + let _ = pat; + } + + fn visit_ty(&mut self, ty: &mut Ty) { + let _ = ty; + } + + fn visit_attribute(&mut self, attr: &mut Attribute) { + let _ = attr; + } + } + + let mut folder = FullyParenthesize; + folder.visit_expr(&mut librustc_expr); + librustc_expr +} + +fn syn_parenthesize(syn_expr: syn::Expr) -> syn::Expr { + use syn::fold::{fold_expr, fold_generic_argument, Fold}; + use syn::{ + token, BinOp, Expr, ExprParen, GenericArgument, Lit, MetaNameValue, Pat, Stmt, Type, + }; + + struct FullyParenthesize; + + fn parenthesize(expr: Expr) -> Expr { + Expr::Paren(ExprParen { + attrs: Vec::new(), + expr: Box::new(expr), + paren_token: token::Paren::default(), + }) + } + + fn needs_paren(expr: &Expr) -> bool { + match expr { + Expr::Group(_) => unreachable!(), + Expr::If(_) | Expr::Unsafe(_) | Expr::Block(_) | Expr::Let(_) => false, + Expr::Binary(_) => !contains_let_chain(expr), + _ => true, + } + } + + fn contains_let_chain(expr: &Expr) -> bool { + match expr { + Expr::Let(_) => true, + Expr::Binary(expr) => { + matches!(expr.op, BinOp::And(_)) + && (contains_let_chain(&expr.left) || contains_let_chain(&expr.right)) + } + _ => false, + } + } + + impl Fold for FullyParenthesize { + fn fold_expr(&mut self, expr: Expr) -> Expr { + let needs_paren = needs_paren(&expr); + let folded = fold_expr(self, expr); + if needs_paren { + parenthesize(folded) + } else { + folded + } + } + + fn fold_generic_argument(&mut self, arg: GenericArgument) -> GenericArgument { + match arg { + GenericArgument::Const(arg) => GenericArgument::Const(match arg { + Expr::Block(_) => fold_expr(self, arg), + // Don't wrap unbraced const generic arg as that's invalid syntax. + _ => arg, + }), + _ => fold_generic_argument(self, arg), + } + } + + fn fold_stmt(&mut self, stmt: Stmt) -> Stmt { + match stmt { + // Don't wrap toplevel expressions in statements. + Stmt::Expr(Expr::Verbatim(_), Some(_)) => stmt, + Stmt::Expr(e, semi) => Stmt::Expr(fold_expr(self, e), semi), + s => s, + } + } + + fn fold_meta_name_value(&mut self, meta: MetaNameValue) -> MetaNameValue { + // Don't turn #[p = "..."] into #[p = ("...")]. + meta + } + + // We don't want to look at expressions that might appear in patterns or + // types yet. We'll look into comparing those in the future. For now + // focus on expressions appearing in other places. + fn fold_pat(&mut self, pat: Pat) -> Pat { + pat + } + + fn fold_type(&mut self, ty: Type) -> Type { + ty + } + + fn fold_lit(&mut self, lit: Lit) -> Lit { + if let Lit::Verbatim(lit) = &lit { + panic!("unexpected verbatim literal: {lit}"); + } + lit + } + } + + let mut folder = FullyParenthesize; + folder.fold_expr(syn_expr) +} + +fn make_parens_invisible(expr: syn::Expr) -> syn::Expr { + use syn::fold::{fold_expr, fold_stmt, Fold}; + use syn::{token, Expr, ExprGroup, ExprParen, Stmt}; + + struct MakeParensInvisible; + + impl Fold for MakeParensInvisible { + fn fold_expr(&mut self, mut expr: Expr) -> Expr { + if let Expr::Paren(paren) = expr { + expr = Expr::Group(ExprGroup { + attrs: paren.attrs, + group_token: token::Group(paren.paren_token.span.join()), + expr: paren.expr, + }); + } + fold_expr(self, expr) + } + + fn fold_stmt(&mut self, stmt: Stmt) -> Stmt { + if let Stmt::Expr(expr @ (Expr::Binary(_) | Expr::Call(_) | Expr::Cast(_)), None) = stmt + { + Stmt::Expr( + Expr::Paren(ExprParen { + attrs: Vec::new(), + paren_token: token::Paren::default(), + expr: Box::new(fold_expr(self, expr)), + }), + None, + ) + } else { + fold_stmt(self, stmt) + } + } + } + + let mut folder = MakeParensInvisible; + folder.fold_expr(expr) +} + +/// Walk through a crate collecting all expressions we can find in it. +fn collect_exprs(file: syn::File) -> Vec<syn::Expr> { + use syn::fold::Fold; + use syn::punctuated::Punctuated; + use syn::{token, ConstParam, Expr, ExprTuple, Pat, Path}; + + struct CollectExprs(Vec<Expr>); + impl Fold for CollectExprs { + fn fold_expr(&mut self, expr: Expr) -> Expr { + match expr { + Expr::Verbatim(_) => {} + _ => self.0.push(expr), + } + + Expr::Tuple(ExprTuple { + attrs: vec![], + elems: Punctuated::new(), + paren_token: token::Paren::default(), + }) + } + + fn fold_pat(&mut self, pat: Pat) -> Pat { + pat + } + + fn fold_path(&mut self, path: Path) -> Path { + // Skip traversing into const generic path arguments + path + } + + fn fold_const_param(&mut self, const_param: ConstParam) -> ConstParam { + const_param + } + } + + let mut folder = CollectExprs(vec![]); + folder.fold_file(file); + folder.0 +} diff --git a/third_party/crates/syn-2.0.119/tests/test_punctuated.rs b/third_party/crates/syn-2.0.119/tests/test_punctuated.rs new file mode 100644 index 0000000..14ea96c --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_punctuated.rs @@ -0,0 +1,92 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +use syn::punctuated::{Pair, Punctuated}; +use syn::Token; + +macro_rules! punctuated { + ($($e:expr,)+) => {{ + let mut seq = ::syn::punctuated::Punctuated::new(); + $( + seq.push($e); + )+ + seq + }}; + + ($($e:expr),+) => { + punctuated!($($e,)+) + }; +} + +macro_rules! check_exact_size_iterator { + ($iter:expr) => {{ + let iter = $iter; + let size_hint = iter.size_hint(); + let len = iter.len(); + let count = iter.count(); + assert_eq!(len, count); + assert_eq!(size_hint, (count, Some(count))); + }}; +} + +#[test] +fn pairs() { + let mut p: Punctuated<_, Token![,]> = punctuated!(2, 3, 4); + + check_exact_size_iterator!(p.pairs()); + check_exact_size_iterator!(p.pairs_mut()); + check_exact_size_iterator!(p.into_pairs()); + + let mut p: Punctuated<_, Token![,]> = punctuated!(2, 3, 4); + + assert_eq!(p.pairs().next_back().map(Pair::into_value), Some(&4)); + assert_eq!( + p.pairs_mut().next_back().map(Pair::into_value), + Some(&mut 4) + ); + assert_eq!(p.into_pairs().next_back().map(Pair::into_value), Some(4)); +} + +#[test] +fn iter() { + let mut p: Punctuated<_, Token![,]> = punctuated!(2, 3, 4); + + check_exact_size_iterator!(p.iter()); + check_exact_size_iterator!(p.iter_mut()); + check_exact_size_iterator!(p.into_iter()); + + let mut p: Punctuated<_, Token![,]> = punctuated!(2, 3, 4); + + assert_eq!(p.iter().next_back(), Some(&4)); + assert_eq!(p.iter_mut().next_back(), Some(&mut 4)); + assert_eq!(p.into_iter().next_back(), Some(4)); +} + +#[test] +fn may_dangle() { + let p: Punctuated<_, Token![,]> = punctuated!(2, 3, 4); + for element in &p { + if *element == 2 { + drop(p); + break; + } + } + + let mut p: Punctuated<_, Token![,]> = punctuated!(2, 3, 4); + for element in &mut p { + if *element == 2 { + drop(p); + break; + } + } +} + +#[test] +#[should_panic = "index out of bounds: the len is 0 but the index is 0"] +fn index_out_of_bounds() { + let p = Punctuated::<syn::Ident, Token![,]>::new(); + let _ = p[0].clone(); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_receiver.rs b/third_party/crates/syn-2.0.119/tests/test_receiver.rs new file mode 100644 index 0000000..9819410 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_receiver.rs @@ -0,0 +1,327 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use syn::{parse_quote, TraitItemFn}; + +#[test] +fn test_by_value() { + let TraitItemFn { sig, .. } = parse_quote! { + fn by_value(self: Self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }) + "#); +} + +#[test] +fn test_by_mut_value() { + let TraitItemFn { sig, .. } = parse_quote! { + fn by_mut(mut self: Self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + mutability: Some, + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }) + "#); +} + +#[test] +fn test_by_ref() { + let TraitItemFn { sig, .. } = parse_quote! { + fn by_ref(self: &Self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + colon_token: Some, + ty: Type::Reference { + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }, + }) + "#); +} + +#[test] +fn test_by_box() { + let TraitItemFn { sig, .. } = parse_quote! { + fn by_box(self: Box<Self>); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Box", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Type(Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }), + ], + }, + }, + ], + }, + }, + }) + "#); +} + +#[test] +fn test_by_pin() { + let TraitItemFn { sig, .. } = parse_quote! { + fn by_pin(self: Pin<Self>); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Pin", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Type(Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }), + ], + }, + }, + ], + }, + }, + }) + "#); +} + +#[test] +fn test_explicit_type() { + let TraitItemFn { sig, .. } = parse_quote! { + fn explicit_type(self: Pin<MyType>); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + colon_token: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Pin", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Type(Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "MyType", + }, + ], + }, + }), + ], + }, + }, + ], + }, + }, + }) + "#); +} + +#[test] +fn test_value_shorthand() { + let TraitItemFn { sig, .. } = parse_quote! { + fn value_shorthand(self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }) + "#); +} + +#[test] +fn test_mut_value_shorthand() { + let TraitItemFn { sig, .. } = parse_quote! { + fn mut_value_shorthand(mut self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + mutability: Some, + ty: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }) + "#); +} + +#[test] +fn test_ref_shorthand() { + let TraitItemFn { sig, .. } = parse_quote! { + fn ref_shorthand(&self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + reference: Some(None), + ty: Type::Reference { + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }, + }) + "#); +} + +#[test] +fn test_ref_shorthand_with_lifetime() { + let TraitItemFn { sig, .. } = parse_quote! { + fn ref_shorthand(&'a self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + reference: Some(Some(Lifetime { + ident: "a", + })), + ty: Type::Reference { + lifetime: Some(Lifetime { + ident: "a", + }), + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }, + }) + "#); +} + +#[test] +fn test_ref_mut_shorthand() { + let TraitItemFn { sig, .. } = parse_quote! { + fn ref_mut_shorthand(&mut self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + reference: Some(None), + mutability: Some, + ty: Type::Reference { + mutability: Some, + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }, + }) + "#); +} + +#[test] +fn test_ref_mut_shorthand_with_lifetime() { + let TraitItemFn { sig, .. } = parse_quote! { + fn ref_mut_shorthand(&'a mut self); + }; + snapshot!(&sig.inputs[0], @r#" + FnArg::Receiver(Receiver { + reference: Some(Some(Lifetime { + ident: "a", + })), + mutability: Some, + ty: Type::Reference { + lifetime: Some(Lifetime { + ident: "a", + }), + mutability: Some, + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Self", + }, + ], + }, + }, + }, + }) + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_round_trip.rs b/third_party/crates/syn-2.0.119/tests/test_round_trip.rs new file mode 100644 index 0000000..c19adb8 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_round_trip.rs @@ -0,0 +1,225 @@ +#![cfg(not(syn_disable_nightly_tests))] +#![cfg(not(miri))] +#![recursion_limit = "1024"] +#![feature(rustc_private)] +#![allow( + clippy::blocks_in_conditions, + clippy::elidable_lifetime_names, + clippy::manual_assert, + clippy::manual_let_else, + clippy::match_like_matches_macro, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] +#![allow(mismatched_lifetime_syntaxes)] + +extern crate rustc_ast; +extern crate rustc_ast_pretty; +extern crate rustc_data_structures; +extern crate rustc_driver; +extern crate rustc_error_messages; +extern crate rustc_errors; +extern crate rustc_expand; +extern crate rustc_parse; +extern crate rustc_session; +extern crate rustc_span; + +use crate::common::eq::SpanlessEq; +use quote::quote; +use rustc_ast::ast::{ + AngleBracketedArg, Crate, GenericArg, GenericArgs, GenericParamKind, Generics, +}; +use rustc_ast::mut_visit::{self, MutVisitor}; +use rustc_ast_pretty::pprust; +use rustc_data_structures::flat_map_in_place::FlatMapInPlace; +use rustc_error_messages::DiagMessage; +use rustc_errors::{Diag, PResult}; +use rustc_parse::lexer::StripTokens; +use rustc_session::parse::ParseSess; +use rustc_span::FileName; +use std::borrow::Cow; +use std::fs; +use std::panic; +use std::path::Path; +use std::process; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; + +#[macro_use] +mod macros; + +mod common; +mod repo; + +#[test] +fn test_round_trip() { + repo::rayon_init(); + repo::clone_rust(); + let abort_after = repo::abort_after(); + if abort_after == 0 { + panic!("skipping all round_trip tests"); + } + + let failed = AtomicUsize::new(0); + + repo::for_each_rust_file(|path| test(path, &failed, abort_after)); + + let failed = failed.into_inner(); + if failed > 0 { + panic!("{} failures", failed); + } +} + +fn test(path: &Path, failed: &AtomicUsize, abort_after: usize) { + let failed = || { + let prev_failed = failed.fetch_add(1, Ordering::Relaxed); + if prev_failed + 1 >= abort_after { + process::exit(1); + } + }; + + let content = fs::read_to_string(path).unwrap(); + + let (back, elapsed) = match panic::catch_unwind(|| { + let start = Instant::now(); + let result = syn::parse_file(&content); + let elapsed = start.elapsed(); + result.map(|krate| (quote!(#krate).to_string(), elapsed)) + }) { + Err(_) => { + errorf!("=== {}: syn panic\n", path.display()); + failed(); + return; + } + Ok(Err(msg)) => { + errorf!("=== {}: syn failed to parse\n{:?}\n", path.display(), msg); + failed(); + return; + } + Ok(Ok(result)) => result, + }; + + let edition = repo::edition(path).parse().unwrap(); + + rustc_span::create_session_if_not_set_then(edition, |_| { + let equal = match panic::catch_unwind(|| { + let sess = ParseSess::new(); + let before = match librustc_parse(content, &sess) { + Ok(before) => before, + Err(diagnostic) => { + errorf!( + "=== {}: ignore - librustc failed to parse original content: {}\n", + path.display(), + translate_message(&diagnostic), + ); + diagnostic.cancel(); + return Err(true); + } + }; + let after = match librustc_parse(back, &sess) { + Ok(after) => after, + Err(diagnostic) => { + errorf!("=== {}: librustc failed to parse", path.display()); + diagnostic.emit(); + return Err(false); + } + }; + Ok((before, after)) + }) { + Err(_) => { + errorf!("=== {}: ignoring librustc panic\n", path.display()); + true + } + Ok(Err(equal)) => equal, + Ok(Ok((mut before, mut after))) => { + normalize(&mut before); + normalize(&mut after); + if SpanlessEq::eq(&before, &after) { + errorf!( + "=== {}: pass in {}ms\n", + path.display(), + elapsed.as_secs() * 1000 + u64::from(elapsed.subsec_nanos()) / 1_000_000 + ); + true + } else { + errorf!( + "=== {}: FAIL\n{}\n!=\n{}\n", + path.display(), + pprust::crate_to_string_for_macros(&before), + pprust::crate_to_string_for_macros(&after), + ); + false + } + } + }; + if !equal { + failed(); + } + }); +} + +fn librustc_parse(content: String, sess: &ParseSess) -> PResult<Crate> { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + let counter = COUNTER.fetch_add(1, Ordering::Relaxed); + let name = FileName::Custom(format!("test_round_trip{}", counter)); + let mut parser = rustc_parse::new_parser_from_source_str( + sess, + name, + content, + StripTokens::ShebangAndFrontmatter, + ) + .unwrap(); + parser.parse_crate_mod() +} + +fn translate_message(diagnostic: &Diag) -> Cow<'static, str> { + match &diagnostic.messages[0].0 { + DiagMessage::Str(msg) | DiagMessage::Inline(msg) => msg.clone(), + } +} + +fn normalize(krate: &mut Crate) { + struct NormalizeVisitor; + + impl MutVisitor for NormalizeVisitor { + fn visit_generic_args(&mut self, e: &mut GenericArgs) { + if let GenericArgs::AngleBracketed(e) = e { + #[derive(Ord, PartialOrd, Eq, PartialEq)] + enum Group { + Lifetimes, + TypesAndConsts, + Constraints, + } + e.args.sort_by_key(|arg| match arg { + AngleBracketedArg::Arg(arg) => match arg { + GenericArg::Lifetime(_) => Group::Lifetimes, + GenericArg::Type(_) | GenericArg::Const(_) => Group::TypesAndConsts, + }, + AngleBracketedArg::Constraint(_) => Group::Constraints, + }); + } + mut_visit::walk_generic_args(self, e); + } + + fn visit_generics(&mut self, e: &mut Generics) { + #[derive(Ord, PartialOrd, Eq, PartialEq)] + enum Group { + Lifetimes, + TypesAndConsts, + } + e.params.sort_by_key(|param| match param.kind { + GenericParamKind::Lifetime => Group::Lifetimes, + GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => { + Group::TypesAndConsts + } + }); + e.params + .flat_map_in_place(|param| self.flat_map_generic_param(param)); + if e.where_clause.predicates.is_empty() { + e.where_clause.has_where_token = false; + } + } + } + + NormalizeVisitor.visit_crate(krate); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_shebang.rs b/third_party/crates/syn-2.0.119/tests/test_shebang.rs new file mode 100644 index 0000000..3b55ddf --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_shebang.rs @@ -0,0 +1,73 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +#[test] +fn test_basic() { + let content = "#!/usr/bin/env rustx\nfn main() {}"; + let file = syn::parse_file(content).unwrap(); + snapshot!(file, @r##" + File { + shebang: Some("#!/usr/bin/env rustx"), + items: [ + Item::Fn { + vis: Visibility::Inherited, + sig: Signature { + ident: "main", + generics: Generics, + output: ReturnType::Default, + }, + block: Block { + stmts: [], + }, + }, + ], + } + "##); +} + +#[test] +fn test_comment() { + let content = "#!//am/i/a/comment\n[allow(dead_code)] fn main() {}"; + let file = syn::parse_file(content).unwrap(); + snapshot!(file, @r#" + File { + attrs: [ + Attribute { + style: AttrStyle::Inner, + meta: Meta::List { + path: Path { + segments: [ + PathSegment { + ident: "allow", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`dead_code`), + }, + }, + ], + items: [ + Item::Fn { + vis: Visibility::Inherited, + sig: Signature { + ident: "main", + generics: Generics, + output: ReturnType::Default, + }, + block: Block { + stmts: [], + }, + }, + ], + } + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_size.rs b/third_party/crates/syn-2.0.119/tests/test_size.rs new file mode 100644 index 0000000..29fd435 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_size.rs @@ -0,0 +1,54 @@ +// Assumes proc-macro2's "span-locations" feature is off. + +use std::mem; +use syn::{Expr, Item, Lit, Pat, Type}; + +#[rustversion::attr(before(2022-11-24), ignore = "requires nightly-2022-11-24 or newer")] +#[rustversion::attr( + since(2022-11-24), + cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit") +)] +#[test] +fn test_expr_size() { + assert_eq!(mem::size_of::<Expr>(), 176); +} + +#[rustversion::attr(before(2022-09-09), ignore = "requires nightly-2022-09-09 or newer")] +#[rustversion::attr( + since(2022-09-09), + cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit") +)] +#[test] +fn test_item_size() { + assert_eq!(mem::size_of::<Item>(), 352); +} + +#[rustversion::attr(before(2023-04-29), ignore = "requires nightly-2023-04-29 or newer")] +#[rustversion::attr( + since(2023-04-29), + cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit") +)] +#[test] +fn test_type_size() { + assert_eq!(mem::size_of::<Type>(), 224); +} + +#[rustversion::attr(before(2023-04-29), ignore = "requires nightly-2023-04-29 or newer")] +#[rustversion::attr( + since(2023-04-29), + cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit") +)] +#[test] +fn test_pat_size() { + assert_eq!(mem::size_of::<Pat>(), 184); +} + +#[rustversion::attr(before(2023-12-20), ignore = "requires nightly-2023-12-20 or newer")] +#[rustversion::attr( + since(2023-12-20), + cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit") +)] +#[test] +fn test_lit_size() { + assert_eq!(mem::size_of::<Lit>(), 24); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_stmt.rs b/third_party/crates/syn-2.0.119/tests/test_stmt.rs new file mode 100644 index 0000000..101c1b1 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_stmt.rs @@ -0,0 +1,337 @@ +#![allow( + clippy::assertions_on_result_states, + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::non_ascii_literal, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::{Delimiter, Group, Ident, Span, TokenStream, TokenTree}; +use quote::{quote, ToTokens as _}; +use syn::parse::Parser as _; +use syn::{Block, Stmt}; + +#[test] +fn test_raw_operator() { + let stmt = syn::parse_str::<Stmt>("let _ = &raw const x;").unwrap(); + + snapshot!(stmt, @r#" + Stmt::Local { + pat: Pat::Wild, + init: Some(LocalInit { + expr: Expr::RawAddr { + mutability: PointerMutability::Const, + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "x", + }, + ], + }, + }, + }, + }), + } + "#); +} + +#[test] +fn test_raw_variable() { + let stmt = syn::parse_str::<Stmt>("let _ = &raw;").unwrap(); + + snapshot!(stmt, @r#" + Stmt::Local { + pat: Pat::Wild, + init: Some(LocalInit { + expr: Expr::Reference { + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "raw", + }, + ], + }, + }, + }, + }), + } + "#); +} + +#[test] +fn test_raw_invalid() { + assert!(syn::parse_str::<Stmt>("let _ = &raw x;").is_err()); +} + +#[test] +fn test_none_group() { + // «∅ async fn f() {} ∅» + let tokens = TokenStream::from_iter([TokenTree::Group(Group::new( + Delimiter::None, + TokenStream::from_iter([ + TokenTree::Ident(Ident::new("async", Span::call_site())), + TokenTree::Ident(Ident::new("fn", Span::call_site())), + TokenTree::Ident(Ident::new("f", Span::call_site())), + TokenTree::Group(Group::new(Delimiter::Parenthesis, TokenStream::new())), + TokenTree::Group(Group::new(Delimiter::Brace, TokenStream::new())), + ]), + ))]); + snapshot!(tokens as Stmt, @r#" + Stmt::Item(Item::Fn { + vis: Visibility::Inherited, + sig: Signature { + asyncness: Some, + ident: "f", + generics: Generics, + output: ReturnType::Default, + }, + block: Block { + stmts: [], + }, + }) + "#); + + let tokens = Group::new(Delimiter::None, quote!(let None = None)).to_token_stream(); + let stmts = Block::parse_within.parse2(tokens).unwrap(); + snapshot!(stmts, @r#" + [ + Stmt::Expr( + Expr::Group { + expr: Expr::Let { + pat: Pat::Ident { + ident: "None", + }, + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "None", + }, + ], + }, + }, + }, + }, + None, + ), + ] + "#); +} + +#[test] +fn test_let_dot_dot() { + let tokens = quote! { + let .. = 10; + }; + + snapshot!(tokens as Stmt, @r#" + Stmt::Local { + pat: Pat::Rest, + init: Some(LocalInit { + expr: Expr::Lit { + lit: 10, + }, + }), + } + "#); +} + +#[test] +fn test_let_else() { + let tokens = quote! { + let Some(x) = None else { return 0; }; + }; + + snapshot!(tokens as Stmt, @r#" + Stmt::Local { + pat: Pat::TupleStruct { + path: Path { + segments: [ + PathSegment { + ident: "Some", + }, + ], + }, + elems: [ + Pat::Ident { + ident: "x", + }, + ], + }, + init: Some(LocalInit { + expr: Expr::Path { + path: Path { + segments: [ + PathSegment { + ident: "None", + }, + ], + }, + }, + diverge: Some(Expr::Block { + block: Block { + stmts: [ + Stmt::Expr( + Expr::Return { + expr: Some(Expr::Lit { + lit: 0, + }), + }, + Some, + ), + ], + }, + }), + }), + } + "#); +} + +#[test] +fn test_macros() { + let tokens = quote! { + fn main() { + macro_rules! mac {} + thread_local! { static FOO } + println!(""); + vec![] + } + }; + + snapshot!(tokens as Stmt, @r#" + Stmt::Item(Item::Fn { + vis: Visibility::Inherited, + sig: Signature { + ident: "main", + generics: Generics, + output: ReturnType::Default, + }, + block: Block { + stmts: [ + Stmt::Item(Item::Macro { + ident: Some("mac"), + mac: Macro { + path: Path { + segments: [ + PathSegment { + ident: "macro_rules", + }, + ], + }, + delimiter: MacroDelimiter::Brace, + tokens: TokenStream(``), + }, + }), + Stmt::Macro { + mac: Macro { + path: Path { + segments: [ + PathSegment { + ident: "thread_local", + }, + ], + }, + delimiter: MacroDelimiter::Brace, + tokens: TokenStream(`static FOO`), + }, + }, + Stmt::Macro { + mac: Macro { + path: Path { + segments: [ + PathSegment { + ident: "println", + }, + ], + }, + delimiter: MacroDelimiter::Paren, + tokens: TokenStream(`""`), + }, + semi_token: Some, + }, + Stmt::Expr( + Expr::Macro { + mac: Macro { + path: Path { + segments: [ + PathSegment { + ident: "vec", + }, + ], + }, + delimiter: MacroDelimiter::Bracket, + tokens: TokenStream(``), + }, + }, + None, + ), + ], + }, + }) + "#); +} + +#[test] +fn test_early_parse_loop() { + // The following is an Expr::Loop followed by Expr::Tuple. It is not an + // Expr::Call. + let tokens = quote! { + loop {} + () + }; + + let stmts = Block::parse_within.parse2(tokens).unwrap(); + + snapshot!(stmts, @r#" + [ + Stmt::Expr( + Expr::Loop { + body: Block { + stmts: [], + }, + }, + None, + ), + Stmt::Expr( + Expr::Tuple, + None, + ), + ] + "#); + + let tokens = quote! { + 'a: loop {} + () + }; + + let stmts = Block::parse_within.parse2(tokens).unwrap(); + + snapshot!(stmts, @r#" + [ + Stmt::Expr( + Expr::Loop { + label: Some(Label { + name: Lifetime { + ident: "a", + }, + }), + body: Block { + stmts: [], + }, + }, + None, + ), + Stmt::Expr( + Expr::Tuple, + None, + ), + ] + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_token_trees.rs b/third_party/crates/syn-2.0.119/tests/test_token_trees.rs new file mode 100644 index 0000000..1b47385 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_token_trees.rs @@ -0,0 +1,38 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::TokenStream; +use quote::quote; +use syn::Lit; + +#[test] +fn test_struct() { + let input = " + #[derive(Debug, Clone)] + pub struct Item { + pub ident: Ident, + pub attrs: Vec<Attribute>, + } + "; + + snapshot!(input as TokenStream, @r##" + TokenStream( + `# [derive (Debug , Clone)] pub struct Item { pub ident : Ident , pub attrs : Vec < Attribute >, }`, + ) + "##); +} + +#[test] +fn test_literal_mangling() { + let code = "0_4"; + let parsed: Lit = syn::parse_str(code).unwrap(); + assert_eq!(code, quote!(#parsed).to_string()); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_ty.rs b/third_party/crates/syn-2.0.119/tests/test_ty.rs new file mode 100644 index 0000000..5f29220 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_ty.rs @@ -0,0 +1,471 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; +use quote::{quote, ToTokens as _}; +use syn::punctuated::Punctuated; +use syn::{parse_quote, token, Token, Type, TypeTuple}; + +#[test] +fn test_mut_self() { + syn::parse_str::<Type>("fn(mut self)").unwrap(); + syn::parse_str::<Type>("fn(mut self,)").unwrap(); + syn::parse_str::<Type>("fn(mut self: ())").unwrap(); + syn::parse_str::<Type>("fn(mut self: ...)").unwrap_err(); + syn::parse_str::<Type>("fn(mut self: mut self)").unwrap_err(); + syn::parse_str::<Type>("fn(mut self::T)").unwrap_err(); +} + +#[test] +fn test_macro_variable_type() { + // mimics the token stream corresponding to `$ty<T>` + let tokens = TokenStream::from_iter([ + TokenTree::Group(Group::new(Delimiter::None, quote! { ty })), + TokenTree::Punct(Punct::new('<', Spacing::Alone)), + TokenTree::Ident(Ident::new("T", Span::call_site())), + TokenTree::Punct(Punct::new('>', Spacing::Alone)), + ]); + + snapshot!(tokens as Type, @r#" + Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "ty", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Type(Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "T", + }, + ], + }, + }), + ], + }, + }, + ], + }, + } + "#); + + // mimics the token stream corresponding to `$ty::<T>` + let tokens = TokenStream::from_iter([ + TokenTree::Group(Group::new(Delimiter::None, quote! { ty })), + TokenTree::Punct(Punct::new(':', Spacing::Joint)), + TokenTree::Punct(Punct::new(':', Spacing::Alone)), + TokenTree::Punct(Punct::new('<', Spacing::Alone)), + TokenTree::Ident(Ident::new("T", Span::call_site())), + TokenTree::Punct(Punct::new('>', Spacing::Alone)), + ]); + + snapshot!(tokens as Type, @r#" + Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "ty", + arguments: PathArguments::AngleBracketed { + colon2_token: Some, + args: [ + GenericArgument::Type(Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "T", + }, + ], + }, + }), + ], + }, + }, + ], + }, + } + "#); +} + +#[test] +fn test_group_angle_brackets() { + // mimics the token stream corresponding to `Option<$ty>` + let tokens = TokenStream::from_iter([ + TokenTree::Ident(Ident::new("Option", Span::call_site())), + TokenTree::Punct(Punct::new('<', Spacing::Alone)), + TokenTree::Group(Group::new(Delimiter::None, quote! { Vec<u8> })), + TokenTree::Punct(Punct::new('>', Spacing::Alone)), + ]); + + snapshot!(tokens as Type, @r#" + Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Option", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Type(Type::Group { + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Vec", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Type(Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "u8", + }, + ], + }, + }), + ], + }, + }, + ], + }, + }, + }), + ], + }, + }, + ], + }, + } + "#); +} + +#[test] +fn test_group_colons() { + // mimics the token stream corresponding to `$ty::Item` + let tokens = TokenStream::from_iter([ + TokenTree::Group(Group::new(Delimiter::None, quote! { Vec<u8> })), + TokenTree::Punct(Punct::new(':', Spacing::Joint)), + TokenTree::Punct(Punct::new(':', Spacing::Alone)), + TokenTree::Ident(Ident::new("Item", Span::call_site())), + ]); + + snapshot!(tokens as Type, @r#" + Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "Vec", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Type(Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "u8", + }, + ], + }, + }), + ], + }, + }, + Token![::], + PathSegment { + ident: "Item", + }, + ], + }, + } + "#); + + let tokens = TokenStream::from_iter([ + TokenTree::Group(Group::new(Delimiter::None, quote! { [T] })), + TokenTree::Punct(Punct::new(':', Spacing::Joint)), + TokenTree::Punct(Punct::new(':', Spacing::Alone)), + TokenTree::Ident(Ident::new("Element", Span::call_site())), + ]); + + snapshot!(tokens as Type, @r#" + Type::Path { + qself: Some(QSelf { + ty: Type::Slice { + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "T", + }, + ], + }, + }, + }, + position: 0, + }), + path: Path { + leading_colon: Some, + segments: [ + PathSegment { + ident: "Element", + }, + ], + }, + } + "#); +} + +#[test] +fn test_trait_object() { + let tokens = quote!(dyn for<'a> Trait<'a> + 'static); + snapshot!(tokens as Type, @r#" + Type::TraitObject { + dyn_token: Some, + bounds: [ + TypeParamBound::Trait(TraitBound { + lifetimes: Some(BoundLifetimes { + lifetimes: [ + GenericParam::Lifetime(LifetimeParam { + lifetime: Lifetime { + ident: "a", + }, + }), + ], + }), + path: Path { + segments: [ + PathSegment { + ident: "Trait", + arguments: PathArguments::AngleBracketed { + args: [ + GenericArgument::Lifetime(Lifetime { + ident: "a", + }), + ], + }, + }, + ], + }, + }), + Token![+], + TypeParamBound::Lifetime { + ident: "static", + }, + ], + } + "#); + + let tokens = quote!(dyn 'a + Trait); + snapshot!(tokens as Type, @r#" + Type::TraitObject { + dyn_token: Some, + bounds: [ + TypeParamBound::Lifetime { + ident: "a", + }, + Token![+], + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Trait", + }, + ], + }, + }), + ], + } + "#); + + // None of the following are valid Rust types. + syn::parse_str::<Type>("for<'a> dyn Trait<'a>").unwrap_err(); + syn::parse_str::<Type>("dyn for<'a> 'a + Trait").unwrap_err(); +} + +#[test] +fn test_trailing_plus() { + #[rustfmt::skip] + let tokens = quote!(impl Trait +); + snapshot!(tokens as Type, @r#" + Type::ImplTrait { + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Trait", + }, + ], + }, + }), + Token![+], + ], + } + "#); + + #[rustfmt::skip] + let tokens = quote!(dyn Trait +); + snapshot!(tokens as Type, @r#" + Type::TraitObject { + dyn_token: Some, + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Trait", + }, + ], + }, + }), + Token![+], + ], + } + "#); + + #[rustfmt::skip] + let tokens = quote!(Trait +); + snapshot!(tokens as Type, @r#" + Type::TraitObject { + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Trait", + }, + ], + }, + }), + Token![+], + ], + } + "#); +} + +#[test] +fn test_tuple_comma() { + let mut expr = TypeTuple { + paren_token: token::Paren::default(), + elems: Punctuated::new(), + }; + snapshot!(expr.to_token_stream() as Type, @"Type::Tuple"); + + expr.elems.push_value(parse_quote!(_)); + // Must not parse to Type::Paren + snapshot!(expr.to_token_stream() as Type, @r#" + Type::Tuple { + elems: [ + Type::Infer, + Token![,], + ], + } + "#); + + expr.elems.push_punct(<Token![,]>::default()); + snapshot!(expr.to_token_stream() as Type, @r#" + Type::Tuple { + elems: [ + Type::Infer, + Token![,], + ], + } + "#); + + expr.elems.push_value(parse_quote!(_)); + snapshot!(expr.to_token_stream() as Type, @r#" + Type::Tuple { + elems: [ + Type::Infer, + Token![,], + Type::Infer, + ], + } + "#); + + expr.elems.push_punct(<Token![,]>::default()); + snapshot!(expr.to_token_stream() as Type, @r#" + Type::Tuple { + elems: [ + Type::Infer, + Token![,], + Type::Infer, + Token![,], + ], + } + "#); +} + +#[test] +fn test_impl_trait_use() { + let tokens = quote! { + impl Sized + use<'_, 'a, A, Test> + }; + + snapshot!(tokens as Type, @r#" + Type::ImplTrait { + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Sized", + }, + ], + }, + }), + Token![+], + TypeParamBound::PreciseCapture(PreciseCapture { + params: [ + CapturedParam::Lifetime(Lifetime { + ident: "_", + }), + Token![,], + CapturedParam::Lifetime(Lifetime { + ident: "a", + }), + Token![,], + CapturedParam::Ident("A"), + Token![,], + CapturedParam::Ident("Test"), + ], + }), + ], + } + "#); + + let trailing = quote! { + impl Sized + use<'_,> + }; + + snapshot!(trailing as Type, @r#" + Type::ImplTrait { + bounds: [ + TypeParamBound::Trait(TraitBound { + path: Path { + segments: [ + PathSegment { + ident: "Sized", + }, + ], + }, + }), + Token![+], + TypeParamBound::PreciseCapture(PreciseCapture { + params: [ + CapturedParam::Lifetime(Lifetime { + ident: "_", + }), + Token![,], + ], + }), + ], + } + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/test_unparenthesize.rs b/third_party/crates/syn-2.0.119/tests/test_unparenthesize.rs new file mode 100644 index 0000000..5fa2e59 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_unparenthesize.rs @@ -0,0 +1,70 @@ +#![cfg(not(miri))] +#![recursion_limit = "1024"] +#![feature(rustc_private)] +#![allow( + clippy::elidable_lifetime_names, + clippy::manual_assert, + clippy::match_like_matches_macro, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +use crate::common::visit::{AsIfPrinted, FlattenParens}; +use quote::ToTokens as _; +use std::fs; +use std::panic; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use syn::visit_mut::VisitMut as _; + +#[macro_use] +mod macros; + +mod common; +mod repo; + +#[test] +fn test_unparenthesize() { + repo::rayon_init(); + repo::clone_rust(); + + let failed = AtomicUsize::new(0); + + repo::for_each_rust_file(|path| test(path, &failed)); + + let failed = failed.into_inner(); + if failed > 0 { + panic!("{} failures", failed); + } +} + +fn test(path: &Path, failed: &AtomicUsize) { + let content = fs::read_to_string(path).unwrap(); + + match panic::catch_unwind(|| -> syn::Result<()> { + let mut before = syn::parse_file(&content)?; + FlattenParens::discard_attrs().visit_file_mut(&mut before); + let printed = before.to_token_stream(); + let mut after = syn::parse2::<syn::File>(printed.clone())?; + FlattenParens::discard_attrs().visit_file_mut(&mut after); + // Normalize features that we expect Syn not to print. + AsIfPrinted.visit_file_mut(&mut before); + if before != after { + errorf!("=== {}\n", path.display()); + if failed.fetch_add(1, Ordering::Relaxed) == 0 { + errorf!("BEFORE:\n{:#?}\nAFTER:\n{:#?}\n", before, after); + } + } + Ok(()) + }) { + Err(_) => { + errorf!("=== {}: syn panic\n", path.display()); + failed.fetch_add(1, Ordering::Relaxed); + } + Ok(Err(msg)) => { + errorf!("=== {}: syn failed to parse\n{:?}\n", path.display(), msg); + failed.fetch_add(1, Ordering::Relaxed); + } + Ok(Ok(())) => {} + } +} diff --git a/third_party/crates/syn-2.0.119/tests/test_visibility.rs b/third_party/crates/syn-2.0.119/tests/test_visibility.rs new file mode 100644 index 0000000..cf15574 --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/test_visibility.rs @@ -0,0 +1,191 @@ +#![allow( + clippy::elidable_lifetime_names, + clippy::needless_lifetimes, + clippy::uninlined_format_args +)] + +#[macro_use] +mod snapshot; + +mod debug; + +use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; +use quote::quote; +use syn::parse::{Parse, ParseStream}; +use syn::{DeriveInput, Result, Visibility}; + +#[derive(Debug)] +struct VisRest { + vis: Visibility, + rest: TokenStream, +} + +impl Parse for VisRest { + fn parse(input: ParseStream) -> Result<Self> { + Ok(VisRest { + vis: input.parse()?, + rest: input.parse()?, + }) + } +} + +macro_rules! assert_vis_parse { + ($input:expr, Ok($p:pat)) => { + assert_vis_parse!($input, Ok($p) + ""); + }; + + ($input:expr, Ok($p:pat) + $rest:expr) => { + let expected = $rest.parse::<TokenStream>().unwrap(); + let parse: VisRest = syn::parse_str($input).unwrap(); + + match parse.vis { + $p => {} + _ => panic!("expected {}, got {:?}", stringify!($p), parse.vis), + } + + // NOTE: Round-trips through `to_string` to avoid potential whitespace + // diffs. + assert_eq!(parse.rest.to_string(), expected.to_string()); + }; + + ($input:expr, Err) => { + syn::parse2::<VisRest>($input.parse().unwrap()).unwrap_err(); + }; +} + +#[test] +fn test_pub() { + assert_vis_parse!("pub", Ok(Visibility::Public(_))); +} + +#[test] +fn test_inherited() { + assert_vis_parse!("", Ok(Visibility::Inherited)); +} + +#[test] +fn test_in() { + assert_vis_parse!("pub(in foo::bar)", Ok(Visibility::Restricted(_))); +} + +#[test] +fn test_pub_crate() { + assert_vis_parse!("pub(crate)", Ok(Visibility::Restricted(_))); +} + +#[test] +fn test_pub_self() { + assert_vis_parse!("pub(self)", Ok(Visibility::Restricted(_))); +} + +#[test] +fn test_pub_super() { + assert_vis_parse!("pub(super)", Ok(Visibility::Restricted(_))); +} + +#[test] +fn test_missing_in() { + assert_vis_parse!("pub(foo::bar)", Ok(Visibility::Public(_)) + "(foo::bar)"); +} + +#[test] +fn test_missing_in_path() { + assert_vis_parse!("pub(in)", Err); +} + +#[test] +fn test_crate_path() { + assert_vis_parse!( + "pub(crate::A, crate::B)", + Ok(Visibility::Public(_)) + "(crate::A, crate::B)" + ); +} + +#[test] +fn test_junk_after_in() { + assert_vis_parse!("pub(in some::path @@garbage)", Err); +} + +#[test] +fn test_inherited_vis_named_field() { + // mimics `struct S { $vis $field: () }` where $vis is empty + let tokens = TokenStream::from_iter([ + TokenTree::Ident(Ident::new("struct", Span::call_site())), + TokenTree::Ident(Ident::new("S", Span::call_site())), + TokenTree::Group(Group::new( + Delimiter::Brace, + TokenStream::from_iter([ + TokenTree::Group(Group::new(Delimiter::None, TokenStream::new())), + TokenTree::Group(Group::new(Delimiter::None, quote!(f))), + TokenTree::Punct(Punct::new(':', Spacing::Alone)), + TokenTree::Group(Group::new(Delimiter::Parenthesis, TokenStream::new())), + ]), + )), + ]); + + snapshot!(tokens as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Named { + named: [ + Field { + vis: Visibility::Inherited, + ident: Some("f"), + colon_token: Some, + ty: Type::Tuple, + }, + ], + }, + }, + } + "#); +} + +#[test] +fn test_inherited_vis_unnamed_field() { + // mimics `struct S($vis $ty);` where $vis is empty + let tokens = TokenStream::from_iter([ + TokenTree::Ident(Ident::new("struct", Span::call_site())), + TokenTree::Ident(Ident::new("S", Span::call_site())), + TokenTree::Group(Group::new( + Delimiter::Parenthesis, + TokenStream::from_iter([ + TokenTree::Group(Group::new(Delimiter::None, TokenStream::new())), + TokenTree::Group(Group::new(Delimiter::None, quote!(str))), + ]), + )), + TokenTree::Punct(Punct::new(';', Spacing::Alone)), + ]); + + snapshot!(tokens as DeriveInput, @r#" + DeriveInput { + vis: Visibility::Inherited, + ident: "S", + generics: Generics, + data: Data::Struct { + fields: Fields::Unnamed { + unnamed: [ + Field { + vis: Visibility::Inherited, + ty: Type::Group { + elem: Type::Path { + path: Path { + segments: [ + PathSegment { + ident: "str", + }, + ], + }, + }, + }, + }, + ], + }, + semi_token: Some, + }, + } + "#); +} diff --git a/third_party/crates/syn-2.0.119/tests/zzz_stable.rs b/third_party/crates/syn-2.0.119/tests/zzz_stable.rs new file mode 100644 index 0000000..a1a670d --- /dev/null +++ b/third_party/crates/syn-2.0.119/tests/zzz_stable.rs @@ -0,0 +1,33 @@ +#![cfg(syn_disable_nightly_tests)] + +use std::io::{self, Write}; +use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; + +const MSG: &str = "\ +‖ +‖ WARNING: +‖ This is not a nightly compiler so not all tests were able to +‖ run. Syn includes tests that compare Syn's parser against the +‖ compiler's parser, which requires access to unstable librustc +‖ data structures and a nightly compiler. +‖ +"; + +#[test] +fn notice() -> io::Result<()> { + let header = "WARNING"; + let index_of_header = MSG.find(header).unwrap(); + let before = &MSG[..index_of_header]; + let after = &MSG[index_of_header + header.len()..]; + + let mut stderr = StandardStream::stderr(ColorChoice::Auto); + stderr.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?; + write!(&mut stderr, "{}", before)?; + stderr.set_color(ColorSpec::new().set_bold(true).set_fg(Some(Color::Yellow)))?; + write!(&mut stderr, "{}", header)?; + stderr.set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))?; + write!(&mut stderr, "{}", after)?; + stderr.reset()?; + + Ok(()) +}