Mar 22, 2021 · Mar 21, 2021 · Mar 21, 2021 · Mar 21, 2021
diff --git a/Cargo.toml b/Cargo.toml
 version = "1.7.1"
 authors = ["kwantam <kwantam@gmail.com>", "Manish Goregaokar <manishsmail@gmail.com>"]

 edition = "2018"
 homepage = "https://github.com/unicode-rs/unicode-segmentation"
 repository = "https://github.com/unicode-rs/unicode-segmentation"
 documentation = "https://unicode-rs.github.io/unicode-segmentation"
diff --git a/scripts/unicode.py b/scripts/unicode.py
    #[inline]
    fn is_alphabetic(c: char) -> bool {
        match c {
            'a' ... 'z' | 'A' ... 'Z' => true,
            'a' ..= 'z' | 'A' ..= 'Z' => true,
            c if c > '\x7f' => super::derived_property::Alphabetic(c),
            _ => false,
        }
    #[inline]
    fn is_numeric(c: char) -> bool {
        match c {
            '0' ... '9' => true,
            '0' ..= '9' => true,
            c if c > '\x7f' => super::general_category::N(c),
            _ => false,
        }
    f.write("""    }

    fn bsearch_range_value_table(c: char, r: &'static [(char, char, %sCat)]) -> (u32, u32, %sCat) {
        use core;
        use core::cmp::Ordering::{Equal, Less, Greater};
        match r.binary_search_by(|&(lo, hi, _)| {
            if lo <= c && c <= hi { Equal }
diff --git a/src/grapheme.rs b/src/grapheme.rs

 use core::cmp;

 use tables::grapheme::GraphemeCat;
 usecrate::tables::grapheme::GraphemeCat;

 /// External iterator for grapheme clusters and byte offsets.
 ///
 }

 fn check_pair(before: GraphemeCat, after: GraphemeCat) -> PairResult {
    use tables::grapheme::GraphemeCat::*;
    usecrate::tables::grapheme::GraphemeCat::*;
    use self::PairResult::*;
    match (before, after) {
        (GC_CR, GC_LF) => NotBreak,  // GB3
    }

    fn grapheme_category(&mut self, ch: char) -> GraphemeCat {
        use tables::grapheme as gr;
        use tables::grapheme::GraphemeCat::*;
        usecrate::tables::grapheme as gr;
        usecrate::tables::grapheme::GraphemeCat::*;

        if ch <= '\u{7e}' {
            // Special-case optimization for ascii, except U+007F.  This
    /// assert_eq!(cursor.is_boundary(&flags[8..], 8), Ok(true));
    /// ```
    pub fn provide_context(&mut self, chunk: &str, chunk_start: usize) {
        use tables::grapheme as gr;
        usecrate::tables::grapheme as gr;
        assert!(chunk_start + chunk.len() == self.pre_context_offset.unwrap());
        self.pre_context_offset = None;
        if self.is_extended && chunk_start + chunk.len() == self.offset {
    }

    fn handle_regional(&mut self, chunk: &str, chunk_start: usize) {
        use tables::grapheme as gr;
        usecrate::tables::grapheme as gr;
        let mut ris_count = self.ris_count.unwrap_or(0);
        for ch in chunk.chars().rev() {
            if self.grapheme_category(ch) != gr::GC_Regional_Indicator {
    }

    fn handle_emoji(&mut self, chunk: &str, chunk_start: usize) {
        use tables::grapheme as gr;
        usecrate::tables::grapheme as gr;
        let mut iter = chunk.chars().rev();
        if let Some(ch) = iter.next() {
            if self.grapheme_category(ch) != gr::GC_ZWJ {
    /// assert_eq!(cursor.is_boundary(flags, 0), Ok(false));
    /// ```
    pub fn is_boundary(&mut self, chunk: &str, chunk_start: usize) -> Result<bool, GraphemeIncomplete> {
        use tables::grapheme as gr;
        usecrate::tables::grapheme as gr;
        if self.state == GraphemeState::Break {
            return Ok(true)
        }
diff --git a/src/sentence.rs b/src/sentence.rs

 // All of the logic for forward iteration over sentences
 mod fwd {
    use tables::sentence::SentenceCat;
    usecrate::tables::sentence::SentenceCat;
    use core::cmp;

    // Describe a parsed part of source string as described in this table:
        if parts[idx] == StatePart::ClosePlus { idx -= 1 }

        if parts[idx] == StatePart::ATerm {
            use tables::sentence as se;
            usecrate::tables::sentence as se;

            for next_char in ahead.chars() {
                //( ¬(OLetter | Upper | Lower | ParaSep | SATerm) )* Lower

        #[inline]
        fn next(&mut self) -> Option<usize> {
            use tables::sentence as se;
            usecrate::tables::sentence as se;

            for next_char in self.string[self.pos..].chars() {
                let position_before = self.pos;
 #[inline]
 pub fn new_unicode_sentences<'b>(s: &'b str) -> UnicodeSentences<'b> {
    use super::UnicodeSegmentation;
    use tables::util::is_alphanumeric;
    usecrate::tables::util::is_alphanumeric;

    fn has_alphanumeric(s: &&str) -> bool { s.chars().any(|c| is_alphanumeric(c)) }
    let has_alphanumeric: fn(&&str) -> bool = has_alphanumeric; // coerce to fn pointer
diff --git a/src/tables.rs b/src/tables.rs
    #[inline]
    fn is_alphabetic(c: char) -> bool {
        match c {
            'a' ... 'z' | 'A' ... 'Z' => true,
            'a' ..= 'z' | 'A' ..= 'Z' => true,
            c if c > '' => super::derived_property::Alphabetic(c),
            _ => false,
        }
    #[inline]
    fn is_numeric(c: char) -> bool {
        match c {
            '0' ... '9' => true,
            '0' ..= '9' => true,
            c if c > '' => super::general_category::N(c),
            _ => false,
        }
    }

    fn bsearch_range_value_table(c: char, r: &'static [(char, char, GraphemeCat)]) -> (u32, u32, GraphemeCat) {
        use core;
        use core::cmp::Ordering::{Equal, Less, Greater};
        match r.binary_search_by(|&(lo, hi, _)| {
            if lo <= c && c <= hi { Equal }
    }

    fn bsearch_range_value_table(c: char, r: &'static [(char, char, WordCat)]) -> (u32, u32, WordCat) {
        use core;
        use core::cmp::Ordering::{Equal, Less, Greater};
        match r.binary_search_by(|&(lo, hi, _)| {
            if lo <= c && c <= hi { Equal }
    }

    fn bsearch_range_value_table(c: char, r: &'static [(char, char, EmojiCat)]) -> (u32, u32, EmojiCat) {
        use core;
        use core::cmp::Ordering::{Equal, Less, Greater};
        match r.binary_search_by(|&(lo, hi, _)| {
            if lo <= c && c <= hi { Equal }
    }

    fn bsearch_range_value_table(c: char, r: &'static [(char, char, SentenceCat)]) -> (u32, u32, SentenceCat) {
        use core;
        use core::cmp::Ordering::{Equal, Less, Greater};
        match r.binary_search_by(|&(lo, hi, _)| {
            if lo <= c && c <= hi { Equal }
diff --git a/src/test.rs b/src/test.rs

 #[test]
 fn test_graphemes() {
    use testdata::{TEST_SAME, TEST_DIFF};
    usecrate::testdata::{TEST_SAME, TEST_DIFF};

    pub const EXTRA_DIFF: &'static [(&'static str,
                                     &'static [&'static str],

 #[test]
 fn test_words() {
    use testdata::TEST_WORD;
    usecrate::testdata::TEST_WORD;

    // Unicode's official tests don't really test longer chains of flag emoji
    // TODO This could be improved with more tests like flag emoji with interspersed Extend chars and ZWJ

 #[test]
 fn test_sentences() {
    use testdata::TEST_SENTENCE;
    usecrate::testdata::TEST_SENTENCE;

    for &(s, w) in TEST_SENTENCE.iter() {
        macro_rules! assert_ {
diff --git a/src/word.rs b/src/word.rs
 use core::cmp;
 use core::iter::Filter;

 use tables::word::WordCat;
 usecrate::tables::word::WordCat;

 /// An iterator over the substrings of a string which, after splitting the string on
 /// [word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries),
 }

 fn is_emoji(ch: char) -> bool {
    use tables::emoji;
    usecrate::tables::emoji;
    emoji::emoji_category(ch).2 == emoji::EmojiCat::EC_Extended_Pictographic
 }

    fn next(&mut self) -> Option<&'a str> {
        use self::UWordBoundsState::*;
        use self::FormatExtendType::*;
        use tables::word as wd;
        usecrate::tables::word as wd;
        if self.string.len() == 0 {
            return None;
        }
    fn next_back(&mut self) -> Option<&'a str> {
        use self::UWordBoundsState::*;
        use self::FormatExtendType::*;
        use tables::word as wd;
        usecrate::tables::word as wd;
        if self.string.len() == 0 {
            return None;
        }

    #[inline]
    fn get_next_cat(&self, idx: usize) -> Option<WordCat> {
        use tables::word as wd;
        usecrate::tables::word as wd;
        let nidx = idx + self.string[idx..].chars().next().unwrap().len_utf8();
        if nidx < self.string.len() {
            let nch = self.string[nidx..].chars().next().unwrap();

    #[inline]
    fn get_prev_cat(&self, idx: usize) -> Option<WordCat> {
        use tables::word as wd;
        usecrate::tables::word as wd;
        if idx > 0 {
            let nch = self.string[..idx].chars().next_back().unwrap();
            Some(wd::word_category(nch).2)

 #[inline]
 fn has_alphanumeric(s: &&str) -> bool {
    use tables::util::is_alphanumeric;
    usecrate::tables::util::is_alphanumeric;

    s.chars().any(|c| is_alphanumeric(c))
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,6 +4,7 @@ name = "unicode-segmentation"
		version = "1.7.1"
		authors = ["kwantam <kwantam@gmail.com>", "Manish Goregaokar <manishsmail@gmail.com>"]

		edition = "2018"
		homepage = "https://github.com/unicode-rs/unicode-segmentation"
		repository = "https://github.com/unicode-rs/unicode-segmentation"
		documentation = "https://unicode-rs.github.io/unicode-segmentation"
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -229,7 +229,7 @@ def emit_util_mod(f):
		#[inline]
		fn is_alphabetic(c: char) -> bool {
		match c {
		'a' ... 'z' \| 'A' ... 'Z' => true,
		'a' ..= 'z' \| 'A' ..= 'Z' => true,
		c if c > '\x7f' => super::derived_property::Alphabetic(c),
		_ => false,
		}
Expand All		@@ -238,7 +238,7 @@ def emit_util_mod(f):
		#[inline]
		fn is_numeric(c: char) -> bool {
		match c {
		'0' ... '9' => true,
		'0' ..= '9' => true,
		c if c > '\x7f' => super::general_category::N(c),
		_ => false,
		}
Expand DownExpand Up		@@ -281,7 +281,6 @@ def emit_break_module(f, break_table, break_cats, name):
		f.write(""" }

		fn bsearch_range_value_table(c: char, r: &'static [(char, char, %sCat)]) -> (u32, u32, %sCat) {
		use core;
		use core::cmp::Ordering::{Equal, Less, Greater};
		match r.binary_search_by(\|&(lo, hi, _)\| {
		if lo <= c && c <= hi { Equal }
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -10,7 +10,7 @@

		use core::cmp;

		use tables::grapheme::GraphemeCat;
		usecrate::tables::grapheme::GraphemeCat;

		/// External iterator for grapheme clusters and byte offsets.
		///
Expand DownExpand Up		@@ -229,7 +229,7 @@ enum PairResult {
		}

		fn check_pair(before: GraphemeCat, after: GraphemeCat) -> PairResult {
		use tables::grapheme::GraphemeCat::*;
		usecrate::tables::grapheme::GraphemeCat::*;
		use self::PairResult::*;
		match (before, after) {
		(GC_CR, GC_LF) => NotBreak, // GB3
Expand DownExpand Up		@@ -295,8 +295,8 @@ impl GraphemeCursor {
		}

		fn grapheme_category(&mut self, ch: char) -> GraphemeCat {
		use tables::grapheme as gr;
		use tables::grapheme::GraphemeCat::*;
		usecrate::tables::grapheme as gr;
		usecrate::tables::grapheme::GraphemeCat::*;

		if ch <= '\u{7e}' {
		// Special-case optimization for ascii, except U+007F. This
Expand DownExpand Up		@@ -387,7 +387,7 @@ impl GraphemeCursor {
		/// assert_eq!(cursor.is_boundary(&flags[8..], 8), Ok(true));
		/// ```
		pub fn provide_context(&mut self, chunk: &str, chunk_start: usize) {
		use tables::grapheme as gr;
		usecrate::tables::grapheme as gr;
		assert!(chunk_start + chunk.len() == self.pre_context_offset.unwrap());
		self.pre_context_offset = None;
		if self.is_extended && chunk_start + chunk.len() == self.offset {
Expand DownExpand Up		@@ -433,7 +433,7 @@ impl GraphemeCursor {
		}

		fn handle_regional(&mut self, chunk: &str, chunk_start: usize) {
		use tables::grapheme as gr;
		usecrate::tables::grapheme as gr;
		let mut ris_count = self.ris_count.unwrap_or(0);
		for ch in chunk.chars().rev() {
		if self.grapheme_category(ch) != gr::GC_Regional_Indicator {
Expand All		@@ -453,7 +453,7 @@ impl GraphemeCursor {
		}

		fn handle_emoji(&mut self, chunk: &str, chunk_start: usize) {
		use tables::grapheme as gr;
		usecrate::tables::grapheme as gr;
		let mut iter = chunk.chars().rev();
		if let Some(ch) = iter.next() {
		if self.grapheme_category(ch) != gr::GC_ZWJ {
Expand DownExpand Up		@@ -506,7 +506,7 @@ impl GraphemeCursor {
		/// assert_eq!(cursor.is_boundary(flags, 0), Ok(false));
		/// ```
		pub fn is_boundary(&mut self, chunk: &str, chunk_start: usize) -> Result<bool, GraphemeIncomplete> {
		use tables::grapheme as gr;
		usecrate::tables::grapheme as gr;
		if self.state == GraphemeState::Break {
		return Ok(true)
		}
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -13,7 +13,7 @@ use core::iter::Filter;

		// All of the logic for forward iteration over sentences
		mod fwd {
		use tables::sentence::SentenceCat;
		usecrate::tables::sentence::SentenceCat;
		use core::cmp;

		// Describe a parsed part of source string as described in this table:
Expand DownExpand Up		@@ -111,7 +111,7 @@ mod fwd {
		if parts[idx] == StatePart::ClosePlus { idx -= 1 }

		if parts[idx] == StatePart::ATerm {
		use tables::sentence as se;
		usecrate::tables::sentence as se;

		for next_char in ahead.chars() {
		//( ¬(OLetter \| Upper \| Lower \| ParaSep \| SATerm) )* Lower
Expand DownExpand Up		@@ -176,7 +176,7 @@ mod fwd {

		#[inline]
		fn next(&mut self) -> Option<usize> {
		use tables::sentence as se;
		usecrate::tables::sentence as se;

		for next_char in self.string[self.pos..].chars() {
		let position_before = self.pos;
Expand DownExpand Up		@@ -331,7 +331,7 @@ pub fn new_sentence_bound_indices<'a>(source: &'a str) -> USentenceBoundIndices<
		#[inline]
		pub fn new_unicode_sentences<'b>(s: &'b str) -> UnicodeSentences<'b> {
		use super::UnicodeSegmentation;
		use tables::util::is_alphanumeric;
		usecrate::tables::util::is_alphanumeric;

		fn has_alphanumeric(s: &&str) -> bool { s.chars().any(\|c\| is_alphanumeric(c)) }
		let has_alphanumeric: fn(&&str) -> bool = has_alphanumeric; // coerce to fn pointer
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -30,7 +30,7 @@ pub mod util {
		#[inline]
		fn is_alphabetic(c: char) -> bool {
		match c {
		'a' ... 'z' \| 'A' ... 'Z' => true,
		'a' ..= 'z' \| 'A' ..= 'Z' => true,
		c if c > '' => super::derived_property::Alphabetic(c),
		_ => false,
		}
Expand All		@@ -39,7 +39,7 @@ pub mod util {
		#[inline]
		fn is_numeric(c: char) -> bool {
		match c {
		'0' ... '9' => true,
		'0' ..= '9' => true,
		c if c > '' => super::general_category::N(c),
		_ => false,
		}
Expand DownExpand Up		@@ -352,7 +352,6 @@ pub mod grapheme {
		}

		fn bsearch_range_value_table(c: char, r: &'static [(char, char, GraphemeCat)]) -> (u32, u32, GraphemeCat) {
		use core;
		use core::cmp::Ordering::{Equal, Less, Greater};
		match r.binary_search_by(\|&(lo, hi, _)\| {
		if lo <= c && c <= hi { Equal }
Expand DownExpand Up		@@ -1003,7 +1002,6 @@ pub mod word {
		}

		fn bsearch_range_value_table(c: char, r: &'static [(char, char, WordCat)]) -> (u32, u32, WordCat) {
		use core;
		use core::cmp::Ordering::{Equal, Less, Greater};
		match r.binary_search_by(\|&(lo, hi, _)\| {
		if lo <= c && c <= hi { Equal }
Expand DownExpand Up		@@ -1479,7 +1477,6 @@ pub mod emoji {
		}

		fn bsearch_range_value_table(c: char, r: &'static [(char, char, EmojiCat)]) -> (u32, u32, EmojiCat) {
		use core;
		use core::cmp::Ordering::{Equal, Less, Greater};
		match r.binary_search_by(\|&(lo, hi, _)\| {
		if lo <= c && c <= hi { Equal }
Expand DownExpand Up		@@ -1583,7 +1580,6 @@ pub mod sentence {
		}

		fn bsearch_range_value_table(c: char, r: &'static [(char, char, SentenceCat)]) -> (u32, u32, SentenceCat) {
		use core;
		use core::cmp::Ordering::{Equal, Less, Greater};
		match r.binary_search_by(\|&(lo, hi, _)\| {
		if lo <= c && c <= hi { Equal }
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -14,7 +14,7 @@ use std::prelude::v1::*;

		#[test]
		fn test_graphemes() {
		use testdata::{TEST_SAME, TEST_DIFF};
		usecrate::testdata::{TEST_SAME, TEST_DIFF};

		pub const EXTRA_DIFF: &'static [(&'static str,
		&'static [&'static str],
Expand DownExpand Up		@@ -88,7 +88,7 @@ fn test_graphemes() {

		#[test]
		fn test_words() {
		use testdata::TEST_WORD;
		usecrate::testdata::TEST_WORD;

		// Unicode's official tests don't really test longer chains of flag emoji
		// TODO This could be improved with more tests like flag emoji with interspersed Extend chars and ZWJ
Expand DownExpand Up		@@ -144,7 +144,7 @@ fn test_words() {

		#[test]
		fn test_sentences() {
		use testdata::TEST_SENTENCE;
		usecrate::testdata::TEST_SENTENCE;

		for &(s, w) in TEST_SENTENCE.iter() {
		macro_rules! assert_ {
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -11,7 +11,7 @@
		use core::cmp;
		use core::iter::Filter;

		use tables::word::WordCat;
		usecrate::tables::word::WordCat;

		/// An iterator over the substrings of a string which, after splitting the string on
		/// [word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries),
Expand DownExpand Up		@@ -170,7 +170,7 @@ enum RegionalState {
		}

		fn is_emoji(ch: char) -> bool {
		use tables::emoji;
		usecrate::tables::emoji;
		emoji::emoji_category(ch).2 == emoji::EmojiCat::EC_Extended_Pictographic
		}

Expand All		@@ -187,7 +187,7 @@ impl<'a> Iterator for UWordBounds<'a> {
		fn next(&mut self) -> Option<&'a str> {
		use self::UWordBoundsState::*;
		use self::FormatExtendType::*;
		use tables::word as wd;
		usecrate::tables::word as wd;
		if self.string.len() == 0 {
		return None;
		}
Expand DownExpand Up		@@ -414,7 +414,7 @@ impl<'a> DoubleEndedIterator for UWordBounds<'a> {
		fn next_back(&mut self) -> Option<&'a str> {
		use self::UWordBoundsState::*;
		use self::FormatExtendType::*;
		use tables::word as wd;
		usecrate::tables::word as wd;
		if self.string.len() == 0 {
		return None;
		}
Expand DownExpand Up		@@ -666,7 +666,7 @@ impl<'a> UWordBounds<'a> {

		#[inline]
		fn get_next_cat(&self, idx: usize) -> Option<WordCat> {
		use tables::word as wd;
		usecrate::tables::word as wd;
		let nidx = idx + self.string[idx..].chars().next().unwrap().len_utf8();
		if nidx < self.string.len() {
		let nch = self.string[nidx..].chars().next().unwrap();
Expand All		@@ -678,7 +678,7 @@ impl<'a> UWordBounds<'a> {

		#[inline]
		fn get_prev_cat(&self, idx: usize) -> Option<WordCat> {
		use tables::word as wd;
		usecrate::tables::word as wd;
		if idx > 0 {
		let nch = self.string[..idx].chars().next_back().unwrap();
		Some(wd::word_category(nch).2)
Expand All		@@ -700,7 +700,7 @@ pub fn new_word_bound_indices<'b>(s: &'b str) -> UWordBoundIndices<'b> {

		#[inline]
		fn has_alphanumeric(s: &&str) -> bool {
		use tables::util::is_alphanumeric;
		usecrate::tables::util::is_alphanumeric;

		s.chars().any(\|c\| is_alphanumeric(c))
		}
Expand Down