Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Track position where normalization failed#114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Open
clarfonthey wants to merge1 commit intounicode-rs:master
base:master
Choose a base branch
Loading
fromclarfonthey:errors
Open
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 0 additions & 17 deletions.travis.yml
View file
Open in desktop

This file was deleted.

7 changes: 3 additions & 4 deletionsCargo.toml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
[package]

name = "unicode-normalization"
version = "0.1.24"
version = "0.2.0"
authors = [
"kwantam <kwantam@gmail.com>",
"Manish Goregaokar <manishsmail@gmail.com>",
Expand All@@ -27,17 +27,16 @@ Decomposition and Recomposition, as described in
Unicode Standard Annex #15.
"""

rust-version = "1.36"
rust-version = "1.84"

edition = "2018"
edition = "2021"

exclude = ["target/*", "Cargo.lock", "scripts/tmp", "*.txt", "tests/*"]

[dependencies.tinyvec]
version = "1"
features = ["alloc"]


[features]
default = ["std"]
std = []
17 changes: 2 additions & 15 deletionsREADME.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
# unicode-normalization

[![Build Status](https://travis-ci.org/unicode-rs/unicode-normalization.svg)](https://travis-ci.org/unicode-rs/unicode-normalization)
[![Docs](https://docs.rs/unicode-normalization/badge.svg)](https://docs.rs/unicode-normalization/)

Unicode character composition and decomposition utilities
as described in
[Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/).

This crate requires Rust 1.36+.
This crate requires Rust 1.84+.

```rust
extern crate unicode_normalization;
Expand All@@ -31,21 +30,9 @@ to your `Cargo.toml`:

```toml
[dependencies]
unicode-normalization = "0.1.24"
unicode-normalization = "0.2.0"
```

## `no_std` + `alloc` support

This crate is completely `no_std` + `alloc` compatible. This can be enabled by disabling the `std` feature, i.e. specifying `default-features = false` for this crate on your `Cargo.toml`.

## Note about MSRV

Dependencies' MSRVs evolve independently of this crate's MSRV.
Old versions of cargo will always try to get the most recent versions of the dependencies.
Therefore, if you are having troubles compiling on an old Rust version, try to install an older version of the incompatible dependency.

For instance, to compile on Rust 1.36, `tinyvec` must be `<=1.6.0`

```sh
cargo update -p tinyvec --precise 1.6.0
```
66 changes: 29 additions & 37 deletionsfuzz/fuzz_targets/unicode-normalization.rs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,74 +4,66 @@
extern crate libfuzzer_sys;

use unicode_normalization::{
is_nfc, is_nfc_quick, is_nfc_stream_safe, is_nfc_stream_safe_quick, is_nfd, is_nfd_quick,
is_nfd_stream_safe, is_nfd_stream_safe_quick, is_nfkc, is_nfkc_quick, is_nfkd, is_nfkd_quick,
IsNormalized, UnicodeNormalization,
check_nfc_quick, check_nfc_stream_safe_quick, check_nfd_quick, check_nfd_stream_safe_quick,
check_nfkc_quick, check_nfkd_quick, is_nfc, is_nfc_stream_safe, is_nfd, is_nfd_stream_safe,
is_nfkc, is_nfkd, UnicodeNormalization,
};

fn from_bool(is_normalized: bool) -> IsNormalized {
if is_normalized {
IsNormalized::Yes
} else {
IsNormalized::No
}
}

fuzz_target!(|input: String| {
// The full predicates imply the quick predicates.
assert_ne!(is_nfc_quick(input.chars()), from_bool(!is_nfc(&input)));
assert_ne!(is_nfd_quick(input.chars()), from_bool(!is_nfd(&input)));
assert_ne!(is_nfkc_quick(input.chars()), from_bool(!is_nfkc(&input)));
assert_ne!(is_nfkd_quick(input.chars()), from_bool(!is_nfkd(&input)));
assert_ne!(
is_nfc_stream_safe_quick(input.chars()),
from_bool(!is_nfc_stream_safe(&input))
assert_eq!(check_nfc_quick(&input).is_ok(),is_nfc(&input));
assert_eq!(check_nfd_quick(&input).is_ok(),is_nfd(&input));
assert_eq!(check_nfkc_quick(&input).is_ok(),is_nfkc(&input));
assert_eq!(check_nfkd_quick(&input).is_ok(),is_nfkd(&input));
assert_eq!(
check_nfc_stream_safe_quick(&input).is_ok(),
is_nfc_stream_safe(&input)
);
assert_ne!(
is_nfd_stream_safe_quick(input.chars()),
from_bool(!is_nfd_stream_safe(&input))
assert_eq!(
check_nfd_stream_safe_quick(&input).is_ok(),
is_nfd_stream_safe(&input)
);

// Check NFC, NFD, NFKC, and NFKD normalization.
let nfc = input.chars().nfc().collect::<String>();
let nfc = input.nfc().collect::<String>();
assert_eq!(nfc.is_empty(), input.is_empty());
assert_ne!(is_nfc_quick(nfc.chars()), IsNormalized::No);
assert!(check_nfc_quick(&nfc).is_ok());
assert!(is_nfc(&nfc));

let nfd = input.chars().nfd().collect::<String>();
let nfd = input.nfd().collect::<String>();
assert!(nfd.len() >= nfc.len());
assert_ne!(is_nfd_quick(nfd.chars()), IsNormalized::No);
assert!(check_nfd_quick(&nfd).is_ok());
assert!(is_nfd(&nfd));

let nfkc = input.chars().nfkc().collect::<String>();
let nfkc = input.nfkc().collect::<String>();
assert_eq!(nfkc.is_empty(), input.is_empty());
assert_ne!(is_nfkc_quick(nfkc.chars()), IsNormalized::No);
assert!(check_nfkc_quick(&nfkc).is_ok());
assert!(is_nfkc(&nfkc));

let nfkd = input.chars().nfkd().collect::<String>();
let nfkd = input.nfkd().collect::<String>();
assert!(nfkd.len() >= nfkc.len());
assert_ne!(is_nfkd_quick(nfkd.chars()), IsNormalized::No);
assert!(check_nfkd_quick(&nfkd).is_ok());
assert!(is_nfkd(&nfkd));

// Check stream-safe.
let nfc_ss = nfc.chars().stream_safe().collect::<String>();
let nfc_ss = nfc.stream_safe().collect::<String>();
assert!(nfc_ss.len() >= nfc.len());
assert_ne!(is_nfc_stream_safe_quick(nfc_ss.chars()), IsNormalized::No);
assert!(check_nfc_stream_safe_quick(&nfc_ss).is_ok());
assert!(is_nfc_stream_safe(&nfc_ss));

let nfd_ss = nfd.chars().stream_safe().collect::<String>();
let nfd_ss = nfd.stream_safe().collect::<String>();
assert!(nfd_ss.len() >= nfd.len());
assert_ne!(is_nfd_stream_safe_quick(nfd_ss.chars()), IsNormalized::No);
assert!(check_nfd_stream_safe_quick(&nfd_ss).is_ok());
assert!(is_nfd_stream_safe(&nfd_ss));

// Check that NFC and NFD preserve stream-safe.
let ss_nfc = input.chars().stream_safe().nfc().collect::<String>();
let ss_nfc = input.stream_safe().nfc().collect::<String>();
assert_eq!(ss_nfc.is_empty(), input.is_empty());
assert_ne!(is_nfc_stream_safe_quick(ss_nfc.chars()), IsNormalized::No);
assert!(check_nfc_stream_safe_quick(&ss_nfc).is_ok());
assert!(is_nfc_stream_safe(&ss_nfc));

let ss_nfd = input.chars().stream_safe().nfd().collect::<String>();
let ss_nfd = input.stream_safe().nfd().collect::<String>();
assert_eq!(ss_nfd.is_empty(), input.is_empty());
assert_ne!(is_nfd_stream_safe_quick(ss_nfd.chars()), IsNormalized::No);
assert!(check_nfd_stream_safe_quick(&ss_nfd).is_ok());
assert!(is_nfd_stream_safe(&ss_nfd));
});
8 changes: 4 additions & 4 deletionsscripts/unicode.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -463,7 +463,8 @@ def gen_public_assigned(general_category_public_assigned, out):
# This could be done as a hash but the table is somewhat small.
out.write("#[inline]\n")
out.write("pub fn is_public_assigned(c: char) -> bool {\n")
out.write(" match c {\n")
out.write(" matches!(\n")
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

This was a clippy suggestion, not required, but I figured I'd make it anyway. It's not stable in 1.36, so, that's why it wasn't being suggested before.

out.write(" c,\n")

start = True
for first, last in general_category_public_assigned:
Expand All@@ -476,10 +477,9 @@ def gen_public_assigned(general_category_public_assigned, out):
out.write("'\\u{%s}'" % hexify(first))
else:
out.write("'\\u{%s}'..='\\u{%s}'" % (hexify(first), hexify(last)))
out.write(" => true,\n")
out.write(",\n")

out.write(" _ => false,\n")
out.write(" }\n")
out.write(" )\n")
out.write("}\n")

def gen_stream_safe(leading, trailing, out):
Expand Down
9 changes: 5 additions & 4 deletionssrc/lib.rs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -34,7 +34,7 @@
//!
//! ```toml
//! [dependencies]
//!unicode-normalization ="0.1.20"
#![doc = concat!("unicode-normalization =\"", env!("CARGO_PKG_VERSION"), "\"")]
//! ```

#![deny(missing_docs, unsafe_code)]
Expand All@@ -54,9 +54,10 @@ extern crate tinyvec;

pub use crate::decompose::Decompositions;
pub use crate::quick_check::{
is_nfc, is_nfc_quick, is_nfc_stream_safe, is_nfc_stream_safe_quick, is_nfd, is_nfd_quick,
is_nfd_stream_safe, is_nfd_stream_safe_quick, is_nfkc, is_nfkc_quick, is_nfkd, is_nfkd_quick,
IsNormalized,
check_nfc, check_nfc_quick, check_nfc_stream_safe, check_nfc_stream_safe_quick, check_nfd,
check_nfd_quick, check_nfd_stream_safe, check_nfd_stream_safe_quick, check_nfkc,
check_nfkc_quick, check_nfkd, check_nfkd_quick, is_nfc, is_nfc_stream_safe, is_nfd,
is_nfd_stream_safe, is_nfkc, is_nfkd, NormalizationError, QuickCheck,
};
pub use crate::recompose::Recompositions;
pub use crate::replace::Replacements;
Expand Down
Loading
[8]ページ先頭
©2009-2025 Movatter.jp